Sdpa dino v2 (#33403)

* add sdpa to dinov2

* fixup

* add dinov2 to sdpa doc

* update doc order

* [run-slow] dinov2

* common to eager

* [run-slow] dinov2

* update attn implementation in common

* update test_modeling_dinov2 to have mask_ration, num_masks and mask_length similar to vit

* [run-slow] dinov2

---------

Co-authored-by: Avishai Elmakies <avishai.elma@cs.huji.ac.il>
This commit is contained in:
Avishai Elmakies
2024-09-21 03:58:00 +03:00
committed by GitHub
parent e71bf70e33
commit 78b2929c05
3 changed files with 55 additions and 2 deletions

View File

@@ -65,6 +65,8 @@ class Dinov2ModelTester:
type_sequence_label_size=10,
initializer_range=0.02,
scope=None,
attn_implementation="eager",
mask_ratio=0.5,
):
self.parent = parent
self.batch_size = batch_size
@@ -83,10 +85,14 @@ class Dinov2ModelTester:
self.type_sequence_label_size = type_sequence_label_size
self.initializer_range = initializer_range
self.scope = scope
self.attn_implementation = attn_implementation
self.mask_ratio = mask_ratio
# in Dinov2, the seq length equals the number of patches + 1 (we add 1 for the [CLS] token)
num_patches = (image_size // patch_size) ** 2
self.seq_length = num_patches + 1
self.num_masks = int(self.mask_ratio * self.seq_length)
self.mask_length = num_patches
def prepare_config_and_inputs(self):
pixel_values = floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size])
@@ -113,6 +119,7 @@ class Dinov2ModelTester:
attention_probs_dropout_prob=self.attention_probs_dropout_prob,
is_decoder=False,
initializer_range=self.initializer_range,
attn_implementation=self.attn_implementation,
)
def create_and_check_model(self, config, pixel_values, labels):