SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Fix 29807, sinusoidal positional encodings overwritten by post_init() (#29813)

Browse Source 
* Check for requires_grad when initing weights

* Add unit test

* Move sinusoidal positional encoding generation after post_init()

* Add modules to skip init list

* Move create_sinusoidal_embeddings to _init_weights
This commit is contained in:
Hovnatan Karapetyan
2024-03-27 09:28:00 +04:00
committed by GitHub
parent cefb819f7a
commit a81cf9ee90
2 changed files with 14 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
tests/models/distilbert/test_modeling_distilbert.py
View File

@@ -37,6 +37,7 @@ if is_torch_available():
DistilBertForTokenClassification,
DistilBertModel,
)
from transformers.models.distilbert.modeling_distilbert import _create_sinusoidal_embeddings
class DistilBertModelTester(object):
@@ -238,6 +239,15 @@ class DistilBertModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCa
config_and_inputs = self.model_tester.prepare_config_and_inputs()
self.model_tester.create_and_check_distilbert_model(*config_and_inputs)
def test_distilbert_model_with_sinusoidal_encodings(self):
config = DistilBertConfig(sinusoidal_pos_embds=True)
model = DistilBertModel(config=config)
sinusoidal_pos_embds = torch.empty((config.max_position_embeddings, config.dim), dtype=torch.float32)
_create_sinusoidal_embeddings(config.max_position_embeddings, config.dim, sinusoidal_pos_embds)
self.model_tester.parent.assertTrue(
torch.equal(model.embeddings.position_embeddings.weight, sinusoidal_pos_embds)
)
def test_for_masked_lm(self):
config_and_inputs = self.model_tester.prepare_config_and_inputs()
self.model_tester.create_and_check_distilbert_for_masked_lm(*config_and_inputs)