From 6244727e059877c54110d6f8207a925d0071ebfc Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Wed, 3 Feb 2021 17:42:16 +0100 Subject: [PATCH] distilbert: fix creation of sinusoidal embeddings when using PyTorch 1.8+ (#9917) --- src/transformers/models/distilbert/modeling_distilbert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/distilbert/modeling_distilbert.py b/src/transformers/models/distilbert/modeling_distilbert.py index d659826874..e478c588b4 100755 --- a/src/transformers/models/distilbert/modeling_distilbert.py +++ b/src/transformers/models/distilbert/modeling_distilbert.py @@ -73,10 +73,10 @@ DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST = [ def create_sinusoidal_embeddings(n_pos, dim, out): position_enc = np.array([[pos / np.power(10000, 2 * (j // 2) / dim) for j in range(dim)] for pos in range(n_pos)]) + out.requires_grad = False out[:, 0::2] = torch.FloatTensor(np.sin(position_enc[:, 0::2])) out[:, 1::2] = torch.FloatTensor(np.cos(position_enc[:, 1::2])) out.detach_() - out.requires_grad = False class Embeddings(nn.Module):