SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

distilbert: fix creation of sinusoidal embeddings when using PyTorch 1.8+ (#9917)

Browse Source
This commit is contained in:
Stefan Schweter
2021-02-03 17:42:16 +01:00
committed by GitHub
parent 2f06f2bcd6
commit 6244727e05
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/transformers/models/distilbert/modeling_distilbert.py
View File

@@ -73,10 +73,10 @@ DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST = [
def create_sinusoidal_embeddings(n_pos, dim, out): def create_sinusoidal_embeddings(n_pos, dim, out):
position_enc = np.array([[pos / np.power(10000, 2 * (j // 2) / dim) for j in range(dim)] for pos in range(n_pos)]) position_enc = np.array([[pos / np.power(10000, 2 * (j // 2) / dim) for j in range(dim)] for pos in range(n_pos)])
out.requires_grad = False
out[:, 0::2] = torch.FloatTensor(np.sin(position_enc[:, 0::2])) out[:, 0::2] = torch.FloatTensor(np.sin(position_enc[:, 0::2]))
out[:, 1::2] = torch.FloatTensor(np.cos(position_enc[:, 1::2])) out[:, 1::2] = torch.FloatTensor(np.cos(position_enc[:, 1::2]))
out.detach_() out.detach_()
out.requires_grad = False
class Embeddings(nn.Module): class Embeddings(nn.Module):