SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Fix sentinel token IDs in data collator for Flax T5 pretraining script (#14477)

Browse Source
This commit is contained in:
Rahul Nadkarni
2021-11-29 08:30:17 -08:00
committed by GitHub
parent 2bd950ca47
commit 8332327dca
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
examples/flax/language-modeling/run_t5_mlm_flax.py
View File

@@ -291,7 +291,7 @@ class FlaxDataCollatorForT5MLM:
start_indices[:, 0] = mask_indices[:, 0] start_indices[:, 0] = mask_indices[:, 0]
sentinel_ids = np.where(start_indices != 0, np.cumsum(start_indices, axis=-1), start_indices) sentinel_ids = np.where(start_indices != 0, np.cumsum(start_indices, axis=-1), start_indices)
sentinel_ids = np.where(sentinel_ids != 0, (sentinel_ids + self.tokenizer.vocab_size - 1), 0) sentinel_ids = np.where(sentinel_ids != 0, (len(self.tokenizer) - sentinel_ids), 0)
sentinel_ids -= mask_indices - start_indices sentinel_ids -= mask_indices - start_indices
return sentinel_ids return sentinel_ids