SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

[FlaxT5 Example] fix flax t5 example pretraining (#15835)

Browse Source
This commit is contained in:
Patrick von Platen
2022-03-04 17:04:43 +01:00
committed by GitHub
parent 01485ceec3
commit 10b76987fc
1 changed files with 3 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
examples/flax/language-modeling/run_t5_mlm_flax.py
View File

@@ -368,7 +368,9 @@ class FlaxDataCollatorForT5MLM:
batch_size = input_ids.shape[0]
input_ids_full = np.where(sentinel_ids != 0, sentinel_ids, input_ids)
input_ids = input_ids_full[input_ids_full > 0].reshape((batch_size, -1))
# input_ids tokens and sentinel tokens are >= 0, tokens < 0 are
# masked tokens coming after sentinel tokens and should be removed
input_ids = input_ids_full[input_ids_full >= 0].reshape((batch_size, -1))
input_ids = np.concatenate(
[input_ids, np.full((batch_size, 1), self.tokenizer.eos_token_id, dtype=np.int32)], axis=-1
)