SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

num_noise_spans should be <= num_items #22246 (#22938)

Browse Source
This commit is contained in:
Alex Punnen
2023-05-02 22:37:30 +05:30
committed by GitHub
parent 9ade58f055
commit 805db1fe13
1 changed files with 3 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
examples/flax/language-modeling/run_t5_mlm_flax.py
View File

@@ -418,13 +418,14 @@ class FlaxDataCollatorForT5MLM:
orig_length = length orig_length = length
num_noise_tokens = int(np.round(length * self.noise_density)) num_noise_tokens = int(np.round(length * self.noise_density))
num_nonnoise_tokens = length - num_noise_tokens
# avoid degeneracy by ensuring positive numbers of noise and nonnoise tokens. # avoid degeneracy by ensuring positive numbers of noise and nonnoise tokens.
num_noise_tokens = min(max(num_noise_tokens, 1), length - 1) num_noise_tokens = min(max(num_noise_tokens, 1), length - 1)
num_noise_spans = int(np.round(num_noise_tokens / self.mean_noise_span_length)) # num_noise_tokens should be less than num_noise_tokens and num_nonnoise_tokens
num_noise_spans = int(np.round(min(num_noise_tokens, num_nonnoise_tokens) / self.mean_noise_span_length))
# avoid degeneracy by ensuring positive number of noise spans # avoid degeneracy by ensuring positive number of noise spans
num_noise_spans = max(num_noise_spans, 1) num_noise_spans = max(num_noise_spans, 1)
num_nonnoise_tokens = length - num_noise_tokens
# pick the lengths of the noise spans and the non-noise spans # pick the lengths of the noise spans and the non-noise spans
def _random_segmentation(num_items, num_segments): def _random_segmentation(num_items, num_segments):