SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Restrain tokenizer.model_max_length default (#9681)

Browse Source 
* Restrain tokenizer.model_max_length default

* Fix indent
This commit is contained in:
Sylvain Gugger
2021-01-20 04:17:39 -05:00
committed by GitHub
parent 7e662e6a3b
commit a1ad16a446
1 changed files with 6 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
examples/language-modeling/run_mlm.py
View File

@@ -338,6 +338,12 @@ def main():
if data_args.max_seq_length is None: if data_args.max_seq_length is None:
max_seq_length = tokenizer.model_max_length max_seq_length = tokenizer.model_max_length
if max_seq_length > 1024:
logger.warn(
f"The tokenizer picked seems to have a very large `model_max_length` ({tokenizer.model_max_length}). "
"Picking 1024 instead. You can change that default value by passing --max_seq_length xxx."
)
max_seq_length = 1024
else: else:
if data_args.max_seq_length > tokenizer.model_max_length: if data_args.max_seq_length > tokenizer.model_max_length:
logger.warn( logger.warn(