[examples] improve block_size warning message (#21463)

This commit is contained in:
Stas Bekman
2023-02-06 08:36:12 -08:00
committed by GitHub
parent 4435c7f52c
commit 3b9a1dc132
5 changed files with 15 additions and 10 deletions

View File

@@ -459,8 +459,9 @@ def main():
block_size = tokenizer.model_max_length
if block_size > 1024:
logger.warning(
f"The tokenizer picked seems to have a very large `model_max_length` ({tokenizer.model_max_length}). "
"Picking 1024 instead. You can change that default value by passing --block_size xxx."
"The chosen tokenizer supports a `model_max_length` that is longer than the default `block_size` value"
" of 1024. If you would like to use a longer `block_size` up to `tokenizer.model_max_length` you can"
" override this default with `--block_size xxx`."
)
block_size = 1024
else: