From 5f2a3d721c514cb160c74d2f2df6b729c2f99b2d Mon Sep 17 00:00:00 2001 From: Poedator <24738311+poedator@users.noreply.github.com> Date: Wed, 24 Feb 2021 17:01:28 +0300 Subject: [PATCH] fix deprecated ref to `tokenizer.max_len` (#10220) This is to fix deprecated reference to `tokenizer.max_len` with `tokenizer.model_max_length` - similar to [issue 8739](https://github.com/huggingface/transformers/issues/8739) and [PR 8604](https://github.com/huggingface/transformers/pull/8604). Example [here](https://colab.research.google.com/gist/poedator/f8776349e5c625ce287fc6fcd312fa1e/tokenizer-max_len-error-in-transformers_glue.ipynb). The error happens when `glue_convert_examples_to_features` is called without `max_length` parameter specified. In that case line 119 with wrong reference gets called. This simple fix should do it. --- src/transformers/data/processors/glue.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/data/processors/glue.py b/src/transformers/data/processors/glue.py index d92b976bce..0e1f244305 100644 --- a/src/transformers/data/processors/glue.py +++ b/src/transformers/data/processors/glue.py @@ -116,7 +116,7 @@ def _glue_convert_examples_to_features( output_mode=None, ): if max_length is None: - max_length = tokenizer.max_len + max_length = tokenizer.model_max_length if task is not None: processor = glue_processors[task]()