From c18f5916a03d0a161d003e95ffff8120d8addc0c Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Mon, 7 Sep 2020 22:22:20 -0700 Subject: [PATCH] typo (#7001) apologies for the tiny PRs, just sending those as I find them. --- src/transformers/tokenization_utils_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/tokenization_utils_base.py b/src/transformers/tokenization_utils_base.py index c0cc919543..43b77951fb 100644 --- a/src/transformers/tokenization_utils_base.py +++ b/src/transformers/tokenization_utils_base.py @@ -1609,7 +1609,7 @@ class PreTrainedTokenizerBase(SpecialTokensMixin): ) tokenizer.add_tokens(token, special_tokens=bool(token in special_tokens)) - # Check all our special tokens are registrered as "no split" token (we don't cut them) and are in the vocab + # Check all our special tokens are registered as "no split" token (we don't cut them) and are in the vocab added_tokens = tokenizer.sanitize_special_tokens() if added_tokens: logger.warning(