From 1b652295c5940a45079c9056864009418c5a8054 Mon Sep 17 00:00:00 2001 From: Lysandre Debut Date: Fri, 21 May 2021 12:50:01 +0200 Subject: [PATCH] Patch recursive import (#11812) --- src/transformers/convert_slow_tokenizer.py | 3 ++- tests/test_tokenization_utils.py | 7 ++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/transformers/convert_slow_tokenizer.py b/src/transformers/convert_slow_tokenizer.py index f2587341c0..2e9ac9066c 100644 --- a/src/transformers/convert_slow_tokenizer.py +++ b/src/transformers/convert_slow_tokenizer.py @@ -25,7 +25,6 @@ from tokenizers import Regex, Tokenizer, decoders, normalizers, pre_tokenizers, from tokenizers.models import BPE, Unigram, WordPiece from .file_utils import requires_backends -from .models.roformer.tokenization_utils import JiebaPreTokenizer class SentencePieceExtractor: @@ -299,6 +298,8 @@ class RobertaConverter(Converter): class RoFormerConverter(Converter): def converted(self) -> Tokenizer: + from .models.roformer.tokenization_utils import JiebaPreTokenizer + vocab = self.original_tokenizer.vocab tokenizer = Tokenizer(WordPiece(vocab, unk_token=str(self.original_tokenizer.unk_token))) diff --git a/tests/test_tokenization_utils.py b/tests/test_tokenization_utils.py index 534d945458..a655b84dc1 100644 --- a/tests/test_tokenization_utils.py +++ b/tests/test_tokenization_utils.py @@ -12,6 +12,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +""" +isort:skip_file +""" import os import pickle import tempfile @@ -20,12 +23,14 @@ from typing import Callable, Optional import numpy as np +# Ensure there are no circular imports when importing the parent class +from transformers import PreTrainedTokenizerFast + from transformers import ( BatchEncoding, BertTokenizer, BertTokenizerFast, PreTrainedTokenizer, - PreTrainedTokenizerFast, TensorType, TokenSpan, is_tokenizers_available,