From 11426641dc525414cc27e9199a1f9bd512326e27 Mon Sep 17 00:00:00 2001 From: Roy Hvaara Date: Thu, 30 Mar 2023 06:16:03 -0700 Subject: [PATCH] Guard imports of PreTrainedTokenizerFast on is_tokenizers_available (#22285) Guard imports that use the tokenizers library --- src/transformers/models/auto/tokenization_auto.py | 7 ++++++- src/transformers/pipelines/__init__.py | 7 +++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py index fd10c5cd11..b23b0c39d6 100644 --- a/src/transformers/models/auto/tokenization_auto.py +++ b/src/transformers/models/auto/tokenization_auto.py @@ -24,7 +24,6 @@ from ...configuration_utils import PretrainedConfig from ...dynamic_module_utils import get_class_from_dynamic_module from ...tokenization_utils import PreTrainedTokenizer from ...tokenization_utils_base import TOKENIZER_CONFIG_FILE -from ...tokenization_utils_fast import PreTrainedTokenizerFast from ...utils import cached_file, extract_commit_hash, is_sentencepiece_available, is_tokenizers_available, logging from ..encoder_decoder import EncoderDecoderConfig from .auto_factory import _LazyAutoMapping @@ -37,6 +36,12 @@ from .configuration_auto import ( ) +if is_tokenizers_available(): + from ...tokenization_utils_fast import PreTrainedTokenizerFast +else: + PreTrainedTokenizerFast = None + + logger = logging.get_logger(__name__) if TYPE_CHECKING: diff --git a/src/transformers/pipelines/__init__.py b/src/transformers/pipelines/__init__.py index c8c0549a46..7beab782c7 100755 --- a/src/transformers/pipelines/__init__.py +++ b/src/transformers/pipelines/__init__.py @@ -32,7 +32,6 @@ from ..models.auto.image_processing_auto import IMAGE_PROCESSOR_MAPPING, AutoIma from ..models.auto.modeling_auto import AutoModelForDepthEstimation from ..models.auto.tokenization_auto import TOKENIZER_MAPPING, AutoTokenizer from ..tokenization_utils import PreTrainedTokenizer -from ..tokenization_utils_fast import PreTrainedTokenizerFast from ..utils import ( HUGGINGFACE_CO_RESOLVE_ENDPOINT, is_kenlm_available, @@ -139,9 +138,13 @@ if is_torch_available(): AutoModelForZeroShotImageClassification, AutoModelForZeroShotObjectDetection, ) + + if TYPE_CHECKING: from ..modeling_tf_utils import TFPreTrainedModel from ..modeling_utils import PreTrainedModel + from ..tokenization_utils_fast import PreTrainedTokenizerFast + logger = logging.get_logger(__name__) @@ -495,7 +498,7 @@ def pipeline( task: str = None, model: Optional = None, config: Optional[Union[str, PretrainedConfig]] = None, - tokenizer: Optional[Union[str, PreTrainedTokenizer, PreTrainedTokenizerFast]] = None, + tokenizer: Optional[Union[str, PreTrainedTokenizer, "PreTrainedTokenizerFast"]] = None, feature_extractor: Optional[Union[str, PreTrainedFeatureExtractor]] = None, image_processor: Optional[Union[str, BaseImageProcessor]] = None, framework: Optional[str] = None,