Guard imports of PreTrainedTokenizerFast on is_tokenizers_available (#22285)

Guard imports that use the tokenizers library
This commit is contained in:
Roy Hvaara
2023-03-30 06:16:03 -07:00
committed by GitHub
parent 4d7a5b5ba3
commit 11426641dc
2 changed files with 11 additions and 3 deletions

View File

@@ -32,7 +32,6 @@ from ..models.auto.image_processing_auto import IMAGE_PROCESSOR_MAPPING, AutoIma
from ..models.auto.modeling_auto import AutoModelForDepthEstimation
from ..models.auto.tokenization_auto import TOKENIZER_MAPPING, AutoTokenizer
from ..tokenization_utils import PreTrainedTokenizer
from ..tokenization_utils_fast import PreTrainedTokenizerFast
from ..utils import (
HUGGINGFACE_CO_RESOLVE_ENDPOINT,
is_kenlm_available,
@@ -139,9 +138,13 @@ if is_torch_available():
AutoModelForZeroShotImageClassification,
AutoModelForZeroShotObjectDetection,
)
if TYPE_CHECKING:
from ..modeling_tf_utils import TFPreTrainedModel
from ..modeling_utils import PreTrainedModel
from ..tokenization_utils_fast import PreTrainedTokenizerFast
logger = logging.get_logger(__name__)
@@ -495,7 +498,7 @@ def pipeline(
task: str = None,
model: Optional = None,
config: Optional[Union[str, PretrainedConfig]] = None,
tokenizer: Optional[Union[str, PreTrainedTokenizer, PreTrainedTokenizerFast]] = None,
tokenizer: Optional[Union[str, PreTrainedTokenizer, "PreTrainedTokenizerFast"]] = None,
feature_extractor: Optional[Union[str, PreTrainedFeatureExtractor]] = None,
image_processor: Optional[Union[str, BaseImageProcessor]] = None,
framework: Optional[str] = None,