Move test model folders (#17034)
* move test model folders (TODO: fix imports and others) * fix (potentially partially) imports (in model test modules) * fix (potentially partially) imports (in tokenization test modules) * fix (potentially partially) imports (in feature extraction test modules) * fix import utils.test_modeling_tf_core * fix path ../fixtures/ * fix imports about generation.test_generation_flax_utils * fix more imports * fix fixture path * fix get_test_dir * update module_to_test_file * fix get_tests_dir from wrong transformers.utils * update config.yml (CircleCI) * fix style * remove missing imports * update new model script * update check_repo * update SPECIAL_MODULE_TO_TEST_MAP * fix style * add __init__ * update self-scheduled * fix add_new_model scripts * check one way to get location back * python setup.py build install * fix import in test auto * update self-scheduled.yml * update slack notification script * Add comments about artifact names * fix for yolos Co-authored-by: ydshieh <ydshieh@users.noreply.github.com>
This commit is contained in:
Yih-Dar
0
tests/models/auto/__init__.py
Normal file
0
tests/models/auto/__init__.py
Normal file
110
tests/models/auto/test_configuration_auto.py
Normal file
110
tests/models/auto/test_configuration_auto.py
Normal file
@@ -0,0 +1,110 @@
|
||||
# coding=utf-8
|
||||
# Copyright 2019-present, the HuggingFace Inc. team.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import importlib
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
import transformers.models.auto
|
||||
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
|
||||
from transformers.models.bert.configuration_bert import BertConfig
|
||||
from transformers.models.roberta.configuration_roberta import RobertaConfig
|
||||
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, get_tests_dir
|
||||
|
||||
|
||||
sys.path.append(str(Path(__file__).parent.parent.parent.parent / "utils"))
|
||||
|
||||
from test_module.custom_configuration import CustomConfig # noqa E402
|
||||
|
||||
|
||||
SAMPLE_ROBERTA_CONFIG = get_tests_dir("fixtures/dummy-config.json")
|
||||
|
||||
|
||||
class AutoConfigTest(unittest.TestCase):
|
||||
def test_module_spec(self):
|
||||
self.assertIsNotNone(transformers.models.auto.__spec__)
|
||||
self.assertIsNotNone(importlib.util.find_spec("transformers.models.auto"))
|
||||
|
||||
def test_config_from_model_shortcut(self):
|
||||
config = AutoConfig.from_pretrained("bert-base-uncased")
|
||||
self.assertIsInstance(config, BertConfig)
|
||||
|
||||
def test_config_model_type_from_local_file(self):
|
||||
config = AutoConfig.from_pretrained(SAMPLE_ROBERTA_CONFIG)
|
||||
self.assertIsInstance(config, RobertaConfig)
|
||||
|
||||
def test_config_model_type_from_model_identifier(self):
|
||||
config = AutoConfig.from_pretrained(DUMMY_UNKNOWN_IDENTIFIER)
|
||||
self.assertIsInstance(config, RobertaConfig)
|
||||
|
||||
def test_config_for_model_str(self):
|
||||
config = AutoConfig.for_model("roberta")
|
||||
self.assertIsInstance(config, RobertaConfig)
|
||||
|
||||
def test_pattern_matching_fallback(self):
|
||||
"""
|
||||
In cases where config.json doesn't include a model_type,
|
||||
perform a few safety checks on the config mapping's order.
|
||||
"""
|
||||
# no key string should be included in a later key string (typical failure case)
|
||||
keys = list(CONFIG_MAPPING.keys())
|
||||
for i, key in enumerate(keys):
|
||||
self.assertFalse(any(key in later_key for later_key in keys[i + 1 :]))
|
||||
|
||||
def test_new_config_registration(self):
|
||||
try:
|
||||
AutoConfig.register("custom", CustomConfig)
|
||||
# Wrong model type will raise an error
|
||||
with self.assertRaises(ValueError):
|
||||
AutoConfig.register("model", CustomConfig)
|
||||
# Trying to register something existing in the Transformers library will raise an error
|
||||
with self.assertRaises(ValueError):
|
||||
AutoConfig.register("bert", BertConfig)
|
||||
|
||||
# Now that the config is registered, it can be used as any other config with the auto-API
|
||||
config = CustomConfig()
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
config.save_pretrained(tmp_dir)
|
||||
new_config = AutoConfig.from_pretrained(tmp_dir)
|
||||
self.assertIsInstance(new_config, CustomConfig)
|
||||
|
||||
finally:
|
||||
if "custom" in CONFIG_MAPPING._extra_content:
|
||||
del CONFIG_MAPPING._extra_content["custom"]
|
||||
|
||||
def test_repo_not_found(self):
|
||||
with self.assertRaisesRegex(
|
||||
EnvironmentError, "bert-base is not a local folder and is not a valid model identifier"
|
||||
):
|
||||
_ = AutoConfig.from_pretrained("bert-base")
|
||||
|
||||
def test_revision_not_found(self):
|
||||
with self.assertRaisesRegex(
|
||||
EnvironmentError, r"aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"
|
||||
):
|
||||
_ = AutoConfig.from_pretrained(DUMMY_UNKNOWN_IDENTIFIER, revision="aaaaaa")
|
||||
|
||||
def test_configuration_not_found(self):
|
||||
with self.assertRaisesRegex(
|
||||
EnvironmentError,
|
||||
"hf-internal-testing/no-config-test-repo does not appear to have a file named config.json.",
|
||||
):
|
||||
_ = AutoConfig.from_pretrained("hf-internal-testing/no-config-test-repo")
|
||||
|
||||
def test_from_pretrained_dynamic_config(self):
|
||||
config = AutoConfig.from_pretrained("hf-internal-testing/test_dynamic_model", trust_remote_code=True)
|
||||
self.assertEqual(config.__class__.__name__, "NewModelConfig")
|
||||
123
tests/models/auto/test_feature_extraction_auto.py
Normal file
123
tests/models/auto/test_feature_extraction_auto.py
Normal file
@@ -0,0 +1,123 @@
|
||||
# coding=utf-8
|
||||
# Copyright 2021 the HuggingFace Inc. team.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from transformers import (
|
||||
CONFIG_MAPPING,
|
||||
FEATURE_EXTRACTOR_MAPPING,
|
||||
AutoConfig,
|
||||
AutoFeatureExtractor,
|
||||
Wav2Vec2Config,
|
||||
Wav2Vec2FeatureExtractor,
|
||||
)
|
||||
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, get_tests_dir
|
||||
|
||||
|
||||
sys.path.append(str(Path(__file__).parent.parent.parent.parent / "utils"))
|
||||
|
||||
from test_module.custom_configuration import CustomConfig # noqa E402
|
||||
from test_module.custom_feature_extraction import CustomFeatureExtractor # noqa E402
|
||||
|
||||
|
||||
SAMPLE_FEATURE_EXTRACTION_CONFIG_DIR = get_tests_dir("fixtures")
|
||||
SAMPLE_FEATURE_EXTRACTION_CONFIG = get_tests_dir("fixtures/dummy_feature_extractor_config.json")
|
||||
SAMPLE_CONFIG = get_tests_dir("fixtures/dummy-config.json")
|
||||
|
||||
|
||||
class AutoFeatureExtractorTest(unittest.TestCase):
|
||||
def test_feature_extractor_from_model_shortcut(self):
|
||||
config = AutoFeatureExtractor.from_pretrained("facebook/wav2vec2-base-960h")
|
||||
self.assertIsInstance(config, Wav2Vec2FeatureExtractor)
|
||||
|
||||
def test_feature_extractor_from_local_directory_from_key(self):
|
||||
config = AutoFeatureExtractor.from_pretrained(SAMPLE_FEATURE_EXTRACTION_CONFIG_DIR)
|
||||
self.assertIsInstance(config, Wav2Vec2FeatureExtractor)
|
||||
|
||||
def test_feature_extractor_from_local_directory_from_config(self):
|
||||
with tempfile.TemporaryDirectory() as tmpdirname:
|
||||
model_config = Wav2Vec2Config()
|
||||
|
||||
# remove feature_extractor_type to make sure config.json alone is enough to load feature processor locally
|
||||
config_dict = AutoFeatureExtractor.from_pretrained(SAMPLE_FEATURE_EXTRACTION_CONFIG_DIR).to_dict()
|
||||
|
||||
config_dict.pop("feature_extractor_type")
|
||||
config = Wav2Vec2FeatureExtractor(**config_dict)
|
||||
|
||||
# save in new folder
|
||||
model_config.save_pretrained(tmpdirname)
|
||||
config.save_pretrained(tmpdirname)
|
||||
|
||||
config = AutoFeatureExtractor.from_pretrained(tmpdirname)
|
||||
|
||||
# make sure private variable is not incorrectly saved
|
||||
dict_as_saved = json.loads(config.to_json_string())
|
||||
self.assertTrue("_processor_class" not in dict_as_saved)
|
||||
|
||||
self.assertIsInstance(config, Wav2Vec2FeatureExtractor)
|
||||
|
||||
def test_feature_extractor_from_local_file(self):
|
||||
config = AutoFeatureExtractor.from_pretrained(SAMPLE_FEATURE_EXTRACTION_CONFIG)
|
||||
self.assertIsInstance(config, Wav2Vec2FeatureExtractor)
|
||||
|
||||
def test_repo_not_found(self):
|
||||
with self.assertRaisesRegex(
|
||||
EnvironmentError, "bert-base is not a local folder and is not a valid model identifier"
|
||||
):
|
||||
_ = AutoFeatureExtractor.from_pretrained("bert-base")
|
||||
|
||||
def test_revision_not_found(self):
|
||||
with self.assertRaisesRegex(
|
||||
EnvironmentError, r"aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"
|
||||
):
|
||||
_ = AutoFeatureExtractor.from_pretrained(DUMMY_UNKNOWN_IDENTIFIER, revision="aaaaaa")
|
||||
|
||||
def test_feature_extractor_not_found(self):
|
||||
with self.assertRaisesRegex(
|
||||
EnvironmentError,
|
||||
"hf-internal-testing/config-no-model does not appear to have a file named preprocessor_config.json.",
|
||||
):
|
||||
_ = AutoFeatureExtractor.from_pretrained("hf-internal-testing/config-no-model")
|
||||
|
||||
def test_from_pretrained_dynamic_feature_extractor(self):
|
||||
model = AutoFeatureExtractor.from_pretrained(
|
||||
"hf-internal-testing/test_dynamic_feature_extractor", trust_remote_code=True
|
||||
)
|
||||
self.assertEqual(model.__class__.__name__, "NewFeatureExtractor")
|
||||
|
||||
def test_new_feature_extractor_registration(self):
|
||||
try:
|
||||
AutoConfig.register("custom", CustomConfig)
|
||||
AutoFeatureExtractor.register(CustomConfig, CustomFeatureExtractor)
|
||||
# Trying to register something existing in the Transformers library will raise an error
|
||||
with self.assertRaises(ValueError):
|
||||
AutoFeatureExtractor.register(Wav2Vec2Config, Wav2Vec2FeatureExtractor)
|
||||
|
||||
# Now that the config is registered, it can be used as any other config with the auto-API
|
||||
feature_extractor = CustomFeatureExtractor.from_pretrained(SAMPLE_FEATURE_EXTRACTION_CONFIG_DIR)
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
feature_extractor.save_pretrained(tmp_dir)
|
||||
new_feature_extractor = AutoFeatureExtractor.from_pretrained(tmp_dir)
|
||||
self.assertIsInstance(new_feature_extractor, CustomFeatureExtractor)
|
||||
|
||||
finally:
|
||||
if "custom" in CONFIG_MAPPING._extra_content:
|
||||
del CONFIG_MAPPING._extra_content["custom"]
|
||||
if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
|
||||
del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]
|
||||
356
tests/models/auto/test_modeling_auto.py
Normal file
356
tests/models/auto/test_modeling_auto.py
Normal file
@@ -0,0 +1,356 @@
|
||||
# coding=utf-8
|
||||
# Copyright 2020 The HuggingFace Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import copy
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from transformers import BertConfig, is_torch_available
|
||||
from transformers.models.auto.configuration_auto import CONFIG_MAPPING
|
||||
from transformers.testing_utils import (
|
||||
DUMMY_UNKNOWN_IDENTIFIER,
|
||||
SMALL_MODEL_IDENTIFIER,
|
||||
require_scatter,
|
||||
require_torch,
|
||||
slow,
|
||||
)
|
||||
|
||||
from ..bert.test_modeling_bert import BertModelTester
|
||||
|
||||
|
||||
sys.path.append(str(Path(__file__).parent.parent.parent.parent / "utils"))
|
||||
|
||||
from test_module.custom_configuration import CustomConfig # noqa E402
|
||||
|
||||
|
||||
if is_torch_available():
|
||||
import torch
|
||||
|
||||
from test_module.custom_modeling import CustomModel
|
||||
from transformers import (
|
||||
AutoConfig,
|
||||
AutoModel,
|
||||
AutoModelForCausalLM,
|
||||
AutoModelForMaskedLM,
|
||||
AutoModelForPreTraining,
|
||||
AutoModelForQuestionAnswering,
|
||||
AutoModelForSeq2SeqLM,
|
||||
AutoModelForSequenceClassification,
|
||||
AutoModelForTableQuestionAnswering,
|
||||
AutoModelForTokenClassification,
|
||||
AutoModelWithLMHead,
|
||||
BertForMaskedLM,
|
||||
BertForPreTraining,
|
||||
BertForQuestionAnswering,
|
||||
BertForSequenceClassification,
|
||||
BertForTokenClassification,
|
||||
BertModel,
|
||||
FunnelBaseModel,
|
||||
FunnelModel,
|
||||
GPT2Config,
|
||||
GPT2LMHeadModel,
|
||||
RobertaForMaskedLM,
|
||||
T5Config,
|
||||
T5ForConditionalGeneration,
|
||||
TapasConfig,
|
||||
TapasForQuestionAnswering,
|
||||
)
|
||||
from transformers.models.auto.modeling_auto import (
|
||||
MODEL_FOR_CAUSAL_LM_MAPPING,
|
||||
MODEL_FOR_MASKED_LM_MAPPING,
|
||||
MODEL_FOR_PRETRAINING_MAPPING,
|
||||
MODEL_FOR_QUESTION_ANSWERING_MAPPING,
|
||||
MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
|
||||
MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
|
||||
MODEL_MAPPING,
|
||||
)
|
||||
from transformers.models.bert.modeling_bert import BERT_PRETRAINED_MODEL_ARCHIVE_LIST
|
||||
from transformers.models.gpt2.modeling_gpt2 import GPT2_PRETRAINED_MODEL_ARCHIVE_LIST
|
||||
from transformers.models.t5.modeling_t5 import T5_PRETRAINED_MODEL_ARCHIVE_LIST
|
||||
from transformers.models.tapas.modeling_tapas import TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST
|
||||
|
||||
|
||||
@require_torch
|
||||
class AutoModelTest(unittest.TestCase):
|
||||
@slow
|
||||
def test_model_from_pretrained(self):
|
||||
for model_name in BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
|
||||
config = AutoConfig.from_pretrained(model_name)
|
||||
self.assertIsNotNone(config)
|
||||
self.assertIsInstance(config, BertConfig)
|
||||
|
||||
model = AutoModel.from_pretrained(model_name)
|
||||
model, loading_info = AutoModel.from_pretrained(model_name, output_loading_info=True)
|
||||
self.assertIsNotNone(model)
|
||||
self.assertIsInstance(model, BertModel)
|
||||
|
||||
self.assertEqual(len(loading_info["missing_keys"]), 0)
|
||||
self.assertEqual(len(loading_info["unexpected_keys"]), 8)
|
||||
self.assertEqual(len(loading_info["mismatched_keys"]), 0)
|
||||
self.assertEqual(len(loading_info["error_msgs"]), 0)
|
||||
|
||||
@slow
|
||||
def test_model_for_pretraining_from_pretrained(self):
|
||||
for model_name in BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
|
||||
config = AutoConfig.from_pretrained(model_name)
|
||||
self.assertIsNotNone(config)
|
||||
self.assertIsInstance(config, BertConfig)
|
||||
|
||||
model = AutoModelForPreTraining.from_pretrained(model_name)
|
||||
model, loading_info = AutoModelForPreTraining.from_pretrained(model_name, output_loading_info=True)
|
||||
self.assertIsNotNone(model)
|
||||
self.assertIsInstance(model, BertForPreTraining)
|
||||
# Only one value should not be initialized and in the missing keys.
|
||||
missing_keys = loading_info.pop("missing_keys")
|
||||
self.assertListEqual(["cls.predictions.decoder.bias"], missing_keys)
|
||||
for key, value in loading_info.items():
|
||||
self.assertEqual(len(value), 0)
|
||||
|
||||
@slow
|
||||
def test_lmhead_model_from_pretrained(self):
|
||||
for model_name in BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
|
||||
config = AutoConfig.from_pretrained(model_name)
|
||||
self.assertIsNotNone(config)
|
||||
self.assertIsInstance(config, BertConfig)
|
||||
|
||||
model = AutoModelWithLMHead.from_pretrained(model_name)
|
||||
model, loading_info = AutoModelWithLMHead.from_pretrained(model_name, output_loading_info=True)
|
||||
self.assertIsNotNone(model)
|
||||
self.assertIsInstance(model, BertForMaskedLM)
|
||||
|
||||
@slow
|
||||
def test_model_for_causal_lm(self):
|
||||
for model_name in GPT2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
|
||||
config = AutoConfig.from_pretrained(model_name)
|
||||
self.assertIsNotNone(config)
|
||||
self.assertIsInstance(config, GPT2Config)
|
||||
|
||||
model = AutoModelForCausalLM.from_pretrained(model_name)
|
||||
model, loading_info = AutoModelForCausalLM.from_pretrained(model_name, output_loading_info=True)
|
||||
self.assertIsNotNone(model)
|
||||
self.assertIsInstance(model, GPT2LMHeadModel)
|
||||
|
||||
@slow
|
||||
def test_model_for_masked_lm(self):
|
||||
for model_name in BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
|
||||
config = AutoConfig.from_pretrained(model_name)
|
||||
self.assertIsNotNone(config)
|
||||
self.assertIsInstance(config, BertConfig)
|
||||
|
||||
model = AutoModelForMaskedLM.from_pretrained(model_name)
|
||||
model, loading_info = AutoModelForMaskedLM.from_pretrained(model_name, output_loading_info=True)
|
||||
self.assertIsNotNone(model)
|
||||
self.assertIsInstance(model, BertForMaskedLM)
|
||||
|
||||
@slow
|
||||
def test_model_for_encoder_decoder_lm(self):
|
||||
for model_name in T5_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
|
||||
config = AutoConfig.from_pretrained(model_name)
|
||||
self.assertIsNotNone(config)
|
||||
self.assertIsInstance(config, T5Config)
|
||||
|
||||
model = AutoModelForSeq2SeqLM.from_pretrained(model_name)
|
||||
model, loading_info = AutoModelForSeq2SeqLM.from_pretrained(model_name, output_loading_info=True)
|
||||
self.assertIsNotNone(model)
|
||||
self.assertIsInstance(model, T5ForConditionalGeneration)
|
||||
|
||||
@slow
|
||||
def test_sequence_classification_model_from_pretrained(self):
|
||||
for model_name in BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
|
||||
config = AutoConfig.from_pretrained(model_name)
|
||||
self.assertIsNotNone(config)
|
||||
self.assertIsInstance(config, BertConfig)
|
||||
|
||||
model = AutoModelForSequenceClassification.from_pretrained(model_name)
|
||||
model, loading_info = AutoModelForSequenceClassification.from_pretrained(
|
||||
model_name, output_loading_info=True
|
||||
)
|
||||
self.assertIsNotNone(model)
|
||||
self.assertIsInstance(model, BertForSequenceClassification)
|
||||
|
||||
@slow
|
||||
def test_question_answering_model_from_pretrained(self):
|
||||
for model_name in BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
|
||||
config = AutoConfig.from_pretrained(model_name)
|
||||
self.assertIsNotNone(config)
|
||||
self.assertIsInstance(config, BertConfig)
|
||||
|
||||
model = AutoModelForQuestionAnswering.from_pretrained(model_name)
|
||||
model, loading_info = AutoModelForQuestionAnswering.from_pretrained(model_name, output_loading_info=True)
|
||||
self.assertIsNotNone(model)
|
||||
self.assertIsInstance(model, BertForQuestionAnswering)
|
||||
|
||||
@slow
|
||||
@require_scatter
|
||||
def test_table_question_answering_model_from_pretrained(self):
|
||||
for model_name in TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST[5:6]:
|
||||
config = AutoConfig.from_pretrained(model_name)
|
||||
self.assertIsNotNone(config)
|
||||
self.assertIsInstance(config, TapasConfig)
|
||||
|
||||
model = AutoModelForTableQuestionAnswering.from_pretrained(model_name)
|
||||
model, loading_info = AutoModelForTableQuestionAnswering.from_pretrained(
|
||||
model_name, output_loading_info=True
|
||||
)
|
||||
self.assertIsNotNone(model)
|
||||
self.assertIsInstance(model, TapasForQuestionAnswering)
|
||||
|
||||
@slow
|
||||
def test_token_classification_model_from_pretrained(self):
|
||||
for model_name in BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
|
||||
config = AutoConfig.from_pretrained(model_name)
|
||||
self.assertIsNotNone(config)
|
||||
self.assertIsInstance(config, BertConfig)
|
||||
|
||||
model = AutoModelForTokenClassification.from_pretrained(model_name)
|
||||
model, loading_info = AutoModelForTokenClassification.from_pretrained(model_name, output_loading_info=True)
|
||||
self.assertIsNotNone(model)
|
||||
self.assertIsInstance(model, BertForTokenClassification)
|
||||
|
||||
def test_from_pretrained_identifier(self):
|
||||
model = AutoModelWithLMHead.from_pretrained(SMALL_MODEL_IDENTIFIER)
|
||||
self.assertIsInstance(model, BertForMaskedLM)
|
||||
self.assertEqual(model.num_parameters(), 14410)
|
||||
self.assertEqual(model.num_parameters(only_trainable=True), 14410)
|
||||
|
||||
def test_from_identifier_from_model_type(self):
|
||||
model = AutoModelWithLMHead.from_pretrained(DUMMY_UNKNOWN_IDENTIFIER)
|
||||
self.assertIsInstance(model, RobertaForMaskedLM)
|
||||
self.assertEqual(model.num_parameters(), 14410)
|
||||
self.assertEqual(model.num_parameters(only_trainable=True), 14410)
|
||||
|
||||
def test_from_pretrained_with_tuple_values(self):
|
||||
# For the auto model mapping, FunnelConfig has two models: FunnelModel and FunnelBaseModel
|
||||
model = AutoModel.from_pretrained("sgugger/funnel-random-tiny")
|
||||
self.assertIsInstance(model, FunnelModel)
|
||||
|
||||
config = copy.deepcopy(model.config)
|
||||
config.architectures = ["FunnelBaseModel"]
|
||||
model = AutoModel.from_config(config)
|
||||
self.assertIsInstance(model, FunnelBaseModel)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
model.save_pretrained(tmp_dir)
|
||||
model = AutoModel.from_pretrained(tmp_dir)
|
||||
self.assertIsInstance(model, FunnelBaseModel)
|
||||
|
||||
def test_from_pretrained_dynamic_model_local(self):
|
||||
try:
|
||||
AutoConfig.register("custom", CustomConfig)
|
||||
AutoModel.register(CustomConfig, CustomModel)
|
||||
|
||||
config = CustomConfig(hidden_size=32)
|
||||
model = CustomModel(config)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
model.save_pretrained(tmp_dir)
|
||||
|
||||
new_model = AutoModel.from_pretrained(tmp_dir, trust_remote_code=True)
|
||||
for p1, p2 in zip(model.parameters(), new_model.parameters()):
|
||||
self.assertTrue(torch.equal(p1, p2))
|
||||
|
||||
finally:
|
||||
if "custom" in CONFIG_MAPPING._extra_content:
|
||||
del CONFIG_MAPPING._extra_content["custom"]
|
||||
if CustomConfig in MODEL_MAPPING._extra_content:
|
||||
del MODEL_MAPPING._extra_content[CustomConfig]
|
||||
|
||||
def test_from_pretrained_dynamic_model_distant(self):
|
||||
model = AutoModel.from_pretrained("hf-internal-testing/test_dynamic_model", trust_remote_code=True)
|
||||
self.assertEqual(model.__class__.__name__, "NewModel")
|
||||
|
||||
# This one uses a relative import to a util file, this checks it is downloaded and used properly.
|
||||
model = AutoModel.from_pretrained("hf-internal-testing/test_dynamic_model_with_util", trust_remote_code=True)
|
||||
self.assertEqual(model.__class__.__name__, "NewModel")
|
||||
|
||||
def test_new_model_registration(self):
|
||||
AutoConfig.register("custom", CustomConfig)
|
||||
|
||||
auto_classes = [
|
||||
AutoModel,
|
||||
AutoModelForCausalLM,
|
||||
AutoModelForMaskedLM,
|
||||
AutoModelForPreTraining,
|
||||
AutoModelForQuestionAnswering,
|
||||
AutoModelForSequenceClassification,
|
||||
AutoModelForTokenClassification,
|
||||
]
|
||||
|
||||
try:
|
||||
for auto_class in auto_classes:
|
||||
with self.subTest(auto_class.__name__):
|
||||
# Wrong config class will raise an error
|
||||
with self.assertRaises(ValueError):
|
||||
auto_class.register(BertConfig, CustomModel)
|
||||
auto_class.register(CustomConfig, CustomModel)
|
||||
# Trying to register something existing in the Transformers library will raise an error
|
||||
with self.assertRaises(ValueError):
|
||||
auto_class.register(BertConfig, BertModel)
|
||||
|
||||
# Now that the config is registered, it can be used as any other config with the auto-API
|
||||
tiny_config = BertModelTester(self).get_config()
|
||||
config = CustomConfig(**tiny_config.to_dict())
|
||||
model = auto_class.from_config(config)
|
||||
self.assertIsInstance(model, CustomModel)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
model.save_pretrained(tmp_dir)
|
||||
new_model = auto_class.from_pretrained(tmp_dir)
|
||||
# The model is a CustomModel but from the new dynamically imported class.
|
||||
self.assertIsInstance(new_model, CustomModel)
|
||||
|
||||
finally:
|
||||
if "custom" in CONFIG_MAPPING._extra_content:
|
||||
del CONFIG_MAPPING._extra_content["custom"]
|
||||
for mapping in (
|
||||
MODEL_MAPPING,
|
||||
MODEL_FOR_PRETRAINING_MAPPING,
|
||||
MODEL_FOR_QUESTION_ANSWERING_MAPPING,
|
||||
MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
|
||||
MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
|
||||
MODEL_FOR_CAUSAL_LM_MAPPING,
|
||||
MODEL_FOR_MASKED_LM_MAPPING,
|
||||
):
|
||||
if CustomConfig in mapping._extra_content:
|
||||
del mapping._extra_content[CustomConfig]
|
||||
|
||||
def test_repo_not_found(self):
|
||||
with self.assertRaisesRegex(
|
||||
EnvironmentError, "bert-base is not a local folder and is not a valid model identifier"
|
||||
):
|
||||
_ = AutoModel.from_pretrained("bert-base")
|
||||
|
||||
def test_revision_not_found(self):
|
||||
with self.assertRaisesRegex(
|
||||
EnvironmentError, r"aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"
|
||||
):
|
||||
_ = AutoModel.from_pretrained(DUMMY_UNKNOWN_IDENTIFIER, revision="aaaaaa")
|
||||
|
||||
def test_model_file_not_found(self):
|
||||
with self.assertRaisesRegex(
|
||||
EnvironmentError,
|
||||
"hf-internal-testing/config-no-model does not appear to have a file named pytorch_model.bin",
|
||||
):
|
||||
_ = AutoModel.from_pretrained("hf-internal-testing/config-no-model")
|
||||
|
||||
def test_model_from_tf_suggestion(self):
|
||||
with self.assertRaisesRegex(EnvironmentError, "Use `from_tf=True` to load this model"):
|
||||
_ = AutoModel.from_pretrained("hf-internal-testing/tiny-bert-tf-only")
|
||||
|
||||
def test_model_from_flax_suggestion(self):
|
||||
with self.assertRaisesRegex(EnvironmentError, "Use `from_flax=True` to load this model"):
|
||||
_ = AutoModel.from_pretrained("hf-internal-testing/tiny-bert-flax-only")
|
||||
101
tests/models/auto/test_modeling_flax_auto.py
Normal file
101
tests/models/auto/test_modeling_flax_auto.py
Normal file
@@ -0,0 +1,101 @@
|
||||
# Copyright 2020 The HuggingFace Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import unittest
|
||||
|
||||
from transformers import AutoConfig, AutoTokenizer, BertConfig, TensorType, is_flax_available
|
||||
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, require_flax, slow
|
||||
|
||||
|
||||
if is_flax_available():
|
||||
import jax
|
||||
from transformers.models.auto.modeling_flax_auto import FlaxAutoModel
|
||||
from transformers.models.bert.modeling_flax_bert import FlaxBertModel
|
||||
from transformers.models.roberta.modeling_flax_roberta import FlaxRobertaModel
|
||||
|
||||
|
||||
@require_flax
|
||||
class FlaxAutoModelTest(unittest.TestCase):
|
||||
@slow
|
||||
def test_bert_from_pretrained(self):
|
||||
for model_name in ["bert-base-cased", "bert-large-uncased"]:
|
||||
with self.subTest(model_name):
|
||||
config = AutoConfig.from_pretrained(model_name)
|
||||
self.assertIsNotNone(config)
|
||||
self.assertIsInstance(config, BertConfig)
|
||||
|
||||
model = FlaxAutoModel.from_pretrained(model_name)
|
||||
self.assertIsNotNone(model)
|
||||
self.assertIsInstance(model, FlaxBertModel)
|
||||
|
||||
@slow
|
||||
def test_roberta_from_pretrained(self):
|
||||
for model_name in ["roberta-base", "roberta-large"]:
|
||||
with self.subTest(model_name):
|
||||
config = AutoConfig.from_pretrained(model_name)
|
||||
self.assertIsNotNone(config)
|
||||
self.assertIsInstance(config, BertConfig)
|
||||
|
||||
model = FlaxAutoModel.from_pretrained(model_name)
|
||||
self.assertIsNotNone(model)
|
||||
self.assertIsInstance(model, FlaxRobertaModel)
|
||||
|
||||
@slow
|
||||
def test_bert_jax_jit(self):
|
||||
for model_name in ["bert-base-cased", "bert-large-uncased"]:
|
||||
tokenizer = AutoTokenizer.from_pretrained(model_name)
|
||||
model = FlaxBertModel.from_pretrained(model_name)
|
||||
tokens = tokenizer("Do you support jax jitted function?", return_tensors=TensorType.JAX)
|
||||
|
||||
@jax.jit
|
||||
def eval(**kwargs):
|
||||
return model(**kwargs)
|
||||
|
||||
eval(**tokens).block_until_ready()
|
||||
|
||||
@slow
|
||||
def test_roberta_jax_jit(self):
|
||||
for model_name in ["roberta-base", "roberta-large"]:
|
||||
tokenizer = AutoTokenizer.from_pretrained(model_name)
|
||||
model = FlaxRobertaModel.from_pretrained(model_name)
|
||||
tokens = tokenizer("Do you support jax jitted function?", return_tensors=TensorType.JAX)
|
||||
|
||||
@jax.jit
|
||||
def eval(**kwargs):
|
||||
return model(**kwargs)
|
||||
|
||||
eval(**tokens).block_until_ready()
|
||||
|
||||
def test_repo_not_found(self):
|
||||
with self.assertRaisesRegex(
|
||||
EnvironmentError, "bert-base is not a local folder and is not a valid model identifier"
|
||||
):
|
||||
_ = FlaxAutoModel.from_pretrained("bert-base")
|
||||
|
||||
def test_revision_not_found(self):
|
||||
with self.assertRaisesRegex(
|
||||
EnvironmentError, r"aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"
|
||||
):
|
||||
_ = FlaxAutoModel.from_pretrained(DUMMY_UNKNOWN_IDENTIFIER, revision="aaaaaa")
|
||||
|
||||
def test_model_file_not_found(self):
|
||||
with self.assertRaisesRegex(
|
||||
EnvironmentError,
|
||||
"hf-internal-testing/config-no-model does not appear to have a file named flax_model.msgpack",
|
||||
):
|
||||
_ = FlaxAutoModel.from_pretrained("hf-internal-testing/config-no-model")
|
||||
|
||||
def test_model_from_pt_suggestion(self):
|
||||
with self.assertRaisesRegex(EnvironmentError, "Use `from_pt=True` to load this model"):
|
||||
_ = FlaxAutoModel.from_pretrained("hf-internal-testing/tiny-bert-pt-only")
|
||||
289
tests/models/auto/test_modeling_tf_auto.py
Normal file
289
tests/models/auto/test_modeling_tf_auto.py
Normal file
@@ -0,0 +1,289 @@
|
||||
# coding=utf-8
|
||||
# Copyright 2020 The HuggingFace Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import copy
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from transformers import CONFIG_MAPPING, AutoConfig, BertConfig, GPT2Config, T5Config, TapasConfig, is_tf_available
|
||||
from transformers.testing_utils import (
|
||||
DUMMY_UNKNOWN_IDENTIFIER,
|
||||
SMALL_MODEL_IDENTIFIER,
|
||||
require_tensorflow_probability,
|
||||
require_tf,
|
||||
slow,
|
||||
)
|
||||
|
||||
from ..bert.test_modeling_bert import BertModelTester
|
||||
|
||||
|
||||
if is_tf_available():
|
||||
from transformers import (
|
||||
TFAutoModel,
|
||||
TFAutoModelForCausalLM,
|
||||
TFAutoModelForMaskedLM,
|
||||
TFAutoModelForPreTraining,
|
||||
TFAutoModelForQuestionAnswering,
|
||||
TFAutoModelForSeq2SeqLM,
|
||||
TFAutoModelForSequenceClassification,
|
||||
TFAutoModelForTableQuestionAnswering,
|
||||
TFAutoModelForTokenClassification,
|
||||
TFAutoModelWithLMHead,
|
||||
TFBertForMaskedLM,
|
||||
TFBertForPreTraining,
|
||||
TFBertForQuestionAnswering,
|
||||
TFBertForSequenceClassification,
|
||||
TFBertModel,
|
||||
TFFunnelBaseModel,
|
||||
TFFunnelModel,
|
||||
TFGPT2LMHeadModel,
|
||||
TFRobertaForMaskedLM,
|
||||
TFT5ForConditionalGeneration,
|
||||
TFTapasForQuestionAnswering,
|
||||
)
|
||||
from transformers.models.auto.modeling_tf_auto import (
|
||||
TF_MODEL_FOR_CAUSAL_LM_MAPPING,
|
||||
TF_MODEL_FOR_MASKED_LM_MAPPING,
|
||||
TF_MODEL_FOR_PRETRAINING_MAPPING,
|
||||
TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
|
||||
TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
|
||||
TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
|
||||
TF_MODEL_MAPPING,
|
||||
)
|
||||
from transformers.models.bert.modeling_tf_bert import TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST
|
||||
from transformers.models.gpt2.modeling_tf_gpt2 import TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST
|
||||
from transformers.models.t5.modeling_tf_t5 import TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST
|
||||
from transformers.models.tapas.modeling_tf_tapas import TF_TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST
|
||||
|
||||
|
||||
class NewModelConfig(BertConfig):
|
||||
model_type = "new-model"
|
||||
|
||||
|
||||
if is_tf_available():
|
||||
|
||||
class TFNewModel(TFBertModel):
|
||||
config_class = NewModelConfig
|
||||
|
||||
|
||||
@require_tf
|
||||
class TFAutoModelTest(unittest.TestCase):
|
||||
@slow
|
||||
def test_model_from_pretrained(self):
|
||||
model_name = "bert-base-cased"
|
||||
config = AutoConfig.from_pretrained(model_name)
|
||||
self.assertIsNotNone(config)
|
||||
self.assertIsInstance(config, BertConfig)
|
||||
|
||||
model = TFAutoModel.from_pretrained(model_name)
|
||||
self.assertIsNotNone(model)
|
||||
self.assertIsInstance(model, TFBertModel)
|
||||
|
||||
@slow
|
||||
def test_model_for_pretraining_from_pretrained(self):
|
||||
model_name = "bert-base-cased"
|
||||
config = AutoConfig.from_pretrained(model_name)
|
||||
self.assertIsNotNone(config)
|
||||
self.assertIsInstance(config, BertConfig)
|
||||
|
||||
model = TFAutoModelForPreTraining.from_pretrained(model_name)
|
||||
self.assertIsNotNone(model)
|
||||
self.assertIsInstance(model, TFBertForPreTraining)
|
||||
|
||||
@slow
|
||||
def test_model_for_causal_lm(self):
|
||||
for model_name in TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
|
||||
config = AutoConfig.from_pretrained(model_name)
|
||||
self.assertIsNotNone(config)
|
||||
self.assertIsInstance(config, GPT2Config)
|
||||
|
||||
model = TFAutoModelForCausalLM.from_pretrained(model_name)
|
||||
model, loading_info = TFAutoModelForCausalLM.from_pretrained(model_name, output_loading_info=True)
|
||||
self.assertIsNotNone(model)
|
||||
self.assertIsInstance(model, TFGPT2LMHeadModel)
|
||||
|
||||
@slow
|
||||
def test_lmhead_model_from_pretrained(self):
|
||||
for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
|
||||
config = AutoConfig.from_pretrained(model_name)
|
||||
self.assertIsNotNone(config)
|
||||
self.assertIsInstance(config, BertConfig)
|
||||
|
||||
model = TFAutoModelWithLMHead.from_pretrained(model_name)
|
||||
self.assertIsNotNone(model)
|
||||
self.assertIsInstance(model, TFBertForMaskedLM)
|
||||
|
||||
@slow
|
||||
def test_model_for_masked_lm(self):
|
||||
for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
|
||||
config = AutoConfig.from_pretrained(model_name)
|
||||
self.assertIsNotNone(config)
|
||||
self.assertIsInstance(config, BertConfig)
|
||||
|
||||
model = TFAutoModelForMaskedLM.from_pretrained(model_name)
|
||||
model, loading_info = TFAutoModelForMaskedLM.from_pretrained(model_name, output_loading_info=True)
|
||||
self.assertIsNotNone(model)
|
||||
self.assertIsInstance(model, TFBertForMaskedLM)
|
||||
|
||||
@slow
|
||||
def test_model_for_encoder_decoder_lm(self):
|
||||
for model_name in TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
|
||||
config = AutoConfig.from_pretrained(model_name)
|
||||
self.assertIsNotNone(config)
|
||||
self.assertIsInstance(config, T5Config)
|
||||
|
||||
model = TFAutoModelForSeq2SeqLM.from_pretrained(model_name)
|
||||
model, loading_info = TFAutoModelForSeq2SeqLM.from_pretrained(model_name, output_loading_info=True)
|
||||
self.assertIsNotNone(model)
|
||||
self.assertIsInstance(model, TFT5ForConditionalGeneration)
|
||||
|
||||
@slow
|
||||
def test_sequence_classification_model_from_pretrained(self):
|
||||
# for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
|
||||
for model_name in ["bert-base-uncased"]:
|
||||
config = AutoConfig.from_pretrained(model_name)
|
||||
self.assertIsNotNone(config)
|
||||
self.assertIsInstance(config, BertConfig)
|
||||
|
||||
model = TFAutoModelForSequenceClassification.from_pretrained(model_name)
|
||||
self.assertIsNotNone(model)
|
||||
self.assertIsInstance(model, TFBertForSequenceClassification)
|
||||
|
||||
@slow
|
||||
def test_question_answering_model_from_pretrained(self):
|
||||
# for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
|
||||
for model_name in ["bert-base-uncased"]:
|
||||
config = AutoConfig.from_pretrained(model_name)
|
||||
self.assertIsNotNone(config)
|
||||
self.assertIsInstance(config, BertConfig)
|
||||
|
||||
model = TFAutoModelForQuestionAnswering.from_pretrained(model_name)
|
||||
self.assertIsNotNone(model)
|
||||
self.assertIsInstance(model, TFBertForQuestionAnswering)
|
||||
|
||||
@slow
|
||||
@require_tensorflow_probability
|
||||
def test_table_question_answering_model_from_pretrained(self):
|
||||
for model_name in TF_TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST[5:6]:
|
||||
config = AutoConfig.from_pretrained(model_name)
|
||||
self.assertIsNotNone(config)
|
||||
self.assertIsInstance(config, TapasConfig)
|
||||
|
||||
model = TFAutoModelForTableQuestionAnswering.from_pretrained(model_name)
|
||||
model, loading_info = TFAutoModelForTableQuestionAnswering.from_pretrained(
|
||||
model_name, output_loading_info=True
|
||||
)
|
||||
self.assertIsNotNone(model)
|
||||
self.assertIsInstance(model, TFTapasForQuestionAnswering)
|
||||
|
||||
def test_from_pretrained_identifier(self):
|
||||
model = TFAutoModelWithLMHead.from_pretrained(SMALL_MODEL_IDENTIFIER)
|
||||
self.assertIsInstance(model, TFBertForMaskedLM)
|
||||
self.assertEqual(model.num_parameters(), 14410)
|
||||
self.assertEqual(model.num_parameters(only_trainable=True), 14410)
|
||||
|
||||
def test_from_identifier_from_model_type(self):
|
||||
model = TFAutoModelWithLMHead.from_pretrained(DUMMY_UNKNOWN_IDENTIFIER)
|
||||
self.assertIsInstance(model, TFRobertaForMaskedLM)
|
||||
self.assertEqual(model.num_parameters(), 14410)
|
||||
self.assertEqual(model.num_parameters(only_trainable=True), 14410)
|
||||
|
||||
def test_from_pretrained_with_tuple_values(self):
|
||||
# For the auto model mapping, FunnelConfig has two models: FunnelModel and FunnelBaseModel
|
||||
model = TFAutoModel.from_pretrained("sgugger/funnel-random-tiny")
|
||||
self.assertIsInstance(model, TFFunnelModel)
|
||||
|
||||
config = copy.deepcopy(model.config)
|
||||
config.architectures = ["FunnelBaseModel"]
|
||||
model = TFAutoModel.from_config(config)
|
||||
self.assertIsInstance(model, TFFunnelBaseModel)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
model.save_pretrained(tmp_dir)
|
||||
model = TFAutoModel.from_pretrained(tmp_dir)
|
||||
self.assertIsInstance(model, TFFunnelBaseModel)
|
||||
|
||||
def test_new_model_registration(self):
|
||||
try:
|
||||
AutoConfig.register("new-model", NewModelConfig)
|
||||
|
||||
auto_classes = [
|
||||
TFAutoModel,
|
||||
TFAutoModelForCausalLM,
|
||||
TFAutoModelForMaskedLM,
|
||||
TFAutoModelForPreTraining,
|
||||
TFAutoModelForQuestionAnswering,
|
||||
TFAutoModelForSequenceClassification,
|
||||
TFAutoModelForTokenClassification,
|
||||
]
|
||||
|
||||
for auto_class in auto_classes:
|
||||
with self.subTest(auto_class.__name__):
|
||||
# Wrong config class will raise an error
|
||||
with self.assertRaises(ValueError):
|
||||
auto_class.register(BertConfig, TFNewModel)
|
||||
auto_class.register(NewModelConfig, TFNewModel)
|
||||
# Trying to register something existing in the Transformers library will raise an error
|
||||
with self.assertRaises(ValueError):
|
||||
auto_class.register(BertConfig, TFBertModel)
|
||||
|
||||
# Now that the config is registered, it can be used as any other config with the auto-API
|
||||
tiny_config = BertModelTester(self).get_config()
|
||||
config = NewModelConfig(**tiny_config.to_dict())
|
||||
model = auto_class.from_config(config)
|
||||
self.assertIsInstance(model, TFNewModel)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
model.save_pretrained(tmp_dir)
|
||||
new_model = auto_class.from_pretrained(tmp_dir)
|
||||
self.assertIsInstance(new_model, TFNewModel)
|
||||
|
||||
finally:
|
||||
if "new-model" in CONFIG_MAPPING._extra_content:
|
||||
del CONFIG_MAPPING._extra_content["new-model"]
|
||||
for mapping in (
|
||||
TF_MODEL_MAPPING,
|
||||
TF_MODEL_FOR_PRETRAINING_MAPPING,
|
||||
TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
|
||||
TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
|
||||
TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
|
||||
TF_MODEL_FOR_CAUSAL_LM_MAPPING,
|
||||
TF_MODEL_FOR_MASKED_LM_MAPPING,
|
||||
):
|
||||
if NewModelConfig in mapping._extra_content:
|
||||
del mapping._extra_content[NewModelConfig]
|
||||
|
||||
def test_repo_not_found(self):
|
||||
with self.assertRaisesRegex(
|
||||
EnvironmentError, "bert-base is not a local folder and is not a valid model identifier"
|
||||
):
|
||||
_ = TFAutoModel.from_pretrained("bert-base")
|
||||
|
||||
def test_revision_not_found(self):
|
||||
with self.assertRaisesRegex(
|
||||
EnvironmentError, r"aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"
|
||||
):
|
||||
_ = TFAutoModel.from_pretrained(DUMMY_UNKNOWN_IDENTIFIER, revision="aaaaaa")
|
||||
|
||||
def test_model_file_not_found(self):
|
||||
with self.assertRaisesRegex(
|
||||
EnvironmentError,
|
||||
"hf-internal-testing/config-no-model does not appear to have a file named tf_model.h5",
|
||||
):
|
||||
_ = TFAutoModel.from_pretrained("hf-internal-testing/config-no-model")
|
||||
|
||||
def test_model_from_pt_suggestion(self):
|
||||
with self.assertRaisesRegex(EnvironmentError, "Use `from_pt=True` to load this model"):
|
||||
_ = TFAutoModel.from_pretrained("hf-internal-testing/tiny-bert-pt-only")
|
||||
235
tests/models/auto/test_modeling_tf_pytorch.py
Normal file
235
tests/models/auto/test_modeling_tf_pytorch.py
Normal file
@@ -0,0 +1,235 @@
|
||||
# coding=utf-8
|
||||
# Copyright 2020 The HuggingFace Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
import unittest
|
||||
|
||||
from transformers import is_tf_available, is_torch_available
|
||||
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, SMALL_MODEL_IDENTIFIER, is_pt_tf_cross_test, slow
|
||||
|
||||
|
||||
if is_tf_available():
|
||||
from transformers import (
|
||||
AutoConfig,
|
||||
BertConfig,
|
||||
GPT2Config,
|
||||
T5Config,
|
||||
TFAutoModel,
|
||||
TFAutoModelForCausalLM,
|
||||
TFAutoModelForMaskedLM,
|
||||
TFAutoModelForPreTraining,
|
||||
TFAutoModelForQuestionAnswering,
|
||||
TFAutoModelForSeq2SeqLM,
|
||||
TFAutoModelForSequenceClassification,
|
||||
TFAutoModelWithLMHead,
|
||||
TFBertForMaskedLM,
|
||||
TFBertForPreTraining,
|
||||
TFBertForQuestionAnswering,
|
||||
TFBertForSequenceClassification,
|
||||
TFBertModel,
|
||||
TFGPT2LMHeadModel,
|
||||
TFRobertaForMaskedLM,
|
||||
TFT5ForConditionalGeneration,
|
||||
)
|
||||
from transformers.models.bert.modeling_tf_bert import TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST
|
||||
from transformers.models.gpt2.modeling_tf_gpt2 import TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST
|
||||
from transformers.models.t5.modeling_tf_t5 import TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST
|
||||
|
||||
if is_torch_available():
|
||||
from transformers import (
|
||||
AutoModel,
|
||||
AutoModelForCausalLM,
|
||||
AutoModelForMaskedLM,
|
||||
AutoModelForPreTraining,
|
||||
AutoModelForQuestionAnswering,
|
||||
AutoModelForSeq2SeqLM,
|
||||
AutoModelForSequenceClassification,
|
||||
AutoModelWithLMHead,
|
||||
BertForMaskedLM,
|
||||
BertForPreTraining,
|
||||
BertForQuestionAnswering,
|
||||
BertForSequenceClassification,
|
||||
BertModel,
|
||||
GPT2LMHeadModel,
|
||||
RobertaForMaskedLM,
|
||||
T5ForConditionalGeneration,
|
||||
)
|
||||
|
||||
|
||||
@is_pt_tf_cross_test
|
||||
class TFPTAutoModelTest(unittest.TestCase):
|
||||
@slow
|
||||
def test_model_from_pretrained(self):
|
||||
# for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
|
||||
for model_name in ["bert-base-uncased"]:
|
||||
config = AutoConfig.from_pretrained(model_name)
|
||||
self.assertIsNotNone(config)
|
||||
self.assertIsInstance(config, BertConfig)
|
||||
|
||||
model = TFAutoModel.from_pretrained(model_name, from_pt=True)
|
||||
self.assertIsNotNone(model)
|
||||
self.assertIsInstance(model, TFBertModel)
|
||||
|
||||
model = AutoModel.from_pretrained(model_name, from_tf=True)
|
||||
self.assertIsNotNone(model)
|
||||
self.assertIsInstance(model, BertModel)
|
||||
|
||||
@slow
|
||||
def test_model_for_pretraining_from_pretrained(self):
|
||||
# for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
|
||||
for model_name in ["bert-base-uncased"]:
|
||||
config = AutoConfig.from_pretrained(model_name)
|
||||
self.assertIsNotNone(config)
|
||||
self.assertIsInstance(config, BertConfig)
|
||||
|
||||
model = TFAutoModelForPreTraining.from_pretrained(model_name, from_pt=True)
|
||||
self.assertIsNotNone(model)
|
||||
self.assertIsInstance(model, TFBertForPreTraining)
|
||||
|
||||
model = AutoModelForPreTraining.from_pretrained(model_name, from_tf=True)
|
||||
self.assertIsNotNone(model)
|
||||
self.assertIsInstance(model, BertForPreTraining)
|
||||
|
||||
@slow
|
||||
def test_model_for_causal_lm(self):
|
||||
for model_name in TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
|
||||
config = AutoConfig.from_pretrained(model_name)
|
||||
self.assertIsNotNone(config)
|
||||
self.assertIsInstance(config, GPT2Config)
|
||||
|
||||
model = TFAutoModelForCausalLM.from_pretrained(model_name, from_pt=True)
|
||||
model, loading_info = TFAutoModelForCausalLM.from_pretrained(
|
||||
model_name, output_loading_info=True, from_pt=True
|
||||
)
|
||||
self.assertIsNotNone(model)
|
||||
self.assertIsInstance(model, TFGPT2LMHeadModel)
|
||||
|
||||
model = AutoModelForCausalLM.from_pretrained(model_name, from_tf=True)
|
||||
model, loading_info = AutoModelForCausalLM.from_pretrained(
|
||||
model_name, output_loading_info=True, from_tf=True
|
||||
)
|
||||
self.assertIsNotNone(model)
|
||||
self.assertIsInstance(model, GPT2LMHeadModel)
|
||||
|
||||
@slow
|
||||
def test_lmhead_model_from_pretrained(self):
|
||||
for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
|
||||
config = AutoConfig.from_pretrained(model_name)
|
||||
self.assertIsNotNone(config)
|
||||
self.assertIsInstance(config, BertConfig)
|
||||
|
||||
model = TFAutoModelWithLMHead.from_pretrained(model_name, from_pt=True)
|
||||
self.assertIsNotNone(model)
|
||||
self.assertIsInstance(model, TFBertForMaskedLM)
|
||||
|
||||
model = AutoModelWithLMHead.from_pretrained(model_name, from_tf=True)
|
||||
self.assertIsNotNone(model)
|
||||
self.assertIsInstance(model, BertForMaskedLM)
|
||||
|
||||
@slow
|
||||
def test_model_for_masked_lm(self):
|
||||
for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
|
||||
config = AutoConfig.from_pretrained(model_name)
|
||||
self.assertIsNotNone(config)
|
||||
self.assertIsInstance(config, BertConfig)
|
||||
|
||||
model = TFAutoModelForMaskedLM.from_pretrained(model_name, from_pt=True)
|
||||
model, loading_info = TFAutoModelForMaskedLM.from_pretrained(
|
||||
model_name, output_loading_info=True, from_pt=True
|
||||
)
|
||||
self.assertIsNotNone(model)
|
||||
self.assertIsInstance(model, TFBertForMaskedLM)
|
||||
|
||||
model = AutoModelForMaskedLM.from_pretrained(model_name, from_tf=True)
|
||||
model, loading_info = AutoModelForMaskedLM.from_pretrained(
|
||||
model_name, output_loading_info=True, from_tf=True
|
||||
)
|
||||
self.assertIsNotNone(model)
|
||||
self.assertIsInstance(model, BertForMaskedLM)
|
||||
|
||||
@slow
|
||||
def test_model_for_encoder_decoder_lm(self):
|
||||
for model_name in TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
|
||||
config = AutoConfig.from_pretrained(model_name)
|
||||
self.assertIsNotNone(config)
|
||||
self.assertIsInstance(config, T5Config)
|
||||
|
||||
model = TFAutoModelForSeq2SeqLM.from_pretrained(model_name, from_pt=True)
|
||||
model, loading_info = TFAutoModelForSeq2SeqLM.from_pretrained(
|
||||
model_name, output_loading_info=True, from_pt=True
|
||||
)
|
||||
self.assertIsNotNone(model)
|
||||
self.assertIsInstance(model, TFT5ForConditionalGeneration)
|
||||
|
||||
model = AutoModelForSeq2SeqLM.from_pretrained(model_name, from_tf=True)
|
||||
model, loading_info = AutoModelForSeq2SeqLM.from_pretrained(
|
||||
model_name, output_loading_info=True, from_tf=True
|
||||
)
|
||||
self.assertIsNotNone(model)
|
||||
self.assertIsInstance(model, T5ForConditionalGeneration)
|
||||
|
||||
@slow
|
||||
def test_sequence_classification_model_from_pretrained(self):
|
||||
# for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
|
||||
for model_name in ["bert-base-uncased"]:
|
||||
config = AutoConfig.from_pretrained(model_name)
|
||||
self.assertIsNotNone(config)
|
||||
self.assertIsInstance(config, BertConfig)
|
||||
|
||||
model = TFAutoModelForSequenceClassification.from_pretrained(model_name, from_pt=True)
|
||||
self.assertIsNotNone(model)
|
||||
self.assertIsInstance(model, TFBertForSequenceClassification)
|
||||
|
||||
model = AutoModelForSequenceClassification.from_pretrained(model_name, from_tf=True)
|
||||
self.assertIsNotNone(model)
|
||||
self.assertIsInstance(model, BertForSequenceClassification)
|
||||
|
||||
@slow
|
||||
def test_question_answering_model_from_pretrained(self):
|
||||
# for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
|
||||
for model_name in ["bert-base-uncased"]:
|
||||
config = AutoConfig.from_pretrained(model_name)
|
||||
self.assertIsNotNone(config)
|
||||
self.assertIsInstance(config, BertConfig)
|
||||
|
||||
model = TFAutoModelForQuestionAnswering.from_pretrained(model_name, from_pt=True)
|
||||
self.assertIsNotNone(model)
|
||||
self.assertIsInstance(model, TFBertForQuestionAnswering)
|
||||
|
||||
model = AutoModelForQuestionAnswering.from_pretrained(model_name, from_tf=True)
|
||||
self.assertIsNotNone(model)
|
||||
self.assertIsInstance(model, BertForQuestionAnswering)
|
||||
|
||||
def test_from_pretrained_identifier(self):
|
||||
model = TFAutoModelWithLMHead.from_pretrained(SMALL_MODEL_IDENTIFIER, from_pt=True)
|
||||
self.assertIsInstance(model, TFBertForMaskedLM)
|
||||
self.assertEqual(model.num_parameters(), 14410)
|
||||
self.assertEqual(model.num_parameters(only_trainable=True), 14410)
|
||||
|
||||
model = AutoModelWithLMHead.from_pretrained(SMALL_MODEL_IDENTIFIER, from_tf=True)
|
||||
self.assertIsInstance(model, BertForMaskedLM)
|
||||
self.assertEqual(model.num_parameters(), 14410)
|
||||
self.assertEqual(model.num_parameters(only_trainable=True), 14410)
|
||||
|
||||
def test_from_identifier_from_model_type(self):
|
||||
model = TFAutoModelWithLMHead.from_pretrained(DUMMY_UNKNOWN_IDENTIFIER, from_pt=True)
|
||||
self.assertIsInstance(model, TFRobertaForMaskedLM)
|
||||
self.assertEqual(model.num_parameters(), 14410)
|
||||
self.assertEqual(model.num_parameters(only_trainable=True), 14410)
|
||||
|
||||
model = AutoModelWithLMHead.from_pretrained(DUMMY_UNKNOWN_IDENTIFIER, from_tf=True)
|
||||
self.assertIsInstance(model, RobertaForMaskedLM)
|
||||
self.assertEqual(model.num_parameters(), 14410)
|
||||
self.assertEqual(model.num_parameters(only_trainable=True), 14410)
|
||||
307
tests/models/auto/test_processor_auto.py
Normal file
307
tests/models/auto/test_processor_auto.py
Normal file
@@ -0,0 +1,307 @@
|
||||
# coding=utf-8
|
||||
# Copyright 2021 the HuggingFace Inc. team.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from shutil import copyfile
|
||||
|
||||
from huggingface_hub import Repository, delete_repo, login
|
||||
from requests.exceptions import HTTPError
|
||||
from transformers import (
|
||||
CONFIG_MAPPING,
|
||||
FEATURE_EXTRACTOR_MAPPING,
|
||||
PROCESSOR_MAPPING,
|
||||
TOKENIZER_MAPPING,
|
||||
AutoConfig,
|
||||
AutoFeatureExtractor,
|
||||
AutoProcessor,
|
||||
AutoTokenizer,
|
||||
Wav2Vec2Config,
|
||||
Wav2Vec2FeatureExtractor,
|
||||
Wav2Vec2Processor,
|
||||
)
|
||||
from transformers.testing_utils import PASS, USER, get_tests_dir, is_staging_test
|
||||
from transformers.tokenization_utils import TOKENIZER_CONFIG_FILE
|
||||
from transformers.utils import FEATURE_EXTRACTOR_NAME, is_tokenizers_available
|
||||
|
||||
|
||||
sys.path.append(str(Path(__file__).parent.parent.parent.parent / "utils"))
|
||||
|
||||
from test_module.custom_configuration import CustomConfig # noqa E402
|
||||
from test_module.custom_feature_extraction import CustomFeatureExtractor # noqa E402
|
||||
from test_module.custom_processing import CustomProcessor # noqa E402
|
||||
from test_module.custom_tokenization import CustomTokenizer # noqa E402
|
||||
|
||||
|
||||
SAMPLE_PROCESSOR_CONFIG = get_tests_dir("fixtures/dummy_feature_extractor_config.json")
|
||||
SAMPLE_VOCAB = get_tests_dir("fixtures/vocab.json")
|
||||
SAMPLE_PROCESSOR_CONFIG_DIR = get_tests_dir("fixtures")
|
||||
|
||||
|
||||
class AutoFeatureExtractorTest(unittest.TestCase):
|
||||
vocab_tokens = ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]
|
||||
|
||||
def test_processor_from_model_shortcut(self):
|
||||
processor = AutoProcessor.from_pretrained("facebook/wav2vec2-base-960h")
|
||||
self.assertIsInstance(processor, Wav2Vec2Processor)
|
||||
|
||||
def test_processor_from_local_directory_from_repo(self):
|
||||
with tempfile.TemporaryDirectory() as tmpdirname:
|
||||
model_config = Wav2Vec2Config()
|
||||
processor = AutoProcessor.from_pretrained("facebook/wav2vec2-base-960h")
|
||||
|
||||
# save in new folder
|
||||
model_config.save_pretrained(tmpdirname)
|
||||
processor.save_pretrained(tmpdirname)
|
||||
|
||||
processor = AutoProcessor.from_pretrained(tmpdirname)
|
||||
|
||||
self.assertIsInstance(processor, Wav2Vec2Processor)
|
||||
|
||||
def test_processor_from_local_directory_from_extractor_config(self):
|
||||
with tempfile.TemporaryDirectory() as tmpdirname:
|
||||
# copy relevant files
|
||||
copyfile(SAMPLE_PROCESSOR_CONFIG, os.path.join(tmpdirname, FEATURE_EXTRACTOR_NAME))
|
||||
copyfile(SAMPLE_VOCAB, os.path.join(tmpdirname, "vocab.json"))
|
||||
|
||||
processor = AutoProcessor.from_pretrained(tmpdirname)
|
||||
|
||||
self.assertIsInstance(processor, Wav2Vec2Processor)
|
||||
|
||||
def test_processor_from_feat_extr_processor_class(self):
|
||||
with tempfile.TemporaryDirectory() as tmpdirname:
|
||||
feature_extractor = Wav2Vec2FeatureExtractor()
|
||||
tokenizer = AutoTokenizer.from_pretrained("facebook/wav2vec2-base-960h")
|
||||
|
||||
processor = Wav2Vec2Processor(feature_extractor, tokenizer)
|
||||
|
||||
# save in new folder
|
||||
processor.save_pretrained(tmpdirname)
|
||||
|
||||
# drop `processor_class` in tokenizer
|
||||
with open(os.path.join(tmpdirname, TOKENIZER_CONFIG_FILE), "r") as f:
|
||||
config_dict = json.load(f)
|
||||
config_dict.pop("processor_class")
|
||||
|
||||
with open(os.path.join(tmpdirname, TOKENIZER_CONFIG_FILE), "w") as f:
|
||||
f.write(json.dumps(config_dict))
|
||||
|
||||
processor = AutoProcessor.from_pretrained(tmpdirname)
|
||||
|
||||
self.assertIsInstance(processor, Wav2Vec2Processor)
|
||||
|
||||
def test_processor_from_tokenizer_processor_class(self):
|
||||
with tempfile.TemporaryDirectory() as tmpdirname:
|
||||
feature_extractor = Wav2Vec2FeatureExtractor()
|
||||
tokenizer = AutoTokenizer.from_pretrained("facebook/wav2vec2-base-960h")
|
||||
|
||||
processor = Wav2Vec2Processor(feature_extractor, tokenizer)
|
||||
|
||||
# save in new folder
|
||||
processor.save_pretrained(tmpdirname)
|
||||
|
||||
# drop `processor_class` in feature extractor
|
||||
with open(os.path.join(tmpdirname, FEATURE_EXTRACTOR_NAME), "r") as f:
|
||||
config_dict = json.load(f)
|
||||
config_dict.pop("processor_class")
|
||||
|
||||
with open(os.path.join(tmpdirname, FEATURE_EXTRACTOR_NAME), "w") as f:
|
||||
f.write(json.dumps(config_dict))
|
||||
|
||||
processor = AutoProcessor.from_pretrained(tmpdirname)
|
||||
|
||||
self.assertIsInstance(processor, Wav2Vec2Processor)
|
||||
|
||||
def test_processor_from_local_directory_from_model_config(self):
|
||||
with tempfile.TemporaryDirectory() as tmpdirname:
|
||||
model_config = Wav2Vec2Config(processor_class="Wav2Vec2Processor")
|
||||
model_config.save_pretrained(tmpdirname)
|
||||
# copy relevant files
|
||||
copyfile(SAMPLE_VOCAB, os.path.join(tmpdirname, "vocab.json"))
|
||||
# create emtpy sample processor
|
||||
with open(os.path.join(tmpdirname, FEATURE_EXTRACTOR_NAME), "w") as f:
|
||||
f.write("{}")
|
||||
|
||||
processor = AutoProcessor.from_pretrained(tmpdirname)
|
||||
|
||||
self.assertIsInstance(processor, Wav2Vec2Processor)
|
||||
|
||||
def test_from_pretrained_dynamic_processor(self):
|
||||
processor = AutoProcessor.from_pretrained("hf-internal-testing/test_dynamic_processor", trust_remote_code=True)
|
||||
self.assertTrue(processor.special_attribute_present)
|
||||
self.assertEqual(processor.__class__.__name__, "NewProcessor")
|
||||
|
||||
feature_extractor = processor.feature_extractor
|
||||
self.assertTrue(feature_extractor.special_attribute_present)
|
||||
self.assertEqual(feature_extractor.__class__.__name__, "NewFeatureExtractor")
|
||||
|
||||
tokenizer = processor.tokenizer
|
||||
self.assertTrue(tokenizer.special_attribute_present)
|
||||
if is_tokenizers_available():
|
||||
self.assertEqual(tokenizer.__class__.__name__, "NewTokenizerFast")
|
||||
|
||||
# Test we can also load the slow version
|
||||
processor = AutoProcessor.from_pretrained(
|
||||
"hf-internal-testing/test_dynamic_processor", trust_remote_code=True, use_fast=False
|
||||
)
|
||||
tokenizer = processor.tokenizer
|
||||
self.assertTrue(tokenizer.special_attribute_present)
|
||||
self.assertEqual(tokenizer.__class__.__name__, "NewTokenizer")
|
||||
else:
|
||||
self.assertEqual(tokenizer.__class__.__name__, "NewTokenizer")
|
||||
|
||||
def test_new_processor_registration(self):
|
||||
try:
|
||||
AutoConfig.register("custom", CustomConfig)
|
||||
AutoFeatureExtractor.register(CustomConfig, CustomFeatureExtractor)
|
||||
AutoTokenizer.register(CustomConfig, slow_tokenizer_class=CustomTokenizer)
|
||||
AutoProcessor.register(CustomConfig, CustomProcessor)
|
||||
# Trying to register something existing in the Transformers library will raise an error
|
||||
with self.assertRaises(ValueError):
|
||||
AutoProcessor.register(Wav2Vec2Config, Wav2Vec2Processor)
|
||||
|
||||
# Now that the config is registered, it can be used as any other config with the auto-API
|
||||
feature_extractor = CustomFeatureExtractor.from_pretrained(SAMPLE_PROCESSOR_CONFIG_DIR)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
vocab_file = os.path.join(tmp_dir, "vocab.txt")
|
||||
with open(vocab_file, "w", encoding="utf-8") as vocab_writer:
|
||||
vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]))
|
||||
tokenizer = CustomTokenizer(vocab_file)
|
||||
|
||||
processor = CustomProcessor(feature_extractor, tokenizer)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
processor.save_pretrained(tmp_dir)
|
||||
new_processor = AutoProcessor.from_pretrained(tmp_dir)
|
||||
self.assertIsInstance(new_processor, CustomProcessor)
|
||||
|
||||
finally:
|
||||
if "custom" in CONFIG_MAPPING._extra_content:
|
||||
del CONFIG_MAPPING._extra_content["custom"]
|
||||
if CustomConfig in FEATURE_EXTRACTOR_MAPPING._extra_content:
|
||||
del FEATURE_EXTRACTOR_MAPPING._extra_content[CustomConfig]
|
||||
if CustomConfig in TOKENIZER_MAPPING._extra_content:
|
||||
del TOKENIZER_MAPPING._extra_content[CustomConfig]
|
||||
if CustomConfig in PROCESSOR_MAPPING._extra_content:
|
||||
del PROCESSOR_MAPPING._extra_content[CustomConfig]
|
||||
|
||||
|
||||
@is_staging_test
|
||||
class ProcessorPushToHubTester(unittest.TestCase):
|
||||
vocab_tokens = ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls._token = login(username=USER, password=PASS)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
try:
|
||||
delete_repo(token=cls._token, name="test-processor")
|
||||
except HTTPError:
|
||||
pass
|
||||
|
||||
try:
|
||||
delete_repo(token=cls._token, name="test-processor-org", organization="valid_org")
|
||||
except HTTPError:
|
||||
pass
|
||||
|
||||
try:
|
||||
delete_repo(token=cls._token, name="test-dynamic-processor")
|
||||
except HTTPError:
|
||||
pass
|
||||
|
||||
def test_push_to_hub(self):
|
||||
processor = Wav2Vec2Processor.from_pretrained(SAMPLE_PROCESSOR_CONFIG_DIR)
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
processor.save_pretrained(
|
||||
os.path.join(tmp_dir, "test-processor"), push_to_hub=True, use_auth_token=self._token
|
||||
)
|
||||
|
||||
new_processor = Wav2Vec2Processor.from_pretrained(f"{USER}/test-processor")
|
||||
for k, v in processor.feature_extractor.__dict__.items():
|
||||
self.assertEqual(v, getattr(new_processor.feature_extractor, k))
|
||||
self.assertDictEqual(new_processor.tokenizer.get_vocab(), processor.tokenizer.get_vocab())
|
||||
|
||||
def test_push_to_hub_in_organization(self):
|
||||
processor = Wav2Vec2Processor.from_pretrained(SAMPLE_PROCESSOR_CONFIG_DIR)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
processor.save_pretrained(
|
||||
os.path.join(tmp_dir, "test-processor-org"),
|
||||
push_to_hub=True,
|
||||
use_auth_token=self._token,
|
||||
organization="valid_org",
|
||||
)
|
||||
|
||||
new_processor = Wav2Vec2Processor.from_pretrained("valid_org/test-processor-org")
|
||||
for k, v in processor.feature_extractor.__dict__.items():
|
||||
self.assertEqual(v, getattr(new_processor.feature_extractor, k))
|
||||
self.assertDictEqual(new_processor.tokenizer.get_vocab(), processor.tokenizer.get_vocab())
|
||||
|
||||
def test_push_to_hub_dynamic_processor(self):
|
||||
CustomFeatureExtractor.register_for_auto_class()
|
||||
CustomTokenizer.register_for_auto_class()
|
||||
CustomProcessor.register_for_auto_class()
|
||||
|
||||
feature_extractor = CustomFeatureExtractor.from_pretrained(SAMPLE_PROCESSOR_CONFIG_DIR)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
vocab_file = os.path.join(tmp_dir, "vocab.txt")
|
||||
with open(vocab_file, "w", encoding="utf-8") as vocab_writer:
|
||||
vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]))
|
||||
tokenizer = CustomTokenizer(vocab_file)
|
||||
|
||||
processor = CustomProcessor(feature_extractor, tokenizer)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
repo = Repository(tmp_dir, clone_from=f"{USER}/test-dynamic-processor", use_auth_token=self._token)
|
||||
processor.save_pretrained(tmp_dir)
|
||||
|
||||
# This has added the proper auto_map field to the feature extractor config
|
||||
self.assertDictEqual(
|
||||
processor.feature_extractor.auto_map,
|
||||
{
|
||||
"AutoFeatureExtractor": "custom_feature_extraction.CustomFeatureExtractor",
|
||||
"AutoProcessor": "custom_processing.CustomProcessor",
|
||||
},
|
||||
)
|
||||
|
||||
# This has added the proper auto_map field to the tokenizer config
|
||||
with open(os.path.join(tmp_dir, "tokenizer_config.json")) as f:
|
||||
tokenizer_config = json.load(f)
|
||||
self.assertDictEqual(
|
||||
tokenizer_config["auto_map"],
|
||||
{
|
||||
"AutoTokenizer": ["custom_tokenization.CustomTokenizer", None],
|
||||
"AutoProcessor": "custom_processing.CustomProcessor",
|
||||
},
|
||||
)
|
||||
|
||||
# The code has been copied from fixtures
|
||||
self.assertTrue(os.path.isfile(os.path.join(tmp_dir, "custom_feature_extraction.py")))
|
||||
self.assertTrue(os.path.isfile(os.path.join(tmp_dir, "custom_tokenization.py")))
|
||||
self.assertTrue(os.path.isfile(os.path.join(tmp_dir, "custom_processing.py")))
|
||||
|
||||
repo.push_to_hub()
|
||||
|
||||
new_processor = AutoProcessor.from_pretrained(f"{USER}/test-dynamic-processor", trust_remote_code=True)
|
||||
# Can't make an isinstance check because the new_processor is from the CustomProcessor class of a dynamic module
|
||||
self.assertEqual(new_processor.__class__.__name__, "CustomProcessor")
|
||||
342
tests/models/auto/test_tokenization_auto.py
Normal file
342
tests/models/auto/test_tokenization_auto.py
Normal file
@@ -0,0 +1,342 @@
|
||||
# coding=utf-8
|
||||
# Copyright 2020 The HuggingFace Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from transformers import (
|
||||
BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
|
||||
GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
|
||||
AutoTokenizer,
|
||||
BertConfig,
|
||||
BertTokenizer,
|
||||
BertTokenizerFast,
|
||||
CTRLTokenizer,
|
||||
GPT2Tokenizer,
|
||||
GPT2TokenizerFast,
|
||||
PreTrainedTokenizerFast,
|
||||
RobertaTokenizer,
|
||||
RobertaTokenizerFast,
|
||||
is_tokenizers_available,
|
||||
)
|
||||
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
|
||||
from transformers.models.auto.tokenization_auto import (
|
||||
TOKENIZER_MAPPING,
|
||||
get_tokenizer_config,
|
||||
tokenizer_class_from_name,
|
||||
)
|
||||
from transformers.models.roberta.configuration_roberta import RobertaConfig
|
||||
from transformers.testing_utils import (
|
||||
DUMMY_DIFF_TOKENIZER_IDENTIFIER,
|
||||
DUMMY_UNKNOWN_IDENTIFIER,
|
||||
SMALL_MODEL_IDENTIFIER,
|
||||
require_tokenizers,
|
||||
slow,
|
||||
)
|
||||
|
||||
|
||||
sys.path.append(str(Path(__file__).parent.parent.parent.parent / "utils"))
|
||||
|
||||
from test_module.custom_configuration import CustomConfig # noqa E402
|
||||
from test_module.custom_tokenization import CustomTokenizer # noqa E402
|
||||
|
||||
|
||||
if is_tokenizers_available():
|
||||
from test_module.custom_tokenization_fast import CustomTokenizerFast
|
||||
|
||||
|
||||
class AutoTokenizerTest(unittest.TestCase):
|
||||
@slow
|
||||
def test_tokenizer_from_pretrained(self):
|
||||
for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
|
||||
tokenizer = AutoTokenizer.from_pretrained(model_name)
|
||||
self.assertIsNotNone(tokenizer)
|
||||
self.assertIsInstance(tokenizer, (BertTokenizer, BertTokenizerFast))
|
||||
self.assertGreater(len(tokenizer), 0)
|
||||
|
||||
for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
|
||||
tokenizer = AutoTokenizer.from_pretrained(model_name)
|
||||
self.assertIsNotNone(tokenizer)
|
||||
self.assertIsInstance(tokenizer, (GPT2Tokenizer, GPT2TokenizerFast))
|
||||
self.assertGreater(len(tokenizer), 0)
|
||||
|
||||
def test_tokenizer_from_pretrained_identifier(self):
|
||||
tokenizer = AutoTokenizer.from_pretrained(SMALL_MODEL_IDENTIFIER)
|
||||
self.assertIsInstance(tokenizer, (BertTokenizer, BertTokenizerFast))
|
||||
self.assertEqual(tokenizer.vocab_size, 12)
|
||||
|
||||
def test_tokenizer_from_model_type(self):
|
||||
tokenizer = AutoTokenizer.from_pretrained(DUMMY_UNKNOWN_IDENTIFIER)
|
||||
self.assertIsInstance(tokenizer, (RobertaTokenizer, RobertaTokenizerFast))
|
||||
self.assertEqual(tokenizer.vocab_size, 20)
|
||||
|
||||
def test_tokenizer_from_tokenizer_class(self):
|
||||
config = AutoConfig.from_pretrained(DUMMY_DIFF_TOKENIZER_IDENTIFIER)
|
||||
self.assertIsInstance(config, RobertaConfig)
|
||||
# Check that tokenizer_type ≠ model_type
|
||||
tokenizer = AutoTokenizer.from_pretrained(DUMMY_DIFF_TOKENIZER_IDENTIFIER, config=config)
|
||||
self.assertIsInstance(tokenizer, (BertTokenizer, BertTokenizerFast))
|
||||
self.assertEqual(tokenizer.vocab_size, 12)
|
||||
|
||||
def test_tokenizer_from_type(self):
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
shutil.copy("./tests/fixtures/vocab.txt", os.path.join(tmp_dir, "vocab.txt"))
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained(tmp_dir, tokenizer_type="bert", use_fast=False)
|
||||
self.assertIsInstance(tokenizer, BertTokenizer)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
shutil.copy("./tests/fixtures/vocab.json", os.path.join(tmp_dir, "vocab.json"))
|
||||
shutil.copy("./tests/fixtures/merges.txt", os.path.join(tmp_dir, "merges.txt"))
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained(tmp_dir, tokenizer_type="gpt2", use_fast=False)
|
||||
self.assertIsInstance(tokenizer, GPT2Tokenizer)
|
||||
|
||||
@require_tokenizers
|
||||
def test_tokenizer_from_type_fast(self):
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
shutil.copy("./tests/fixtures/vocab.txt", os.path.join(tmp_dir, "vocab.txt"))
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained(tmp_dir, tokenizer_type="bert")
|
||||
self.assertIsInstance(tokenizer, BertTokenizerFast)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
shutil.copy("./tests/fixtures/vocab.json", os.path.join(tmp_dir, "vocab.json"))
|
||||
shutil.copy("./tests/fixtures/merges.txt", os.path.join(tmp_dir, "merges.txt"))
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained(tmp_dir, tokenizer_type="gpt2")
|
||||
self.assertIsInstance(tokenizer, GPT2TokenizerFast)
|
||||
|
||||
def test_tokenizer_from_type_incorrect_name(self):
|
||||
with pytest.raises(ValueError):
|
||||
AutoTokenizer.from_pretrained("./", tokenizer_type="xxx")
|
||||
|
||||
@require_tokenizers
|
||||
def test_tokenizer_identifier_with_correct_config(self):
|
||||
for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
|
||||
tokenizer = tokenizer_class.from_pretrained("wietsedv/bert-base-dutch-cased")
|
||||
self.assertIsInstance(tokenizer, (BertTokenizer, BertTokenizerFast))
|
||||
|
||||
if isinstance(tokenizer, BertTokenizer):
|
||||
self.assertEqual(tokenizer.basic_tokenizer.do_lower_case, False)
|
||||
else:
|
||||
self.assertEqual(tokenizer.do_lower_case, False)
|
||||
|
||||
self.assertEqual(tokenizer.model_max_length, 512)
|
||||
|
||||
@require_tokenizers
|
||||
def test_tokenizer_identifier_non_existent(self):
|
||||
for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
|
||||
with self.assertRaisesRegex(
|
||||
EnvironmentError,
|
||||
"julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier",
|
||||
):
|
||||
_ = tokenizer_class.from_pretrained("julien-c/herlolip-not-exists")
|
||||
|
||||
def test_model_name_edge_cases_in_mappings(self):
|
||||
# tests: https://github.com/huggingface/transformers/pull/13251
|
||||
# 1. models with `-`, e.g. xlm-roberta -> xlm_roberta
|
||||
# 2. models that don't remap 1-1 from model-name to model file, e.g., openai-gpt -> openai
|
||||
tokenizers = TOKENIZER_MAPPING.values()
|
||||
tokenizer_names = []
|
||||
|
||||
for slow_tok, fast_tok in tokenizers:
|
||||
if slow_tok is not None:
|
||||
tokenizer_names.append(slow_tok.__name__)
|
||||
|
||||
if fast_tok is not None:
|
||||
tokenizer_names.append(fast_tok.__name__)
|
||||
|
||||
for tokenizer_name in tokenizer_names:
|
||||
# must find the right class
|
||||
tokenizer_class_from_name(tokenizer_name)
|
||||
|
||||
@require_tokenizers
|
||||
def test_from_pretrained_use_fast_toggle(self):
|
||||
self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased", use_fast=False), BertTokenizer)
|
||||
self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased"), BertTokenizerFast)
|
||||
|
||||
@require_tokenizers
|
||||
def test_do_lower_case(self):
|
||||
tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased", do_lower_case=False)
|
||||
sample = "Hello, world. How are you?"
|
||||
tokens = tokenizer.tokenize(sample)
|
||||
self.assertEqual("[UNK]", tokens[0])
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained("microsoft/mpnet-base", do_lower_case=False)
|
||||
tokens = tokenizer.tokenize(sample)
|
||||
self.assertEqual("[UNK]", tokens[0])
|
||||
|
||||
@require_tokenizers
|
||||
def test_PreTrainedTokenizerFast_from_pretrained(self):
|
||||
tokenizer = AutoTokenizer.from_pretrained("robot-test/dummy-tokenizer-fast-with-model-config")
|
||||
self.assertEqual(type(tokenizer), PreTrainedTokenizerFast)
|
||||
self.assertEqual(tokenizer.model_max_length, 512)
|
||||
self.assertEqual(tokenizer.vocab_size, 30000)
|
||||
self.assertEqual(tokenizer.unk_token, "[UNK]")
|
||||
self.assertEqual(tokenizer.padding_side, "right")
|
||||
self.assertEqual(tokenizer.truncation_side, "right")
|
||||
|
||||
def test_auto_tokenizer_from_local_folder(self):
|
||||
tokenizer = AutoTokenizer.from_pretrained(SMALL_MODEL_IDENTIFIER)
|
||||
self.assertIsInstance(tokenizer, (BertTokenizer, BertTokenizerFast))
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
tokenizer.save_pretrained(tmp_dir)
|
||||
tokenizer2 = AutoTokenizer.from_pretrained(tmp_dir)
|
||||
|
||||
self.assertIsInstance(tokenizer2, tokenizer.__class__)
|
||||
self.assertEqual(tokenizer2.vocab_size, 12)
|
||||
|
||||
def test_auto_tokenizer_fast_no_slow(self):
|
||||
tokenizer = AutoTokenizer.from_pretrained("ctrl")
|
||||
# There is no fast CTRL so this always gives us a slow tokenizer.
|
||||
self.assertIsInstance(tokenizer, CTRLTokenizer)
|
||||
|
||||
def test_get_tokenizer_config(self):
|
||||
# Check we can load the tokenizer config of an online model.
|
||||
config = get_tokenizer_config("bert-base-cased")
|
||||
# If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
|
||||
self.assertEqual(config, {"do_lower_case": False})
|
||||
|
||||
# This model does not have a tokenizer_config so we get back an empty dict.
|
||||
config = get_tokenizer_config(SMALL_MODEL_IDENTIFIER)
|
||||
self.assertDictEqual(config, {})
|
||||
|
||||
# A tokenizer saved with `save_pretrained` always creates a tokenizer config.
|
||||
tokenizer = AutoTokenizer.from_pretrained(SMALL_MODEL_IDENTIFIER)
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
tokenizer.save_pretrained(tmp_dir)
|
||||
config = get_tokenizer_config(tmp_dir)
|
||||
|
||||
# Check the class of the tokenizer was properly saved (note that it always saves the slow class).
|
||||
self.assertEqual(config["tokenizer_class"], "BertTokenizer")
|
||||
# Check other keys just to make sure the config was properly saved /reloaded.
|
||||
self.assertEqual(config["name_or_path"], SMALL_MODEL_IDENTIFIER)
|
||||
|
||||
def test_new_tokenizer_registration(self):
|
||||
try:
|
||||
AutoConfig.register("custom", CustomConfig)
|
||||
|
||||
AutoTokenizer.register(CustomConfig, slow_tokenizer_class=CustomTokenizer)
|
||||
# Trying to register something existing in the Transformers library will raise an error
|
||||
with self.assertRaises(ValueError):
|
||||
AutoTokenizer.register(BertConfig, slow_tokenizer_class=BertTokenizer)
|
||||
|
||||
tokenizer = CustomTokenizer.from_pretrained(SMALL_MODEL_IDENTIFIER)
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
tokenizer.save_pretrained(tmp_dir)
|
||||
|
||||
new_tokenizer = AutoTokenizer.from_pretrained(tmp_dir)
|
||||
self.assertIsInstance(new_tokenizer, CustomTokenizer)
|
||||
|
||||
finally:
|
||||
if "custom" in CONFIG_MAPPING._extra_content:
|
||||
del CONFIG_MAPPING._extra_content["custom"]
|
||||
if CustomConfig in TOKENIZER_MAPPING._extra_content:
|
||||
del TOKENIZER_MAPPING._extra_content[CustomConfig]
|
||||
|
||||
@require_tokenizers
|
||||
def test_new_tokenizer_fast_registration(self):
|
||||
try:
|
||||
AutoConfig.register("custom", CustomConfig)
|
||||
|
||||
# Can register in two steps
|
||||
AutoTokenizer.register(CustomConfig, slow_tokenizer_class=CustomTokenizer)
|
||||
self.assertEqual(TOKENIZER_MAPPING[CustomConfig], (CustomTokenizer, None))
|
||||
AutoTokenizer.register(CustomConfig, fast_tokenizer_class=CustomTokenizerFast)
|
||||
self.assertEqual(TOKENIZER_MAPPING[CustomConfig], (CustomTokenizer, CustomTokenizerFast))
|
||||
|
||||
del TOKENIZER_MAPPING._extra_content[CustomConfig]
|
||||
# Can register in one step
|
||||
AutoTokenizer.register(
|
||||
CustomConfig, slow_tokenizer_class=CustomTokenizer, fast_tokenizer_class=CustomTokenizerFast
|
||||
)
|
||||
self.assertEqual(TOKENIZER_MAPPING[CustomConfig], (CustomTokenizer, CustomTokenizerFast))
|
||||
|
||||
# Trying to register something existing in the Transformers library will raise an error
|
||||
with self.assertRaises(ValueError):
|
||||
AutoTokenizer.register(BertConfig, fast_tokenizer_class=BertTokenizerFast)
|
||||
|
||||
# We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
|
||||
# and that model does not have a tokenizer.json
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
bert_tokenizer = BertTokenizerFast.from_pretrained(SMALL_MODEL_IDENTIFIER)
|
||||
bert_tokenizer.save_pretrained(tmp_dir)
|
||||
tokenizer = CustomTokenizerFast.from_pretrained(tmp_dir)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
tokenizer.save_pretrained(tmp_dir)
|
||||
|
||||
new_tokenizer = AutoTokenizer.from_pretrained(tmp_dir)
|
||||
self.assertIsInstance(new_tokenizer, CustomTokenizerFast)
|
||||
|
||||
new_tokenizer = AutoTokenizer.from_pretrained(tmp_dir, use_fast=False)
|
||||
self.assertIsInstance(new_tokenizer, CustomTokenizer)
|
||||
|
||||
finally:
|
||||
if "custom" in CONFIG_MAPPING._extra_content:
|
||||
del CONFIG_MAPPING._extra_content["custom"]
|
||||
if CustomConfig in TOKENIZER_MAPPING._extra_content:
|
||||
del TOKENIZER_MAPPING._extra_content[CustomConfig]
|
||||
|
||||
def test_from_pretrained_dynamic_tokenizer(self):
|
||||
tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer", trust_remote_code=True)
|
||||
self.assertTrue(tokenizer.special_attribute_present)
|
||||
if is_tokenizers_available():
|
||||
self.assertEqual(tokenizer.__class__.__name__, "NewTokenizerFast")
|
||||
|
||||
# Test we can also load the slow version
|
||||
tokenizer = AutoTokenizer.from_pretrained(
|
||||
"hf-internal-testing/test_dynamic_tokenizer", trust_remote_code=True, use_fast=False
|
||||
)
|
||||
self.assertTrue(tokenizer.special_attribute_present)
|
||||
self.assertEqual(tokenizer.__class__.__name__, "NewTokenizer")
|
||||
else:
|
||||
self.assertEqual(tokenizer.__class__.__name__, "NewTokenizer")
|
||||
|
||||
def test_from_pretrained_dynamic_tokenizer_legacy_format(self):
|
||||
tokenizer = AutoTokenizer.from_pretrained(
|
||||
"hf-internal-testing/test_dynamic_tokenizer_legacy", trust_remote_code=True
|
||||
)
|
||||
self.assertTrue(tokenizer.special_attribute_present)
|
||||
if is_tokenizers_available():
|
||||
self.assertEqual(tokenizer.__class__.__name__, "NewTokenizerFast")
|
||||
|
||||
# Test we can also load the slow version
|
||||
tokenizer = AutoTokenizer.from_pretrained(
|
||||
"hf-internal-testing/test_dynamic_tokenizer_legacy", trust_remote_code=True, use_fast=False
|
||||
)
|
||||
self.assertTrue(tokenizer.special_attribute_present)
|
||||
self.assertEqual(tokenizer.__class__.__name__, "NewTokenizer")
|
||||
else:
|
||||
self.assertEqual(tokenizer.__class__.__name__, "NewTokenizer")
|
||||
|
||||
def test_repo_not_found(self):
|
||||
with self.assertRaisesRegex(
|
||||
EnvironmentError, "bert-base is not a local folder and is not a valid model identifier"
|
||||
):
|
||||
_ = AutoTokenizer.from_pretrained("bert-base")
|
||||
|
||||
def test_revision_not_found(self):
|
||||
with self.assertRaisesRegex(
|
||||
EnvironmentError, r"aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"
|
||||
):
|
||||
_ = AutoTokenizer.from_pretrained(DUMMY_UNKNOWN_IDENTIFIER, revision="aaaaaa")
|
||||
Reference in New Issue
Block a user