SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Simplify soft dependencies and update the dummy-creation process (#36827)

Browse Source 
* Reverse dependency map shouldn't be created when test_all is set

* [test_all] Remove dummies

* Modular fixes

* Update utils/check_repo.py

Co-authored-by: Pablo Montalvo <39954772+molbap@users.noreply.github.com>

* [test_all] Better docs

* [test_all] Update src/transformers/commands/chat.py

Co-authored-by: Joao Gante <joaofranciscocardosogante@gmail.com>

* [test_all] Remove deprecated AdaptiveEmbeddings from the tests

* [test_all] Doc builder

* [test_all] is_dummy

* [test_all] Import utils

* [test_all] Doc building should not require all deps

---------

Co-authored-by: Pablo Montalvo <39954772+molbap@users.noreply.github.com>
Co-authored-by: Joao Gante <joaofranciscocardosogante@gmail.com>
This commit is contained in:
Lysandre Debut
2025-04-11 11:08:36 +02:00
committed by GitHub
parent 931126b929
commit 54a123f068
295 changed files with 2167 additions and 27702 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
tests/models/colpali/test_processing_colpali.py
View File

@@ -8,7 +8,6 @@ from transformers import GemmaTokenizer
from transformers.models.colpali.processing_colpali import ColPaliProcessor
from transformers.testing_utils import get_tests_dir, require_torch, require_vision
from transformers.utils import is_vision_available
from transformers.utils.dummy_vision_objects import SiglipImageProcessor
from ...test_processing_common import ProcessorTesterMixin

14
tests/pipelines/test_pipelines_mask_generation.py
View File

@@ -19,7 +19,8 @@ from huggingface_hub.utils import insecure_hashlib
from transformers import (
MODEL_FOR_MASK_GENERATION_MAPPING,
TF_MODEL_FOR_MASK_GENERATION_MAPPING,
is_tf_available,
is_torch_available,
is_vision_available,
pipeline,
)
@@ -34,6 +35,17 @@ from transformers.testing_utils import (
)
if is_tf_available():
from transformers import TF_MODEL_FOR_MASK_GENERATION_MAPPING
else:
TF_MODEL_FOR_MASK_GENERATION_MAPPING = None
if is_torch_available():
from transformers import MODEL_FOR_MASK_GENERATION_MAPPING
else:
MODEL_FOR_MASK_GENERATION_MAPPING = None
if is_vision_available():
from PIL import Image
else:

4
tests/pipelines/test_pipelines_question_answering.py
View File

@@ -51,9 +51,9 @@ class QAPipelineTests(unittest.TestCase):
model_mapping = MODEL_FOR_QUESTION_ANSWERING_MAPPING
tf_model_mapping = TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING
if model_mapping is not None:
if not hasattr(model_mapping, "is_dummy"):
model_mapping = {config: model for config, model in model_mapping.items() if config.__name__ not in _TO_SKIP}
if tf_model_mapping is not None:
if not hasattr(tf_model_mapping, "is_dummy"):
tf_model_mapping = {
config: model for config, model in tf_model_mapping.items() if config.__name__ not in _TO_SKIP
}

4
tests/pipelines/test_pipelines_text_classification.py
View File

@@ -48,9 +48,9 @@ class TextClassificationPipelineTests(unittest.TestCase):
model_mapping = MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING
tf_model_mapping = TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING
if model_mapping is not None:
if not hasattr(model_mapping, "is_dummy"):
model_mapping = {config: model for config, model in model_mapping.items() if config.__name__ not in _TO_SKIP}
if tf_model_mapping is not None:
if not hasattr(tf_model_mapping, "is_dummy"):
tf_model_mapping = {
config: model for config, model in tf_model_mapping.items() if config.__name__ not in _TO_SKIP
}

4
tests/pipelines/test_pipelines_token_classification.py
View File

@@ -54,9 +54,9 @@ class TokenClassificationPipelineTests(unittest.TestCase):
model_mapping = MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING
tf_model_mapping = TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING
if model_mapping is not None:
if not hasattr(model_mapping, "is_dummy"):
model_mapping = {config: model for config, model in model_mapping.items() if config.__name__ not in _TO_SKIP}
if tf_model_mapping is not None:
if not hasattr(tf_model_mapping, "is_dummy"):
tf_model_mapping = {
config: model for config, model in tf_model_mapping.items() if config.__name__ not in _TO_SKIP
}

4
tests/pipelines/test_pipelines_zero_shot.py
View File

@@ -46,9 +46,9 @@ class ZeroShotClassificationPipelineTests(unittest.TestCase):
model_mapping = MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING
tf_model_mapping = TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING
if model_mapping is not None:
if not hasattr(model_mapping, "is_dummy"):
model_mapping = {config: model for config, model in model_mapping.items() if config.__name__ not in _TO_SKIP}
if tf_model_mapping is not None:
if not hasattr(tf_model_mapping, "is_dummy"):
tf_model_mapping = {
config: model for config, model in tf_model_mapping.items() if config.__name__ not in _TO_SKIP
}

126
tests/repo_utils/test_check_dummies.py
View File

@@ -1,126 +0,0 @@
# Copyright 2022 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import sys
import unittest
git_repo_path = os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
sys.path.append(os.path.join(git_repo_path, "utils"))
import check_dummies # noqa: E402
from check_dummies import create_dummy_files, create_dummy_object, find_backend, read_init # noqa: E402
# Align TRANSFORMERS_PATH in check_dummies with the current path
check_dummies.PATH_TO_TRANSFORMERS = os.path.join(git_repo_path, "src", "transformers")
DUMMY_CONSTANT = """
{0} = None
"""
DUMMY_CLASS = """
class {0}(metaclass=DummyObject):
_backends = {1}
def __init__(self, *args, **kwargs):
requires_backends(self, {1})
"""
DUMMY_FUNCTION = """
def {0}(*args, **kwargs):
requires_backends({0}, {1})
"""
class CheckDummiesTester(unittest.TestCase):
def test_find_backend(self):
no_backend = find_backend(' _import_structure["models.albert"].append("AlbertTokenizerFast")')
self.assertIsNone(no_backend)
simple_backend = find_backend(" if not is_tokenizers_available():")
self.assertEqual(simple_backend, "tokenizers")
backend_with_underscore = find_backend(" if not is_tensorflow_text_available():")
self.assertEqual(backend_with_underscore, "tensorflow_text")
double_backend = find_backend(" if not (is_sentencepiece_available() and is_tokenizers_available()):")
self.assertEqual(double_backend, "sentencepiece_and_tokenizers")
double_backend_with_underscore = find_backend(
" if not (is_sentencepiece_available() and is_tensorflow_text_available()):"
)
self.assertEqual(double_backend_with_underscore, "sentencepiece_and_tensorflow_text")
triple_backend = find_backend(
" if not (is_sentencepiece_available() and is_tokenizers_available() and is_vision_available()):"
)
self.assertEqual(triple_backend, "sentencepiece_and_tokenizers_and_vision")
def test_read_init(self):
objects = read_init()
# We don't assert on the exact list of keys to allow for smooth grow of backend-specific objects
self.assertIn("torch", objects)
self.assertIn("tensorflow_text", objects)
self.assertIn("sentencepiece_and_tokenizers", objects)
# Likewise, we can't assert on the exact content of a key
self.assertIn("BertModel", objects["torch"])
self.assertIn("TFBertModel", objects["tf"])
self.assertIn("FlaxBertModel", objects["flax"])
self.assertIn("BertModel", objects["torch"])
self.assertIn("TFBertTokenizer", objects["tensorflow_text"])
self.assertIn("convert_slow_tokenizer", objects["sentencepiece_and_tokenizers"])
def test_create_dummy_object(self):
dummy_constant = create_dummy_object("CONSTANT", "'torch'")
self.assertEqual(dummy_constant, "\nCONSTANT = None\n")
dummy_function = create_dummy_object("function", "'torch'")
self.assertEqual(
dummy_function, "\ndef function(*args, **kwargs):\n requires_backends(function, 'torch')\n"
)
expected_dummy_class = """
class FakeClass(metaclass=DummyObject):
_backends = 'torch'
def __init__(self, *args, **kwargs):
requires_backends(self, 'torch')
"""
dummy_class = create_dummy_object("FakeClass", "'torch'")
self.assertEqual(dummy_class, expected_dummy_class)
def test_create_dummy_files(self):
expected_dummy_pytorch_file = """# This file is autogenerated by the command `make fix-copies`, do not edit.
from ..utils import DummyObject, requires_backends
CONSTANT = None
def function(*args, **kwargs):
requires_backends(function, ["torch"])
class FakeClass(metaclass=DummyObject):
_backends = ["torch"]
def __init__(self, *args, **kwargs):
requires_backends(self, ["torch"])
"""
dummy_files = create_dummy_files({"torch": ["CONSTANT", "function", "FakeClass"]})
self.assertEqual(dummy_files["torch"], expected_dummy_pytorch_file)

4
tests/test_modeling_common.py
View File

@@ -119,7 +119,7 @@ if is_torch_available():
from safetensors.torch import save_file as safe_save_file
from torch import nn
from transformers import MODEL_MAPPING, AdaptiveEmbedding
from transformers import MODEL_MAPPING
from transformers.cache_utils import Cache, DynamicCache
from transformers.modeling_utils import load_state_dict, no_init_weights
from transformers.pytorch_utils import id_tensor_storage
@@ -2095,7 +2095,7 @@ class ModelTesterMixin:
for model_class in self.all_model_classes:
model = model_class(config)
self.assertIsInstance(model.get_input_embeddings(), (nn.Embedding, AdaptiveEmbedding))
self.assertIsInstance(model.get_input_embeddings(), nn.Embedding)
new_input_embedding_layer = nn.Embedding(10, 10)
model.set_input_embeddings(new_input_embedding_layer)

4
tests/utils/import_structures/failing_export.py
View File

@@ -14,10 +14,10 @@
# fmt: off
from transformers.utils.import_utils import export
from transformers.utils.import_utils import requires
@export(backends=("random_item_that_should_not_exist",))
@requires(backends=("random_item_that_should_not_exist",))
class A0:
def __init__(self):
pass

20
tests/utils/import_structures/import_structure_raw_register.py
View File

@@ -14,32 +14,32 @@
# fmt: off
from transformers.utils.import_utils import export
from transformers.utils.import_utils import requires
@export()
@requires()
class A0:
def __init__(self):
pass
@export()
@requires()
def a0():
pass
@export(backends=("torch", "tf"))
@requires(backends=("torch", "tf"))
class A1:
def __init__(self):
pass
@export(backends=("torch", "tf"))
@requires(backends=("torch", "tf"))
def a1():
pass
@export(
@requires(
backends=("torch", "tf")
)
class A2:
@@ -47,14 +47,14 @@ class A2:
pass
@export(
@requires(
backends=("torch", "tf")
)
def a2():
pass
@export(
@requires(
backends=(
"torch",
"tf"
@@ -65,7 +65,7 @@ class A3:
pass
@export(
@requires(
backends=(
"torch",
"tf"
@@ -74,7 +74,7 @@ class A3:
def a3():
pass
@export(backends=())
@requires(backends=())
class A4:
def __init__(self):
pass

18
tests/utils/import_structures/import_structure_register_with_comments.py
View File

@@ -14,49 +14,49 @@
# fmt: off
from transformers.utils.import_utils import export
from transformers.utils.import_utils import requires
@export()
@requires()
# That's a statement
class B0:
def __init__(self):
pass
@export()
@requires()
# That's a statement
def b0():
pass
@export(backends=("torch", "tf"))
@requires(backends=("torch", "tf"))
# That's a statement
class B1:
def __init__(self):
pass
@export(backends=("torch", "tf"))
@requires(backends=("torch", "tf"))
# That's a statement
def b1():
pass
@export(backends=("torch", "tf"))
@requires(backends=("torch", "tf"))
# That's a statement
class B2:
def __init__(self):
pass
@export(backends=("torch", "tf"))
@requires(backends=("torch", "tf"))
# That's a statement
def b2():
pass
@export(
@requires(
backends=(
"torch",
"tf"
@@ -68,7 +68,7 @@ class B3:
pass
@export(
@requires(
backends=(
"torch",
"tf"

18
tests/utils/import_structures/import_structure_register_with_duplicates.py
View File

@@ -14,47 +14,47 @@
# fmt: off
from transformers.utils.import_utils import export
from transformers.utils.import_utils import requires
@export(backends=("torch", "torch"))
@requires(backends=("torch", "torch"))
class C0:
def __init__(self):
pass
@export(backends=("torch", "torch"))
@requires(backends=("torch", "torch"))
def c0():
pass
@export(backends=("torch", "torch"))
@requires(backends=("torch", "torch"))
# That's a statement
class C1:
def __init__(self):
pass
@export(backends=("torch", "torch"))
@requires(backends=("torch", "torch"))
# That's a statement
def c1():
pass
@export(backends=("torch", "torch"))
@requires(backends=("torch", "torch"))
# That's a statement
class C2:
def __init__(self):
pass
@export(backends=("torch", "torch"))
@requires(backends=("torch", "torch"))
# That's a statement
def c2():
pass
@export(
@requires(
backends=(
"torch",
"torch"
@@ -66,7 +66,7 @@ class C3:
pass
@export(
@requires(
backends=(
"torch",
"torch"