SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Revamp test selection for the example tests (#23737)

Browse Source 
* Revamp test selection for the example tests

* Rename old XLA test and fake modif in run_glue

* Fixes

* Fake Trainer modif

* Remove fake modifs
This commit is contained in:
Sylvain Gugger
2023-05-25 09:38:21 -04:00
committed by GitHub
parent 7d4fe85ef3
commit 6e4bc67099
5 changed files with 172 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

113
tests/repo_utils/test_tests_fetcher.py
View File

@@ -42,6 +42,7 @@ from tests_fetcher import ( # noqa: E402
get_module_dependencies,
get_tree_starting_at,
infer_tests_to_run,
init_test_examples_dependencies,
parse_commit_message,
print_tree_deps_of,
)
@@ -149,7 +150,19 @@ def create_tmp_repo(tmp_dir, models=None):
f"from transformers import {cls}Config, {cls}Model\nfrom ...test_modeling_common import ModelTesterMixin\n\ncode"
)
repo.index.add(["src", "tests"])
example_dir = tmp_dir / "examples"
example_dir.mkdir(exist_ok=True)
for framework in ["flax", "pytorch", "tensorflow"]:
framework_dir = example_dir / framework
framework_dir.mkdir(exist_ok=True)
with open(framework_dir / f"test_{framework}_examples.py", "w") as f:
f.write("""test_args = "run_glue.py"\n""")
glue_dir = framework_dir / "text-classification"
glue_dir.mkdir(exist_ok=True)
with open(glue_dir / "run_glue.py", "w") as f:
f.write("from transformers import BertModel\n\ncode")
repo.index.add(["examples", "src", "tests"])
repo.index.commit("Initial commit")
repo.create_head("main")
repo.head.reference = repo.refs.main
@@ -164,12 +177,14 @@ def patch_transformer_repo_path(new_folder):
"""
old_repo_path = tests_fetcher.PATH_TO_REPO
tests_fetcher.PATH_TO_REPO = Path(new_folder).resolve()
tests_fetcher.PATH_TO_EXAMPLES = tests_fetcher.PATH_TO_REPO / "examples"
tests_fetcher.PATH_TO_TRANFORMERS = tests_fetcher.PATH_TO_REPO / "src/transformers"
tests_fetcher.PATH_TO_TESTS = tests_fetcher.PATH_TO_REPO / "tests"
try:
yield
finally:
tests_fetcher.PATH_TO_REPO = old_repo_path
tests_fetcher.PATH_TO_EXAMPLES = tests_fetcher.PATH_TO_REPO / "examples"
tests_fetcher.PATH_TO_TRANFORMERS = tests_fetcher.PATH_TO_REPO / "src/transformers"
tests_fetcher.PATH_TO_TESTS = tests_fetcher.PATH_TO_REPO / "tests"
@@ -409,6 +424,17 @@ class TestFetcherTester(unittest.TestCase):
with patch_transformer_repo_path(tmp_folder):
assert get_module_dependencies(BERT_MODELING_FILE) == expected_bert_dependencies
# Test with an example
create_tmp_repo(tmp_folder)
expected_example_dependencies = ["src/transformers/models/bert/modeling_bert.py"]
with patch_transformer_repo_path(tmp_folder):
assert (
get_module_dependencies("examples/pytorch/text-classification/run_glue.py")
== expected_example_dependencies
)
def test_create_reverse_dependency_tree(self):
with tempfile.TemporaryDirectory() as tmp_folder:
tmp_folder = Path(tmp_folder)
@@ -494,6 +520,33 @@ src/transformers/configuration_utils.py
assert cs.out.strip() in [expected_std_out, expected_std_out_2]
def test_init_test_examples_dependencies(self):
with tempfile.TemporaryDirectory() as tmp_folder:
tmp_folder = Path(tmp_folder)
create_tmp_repo(tmp_folder)
expected_example_deps = {
"examples/flax/test_flax_examples.py": ["examples/flax/text-classification/run_glue.py"],
"examples/pytorch/test_pytorch_examples.py": ["examples/pytorch/text-classification/run_glue.py"],
"examples/tensorflow/test_tensorflow_examples.py": [
"examples/tensorflow/text-classification/run_glue.py"
],
}
expected_examples = {
"examples/flax/test_flax_examples.py",
"examples/flax/text-classification/run_glue.py",
"examples/pytorch/test_pytorch_examples.py",
"examples/pytorch/text-classification/run_glue.py",
"examples/tensorflow/test_tensorflow_examples.py",
"examples/tensorflow/text-classification/run_glue.py",
}
with patch_transformer_repo_path(tmp_folder):
example_deps, all_examples = init_test_examples_dependencies()
assert example_deps == expected_example_deps
assert {str(f.relative_to(tmp_folder)) for f in all_examples} == expected_examples
def test_create_reverse_dependency_map(self):
with tempfile.TemporaryDirectory() as tmp_folder:
tmp_folder = Path(tmp_folder)
@@ -506,6 +559,12 @@ src/transformers/configuration_utils.py
"src/transformers/__init__.py",
"src/transformers/models/bert/__init__.py",
"tests/models/bert/test_modeling_bert.py",
"examples/flax/test_flax_examples.py",
"examples/flax/text-classification/run_glue.py",
"examples/pytorch/test_pytorch_examples.py",
"examples/pytorch/text-classification/run_glue.py",
"examples/tensorflow/test_tensorflow_examples.py",
"examples/tensorflow/text-classification/run_glue.py",
}
assert set(reverse_map["src/transformers/models/bert/modeling_bert.py"]) == expected_bert_deps
@@ -521,6 +580,12 @@ src/transformers/configuration_utils.py
"src/transformers/modeling_utils.py",
"tests/test_modeling_common.py",
"tests/models/bert/test_modeling_bert.py",
"examples/flax/test_flax_examples.py",
"examples/flax/text-classification/run_glue.py",
"examples/pytorch/test_pytorch_examples.py",
"examples/pytorch/text-classification/run_glue.py",
"examples/tensorflow/test_tensorflow_examples.py",
"examples/tensorflow/text-classification/run_glue.py",
}
assert set(reverse_map["src/transformers/__init__.py"]) == expected_init_deps
@@ -529,6 +594,12 @@ src/transformers/configuration_utils.py
"src/transformers/models/bert/configuration_bert.py",
"src/transformers/models/bert/modeling_bert.py",
"tests/models/bert/test_modeling_bert.py",
"examples/flax/test_flax_examples.py",
"examples/flax/text-classification/run_glue.py",
"examples/pytorch/test_pytorch_examples.py",
"examples/pytorch/text-classification/run_glue.py",
"examples/tensorflow/test_tensorflow_examples.py",
"examples/tensorflow/text-classification/run_glue.py",
}
assert set(reverse_map["src/transformers/models/bert/__init__.py"]) == expected_init_deps
@@ -543,6 +614,12 @@ src/transformers/configuration_utils.py
"src/transformers/models/bert/configuration_bert.py",
"src/transformers/models/bert/modeling_bert.py",
"tests/models/bert/test_modeling_bert.py",
"examples/flax/test_flax_examples.py",
"examples/flax/text-classification/run_glue.py",
"examples/pytorch/test_pytorch_examples.py",
"examples/pytorch/text-classification/run_glue.py",
"examples/tensorflow/test_tensorflow_examples.py",
"examples/tensorflow/text-classification/run_glue.py",
}
assert set(reverse_map["src/transformers/models/bert/__init__.py"]) == expected_init_deps
@@ -554,13 +631,26 @@ src/transformers/configuration_utils.py
with patch_transformer_repo_path(tmp_folder):
test_map = create_module_to_test_map(filter_models=True)
expected_bert_tests = {
"examples/flax/test_flax_examples.py",
"examples/pytorch/test_pytorch_examples.py",
"examples/tensorflow/test_tensorflow_examples.py",
"tests/models/bert/test_modeling_bert.py",
}
for model in models:
assert test_map[f"src/transformers/models/{model}/modeling_{model}.py"] == [
f"tests/models/{model}/test_modeling_{model}.py"
]
if model != "bert":
assert test_map[f"src/transformers/models/{model}/modeling_{model}.py"] == [
f"tests/models/{model}/test_modeling_{model}.py"
]
else:
assert set(test_map[f"src/transformers/models/{model}/modeling_{model}.py"]) == expected_bert_tests
# Init got filtered
expected_init_tests = {
"examples/flax/test_flax_examples.py",
"examples/pytorch/test_pytorch_examples.py",
"examples/tensorflow/test_tensorflow_examples.py",
"tests/test_modeling_common.py",
"tests/models/bert/test_modeling_bert.py",
"tests/models/gpt2/test_modeling_gpt2.py",
@@ -575,12 +665,21 @@ src/transformers/configuration_utils.py
commit_changes("src/transformers/models/bert/modeling_bert.py", BERT_MODEL_FILE_NEW_CODE, repo)
example_tests = {
"examples/flax/test_flax_examples.py",
"examples/pytorch/test_pytorch_examples.py",
"examples/tensorflow/test_tensorflow_examples.py",
}
with patch_transformer_repo_path(tmp_folder):
infer_tests_to_run(tmp_folder / "test-output.txt", diff_with_last_commit=True)
with open(tmp_folder / "test-output.txt", "r") as f:
tests_to_run = f.read()
with open(tmp_folder / "examples_test_list.txt", "r") as f:
example_tests_to_run = f.read()
assert tests_to_run == "tests/models/bert/test_modeling_bert.py"
assert set(example_tests_to_run.split(" ")) == example_tests
# Fake a new model addition
repo = create_tmp_repo(tmp_folder, models=models)
@@ -617,6 +716,8 @@ src/transformers/configuration_utils.py
infer_tests_to_run(tmp_folder / "test-output.txt")
with open(tmp_folder / "test-output.txt", "r") as f:
tests_to_run = f.read()
with open(tmp_folder / "examples_test_list.txt", "r") as f:
example_tests_to_run = f.read()
expected_tests = {
"tests/models/bert/test_modeling_bert.py",
@@ -625,15 +726,19 @@ src/transformers/configuration_utils.py
"tests/test_modeling_common.py",
}
assert set(tests_to_run.split(" ")) == expected_tests
assert set(example_tests_to_run.split(" ")) == example_tests
with patch_transformer_repo_path(tmp_folder):
infer_tests_to_run(tmp_folder / "test-output.txt", filter_models=False)
with open(tmp_folder / "test-output.txt", "r") as f:
tests_to_run = f.read()
with open(tmp_folder / "examples_test_list.txt", "r") as f:
example_tests_to_run = f.read()
expected_tests = [f"tests/models/{name}/test_modeling_{name}.py" for name in models + ["t5"]]
expected_tests = set(expected_tests + ["tests/test_modeling_common.py"])
assert set(tests_to_run.split(" ")) == expected_tests
assert set(example_tests_to_run.split(" ")) == example_tests
def test_infer_tests_to_run_with_test_modifs(self):
with tempfile.TemporaryDirectory() as tmp_folder: