From 481de7204c6e065616d4f848ea7f69a2287727df Mon Sep 17 00:00:00 2001 From: Matt Date: Wed, 30 Apr 2025 08:52:21 +0100 Subject: [PATCH] Skip is_flaky tests in the CI (#37723) * No more red flaky tests in the CI! * Remove the CircleCI logic as well * Revert most changes including is_flaky behaviour * make fixup * Move to a more sensible place * Mark a flaky test that failed on this PR! * correct import * update * update * update * update --------- Co-authored-by: ydshieh --- .circleci/create_circleci_config.py | 4 ++++ src/transformers/testing_utils.py | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.circleci/create_circleci_config.py b/.circleci/create_circleci_config.py index 2e87b4c2e1..d3a41a70a5 100644 --- a/.circleci/create_circleci_config.py +++ b/.circleci/create_circleci_config.py @@ -28,6 +28,8 @@ COMMON_ENV_VARIABLES = { "TRANSFORMERS_IS_CI": True, "PYTEST_TIMEOUT": 120, "RUN_PIPELINE_TESTS": False, + # will be adjust in `CircleCIJob.to_dict`. + "RUN_FLAKY": True, } # Disable the use of {"s": None} as the output is way too long, causing the navigation on CircleCI impractical COMMON_PYTEST_OPTIONS = {"max-worker-restart": 0, "vvv": None, "rsfE":None} @@ -126,6 +128,8 @@ class CircleCIJob: def to_dict(self): env = COMMON_ENV_VARIABLES.copy() + # Do not run tests decorated by @is_flaky on pull requests + env['RUN_FLAKY'] = os.environ.get("CIRCLE_PULL_REQUEST", "") == "" env.update(self.additional_env) job = { diff --git a/src/transformers/testing_utils.py b/src/transformers/testing_utils.py index 687d745775..4a1fb68979 100644 --- a/src/transformers/testing_utils.py +++ b/src/transformers/testing_utils.py @@ -240,6 +240,7 @@ def parse_int_from_env(key, default=None): _run_slow_tests = parse_flag_from_env("RUN_SLOW", default=False) +_run_flaky_tests = parse_flag_from_env("RUN_FLAKY", default=True) _run_custom_tokenizers = parse_flag_from_env("RUN_CUSTOM_TOKENIZERS", default=False) _run_staging = parse_flag_from_env("HUGGINGFACE_CO_STAGING", default=False) _run_pipeline_tests = parse_flag_from_env("RUN_PIPELINE_TESTS", default=True) @@ -2614,7 +2615,7 @@ def is_flaky(max_attempts: int = 5, wait_before_retry: Optional[float] = None, d return test_func_ref(*args, **kwargs) - return wrapper + return unittest.skipUnless(_run_flaky_tests, "test is flaky")(wrapper) return decorator