diff --git a/.circleci/create_circleci_config.py b/.circleci/create_circleci_config.py index 156765d30f..eba90dac36 100644 --- a/.circleci/create_circleci_config.py +++ b/.circleci/create_circleci_config.py @@ -33,6 +33,26 @@ COMMON_ENV_VARIABLES = { COMMON_PYTEST_OPTIONS = {"max-worker-restart": 0, "dist": "loadfile", "vvv": None, "rsfE":None} DEFAULT_DOCKER_IMAGE = [{"image": "cimg/python:3.8.12"}] +# Strings that commonly appear in the output of flaky tests when they fail. These are used with `pytest-rerunfailures` +# to rerun the tests that match these patterns. +FLAKY_TEST_FAILURE_PATTERNS = [ + "OSError", # Machine/connection transient error + "Timeout", # Machine/connection transient error + "ConnectionError", # Connection transient error + "FileNotFoundError", # Raised by `datasets` on Hub failures + "PIL.UnidentifiedImageError", # Raised by `PIL.Image.open` on connection issues + "HTTPError.*502", # Hub-related + "HTTPError.*504", # Hub-related + "AssertionError: Tensor-likes are not close!", # `torch.testing.assert_close`, we might have unlucky random values + # TODO: error downloading tokenizer's `merged.txt` from hub can cause all the exceptions below. Throw and handle + # them under a single message. + "TypeError: expected str, bytes or os.PathLike object, not NoneType", + "TypeError: stat: path should be string, bytes, os.PathLike or integer, not NoneType", + "Converting from Tiktoken failed", + "KeyError: > ' steps = [ "checkout", @@ -152,7 +174,7 @@ class CircleCIJob: }, {"run": { "name": "Run tests", - "command": f"({timeout_cmd} python3 -m pytest {marker_cmd} -n {self.pytest_num_workers} {additional_flags} {' '.join(pytest_flags)} $(cat splitted_tests.txt) | tee tests_output.txt)"} + "command": f"({timeout_cmd} python3 -m pytest {marker_cmd} -n {self.pytest_num_workers} {junit_flags} {repeat_on_failure_flags} {' '.join(pytest_flags)} $(cat splitted_tests.txt) | tee tests_output.txt)"} }, {"run": {"name": "Expand to show skipped tests", "when": "always", "command": f"python3 .circleci/parse_test_outputs.py --file tests_output.txt --skip"}}, {"run": {"name": "Failed tests: show reasons", "when": "always", "command": f"python3 .circleci/parse_test_outputs.py --file tests_output.txt --fail"}}, diff --git a/docker/custom-tokenizers.dockerfile b/docker/custom-tokenizers.dockerfile index 87601bcf3f..2261c44099 100644 --- a/docker/custom-tokenizers.dockerfile +++ b/docker/custom-tokenizers.dockerfile @@ -1,5 +1,6 @@ FROM python:3.9-slim ENV PYTHONDONTWRITEBYTECODE=1 +ARG REF=main USER root RUN apt-get update && apt-get install -y libsndfile1-dev espeak-ng time git cmake wget xz-utils build-essential g++5 libprotobuf-dev protobuf-compiler ENV UV_PYTHON=/usr/local/bin/python @@ -16,11 +17,11 @@ RUN make install -j 10 RUN uv pip install --no-cache --upgrade 'torch' --index-url https://download.pytorch.org/whl/cpu -RUN uv pip install --no-cache-dir --no-deps accelerate --extra-index-url https://download.pytorch.org/whl/cpu -RUN uv pip install --no-cache-dir "transformers[ja,testing,sentencepiece,jieba,spacy,ftfy,rjieba]" unidic unidic-lite +RUN uv pip install --no-cache-dir --no-deps accelerate --extra-index-url https://download.pytorch.org/whl/cpu +RUN uv pip install --no-cache-dir "git+https://github.com/huggingface/transformers.git@${REF}#egg=transformers[ja,testing,sentencepiece,jieba,spacy,ftfy,rjieba]" unidic unidic-lite # spacy is not used so not tested. Causes to failures. TODO fix later RUN python3 -m unidic download RUN pip uninstall -y transformers RUN apt-get clean && rm -rf /var/lib/apt/lists/* -RUN apt remove -y g++ cmake xz-utils libprotobuf-dev protobuf-compiler \ No newline at end of file +RUN apt remove -y g++ cmake xz-utils libprotobuf-dev protobuf-compiler diff --git a/docker/examples-tf.dockerfile b/docker/examples-tf.dockerfile index ffbaafd8b8..16899e2a44 100644 --- a/docker/examples-tf.dockerfile +++ b/docker/examples-tf.dockerfile @@ -1,12 +1,13 @@ FROM python:3.9-slim ENV PYTHONDONTWRITEBYTECODE=1 +ARG REF=main USER root RUN apt-get update && apt-get install -y libsndfile1-dev espeak-ng time git RUN apt-get install -y g++ cmake ENV UV_PYTHON=/usr/local/bin/python RUN pip --no-cache-dir install uv && uv venv RUN uv pip install --no-cache-dir -U pip setuptools albumentations seqeval -RUN pip install --upgrade --no-cache-dir "transformers[tf-cpu,sklearn,testing,sentencepiece,tf-speech,vision]" -RUN uv pip install --no-cache-dir "protobuf==3.20.3" +RUN pip install --upgrade --no-cache-dir "git+https://github.com/huggingface/transformers.git@${REF}#egg=transformers[tf-cpu,sklearn,testing,sentencepiece,tf-speech,vision]" +RUN uv pip install --no-cache-dir "protobuf==3.20.3" RUN pip uninstall -y transformers -RUN apt-get clean && rm -rf /var/lib/apt/lists/* \ No newline at end of file +RUN apt-get clean && rm -rf /var/lib/apt/lists/* diff --git a/docker/examples-torch.dockerfile b/docker/examples-torch.dockerfile index f9c7257b9c..ec868cb10a 100644 --- a/docker/examples-torch.dockerfile +++ b/docker/examples-torch.dockerfile @@ -1,11 +1,12 @@ FROM python:3.9-slim ENV PYTHONDONTWRITEBYTECODE=1 +ARG REF=main USER root RUN apt-get update && apt-get install -y --no-install-recommends libsndfile1-dev espeak-ng time git g++ cmake pkg-config openssh-client git ENV UV_PYTHON=/usr/local/bin/python RUN pip --no-cache-dir install uv && uv venv && uv pip install --no-cache-dir -U pip setuptools RUN pip install --no-cache-dir 'torch' 'torchvision' 'torchaudio' --index-url https://download.pytorch.org/whl/cpu -RUN uv pip install --no-deps timm accelerate --extra-index-url https://download.pytorch.org/whl/cpu -RUN uv pip install --no-cache-dir librosa "transformers[sklearn,sentencepiece,vision,testing]" seqeval albumentations jiwer +RUN uv pip install --no-deps timm accelerate --extra-index-url https://download.pytorch.org/whl/cpu +RUN uv pip install --no-cache-dir librosa "git+https://github.com/huggingface/transformers.git@${REF}#egg=transformers[sklearn,sentencepiece,vision,testing]" seqeval albumentations jiwer RUN pip uninstall -y transformers -RUN apt-get clean && rm -rf /var/lib/apt/lists/* \ No newline at end of file +RUN apt-get clean && rm -rf /var/lib/apt/lists/* diff --git a/setup.py b/setup.py index fcb62c6157..e194982185 100644 --- a/setup.py +++ b/setup.py @@ -150,6 +150,7 @@ _deps = [ "pydantic", "pytest>=7.2.0,<8.0.0", "pytest-asyncio", + "pytest-rerunfailures", "pytest-timeout", "pytest-xdist", "pytest-order", @@ -326,6 +327,7 @@ extras["testing"] = ( "pytest-rich", "pytest-xdist", "pytest-order", + "pytest-rerunfailures", "timeout-decorator", "parameterized", "psutil", diff --git a/src/transformers/dependency_versions_table.py b/src/transformers/dependency_versions_table.py index 28ae446366..3fc3aafdbe 100644 --- a/src/transformers/dependency_versions_table.py +++ b/src/transformers/dependency_versions_table.py @@ -56,6 +56,7 @@ deps = { "pydantic": "pydantic", "pytest": "pytest>=7.2.0,<8.0.0", "pytest-asyncio": "pytest-asyncio", + "pytest-rerunfailures": "pytest-rerunfailures", "pytest-timeout": "pytest-timeout", "pytest-xdist": "pytest-xdist", "pytest-order": "pytest-order", diff --git a/src/transformers/testing_utils.py b/src/transformers/testing_utils.py index f6577469bf..6d25738691 100644 --- a/src/transformers/testing_utils.py +++ b/src/transformers/testing_utils.py @@ -2518,6 +2518,11 @@ def is_flaky(max_attempts: int = 5, wait_before_retry: Optional[float] = None, d """ To decorate flaky tests. They will be retried on failures. + Please note that our push tests use `pytest-rerunfailures`, which prompts the CI to rerun certain types of + failed tests. More specifically, if the test exception contains any substring in `FLAKY_TEST_FAILURE_PATTERNS` + (in `.circleci/create_circleci_config.py`), it will be rerun. If you find a recurrent pattern of failures, + expand `FLAKY_TEST_FAILURE_PATTERNS` in our CI configuration instead of using `is_flaky`. + Args: max_attempts (`int`, *optional*, defaults to 5): The maximum number of attempts to retry the flaky test.