From 050ef1451682a64a70394f372021572c59dc4c28 Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Wed, 5 Jul 2023 16:49:10 +0200 Subject: [PATCH] Unpin `huggingface_hub` (#24667) * fix * fix * fix * [test all] commit * [test all] commit * [test all] commit --------- Co-authored-by: ydshieh --- .circleci/config.yml | 44 ++++++------ .circleci/create_circleci_config.py | 104 ++++++++++++++-------------- 2 files changed, 74 insertions(+), 74 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 99d0e05c64..0756e0d4b7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -30,9 +30,9 @@ jobs: parallelism: 1 steps: - checkout - - run: pip install --upgrade pip - - run: pip install -U GitPython - - run: pip install -U . + - run: pip install --upgrade --upgrade-strategy eager pip + - run: pip install -U --upgrade-strategy eager GitPython + - run: pip install -U --upgrade-strategy eager . - run: mkdir -p test_preparation - run: python utils/tests_fetcher.py | tee tests_fetched_summary.txt - store_artifacts: @@ -104,9 +104,9 @@ jobs: parallelism: 1 steps: - checkout - - run: pip install --upgrade pip - - run: pip install -U GitPython - - run: pip install -U . + - run: pip install --upgrade --upgrade-strategy eager pip + - run: pip install -U --upgrade-strategy eager GitPython + - run: pip install -U --upgrade-strategy eager . - run: | mkdir test_preparation echo -n "tests" > test_preparation/test_list.txt @@ -136,20 +136,20 @@ jobs: - checkout - restore_cache: keys: - - v0.6-code_quality-{{ checksum "setup.py" }} - - v0.6-code-quality + - v0.7-code_quality-{{ checksum "setup.py" }} + - v0.7-code-quality - restore_cache: keys: - - v0.6-code_quality-{{ checksum "setup.py" }}-site-packages - - v0.6-code-quality-site-packages - - run: pip install --upgrade pip - - run: pip install -U .[all,quality] + - v0.7-code_quality-{{ checksum "setup.py" }}-site-packages + - v0.7-code-quality-site-packages + - run: pip install --upgrade --upgrade-strategy eager pip + - run: pip install -U --upgrade-strategy eager .[all,quality] - save_cache: - key: v0.6-code_quality-{{ checksum "setup.py" }} + key: v0.7-code_quality-{{ checksum "setup.py" }} paths: - '~/.cache/pip' - save_cache: - key: v0.6-code_quality-{{ checksum "setup.py" }}-site-packages + key: v0.7-code_quality-{{ checksum "setup.py" }}-site-packages paths: - '~/.pyenv/versions/' - run: @@ -177,20 +177,20 @@ jobs: - checkout - restore_cache: keys: - - v0.6-repository_consistency-{{ checksum "setup.py" }} - - v0.6-repository_consistency + - v0.7-repository_consistency-{{ checksum "setup.py" }} + - v0.7-repository_consistency - restore_cache: keys: - - v0.6-repository_consistency-{{ checksum "setup.py" }}-site-packages - - v0.6-repository_consistency-site-packages - - run: pip install --upgrade pip - - run: pip install -U .[all,quality] + - v0.7-repository_consistency-{{ checksum "setup.py" }}-site-packages + - v0.7-repository_consistency-site-packages + - run: pip install --upgrade --upgrade-strategy eager pip + - run: pip install -U --upgrade-strategy eager .[all,quality] - save_cache: - key: v0.6-repository_consistency-{{ checksum "setup.py" }} + key: v0.7-repository_consistency-{{ checksum "setup.py" }} paths: - '~/.cache/pip' - save_cache: - key: v0.6-repository_consistency-{{ checksum "setup.py" }}-site-packages + key: v0.7-repository_consistency-{{ checksum "setup.py" }}-site-packages paths: - '~/.pyenv/versions/' - run: diff --git a/.circleci/create_circleci_config.py b/.circleci/create_circleci_config.py index 246dc339d7..f44b500a80 100644 --- a/.circleci/create_circleci_config.py +++ b/.circleci/create_circleci_config.py @@ -52,7 +52,7 @@ class CircleCIJob: name: str additional_env: Dict[str, Any] = None cache_name: str = None - cache_version: str = "0.6" + cache_version: str = "0.7" docker_image: List[Dict[str, str]] = None install_steps: List[str] = None marker: Optional[str] = None @@ -116,8 +116,6 @@ class CircleCIJob: }, ] steps.extend([{"run": l} for l in self.install_steps]) - # TODO (ydshieh): Remove this line after the next release (the one after 2023/06/19) of `huggingface_hub` - steps.append({"run": {"name": "Split tests", "command": "pip uninstall -y huggingface_hub && pip install -U git+https://github.com/huggingface/huggingface_hub.git@e4a419bf6bbaa95d14704cc781d3e81a49cef413"}}) steps.append( { "save_cache": { @@ -244,10 +242,10 @@ torch_and_tf_job = CircleCIJob( install_steps=[ "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng git-lfs cmake", "git lfs install", - "pip install --upgrade pip", - "pip install -U .[sklearn,tf-cpu,torch,testing,sentencepiece,torch-speech,vision]", - "pip install -U tensorflow_probability", - "pip install -U git+https://github.com/huggingface/accelerate", + "pip install --upgrade --upgrade-strategy eager pip", + "pip install -U --upgrade-strategy eager .[sklearn,tf-cpu,torch,testing,sentencepiece,torch-speech,vision]", + "pip install -U --upgrade-strategy eager tensorflow_probability", + "pip install -U --upgrade-strategy eager git+https://github.com/huggingface/accelerate", ], marker="is_pt_tf_cross_test", pytest_options={"rA": None, "durations": 0}, @@ -259,9 +257,9 @@ torch_and_flax_job = CircleCIJob( additional_env={"RUN_PT_FLAX_CROSS_TESTS": True}, install_steps=[ "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng", - "pip install -U --upgrade pip", - "pip install -U .[sklearn,flax,torch,testing,sentencepiece,torch-speech,vision]", - "pip install -U git+https://github.com/huggingface/accelerate", + "pip install -U --upgrade-strategy eager --upgrade pip", + "pip install -U --upgrade-strategy eager .[sklearn,flax,torch,testing,sentencepiece,torch-speech,vision]", + "pip install -U --upgrade-strategy eager git+https://github.com/huggingface/accelerate", ], marker="is_pt_flax_cross_test", pytest_options={"rA": None, "durations": 0}, @@ -272,9 +270,9 @@ torch_job = CircleCIJob( "torch", install_steps=[ "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng time", - "pip install --upgrade pip", - "pip install -U .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm]", - "pip install -U git+https://github.com/huggingface/accelerate", + "pip install --upgrade --upgrade-strategy eager pip", + "pip install -U --upgrade-strategy eager .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm]", + "pip install -U --upgrade-strategy eager git+https://github.com/huggingface/accelerate", ], parallelism=1, pytest_num_workers=3, @@ -285,9 +283,9 @@ tf_job = CircleCIJob( "tf", install_steps=[ "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng cmake", - "pip install --upgrade pip", - "pip install -U .[sklearn,tf-cpu,testing,sentencepiece,tf-speech,vision]", - "pip install -U tensorflow_probability", + "pip install --upgrade --upgrade-strategy eager pip", + "pip install -U --upgrade-strategy eager .[sklearn,tf-cpu,testing,sentencepiece,tf-speech,vision]", + "pip install -U --upgrade-strategy eager tensorflow_probability", ], parallelism=1, pytest_num_workers=6, @@ -299,8 +297,8 @@ flax_job = CircleCIJob( "flax", install_steps=[ "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng", - "pip install --upgrade pip", - "pip install -U .[flax,testing,sentencepiece,flax-speech,vision]", + "pip install --upgrade --upgrade-strategy eager pip", + "pip install -U --upgrade-strategy eager .[flax,testing,sentencepiece,flax-speech,vision]", ], parallelism=1, pytest_options={"rA": None}, @@ -312,8 +310,8 @@ pipelines_torch_job = CircleCIJob( additional_env={"RUN_PIPELINE_TESTS": True}, install_steps=[ "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng", - "pip install --upgrade pip", - "pip install -U .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm,video]", + "pip install --upgrade --upgrade-strategy eager pip", + "pip install -U --upgrade-strategy eager .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm,video]", ], pytest_options={"rA": None}, marker="is_pipeline_test", @@ -325,9 +323,9 @@ pipelines_tf_job = CircleCIJob( additional_env={"RUN_PIPELINE_TESTS": True}, install_steps=[ "sudo apt-get -y update && sudo apt-get install -y cmake", - "pip install --upgrade pip", - "pip install -U .[sklearn,tf-cpu,testing,sentencepiece,vision]", - "pip install -U tensorflow_probability", + "pip install --upgrade --upgrade-strategy eager pip", + "pip install -U --upgrade-strategy eager .[sklearn,tf-cpu,testing,sentencepiece,vision]", + "pip install -U --upgrade-strategy eager tensorflow_probability", ], pytest_options={"rA": None}, marker="is_pipeline_test", @@ -349,8 +347,8 @@ custom_tokenizers_job = CircleCIJob( "sudo cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local\n" "sudo make install\n", }, - "pip install --upgrade pip", - "pip install -U .[ja,testing,sentencepiece,jieba,spacy,ftfy,rjieba]", + "pip install --upgrade --upgrade-strategy eager pip", + "pip install -U --upgrade-strategy eager .[ja,testing,sentencepiece,jieba,spacy,ftfy,rjieba]", "python -m unidic download", ], parallelism=None, @@ -368,9 +366,9 @@ examples_torch_job = CircleCIJob( cache_name="torch_examples", install_steps=[ "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng", - "pip install --upgrade pip", - "pip install -U .[sklearn,torch,sentencepiece,testing,torch-speech]", - "pip install -U -r examples/pytorch/_tests_requirements.txt", + "pip install --upgrade --upgrade-strategy eager pip", + "pip install -U --upgrade-strategy eager .[sklearn,torch,sentencepiece,testing,torch-speech]", + "pip install -U --upgrade-strategy eager -r examples/pytorch/_tests_requirements.txt", ], ) @@ -380,9 +378,9 @@ examples_tensorflow_job = CircleCIJob( cache_name="tensorflow_examples", install_steps=[ "sudo apt-get -y update && sudo apt-get install -y cmake", - "pip install --upgrade pip", - "pip install -U .[sklearn,tensorflow,sentencepiece,testing]", - "pip install -U -r examples/tensorflow/_tests_requirements.txt", + "pip install --upgrade --upgrade-strategy eager pip", + "pip install -U --upgrade-strategy eager .[sklearn,tensorflow,sentencepiece,testing]", + "pip install -U --upgrade-strategy eager -r examples/tensorflow/_tests_requirements.txt", ], ) @@ -391,9 +389,9 @@ examples_flax_job = CircleCIJob( "examples_flax", cache_name="flax_examples", install_steps=[ - "pip install --upgrade pip", - "pip install -U .[flax,testing,sentencepiece]", - "pip install -U -r examples/flax/_tests_requirements.txt", + "pip install --upgrade --upgrade-strategy eager pip", + "pip install -U --upgrade-strategy eager .[flax,testing,sentencepiece]", + "pip install -U --upgrade-strategy eager -r examples/flax/_tests_requirements.txt", ], ) @@ -404,8 +402,8 @@ hub_job = CircleCIJob( "sudo apt-get -y update && sudo apt-get install git-lfs", 'git config --global user.email "ci@dummy.com"', 'git config --global user.name "ci"', - "pip install --upgrade pip", - "pip install -U .[torch,sentencepiece,testing]", + "pip install --upgrade --upgrade-strategy eager pip", + "pip install -U --upgrade-strategy eager .[torch,sentencepiece,testing]", ], marker="is_staging_test", pytest_num_workers=1, @@ -416,8 +414,8 @@ onnx_job = CircleCIJob( "onnx", install_steps=[ "sudo apt-get -y update && sudo apt-get install -y cmake", - "pip install --upgrade pip", - "pip install -U .[torch,tf,testing,sentencepiece,onnxruntime,vision,rjieba]", + "pip install --upgrade --upgrade-strategy eager pip", + "pip install -U --upgrade-strategy eager .[torch,tf,testing,sentencepiece,onnxruntime,vision,rjieba]", ], pytest_options={"k onnx": None}, pytest_num_workers=1, @@ -428,14 +426,16 @@ exotic_models_job = CircleCIJob( "exotic_models", install_steps=[ "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev", - "pip install --upgrade pip", - "pip install -U .[torch,testing,vision]", - "pip install -U torchvision", - "pip install -U scipy", - "pip install -U 'git+https://github.com/facebookresearch/detectron2.git'", + "pip install --upgrade --upgrade-strategy eager pip", + "pip install -U --upgrade-strategy eager .[torch,testing,vision]", + "pip install -U --upgrade-strategy eager torchvision", + "pip install -U --upgrade-strategy eager scipy", + "pip install -U --upgrade-strategy eager 'git+https://github.com/facebookresearch/detectron2.git'", "sudo apt install tesseract-ocr", - "pip install -U pytesseract", - "pip install -U natten", + "pip install -U --upgrade-strategy eager pytesseract", + "pip install -U --upgrade-strategy eager natten", + # TODO (ydshieh): Remove this line once `https://github.com/facebookresearch/detectron2/issues/5010` is resolved + 'pip install -U --upgrade-strategy eager "Pillow<10.0.0"', ], tests_to_run=[ "tests/models/*layoutlmv*", @@ -450,8 +450,8 @@ exotic_models_job = CircleCIJob( repo_utils_job = CircleCIJob( "repo_utils", install_steps=[ - "pip install --upgrade pip", - "pip install -U .[quality,testing,torch]", + "pip install --upgrade --upgrade-strategy eager pip", + "pip install -U --upgrade-strategy eager .[quality,testing,torch]", ], parallelism=None, pytest_num_workers=1, @@ -471,11 +471,11 @@ doc_test_job = CircleCIJob( additional_env={"TRANSFORMERS_VERBOSITY": "error", "DATASETS_VERBOSITY": "error", "SKIP_CUDA_DOCTEST": "1"}, install_steps=[ "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng time ffmpeg", - "pip install --upgrade pip", - "pip install -U -e .[dev]", - "pip install -U git+https://github.com/huggingface/accelerate", - "pip install --upgrade pytest pytest-sugar", - "pip install -U natten", + "pip install --upgrade --upgrade-strategy eager pip", + "pip install -U --upgrade-strategy eager -e .[dev]", + "pip install -U --upgrade-strategy eager git+https://github.com/huggingface/accelerate", + "pip install --upgrade --upgrade-strategy eager pytest pytest-sugar", + "pip install -U --upgrade-strategy eager natten", "find -name __pycache__ -delete", "find . -name \*.pyc -delete", # Add an empty file to keep the test step running correctly even no file is selected to be tested.