SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Unpin huggingface_hub (#24667)

Browse Source 
* fix

* fix

* fix

* [test all] commit

* [test all] commit

* [test all] commit

---------

Co-authored-by: ydshieh <ydshieh@users.noreply.github.com>
This commit is contained in:
Yih-Dar
2023-07-05 16:49:10 +02:00
committed by GitHub
parent bd9dfc23b9
commit 050ef14516
2 changed files with 74 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
.circleci/config.yml
View File

@@ -30,9 +30,9 @@ jobs:
parallelism: 1
steps:
- checkout
- run: pip install --upgrade pip
- run: pip install -U GitPython
- run: pip install -U .
- run: pip install --upgrade --upgrade-strategy eager pip
- run: pip install -U --upgrade-strategy eager GitPython
- run: pip install -U --upgrade-strategy eager .
- run: mkdir -p test_preparation
- run: python utils/tests_fetcher.py | tee tests_fetched_summary.txt
- store_artifacts:
@@ -104,9 +104,9 @@ jobs:
parallelism: 1
steps:
- checkout
- run: pip install --upgrade pip
- run: pip install -U GitPython
- run: pip install -U .
- run: pip install --upgrade --upgrade-strategy eager pip
- run: pip install -U --upgrade-strategy eager GitPython
- run: pip install -U --upgrade-strategy eager .
- run: |
mkdir test_preparation
echo -n "tests" > test_preparation/test_list.txt
@@ -136,20 +136,20 @@ jobs:
- checkout
- restore_cache:
keys:
- v0.6-code_quality-{{ checksum "setup.py" }}
- v0.6-code-quality
- v0.7-code_quality-{{ checksum "setup.py" }}
- v0.7-code-quality
- restore_cache:
keys:
- v0.6-code_quality-{{ checksum "setup.py" }}-site-packages
- v0.6-code-quality-site-packages
- run: pip install --upgrade pip
- run: pip install -U .[all,quality]
- v0.7-code_quality-{{ checksum "setup.py" }}-site-packages
- v0.7-code-quality-site-packages
- run: pip install --upgrade --upgrade-strategy eager pip
- run: pip install -U --upgrade-strategy eager .[all,quality]
- save_cache:
key: v0.6-code_quality-{{ checksum "setup.py" }}
key: v0.7-code_quality-{{ checksum "setup.py" }}
paths:
- '~/.cache/pip'
- save_cache:
key: v0.6-code_quality-{{ checksum "setup.py" }}-site-packages
key: v0.7-code_quality-{{ checksum "setup.py" }}-site-packages
paths:
- '~/.pyenv/versions/'
- run:
@@ -177,20 +177,20 @@ jobs:
- checkout
- restore_cache:
keys:
- v0.6-repository_consistency-{{ checksum "setup.py" }}
- v0.6-repository_consistency
- v0.7-repository_consistency-{{ checksum "setup.py" }}
- v0.7-repository_consistency
- restore_cache:
keys:
- v0.6-repository_consistency-{{ checksum "setup.py" }}-site-packages
- v0.6-repository_consistency-site-packages
- run: pip install --upgrade pip
- run: pip install -U .[all,quality]
- v0.7-repository_consistency-{{ checksum "setup.py" }}-site-packages
- v0.7-repository_consistency-site-packages
- run: pip install --upgrade --upgrade-strategy eager pip
- run: pip install -U --upgrade-strategy eager .[all,quality]
- save_cache:
key: v0.6-repository_consistency-{{ checksum "setup.py" }}
key: v0.7-repository_consistency-{{ checksum "setup.py" }}
paths:
- '~/.cache/pip'
- save_cache:
key: v0.6-repository_consistency-{{ checksum "setup.py" }}-site-packages
key: v0.7-repository_consistency-{{ checksum "setup.py" }}-site-packages
paths:
- '~/.pyenv/versions/'
- run:

104
.circleci/create_circleci_config.py
View File

@@ -52,7 +52,7 @@ class CircleCIJob:
name: str
additional_env: Dict[str, Any] = None
cache_name: str = None
cache_version: str = "0.6"
cache_version: str = "0.7"
docker_image: List[Dict[str, str]] = None
install_steps: List[str] = None
marker: Optional[str] = None
@@ -116,8 +116,6 @@ class CircleCIJob:
},
]
steps.extend([{"run": l} for l in self.install_steps])
# TODO (ydshieh): Remove this line after the next release (the one after 2023/06/19) of `huggingface_hub`
steps.append({"run": {"name": "Split tests", "command": "pip uninstall -y huggingface_hub && pip install -U git+https://github.com/huggingface/huggingface_hub.git@e4a419bf6bbaa95d14704cc781d3e81a49cef413"}})
steps.append(
{
"save_cache": {
@@ -244,10 +242,10 @@ torch_and_tf_job = CircleCIJob(
install_steps=[
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng git-lfs cmake",
"git lfs install",
"pip install --upgrade pip",
"pip install -U .[sklearn,tf-cpu,torch,testing,sentencepiece,torch-speech,vision]",
"pip install -U tensorflow_probability",
"pip install -U git+https://github.com/huggingface/accelerate",
"pip install --upgrade --upgrade-strategy eager pip",
"pip install -U --upgrade-strategy eager .[sklearn,tf-cpu,torch,testing,sentencepiece,torch-speech,vision]",
"pip install -U --upgrade-strategy eager tensorflow_probability",
"pip install -U --upgrade-strategy eager git+https://github.com/huggingface/accelerate",
],
marker="is_pt_tf_cross_test",
pytest_options={"rA": None, "durations": 0},
@@ -259,9 +257,9 @@ torch_and_flax_job = CircleCIJob(
additional_env={"RUN_PT_FLAX_CROSS_TESTS": True},
install_steps=[
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng",
"pip install -U --upgrade pip",
"pip install -U .[sklearn,flax,torch,testing,sentencepiece,torch-speech,vision]",
"pip install -U git+https://github.com/huggingface/accelerate",
"pip install -U --upgrade-strategy eager --upgrade pip",
"pip install -U --upgrade-strategy eager .[sklearn,flax,torch,testing,sentencepiece,torch-speech,vision]",
"pip install -U --upgrade-strategy eager git+https://github.com/huggingface/accelerate",
],
marker="is_pt_flax_cross_test",
pytest_options={"rA": None, "durations": 0},
@@ -272,9 +270,9 @@ torch_job = CircleCIJob(
"torch",
install_steps=[
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng time",
"pip install --upgrade pip",
"pip install -U .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm]",
"pip install -U git+https://github.com/huggingface/accelerate",
"pip install --upgrade --upgrade-strategy eager pip",
"pip install -U --upgrade-strategy eager .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm]",
"pip install -U --upgrade-strategy eager git+https://github.com/huggingface/accelerate",
],
parallelism=1,
pytest_num_workers=3,
@@ -285,9 +283,9 @@ tf_job = CircleCIJob(
"tf",
install_steps=[
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng cmake",
"pip install --upgrade pip",
"pip install -U .[sklearn,tf-cpu,testing,sentencepiece,tf-speech,vision]",
"pip install -U tensorflow_probability",
"pip install --upgrade --upgrade-strategy eager pip",
"pip install -U --upgrade-strategy eager .[sklearn,tf-cpu,testing,sentencepiece,tf-speech,vision]",
"pip install -U --upgrade-strategy eager tensorflow_probability",
],
parallelism=1,
pytest_num_workers=6,
@@ -299,8 +297,8 @@ flax_job = CircleCIJob(
"flax",
install_steps=[
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng",
"pip install --upgrade pip",
"pip install -U .[flax,testing,sentencepiece,flax-speech,vision]",
"pip install --upgrade --upgrade-strategy eager pip",
"pip install -U --upgrade-strategy eager .[flax,testing,sentencepiece,flax-speech,vision]",
],
parallelism=1,
pytest_options={"rA": None},
@@ -312,8 +310,8 @@ pipelines_torch_job = CircleCIJob(
additional_env={"RUN_PIPELINE_TESTS": True},
install_steps=[
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng",
"pip install --upgrade pip",
"pip install -U .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm,video]",
"pip install --upgrade --upgrade-strategy eager pip",
"pip install -U --upgrade-strategy eager .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm,video]",
],
pytest_options={"rA": None},
marker="is_pipeline_test",
@@ -325,9 +323,9 @@ pipelines_tf_job = CircleCIJob(
additional_env={"RUN_PIPELINE_TESTS": True},
install_steps=[
"sudo apt-get -y update && sudo apt-get install -y cmake",
"pip install --upgrade pip",
"pip install -U .[sklearn,tf-cpu,testing,sentencepiece,vision]",
"pip install -U tensorflow_probability",
"pip install --upgrade --upgrade-strategy eager pip",
"pip install -U --upgrade-strategy eager .[sklearn,tf-cpu,testing,sentencepiece,vision]",
"pip install -U --upgrade-strategy eager tensorflow_probability",
],
pytest_options={"rA": None},
marker="is_pipeline_test",
@@ -349,8 +347,8 @@ custom_tokenizers_job = CircleCIJob(
"sudo cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local\n"
"sudo make install\n",
},
"pip install --upgrade pip",
"pip install -U .[ja,testing,sentencepiece,jieba,spacy,ftfy,rjieba]",
"pip install --upgrade --upgrade-strategy eager pip",
"pip install -U --upgrade-strategy eager .[ja,testing,sentencepiece,jieba,spacy,ftfy,rjieba]",
"python -m unidic download",
],
parallelism=None,
@@ -368,9 +366,9 @@ examples_torch_job = CircleCIJob(
cache_name="torch_examples",
install_steps=[
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng",
"pip install --upgrade pip",
"pip install -U .[sklearn,torch,sentencepiece,testing,torch-speech]",
"pip install -U -r examples/pytorch/_tests_requirements.txt",
"pip install --upgrade --upgrade-strategy eager pip",
"pip install -U --upgrade-strategy eager .[sklearn,torch,sentencepiece,testing,torch-speech]",
"pip install -U --upgrade-strategy eager -r examples/pytorch/_tests_requirements.txt",
],
)
@@ -380,9 +378,9 @@ examples_tensorflow_job = CircleCIJob(
cache_name="tensorflow_examples",
install_steps=[
"sudo apt-get -y update && sudo apt-get install -y cmake",
"pip install --upgrade pip",
"pip install -U .[sklearn,tensorflow,sentencepiece,testing]",
"pip install -U -r examples/tensorflow/_tests_requirements.txt",
"pip install --upgrade --upgrade-strategy eager pip",
"pip install -U --upgrade-strategy eager .[sklearn,tensorflow,sentencepiece,testing]",
"pip install -U --upgrade-strategy eager -r examples/tensorflow/_tests_requirements.txt",
],
)
@@ -391,9 +389,9 @@ examples_flax_job = CircleCIJob(
"examples_flax",
cache_name="flax_examples",
install_steps=[
"pip install --upgrade pip",
"pip install -U .[flax,testing,sentencepiece]",
"pip install -U -r examples/flax/_tests_requirements.txt",
"pip install --upgrade --upgrade-strategy eager pip",
"pip install -U --upgrade-strategy eager .[flax,testing,sentencepiece]",
"pip install -U --upgrade-strategy eager -r examples/flax/_tests_requirements.txt",
],
)
@@ -404,8 +402,8 @@ hub_job = CircleCIJob(
"sudo apt-get -y update && sudo apt-get install git-lfs",
'git config --global user.email "ci@dummy.com"',
'git config --global user.name "ci"',
"pip install --upgrade pip",
"pip install -U .[torch,sentencepiece,testing]",
"pip install --upgrade --upgrade-strategy eager pip",
"pip install -U --upgrade-strategy eager .[torch,sentencepiece,testing]",
],
marker="is_staging_test",
pytest_num_workers=1,
@@ -416,8 +414,8 @@ onnx_job = CircleCIJob(
"onnx",
install_steps=[
"sudo apt-get -y update && sudo apt-get install -y cmake",
"pip install --upgrade pip",
"pip install -U .[torch,tf,testing,sentencepiece,onnxruntime,vision,rjieba]",
"pip install --upgrade --upgrade-strategy eager pip",
"pip install -U --upgrade-strategy eager .[torch,tf,testing,sentencepiece,onnxruntime,vision,rjieba]",
],
pytest_options={"k onnx": None},
pytest_num_workers=1,
@@ -428,14 +426,16 @@ exotic_models_job = CircleCIJob(
"exotic_models",
install_steps=[
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev",
"pip install --upgrade pip",
"pip install -U .[torch,testing,vision]",
"pip install -U torchvision",
"pip install -U scipy",
"pip install -U 'git+https://github.com/facebookresearch/detectron2.git'",
"pip install --upgrade --upgrade-strategy eager pip",
"pip install -U --upgrade-strategy eager .[torch,testing,vision]",
"pip install -U --upgrade-strategy eager torchvision",
"pip install -U --upgrade-strategy eager scipy",
"pip install -U --upgrade-strategy eager 'git+https://github.com/facebookresearch/detectron2.git'",
"sudo apt install tesseract-ocr",
"pip install -U pytesseract",
"pip install -U natten",
"pip install -U --upgrade-strategy eager pytesseract",
"pip install -U --upgrade-strategy eager natten",
# TODO (ydshieh): Remove this line once `https://github.com/facebookresearch/detectron2/issues/5010` is resolved
'pip install -U --upgrade-strategy eager "Pillow<10.0.0"',
],
tests_to_run=[
"tests/models/*layoutlmv*",
@@ -450,8 +450,8 @@ exotic_models_job = CircleCIJob(
repo_utils_job = CircleCIJob(
"repo_utils",
install_steps=[
"pip install --upgrade pip",
"pip install -U .[quality,testing,torch]",
"pip install --upgrade --upgrade-strategy eager pip",
"pip install -U --upgrade-strategy eager .[quality,testing,torch]",
],
parallelism=None,
pytest_num_workers=1,
@@ -471,11 +471,11 @@ doc_test_job = CircleCIJob(
additional_env={"TRANSFORMERS_VERBOSITY": "error", "DATASETS_VERBOSITY": "error", "SKIP_CUDA_DOCTEST": "1"},
install_steps=[
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng time ffmpeg",
"pip install --upgrade pip",
"pip install -U -e .[dev]",
"pip install -U git+https://github.com/huggingface/accelerate",
"pip install --upgrade pytest pytest-sugar",
"pip install -U natten",
"pip install --upgrade --upgrade-strategy eager pip",
"pip install -U --upgrade-strategy eager -e .[dev]",
"pip install -U --upgrade-strategy eager git+https://github.com/huggingface/accelerate",
"pip install --upgrade --upgrade-strategy eager pytest pytest-sugar",
"pip install -U --upgrade-strategy eager natten",
"find -name __pycache__ -delete",
"find . -name \*.pyc -delete",
# Add an empty file to keep the test step running correctly even no file is selected to be tested.