[CI update] Try to use dockers and no cache (#29202)

* change cis * nits * update * minor updates * [push-ci-image] * nit [push-ci-image] * nitsssss * [build-ci-image] * [push-ci-image] * [push-ci-image] * both * [push-ci-image] * this? * [push-ci-image] * pypi-kenlm needs g++ * [push-ci-image] * nit * more nits [push-ci-image] * nits [push-ci-image] * [push-ci-image] * [push-ci-image] * [push-ci-image] * add vision * [push-ci-image] * [push-ci-image] * add new dummy file but will need to update them [push-ci-image] * [push-ci-image] * show package size as well * [push-ci-image] * potentially ignore failures * workflow updates * nits [push-ci-image] * [push-ci-image] * fix consistency * clean nciida triton * also show big packages [push-ci-image] * nit * update * another one * line escape? * add accelerate [push-ci-image] * updates [push-ci-image] * nits to run tests, no push-ci * try to parse skip reason to make sure nothing is skipped that should no be skippped * nit? * always show skipped reasons * nits * better parsing of the test outputs * action="store_true", * failure on failed * show matched * debug * update short summary with skipped, failed and errors * nits * nits * coolu pdates * remove docbuilder * fix * always run checks * oups * nits * don't error out on library printing * non zero exi codes * no warning * nit * WAT? * format nit * [push-ci-image] * fail if fail is needed * [push-ci-image] * sound file for torch light? * [push-ci-image] * order is important [push-ci-image] * [push-ci-image] reduce even further * [push-ci-image] * use pytest rich ! * yes [push-ci-image] * oupsy * bring back the full traceback, but pytest rich should help * nit * [push-ci-image] * re run * nit * [push-ci-image] * [push-ci-image] * [push-ci-image] * empty push to trigger * [push-ci-image] * nit? [push-ci-image] * empty * try to install timm with no deps * [push-ci-image] * oups [push-ci-image] * [push-ci-image] * [push-ci-image] ? * [push-ci-image] open ssh client for git checkout fast * empty for torch light * updates [push-ci-image] * nit * @v4 for checkout * [push-ci-image] * [push-ci-image] * fix fetch tests with parallelism * [push-ci-image] * more parallelism * nit * more nits * empty to re-trigger * empty to re-trigger * split by timing * did not work with previous commit * junit.xml * no path? * mmm this? * junitxml format * split by timing * nit * fix junit family * now we can test if the xunit1 is compatible! * this? * fully list tests * update * update * oups * finally * use classname * remove working directory to make sure the path does not interfere * okay no juni should have the correct path * name split? * sort by classname is what make most sense * some testing * naem * oups * test something fun * autodetect * 18? * nit * file size? * uip * 4 is best * update to see versions * better print * [push-ci-image] * [push-ci-image] * please install the correct keras version * [push-ci-image] * [push-ci-image] * [push-ci-image] * [push-ci-image] * [push-ci-image] * uv is fucking me up * [push-ci-image] * [push-ci-image] * [push-ci-image] * nits * [push-ci-image] * [push-ci-image] * install issues an pins * tapas as well * nits * more paralellism * short tb * soundfile * soundfile * [push-ci-image] * [push-ci-image] * [push-ci-image] * oups * [push-ci-image] * fix some things * [push-ci-image] * [push-ci-image] * [push-ci-image] * [push-ci-image] * use torch-light for hub * small git lfs for hub job * [push-ci-image] * [push-ci-image] * [push-ci-image] * [push-ci-image] * fix tf tapas * [push-ci-image] * nits * [push-ci-image] * don't update the test * [push-ci-image] * [push-ci-image] * [push-ci-image] * no use them * [push-ci-image] * [push-ci-image] * [push-ci-image] * [push-ci-image] * update tf proba * [push-ci-image] * [push-ci-image] * woops * [push-ci-image] * [push-ci-image] * [push-ci-image] * [push-ci-image] * [push-ci-image] * [push-ci-image] * test with built dockers * [push-ci-image] * skip annoying tests * revert fix copy * update test values * update * last skip and fixup * nit * ALL GOOOD * quality * Update tests/models/layoutlmv2/test_image_processing_layoutlmv2.py * Update docker/quality.dockerfile Co-authored-by: Lysandre Debut <hi@lysand.re> * Update src/transformers/models/tapas/modeling_tf_tapas.py Co-authored-by: Lysandre Debut <hi@lysand.re> * Apply suggestions from code review Co-authored-by: Lysandre Debut <hi@lysand.re> * use torch-speed * updates * [push-ci-image] * [push-ci-image] * [push-ci-image] * [push-ci-image] * fuck ken-lm [push-ci-image] * [push-ci-image] * [push-ci-image] --------- Co-authored-by: Lysandre Debut <hi@lysand.re>
2024-05-06 10:10:32 +02:00
parent 91d155ea92
commit 307f632bb2
25 changed files with 430 additions and 291 deletions
--- a/setup.py
+++ b/setup.py
@@ -116,7 +116,6 @@ _deps = [
    "ftfy",
    "fugashi>=1.0",
    "GitPython<3.1.19",
-    "hf-doc-builder>=0.3.0",
    "huggingface-hub>=0.19.3,<1.0",
    "importlib_metadata",
    "ipadic>=1.0.0,<2.0",
@@ -126,7 +125,7 @@ _deps = [
    "jieba",
    "kenlm",
    # Keras pin - this is to make sure Keras 3 doesn't destroy us. Remove or change when we have proper support.
-    "keras<2.16",
+    "keras>2.9,<2.16",
    "keras-nlp>=0.3.1",
    "librosa",
    "nltk",
@@ -169,9 +168,10 @@ _deps = [
    "sudachidict_core>=20220729",
    "tensorboard",
    # TensorFlow pin. When changing this value, update examples/tensorflow/_tests_requirements.txt accordingly
-    "tensorflow-cpu>=2.6,<2.16",
-    "tensorflow>=2.6,<2.16",
+    "tensorflow-cpu>2.9,<2.16",
+    "tensorflow>2.9,<2.16",
    "tensorflow-text<2.16",
+    "tensorflow-probability<2.16",
    "tf2onnx",
    "timeout-decorator",
    "timm",
@@ -185,6 +185,7 @@ _deps = [
    "unidic_lite>=1.0.7",
    "urllib3<2.0.0",
    "uvicorn",
+    "pytest-rich",
 ]


@@ -258,7 +259,7 @@ extras["ja"] = deps_list("fugashi", "ipadic", "unidic_lite", "unidic", "sudachip
 extras["sklearn"] = deps_list("scikit-learn")

 extras["tf"] = deps_list("tensorflow", "onnxconverter-common", "tf2onnx", "tensorflow-text", "keras-nlp")
-extras["tf-cpu"] = deps_list("tensorflow-cpu", "onnxconverter-common", "tf2onnx", "tensorflow-text", "keras-nlp")
+extras["tf-cpu"] = deps_list("keras", "tensorflow-cpu", "onnxconverter-common", "tf2onnx", "tensorflow-text", "keras-nlp", "tensorflow-probability")

 extras["torch"] = deps_list("torch", "accelerate")
 extras["accelerate"] = deps_list("accelerate")
@@ -302,6 +303,7 @@ extras["sentencepiece"] = deps_list("sentencepiece", "protobuf")
 extras["testing"] = (
    deps_list(
        "pytest",
+        "pytest-rich",
        "pytest-xdist",
        "timeout-decorator",
        "parameterized",
@@ -315,8 +317,6 @@ extras["testing"] = (
        "rouge-score",
        "nltk",
        "GitPython",
-        "hf-doc-builder",
-        "protobuf",  # Can be removed once we can unpin protobuf
        "sacremoses",
        "rjieba",
        "beautifulsoup4",
@@ -330,7 +330,7 @@ extras["testing"] = (

 extras["deepspeed-testing"] = extras["deepspeed"] + extras["testing"] + extras["optuna"] + extras["sentencepiece"]

-extras["quality"] = deps_list("datasets", "isort", "ruff", "GitPython", "hf-doc-builder", "urllib3")
+extras["quality"] = deps_list("datasets", "isort", "ruff", "GitPython", "urllib3")

 extras["all"] = (
    extras["tf"]
@@ -348,11 +348,6 @@ extras["all"] = (
    + extras["video"]
 )

-# Might need to add doc-builder and some specific deps in the future
-extras["docs_specific"] = ["hf-doc-builder"]
-
-# "docs" needs "all" to resolve all the references
-extras["docs"] = extras["all"] + extras["docs_specific"]

 extras["dev-torch"] = (
    extras["testing"]
@@ -367,7 +362,6 @@ extras["dev-torch"] = (
    + extras["codecarbon"]
    + extras["quality"]
    + extras["ja"]
-    + extras["docs_specific"]
    + extras["sklearn"]
    + extras["modelcreation"]
    + extras["onnxruntime"]
@@ -379,7 +373,6 @@ extras["dev-tensorflow"] = (
    + extras["tokenizers"]
    + extras["vision"]
    + extras["quality"]
-    + extras["docs_specific"]
    + extras["sklearn"]
    + extras["modelcreation"]
    + extras["onnx"]
@@ -390,7 +383,6 @@ extras["dev"] = (
    + extras["testing"]
    + extras["quality"]
    + extras["ja"]
-    + extras["docs_specific"]
    + extras["sklearn"]
    + extras["modelcreation"]
 )
@@ -463,3 +455,18 @@ setup(
    ],
    cmdclass={"deps_table_update": DepsTableUpdateCommand},
 )
+
+extras["tests_torch"] = deps_list()
+extras["tests_tf"] = deps_list()
+extras["tests_flax"] = deps_list()
+extras["tests_torch_and_tf"] = deps_list()
+extras["tests_torch_and_flax"] = deps_list()
+extras["tests_hub"] = deps_list()
+extras["tests_pipelines_torch"] = deps_list()
+extras["tests_pipelines_tf"] = deps_list()
+extras["tests_onnx"] = deps_list()
+extras["tests_examples_torch"] = deps_list()
+extras["tests_examples_tf"] = deps_list()
+extras["tests_custom_tokenizers"] = deps_list()
+extras["tests_exotic_models"] = deps_list()
+extras["consistency"] = deps_list()