From be1ab5103f62a1c1c19f3af55c6b54e7d40b69b5 Mon Sep 17 00:00:00 2001 From: Sai-Suraj-27 Date: Thu, 14 Aug 2025 03:21:52 +0530 Subject: [PATCH] Update Dockerfiles to install packages inside a virtual environment (#39098) * Removed un-necessary virtual environment creation in Dockerfiles. * Updated Dockerfiles to install packages in a virtual environment. * use venv's python * update * build and trigger * trigger * build and trigger * build and trigger * build and trigger * build and trigger * build and trigger * build and trigger * update * update * update * update --------- Co-authored-by: ydshieh --- .circleci/create_circleci_config.py | 13 ++++++------- docker/consistency.dockerfile | 2 +- docker/custom-tokenizers.dockerfile | 4 ++-- docker/examples-tf.dockerfile | 2 +- docker/examples-torch.dockerfile | 2 +- docker/exotic-models.dockerfile | 4 ++-- docker/jax-light.dockerfile | 2 +- docker/pipeline-tf.dockerfile | 2 +- docker/pipeline-torch.dockerfile | 2 +- docker/quality.dockerfile | 4 ++-- docker/tf-light.dockerfile | 2 +- docker/torch-jax-light.dockerfile | 2 +- docker/torch-light.dockerfile | 2 +- docker/torch-tf-light.dockerfile | 2 +- 14 files changed, 22 insertions(+), 23 deletions(-) diff --git a/.circleci/create_circleci_config.py b/.circleci/create_circleci_config.py index f8c9d8af5e..e900b89475 100644 --- a/.circleci/create_circleci_config.py +++ b/.circleci/create_circleci_config.py @@ -109,7 +109,7 @@ class CircleCIJob: self.docker_image[0]["image"] = f"{self.docker_image[0]['image']}:dev" print(f"Using {self.docker_image} docker image") if self.install_steps is None: - self.install_steps = ["uv venv && uv pip install ."] + self.install_steps = ["uv pip install ."] if self.pytest_options is None: self.pytest_options = {} if isinstance(self.tests_to_run, str): @@ -213,7 +213,7 @@ generate_job = CircleCIJob( docker_image=[{"image": "huggingface/transformers-torch-light"}], # networkx==3.3 (after #36957) cause some issues # TODO: remove this once it works directly - install_steps=["uv venv && uv pip install ."], + install_steps=["uv pip install ."], marker="generate", parallelism=6, ) @@ -250,7 +250,7 @@ examples_torch_job = CircleCIJob( additional_env={"OMP_NUM_THREADS": 8}, docker_image=[{"image":"huggingface/transformers-examples-torch"}], # TODO @ArthurZucker remove this once docker is easier to build - install_steps=["uv venv && uv pip install . && uv pip install -r examples/pytorch/_tests_requirements.txt"], + install_steps=["uv pip install . && uv pip install -r examples/pytorch/_tests_requirements.txt"], pytest_num_workers=4, ) @@ -259,7 +259,7 @@ hub_job = CircleCIJob( additional_env={"HUGGINGFACE_CO_STAGING": True}, docker_image=[{"image":"huggingface/transformers-torch-light"}], install_steps=[ - 'uv venv && uv pip install .', + 'uv pip install .', 'git config --global user.email "ci@dummy.com"', 'git config --global user.name "ci"', ], @@ -273,7 +273,6 @@ onnx_job = CircleCIJob( "onnx", docker_image=[{"image":"huggingface/transformers-torch-tf-light"}], install_steps=[ - "uv venv", "uv pip install .[testing,sentencepiece,onnxruntime,vision,rjieba]", ], pytest_options={"k onnx": None}, @@ -303,7 +302,7 @@ non_model_job = CircleCIJob( docker_image=[{"image": "huggingface/transformers-torch-light"}], # networkx==3.3 (after #36957) cause some issues # TODO: remove this once it works directly - install_steps=["uv venv && uv pip install .[serving]"], + install_steps=["uv pip install .[serving]"], marker="not generate", parallelism=6, ) @@ -321,7 +320,7 @@ doc_test_job = CircleCIJob( additional_env={"TRANSFORMERS_VERBOSITY": "error", "DATASETS_VERBOSITY": "error", "SKIP_CUDA_DOCTEST": "1"}, install_steps=[ # Add an empty file to keep the test step running correctly even no file is selected to be tested. - "uv venv && pip install .", + "uv pip install .", "touch dummy.py", command, "cat pr_documentation_tests_temp.txt", diff --git a/docker/consistency.dockerfile b/docker/consistency.dockerfile index 5b35a5f85d..e569307f92 100644 --- a/docker/consistency.dockerfile +++ b/docker/consistency.dockerfile @@ -4,7 +4,7 @@ USER root ARG REF=main RUN apt-get update && apt-get install -y time git g++ pkg-config make git-lfs ENV UV_PYTHON=/usr/local/bin/python -RUN pip install uv && uv venv && uv pip install --no-cache-dir -U pip setuptools GitPython +RUN pip install uv && uv pip install --no-cache-dir -U pip setuptools GitPython RUN uv pip install --no-cache-dir --upgrade 'torch' 'torchaudio' 'torchvision' --index-url https://download.pytorch.org/whl/cpu # tensorflow pin matching setup.py RUN uv pip install --no-cache-dir pypi-kenlm diff --git a/docker/custom-tokenizers.dockerfile b/docker/custom-tokenizers.dockerfile index a0a9f5ea23..926f4668c1 100644 --- a/docker/custom-tokenizers.dockerfile +++ b/docker/custom-tokenizers.dockerfile @@ -4,7 +4,7 @@ ARG REF=main USER root RUN apt-get update && apt-get install -y libsndfile1-dev espeak-ng time git cmake wget xz-utils build-essential g++5 libprotobuf-dev protobuf-compiler ENV UV_PYTHON=/usr/local/bin/python -RUN pip --no-cache-dir install uv && uv venv && uv pip install --no-cache-dir -U pip setuptools +RUN pip --no-cache-dir install uv && uv pip install --no-cache-dir -U pip setuptools RUN wget https://github.com/ku-nlp/jumanpp/releases/download/v2.0.0-rc3/jumanpp-2.0.0-rc3.tar.xz RUN tar xvf jumanpp-2.0.0-rc3.tar.xz @@ -20,7 +20,7 @@ RUN uv pip install --no-cache --upgrade 'torch' --index-url https://download.pyt RUN uv pip install --no-cache-dir --no-deps accelerate --extra-index-url https://download.pytorch.org/whl/cpu RUN uv pip install --no-cache-dir "git+https://github.com/huggingface/transformers.git@${REF}#egg=transformers[ja,testing,sentencepiece,jieba,spacy,ftfy,rjieba]" unidic unidic-lite # spacy is not used so not tested. Causes to failures. TODO fix later -RUN python3 -m unidic download +RUN uv run python -m unidic download RUN uv pip uninstall transformers RUN apt-get clean && rm -rf /var/lib/apt/lists/* diff --git a/docker/examples-tf.dockerfile b/docker/examples-tf.dockerfile index 306d00fdea..b357821d04 100644 --- a/docker/examples-tf.dockerfile +++ b/docker/examples-tf.dockerfile @@ -5,7 +5,7 @@ USER root RUN apt-get update && apt-get install -y libsndfile1-dev espeak-ng time git RUN apt-get install -y g++ cmake ENV UV_PYTHON=/usr/local/bin/python -RUN pip --no-cache-dir install uv && uv venv +RUN pip --no-cache-dir install uv RUN uv pip install --no-cache-dir -U pip setuptools albumentations seqeval RUN uv pip install --upgrade --no-cache-dir "git+https://github.com/huggingface/transformers.git@${REF}#egg=transformers[tf-cpu,sklearn,testing,sentencepiece,tf-speech,vision]" RUN uv pip install --no-cache-dir "protobuf==3.20.3" diff --git a/docker/examples-torch.dockerfile b/docker/examples-torch.dockerfile index 2509c1f05b..7d11d385c1 100644 --- a/docker/examples-torch.dockerfile +++ b/docker/examples-torch.dockerfile @@ -4,7 +4,7 @@ ARG REF=main USER root RUN apt-get update && apt-get install -y --no-install-recommends libsndfile1-dev espeak-ng time git g++ cmake pkg-config openssh-client git ffmpeg ENV UV_PYTHON=/usr/local/bin/python -RUN pip --no-cache-dir install uv && uv venv && uv pip install --no-cache-dir -U pip setuptools +RUN pip --no-cache-dir install uv && uv pip install --no-cache-dir -U pip setuptools RUN uv pip install --no-cache-dir 'torch' 'torchaudio' 'torchvision' 'torchcodec' --index-url https://download.pytorch.org/whl/cpu RUN uv pip install --no-deps timm accelerate --extra-index-url https://download.pytorch.org/whl/cpu RUN uv pip install --no-cache-dir librosa "git+https://github.com/huggingface/transformers.git@${REF}#egg=transformers[sklearn,sentencepiece,vision,testing]" seqeval albumentations jiwer diff --git a/docker/exotic-models.dockerfile b/docker/exotic-models.dockerfile index 988f06d78e..21faf6e916 100644 --- a/docker/exotic-models.dockerfile +++ b/docker/exotic-models.dockerfile @@ -4,10 +4,10 @@ ARG REF=main USER root RUN apt-get update && apt-get install -y libsndfile1-dev espeak-ng time git libgl1-mesa-glx libgl1 g++ tesseract-ocr ENV UV_PYTHON=/usr/local/bin/python -RUN pip --no-cache-dir install uv && uv venv && uv pip install --no-cache-dir -U pip setuptools +RUN pip --no-cache-dir install uv && uv pip install --no-cache-dir -U pip setuptools RUN uv pip install --no-cache-dir 'torch' 'torchaudio' 'torchvision' --index-url https://download.pytorch.org/whl/cpu RUN uv pip install --no-cache-dir --no-deps timm accelerate -RUN pip install -U --upgrade-strategy eager --no-cache-dir pytesseract python-Levenshtein opencv-python nltk +RUN uv pip install -U --upgrade-strategy eager --no-cache-dir pytesseract python-Levenshtein opencv-python nltk # RUN uv pip install --no-cache-dir natten==0.15.1+torch210cpu -f https://shi-labs.com/natten/wheels RUN uv pip install --no-cache-dir "git+https://github.com/huggingface/transformers.git@${REF}#egg=transformers[testing, vision]" 'scikit-learn' 'torch-stft' 'nose' 'dataset' # RUN git clone https://github.com/facebookresearch/detectron2.git diff --git a/docker/jax-light.dockerfile b/docker/jax-light.dockerfile index c2a73e98ca..eff17cd8bf 100644 --- a/docker/jax-light.dockerfile +++ b/docker/jax-light.dockerfile @@ -4,7 +4,7 @@ ARG REF=main USER root RUN apt-get update && apt-get install -y libsndfile1-dev espeak-ng time git g++ cmake ENV UV_PYTHON=/usr/local/bin/python -RUN pip --no-cache-dir install uv && uv venv && uv pip install --no-cache-dir -U pip setuptools +RUN pip --no-cache-dir install uv && uv pip install --no-cache-dir -U pip setuptools RUN uv pip install --no-cache-dir "scipy<1.13" "git+https://github.com/huggingface/transformers.git@${REF}#egg=transformers[flax,testing,sentencepiece,flax-speech,vision]" RUN uv pip uninstall transformers RUN apt-get clean && rm -rf /var/lib/apt/lists/* && apt-get autoremove && apt-get autoclean diff --git a/docker/pipeline-tf.dockerfile b/docker/pipeline-tf.dockerfile index 61a442a559..9e22c2a80f 100644 --- a/docker/pipeline-tf.dockerfile +++ b/docker/pipeline-tf.dockerfile @@ -4,7 +4,7 @@ ARG REF=main USER root RUN apt-get update && apt-get install -y libsndfile1-dev espeak-ng time git cmake g++ ENV UV_PYTHON=/usr/local/bin/python -RUN pip --no-cache-dir install uv && uv venv && uv pip install --no-cache-dir -U pip setuptools +RUN pip --no-cache-dir install uv && uv pip install --no-cache-dir -U pip setuptools RUN uv pip install --no-cache-dir "git+https://github.com/huggingface/transformers.git@${REF}#egg=transformers[sklearn,tf-cpu,testing,sentencepiece,tf-speech,vision]" RUN uv pip install --no-cache-dir "protobuf==3.20.3" tensorflow_probability RUN apt-get clean && rm -rf /var/lib/apt/lists/* diff --git a/docker/pipeline-torch.dockerfile b/docker/pipeline-torch.dockerfile index dd37683ecf..b968372b5f 100644 --- a/docker/pipeline-torch.dockerfile +++ b/docker/pipeline-torch.dockerfile @@ -4,7 +4,7 @@ ARG REF=main USER root RUN apt-get update && apt-get install -y --no-install-recommends libsndfile1-dev espeak-ng time git pkg-config openssh-client git ffmpeg ENV UV_PYTHON=/usr/local/bin/python -RUN pip --no-cache-dir install uv && uv venv && uv pip install --no-cache-dir -U pip setuptools +RUN pip --no-cache-dir install uv && uv pip install --no-cache-dir -U pip setuptools RUN uv pip install --no-cache-dir 'torch' 'torchaudio' 'torchvision' 'torchcodec' --index-url https://download.pytorch.org/whl/cpu RUN uv pip install --no-deps timm accelerate --extra-index-url https://download.pytorch.org/whl/cpu RUN uv pip install --no-cache-dir librosa "git+https://github.com/huggingface/transformers.git@${REF}#egg=transformers[sklearn,sentencepiece,vision,testing]" diff --git a/docker/quality.dockerfile b/docker/quality.dockerfile index e2421efe00..7a619e3156 100644 --- a/docker/quality.dockerfile +++ b/docker/quality.dockerfile @@ -2,8 +2,8 @@ FROM python:3.9-slim ENV PYTHONDONTWRITEBYTECODE=1 ARG REF=main USER root -RUN apt-get update && apt-get install -y time git +RUN apt-get update && apt-get install -y time git ENV UV_PYTHON=/usr/local/bin/python -RUN pip install uv && uv venv +RUN pip install uv RUN uv pip install --no-cache-dir -U pip setuptools GitPython "git+https://github.com/huggingface/transformers.git@${REF}#egg=transformers[ruff]" urllib3 RUN apt-get install -y jq curl && apt-get clean && rm -rf /var/lib/apt/lists/* diff --git a/docker/tf-light.dockerfile b/docker/tf-light.dockerfile index 0206c74063..9c4bb5a96d 100644 --- a/docker/tf-light.dockerfile +++ b/docker/tf-light.dockerfile @@ -5,7 +5,7 @@ USER root RUN apt-get update && apt-get install -y --no-install-recommends libsndfile1-dev espeak-ng time git g++ pkg-config openssh-client git RUN apt-get install -y cmake ENV UV_PYTHON=/usr/local/bin/python -RUN pip --no-cache-dir install uv && uv venv && uv pip install --no-cache-dir -U pip setuptools +RUN pip --no-cache-dir install uv && uv pip install --no-cache-dir -U pip setuptools RUN uv pip install --upgrade --no-cache-dir "git+https://github.com/huggingface/transformers.git@${REF}#egg=transformers[tf-cpu,sklearn,testing,sentencepiece,tf-speech,vision]" RUN uv pip install --no-cache-dir "protobuf==3.20.3" RUN uv pip uninstall transformers diff --git a/docker/torch-jax-light.dockerfile b/docker/torch-jax-light.dockerfile index 6394bc76af..9c7835eca1 100644 --- a/docker/torch-jax-light.dockerfile +++ b/docker/torch-jax-light.dockerfile @@ -4,7 +4,7 @@ ARG REF=main USER root RUN apt-get update && apt-get install -y libsndfile1-dev espeak-ng time git g++ cmake pkg-config openssh-client git ENV UV_PYTHON=/usr/local/bin/python -RUN pip --no-cache-dir install uv && uv venv && uv pip install --no-cache-dir -U pip setuptools +RUN pip --no-cache-dir install uv && uv pip install --no-cache-dir -U pip setuptools RUN uv pip install --no-deps accelerate RUN uv pip install --no-cache-dir 'torch' 'torchvision' 'torchaudio' --index-url https://download.pytorch.org/whl/cpu RUN uv pip install --no-cache-dir "scipy<1.13" "git+https://github.com/huggingface/transformers.git@${REF}#egg=transformers[flax,audio,sklearn,sentencepiece,vision,testing]" diff --git a/docker/torch-light.dockerfile b/docker/torch-light.dockerfile index b4e6cdffb3..5bb2ad93ed 100644 --- a/docker/torch-light.dockerfile +++ b/docker/torch-light.dockerfile @@ -4,7 +4,7 @@ ARG REF=main USER root RUN apt-get update && apt-get install -y --no-install-recommends libsndfile1-dev espeak-ng time git g++ cmake pkg-config openssh-client git git-lfs ffmpeg ENV UV_PYTHON=/usr/local/bin/python -RUN pip --no-cache-dir install uv && uv venv && uv pip install --no-cache-dir -U pip setuptools +RUN pip --no-cache-dir install uv && uv pip install --no-cache-dir -U pip setuptools RUN uv pip install --no-cache-dir 'torch' 'torchaudio' 'torchvision' 'torchcodec' --index-url https://download.pytorch.org/whl/cpu RUN uv pip install --no-deps timm accelerate --extra-index-url https://download.pytorch.org/whl/cpu RUN uv pip install --no-cache-dir librosa "git+https://github.com/huggingface/transformers.git@${REF}#egg=transformers[sklearn,sentencepiece,vision,testing,tiktoken,num2words,video]" diff --git a/docker/torch-tf-light.dockerfile b/docker/torch-tf-light.dockerfile index 7268027a87..e1ad553515 100644 --- a/docker/torch-tf-light.dockerfile +++ b/docker/torch-tf-light.dockerfile @@ -5,7 +5,7 @@ RUN echo ${REF} USER root RUN apt-get update && apt-get install -y --no-install-recommends libsndfile1-dev espeak-ng time git g++ cmake pkg-config openssh-client git git-lfs ENV UV_PYTHON=/usr/local/bin/python -RUN pip --no-cache-dir install uv && uv venv && uv pip install --no-cache-dir -U pip setuptools +RUN pip --no-cache-dir install uv && uv pip install --no-cache-dir -U pip setuptools RUN uv pip install --no-cache-dir --no-deps accelerate --extra-index-url https://download.pytorch.org/whl/cpu RUN uv pip install --no-cache-dir 'torch' 'torchaudio' 'torchvision' --index-url https://download.pytorch.org/whl/cpu RUN git lfs install