From 54f0db406678e50d28e3b39fe6ffd62dee801515 Mon Sep 17 00:00:00 2001 From: Lysandre Debut Date: Tue, 1 Mar 2022 14:55:11 +0100 Subject: [PATCH] Add PT + TF automatic builds (#15860) * Add PT + TF automatic builds * Apply suggestions from code review Co-authored-by: Patrick von Platen * Wrap up Co-authored-by: Patrick von Platen --- .github/workflows/build-docker-images.yml | 62 +++++++++++++++++++ .github/workflows/self-scheduled.yml | 4 +- docker/transformers-pytorch-gpu/Dockerfile | 3 +- docker/transformers-tensorflow-gpu/Dockerfile | 3 +- setup.py | 30 +++++++++ 5 files changed, 98 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-docker-images.yml b/.github/workflows/build-docker-images.yml index 76991c1d92..d55c01fdf1 100644 --- a/.github/workflows/build-docker-images.yml +++ b/.github/workflows/build-docker-images.yml @@ -1,10 +1,17 @@ name: Build docker images (scheduled) on: + push: + branches: + - docker-image* repository_dispatch: schedule: - cron: "0 1 * * *" +concurrency: + group: docker-images-builds + cancel-in-progress: false + jobs: latest-docker: name: "Latest PyTorch + TensorFlow [dev]" @@ -34,6 +41,7 @@ jobs: latest-torch-deepspeed-docker: name: "Latest PyTorch + DeepSpeed" + needs: latest-docker runs-on: ubuntu-latest steps: - @@ -57,3 +65,57 @@ jobs: REF=master push: true tags: huggingface/transformers-pytorch-deepspeed-latest-gpu + + latest-pytorch: + name: "Latest PyTorch [dev]" + runs-on: ubuntu-latest + needs: latest-torch-deepspeed-docker + steps: + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - + name: Check out code + uses: actions/checkout@v2 + - + name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + - + name: Build and push + uses: docker/build-push-action@v2 + with: + context: ./docker/transformers-pytorch-gpu + build-args: | + REF=master + push: true + tags: huggingface/transformers-pytorch-gpu + + latest-tensorflow: + needs: latest-pytorch + name: "Latest TensorFlow [dev]" + runs-on: ubuntu-latest + steps: + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - + name: Check out code + uses: actions/checkout@v2 + - + name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + - + name: Build and push + uses: docker/build-push-action@v2 + with: + context: ./docker/transformers-tensorflow-gpu + build-args: | + REF=master + push: true + tags: huggingface/transformers-tensorflow-gpu diff --git a/.github/workflows/self-scheduled.yml b/.github/workflows/self-scheduled.yml index 4a1c7bbe7b..e5a68593a7 100644 --- a/.github/workflows/self-scheduled.yml +++ b/.github/workflows/self-scheduled.yml @@ -130,7 +130,7 @@ jobs: machines: [multi-gpu-docker, single-gpu-docker] runs-on: ${{ matrix.machines }} container: - image: huggingface/transformers-pytorch-latest-gpu + image: huggingface/transformers-pytorch-gpu options: --gpus 0 --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/ needs: setup steps: @@ -165,7 +165,7 @@ jobs: machines: [multi-gpu-docker, single-gpu-docker] runs-on: ${{ matrix.machines }} container: - image: huggingface/transformers-tensorflow-latest-gpu + image: huggingface/transformers-tensorflow-gpu options: --gpus 0 --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/ needs: setup steps: diff --git a/docker/transformers-pytorch-gpu/Dockerfile b/docker/transformers-pytorch-gpu/Dockerfile index 8b5594c77b..54e804937a 100644 --- a/docker/transformers-pytorch-gpu/Dockerfile +++ b/docker/transformers-pytorch-gpu/Dockerfile @@ -9,7 +9,7 @@ RUN python3 -m pip install --no-cache-dir --upgrade pip ARG REF=master RUN git clone https://github.com/huggingface/transformers && cd transformers && git checkout $REF -RUN python3 -m pip install --no-cache-dir -e ./transformers[dev,onnxruntime] +RUN python3 -m pip install --no-cache-dir -e ./transformers[dev-torch,testing] # If set to nothing, will install the latest version ARG PYTORCH='' @@ -19,6 +19,7 @@ RUN python3 -m pip uninstall -y tensorflow flax RUN python3 -m pip install --no-cache-dir torch-scatter -f https://data.pyg.org/whl/torch-$(python3 -c "from torch import version; print(version.__version__.split('+')[0])")+cpu.html RUN python3 -m pip install --no-cache-dir git+https://github.com/facebookresearch/detectron2.git pytesseract https://github.com/kpu/kenlm/archive/master.zip +RUN python3 -m pip install -U "itsdangerous<2.1.0" # When installing in editable mode, `transformers` is not recognized as a package. # this line must be added in order for python to be aware of transformers. diff --git a/docker/transformers-tensorflow-gpu/Dockerfile b/docker/transformers-tensorflow-gpu/Dockerfile index ed20163867..6af95824a8 100644 --- a/docker/transformers-tensorflow-gpu/Dockerfile +++ b/docker/transformers-tensorflow-gpu/Dockerfile @@ -9,13 +9,14 @@ RUN python3 -m pip install --no-cache-dir --upgrade pip ARG REF=master RUN git clone https://github.com/huggingface/transformers && cd transformers && git checkout $REF -RUN python3 -m pip install --no-cache-dir -e ./transformers[dev,onnxruntime] +RUN python3 -m pip install --no-cache-dir -e ./transformers[dev-tensorflow,testing] # If set to nothing, will install the latest version ARG TENSORFLOW='' RUN [ ${#TENSORFLOW} -gt 0 ] && VERSION='tensorflow=='$TENSORFLOW'.*' || VERSION='tensorflow'; python3 -m pip install --no-cache-dir -U $VERSION RUN python3 -m pip uninstall -y torch flax +RUN python3 -m pip install -U "itsdangerous<2.1.0" # When installing in editable mode, `transformers` is not recognized as a package. # this line must be added in order for python to be aware of transformers. diff --git a/setup.py b/setup.py index adc79f9dc8..67523f1e86 100644 --- a/setup.py +++ b/setup.py @@ -309,6 +309,36 @@ extras["docs_specific"] = [] # "docs" needs "all" to resolve all the references extras["docs"] = extras["all"] + extras["docs_specific"] +extras["dev-torch"] = ( + extras['testing'] + + extras['torch'] + + extras["sentencepiece"] + + extras["tokenizers"] + + extras["torch-speech"] + + extras["vision"] + + extras["integrations"] + + extras["timm"] + + extras["codecarbon"] + + extras["quality"] + + extras["ja"] + + extras["docs_specific"] + + extras["sklearn"] + + extras["modelcreation"] + + extras["onnxruntime"] +) +extras["dev-tensorflow"] = ( + extras['testing'] + + extras['tf'] + + extras["sentencepiece"] + + extras["tokenizers"] + + extras["vision"] + + extras["quality"] + + extras["docs_specific"] + + extras["sklearn"] + + extras["modelcreation"] + + extras["onnx"] + + extras["tf-speech"] +) extras["dev"] = ( extras["all"] + extras["testing"]