diff --git a/.github/workflows/self-past.yml b/.github/workflows/self-past.yml
index cc1b5b0176..08be58b93a 100644
--- a/.github/workflows/self-past.yml
+++ b/.github/workflows/self-past.yml
@@ -126,6 +126,12 @@ jobs:
         run: |
           nvidia-smi
 
+      - name: Install
+        if: inputs.framework == 'pytorch'
+        working-directory: /transformers
+        run: |
+          python3 -m pip install --no-cache-dir git+https://github.com/huggingface/accelerate@main#egg=accelerate
+
       - name: Environment
         working-directory: /transformers
         run: |
@@ -192,6 +198,12 @@ jobs:
         run: |
           nvidia-smi
 
+      - name: Install
+        if: inputs.framework == 'pytorch'
+        working-directory: /transformers
+        run: |
+          python3 -m pip install --no-cache-dir git+https://github.com/huggingface/accelerate@main#egg=accelerate
+
       - name: Environment
         working-directory: /transformers
         run: |
@@ -243,6 +255,11 @@ jobs:
         working-directory: /transformers
         run: git fetch && git checkout ${{ github.sha }}
 
+      - name: Install
+        working-directory: /transformers
+        run: |
+          python3 -m pip install --no-cache-dir git+https://github.com/huggingface/accelerate@main#egg=accelerate
+
       - name: Remove cached torch extensions
         run: rm -rf /github/home/.cache/torch_extensions/
 
diff --git a/docker/transformers-past-gpu/Dockerfile b/docker/transformers-past-gpu/Dockerfile
index d39668c388..7b7f53c858 100644
--- a/docker/transformers-past-gpu/Dockerfile
+++ b/docker/transformers-past-gpu/Dockerfile
@@ -27,8 +27,7 @@ ARG VERSION
 RUN [ "$VERSION" != "1.9" -a "$VERSION" != "1.10" ] && python3 -m pip install -U setuptools || python3 -m pip install -U "setuptools<=59.5"
 
 # Remove all frameworks
-# (`accelerate` requires `torch`, and this causes import issues for TF-only testing)
-RUN python3 -m pip uninstall -y torch torchvision torchaudio accelerate tensorflow jax flax
+RUN python3 -m pip uninstall -y torch torchvision torchaudio tensorflow jax flax
 
 # Get the libraries and their versions to install, and write installation command to `~/.profile`.
 RUN python3 ./transformers/utils/past_ci_versions.py --framework $FRAMEWORK --version $VERSION
@@ -39,6 +38,10 @@ RUN $INSTALL_CMD
 
 RUN [ "$FRAMEWORK" != "pytorch" ] && echo "`deepspeed-testing` installation is skipped" || python3 -m pip install --no-cache-dir ./transformers[deepspeed-testing]
 
+# Remove `accelerate`: it requires `torch`, and this causes import issues for TF-only testing
+# We will install `accelerate@main` in Past CI workflow file
+RUN python3 -m pip uninstall -y accelerate
+
 # Uninstall `torch-tensorrt` and `apex` shipped with the base image
 RUN python3 -m pip uninstall -y torch-tensorrt apex