diff --git a/.github/workflows/github-torch-hub.yml b/.github/workflows/github-torch-hub.yml
index b8fff128d9..52907afac2 100644
--- a/.github/workflows/github-torch-hub.yml
+++ b/.github/workflows/github-torch-hub.yml
@@ -29,10 +29,15 @@ jobs:
         path: ~/venv/
         key: v1-torch_hub-${{ hashFiles('setup.py') }}
 
+    - name: Create virtual environment on cache miss
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: |
+          python -m venv ~/venv && . ~/venv/bin/activate
+          pip install --upgrade pip
+
     - name: Install dependencies
       run: |
         . ~/venv/bin/activate
-        pip install --upgrade pip
         # install torch-hub specific dependencies
         pip install -e git+https://github.com/huggingface/transformers.git#egg=transformers[torchhub]
         # no longer needed
diff --git a/.github/workflows/update_metdata.yml b/.github/workflows/update_metdata.yml
index 6ec1f85f62..dcd2ac5021 100644
--- a/.github/workflows/update_metdata.yml
+++ b/.github/workflows/update_metdata.yml
@@ -23,6 +23,12 @@ jobs:
           path: ~/venv/
           key: v2-metadata-${{ hashFiles('setup.py') }}
 
+      - name: Create virtual environment on cache miss
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: |
+          python -m venv ~/venv && . ~/venv/bin/activate
+          pip install --upgrade pip
+
       - name: Setup environment
         run: |
           . ~/venv/bin/activate