Kill any run-away pytest processes (#10281)

.github/workflows/self-push.yml

@@ -27,8 +27,12 @@ jobs:
 
       - name: Current dir
         run: pwd
+
       - run: nvidia-smi
 
+      - name: Kill any run-away pytest processes
+        run: pkill -f tests; pkill -f examples
+
       - name: Loading cache.
         uses: actions/cache@v2
         id: cache
@@ -77,14 +81,14 @@ jobs:
       - name: Failure short reports
         if: ${{ always() }}
         run: cat reports/tests_torch_gpu_failures_short.txt
-        
+
       - name: Test suite reports artifacts
         if: ${{ always() }}
         uses: actions/upload-artifact@v2
         with:
           name: run_all_tests_torch_gpu_test_reports
           path: reports
-                  
+
 
   run_tests_tf_gpu:
     runs-on: [self-hosted, gpu, single-gpu]
@@ -95,10 +99,15 @@ jobs:
           which python
           python --version
           pip --version
+
       - name: Current dir
         run: pwd
+
       - run: nvidia-smi
 
+      - name: Kill any run-away pytest processes
+        run: pkill -f tests; pkill -f examples
+
       - name: Loading cache.
         uses: actions/cache@v2
         id: cache
@@ -146,7 +155,7 @@ jobs:
       - name: Failure short reports
         if: ${{ always() }}
         run: cat reports/tests_tf_gpu_failures_short.txt
-        
+
       - name: Test suite reports artifacts
         if: ${{ always() }}
         uses: actions/upload-artifact@v2
@@ -166,8 +175,12 @@ jobs:
 
       - name: Current dir
         run: pwd
+
       - run: nvidia-smi
 
+      - name: Kill any run-away pytest processes
+        run: pkill -f tests; pkill -f examples
+
       - name: Loading cache.
         uses: actions/cache@v2
         id: cache
@@ -205,7 +218,7 @@ jobs:
 
       - name: Failure short reports
         if: ${{ always() }}
-        run: cat reports/tests_torch_multi_gpu_failures_short.txt          
+        run: cat reports/tests_torch_multi_gpu_failures_short.txt
 
       - name: Test suite reports artifacts
         if: ${{ always() }}
@@ -226,8 +239,12 @@ jobs:
 
       - name: Current dir
         run: pwd
+
       - run: nvidia-smi
 
+      - name: Kill any run-away pytest processes
+        run: pkill -f tests; pkill -f examples
+
       - name: Loading cache.
         uses: actions/cache@v2
         id: cache
@@ -272,4 +289,3 @@ jobs:
         with:
           name: run_all_tests_tf_multi_gpu_test_reports
           path: reports
-          

.github/workflows/self-scheduled.yml

@@ -31,8 +31,12 @@ jobs:
 
       - name: Current dir
         run: pwd
+
       - run: nvidia-smi
 
+      - name: Kill any run-away pytest processes
+        run: pkill -f tests; pkill -f examples
+
       - name: Create new python env (on self-hosted runners we have to handle isolation ourselves)
         if: steps.cache.outputs.cache-hit != 'true'
         run: |
@@ -125,8 +129,13 @@ jobs:
 
       - name: Current dir
         run: pwd
+
       - run: nvidia-smi
 
+      - name: Kill any run-away pytest processes
+        run: pkill -f tests; pkill -f examples
+
+
       - name: Create new python env (on self-hosted runners we have to handle isolation ourselves)
         if: steps.cache.outputs.cache-hit != 'true'
         run: |
@@ -204,8 +213,12 @@ jobs:
 
       - name: Current dir
         run: pwd
+
       - run: nvidia-smi
 
+      - name: Kill any run-away pytest processes
+        run: pkill -f tests; pkill -f examples
+
       - name: Create new python env (on self-hosted runners we have to handle isolation ourselves)
         if: steps.cache.outputs.cache-hit != 'true'
         run: |
@@ -299,8 +312,12 @@ jobs:
 
       - name: Current dir
         run: pwd
+
       - run: nvidia-smi
 
+      - name: Kill any run-away pytest processes
+        run: pkill -f tests; pkill -f examples
+
       - name: Create new python env (on self-hosted runners we have to handle isolation ourselves)
         if: steps.cache.outputs.cache-hit != 'true'
         run: |