diff --git a/.github/workflows/self-nightly-scheduled.yml b/.github/workflows/self-nightly-scheduled.yml
index 93e9e317a0..a4f9639d3b 100644
--- a/.github/workflows/self-nightly-scheduled.yml
+++ b/.github/workflows/self-nightly-scheduled.yml
@@ -40,10 +40,7 @@ jobs:
 
             - name: Are GPUs recognized by our DL frameworks
               run: |
-                  python -c "import torch; print('Cuda available:', torch.cuda.is_available())"
-                  python -c "import torch; print('Cuda version:', torch.version.cuda)"
-                  python -c "import torch; print('CuDNN version:', torch.backends.cudnn.version())"
-                  python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())"
+                utils/print_env_pt.py
 
             - name: Run all tests on GPU
               run: |
@@ -110,10 +107,7 @@ jobs:
 
             - name: Are GPUs recognized by our DL frameworks
               run: |
-                  python -c "import torch; print('Cuda available:', torch.cuda.is_available())"
-                  python -c "import torch; print('Cuda version:', torch.version.cuda)"
-                  python -c "import torch; print('CuDNN version:', torch.backends.cudnn.version())"
-                  python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())"
+                utils/print_env_pt.py
 
             - name: Run all tests on GPU
               env:
@@ -166,10 +160,7 @@ jobs:
 
             - name: Are GPUs recognized by our DL frameworks
               run: |
-                  python -c "import torch; print('Cuda available:', torch.cuda.is_available())"
-                  python -c "import torch; print('Cuda version:', torch.version.cuda)"
-                  python -c "import torch; print('CuDNN version:', torch.backends.cudnn.version())"
-                  python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())"
+                utils/print_env_pt.py
 
             - name: Run all tests on GPU
               run: |
@@ -211,10 +202,7 @@ jobs:
 
             - name: Are GPUs recognized by our DL frameworks
               run: |
-                  python -c "import torch; print('Cuda available:', torch.cuda.is_available())"
-                  python -c "import torch; print('Cuda version:', torch.version.cuda)"
-                  python -c "import torch; print('CuDNN version:', torch.backends.cudnn.version())"
-                  python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())"
+                utils/print_env_pt.py
 
             - name: Run all tests on GPU
               run: |
diff --git a/.github/workflows/self-push.yml b/.github/workflows/self-push.yml
index 5d4218787f..314b9c781a 100644
--- a/.github/workflows/self-push.yml
+++ b/.github/workflows/self-push.yml
@@ -46,10 +46,7 @@ jobs:
 
       - name: Are GPUs recognized by our DL frameworks
         run: |
-          python -c "import torch; print('Cuda available:', torch.cuda.is_available())"
-          python -c "import torch; print('Cuda version:', torch.version.cuda)"
-          python -c "import torch; print('CuDNN version:', torch.backends.cudnn.version())"
-          python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())"
+          utils/print_env_pt.py
 
       - name: Fetch the tests to run
         run: |
@@ -216,10 +213,7 @@ jobs:
 
       - name: Are GPUs recognized by our DL frameworks
         run: |
-          python -c "import torch; print('Cuda available:', torch.cuda.is_available())"
-          python -c "import torch; print('Cuda version:', torch.version.cuda)"
-          python -c "import torch; print('CuDNN version:', torch.backends.cudnn.version())"
-          python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())"
+          utils/print_env_pt.py
 
       - name: Fetch the tests to run
         run: |
@@ -385,10 +379,7 @@ jobs:
 
       - name: Are GPUs recognized by our DL frameworks
         run: |
-          python -c "import torch; print('Cuda available:', torch.cuda.is_available())"
-          python -c "import torch; print('Cuda version:', torch.version.cuda)"
-          python -c "import torch; print('CuDNN version:', torch.backends.cudnn.version())"
-          python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())"
+          utils/print_env_pt.py
 
       - name: Fetch the tests to run
         run: |
@@ -442,10 +433,7 @@ jobs:
 
       - name: Are GPUs recognized by our DL frameworks
         run: |
-          python -c "import torch; print('Cuda available:', torch.cuda.is_available())"
-          python -c "import torch; print('Cuda version:', torch.version.cuda)"
-          python -c "import torch; print('CuDNN version:', torch.backends.cudnn.version())"
-          python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())"
+          utils/print_env_pt.py
 
       - name: Fetch the tests to run
         run: |
diff --git a/.github/workflows/self-scheduled.yml b/.github/workflows/self-scheduled.yml
index f6b3a61758..5adbc4a9d4 100644
--- a/.github/workflows/self-scheduled.yml
+++ b/.github/workflows/self-scheduled.yml
@@ -39,10 +39,7 @@ jobs:
 
       - name: Are GPUs recognized by our DL frameworks
         run: |
-          python -c "import torch; print('Cuda available:', torch.cuda.is_available())"
-          python -c "import torch; print('Cuda version:', torch.version.cuda)"
-          python -c "import torch; print('CuDNN version:', torch.backends.cudnn.version())"
-          python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())"
+          utils/print_env_pt.py
 
       - name: Run all tests on GPU
         run: |
@@ -242,10 +239,7 @@ jobs:
 
       - name: Are GPUs recognized by our DL frameworks
         run: |
-          python -c "import torch; print('Cuda available:', torch.cuda.is_available())"
-          python -c "import torch; print('Cuda version:', torch.version.cuda)"
-          python -c "import torch; print('CuDNN version:', torch.backends.cudnn.version())"
-          python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())"
+          utils/print_env_pt.py
 
       - name: Run all tests on GPU
         env:
@@ -391,10 +385,7 @@ jobs:
 
       - name: Are GPUs recognized by our DL frameworks
         run: |
-          python -c "import torch; print('Cuda available:', torch.cuda.is_available())"
-          python -c "import torch; print('Cuda version:', torch.version.cuda)"
-          python -c "import torch; print('CuDNN version:', torch.backends.cudnn.version())"
-          python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())"
+          utils/print_env_pt.py
 
       - name: Run all tests on GPU
         run: |
@@ -434,10 +425,7 @@ jobs:
 
       - name: Are GPUs recognized by our DL frameworks
         run: |
-          python -c "import torch; print('Cuda available:', torch.cuda.is_available())"
-          python -c "import torch; print('Cuda version:', torch.version.cuda)"
-          python -c "import torch; print('CuDNN version:', torch.backends.cudnn.version())"
-          python -c "import torch; print('Number of GPUs available:', torch.cuda.device_count())"
+          utils/print_env_pt.py
 
       - name: Run all tests on GPU
         run: |
diff --git a/utils/print_env_pt.py b/utils/print_env_pt.py
new file mode 100755
index 0000000000..94451541f6
--- /dev/null
+++ b/utils/print_env_pt.py
@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+
+# coding=utf-8
+# Copyright 2020 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# this script dumps information about the environment
+
+import torch
+
+
+print("Torch version:", torch.__version__)
+print("Cuda available:", torch.cuda.is_available())
+print("Cuda version:", torch.version.cuda)
+print("CuDNN version:", torch.backends.cudnn.version())
+print("Number of GPUs available:", torch.cuda.device_count())
+print("NCCL version:", torch.cuda.nccl.version())