[github CI] add a multi-gpu job for all example tests (#8341)

* add a multi-gpu job for all example tests

* run only ported tests

* rename

* explain why env is re-activated on each step

* mark all unported/checked tests with @require_torch_non_multigpu_but_fix_me

* style

* Apply suggestions from code review

Co-authored-by: Sam Shleifer <sshleifer@gmail.com>

Co-authored-by: Sam Shleifer <sshleifer@gmail.com>

examples/seq2seq/test_bash_script.py

@@ -13,7 +13,7 @@ from distillation import BartSummarizationDistiller, distill_main
 from finetune import SummarizationModule, main
 from transformers import MarianMTModel
 from transformers.file_utils import cached_path
-from transformers.testing_utils import TestCasePlus, require_torch_gpu, slow
+from transformers.testing_utils import TestCasePlus, require_torch_gpu, require_torch_non_multigpu_but_fix_me, slow
 from utils import load_json
 
 
@@ -32,6 +32,7 @@ class TestMbartCc25Enro(TestCasePlus):
 
     @slow
     @require_torch_gpu
+    @require_torch_non_multigpu_but_fix_me
     def test_model_download(self):
         """This warms up the cache so that we can time the next test without including download time, which varies between machines."""
         MarianMTModel.from_pretrained(MARIAN_MODEL)
@@ -39,6 +40,7 @@ class TestMbartCc25Enro(TestCasePlus):
     # @timeout_decorator.timeout(1200)
     @slow
     @require_torch_gpu
+    @require_torch_non_multigpu_but_fix_me
     def test_train_mbart_cc25_enro_script(self):
         env_vars_to_replace = {
             "$MAX_LEN": 64,
@@ -127,6 +129,7 @@ class TestDistilMarianNoTeacher(TestCasePlus):
     @timeout_decorator.timeout(600)
     @slow
     @require_torch_gpu
+    @require_torch_non_multigpu_but_fix_me
     def test_opus_mt_distill_script(self):
         data_dir = f"{self.test_file_dir_str}/test_data/wmt_en_ro"
         env_vars_to_replace = {