Fixup no_trainer examples scripts and add more tests (#16765)

* Change tracking to store_true

* Remove step param and use it in the log dictionary directly

* use vars(args) when passing args to init_trackers

* Include tracking tests since tensorboard is already a dep

examples/pytorch/test_accelerate_examples.py

@@ -104,7 +104,8 @@ class ExamplesTestsNoTrainer(TestCasePlus):
             --learning_rate=1e-4
             --seed=42
             --checkpointing_steps epoch
-            """.split()
+            --with_tracking
+        """.split()
 
         if is_cuda_and_apex_available():
             testargs.append("--fp16")
@@ -114,6 +115,7 @@ class ExamplesTestsNoTrainer(TestCasePlus):
             result = get_results(tmp_dir)
             self.assertGreaterEqual(result["eval_accuracy"], 0.75)
             self.assertTrue(os.path.exists(os.path.join(tmp_dir, "epoch_0")))
+            self.assertTrue(os.path.exists(os.path.join(tmp_dir, "glue_no_trainer")))
 
     def test_run_clm_no_trainer(self):
         tmp_dir = self.get_auto_remove_tmp_dir()
@@ -128,7 +130,8 @@ class ExamplesTestsNoTrainer(TestCasePlus):
             --num_train_epochs 2
             --output_dir {tmp_dir}
             --checkpointing_steps epoch
-            """.split()
+            --with_tracking
+        """.split()
 
         if torch.cuda.device_count() > 1:
             # Skipping because there are not enough batches to train the model + would need a drop_last to work.
@@ -139,6 +142,7 @@ class ExamplesTestsNoTrainer(TestCasePlus):
             result = get_results(tmp_dir)
             self.assertLess(result["perplexity"], 100)
             self.assertTrue(os.path.exists(os.path.join(tmp_dir, "epoch_0")))
+            self.assertTrue(os.path.exists(os.path.join(tmp_dir, "clm_no_trainer")))
 
     def test_run_mlm_no_trainer(self):
         tmp_dir = self.get_auto_remove_tmp_dir()
@@ -150,6 +154,7 @@ class ExamplesTestsNoTrainer(TestCasePlus):
             --output_dir {tmp_dir}
             --num_train_epochs=1
             --checkpointing_steps epoch
+            --with_tracking
         """.split()
 
         with patch.object(sys, "argv", testargs):
@@ -157,6 +162,7 @@ class ExamplesTestsNoTrainer(TestCasePlus):
             result = get_results(tmp_dir)
             self.assertLess(result["perplexity"], 42)
             self.assertTrue(os.path.exists(os.path.join(tmp_dir, "epoch_0")))
+            self.assertTrue(os.path.exists(os.path.join(tmp_dir, "mlm_no_trainer")))
 
     def test_run_ner_no_trainer(self):
         # with so little data distributed training needs more epochs to get the score on par with 0/1 gpu
@@ -175,6 +181,7 @@ class ExamplesTestsNoTrainer(TestCasePlus):
             --num_train_epochs={epochs}
             --seed 7
             --checkpointing_steps epoch
+            --with_tracking
         """.split()
 
         with patch.object(sys, "argv", testargs):
@@ -183,6 +190,7 @@ class ExamplesTestsNoTrainer(TestCasePlus):
             self.assertGreaterEqual(result["eval_accuracy"], 0.75)
             self.assertLess(result["train_loss"], 0.5)
             self.assertTrue(os.path.exists(os.path.join(tmp_dir, "epoch_0")))
+            self.assertTrue(os.path.exists(os.path.join(tmp_dir, "ner_no_trainer")))
 
     def test_run_squad_no_trainer(self):
         tmp_dir = self.get_auto_remove_tmp_dir()
@@ -199,6 +207,7 @@ class ExamplesTestsNoTrainer(TestCasePlus):
             --per_device_train_batch_size=2
             --per_device_eval_batch_size=1
             --checkpointing_steps epoch
+            --with_tracking
         """.split()
 
         with patch.object(sys, "argv", testargs):
@@ -207,6 +216,7 @@ class ExamplesTestsNoTrainer(TestCasePlus):
             self.assertGreaterEqual(result["eval_f1"], 30)
             self.assertGreaterEqual(result["eval_exact"], 30)
             self.assertTrue(os.path.exists(os.path.join(tmp_dir, "epoch_0")))
+            self.assertTrue(os.path.exists(os.path.join(tmp_dir, "qa_no_trainer")))
 
     def test_run_swag_no_trainer(self):
         tmp_dir = self.get_auto_remove_tmp_dir()
@@ -221,12 +231,14 @@ class ExamplesTestsNoTrainer(TestCasePlus):
             --learning_rate=2e-4
             --per_device_train_batch_size=2
             --per_device_eval_batch_size=1
+            --with_tracking
         """.split()
 
         with patch.object(sys, "argv", testargs):
             run_swag_no_trainer.main()
             result = get_results(tmp_dir)
             self.assertGreaterEqual(result["eval_accuracy"], 0.8)
+            self.assertTrue(os.path.exists(os.path.join(tmp_dir, "swag_no_trainer")))
 
     @slow
     def test_run_summarization_no_trainer(self):
@@ -243,6 +255,7 @@ class ExamplesTestsNoTrainer(TestCasePlus):
             --per_device_train_batch_size=2
             --per_device_eval_batch_size=1
             --checkpointing_steps epoch
+            --with_tracking
         """.split()
 
         with patch.object(sys, "argv", testargs):
@@ -253,6 +266,7 @@ class ExamplesTestsNoTrainer(TestCasePlus):
             self.assertGreaterEqual(result["eval_rougeL"], 7)
             self.assertGreaterEqual(result["eval_rougeLsum"], 7)
             self.assertTrue(os.path.exists(os.path.join(tmp_dir, "epoch_0")))
+            self.assertTrue(os.path.exists(os.path.join(tmp_dir, "summarization_no_trainer")))
 
     @slow
     def test_run_translation_no_trainer(self):
@@ -273,6 +287,7 @@ class ExamplesTestsNoTrainer(TestCasePlus):
             --source_lang en_XX
             --target_lang ro_RO
             --checkpointing_steps epoch
+            --with_tracking
         """.split()
 
         with patch.object(sys, "argv", testargs):
@@ -280,3 +295,4 @@ class ExamplesTestsNoTrainer(TestCasePlus):
             result = get_results(tmp_dir)
             self.assertGreaterEqual(result["eval_bleu"], 30)
             self.assertTrue(os.path.exists(os.path.join(tmp_dir, "epoch_0")))
+            self.assertTrue(os.path.exists(os.path.join(tmp_dir, "translation_no_trainer")))