[lightning_base] fix s2s logging, only make train_loader once (#6404)

examples/token-classification/test_ner_examples.py

@@ -4,6 +4,7 @@ import unittest
 from unittest.mock import patch
 
 import run_ner
+from transformers.testing_utils import slow
 
 
 logging.basicConfig(level=logging.INFO)
@@ -12,6 +13,7 @@ logger = logging.getLogger()
 
 
 class ExamplesTests(unittest.TestCase):
+    @slow
     def test_run_ner(self):
         stream_handler = logging.StreamHandler(sys.stdout)
         logger.addHandler(stream_handler)
@@ -31,3 +33,23 @@ class ExamplesTests(unittest.TestCase):
         with patch.object(sys, "argv", ["run.py"] + testargs):
             result = run_ner.main()
             self.assertLess(result["eval_loss"], 1.5)
+
+    def test_run_ner_pl(self):
+        stream_handler = logging.StreamHandler(sys.stdout)
+        logger.addHandler(stream_handler)
+
+        testargs = """
+            --model_name distilbert-base-german-cased
+            --output_dir ./tests/fixtures/tests_samples/temp_dir
+            --overwrite_output_dir
+            --data_dir ./tests/fixtures/tests_samples/GermEval
+            --labels ./tests/fixtures/tests_samples/GermEval/labels.txt
+            --max_seq_length 128
+            --num_train_epochs 6
+            --logging_steps 1
+            --do_train
+            --do_eval
+            """.split()
+        with patch.object(sys, "argv", ["run.py"] + testargs):
+            result = run_ner.main()
+            self.assertLess(result["eval_loss"], 1.5)