adding TRANSFORMERS_VERBOSITY env var (#6961)

* introduce TRANSFORMERS_VERBOSITY env var + test + test helpers

* cleanup

* remove helper function

docs/source/main_classes/logging.rst

@@ -1,15 +1,23 @@
 Logging
 -------
 
-🤗 Transformers has a centralized logging system, so that you can setup the verbosity of the library easily. To
+🤗 Transformers has a centralized logging system, so that you can setup the verbosity of the library easily.
-change the level of verbosity, just use one of the direct setters. For instance, here is how to change the verbosity to
+
-the INFO level.
+Currently the default verbosity of the library is ``WARNING``.
+
+To change the level of verbosity, just use one of the direct setters. For instance, here is how to change the verbosity to the INFO level.
 
 .. code-block:: python
 
     import transformers
     transformers.logging.set_verbosity_info()
 
+You can also use the environment variable ``TRANSFORMERS_VERBOSITY`` to override the default verbosity. You can set it to one of the following: ``debug``, ``info``, ``warning``, ``error``, ``critical``. For example:
+
+.. code-block:: bash
+               
+    TRANSFORMERS_VERBOSITY=error ./myprogram.py
+
 All the methods of this logging module are documented below, the main ones are
 :func:`transformers.logging.get_verbosity` to get the current level of verbosity in the logger and
 :func:`transformers.logging.set_verbosity` to set the verbosity to the level of your choice. In order (from the least

src/transformers/testing_utils.py

@@ -1,4 +1,5 @@
 import inspect
+import logging
 import os
 import re
 import shutil
@@ -270,6 +271,46 @@ class CaptureStderr(CaptureStd):
         super().__init__(out=False)
 
 
+class CaptureLogger:
+    """Context manager to capture `logging` streams
+
+    Args:
+    - logger: 'logging` logger object
+
+    Results:
+        The captured output is available via `self.out`
+
+    Example:
+
+    from transformers import logging
+    from transformers.testing_utils import CaptureLogger
+
+    msg = "Testing 1, 2, 3"
+    logging.set_verbosity_info()
+    logger = logging.get_logger("transformers.tokenization_bart")
+    with CaptureLogger(logger) as cl:
+        logger.info(msg)
+    assert cl.out, msg+"\n"
+    """
+
+    def __init__(self, logger):
+        self.logger = logger
+        self.io = StringIO()
+        self.sh = logging.StreamHandler(self.io)
+        self.out = ""
+
+    def __enter__(self):
+        self.logger.addHandler(self.sh)
+        return self
+
+    def __exit__(self, *exc):
+        self.logger.removeHandler(self.sh)
+        self.out = self.io.getvalue()
+
+    def __repr__(self):
+        return f"captured: {self.out}\n"
+
+
 class TestCasePlus(unittest.TestCase):
     """This class extends `unittest.TestCase` with additional features.
 
@@ -357,3 +398,14 @@ class TestCasePlus(unittest.TestCase):
         for path in self.teardown_tmp_dirs:
             shutil.rmtree(path, ignore_errors=True)
         self.teardown_tmp_dirs = []
+
+
+def mockenv(**kwargs):
+    """this is a convenience wrapper, that allows this:
+
+    @mockenv(USE_CUDA=True, USE_TF=False)
+    def test_something():
+        use_cuda = os.getenv("USE_CUDA", False)
+        use_tf = os.getenv("USE_TF", False)
+    """
+    return unittest.mock.patch.dict(os.environ, kwargs)

src/transformers/utils/logging.py

@@ -15,6 +15,7 @@
 """ Logging utilities. """
 
 import logging
+import os
 import threading
 from logging import CRITICAL  # NOQA
 from logging import DEBUG  # NOQA
@@ -30,6 +31,33 @@ from typing import Optional
 _lock = threading.Lock()
 _default_handler: Optional[logging.Handler] = None
 
+log_levels = {
+    "debug": logging.DEBUG,
+    "info": logging.INFO,
+    "warning": logging.WARNING,
+    "error": logging.ERROR,
+    "critical": logging.CRITICAL,
+}
+
+_default_log_level = logging.WARNING
+
+
+def _get_default_logging_level():
+    """
+    If TRANSFORMERS_VERBOSITY env var is set to one of the valid choices return that as the new default level.
+    If it is not - fall back to ``_default_log_level``
+    """
+    env_level_str = os.getenv("TRANSFORMERS_VERBOSITY", None)
+    if env_level_str:
+        if env_level_str in log_levels:
+            return log_levels[env_level_str]
+        else:
+            logging.getLogger().warning(
+                f"Unknown option TRANSFORMERS_VERBOSITY={env_level_str}, "
+                f"has to be one of: { ', '.join(log_levels.keys()) }"
+            )
+    return _default_log_level
+
 
 def _get_library_name() -> str:
 
@@ -54,7 +82,7 @@ def _configure_library_root_logger() -> None:
         # Apply our default configuration to the library root logger.
         library_root_logger = _get_library_root_logger()
         library_root_logger.addHandler(_default_handler)
-        library_root_logger.setLevel(logging.WARN)
+        library_root_logger.setLevel(_get_default_logging_level())
         library_root_logger.propagate = False
 
 

tests/test_logging.py

@@ -1,12 +1,18 @@
+import os
 import unittest
 
+import transformers.tokenization_bart
 from transformers import logging
+from transformers.testing_utils import CaptureLogger, mockenv
 
 
 class HfArgumentParserTest(unittest.TestCase):
     def test_set_level(self):
         logger = logging.get_logger()
 
+        # the current default level is logging.WARNING
+        level_origin = logging.get_verbosity()
+
         logging.set_verbosity_error()
         self.assertEqual(logger.getEffectiveLevel(), logging.get_verbosity())
 
@@ -18,3 +24,68 @@ class HfArgumentParserTest(unittest.TestCase):
 
         logging.set_verbosity_debug()
         self.assertEqual(logger.getEffectiveLevel(), logging.get_verbosity())
+
+        # restore to the original level
+        logging.set_verbosity(level_origin)
+
+    def test_integration(self):
+        level_origin = logging.get_verbosity()
+
+        logger = logging.get_logger("transformers.tokenization_bart")
+        msg = "Testing 1, 2, 3"
+
+        # should be able to log warnings (if default settings weren't overriden by `pytest --log-level-all`)
+        if level_origin <= logging.WARNING:
+            with CaptureLogger(logger) as cl:
+                logger.warn(msg)
+            self.assertEqual(cl.out, msg + "\n")
+
+        # this is setting the level for all of `transformers.*` loggers
+        logging.set_verbosity_error()
+
+        # should not be able to log warnings
+        with CaptureLogger(logger) as cl:
+            logger.warn(msg)
+        self.assertEqual(cl.out, "")
+
+        # should be able to log warnings again
+        logging.set_verbosity_warning()
+        with CaptureLogger(logger) as cl:
+            logger.warning(msg)
+        self.assertEqual(cl.out, msg + "\n")
+
+        # restore to the original level
+        logging.set_verbosity(level_origin)
+
+    @mockenv(TRANSFORMERS_VERBOSITY="error")
+    def test_env_override(self):
+        # reset for the env var to take effect, next time some logger call is made
+        transformers.utils.logging._reset_library_root_logger()
+        # this action activates the env var
+        _ = logging.get_logger("transformers.tokenization_bart")
+
+        env_level_str = os.getenv("TRANSFORMERS_VERBOSITY", None)
+        env_level = logging.log_levels[env_level_str]
+
+        current_level = logging.get_verbosity()
+        self.assertEqual(
+            env_level,
+            current_level,
+            f"TRANSFORMERS_VERBOSITY={env_level_str}/{env_level}, but internal verbosity is {current_level}",
+        )
+
+        # restore to the original level
+        os.environ["TRANSFORMERS_VERBOSITY"] = ""
+        transformers.utils.logging._reset_library_root_logger()
+
+    @mockenv(TRANSFORMERS_VERBOSITY="super-error")
+    def test_env_invalid_override(self):
+        # reset for the env var to take effect, next time some logger call is made
+        transformers.utils.logging._reset_library_root_logger()
+        logger = logging.logging.getLogger()
+        with CaptureLogger(logger) as cl:
+            # this action activates the env var
+            logging.get_logger("transformers.tokenization_bart")
+        self.assertIn("Unknown option TRANSFORMERS_VERBOSITY=super-error", cl.out)
+
+        # no need to restore as nothing was changed