diff --git a/docs/source/main_classes/logging.rst b/docs/source/main_classes/logging.rst index fba1edb08e..e73f1b84f5 100644 --- a/docs/source/main_classes/logging.rst +++ b/docs/source/main_classes/logging.rst @@ -1,15 +1,23 @@ Logging ------- -🤗 Transformers has a centralized logging system, so that you can setup the verbosity of the library easily. To -change the level of verbosity, just use one of the direct setters. For instance, here is how to change the verbosity to -the INFO level. +🤗 Transformers has a centralized logging system, so that you can setup the verbosity of the library easily. + +Currently the default verbosity of the library is ``WARNING``. + +To change the level of verbosity, just use one of the direct setters. For instance, here is how to change the verbosity to the INFO level. .. code-block:: python import transformers transformers.logging.set_verbosity_info() +You can also use the environment variable ``TRANSFORMERS_VERBOSITY`` to override the default verbosity. You can set it to one of the following: ``debug``, ``info``, ``warning``, ``error``, ``critical``. For example: + +.. code-block:: bash + + TRANSFORMERS_VERBOSITY=error ./myprogram.py + All the methods of this logging module are documented below, the main ones are :func:`transformers.logging.get_verbosity` to get the current level of verbosity in the logger and :func:`transformers.logging.set_verbosity` to set the verbosity to the level of your choice. In order (from the least diff --git a/src/transformers/testing_utils.py b/src/transformers/testing_utils.py index 5a48ee16f0..af2333c176 100644 --- a/src/transformers/testing_utils.py +++ b/src/transformers/testing_utils.py @@ -1,4 +1,5 @@ import inspect +import logging import os import re import shutil @@ -270,6 +271,46 @@ class CaptureStderr(CaptureStd): super().__init__(out=False) +class CaptureLogger: + """Context manager to capture `logging` streams + + Args: + - logger: 'logging` logger object + + Results: + The captured output is available via `self.out` + + Example: + + from transformers import logging + from transformers.testing_utils import CaptureLogger + + msg = "Testing 1, 2, 3" + logging.set_verbosity_info() + logger = logging.get_logger("transformers.tokenization_bart") + with CaptureLogger(logger) as cl: + logger.info(msg) + assert cl.out, msg+"\n" + """ + + def __init__(self, logger): + self.logger = logger + self.io = StringIO() + self.sh = logging.StreamHandler(self.io) + self.out = "" + + def __enter__(self): + self.logger.addHandler(self.sh) + return self + + def __exit__(self, *exc): + self.logger.removeHandler(self.sh) + self.out = self.io.getvalue() + + def __repr__(self): + return f"captured: {self.out}\n" + + class TestCasePlus(unittest.TestCase): """This class extends `unittest.TestCase` with additional features. @@ -357,3 +398,14 @@ class TestCasePlus(unittest.TestCase): for path in self.teardown_tmp_dirs: shutil.rmtree(path, ignore_errors=True) self.teardown_tmp_dirs = [] + + +def mockenv(**kwargs): + """this is a convenience wrapper, that allows this: + + @mockenv(USE_CUDA=True, USE_TF=False) + def test_something(): + use_cuda = os.getenv("USE_CUDA", False) + use_tf = os.getenv("USE_TF", False) + """ + return unittest.mock.patch.dict(os.environ, kwargs) diff --git a/src/transformers/utils/logging.py b/src/transformers/utils/logging.py index 1987718ddb..f677f4e7f1 100644 --- a/src/transformers/utils/logging.py +++ b/src/transformers/utils/logging.py @@ -15,6 +15,7 @@ """ Logging utilities. """ import logging +import os import threading from logging import CRITICAL # NOQA from logging import DEBUG # NOQA @@ -30,6 +31,33 @@ from typing import Optional _lock = threading.Lock() _default_handler: Optional[logging.Handler] = None +log_levels = { + "debug": logging.DEBUG, + "info": logging.INFO, + "warning": logging.WARNING, + "error": logging.ERROR, + "critical": logging.CRITICAL, +} + +_default_log_level = logging.WARNING + + +def _get_default_logging_level(): + """ + If TRANSFORMERS_VERBOSITY env var is set to one of the valid choices return that as the new default level. + If it is not - fall back to ``_default_log_level`` + """ + env_level_str = os.getenv("TRANSFORMERS_VERBOSITY", None) + if env_level_str: + if env_level_str in log_levels: + return log_levels[env_level_str] + else: + logging.getLogger().warning( + f"Unknown option TRANSFORMERS_VERBOSITY={env_level_str}, " + f"has to be one of: { ', '.join(log_levels.keys()) }" + ) + return _default_log_level + def _get_library_name() -> str: @@ -54,7 +82,7 @@ def _configure_library_root_logger() -> None: # Apply our default configuration to the library root logger. library_root_logger = _get_library_root_logger() library_root_logger.addHandler(_default_handler) - library_root_logger.setLevel(logging.WARN) + library_root_logger.setLevel(_get_default_logging_level()) library_root_logger.propagate = False diff --git a/tests/test_logging.py b/tests/test_logging.py index 9f5e3f9b74..e94bf53b42 100644 --- a/tests/test_logging.py +++ b/tests/test_logging.py @@ -1,12 +1,18 @@ +import os import unittest +import transformers.tokenization_bart from transformers import logging +from transformers.testing_utils import CaptureLogger, mockenv class HfArgumentParserTest(unittest.TestCase): def test_set_level(self): logger = logging.get_logger() + # the current default level is logging.WARNING + level_origin = logging.get_verbosity() + logging.set_verbosity_error() self.assertEqual(logger.getEffectiveLevel(), logging.get_verbosity()) @@ -18,3 +24,68 @@ class HfArgumentParserTest(unittest.TestCase): logging.set_verbosity_debug() self.assertEqual(logger.getEffectiveLevel(), logging.get_verbosity()) + + # restore to the original level + logging.set_verbosity(level_origin) + + def test_integration(self): + level_origin = logging.get_verbosity() + + logger = logging.get_logger("transformers.tokenization_bart") + msg = "Testing 1, 2, 3" + + # should be able to log warnings (if default settings weren't overriden by `pytest --log-level-all`) + if level_origin <= logging.WARNING: + with CaptureLogger(logger) as cl: + logger.warn(msg) + self.assertEqual(cl.out, msg + "\n") + + # this is setting the level for all of `transformers.*` loggers + logging.set_verbosity_error() + + # should not be able to log warnings + with CaptureLogger(logger) as cl: + logger.warn(msg) + self.assertEqual(cl.out, "") + + # should be able to log warnings again + logging.set_verbosity_warning() + with CaptureLogger(logger) as cl: + logger.warning(msg) + self.assertEqual(cl.out, msg + "\n") + + # restore to the original level + logging.set_verbosity(level_origin) + + @mockenv(TRANSFORMERS_VERBOSITY="error") + def test_env_override(self): + # reset for the env var to take effect, next time some logger call is made + transformers.utils.logging._reset_library_root_logger() + # this action activates the env var + _ = logging.get_logger("transformers.tokenization_bart") + + env_level_str = os.getenv("TRANSFORMERS_VERBOSITY", None) + env_level = logging.log_levels[env_level_str] + + current_level = logging.get_verbosity() + self.assertEqual( + env_level, + current_level, + f"TRANSFORMERS_VERBOSITY={env_level_str}/{env_level}, but internal verbosity is {current_level}", + ) + + # restore to the original level + os.environ["TRANSFORMERS_VERBOSITY"] = "" + transformers.utils.logging._reset_library_root_logger() + + @mockenv(TRANSFORMERS_VERBOSITY="super-error") + def test_env_invalid_override(self): + # reset for the env var to take effect, next time some logger call is made + transformers.utils.logging._reset_library_root_logger() + logger = logging.logging.getLogger() + with CaptureLogger(logger) as cl: + # this action activates the env var + logging.get_logger("transformers.tokenization_bart") + self.assertIn("Unknown option TRANSFORMERS_VERBOSITY=super-error", cl.out) + + # no need to restore as nothing was changed