From 66446909b236c17498276857fa88e23d2c91d004 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Thu, 8 Apr 2021 13:13:17 -0700 Subject: [PATCH] [tests] relocate core integration tests (#11146) * relocate core integration tests * add sys.path context manager * cleanup * try * try2 * fix path * doc * style * add dep * add 2 more deps --- docs/source/main_classes/trainer.rst | 4 ++-- docs/source/testing.rst | 23 ++++++++++++++++--- setup.py | 13 +++++++---- src/transformers/dependency_versions_table.py | 9 +++++--- src/transformers/testing_utils.py | 22 ++++++++++++++++++ .../deepspeed/ds_config_zero2.json | 0 .../deepspeed/ds_config_zero3.json | 0 .../deepspeed/test_deepspeed.py | 11 +++++---- .../extended}/test_trainer_ext.py | 5 ++-- 9 files changed, 68 insertions(+), 19 deletions(-) rename {examples/tests => tests}/deepspeed/ds_config_zero2.json (100%) rename {examples/tests => tests}/deepspeed/ds_config_zero3.json (100%) rename {examples/tests => tests}/deepspeed/test_deepspeed.py (98%) rename {examples/tests/trainer => tests/extended}/test_trainer_ext.py (98%) diff --git a/docs/source/main_classes/trainer.rst b/docs/source/main_classes/trainer.rst index 2e323aaa28..bc9f248827 100644 --- a/docs/source/main_classes/trainer.rst +++ b/docs/source/main_classes/trainer.rst @@ -525,7 +525,7 @@ Here is an example of running ``run_translation.py`` under DeepSpeed deploying a .. code-block:: bash deepspeed examples/seq2seq/run_translation.py \ - --deepspeed examples/tests/deepspeed/ds_config.json \ + --deepspeed tests/deepspeed/ds_config.json \ --model_name_or_path t5-small --per_device_train_batch_size 1 \ --output_dir output_dir --overwrite_output_dir --fp16 \ --do_train --max_train_samples 500 --num_train_epochs 1 \ @@ -550,7 +550,7 @@ To deploy DeepSpeed with one GPU adjust the :class:`~transformers.Trainer` comma .. code-block:: bash deepspeed --num_gpus=1 examples/seq2seq/run_translation.py \ - --deepspeed examples/tests/deepspeed/ds_config.json \ + --deepspeed tests/deepspeed/ds_config.json \ --model_name_or_path t5-small --per_device_train_batch_size 1 \ --output_dir output_dir --overwrite_output_dir --fp16 \ --do_train --max_train_samples 500 --num_train_epochs 1 \ diff --git a/docs/source/testing.rst b/docs/source/testing.rst index 10ad3e2311..9a4efb06fc 100644 --- a/docs/source/testing.rst +++ b/docs/source/testing.rst @@ -1,4 +1,4 @@ -.. +.. Copyright 2020 The HuggingFace Team. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with @@ -388,7 +388,7 @@ For a single or a group of tests via ``pytest`` (after ``pip install pytest-pspe .. code-block:: bash - pytest --pspec tests/test_optimization.py + pytest --pspec tests/test_optimization.py @@ -672,7 +672,7 @@ and it will list: test_this2.py::test_floor[integer-1-1.0] test_this2.py::test_floor[negative--1.5--2.0] - test_this2.py::test_floor[large fraction-1.6-1] + test_this2.py::test_floor[large fraction-1.6-1] So now you can run just the specific test: @@ -795,6 +795,23 @@ leave any data in there. otherwise. +Temporary sys.path override +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you need to temporary override ``sys.path`` to import from another test for example, you can use the +``ExtendSysPath`` context manager. Example: + + +.. code-block:: python + + import os + from transformers.testing_utils import ExtendSysPath + bindir = os.path.abspath(os.path.dirname(__file__)) + with ExtendSysPath(f"{bindir}/.."): + from test_trainer import TrainerIntegrationCommon # noqa + + + Skipping tests ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/setup.py b/setup.py index 45df48f68b..c3583a3070 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ To create the package for pypi. 1. Run `make pre-release` (or `make pre-patch` for a patch release) then run `make fix-copies` to fix the index of the documentation. - + 2. Run Tests for Amazon Sagemaker. The documentation is located in `./tests/sagemaker/README.md`, otherwise @philschmid. 3. Unpin specific versions from setup.py that use a git install. @@ -85,6 +85,7 @@ if stale_egg_info.exists(): # 1. all dependencies should be listed here with their version requirements if any # 2. once modified, run: `make deps_table_update` to update src/transformers/dependency_versions_table.py _deps = [ + "Pillow", "black>=20.8b1", "cookiecutter==1.7.2", "dataclasses", @@ -102,13 +103,13 @@ _deps = [ "jax>=0.2.8", "jaxlib>=0.1.59", "keras2onnx", + "nltk", "numpy>=1.17", "onnxconverter-common", "onnxruntime-tools>=1.4.2", "onnxruntime>=1.4.0", "packaging", "parameterized", - "Pillow", "protobuf", "psutil", "pydantic", @@ -119,15 +120,18 @@ _deps = [ "recommonmark", "regex!=2019.12.17", "requests", + "rouge-score", + "sacrebleu>=1.4.12", "sacremoses", + "sagemaker>=2.31.0", "scikit-learn", "sentencepiece==0.1.91", "soundfile", "sphinx-copybutton", "sphinx-markdown-tables", "sphinx-rtd-theme==0.4.3", # sphinx-rtd-theme==0.5.0 introduced big changes in the style. - "sphinxext-opengraph==0.4.1", "sphinx==3.2.1", + "sphinxext-opengraph==0.4.1", "starlette", "tensorflow-cpu>=2.3", "tensorflow>=2.3", @@ -139,7 +143,6 @@ _deps = [ "unidic>=1.0.2", "unidic_lite>=1.0.7", "uvicorn", - "sagemaker>=2.31.0", ] @@ -238,7 +241,7 @@ extras["vision"] = deps_list("Pillow") extras["sentencepiece"] = deps_list("sentencepiece", "protobuf") extras["testing"] = ( deps_list( - "pytest", "pytest-xdist", "timeout-decorator", "parameterized", "psutil", "datasets", "pytest-sugar", "black" + "pytest", "pytest-xdist", "timeout-decorator", "parameterized", "psutil", "datasets", "pytest-sugar", "black", "sacrebleu", "rouge-score", "nltk" ) + extras["retrieval"] + extras["modelcreation"] diff --git a/src/transformers/dependency_versions_table.py b/src/transformers/dependency_versions_table.py index b53407ad3e..43f4c028fe 100644 --- a/src/transformers/dependency_versions_table.py +++ b/src/transformers/dependency_versions_table.py @@ -2,6 +2,7 @@ # 1. modify the `_deps` dict in setup.py # 2. run `make deps_table_update`` deps = { + "Pillow": "Pillow", "black": "black>=20.8b1", "cookiecutter": "cookiecutter==1.7.2", "dataclasses": "dataclasses", @@ -19,13 +20,13 @@ deps = { "jax": "jax>=0.2.8", "jaxlib": "jaxlib>=0.1.59", "keras2onnx": "keras2onnx", + "nltk": "nltk", "numpy": "numpy>=1.17", "onnxconverter-common": "onnxconverter-common", "onnxruntime-tools": "onnxruntime-tools>=1.4.2", "onnxruntime": "onnxruntime>=1.4.0", "packaging": "packaging", "parameterized": "parameterized", - "Pillow": "Pillow", "protobuf": "protobuf", "psutil": "psutil", "pydantic": "pydantic", @@ -36,15 +37,18 @@ deps = { "recommonmark": "recommonmark", "regex": "regex!=2019.12.17", "requests": "requests", + "rouge-score": "rouge-score", + "sacrebleu": "sacrebleu>=1.4.12", "sacremoses": "sacremoses", + "sagemaker": "sagemaker>=2.31.0", "scikit-learn": "scikit-learn", "sentencepiece": "sentencepiece==0.1.91", "soundfile": "soundfile", "sphinx-copybutton": "sphinx-copybutton", "sphinx-markdown-tables": "sphinx-markdown-tables", "sphinx-rtd-theme": "sphinx-rtd-theme==0.4.3", - "sphinxext-opengraph": "sphinxext-opengraph==0.4.1", "sphinx": "sphinx==3.2.1", + "sphinxext-opengraph": "sphinxext-opengraph==0.4.1", "starlette": "starlette", "tensorflow-cpu": "tensorflow-cpu>=2.3", "tensorflow": "tensorflow>=2.3", @@ -56,5 +60,4 @@ deps = { "unidic": "unidic>=1.0.2", "unidic_lite": "unidic_lite>=1.0.7", "uvicorn": "uvicorn", - "sagemaker": "sagemaker>=2.31.0", } diff --git a/src/transformers/testing_utils.py b/src/transformers/testing_utils.py index 3f1273a7c9..a5c4e7d2b8 100644 --- a/src/transformers/testing_utils.py +++ b/src/transformers/testing_utils.py @@ -24,6 +24,7 @@ import unittest from distutils.util import strtobool from io import StringIO from pathlib import Path +from typing import Iterator, Union from .file_utils import ( is_datasets_available, @@ -621,6 +622,27 @@ class CaptureLogger: return f"captured: {self.out}\n" +@contextlib.contextmanager +# adapted from https://stackoverflow.com/a/64789046/9201239 +def ExtendSysPath(path: Union[str, os.PathLike]) -> Iterator[None]: + """ + Temporary add given path to `sys.path`. + + Usage :: + + with ExtendSysPath('/path/to/dir'): + mymodule = importlib.import_module('mymodule') + + """ + + path = os.fspath(path) + try: + sys.path.insert(0, path) + yield + finally: + sys.path.remove(path) + + class TestCasePlus(unittest.TestCase): """ This class extends `unittest.TestCase` with additional features. diff --git a/examples/tests/deepspeed/ds_config_zero2.json b/tests/deepspeed/ds_config_zero2.json similarity index 100% rename from examples/tests/deepspeed/ds_config_zero2.json rename to tests/deepspeed/ds_config_zero2.json diff --git a/examples/tests/deepspeed/ds_config_zero3.json b/tests/deepspeed/ds_config_zero3.json similarity index 100% rename from examples/tests/deepspeed/ds_config_zero3.json rename to tests/deepspeed/ds_config_zero3.json diff --git a/examples/tests/deepspeed/test_deepspeed.py b/tests/deepspeed/test_deepspeed.py similarity index 98% rename from examples/tests/deepspeed/test_deepspeed.py rename to tests/deepspeed/test_deepspeed.py index b9c9b46167..9baaf3085b 100644 --- a/examples/tests/deepspeed/test_deepspeed.py +++ b/tests/deepspeed/test_deepspeed.py @@ -16,16 +16,16 @@ import dataclasses import io import json import os -import sys import unittest from copy import deepcopy from parameterized import parameterized -from transformers import TrainingArguments +from transformers import TrainingArguments, is_torch_available from transformers.file_utils import WEIGHTS_NAME from transformers.integrations import is_deepspeed_available from transformers.testing_utils import ( CaptureLogger, + ExtendSysPath, TestCasePlus, execute_subprocess_async, get_gpu_count, @@ -38,8 +38,11 @@ from transformers.trainer_utils import set_seed bindir = os.path.abspath(os.path.dirname(__file__)) -sys.path.append(f"{bindir}/../../../tests") -from test_trainer import TrainerIntegrationCommon, get_regression_trainer # noqa +with ExtendSysPath(f"{bindir}/.."): + from test_trainer import TrainerIntegrationCommon # noqa + + if is_torch_available(): + from test_trainer import get_regression_trainer # noqa set_seed(42) diff --git a/examples/tests/trainer/test_trainer_ext.py b/tests/extended/test_trainer_ext.py similarity index 98% rename from examples/tests/trainer/test_trainer_ext.py rename to tests/extended/test_trainer_ext.py index 82ec2f625c..6d13f9a4cc 100644 --- a/examples/tests/trainer/test_trainer_ext.py +++ b/tests/extended/test_trainer_ext.py @@ -21,6 +21,7 @@ from unittest.mock import patch from transformers.file_utils import is_apex_available from transformers.integrations import is_fairscale_available from transformers.testing_utils import ( + ExtendSysPath, TestCasePlus, execute_subprocess_async, get_gpu_count, @@ -34,8 +35,8 @@ from transformers.trainer_utils import set_seed bindir = os.path.abspath(os.path.dirname(__file__)) -sys.path.append(f"{bindir}/../../seq2seq") -from run_translation import main # noqa +with ExtendSysPath(f"{bindir}/../../examples/seq2seq"): + from run_translation import main # noqa set_seed(42)