SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Examples reorg (#11350)

Browse Source 
* Base move

* Examples reorganization

* Update references

* Put back test data

* Move conftest

* More fixes

* Move test data to test fixtures

* Update path

* Apply suggestions from code review

Co-authored-by: Lysandre Debut <lysandre@huggingface.co>

* Address review comments and clean

Co-authored-by: Lysandre Debut <lysandre@huggingface.co>
This commit is contained in:
Sylvain Gugger
2021-04-21 11:11:20 -04:00
committed by GitHub
parent ca7ff64f5b
commit dabeb15292
105 changed files with 1062 additions and 560 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
tests/deepspeed/test_deepspeed.py
View File

@@ -536,7 +536,7 @@ class TestDeepSpeedWithLauncher(TestCasePlus):
remove_args_str: str = None,
):
max_len = 32
data_dir = self.examples_dir / "test_data/wmt_en_ro"
data_dir = self.test_file_dir / "../fixtures/tests_samples/wmt_en_ro"
output_dir = self.get_auto_remove_tmp_dir()
args = f"""
--model_name_or_path {model_name}
@@ -594,7 +594,7 @@ class TestDeepSpeedWithLauncher(TestCasePlus):
args = [x for x in args if x not in remove_args]
ds_args = f"--deepspeed {self.test_file_dir_str}/ds_config_{stage}.json".split()
script = [f"{self.examples_dir_str}/seq2seq/run_translation.py"]
script = [f"{self.examples_dir_str}/pytorch/translation/run_translation.py"]
launcher = self.get_launcher(distributed)
cmd = launcher + script + args + ds_args
@@ -629,7 +629,7 @@ class TestDeepSpeedWithLauncher(TestCasePlus):
""".split()
ds_args = f"--deepspeed {self.test_file_dir_str}/ds_config_{stage}.json".split()
script = [f"{self.examples_dir_str}/language-modeling/run_clm.py"]
script = [f"{self.examples_dir_str}/pytorch/language-modeling/run_clm.py"]
launcher = self.get_launcher(distributed=True)
cmd = launcher + script + args + ds_args