From 5ab21b072fa2a122da930386381d23f95de06e28 Mon Sep 17 00:00:00 2001 From: Sam Shleifer Date: Fri, 28 Aug 2020 17:05:52 -0400 Subject: [PATCH] [s2s] Test hub configs in self-scheduled CI (#6809) --- examples/seq2seq/test_seq2seq_examples.py | 24 +++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/examples/seq2seq/test_seq2seq_examples.py b/examples/seq2seq/test_seq2seq_examples.py index 3747c0ac7f..e7c795b7c5 100644 --- a/examples/seq2seq/test_seq2seq_examples.py +++ b/examples/seq2seq/test_seq2seq_examples.py @@ -13,9 +13,10 @@ import torch from torch.utils.data import DataLoader import lightning_base -from transformers import AutoModelForSeq2SeqLM, AutoTokenizer +from transformers import AutoConfig, AutoModelForSeq2SeqLM, AutoTokenizer +from transformers.hf_api import HfApi from transformers.modeling_bart import shift_tokens_right -from transformers.testing_utils import CaptureStderr, CaptureStdout, require_multigpu +from transformers.testing_utils import CaptureStderr, CaptureStdout, require_multigpu, require_torch_and_cuda, slow from .distillation import distill_main, evaluate_checkpoint from .finetune import SummarizationModule, main @@ -116,6 +117,25 @@ class TestSummarizationDistiller(unittest.TestCase): logging.disable(logging.CRITICAL) # remove noisy download output from tracebacks return cls + @slow + @require_torch_and_cuda + def test_hub_configs(self): + """I put require_torch_and_cuda cause I only want this to run with self-scheduled.""" + + model_list = HfApi().model_list() + org = "sshleifer" + model_ids = [x.modelId for x in model_list if x.modelId.startswith(org)] + allowed_to_be_broken = ["sshleifer/blenderbot-3B", "sshleifer/blenderbot-90M"] + failures = [] + for m in model_ids: + if m in allowed_to_be_broken: + continue + try: + AutoConfig.from_pretrained(m) + except Exception: + failures.append(m) + assert not failures, f"The following models could not be loaded through AutoConfig: {failures}" + @require_multigpu def test_multigpu(self): updates = dict(