From 89a78be51f1c2afd263da66ec76d3297432f0c2a Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Thu, 16 Jul 2020 15:15:10 +0200 Subject: [PATCH] fix benchmark for longformer (#5808) --- src/transformers/benchmark/benchmark.py | 12 ++++++++++-- src/transformers/benchmark/benchmark_tf.py | 12 ++++++++++-- tests/test_benchmark.py | 18 ++++++++++++++++++ 3 files changed, 38 insertions(+), 4 deletions(-) diff --git a/src/transformers/benchmark/benchmark.py b/src/transformers/benchmark/benchmark.py index 035487f5b8..78c4b137bb 100644 --- a/src/transformers/benchmark/benchmark.py +++ b/src/transformers/benchmark/benchmark.py @@ -88,7 +88,11 @@ class PyTorchBenchmark(Benchmark): if self.args.torchscript: config.torchscript = True - has_model_class_in_config = hasattr(config, "architectures") and len(config.architectures) > 0 + has_model_class_in_config = ( + hasattr(config, "architectures") + and isinstance(config.architectures, list) + and len(config.architectures) > 0 + ) if not self.args.only_pretrain_model and has_model_class_in_config: try: model_class = config.architectures[0] @@ -138,7 +142,11 @@ class PyTorchBenchmark(Benchmark): def _prepare_train_func(self, model_name: str, batch_size: int, sequence_length: int) -> Callable[[], None]: config = self.config_dict[model_name] - has_model_class_in_config = hasattr(config, "architectures") and len(config.architectures) > 0 + has_model_class_in_config = ( + hasattr(config, "architectures") + and isinstance(config.architectures, list) + and len(config.architectures) > 0 + ) if not self.args.only_pretrain_model and has_model_class_in_config: try: model_class = config.architectures[0] diff --git a/src/transformers/benchmark/benchmark_tf.py b/src/transformers/benchmark/benchmark_tf.py index aecd5f75aa..707a68aa3c 100644 --- a/src/transformers/benchmark/benchmark_tf.py +++ b/src/transformers/benchmark/benchmark_tf.py @@ -132,7 +132,11 @@ class TensorFlowBenchmark(Benchmark): if self.args.fp16: raise NotImplementedError("Mixed precision is currently not supported.") - has_model_class_in_config = hasattr(config, "architectures") and len(config.architectures) > 0 + has_model_class_in_config = ( + hasattr(config, "architectures") + and isinstance(config.architectures, list) + and len(config.architectures) > 0 + ) if not self.args.only_pretrain_model and has_model_class_in_config: try: model_class = "TF" + config.architectures[0] # prepend 'TF' for tensorflow model @@ -172,7 +176,11 @@ class TensorFlowBenchmark(Benchmark): if self.args.fp16: raise NotImplementedError("Mixed precision is currently not supported.") - has_model_class_in_config = hasattr(config, "architectures") and len(config.architectures) > 0 + has_model_class_in_config = ( + hasattr(config, "architectures") + and isinstance(config.architectures, list) + and len(config.architectures) > 0 + ) if not self.args.only_pretrain_model and has_model_class_in_config: try: model_class = "TF" + config.architectures[0] # prepend 'TF' for tensorflow model diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index 1d96180f07..fb8e3ae24c 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -86,6 +86,24 @@ class BenchmarkTest(unittest.TestCase): self.check_results_dict_not_empty(results.time_inference_result) self.check_results_dict_not_empty(results.memory_inference_result) + def test_inference_no_model_no_architecuters(self): + MODEL_ID = "sshleifer/tiny-gpt2" + config = AutoConfig.from_pretrained(MODEL_ID) + # set architectures equal to `None` + config.architectures = None + benchmark_args = PyTorchBenchmarkArguments( + models=[MODEL_ID], + training=True, + no_inference=False, + sequence_lengths=[8], + batch_sizes=[1], + no_multi_process=True, + ) + benchmark = PyTorchBenchmark(benchmark_args, configs=[config]) + results = benchmark.run() + self.check_results_dict_not_empty(results.time_inference_result) + self.check_results_dict_not_empty(results.memory_inference_result) + def test_train_no_configs(self): MODEL_ID = "sshleifer/tiny-gpt2" benchmark_args = PyTorchBenchmarkArguments(