From 2ab75add4b30c2fc44a8bf575156d448d9ed87a7 Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Mon, 17 Jul 2023 22:37:28 +0200 Subject: [PATCH] Remove `tests/onnx` (#24868) fix Co-authored-by: ydshieh --- tests/onnx/__init__.py | 0 tests/onnx/test_features.py | 111 -------- tests/onnx/test_onnx.py | 195 ------------- tests/onnx/test_onnx_v2.py | 526 ------------------------------------ 4 files changed, 832 deletions(-) delete mode 100644 tests/onnx/__init__.py delete mode 100644 tests/onnx/test_features.py delete mode 100644 tests/onnx/test_onnx.py delete mode 100644 tests/onnx/test_onnx_v2.py diff --git a/tests/onnx/__init__.py b/tests/onnx/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/onnx/test_features.py b/tests/onnx/test_features.py deleted file mode 100644 index 4590ff0cc8..0000000000 --- a/tests/onnx/test_features.py +++ /dev/null @@ -1,111 +0,0 @@ -from tempfile import TemporaryDirectory -from unittest import TestCase -from unittest.mock import MagicMock, patch - -from transformers import AutoModel, TFAutoModel -from transformers.onnx import FeaturesManager -from transformers.testing_utils import SMALL_MODEL_IDENTIFIER, require_tf, require_torch - - -@require_torch -@require_tf -class DetermineFrameworkTest(TestCase): - """ - Test `FeaturesManager.determine_framework` - """ - - def setUp(self): - self.test_model = SMALL_MODEL_IDENTIFIER - self.framework_pt = "pt" - self.framework_tf = "tf" - - def _setup_pt_ckpt(self, save_dir): - model_pt = AutoModel.from_pretrained(self.test_model) - model_pt.save_pretrained(save_dir) - - def _setup_tf_ckpt(self, save_dir): - model_tf = TFAutoModel.from_pretrained(self.test_model, from_pt=True) - model_tf.save_pretrained(save_dir) - - def test_framework_provided(self): - """ - Ensure the that the provided framework is returned. - """ - mock_framework = "mock_framework" - - # Framework provided - return whatever the user provides - result = FeaturesManager.determine_framework(self.test_model, mock_framework) - self.assertEqual(result, mock_framework) - - # Local checkpoint and framework provided - return provided framework - # PyTorch checkpoint - with TemporaryDirectory() as local_pt_ckpt: - self._setup_pt_ckpt(local_pt_ckpt) - result = FeaturesManager.determine_framework(local_pt_ckpt, mock_framework) - self.assertEqual(result, mock_framework) - - # TensorFlow checkpoint - with TemporaryDirectory() as local_tf_ckpt: - self._setup_tf_ckpt(local_tf_ckpt) - result = FeaturesManager.determine_framework(local_tf_ckpt, mock_framework) - self.assertEqual(result, mock_framework) - - def test_checkpoint_provided(self): - """ - Ensure that the determined framework is the one used for the local checkpoint. - - For the functionality to execute, local checkpoints are provided but framework is not. - """ - # PyTorch checkpoint - with TemporaryDirectory() as local_pt_ckpt: - self._setup_pt_ckpt(local_pt_ckpt) - result = FeaturesManager.determine_framework(local_pt_ckpt) - self.assertEqual(result, self.framework_pt) - - # TensorFlow checkpoint - with TemporaryDirectory() as local_tf_ckpt: - self._setup_tf_ckpt(local_tf_ckpt) - result = FeaturesManager.determine_framework(local_tf_ckpt) - self.assertEqual(result, self.framework_tf) - - # Invalid local checkpoint - with TemporaryDirectory() as local_invalid_ckpt: - with self.assertRaises(FileNotFoundError): - result = FeaturesManager.determine_framework(local_invalid_ckpt) - - def test_from_environment(self): - """ - Ensure that the determined framework is the one available in the environment. - - For the functionality to execute, framework and local checkpoints are not provided. - """ - # Framework not provided, hub model is used (no local checkpoint directory) - # TensorFlow not in environment -> use PyTorch - mock_tf_available = MagicMock(return_value=False) - with patch("transformers.onnx.features.is_tf_available", mock_tf_available): - result = FeaturesManager.determine_framework(self.test_model) - self.assertEqual(result, self.framework_pt) - - # PyTorch not in environment -> use TensorFlow - mock_torch_available = MagicMock(return_value=False) - with patch("transformers.onnx.features.is_torch_available", mock_torch_available): - result = FeaturesManager.determine_framework(self.test_model) - self.assertEqual(result, self.framework_tf) - - # Both in environment -> use PyTorch - mock_tf_available = MagicMock(return_value=True) - mock_torch_available = MagicMock(return_value=True) - with patch("transformers.onnx.features.is_tf_available", mock_tf_available), patch( - "transformers.onnx.features.is_torch_available", mock_torch_available - ): - result = FeaturesManager.determine_framework(self.test_model) - self.assertEqual(result, self.framework_pt) - - # Both not in environment -> raise error - mock_tf_available = MagicMock(return_value=False) - mock_torch_available = MagicMock(return_value=False) - with patch("transformers.onnx.features.is_tf_available", mock_tf_available), patch( - "transformers.onnx.features.is_torch_available", mock_torch_available - ): - with self.assertRaises(EnvironmentError): - result = FeaturesManager.determine_framework(self.test_model) diff --git a/tests/onnx/test_onnx.py b/tests/onnx/test_onnx.py deleted file mode 100644 index 851934ef30..0000000000 --- a/tests/onnx/test_onnx.py +++ /dev/null @@ -1,195 +0,0 @@ -# Copyright 2020 The HuggingFace Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import unittest -from pathlib import Path -from tempfile import NamedTemporaryFile, TemporaryDirectory - -from transformers import BertConfig, BertTokenizerFast, FeatureExtractionPipeline -from transformers.convert_graph_to_onnx import ( - convert, - ensure_valid_input, - generate_identified_filename, - infer_shapes, - quantize, -) -from transformers.testing_utils import require_tf, require_tokenizers, require_torch, slow - - -class FuncContiguousArgs: - def forward(self, input_ids, token_type_ids, attention_mask): - return None - - -class FuncNonContiguousArgs: - def forward(self, input_ids, some_other_args, token_type_ids, attention_mask): - return None - - -class OnnxExportTestCase(unittest.TestCase): - MODEL_TO_TEST = [ - # (model_name, model_kwargs) - ("bert-base-cased", {}), - ("gpt2", {"use_cache": False}), # We don't support exporting GPT2 past keys anymore - ] - - @require_tf - @slow - def test_export_tensorflow(self): - for model, model_kwargs in OnnxExportTestCase.MODEL_TO_TEST: - self._test_export(model, "tf", 12, **model_kwargs) - - @require_torch - @slow - def test_export_pytorch(self): - for model, model_kwargs in OnnxExportTestCase.MODEL_TO_TEST: - self._test_export(model, "pt", 12, **model_kwargs) - - @require_torch - @slow - def test_export_custom_bert_model(self): - from transformers import BertModel - - vocab = ["[UNK]", "[SEP]", "[CLS]", "[PAD]", "[MASK]", "some", "other", "words"] - with NamedTemporaryFile(mode="w+t") as vocab_file: - vocab_file.write("\n".join(vocab)) - vocab_file.flush() - tokenizer = BertTokenizerFast(vocab_file.name) - - with TemporaryDirectory() as bert_save_dir: - model = BertModel(BertConfig(vocab_size=len(vocab))) - model.save_pretrained(bert_save_dir) - self._test_export(bert_save_dir, "pt", 12, tokenizer) - - @require_tf - @slow - def test_quantize_tf(self): - for model, model_kwargs in OnnxExportTestCase.MODEL_TO_TEST: - path = self._test_export(model, "tf", 12, **model_kwargs) - quantized_path = quantize(Path(path)) - - # Ensure the actual quantized model is not bigger than the original one - if quantized_path.stat().st_size >= Path(path).stat().st_size: - self.fail("Quantized model is bigger than initial ONNX model") - - @require_torch - @slow - def test_quantize_pytorch(self): - for model, model_kwargs in OnnxExportTestCase.MODEL_TO_TEST: - path = self._test_export(model, "pt", 12, **model_kwargs) - quantized_path = quantize(path) - - # Ensure the actual quantized model is not bigger than the original one - if quantized_path.stat().st_size >= Path(path).stat().st_size: - self.fail("Quantized model is bigger than initial ONNX model") - - def _test_export(self, model, framework, opset, tokenizer=None, **model_kwargs): - try: - # Compute path - with TemporaryDirectory() as tempdir: - path = Path(tempdir).joinpath("model.onnx") - - # Remove folder if exists - if path.parent.exists(): - path.parent.rmdir() - - # Export - convert(framework, model, path, opset, tokenizer, **model_kwargs) - - return path - except Exception as e: - self.fail(e) - - @require_torch - @require_tokenizers - @slow - def test_infer_dynamic_axis_pytorch(self): - """ - Validate the dynamic axis generated for each parameters are correct - """ - from transformers import BertModel - - model = BertModel(BertConfig.from_pretrained("lysandre/tiny-bert-random")) - tokenizer = BertTokenizerFast.from_pretrained("lysandre/tiny-bert-random") - self._test_infer_dynamic_axis(model, tokenizer, "pt") - - @require_tf - @require_tokenizers - @slow - def test_infer_dynamic_axis_tf(self): - """ - Validate the dynamic axis generated for each parameters are correct - """ - from transformers import TFBertModel - - model = TFBertModel(BertConfig.from_pretrained("lysandre/tiny-bert-random")) - tokenizer = BertTokenizerFast.from_pretrained("lysandre/tiny-bert-random") - self._test_infer_dynamic_axis(model, tokenizer, "tf") - - def _test_infer_dynamic_axis(self, model, tokenizer, framework): - feature_extractor = FeatureExtractionPipeline(model, tokenizer) - - variable_names = ["input_ids", "token_type_ids", "attention_mask", "output_0", "output_1"] - input_vars, output_vars, shapes, tokens = infer_shapes(feature_extractor, framework) - - # Assert all variables are present - self.assertEqual(len(shapes), len(variable_names)) - self.assertTrue(all(var_name in shapes for var_name in variable_names)) - self.assertSequenceEqual(variable_names[:3], input_vars) - self.assertSequenceEqual(variable_names[3:], output_vars) - - # Assert inputs are {0: batch, 1: sequence} - for var_name in ["input_ids", "token_type_ids", "attention_mask"]: - self.assertDictEqual(shapes[var_name], {0: "batch", 1: "sequence"}) - - # Assert outputs are {0: batch, 1: sequence} and {0: batch} - self.assertDictEqual(shapes["output_0"], {0: "batch", 1: "sequence"}) - self.assertDictEqual(shapes["output_1"], {0: "batch"}) - - def test_ensure_valid_input(self): - """ - Validate parameters are correctly exported - GPT2 has "past" parameter in the middle of input_ids, token_type_ids and attention_mask. - ONNX doesn't support export with a dictionary, only a tuple. Thus we need to ensure we remove - token_type_ids and attention_mask for now to not having a None tensor in the middle - """ - # All generated args are valid - input_names = ["input_ids", "attention_mask", "token_type_ids"] - tokens = {"input_ids": [1, 2, 3, 4], "attention_mask": [0, 0, 0, 0], "token_type_ids": [1, 1, 1, 1]} - ordered_input_names, inputs_args = ensure_valid_input(FuncContiguousArgs(), tokens, input_names) - - # Should have exactly the same number of args (all are valid) - self.assertEqual(len(inputs_args), 3) - - # Should have exactly the same input names - self.assertEqual(set(ordered_input_names), set(input_names)) - - # Parameter should be reordered according to their respective place in the function: - # (input_ids, token_type_ids, attention_mask) - self.assertEqual(inputs_args, (tokens["input_ids"], tokens["token_type_ids"], tokens["attention_mask"])) - - # Generated args are interleaved with another args (for instance parameter "past" in GPT2) - ordered_input_names, inputs_args = ensure_valid_input(FuncNonContiguousArgs(), tokens, input_names) - - # Should have exactly the one arg (all before the one not provided "some_other_args") - self.assertEqual(len(inputs_args), 1) - self.assertEqual(len(ordered_input_names), 1) - - # Should have only "input_ids" - self.assertEqual(inputs_args[0], tokens["input_ids"]) - self.assertEqual(ordered_input_names[0], "input_ids") - - def test_generate_identified_name(self): - generated = generate_identified_filename(Path("/home/something/my_fake_model.onnx"), "-test") - self.assertEqual("/home/something/my_fake_model-test.onnx", generated.as_posix()) diff --git a/tests/onnx/test_onnx_v2.py b/tests/onnx/test_onnx_v2.py deleted file mode 100644 index e160cd77f9..0000000000 --- a/tests/onnx/test_onnx_v2.py +++ /dev/null @@ -1,526 +0,0 @@ -import os -import unittest -from pathlib import Path -from tempfile import NamedTemporaryFile -from unittest import TestCase -from unittest.mock import patch - -import pytest -from packaging import version -from parameterized import parameterized - -from transformers import AutoConfig, PreTrainedTokenizerBase, is_tf_available, is_torch_available -from transformers.onnx import ( - EXTERNAL_DATA_FORMAT_SIZE_LIMIT, - OnnxConfig, - OnnxConfigWithPast, - ParameterFormat, - validate_model_outputs, -) -from transformers.onnx.utils import ( - compute_effective_axis_dimension, - compute_serialized_parameters_size, - get_preprocessor, -) -from transformers.testing_utils import require_onnx, require_rjieba, require_tf, require_torch, require_vision, slow - - -if is_torch_available() or is_tf_available(): - from transformers.onnx.features import FeaturesManager - -if is_torch_available(): - import torch - - from transformers.models.deberta import modeling_deberta - - -@require_onnx -class OnnxUtilsTestCaseV2(TestCase): - """ - Cover all the utilities involved to export ONNX models - """ - - def test_compute_effective_axis_dimension(self): - """ - When exporting ONNX model with dynamic axis (batch or sequence) we set batch_size and/or sequence_length = -1. - We cannot generate an effective tensor with axis dim == -1, so we trick by using some "fixed" values - (> 1 to avoid ONNX squeezing the axis). - - This test ensure we are correctly replacing generated batch / sequence tensor with axis > 1 - """ - - # Dynamic axis (batch, no token added by the tokenizer) - self.assertEqual(compute_effective_axis_dimension(-1, fixed_dimension=2, num_token_to_add=0), 2) - - # Static axis (batch, no token added by the tokenizer) - self.assertEqual(compute_effective_axis_dimension(0, fixed_dimension=2, num_token_to_add=0), 2) - - # Dynamic axis (sequence, token added by the tokenizer 2 (no pair)) - self.assertEqual(compute_effective_axis_dimension(0, fixed_dimension=8, num_token_to_add=2), 6) - self.assertEqual(compute_effective_axis_dimension(0, fixed_dimension=8, num_token_to_add=2), 6) - - # Dynamic axis (sequence, token added by the tokenizer 3 (pair)) - self.assertEqual(compute_effective_axis_dimension(0, fixed_dimension=8, num_token_to_add=3), 5) - self.assertEqual(compute_effective_axis_dimension(0, fixed_dimension=8, num_token_to_add=3), 5) - - def test_compute_parameters_serialized_size(self): - """ - This test ensures we compute a "correct" approximation of the underlying storage requirement (size) for all the - parameters for the specified parameter's dtype. - """ - self.assertEqual(compute_serialized_parameters_size(2, ParameterFormat.Float), 2 * ParameterFormat.Float.size) - - def test_flatten_output_collection_property(self): - """ - This test ensures we correctly flatten nested collection such as the one we use when returning past_keys. - past_keys = Tuple[Tuple] - - ONNX exporter will export nested collections as ${collection_name}.${level_idx_0}.${level_idx_1}...${idx_n} - """ - self.assertEqual( - OnnxConfig.flatten_output_collection_property("past_key", [[0], [1], [2]]), - { - "past_key.0": 0, - "past_key.1": 1, - "past_key.2": 2, - }, - ) - - -class OnnxConfigTestCaseV2(TestCase): - """ - Cover the test for models default. - - Default means no specific features is being enabled on the model. - """ - - @patch.multiple(OnnxConfig, __abstractmethods__=set()) - def test_use_external_data_format(self): - """ - External data format is required only if the serialized size of the parameters if bigger than 2Gb - """ - TWO_GB_LIMIT = EXTERNAL_DATA_FORMAT_SIZE_LIMIT - - # No parameters - self.assertFalse(OnnxConfig.use_external_data_format(0)) - - # Some parameters - self.assertFalse(OnnxConfig.use_external_data_format(1)) - - # Almost 2Gb parameters - self.assertFalse(OnnxConfig.use_external_data_format((TWO_GB_LIMIT - 1) // ParameterFormat.Float.size)) - - # Exactly 2Gb parameters - self.assertTrue(OnnxConfig.use_external_data_format(TWO_GB_LIMIT)) - - # More than 2Gb parameters - self.assertTrue(OnnxConfig.use_external_data_format((TWO_GB_LIMIT + 1) // ParameterFormat.Float.size)) - - -class OnnxConfigWithPastTestCaseV2(TestCase): - """ - Cover the tests for model which have use_cache feature (i.e. "with_past" for ONNX) - """ - - SUPPORTED_WITH_PAST_CONFIGS = {} - # SUPPORTED_WITH_PAST_CONFIGS = { - # ("BART", BartConfig), - # ("GPT2", GPT2Config), - # # ("T5", T5Config) - # } - - @patch.multiple(OnnxConfigWithPast, __abstractmethods__=set()) - def test_use_past(self): - """ - Ensure the use_past variable is correctly being set - """ - for name, config in OnnxConfigWithPastTestCaseV2.SUPPORTED_WITH_PAST_CONFIGS: - with self.subTest(name): - self.assertFalse( - OnnxConfigWithPast.from_model_config(config()).use_past, - "OnnxConfigWithPast.from_model_config() should not use_past", - ) - - self.assertTrue( - OnnxConfigWithPast.with_past(config()).use_past, - "OnnxConfigWithPast.from_model_config() should use_past", - ) - - @patch.multiple(OnnxConfigWithPast, __abstractmethods__=set()) - def test_values_override(self): - """ - Ensure the use_past variable correctly set the `use_cache` value in model's configuration - """ - for name, config in OnnxConfigWithPastTestCaseV2.SUPPORTED_WITH_PAST_CONFIGS: - with self.subTest(name): - # without past - onnx_config_default = OnnxConfigWithPast.from_model_config(config()) - self.assertIsNotNone(onnx_config_default.values_override, "values_override should not be None") - self.assertIn("use_cache", onnx_config_default.values_override, "use_cache should be present") - self.assertFalse( - onnx_config_default.values_override["use_cache"], "use_cache should be False if not using past" - ) - - # with past - onnx_config_default = OnnxConfigWithPast.with_past(config()) - self.assertIsNotNone(onnx_config_default.values_override, "values_override should not be None") - self.assertIn("use_cache", onnx_config_default.values_override, "use_cache should be present") - self.assertTrue( - onnx_config_default.values_override["use_cache"], "use_cache should be False if not using past" - ) - - -PYTORCH_EXPORT_MODELS = { - ("albert", "hf-internal-testing/tiny-random-AlbertModel"), - ("bert", "hf-internal-testing/tiny-random-BertModel"), - ("beit", "microsoft/beit-base-patch16-224"), - ("big-bird", "hf-internal-testing/tiny-random-BigBirdModel"), - ("camembert", "camembert-base"), - ("clip", "hf-internal-testing/tiny-random-CLIPModel"), - ("convbert", "hf-internal-testing/tiny-random-ConvBertModel"), - ("codegen", "hf-internal-testing/tiny-random-CodeGenModel"), - ("data2vec-text", "hf-internal-testing/tiny-random-Data2VecTextModel"), - ("data2vec-vision", "facebook/data2vec-vision-base"), - ("deberta", "hf-internal-testing/tiny-random-DebertaModel"), - ("deberta-v2", "hf-internal-testing/tiny-random-DebertaV2Model"), - ("deit", "facebook/deit-small-patch16-224"), - ("convnext", "facebook/convnext-tiny-224"), - ("detr", "facebook/detr-resnet-50"), - ("distilbert", "hf-internal-testing/tiny-random-DistilBertModel"), - ("electra", "hf-internal-testing/tiny-random-ElectraModel"), - ("groupvit", "nvidia/groupvit-gcc-yfcc"), - ("ibert", "kssteven/ibert-roberta-base"), - ("imagegpt", "openai/imagegpt-small"), - ("levit", "facebook/levit-128S"), - ("layoutlm", "hf-internal-testing/tiny-random-LayoutLMModel"), - ("layoutlmv3", "microsoft/layoutlmv3-base"), - ("longformer", "allenai/longformer-base-4096"), - ("mobilebert", "hf-internal-testing/tiny-random-MobileBertModel"), - ("mobilenet_v1", "google/mobilenet_v1_0.75_192"), - ("mobilenet_v2", "google/mobilenet_v2_0.35_96"), - ("mobilevit", "apple/mobilevit-small"), - ("owlvit", "google/owlvit-base-patch32"), - ("perceiver", "hf-internal-testing/tiny-random-PerceiverModel", ("masked-lm", "sequence-classification")), - ("perceiver", "hf-internal-testing/tiny-random-PerceiverModel", ("image-classification",)), - ("poolformer", "sail/poolformer_s12"), - ("rembert", "google/rembert"), - ("resnet", "microsoft/resnet-50"), - ("roberta", "hf-internal-testing/tiny-random-RobertaModel"), - ("roformer", "hf-internal-testing/tiny-random-RoFormerModel"), - ("segformer", "nvidia/segformer-b0-finetuned-ade-512-512"), - ("squeezebert", "hf-internal-testing/tiny-random-SqueezeBertModel"), - ("swin", "microsoft/swin-tiny-patch4-window7-224"), - ("vit", "google/vit-base-patch16-224"), - ("yolos", "hustvl/yolos-tiny"), - ("whisper", "openai/whisper-tiny.en"), - ("xlm", "hf-internal-testing/tiny-random-XLMModel"), - ("xlm-roberta", "hf-internal-testing/tiny-random-XLMRobertaXLModel"), -} - -PYTORCH_EXPORT_ENCODER_DECODER_MODELS = { - ("vision-encoder-decoder", "nlpconnect/vit-gpt2-image-captioning"), -} - -PYTORCH_EXPORT_WITH_PAST_MODELS = { - ("bloom", "hf-internal-testing/tiny-random-BloomModel"), - ("gpt2", "hf-internal-testing/tiny-random-GPT2Model"), - ("gpt-neo", "hf-internal-testing/tiny-random-GPTNeoModel"), -} - -PYTORCH_EXPORT_SEQ2SEQ_WITH_PAST_MODELS = { - ("bart", "hf-internal-testing/tiny-random-BartModel"), - ("bigbird-pegasus", "hf-internal-testing/tiny-random-BigBirdPegasusModel"), - ("blenderbot-small", "facebook/blenderbot_small-90M"), - ("blenderbot", "hf-internal-testing/tiny-random-BlenderbotModel"), - ("longt5", "hf-internal-testing/tiny-random-LongT5Model"), - ("marian", "Helsinki-NLP/opus-mt-en-de"), - ("mbart", "sshleifer/tiny-mbart"), - ("mt5", "google/mt5-base"), - ("m2m-100", "hf-internal-testing/tiny-random-M2M100Model"), - ("t5", "hf-internal-testing/tiny-random-T5Model"), -} - -# TODO(lewtun): Include the same model types in `PYTORCH_EXPORT_MODELS` once TensorFlow has parity with the PyTorch model implementations. -TENSORFLOW_EXPORT_DEFAULT_MODELS = { - ("albert", "hf-internal-testing/tiny-albert"), - ("bert", "hf-internal-testing/tiny-random-BertModel"), - ("camembert", "camembert-base"), - ("distilbert", "hf-internal-testing/tiny-random-DistilBertModel"), - ("roberta", "hf-internal-testing/tiny-random-RobertaModel"), -} - -# TODO(lewtun): Include the same model types in `PYTORCH_EXPORT_WITH_PAST_MODELS` once TensorFlow has parity with the PyTorch model implementations. -TENSORFLOW_EXPORT_WITH_PAST_MODELS = {} - -# TODO(lewtun): Include the same model types in `PYTORCH_EXPORT_SEQ2SEQ_WITH_PAST_MODELS` once TensorFlow has parity with the PyTorch model implementations. -TENSORFLOW_EXPORT_SEQ2SEQ_WITH_PAST_MODELS = {} - - -def _get_models_to_test(export_models_list): - models_to_test = [] - if is_torch_available() or is_tf_available(): - for name, model, *features in export_models_list: - if features: - feature_config_mapping = { - feature: FeaturesManager.get_config(name, feature) for _ in features for feature in _ - } - else: - # pre-process the model names - model_type = name.replace("_", "-") - model_name = getattr(model, "name", "") - feature_config_mapping = FeaturesManager.get_supported_features_for_model_type( - model_type, model_name=model_name - ) - - for feature, onnx_config_class_constructor in feature_config_mapping.items(): - models_to_test.append((f"{name}_{feature}", name, model, feature, onnx_config_class_constructor)) - return sorted(models_to_test) - else: - # Returning some dummy test that should not be ever called because of the @require_torch / @require_tf - # decorators. - # The reason for not returning an empty list is because parameterized.expand complains when it's empty. - return [("dummy", "dummy", "dummy", "dummy", OnnxConfig.from_model_config)] - - -class OnnxExportTestCaseV2(TestCase): - """ - Integration tests ensuring supported models are correctly exported - """ - - def _onnx_export( - self, test_name, name, model_name, feature, onnx_config_class_constructor, device="cpu", framework="pt" - ): - from transformers.onnx import export - - model_class = FeaturesManager.get_model_class_for_feature(feature, framework=framework) - config = AutoConfig.from_pretrained(model_name) - model = model_class.from_config(config) - - # Dynamic axes aren't supported for YOLO-like models. This means they cannot be exported to ONNX on CUDA devices. - # See: https://github.com/ultralytics/yolov5/pull/8378 - if model.__class__.__name__.startswith("Yolos") and device != "cpu": - return - - # ONNX inference fails with the following name, feature, framework parameterizations - # See: https://github.com/huggingface/transformers/issues/19357 - if (name, feature, framework) in { - ("deberta-v2", "question-answering", "pt"), - ("deberta-v2", "multiple-choice", "pt"), - ("roformer", "multiple-choice", "pt"), - ("groupvit", "default", "pt"), - ("perceiver", "masked-lm", "pt"), - ("perceiver", "sequence-classification", "pt"), - ("perceiver", "image-classification", "pt"), - ("bert", "multiple-choice", "tf"), - ("camembert", "multiple-choice", "tf"), - ("roberta", "multiple-choice", "tf"), - }: - return - - onnx_config = onnx_config_class_constructor(model.config) - - if is_torch_available(): - from transformers.utils import get_torch_version - - if version.parse(get_torch_version()) < onnx_config.torch_onnx_minimum_version: - pytest.skip( - "Skipping due to incompatible PyTorch version. Minimum required is" - f" {onnx_config.torch_onnx_minimum_version}, got: {get_torch_version()}" - ) - - preprocessor = get_preprocessor(model_name) - - # Useful for causal lm models that do not use pad tokens. - if isinstance(preprocessor, PreTrainedTokenizerBase) and not getattr(config, "pad_token_id", None): - config.pad_token_id = preprocessor.eos_token_id - - with NamedTemporaryFile("w") as output: - try: - onnx_inputs, onnx_outputs = export( - preprocessor, model, onnx_config, onnx_config.default_onnx_opset, Path(output.name), device=device - ) - validate_model_outputs( - onnx_config, - preprocessor, - model, - Path(output.name), - onnx_outputs, - onnx_config.atol_for_validation, - ) - except (RuntimeError, ValueError) as e: - self.fail(f"{name}, {feature} -> {e}") - - def _onnx_export_encoder_decoder_models( - self, test_name, name, model_name, feature, onnx_config_class_constructor, device="cpu" - ): - from transformers import AutoFeatureExtractor, AutoTokenizer - from transformers.onnx import export - - model_class = FeaturesManager.get_model_class_for_feature(feature) - config = AutoConfig.from_pretrained(model_name) - model = model_class.from_config(config) - - onnx_config = onnx_config_class_constructor(model.config) - - if is_torch_available(): - from transformers.utils import get_torch_version - - if version.parse(get_torch_version()) < onnx_config.torch_onnx_minimum_version: - pytest.skip( - "Skipping due to incompatible PyTorch version. Minimum required is" - f" {onnx_config.torch_onnx_minimum_version}, got: {get_torch_version()}" - ) - - encoder_model = model.get_encoder() - decoder_model = model.get_decoder() - - encoder_onnx_config = onnx_config.get_encoder_config(encoder_model.config) - decoder_onnx_config = onnx_config.get_decoder_config(encoder_model.config, decoder_model.config, feature) - - preprocessor = AutoFeatureExtractor.from_pretrained(model_name) - - onnx_opset = max(encoder_onnx_config.default_onnx_opset, decoder_onnx_config.default_onnx_opset) - - with NamedTemporaryFile("w") as encoder_output: - onnx_inputs, onnx_outputs = export( - preprocessor, encoder_model, encoder_onnx_config, onnx_opset, Path(encoder_output.name), device=device - ) - validate_model_outputs( - encoder_onnx_config, - preprocessor, - encoder_model, - Path(encoder_output.name), - onnx_outputs, - encoder_onnx_config.atol_for_validation, - ) - - preprocessor = AutoTokenizer.from_pretrained(model_name) - - with NamedTemporaryFile("w") as decoder_output: - _, onnx_outputs = export( - preprocessor, - decoder_model, - decoder_onnx_config, - onnx_config.default_onnx_opset, - Path(decoder_output.name), - device=device, - ) - validate_model_outputs( - decoder_onnx_config, - preprocessor, - decoder_model, - Path(decoder_output.name), - onnx_outputs, - decoder_onnx_config.atol_for_validation, - ) - - @parameterized.expand(_get_models_to_test(PYTORCH_EXPORT_MODELS)) - @slow - @require_torch - @require_vision - @require_rjieba - def test_pytorch_export(self, test_name, name, model_name, feature, onnx_config_class_constructor): - self._onnx_export(test_name, name, model_name, feature, onnx_config_class_constructor) - - @parameterized.expand(_get_models_to_test(PYTORCH_EXPORT_MODELS)) - @slow - @require_torch - @require_vision - @require_rjieba - def test_pytorch_export_on_cuda(self, test_name, name, model_name, feature, onnx_config_class_constructor): - self._onnx_export(test_name, name, model_name, feature, onnx_config_class_constructor, device="cuda") - - @parameterized.expand(_get_models_to_test(PYTORCH_EXPORT_ENCODER_DECODER_MODELS)) - @slow - @require_torch - @require_vision - @require_rjieba - def test_pytorch_export_encoder_decoder_models( - self, test_name, name, model_name, feature, onnx_config_class_constructor - ): - self._onnx_export_encoder_decoder_models(test_name, name, model_name, feature, onnx_config_class_constructor) - - @parameterized.expand(_get_models_to_test(PYTORCH_EXPORT_ENCODER_DECODER_MODELS)) - @slow - @require_torch - @require_vision - @require_rjieba - def test_pytorch_export_encoder_decoder_models_on_cuda( - self, test_name, name, model_name, feature, onnx_config_class_constructor - ): - self._onnx_export_encoder_decoder_models( - test_name, name, model_name, feature, onnx_config_class_constructor, device="cuda" - ) - - @parameterized.expand(_get_models_to_test(PYTORCH_EXPORT_WITH_PAST_MODELS)) - @slow - @require_torch - def test_pytorch_export_with_past(self, test_name, name, model_name, feature, onnx_config_class_constructor): - self._onnx_export(test_name, name, model_name, feature, onnx_config_class_constructor) - - @parameterized.expand(_get_models_to_test(PYTORCH_EXPORT_SEQ2SEQ_WITH_PAST_MODELS)) - @slow - @require_torch - def test_pytorch_export_seq2seq_with_past( - self, test_name, name, model_name, feature, onnx_config_class_constructor - ): - self._onnx_export(test_name, name, model_name, feature, onnx_config_class_constructor) - - @parameterized.expand(_get_models_to_test(TENSORFLOW_EXPORT_DEFAULT_MODELS)) - @slow - @require_tf - @require_vision - def test_tensorflow_export(self, test_name, name, model_name, feature, onnx_config_class_constructor): - self._onnx_export(test_name, name, model_name, feature, onnx_config_class_constructor, framework="tf") - - @parameterized.expand(_get_models_to_test(TENSORFLOW_EXPORT_WITH_PAST_MODELS), skip_on_empty=True) - @slow - @require_tf - def test_tensorflow_export_with_past(self, test_name, name, model_name, feature, onnx_config_class_constructor): - self._onnx_export(test_name, name, model_name, feature, onnx_config_class_constructor, framework="tf") - - @parameterized.expand(_get_models_to_test(TENSORFLOW_EXPORT_SEQ2SEQ_WITH_PAST_MODELS), skip_on_empty=True) - @slow - @require_tf - def test_tensorflow_export_seq2seq_with_past( - self, test_name, name, model_name, feature, onnx_config_class_constructor - ): - self._onnx_export(test_name, name, model_name, feature, onnx_config_class_constructor, framework="tf") - - -class StableDropoutTestCase(TestCase): - """Tests export of StableDropout module.""" - - @unittest.skip("torch 2.0.0 gives `torch.onnx.errors.OnnxExporterError: Module onnx is not installed!`.") - @require_torch - @pytest.mark.filterwarnings("ignore:.*Dropout.*:UserWarning:torch.onnx.*") # torch.onnx is spammy. - def test_training(self): - """Tests export of StableDropout in training mode.""" - devnull = open(os.devnull, "wb") - # drop_prob must be > 0 for the test to be meaningful - sd = modeling_deberta.StableDropout(0.1) - # Avoid warnings in training mode - do_constant_folding = False - # Dropout is a no-op in inference mode - training = torch.onnx.TrainingMode.PRESERVE - input = (torch.randn(2, 2),) - - torch.onnx.export( - sd, - input, - devnull, - opset_version=12, # Minimum supported - do_constant_folding=do_constant_folding, - training=training, - ) - - # Expected to fail with opset_version < 12 - with self.assertRaises(Exception): - torch.onnx.export( - sd, - input, - devnull, - opset_version=11, - do_constant_folding=do_constant_folding, - training=training, - )