From 6e1e88fd38fe9b1d294a4782c63aeb047362a1fe Mon Sep 17 00:00:00 2001 From: lewtun Date: Mon, 14 Mar 2022 08:40:42 +0100 Subject: [PATCH] Add TFCamembertForCausalLM and ONNX integration test (#16073) * Make Camembert great again! * Add Camembert to TensorFlow ONNX tests --- docs/source/model_doc/camembert.mdx | 4 ++++ src/transformers/__init__.py | 2 ++ src/transformers/models/auto/modeling_tf_auto.py | 1 + src/transformers/models/camembert/__init__.py | 2 ++ .../models/camembert/modeling_tf_camembert.py | 13 +++++++++++++ src/transformers/utils/dummy_tf_objects.py | 7 +++++++ tests/onnx/test_onnx_v2.py | 1 + 7 files changed, 30 insertions(+) diff --git a/docs/source/model_doc/camembert.mdx b/docs/source/model_doc/camembert.mdx index 448c35cdf5..a35d5aefca 100644 --- a/docs/source/model_doc/camembert.mdx +++ b/docs/source/model_doc/camembert.mdx @@ -85,6 +85,10 @@ This model was contributed by [camembert](https://huggingface.co/camembert). The [[autodoc]] TFCamembertModel +## TFCamembertForCasualLM + +[[autodoc]] TFCamembertForCausalLM + ## TFCamembertForMaskedLM [[autodoc]] TFCamembertForMaskedLM diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index e357e0b15a..19dddc1b38 100755 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -1744,6 +1744,7 @@ if is_tf_available(): _import_structure["models.camembert"].extend( [ "TF_CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST", + "TFCamembertForCausalLM", "TFCamembertForMaskedLM", "TFCamembertForMultipleChoice", "TFCamembertForQuestionAnswering", @@ -3812,6 +3813,7 @@ if TYPE_CHECKING: ) from .models.camembert import ( TF_CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST, + TFCamembertForCausalLM, TFCamembertForMaskedLM, TFCamembertForMultipleChoice, TFCamembertForQuestionAnswering, diff --git a/src/transformers/models/auto/modeling_tf_auto.py b/src/transformers/models/auto/modeling_tf_auto.py index 1b95cfa01d..34f59393af 100644 --- a/src/transformers/models/auto/modeling_tf_auto.py +++ b/src/transformers/models/auto/modeling_tf_auto.py @@ -139,6 +139,7 @@ TF_MODEL_WITH_LM_HEAD_MAPPING_NAMES = OrderedDict( TF_MODEL_FOR_CAUSAL_LM_MAPPING_NAMES = OrderedDict( [ # Model for Causal LM mapping + ("camembert", "TFCamembertForCausalLM"), ("rembert", "TFRemBertForCausalLM"), ("roformer", "TFRoFormerForCausalLM"), ("roberta", "TFRobertaForCausalLM"), diff --git a/src/transformers/models/camembert/__init__.py b/src/transformers/models/camembert/__init__.py index 3eb99ad948..7be6e7fbb6 100644 --- a/src/transformers/models/camembert/__init__.py +++ b/src/transformers/models/camembert/__init__.py @@ -52,6 +52,7 @@ if is_torch_available(): if is_tf_available(): _import_structure["modeling_tf_camembert"] = [ "TF_CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST", + "TFCamembertForCausalLM", "TFCamembertForMaskedLM", "TFCamembertForMultipleChoice", "TFCamembertForQuestionAnswering", @@ -85,6 +86,7 @@ if TYPE_CHECKING: if is_tf_available(): from .modeling_tf_camembert import ( TF_CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST, + TFCamembertForCausalLM, TFCamembertForMaskedLM, TFCamembertForMultipleChoice, TFCamembertForQuestionAnswering, diff --git a/src/transformers/models/camembert/modeling_tf_camembert.py b/src/transformers/models/camembert/modeling_tf_camembert.py index b46246465b..9e04d95be6 100644 --- a/src/transformers/models/camembert/modeling_tf_camembert.py +++ b/src/transformers/models/camembert/modeling_tf_camembert.py @@ -18,6 +18,7 @@ from ...file_utils import add_start_docstrings from ...utils import logging from ..roberta.modeling_tf_roberta import ( + TFRobertaForCausalLM, TFRobertaForMaskedLM, TFRobertaForMultipleChoice, TFRobertaForQuestionAnswering, @@ -161,3 +162,15 @@ class TFCamembertForQuestionAnswering(TFRobertaForQuestionAnswering): """ config_class = CamembertConfig + + +@add_start_docstrings( + """CamemBERT Model with a `language modeling` head on top for CLM fine-tuning.""", CAMEMBERT_START_DOCSTRING +) +class TFCamembertForCausalLM(TFRobertaForCausalLM): + """ + This class overrides [`TFRobertaForCausalLM`]. Please check the superclass for the appropriate documentation + alongside usage examples. + """ + + config_class = CamembertConfig diff --git a/src/transformers/utils/dummy_tf_objects.py b/src/transformers/utils/dummy_tf_objects.py index 2949532101..631b4a9a5d 100644 --- a/src/transformers/utils/dummy_tf_objects.py +++ b/src/transformers/utils/dummy_tf_objects.py @@ -537,6 +537,13 @@ class TFBlenderbotSmallPreTrainedModel(metaclass=DummyObject): TF_CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST = None +class TFCamembertForCausalLM(metaclass=DummyObject): + _backends = ["tf"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["tf"]) + + class TFCamembertForMaskedLM(metaclass=DummyObject): _backends = ["tf"] diff --git a/tests/onnx/test_onnx_v2.py b/tests/onnx/test_onnx_v2.py index 26ef4370e2..e6076dd076 100644 --- a/tests/onnx/test_onnx_v2.py +++ b/tests/onnx/test_onnx_v2.py @@ -200,6 +200,7 @@ PYTORCH_EXPORT_SEQ2SEQ_WITH_PAST_MODELS = { TENSORFLOW_EXPORT_DEFAULT_MODELS = { ("albert", "hf-internal-testing/tiny-albert"), ("bert", "bert-base-cased"), + ("camembert", "camembert-base"), ("distilbert", "distilbert-base-cased"), ("roberta", "roberta-base"), }