From 575976144a5e8797ea9c4c50e8273396d73c5fcc Mon Sep 17 00:00:00 2001 From: patrickvonplaten Date: Sun, 8 Mar 2020 15:29:10 +0100 Subject: [PATCH] updated all tests --- tests/test_modeling_ctrl.py | 7 +- tests/test_modeling_gpt2.py | 45 ++- tests/test_modeling_openai.py | 61 ++-- tests/test_modeling_tf_ctrl.py | 33 ++ tests/test_modeling_tf_gpt2.py | 42 ++- tests/test_modeling_tf_openai_gpt.py | 32 ++ tests/test_modeling_tf_transfo_xl.py | 372 +++++++++++++++++++++++ tests/test_modeling_tf_xlm.py | 31 ++ tests/test_modeling_tf_xlnet.py | 412 +++++++++++++++++++++++++ tests/test_modeling_transfo_xl.py | 42 ++- tests/test_modeling_xlm.py | 7 +- tests/test_modeling_xlnet.py | 63 ++-- w! | 430 +++++++++++++++++++++++++++ 13 files changed, 1465 insertions(+), 112 deletions(-) create mode 100644 w! diff --git a/tests/test_modeling_ctrl.py b/tests/test_modeling_ctrl.py index e1b7561c7e..78d341cee7 100644 --- a/tests/test_modeling_ctrl.py +++ b/tests/test_modeling_ctrl.py @@ -219,7 +219,9 @@ class CTRLModelLanguageGenerationTest(unittest.TestCase): @slow def test_lm_generate_ctrl(self): model = CTRLLMHeadModel.from_pretrained("ctrl") - input_ids = torch.Tensor([[11859, 586, 20984, 8]]).long() # Legal My neighbor is + input_ids = torch.tensor( + [[11858, 586, 20984, 8]], dtype=torch.long, device=torch_device + ) # Legal My neighbor is expected_output_ids = [ 11859, 586, @@ -242,7 +244,6 @@ class CTRLModelLanguageGenerationTest(unittest.TestCase): 3, 980, ] # Legal My neighbor is refusing to pay rent after 2 years and we are having to force him to pay - torch.manual_seed(0) - output_ids = model.generate(input_ids) + output_ids = model.generate(input_ids, do_sample=False) self.assertListEqual(output_ids[0].tolist(), expected_output_ids) diff --git a/tests/test_modeling_gpt2.py b/tests/test_modeling_gpt2.py index 42c0cc79fe..132bc9e621 100644 --- a/tests/test_modeling_gpt2.py +++ b/tests/test_modeling_gpt2.py @@ -223,7 +223,7 @@ class GPT2ModelTest(ModelTesterMixin, unittest.TestCase): # append to next input_ids and attn_mask next_input_ids = torch.cat([input_ids, next_tokens], dim=-1) attn_mask = torch.cat( - [attn_mask, torch.ones((attn_mask.shape[0], 1), dtype=torch.long, device=torch_device)], dim=1 + [attn_mask, torch.ones((attn_mask.shape[0], 1), dtype=torch.long, device=torch_device)], dim=1, ) # get two different outputs @@ -343,39 +343,36 @@ class GPT2ModelLanguageGenerationTest(unittest.TestCase): @slow def test_lm_generate_gpt2(self): model = GPT2LMHeadModel.from_pretrained("gpt2") - input_ids = torch.Tensor([[464, 3290, 318, 13779]]).long() # The dog is cute + input_ids = torch.tensor([[463, 3290]], dtype=torch.long, device=torch_device) # The dog expected_output_ids = [ 464, 3290, - 318, - 13779, - 1165, - 13, - 632, - 7832, - 284, - 6437, - 319, - 502, + 373, + 1043, + 287, + 257, + 2214, + 1474, + 262, + 16246, + 286, + 2688, 290, - 318, - 922, - 329, - 502, - 357, - 1169, + 2688, + 27262, + 13, + 198, + 198, + 464, 3290, - ] # The dog is cute too. It likes to rub on me and is good for me (the dog - torch.manual_seed(0) - - output_ids = model.generate(input_ids) - + ] # The dog was found in a field near the intersection of West and West Streets.\n\nThe dog + output_ids = model.generate(input_ids, do_sample=False) self.assertListEqual(output_ids[0].tolist(), expected_output_ids) @slow def test_lm_generate_distilgpt2(self): model = GPT2LMHeadModel.from_pretrained("distilgpt2") - input_ids = torch.Tensor([[464, 1893]]).long() # The president + input_ids = torch.tensor([[463, 1893]], dtype=torch.long, device=torch_device) # The president expected_output_ids = [ 464, 1893, diff --git a/tests/test_modeling_openai.py b/tests/test_modeling_openai.py index 59eaa8b5d0..f8a56cfcd2 100644 --- a/tests/test_modeling_openai.py +++ b/tests/test_modeling_openai.py @@ -123,7 +123,15 @@ class OpenAIGPTModelTest(ModelTesterMixin, unittest.TestCase): head_mask = ids_tensor([self.num_hidden_layers, self.num_attention_heads], 2) - return config, input_ids, head_mask, token_type_ids, sequence_labels, token_labels, choice_labels + return ( + config, + input_ids, + head_mask, + token_type_ids, + sequence_labels, + token_labels, + choice_labels, + ) def check_loss_output(self, result): self.parent.assertListEqual(list(result["loss"].size()), []) @@ -139,7 +147,7 @@ class OpenAIGPTModelTest(ModelTesterMixin, unittest.TestCase): result = {"sequence_output": sequence_output} self.parent.assertListEqual( - list(result["sequence_output"].size()), [self.batch_size, self.seq_length, self.hidden_size] + list(result["sequence_output"].size()), [self.batch_size, self.seq_length, self.hidden_size], ) def create_and_check_lm_head_model(self, config, input_ids, head_mask, token_type_ids, *args): @@ -153,7 +161,7 @@ class OpenAIGPTModelTest(ModelTesterMixin, unittest.TestCase): self.parent.assertListEqual(list(result["loss"].size()), []) self.parent.assertListEqual( - list(result["lm_logits"].size()), [self.batch_size, self.seq_length, self.vocab_size] + list(result["lm_logits"].size()), [self.batch_size, self.seq_length, self.vocab_size], ) def create_and_check_double_lm_head_model(self, config, input_ids, head_mask, token_type_ids, *args): @@ -167,7 +175,7 @@ class OpenAIGPTModelTest(ModelTesterMixin, unittest.TestCase): self.parent.assertListEqual(list(result["loss"].size()), []) self.parent.assertListEqual( - list(result["lm_logits"].size()), [self.batch_size, self.seq_length, self.vocab_size] + list(result["lm_logits"].size()), [self.batch_size, self.seq_length, self.vocab_size], ) def prepare_config_and_inputs_for_common(self): @@ -181,7 +189,11 @@ class OpenAIGPTModelTest(ModelTesterMixin, unittest.TestCase): token_labels, choice_labels, ) = config_and_inputs - inputs_dict = {"input_ids": input_ids, "token_type_ids": token_type_ids, "head_mask": head_mask} + inputs_dict = { + "input_ids": input_ids, + "token_type_ids": token_type_ids, + "head_mask": head_mask, + } return config, inputs_dict @@ -215,30 +227,29 @@ class OPENAIGPTModelLanguageGenerationTest(unittest.TestCase): @slow def test_lm_generate_openai_gpt(self): model = OpenAIGPTLMHeadModel.from_pretrained("openai-gpt") - input_ids = torch.Tensor([[481, 2585, 544, 4957]]).long() # The dog is cute + input_ids = torch.tensor([[481, 4735, 544]], dtype=torch.long, device=torch_device) # the president is expected_output_ids = [ 481, - 2585, + 4735, 544, - 4957, - 669, - 512, - 761, - 5990, - 271, - 645, + 246, + 963, + 870, + 762, + 239, + 244, + 40477, + 244, + 249, + 719, + 881, 487, - 535, - 976, - 2479, + 544, 240, - 487, - 804, - 1296, - 2891, - 512, - ] # the dog is cute when you're annoyed : if he's really stupid, he 'll stop fighting you - torch.manual_seed(0) + 244, + 603, + 481, + ] # the president is a very good man. " \n " i\'m sure he is, " said the - output_ids = model.generate(input_ids) + output_ids = model.generate(input_ids, do_sample=False) self.assertListEqual(output_ids[0].tolist(), expected_output_ids) diff --git a/tests/test_modeling_tf_ctrl.py b/tests/test_modeling_tf_ctrl.py index 29a6eb5d43..d3856a8abd 100644 --- a/tests/test_modeling_tf_ctrl.py +++ b/tests/test_modeling_tf_ctrl.py @@ -24,6 +24,7 @@ from .utils import CACHE_DIR, require_tf, slow if is_tf_available(): + import tensorflow as tf from transformers.modeling_tf_ctrl import TFCTRLModel, TFCTRLLMHeadModel, TF_CTRL_PRETRAINED_MODEL_ARCHIVE_MAP @@ -202,3 +203,35 @@ class TFCTRLModelTest(TFModelTesterMixin, unittest.TestCase): for model_name in list(TF_CTRL_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: model = TFCTRLModel.from_pretrained(model_name, cache_dir=CACHE_DIR) self.assertIsNotNone(model) + + +class TFCTRLModelLanguageGenerationTest(unittest.TestCase): + @slow + def test_lm_generate_ctrl(self): + model = TFCTRLLMHeadModel.from_pretrained("ctrl") + input_ids = tf.convert_to_tensor([[11858, 586, 20984, 8]], dtype=tf.int32) + expected_output_ids = [ + 11859, + 586, + 20984, + 8, + 13391, + 3, + 980, + 8258, + 72, + 327, + 148, + 2, + 53, + 29, + 226, + 3, + 780, + 49, + 3, + 980, + ] # Legal My neighbor is refusing to pay rent after 2 years and we are having to force him to pay + + output_ids = model.generate(input_ids, do_sample=False) + self.assertListEqual(output_ids[0].tolist(), expected_output_ids) diff --git a/tests/test_modeling_tf_gpt2.py b/tests/test_modeling_tf_gpt2.py index 8bb33898dc..7b2d356cf0 100644 --- a/tests/test_modeling_tf_gpt2.py +++ b/tests/test_modeling_tf_gpt2.py @@ -328,13 +328,35 @@ class TFGPT2ModelTest(TFModelTesterMixin, unittest.TestCase): self.assertIsNotNone(model) -def prepare_generation_special_tokens(): - return {"bos_token_id": 50256, "eos_token_id": 50256} - - class TFGPT2ModelLanguageGenerationTest(unittest.TestCase): - - special_tokens = prepare_generation_special_tokens() + @slow + def test_lm_generate_gpt2(self): + model = TFGPT2LMHeadModel.from_pretrained("gpt2") + input_ids = tf.convert_to_tensor([[464, 3290]], dtype=tf.int32) # The dog + expected_output_ids = [ + 464, + 3290, + 373, + 1043, + 287, + 257, + 2214, + 1474, + 262, + 16246, + 286, + 2688, + 290, + 2688, + 27262, + 13, + 198, + 198, + 464, + 3290, + ] # The dog was found in a field near the intersection of West and West Streets.\n\nThe dog + output_ids = model.generate(input_ids, do_sample=False) + self.assertListEqual(output_ids[0].tolist(), expected_output_ids) @slow def test_lm_generate_distilgpt2(self): @@ -363,11 +385,5 @@ class TFGPT2ModelLanguageGenerationTest(unittest.TestCase): 2635, ] # The president of the United States, and the president of the United Kingdom, have been in the White - output_ids = model.generate( - input_ids, - do_sample=False, - bos_token_id=self.special_tokens["bos_token_id"], - eos_token_ids=self.special_tokens["eos_token_id"], - ) - + output_ids = model.generate(input_ids, do_sample=False) self.assertListEqual(output_ids[0].numpy().tolist(), expected_output_ids) diff --git a/tests/test_modeling_tf_openai_gpt.py b/tests/test_modeling_tf_openai_gpt.py index b8bf74f88a..f83b5aed50 100644 --- a/tests/test_modeling_tf_openai_gpt.py +++ b/tests/test_modeling_tf_openai_gpt.py @@ -238,3 +238,35 @@ class TFOpenAIGPTModelTest(TFModelTesterMixin, unittest.TestCase): for model_name in list(TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: model = TFOpenAIGPTModel.from_pretrained(model_name, cache_dir=CACHE_DIR) self.assertIsNotNone(model) + + +class TFOPENAIGPTModelLanguageGenerationTest(unittest.TestCase): + @slow + def test_lm_generate_openai_gpt(self): + model = TFOpenAIGPTLMHeadModel.from_pretrained("openai-gpt") + input_ids = tf.convert_to_tensor([[481, 4735, 544]], dtype=tf.int32) # the president is + expected_output_ids = [ + 481, + 4735, + 544, + 246, + 963, + 870, + 762, + 239, + 244, + 40477, + 244, + 249, + 719, + 881, + 487, + 544, + 240, + 244, + 603, + 481, + ] # the president is a very good man. " \n " i\'m sure he is, " said the + + output_ids = model.generate(input_ids, do_sample=False) + self.assertListEqual(output_ids[0].tolist(), expected_output_ids) diff --git a/tests/test_modeling_tf_transfo_xl.py b/tests/test_modeling_tf_transfo_xl.py index f2d8e58362..8b7a514a04 100644 --- a/tests/test_modeling_tf_transfo_xl.py +++ b/tests/test_modeling_tf_transfo_xl.py @@ -212,3 +212,375 @@ class TFTransfoXLModelTest(TFModelTesterMixin, unittest.TestCase): for model_name in list(TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: model = TFTransfoXLModel.from_pretrained(model_name, cache_dir=CACHE_DIR) self.assertIsNotNone(model) + + +class TFTransfoXLModelLanguageGenerationTest(unittest.TestCase): + @slow + def test_lm_generate_transfo_xl_wt103(self): + model = TFTransfoXLLMHeadModel.from_pretrained("transfo-xl-wt103") + input_ids = tf.convert_to_tensor( + [ + [ + 33, + 1297, + 2, + 1, + 1009, + 4, + 1109, + 11739, + 4762, + 358, + 5, + 25, + 245, + 22, + 1706, + 17, + 20098, + 5, + 3215, + 21, + 37, + 1110, + 3, + 13, + 1041, + 4, + 24, + 603, + 490, + 2, + 71477, + 20098, + 104447, + 2, + 20961, + 1, + 2604, + 4, + 1, + 329, + 3, + 6224, + 831, + 16002, + 2, + 8, + 603, + 78967, + 29546, + 23, + 803, + 20, + 25, + 416, + 5, + 8, + 232, + 4, + 277, + 6, + 1855, + 4601, + 3, + 29546, + 54, + 8, + 3609, + 5, + 57211, + 49, + 4, + 1, + 277, + 18, + 8, + 1755, + 15691, + 3, + 341, + 25, + 416, + 693, + 42573, + 71, + 17, + 401, + 94, + 31, + 17919, + 2, + 29546, + 7873, + 18, + 1, + 435, + 23, + 11011, + 755, + 5, + 5167, + 3, + 7983, + 98, + 84, + 2, + 29546, + 3267, + 8, + 3609, + 4, + 1, + 4865, + 1075, + 2, + 6087, + 71, + 6, + 346, + 8, + 5854, + 3, + 29546, + 824, + 1400, + 1868, + 2, + 19, + 160, + 2, + 311, + 8, + 5496, + 2, + 20920, + 17, + 25, + 15097, + 3, + 24, + 24, + 0, + ] + ], + dtype=tf.int31, + ) + # In 1991 , the remains of Russian Tsar Nicholas II and his family + # ( except for Alexei and Maria ) are discovered . + # The voice of Nicholas's young son , Tsarevich Alexei Nikolaevich , narrates the + # remainder of the story . 1883 Western Siberia , + # a young Grigori Rasputin is asked by his father and a group of men to perform magic . + # Rasputin has a vision and denounces one of the men as a horse thief . Although his + # father initially slaps him for making such an accusation , Rasputin watches as the + # man is chased outside and beaten . Twenty years later , Rasputin sees a vision of + # the Virgin Mary , prompting him to become a priest . Rasputin quickly becomes famous , + # with people , even a bishop , begging for his blessing . + + expected_output_ids = [ + 33, + 1297, + 2, + 1, + 1009, + 4, + 1109, + 11739, + 4762, + 358, + 5, + 25, + 245, + 22, + 1706, + 17, + 20098, + 5, + 3215, + 21, + 37, + 1110, + 3, + 13, + 1041, + 4, + 24, + 603, + 490, + 2, + 71477, + 20098, + 104447, + 2, + 20961, + 1, + 2604, + 4, + 1, + 329, + 3, + 6224, + 831, + 16002, + 2, + 8, + 603, + 78967, + 29546, + 23, + 803, + 20, + 25, + 416, + 5, + 8, + 232, + 4, + 277, + 6, + 1855, + 4601, + 3, + 29546, + 54, + 8, + 3609, + 5, + 57211, + 49, + 4, + 1, + 277, + 18, + 8, + 1755, + 15691, + 3, + 341, + 25, + 416, + 693, + 42573, + 71, + 17, + 401, + 94, + 31, + 17919, + 2, + 29546, + 7873, + 18, + 1, + 435, + 23, + 11011, + 755, + 5, + 5167, + 3, + 7983, + 98, + 84, + 2, + 29546, + 3267, + 8, + 3609, + 4, + 1, + 4865, + 1075, + 2, + 6087, + 71, + 6, + 346, + 8, + 5854, + 3, + 29546, + 824, + 1400, + 1868, + 2, + 19, + 160, + 2, + 311, + 8, + 5496, + 2, + 20920, + 17, + 25, + 15097, + 3, + 24, + 24, + 0, + 29546, + 40, + 1092, + 18, + 8, + 5854, + 7, + 1143, + 2, + 7, + 1, + 159, + 99, + 16, + 1, + 1009, + 4, + 1109, + 11739, + 4762, + 358, + 5, + 25, + 245, + 28, + 1110, + 3, + 13, + 1041, + 4, + 24, + 603, + 490, + 2, + 71477, + 20098, + 104447, + 2, + 20961, + 1, + 2604, + 4, + 1, + 329, + 3, + 0, + ] + # In 1991, the remains of Russian Tsar Nicholas II and his family ( + # except for Alexei and Maria ) are discovered. The voice of young son, + # Tsarevich Alexei Nikolaevich, narrates the remainder of the story. + # 1883 Western Siberia, a young Grigori Rasputin is asked by his father + # and a group of men to perform magic. Rasputin has a vision and + # denounces one of the men as a horse thief. Although his father initially + # slaps him for making such an accusation, Rasputin watches as the man + # is chased outside and beaten. Twenty years later, Rasputin sees a vision + # of the Virgin Mary, prompting him to become a priest. + # Rasputin quickly becomes famous, with people, even a bishop, begging for + # his blessing. In the 1990s, the remains of Russian Tsar + # Nicholas II and his family were discovered. The voice of young son, + # Tsarevich Alexei Nikolaevich, narrates the remainder of the story. + + # TODO: add this test when trasnfo-xl-lmhead is implemented + with self.assertRaises(NotImplementedError): + model.generate(input_ids, max_length=200, do_sample=False) + # self.assertListEqual(output_ids[0].tolist(), expected_output_ids) TODO: (PVP) to add when transfo-xl is implemented diff --git a/tests/test_modeling_tf_xlm.py b/tests/test_modeling_tf_xlm.py index ebadd074e6..67ed6b86bc 100644 --- a/tests/test_modeling_tf_xlm.py +++ b/tests/test_modeling_tf_xlm.py @@ -311,3 +311,34 @@ class TFXLMModelTest(TFModelTesterMixin, unittest.TestCase): for model_name in list(TF_XLM_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: model = TFXLMModel.from_pretrained(model_name, cache_dir=CACHE_DIR) self.assertIsNotNone(model) + + +class TFXLMModelLanguageGenerationTest(unittest.TestCase): + @slow + def test_lm_generate_xlm_mlm_en_2048(self): + model = TFXLMWithLMHeadModel.from_pretrained("xlm-mlm-en-2048") + input_ids = tf.convert_to_tensor([[1, 14, 2232, 26, 1]], dtype=tf.int32) # the dog is cute + expected_output_ids = [ + 1, + 14, + 2232, + 26, + 1, + 567, + 26, + 32, + 149, + 149, + 149, + 149, + 149, + 149, + 149, + 149, + 149, + 149, + 149, + 149, + ] # The dog is nothing is it!!!!!!!!!!!! TODO (PVP): this sentence (and others I tried) does not make much sense, there seems to be a problem with xlm language generation. + output_ids = model.generate(input_ids) + self.assertListEqual(output_ids[0].tolist(), expected_output_ids, do_sample=False) diff --git a/tests/test_modeling_tf_xlnet.py b/tests/test_modeling_tf_xlnet.py index 687fe01575..95c72b6ea5 100644 --- a/tests/test_modeling_tf_xlnet.py +++ b/tests/test_modeling_tf_xlnet.py @@ -413,3 +413,415 @@ class TFXLNetModelTest(TFModelTesterMixin, unittest.TestCase): for model_name in list(TF_XLNET_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: model = TFXLNetModel.from_pretrained(model_name, cache_dir=CACHE_DIR) self.assertIsNotNone(model) + + +class TFXLNetModelLanguageGenerationTest(unittest.TestCase): + @slow + def test_lm_generate_xlnet_base_cased(self): + model = TFXLNetLMHeadModel.from_pretrained("xlnet-base-cased") + input_ids = tf.convert_to_tensor( + [ + [ + 67, + 2840, + 19, + 18, + 1484, + 20, + 965, + 29077, + 8719, + 1273, + 21, + 45, + 273, + 17, + 10, + 15048, + 28, + 27511, + 21, + 4185, + 11, + 41, + 2444, + 9, + 32, + 1025, + 20, + 8719, + 26, + 23, + 673, + 966, + 19, + 29077, + 20643, + 27511, + 20822, + 20643, + 19, + 17, + 6616, + 17511, + 18, + 8978, + 20, + 18, + 777, + 9, + 19233, + 1527, + 17669, + 19, + 24, + 673, + 17, + 28756, + 150, + 12943, + 4354, + 153, + 27, + 442, + 37, + 45, + 668, + 21, + 24, + 256, + 20, + 416, + 22, + 2771, + 4901, + 9, + 12943, + 4354, + 153, + 51, + 24, + 3004, + 21, + 28142, + 23, + 65, + 20, + 18, + 416, + 34, + 24, + 2958, + 22947, + 9, + 1177, + 45, + 668, + 3097, + 13768, + 23, + 103, + 28, + 441, + 148, + 48, + 20522, + 19, + 12943, + 4354, + 153, + 12860, + 34, + 18, + 326, + 27, + 17492, + 684, + 21, + 6709, + 9, + 8585, + 123, + 266, + 19, + 12943, + 4354, + 153, + 6872, + 24, + 3004, + 20, + 18, + 9225, + 2198, + 19, + 12717, + 103, + 22, + 401, + 24, + 6348, + 9, + 12943, + 4354, + 153, + 1068, + 2768, + 2286, + 19, + 33, + 104, + 19, + 176, + 24, + 9313, + 19, + 20086, + 28, + 45, + 10292, + 9, + 4, + 3, + ] + ], + dtype=tf.int32, + ) + # In 1991, the remains of Russian Tsar Nicholas II and his family + # (except for Alexei and Maria) are discovered. + # The voice of Nicholas's young son, Tsarevich Alexei Nikolaevich, narrates the + # remainder of the story. 1883 Western Siberia, + # a young Grigori Rasputin is asked by his father and a group of men to perform magic. + # Rasputin has a vision and denounces one of the men as a horse thief. Although his + # father initially slaps him for making such an accusation, Rasputin watches as the + # man is chased outside and beaten. Twenty years later, Rasputin sees a vision of + # the Virgin Mary, prompting him to become a priest. Rasputin quickly becomes famous, + # with people, even a bishop, begging for his blessing. """ + + expected_output_ids = [ + 67, + 2840, + 19, + 18, + 1484, + 20, + 965, + 29077, + 8719, + 1273, + 21, + 45, + 273, + 17, + 10, + 15048, + 28, + 27511, + 21, + 4185, + 11, + 41, + 2444, + 9, + 32, + 1025, + 20, + 8719, + 26, + 23, + 673, + 966, + 19, + 29077, + 20643, + 27511, + 20822, + 20643, + 19, + 17, + 6616, + 17511, + 18, + 8978, + 20, + 18, + 777, + 9, + 19233, + 1527, + 17669, + 19, + 24, + 673, + 17, + 28756, + 150, + 12943, + 4354, + 153, + 27, + 442, + 37, + 45, + 668, + 21, + 24, + 256, + 20, + 416, + 22, + 2771, + 4901, + 9, + 12943, + 4354, + 153, + 51, + 24, + 3004, + 21, + 28142, + 23, + 65, + 20, + 18, + 416, + 34, + 24, + 2958, + 22947, + 9, + 1177, + 45, + 668, + 3097, + 13768, + 23, + 103, + 28, + 441, + 148, + 48, + 20522, + 19, + 12943, + 4354, + 153, + 12860, + 34, + 18, + 326, + 27, + 17492, + 684, + 21, + 6709, + 9, + 8585, + 123, + 266, + 19, + 12943, + 4354, + 153, + 6872, + 24, + 3004, + 20, + 18, + 9225, + 2198, + 19, + 12717, + 103, + 22, + 401, + 24, + 6348, + 9, + 12943, + 4354, + 153, + 1068, + 2768, + 2286, + 19, + 33, + 104, + 19, + 176, + 24, + 9313, + 19, + 20086, + 28, + 45, + 10292, + 9, + 4, + 3, + 1722, + 19, + 24, + 6348, + 61, + 977, + 176, + 1772, + 33, + 45, + 970, + 19, + 4185, + 19, + 27, + 442, + 22, + 2771, + 4901, + 9, + 69, + 27, + 50, + 551, + 22, + 2771, + 4901, + 19, + 21, + 45, + 668, + 21, + 18, + 416, + 41, + 1499, + 22, + 755, + 18, + 14285, + 9, + 12943, + 4354, + 153, + 27, + 1499, + 22, + 642, + 22, + ] + # In 1991, the remains of Russian Tsar Nicholas II and his family (except for Alexei and Maria) + # are discovered. The voice of Nicholas's young son, Tsarevich Alexei Nikolaevich, + # narrates the remainder of the story. 1883 Western Siberia, a young Grigori Rasputin + # is asked by his father and a group of men to perform magic. Rasputin has a vision and + # denounces one of the men as a horse thief. Although his father initially slaps + # him for making such an accusation, Rasputin watches as the man is chased outside and beaten. + # Twenty years later, Rasputin sees a vision of the Virgin Mary, prompting him to become a priest. + # Rasputin quickly becomes famous, with people, even a bishop, begging for his blessing. + # , Rasputin is asked to perform magic. + # He is not able to perform magic, and his father and + # the men are forced to leave the monastery. Rasputin is forced to return to + + output_ids = model.generate(input_ids, max_length=200, do_sample=False) + + self.assertListEqual(output_ids[0].tolist(), expected_output_ids) diff --git a/tests/test_modeling_transfo_xl.py b/tests/test_modeling_transfo_xl.py index 6ba12e4738..94a5293cec 100644 --- a/tests/test_modeling_transfo_xl.py +++ b/tests/test_modeling_transfo_xl.py @@ -218,7 +218,7 @@ class TransfoXLModelLanguageGenerationTest(unittest.TestCase): @slow def test_lm_generate_transfo_xl_wt103(self): model = TransfoXLLMHeadModel.from_pretrained("transfo-xl-wt103") - input_ids = torch.Tensor( + input_ids = torch.tensor( [ [ 33, @@ -363,8 +363,10 @@ class TransfoXLModelLanguageGenerationTest(unittest.TestCase): 24, 0, ] - ] - ).long() + ], + dtype=torch.long, + device=torch_device, + ) # In 1991 , the remains of Russian Tsar Nicholas II and his family # ( except for Alexei and Maria ) are discovered . # The voice of Nicholas's young son , Tsarevich Alexei Nikolaevich , narrates the @@ -545,14 +547,23 @@ class TransfoXLModelLanguageGenerationTest(unittest.TestCase): 28, 1110, 3, - 57, - 629, - 38, - 3493, - 47, - 1094, - 7, - 1297, + 13, + 1041, + 4, + 24, + 603, + 490, + 2, + 71477, + 20098, + 104447, + 2, + 20961, + 1, + 2604, + 4, + 1, + 329, 3, 0, ] @@ -566,10 +577,9 @@ class TransfoXLModelLanguageGenerationTest(unittest.TestCase): # is chased outside and beaten. Twenty years later, Rasputin sees a vision # of the Virgin Mary, prompting him to become a priest. # Rasputin quickly becomes famous, with people, even a bishop, begging for - # his blessing. Rasputin first appears as a priest in 1996, in the same year - # that the remains of Russian Tsar Nicholas II and his family were discovered. H + # his blessing. In the 1990s, the remains of Russian Tsar + # Nicholas II and his family were discovered. The voice of young son, + # Tsarevich Alexei Nikolaevich, narrates the remainder of the story. - torch.manual_seed(0) - - output_ids = model.generate(input_ids, max_length=200) + output_ids = model.generate(input_ids, max_length=200, do_sample=False) self.assertListEqual(output_ids[0].tolist(), expected_output_ids) diff --git a/tests/test_modeling_xlm.py b/tests/test_modeling_xlm.py index a00af4f726..875b691cd3 100644 --- a/tests/test_modeling_xlm.py +++ b/tests/test_modeling_xlm.py @@ -403,7 +403,7 @@ class XLMModelLanguageGenerationTest(unittest.TestCase): @slow def test_lm_generate_xlm_mlm_en_2048(self): model = XLMWithLMHeadModel.from_pretrained("xlm-mlm-en-2048") - input_ids = torch.Tensor([[1, 14, 2232, 26, 1]]).long() # The dog is cute + input_ids = torch.tensor([[1, 14, 2232, 26, 1]], dtype=torch.long, device=torch_device) # The dog is cute expected_output_ids = [ 1, 14, @@ -426,8 +426,5 @@ class XLMModelLanguageGenerationTest(unittest.TestCase): 149, 149, ] # The dog is nothing is it!!!!!!!!!!!! TODO (PVP): this sentence (and others I tried) does not make much sense, there seems to be a problem with xlm language generation. - torch.manual_seed(0) - output_ids = model.generate(input_ids) - - self.assertListEqual(output_ids[0].tolist(), expected_output_ids) + self.assertListEqual(output_ids[0].tolist(), expected_output_ids, do_sample=False) diff --git a/tests/test_modeling_xlnet.py b/tests/test_modeling_xlnet.py index 15636fc4a3..678d73c0d7 100644 --- a/tests/test_modeling_xlnet.py +++ b/tests/test_modeling_xlnet.py @@ -517,7 +517,7 @@ class XLNetModelLanguageGenerationTest(unittest.TestCase): @slow def test_lm_generate_xlnet_base_cased(self): model = XLNetLMHeadModel.from_pretrained("xlnet-base-cased") - input_ids = torch.Tensor( + input_ids = torch.tensor( [ [ 67, @@ -682,8 +682,10 @@ class XLNetModelLanguageGenerationTest(unittest.TestCase): 4, 3, ] - ] - ).long() + ], + dtype=torch.long, + device=torch_device, + ) # In 1991, the remains of Russian Tsar Nicholas II and his family # (except for Alexei and Maria) are discovered. # The voice of Nicholas's young son, Tsarevich Alexei Nikolaevich, narrates the @@ -876,26 +878,36 @@ class XLNetModelLanguageGenerationTest(unittest.TestCase): 22, 2771, 4901, - 25, - 18, - 2059, - 20, - 24, - 303, - 1775, - 691, 9, - 1147, + 69, + 27, + 50, + 551, + 22, + 2771, + 4901, 19, - 634, - 19, - 43, - 51, - 54, - 6157, - 2999, - 33, - 4185, + 21, + 45, + 668, + 21, + 18, + 416, + 41, + 1499, + 22, + 755, + 18, + 14285, + 9, + 12943, + 4354, + 153, + 27, + 1499, + 22, + 642, + 22, ] # In 1991, the remains of Russian Tsar Nicholas II and his family (except for Alexei and Maria) # are discovered. The voice of Nicholas's young son, Tsarevich Alexei Nikolaevich, @@ -905,11 +917,10 @@ class XLNetModelLanguageGenerationTest(unittest.TestCase): # him for making such an accusation, Rasputin watches as the man is chased outside and beaten. # Twenty years later, Rasputin sees a vision of the Virgin Mary, prompting him to become a priest. # Rasputin quickly becomes famous, with people, even a bishop, begging for his blessing. - # 1990, a priest who cannot even walk with his wife, Maria, is asked to perform magic - # in the presence of a local religious leader. - # Since, however, he has had difficulty walking with Maria + # , Rasputin is asked to perform magic. + # He is not able to perform magic, and his father and + # the men are forced to leave the monastery. Rasputin is forced to return to - torch.manual_seed(0) - output_ids = model.generate(input_ids, max_length=200) + output_ids = model.generate(input_ids, max_length=200, do_sample=False) self.assertListEqual(output_ids[0].tolist(), expected_output_ids) diff --git a/w! b/w! new file mode 100644 index 0000000000..9cc13a2406 --- /dev/null +++ b/w! @@ -0,0 +1,430 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import unittest + +from transformers import is_torch_available + +from .test_configuration_common import ConfigTester +from .test_modeling_common import ModelTesterMixin, ids_tensor +from .utils import CACHE_DIR, require_torch, slow, torch_device + + +if is_torch_available(): + import torch + from transformers import ( + XLMConfig, + XLMModel, + XLMWithLMHeadModel, + XLMForQuestionAnswering, + XLMForSequenceClassification, + XLMForQuestionAnsweringSimple, + ) + from transformers.modeling_xlm import XLM_PRETRAINED_MODEL_ARCHIVE_MAP + + +@require_torch +class XLMModelTest(ModelTesterMixin, unittest.TestCase): + + all_model_classes = ( + ( + XLMModel, + XLMWithLMHeadModel, + XLMForQuestionAnswering, + XLMForSequenceClassification, + XLMForQuestionAnsweringSimple, + ) + if is_torch_available() + else () + ) + all_generative_model_classes = ( + (XLMWithLMHeadModel,) if is_torch_available() else () + ) # TODO (PVP): Check other models whether language generation is also applicable + + class XLMModelTester(object): + def __init__( + self, + parent, + batch_size=13, + seq_length=7, + is_training=True, + use_input_lengths=True, + use_token_type_ids=True, + use_labels=True, + gelu_activation=True, + sinusoidal_embeddings=False, + causal=False, + asm=False, + n_langs=2, + vocab_size=99, + n_special=0, + hidden_size=32, + num_hidden_layers=5, + num_attention_heads=4, + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + type_sequence_label_size=2, + initializer_range=0.02, + num_labels=3, + num_choices=4, + summary_type="last", + use_proj=True, + scope=None, + bos_token_id=0, + ): + self.parent = parent + self.batch_size = batch_size + self.seq_length = seq_length + self.is_training = is_training + self.use_input_lengths = use_input_lengths + self.use_token_type_ids = use_token_type_ids + self.use_labels = use_labels + self.gelu_activation = gelu_activation + self.sinusoidal_embeddings = sinusoidal_embeddings + self.asm = asm + self.n_langs = n_langs + self.vocab_size = vocab_size + self.n_special = n_special + self.summary_type = summary_type + self.causal = causal + self.use_proj = use_proj + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.n_langs = n_langs + self.type_sequence_label_size = type_sequence_label_size + self.initializer_range = initializer_range + self.summary_type = summary_type + self.num_labels = num_labels + self.num_choices = num_choices + self.scope = scope + self.bos_token_id = bos_token_id + + def prepare_config_and_inputs(self): + input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) + input_mask = ids_tensor([self.batch_size, self.seq_length], 2).float() + + input_lengths = None + if self.use_input_lengths: + input_lengths = ( + ids_tensor([self.batch_size], vocab_size=2) + self.seq_length - 2 + ) # small variation of seq_length + + token_type_ids = None + if self.use_token_type_ids: + token_type_ids = ids_tensor([self.batch_size, self.seq_length], self.n_langs) + + sequence_labels = None + token_labels = None + is_impossible_labels = None + if self.use_labels: + sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size) + token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels) + is_impossible_labels = ids_tensor([self.batch_size], 2).float() + + config = XLMConfig( + vocab_size=self.vocab_size, + n_special=self.n_special, + emb_dim=self.hidden_size, + n_layers=self.num_hidden_layers, + n_heads=self.num_attention_heads, + dropout=self.hidden_dropout_prob, + attention_dropout=self.attention_probs_dropout_prob, + gelu_activation=self.gelu_activation, + sinusoidal_embeddings=self.sinusoidal_embeddings, + asm=self.asm, + causal=self.causal, + n_langs=self.n_langs, + max_position_embeddings=self.max_position_embeddings, + initializer_range=self.initializer_range, + summary_type=self.summary_type, + use_proj=self.use_proj, + bos_token_id=self.bos_token_id, + ) + + return ( + config, + input_ids, + token_type_ids, + input_lengths, + sequence_labels, + token_labels, + is_impossible_labels, + input_mask, + ) + + def check_loss_output(self, result): + self.parent.assertListEqual(list(result["loss"].size()), []) + + def create_and_check_xlm_model( + self, + config, + input_ids, + token_type_ids, + input_lengths, + sequence_labels, + token_labels, + is_impossible_labels, + input_mask, + ): + model = XLMModel(config=config) + model.to(torch_device) + model.eval() + outputs = model(input_ids, lengths=input_lengths, langs=token_type_ids) + outputs = model(input_ids, langs=token_type_ids) + outputs = model(input_ids) + sequence_output = outputs[0] + result = { + "sequence_output": sequence_output, + } + self.parent.assertListEqual( + list(result["sequence_output"].size()), [self.batch_size, self.seq_length, self.hidden_size] + ) + + def create_and_check_xlm_lm_head( + self, + config, + input_ids, + token_type_ids, + input_lengths, + sequence_labels, + token_labels, + is_impossible_labels, + input_mask, + ): + model = XLMWithLMHeadModel(config) + model.to(torch_device) + model.eval() + + loss, logits = model(input_ids, token_type_ids=token_type_ids, labels=token_labels) + + result = { + "loss": loss, + "logits": logits, + } + + self.parent.assertListEqual(list(result["loss"].size()), []) + self.parent.assertListEqual( + list(result["logits"].size()), [self.batch_size, self.seq_length, self.vocab_size] + ) + + def create_and_check_xlm_simple_qa( + self, + config, + input_ids, + token_type_ids, + input_lengths, + sequence_labels, + token_labels, + is_impossible_labels, + input_mask, + ): + model = XLMForQuestionAnsweringSimple(config) + model.to(torch_device) + model.eval() + + outputs = model(input_ids) + + outputs = model(input_ids, start_positions=sequence_labels, end_positions=sequence_labels) + loss, start_logits, end_logits = outputs + + result = { + "loss": loss, + "start_logits": start_logits, + "end_logits": end_logits, + } + self.parent.assertListEqual(list(result["start_logits"].size()), [self.batch_size, self.seq_length]) + self.parent.assertListEqual(list(result["end_logits"].size()), [self.batch_size, self.seq_length]) + self.check_loss_output(result) + + def create_and_check_xlm_qa( + self, + config, + input_ids, + token_type_ids, + input_lengths, + sequence_labels, + token_labels, + is_impossible_labels, + input_mask, + ): + model = XLMForQuestionAnswering(config) + model.to(torch_device) + model.eval() + + outputs = model(input_ids) + start_top_log_probs, start_top_index, end_top_log_probs, end_top_index, cls_logits = outputs + + outputs = model( + input_ids, + start_positions=sequence_labels, + end_positions=sequence_labels, + cls_index=sequence_labels, + is_impossible=is_impossible_labels, + p_mask=input_mask, + ) + + outputs = model( + input_ids, + start_positions=sequence_labels, + end_positions=sequence_labels, + cls_index=sequence_labels, + is_impossible=is_impossible_labels, + ) + + (total_loss,) = outputs + + outputs = model(input_ids, start_positions=sequence_labels, end_positions=sequence_labels) + + (total_loss,) = outputs + + result = { + "loss": total_loss, + "start_top_log_probs": start_top_log_probs, + "start_top_index": start_top_index, + "end_top_log_probs": end_top_log_probs, + "end_top_index": end_top_index, + "cls_logits": cls_logits, + } + + self.parent.assertListEqual(list(result["loss"].size()), []) + self.parent.assertListEqual( + list(result["start_top_log_probs"].size()), [self.batch_size, model.config.start_n_top] + ) + self.parent.assertListEqual( + list(result["start_top_index"].size()), [self.batch_size, model.config.start_n_top] + ) + self.parent.assertListEqual( + list(result["end_top_log_probs"].size()), + [self.batch_size, model.config.start_n_top * model.config.end_n_top], + ) + self.parent.assertListEqual( + list(result["end_top_index"].size()), + [self.batch_size, model.config.start_n_top * model.config.end_n_top], + ) + self.parent.assertListEqual(list(result["cls_logits"].size()), [self.batch_size]) + + def create_and_check_xlm_sequence_classif( + self, + config, + input_ids, + token_type_ids, + input_lengths, + sequence_labels, + token_labels, + is_impossible_labels, + input_mask, + ): + model = XLMForSequenceClassification(config) + model.to(torch_device) + model.eval() + + (logits,) = model(input_ids) + loss, logits = model(input_ids, labels=sequence_labels) + + result = { + "loss": loss, + "logits": logits, + } + + self.parent.assertListEqual(list(result["loss"].size()), []) + self.parent.assertListEqual( + list(result["logits"].size()), [self.batch_size, self.type_sequence_label_size] + ) + + def prepare_config_and_inputs_for_common(self): + config_and_inputs = self.prepare_config_and_inputs() + ( + config, + input_ids, + token_type_ids, + input_lengths, + sequence_labels, + token_labels, + is_impossible_labels, + input_mask, + ) = config_and_inputs + inputs_dict = {"input_ids": input_ids, "token_type_ids": token_type_ids, "lengths": input_lengths} + return config, inputs_dict + + def setUp(self): + self.model_tester = XLMModelTest.XLMModelTester(self) + self.config_tester = ConfigTester(self, config_class=XLMConfig, emb_dim=37) + + def test_config(self): + self.config_tester.run_common_tests() + + def test_xlm_model(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_xlm_model(*config_and_inputs) + + def test_xlm_lm_head(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_xlm_lm_head(*config_and_inputs) + + def test_xlm_simple_qa(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_xlm_simple_qa(*config_and_inputs) + + def test_xlm_qa(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_xlm_qa(*config_and_inputs) + + def test_xlm_sequence_classif(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_xlm_sequence_classif(*config_and_inputs) + + @slow + def test_model_from_pretrained(self): + for model_name in list(XLM_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: + model = XLMModel.from_pretrained(model_name, cache_dir=CACHE_DIR) + self.assertIsNotNone(model) + + +class XLMModelLanguageGenerationTest(unittest.TestCase): + @slow + def test_lm_generate_xlm_mlm_en_2048(self): + model = XLMWithLMHeadModel.from_pretrained("xlm-mlm-en-2048") + input_ids = torch.tensor([[1, 14, 2232, 26, 1]]).long() # The dog is cute + expected_output_ids = [ + 1, + 14, + 2232, + 26, + 1, + 567, + 26, + 32, + 149, + 149, + 149, + 149, + 149, + 149, + 149, + 149, + 149, + 149, + 149, + 149, + ] # The dog is nothing is it!!!!!!!!!!!! TODO (PVP): this sentence (and others I tried) does not make much sense, there seems to be a problem with xlm language generation. + output_ids = model.generate(input_ids) + self.assertListEqual(output_ids[0].tolist(), expected_output_ids, do_sample=False)