From cbf4391177dc2d5c5589be780404261307f777bb Mon Sep 17 00:00:00 2001
From: Patrick von Platen <patrick.v.platen@gmail.com>
Date: Thu, 24 Feb 2022 19:23:34 +0100
Subject: [PATCH] [TFXLNet] Correct tf xlnet generate (#15822)

* [TFXLNet] Correct tf xlnet

* adapt test comment
---
 tests/xlnet/test_modeling_tf_xlnet.py | 373 +-------------------------
 tests/xlnet/test_modeling_xlnet.py    | 367 +------------------------
 2 files changed, 15 insertions(+), 725 deletions(-)

diff --git a/tests/xlnet/test_modeling_tf_xlnet.py b/tests/xlnet/test_modeling_tf_xlnet.py
index 6dde2c0518..4b92581a0e 100644
--- a/tests/xlnet/test_modeling_tf_xlnet.py
+++ b/tests/xlnet/test_modeling_tf_xlnet.py
@@ -461,174 +461,17 @@ class TFXLNetModelLanguageGenerationTest(unittest.TestCase):
     @slow
     def test_lm_generate_xlnet_base_cased(self):
         model = TFXLNetLMHeadModel.from_pretrained("xlnet-base-cased")
+        # fmt: off
         input_ids = tf.convert_to_tensor(
             [
                 [
-                    67,
-                    2840,
-                    19,
-                    18,
-                    1484,
-                    20,
-                    965,
-                    29077,
-                    8719,
-                    1273,
-                    21,
-                    45,
-                    273,
-                    17,
-                    10,
-                    15048,
-                    28,
-                    27511,
-                    21,
-                    4185,
-                    11,
-                    41,
-                    2444,
-                    9,
-                    32,
-                    1025,
-                    20,
-                    8719,
-                    26,
-                    23,
-                    673,
-                    966,
-                    19,
-                    29077,
-                    20643,
-                    27511,
-                    20822,
-                    20643,
-                    19,
-                    17,
-                    6616,
-                    17511,
-                    18,
-                    8978,
-                    20,
-                    18,
-                    777,
-                    9,
-                    19233,
-                    1527,
-                    17669,
-                    19,
-                    24,
-                    673,
-                    17,
-                    28756,
-                    150,
-                    12943,
-                    4354,
-                    153,
-                    27,
-                    442,
-                    37,
-                    45,
-                    668,
-                    21,
-                    24,
-                    256,
-                    20,
-                    416,
-                    22,
-                    2771,
-                    4901,
-                    9,
-                    12943,
-                    4354,
-                    153,
-                    51,
-                    24,
-                    3004,
-                    21,
-                    28142,
-                    23,
-                    65,
-                    20,
-                    18,
-                    416,
-                    34,
-                    24,
-                    2958,
-                    22947,
-                    9,
-                    1177,
-                    45,
-                    668,
-                    3097,
-                    13768,
-                    23,
-                    103,
-                    28,
-                    441,
-                    148,
-                    48,
-                    20522,
-                    19,
-                    12943,
-                    4354,
-                    153,
-                    12860,
-                    34,
-                    18,
-                    326,
-                    27,
-                    17492,
-                    684,
-                    21,
-                    6709,
-                    9,
-                    8585,
-                    123,
-                    266,
-                    19,
-                    12943,
-                    4354,
-                    153,
-                    6872,
-                    24,
-                    3004,
-                    20,
-                    18,
-                    9225,
-                    2198,
-                    19,
-                    12717,
-                    103,
-                    22,
-                    401,
-                    24,
-                    6348,
-                    9,
-                    12943,
-                    4354,
-                    153,
-                    1068,
-                    2768,
-                    2286,
-                    19,
-                    33,
-                    104,
-                    19,
-                    176,
-                    24,
-                    9313,
-                    19,
-                    20086,
-                    28,
-                    45,
-                    10292,
-                    9,
-                    4,
-                    3,
+                    67, 2840, 19, 18, 1484, 20, 965, 29077, 8719, 1273, 21, 45, 273, 17, 10, 15048, 28, 27511, 21, 4185, 11, 41, 2444, 9, 32, 1025, 20, 8719, 26, 23, 673, 966, 19, 29077, 20643, 27511, 20822, 20643, 19, 17, 6616, 17511, 18, 8978, 20, 18, 777, 9, 19233, 1527, 17669, 19, 24, 673, 17, 28756, 150, 12943, 4354, 153, 27, 442, 37, 45, 668, 21, 24, 256, 20, 416, 22, 2771, 4901, 9, 12943, 4354, 153, 51, 24, 3004, 21, 28142, 23, 65, 20, 18, 416, 34, 24, 2958, 22947, 9, 1177, 45, 668, 3097, 13768, 23, 103, 28, 441, 148, 48, 20522, 19, 12943, 4354, 153, 12860, 34, 18, 326, 27, 17492, 684, 21, 6709, 9, 8585, 123, 266, 19, 12943, 4354, 153, 6872, 24, 3004, 20, 18, 9225, 2198, 19, 12717, 103, 22, 401, 24, 6348, 9, 12943, 4354, 153, 1068, 2768, 2286, 19, 33, 104, 19, 176, 24, 9313, 19, 20086, 28, 45, 10292, 9, 4, 3,
                 ]
             ],
             dtype=tf.int32,
         )
+        # fmt: on
+
         #  In 1991, the remains of Russian Tsar Nicholas II and his family
         #  (except for Alexei and Maria) are discovered.
         #  The voice of Nicholas's young son, Tsarevich Alexei Nikolaevich, narrates the
@@ -640,208 +483,11 @@ class TFXLNetModelLanguageGenerationTest(unittest.TestCase):
         #  the Virgin Mary, prompting him to become a priest. Rasputin quickly becomes famous,
         #  with people, even a bishop, begging for his blessing. """
 
+        # fmt: off
         expected_output_ids = [
-            67,
-            2840,
-            19,
-            18,
-            1484,
-            20,
-            965,
-            29077,
-            8719,
-            1273,
-            21,
-            45,
-            273,
-            17,
-            10,
-            15048,
-            28,
-            27511,
-            21,
-            4185,
-            11,
-            41,
-            2444,
-            9,
-            32,
-            1025,
-            20,
-            8719,
-            26,
-            23,
-            673,
-            966,
-            19,
-            29077,
-            20643,
-            27511,
-            20822,
-            20643,
-            19,
-            17,
-            6616,
-            17511,
-            18,
-            8978,
-            20,
-            18,
-            777,
-            9,
-            19233,
-            1527,
-            17669,
-            19,
-            24,
-            673,
-            17,
-            28756,
-            150,
-            12943,
-            4354,
-            153,
-            27,
-            442,
-            37,
-            45,
-            668,
-            21,
-            24,
-            256,
-            20,
-            416,
-            22,
-            2771,
-            4901,
-            9,
-            12943,
-            4354,
-            153,
-            51,
-            24,
-            3004,
-            21,
-            28142,
-            23,
-            65,
-            20,
-            18,
-            416,
-            34,
-            24,
-            2958,
-            22947,
-            9,
-            1177,
-            45,
-            668,
-            3097,
-            13768,
-            23,
-            103,
-            28,
-            441,
-            148,
-            48,
-            20522,
-            19,
-            12943,
-            4354,
-            153,
-            12860,
-            34,
-            18,
-            326,
-            27,
-            17492,
-            684,
-            21,
-            6709,
-            9,
-            8585,
-            123,
-            266,
-            19,
-            12943,
-            4354,
-            153,
-            6872,
-            24,
-            3004,
-            20,
-            18,
-            9225,
-            2198,
-            19,
-            12717,
-            103,
-            22,
-            401,
-            24,
-            6348,
-            9,
-            12943,
-            4354,
-            153,
-            1068,
-            2768,
-            2286,
-            19,
-            33,
-            104,
-            19,
-            176,
-            24,
-            9313,
-            19,
-            20086,
-            28,
-            45,
-            10292,
-            9,
-            4,
-            3,
-            19,
-            12943,
-            4354,
-            153,
-            27,
-            442,
-            22,
-            2771,
-            4901,
-            9,
-            69,
-            27,
-            50,
-            551,
-            22,
-            2771,
-            4901,
-            19,
-            21,
-            45,
-            668,
-            21,
-            18,
-            416,
-            41,
-            1499,
-            22,
-            755,
-            18,
-            14285,
-            9,
-            12943,
-            4354,
-            153,
-            27,
-            1499,
-            22,
-            642,
-            22,
+            67, 2840, 19, 18, 1484, 20, 965, 29077, 8719, 1273, 21, 45, 273, 17, 10, 15048, 28, 27511, 21, 4185, 11, 41, 2444, 9, 32, 1025, 20, 8719, 26, 23, 673, 966, 19, 29077, 20643, 27511, 20822, 20643, 19, 17, 6616, 17511, 18, 8978, 20, 18, 777, 9, 19233, 1527, 17669, 19, 24, 673, 17, 28756, 150, 12943, 4354, 153, 27, 442, 37, 45, 668, 21, 24, 256, 20, 416, 22, 2771, 4901, 9, 12943, 4354, 153, 51, 24, 3004, 21, 28142, 23, 65, 20, 18, 416, 34, 24, 2958, 22947, 9, 1177, 45, 668, 3097, 13768, 23, 103, 28, 441, 148, 48, 20522, 19, 12943, 4354, 153, 12860, 34, 18, 326, 27, 17492, 684, 21, 6709, 9, 8585, 123, 266, 19, 12943, 4354, 153, 6872, 24, 3004, 20, 18, 9225, 2198, 19, 12717, 103, 22, 401, 24, 6348, 9, 12943, 4354, 153, 1068, 2768, 2286, 19, 33, 104, 19, 176, 24, 9313, 19, 20086, 28, 45, 10292, 9, 4, 3, 19, 12943, 4354, 153, 27, 442, 22, 2771, 4901, 9, 69, 27, 442, 22, 2771, 24, 11335, 20, 18, 9225, 2198, 9, 69, 27, 442, 22, 2771, 24, 11335, 20, 18, 9225, 2198, 9, 69, 27, 442, 22, 2771,
         ]
+        # fmt: on
         #  In 1991, the remains of Russian Tsar Nicholas II and his family (except for Alexei and Maria)
         #  are discovered. The voice of Nicholas's young son, Tsarevich Alexei Nikolaevich,
         #  narrates the remainder of the story. 1883 Western Siberia, a young Grigori Rasputin
@@ -850,9 +496,8 @@ class TFXLNetModelLanguageGenerationTest(unittest.TestCase):
         #  him for making such an accusation, Rasputin watches as the man is chased outside and beaten.
         #  Twenty years later, Rasputin sees a vision of the Virgin Mary, prompting him to become a priest.
         #  Rasputin quickly becomes famous, with people, even a bishop, begging for his blessing.
-        #  <sep><cls>, Rasputin is asked to perform magic.
-        #  He is not able to perform magic, and his father and
-        # the men are forced to leave the monastery. Rasputin is forced to return to
+        #  <sep><cls>, Rasputin is asked to perform magic. He is asked to perform a ritual of the Virgin Mary.
+        #  He is asked to perform a ritual of the Virgin Mary. He is asked to perform
 
         output_ids = model.generate(input_ids, max_length=200, do_sample=False)
 
diff --git a/tests/xlnet/test_modeling_xlnet.py b/tests/xlnet/test_modeling_xlnet.py
index 46ed90460c..420d22cc1e 100644
--- a/tests/xlnet/test_modeling_xlnet.py
+++ b/tests/xlnet/test_modeling_xlnet.py
@@ -673,175 +673,17 @@ class XLNetModelLanguageGenerationTest(unittest.TestCase):
     def test_lm_generate_xlnet_base_cased(self):
         model = XLNetLMHeadModel.from_pretrained("xlnet-base-cased")
         model.to(torch_device)
+        # fmt: off
         input_ids = torch.tensor(
             [
                 [
-                    67,
-                    2840,
-                    19,
-                    18,
-                    1484,
-                    20,
-                    965,
-                    29077,
-                    8719,
-                    1273,
-                    21,
-                    45,
-                    273,
-                    17,
-                    10,
-                    15048,
-                    28,
-                    27511,
-                    21,
-                    4185,
-                    11,
-                    41,
-                    2444,
-                    9,
-                    32,
-                    1025,
-                    20,
-                    8719,
-                    26,
-                    23,
-                    673,
-                    966,
-                    19,
-                    29077,
-                    20643,
-                    27511,
-                    20822,
-                    20643,
-                    19,
-                    17,
-                    6616,
-                    17511,
-                    18,
-                    8978,
-                    20,
-                    18,
-                    777,
-                    9,
-                    19233,
-                    1527,
-                    17669,
-                    19,
-                    24,
-                    673,
-                    17,
-                    28756,
-                    150,
-                    12943,
-                    4354,
-                    153,
-                    27,
-                    442,
-                    37,
-                    45,
-                    668,
-                    21,
-                    24,
-                    256,
-                    20,
-                    416,
-                    22,
-                    2771,
-                    4901,
-                    9,
-                    12943,
-                    4354,
-                    153,
-                    51,
-                    24,
-                    3004,
-                    21,
-                    28142,
-                    23,
-                    65,
-                    20,
-                    18,
-                    416,
-                    34,
-                    24,
-                    2958,
-                    22947,
-                    9,
-                    1177,
-                    45,
-                    668,
-                    3097,
-                    13768,
-                    23,
-                    103,
-                    28,
-                    441,
-                    148,
-                    48,
-                    20522,
-                    19,
-                    12943,
-                    4354,
-                    153,
-                    12860,
-                    34,
-                    18,
-                    326,
-                    27,
-                    17492,
-                    684,
-                    21,
-                    6709,
-                    9,
-                    8585,
-                    123,
-                    266,
-                    19,
-                    12943,
-                    4354,
-                    153,
-                    6872,
-                    24,
-                    3004,
-                    20,
-                    18,
-                    9225,
-                    2198,
-                    19,
-                    12717,
-                    103,
-                    22,
-                    401,
-                    24,
-                    6348,
-                    9,
-                    12943,
-                    4354,
-                    153,
-                    1068,
-                    2768,
-                    2286,
-                    19,
-                    33,
-                    104,
-                    19,
-                    176,
-                    24,
-                    9313,
-                    19,
-                    20086,
-                    28,
-                    45,
-                    10292,
-                    9,
-                    4,
-                    3,
+                    67, 2840, 19, 18, 1484, 20, 965, 29077, 8719, 1273, 21, 45, 273, 17, 10, 15048, 28, 27511, 21, 4185, 11, 41, 2444, 9, 32, 1025, 20, 8719, 26, 23, 673, 966, 19, 29077, 20643, 27511, 20822, 20643, 19, 17, 6616, 17511, 18, 8978, 20, 18, 777, 9, 19233, 1527, 17669, 19, 24, 673, 17, 28756, 150, 12943, 4354, 153, 27, 442, 37, 45, 668, 21, 24, 256, 20, 416, 22, 2771, 4901, 9, 12943, 4354, 153, 51, 24, 3004, 21, 28142, 23, 65, 20, 18, 416, 34, 24, 2958, 22947, 9, 1177, 45, 668, 3097, 13768, 23, 103, 28, 441, 148, 48, 20522, 19, 12943, 4354, 153, 12860, 34, 18, 326, 27, 17492, 684, 21, 6709, 9, 8585, 123, 266, 19, 12943, 4354, 153, 6872, 24, 3004, 20, 18, 9225, 2198, 19, 12717, 103, 22, 401, 24, 6348, 9, 12943, 4354, 153, 1068, 2768, 2286, 19, 33, 104, 19, 176, 24, 9313, 19, 20086, 28, 45, 10292, 9, 4, 3,
                 ]
             ],
             dtype=torch.long,
             device=torch_device,
         )
+        # fmt: on
         #  In 1991, the remains of Russian Tsar Nicholas II and his family
         #  (except for Alexei and Maria) are discovered.
         #  The voice of Nicholas's young son, Tsarevich Alexei Nikolaevich, narrates the
@@ -853,208 +695,11 @@ class XLNetModelLanguageGenerationTest(unittest.TestCase):
         #  the Virgin Mary, prompting him to become a priest. Rasputin quickly becomes famous,
         #  with people, even a bishop, begging for his blessing. """
 
+        # fmt: off
         expected_output_ids = [
-            67,
-            2840,
-            19,
-            18,
-            1484,
-            20,
-            965,
-            29077,
-            8719,
-            1273,
-            21,
-            45,
-            273,
-            17,
-            10,
-            15048,
-            28,
-            27511,
-            21,
-            4185,
-            11,
-            41,
-            2444,
-            9,
-            32,
-            1025,
-            20,
-            8719,
-            26,
-            23,
-            673,
-            966,
-            19,
-            29077,
-            20643,
-            27511,
-            20822,
-            20643,
-            19,
-            17,
-            6616,
-            17511,
-            18,
-            8978,
-            20,
-            18,
-            777,
-            9,
-            19233,
-            1527,
-            17669,
-            19,
-            24,
-            673,
-            17,
-            28756,
-            150,
-            12943,
-            4354,
-            153,
-            27,
-            442,
-            37,
-            45,
-            668,
-            21,
-            24,
-            256,
-            20,
-            416,
-            22,
-            2771,
-            4901,
-            9,
-            12943,
-            4354,
-            153,
-            51,
-            24,
-            3004,
-            21,
-            28142,
-            23,
-            65,
-            20,
-            18,
-            416,
-            34,
-            24,
-            2958,
-            22947,
-            9,
-            1177,
-            45,
-            668,
-            3097,
-            13768,
-            23,
-            103,
-            28,
-            441,
-            148,
-            48,
-            20522,
-            19,
-            12943,
-            4354,
-            153,
-            12860,
-            34,
-            18,
-            326,
-            27,
-            17492,
-            684,
-            21,
-            6709,
-            9,
-            8585,
-            123,
-            266,
-            19,
-            12943,
-            4354,
-            153,
-            6872,
-            24,
-            3004,
-            20,
-            18,
-            9225,
-            2198,
-            19,
-            12717,
-            103,
-            22,
-            401,
-            24,
-            6348,
-            9,
-            12943,
-            4354,
-            153,
-            1068,
-            2768,
-            2286,
-            19,
-            33,
-            104,
-            19,
-            176,
-            24,
-            9313,
-            19,
-            20086,
-            28,
-            45,
-            10292,
-            9,
-            4,
-            3,
-            19,
-            12943,
-            4354,
-            153,
-            27,
-            442,
-            22,
-            2771,
-            4901,
-            9,
-            69,
-            27,
-            442,
-            22,
-            2771,
-            24,
-            11335,
-            20,
-            18,
-            9225,
-            2198,
-            9,
-            69,
-            27,
-            442,
-            22,
-            2771,
-            24,
-            11335,
-            20,
-            18,
-            9225,
-            2198,
-            9,
-            69,
-            27,
-            442,
-            22,
-            2771,
+            67, 2840, 19, 18, 1484, 20, 965, 29077, 8719, 1273, 21, 45, 273, 17, 10, 15048, 28, 27511, 21, 4185, 11, 41, 2444, 9, 32, 1025, 20, 8719, 26, 23, 673, 966, 19, 29077, 20643, 27511, 20822, 20643, 19, 17, 6616, 17511, 18, 8978, 20, 18, 777, 9, 19233, 1527, 17669, 19, 24, 673, 17, 28756, 150, 12943, 4354, 153, 27, 442, 37, 45, 668, 21, 24, 256, 20, 416, 22, 2771, 4901, 9, 12943, 4354, 153, 51, 24, 3004, 21, 28142, 23, 65, 20, 18, 416, 34, 24, 2958, 22947, 9, 1177, 45, 668, 3097, 13768, 23, 103, 28, 441, 148, 48, 20522, 19, 12943, 4354, 153, 12860, 34, 18, 326, 27, 17492, 684, 21, 6709, 9, 8585, 123, 266, 19, 12943, 4354, 153, 6872, 24, 3004, 20, 18, 9225, 2198, 19, 12717, 103, 22, 401, 24, 6348, 9, 12943, 4354, 153, 1068, 2768, 2286, 19, 33, 104, 19, 176, 24, 9313, 19, 20086, 28, 45, 10292, 9, 4, 3, 19, 12943, 4354, 153, 27, 442, 22, 2771, 4901, 9, 69, 27, 442, 22, 2771, 24, 11335, 20, 18, 9225, 2198, 9, 69, 27, 442, 22, 2771, 24, 11335, 20, 18, 9225, 2198, 9, 69, 27, 442, 22, 2771,
         ]
+        # fmt: on
         #  In 1991, the remains of Russian Tsar Nicholas II and his family (except for Alexei and Maria)
         #  are discovered. The voice of Nicholas's young son, Tsarevich Alexei Nikolaevich,
         #  narrates the remainder of the story. 1883 Western Siberia, a young Grigori Rasputin