fix train_new_from_iterator in the case of byte-level tokenizers (#17549)

tests/models/bart/test_modeling_bart.py

@@ -150,6 +150,7 @@ class BartModelTester:
     def get_pipeline_config(self):
         config = self.get_config()
         config.max_position_embeddings = 100
+        config.vocab_size = 300
         return config
 
     def prepare_config_and_inputs_for_common(self):

tests/models/blenderbot/test_modeling_blenderbot.py

@@ -140,6 +140,7 @@ class BlenderbotModelTester:
     def get_pipeline_config(self):
         config = self.get_config()
         config.max_position_embeddings = 100
+        config.vocab_size = 300
         return config
 
     def prepare_config_and_inputs_for_common(self):

tests/models/deberta/test_modeling_deberta.py

@@ -130,6 +130,11 @@ class DebertaModelTester(object):
             pos_att_type=self.pos_att_type,
         )
 
+    def get_pipeline_config(self):
+        config = self.get_config()
+        config.vocab_size = 300
+        return config
+
     def check_loss_output(self, result):
         self.parent.assertListEqual(list(result.loss.size()), [])
 

tests/models/gpt2/test_modeling_gpt2.py

@@ -166,6 +166,11 @@ class GPT2ModelTester:
             reorder_and_upcast_attn=reorder_and_upcast_attn,
         )
 
+    def get_pipeline_config(self):
+        config = self.get_config()
+        config.vocab_size = 300
+        return config
+
     def prepare_config_and_inputs_for_decoder(self):
         (
             config,

tests/models/gpt_neo/test_modeling_gpt_neo.py

@@ -151,6 +151,11 @@ class GPTNeoModelTester:
             attention_types=self.attention_types,
         )
 
+    def get_pipeline_config(self):
+        config = self.get_config()
+        config.vocab_size = 300
+        return config
+
     def prepare_config_and_inputs_for_decoder(self):
         (
             config,

tests/models/gptj/test_modeling_gptj.py

@@ -155,6 +155,11 @@ class GPTJModelTester:
             rotary_dim=self.rotary_dim,
         )
 
+    def get_pipeline_config(self):
+        config = self.get_config()
+        config.vocab_size = 300
+        return config
+
     def prepare_config_and_inputs_for_decoder(self):
         (
             config,

tests/models/ibert/test_modeling_ibert.py

@@ -116,6 +116,11 @@ class IBertModelTester:
             quant_mode=True,
         )
 
+    def get_pipeline_config(self):
+        config = self.get_config()
+        config.vocab_size = 300
+        return config
+
     def create_and_check_model(
         self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
     ):

tests/models/led/test_modeling_led.py

@@ -163,6 +163,7 @@ class LEDModelTester:
     def get_pipeline_config(self):
         config = self.get_config()
         config.max_position_embeddings = 100
+        config.vocab_size = 300
         return config
 
     def prepare_config_and_inputs_for_common(self):

tests/models/longformer/test_modeling_longformer.py

@@ -113,6 +113,11 @@ class LongformerModelTester:
             attention_window=self.attention_window,
         )
 
+    def get_pipeline_config(self):
+        config = self.get_config()
+        config.vocab_size = 300
+        return config
+
     def create_and_check_attention_mask_determinism(
         self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
     ):

tests/models/roberta/test_modeling_roberta.py

@@ -112,6 +112,11 @@ class RobertaModelTester:
             initializer_range=self.initializer_range,
         )
 
+    def get_pipeline_config(self):
+        config = self.get_config()
+        config.vocab_size = 300
+        return config
+
     def prepare_config_and_inputs_for_decoder(self):
         (
             config,

tests/models/yoso/test_modeling_yoso.py

@@ -126,6 +126,11 @@ class YosoModelTester:
             initializer_range=self.initializer_range,
         )
 
+    def get_pipeline_config(self):
+        config = self.get_config()
+        config.vocab_size = 300
+        return config
+
     def prepare_config_and_inputs_for_decoder(self):
         (
             config,