fix train_new_from_iterator in the case of byte-level tokenizers (#17549)

2022-06-08 15:30:41 +02:00
parent 264128cb9d
commit ae7bae8fe7
13 changed files with 56 additions and 0 deletions
--- a/tests/models/led/test_modeling_led.py
+++ b/tests/models/led/test_modeling_led.py
@@ -163,6 +163,7 @@ class LEDModelTester:
    def get_pipeline_config(self):
        config = self.get_config()
        config.max_position_embeddings = 100
+        config.vocab_size = 300
        return config

    def prepare_config_and_inputs_for_common(self):