fix train_new_from_iterator in the case of byte-level tokenizers (#17549)

2022-06-08 15:30:41 +02:00
parent 264128cb9d
commit ae7bae8fe7
13 changed files with 56 additions and 0 deletions
--- a/tests/models/ibert/test_modeling_ibert.py
+++ b/tests/models/ibert/test_modeling_ibert.py
@@ -116,6 +116,11 @@ class IBertModelTester:
            quant_mode=True,
        )

+    def get_pipeline_config(self):
+        config = self.get_config()
+        config.vocab_size = 300
+        return config
+
    def create_and_check_model(
        self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
    ):