fix train_new_from_iterator in the case of byte-level tokenizers (#17549)

2022-06-08 15:30:41 +02:00
parent 264128cb9d
commit ae7bae8fe7
13 changed files with 56 additions and 0 deletions
--- a/tests/models/deberta/test_modeling_deberta.py
+++ b/tests/models/deberta/test_modeling_deberta.py
@@ -130,6 +130,11 @@ class DebertaModelTester(object):
            pos_att_type=self.pos_att_type,
        )

+    def get_pipeline_config(self):
+        config = self.get_config()
+        config.vocab_size = 300
+        return config
+
    def check_loss_output(self, result):
        self.parent.assertListEqual(list(result.loss.size()), [])