Override build_inputs_with_special_tokens for fast tokenizers (#2912)

* Override build_inputs_with_special_tokens for fast impl + unittest. Signed-off-by: Morgan Funtowicz <morgan@huggingface.co> * Quality + format. Signed-off-by: Morgan Funtowicz <morgan@huggingface.co>
2020-02-19 22:09:51 +01:00
parent 59c23ad9c9
commit e676764241
4 changed files with 68 additions and 0 deletions
--- a/src/transformers/tokenization_bert.py
+++ b/src/transformers/tokenization_bert.py
@@ -572,3 +572,11 @@ class BertTokenizerFast(PreTrainedTokenizerFast):
        )

        self.do_lower_case = do_lower_case
+
+    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
+        output = [self.cls_token_id] + token_ids_0 + [self.sep_token_id]
+
+        if token_ids_1:
+            output += token_ids_1 + [self.sep_token_id]
+
+        return output