Updated the GLUE data processor. Corrections to RoBERTa and XLNet.

2019-09-02 21:46:11 -04:00
parent c3df2136e1
commit bac332fec0
3 changed files with 4 additions and 56 deletions
--- a/pytorch_transformers/tokenization_roberta.py
+++ b/pytorch_transformers/tokenization_roberta.py
@@ -98,7 +98,7 @@ class RobertaTokenizer(GPT2Tokenizer):
        if output_mask:
            return (
                cls + token_ids_0 + sep + sep + token_ids_1 + sep,
-                [0] * len(cls + token_ids_0 + sep) + [1] * len(sep + token_ids_1 + sep)
+                [0] * len(cls + token_ids_0 + sep + sep) + [1] * len(token_ids_1 + sep)
            )
        else:
            return cls + token_ids_0 + sep + sep + token_ids_1 + sep