SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

fix for python2

Browse Source
This commit is contained in:
thomwolf
2019-08-30 12:29:31 +02:00
parent 4e6a3172ce
commit ca1a00a302
2 changed files with 4 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
pytorch_transformers/tests/tokenization_gpt2_test.py
View File

@@ -17,6 +17,7 @@ from __future__ import absolute_import, division, print_function, unicode_litera
import os import os
import unittest import unittest
import json import json
from io import open
from pytorch_transformers.tokenization_gpt2 import GPT2Tokenizer, VOCAB_FILES_NAMES from pytorch_transformers.tokenization_gpt2 import GPT2Tokenizer, VOCAB_FILES_NAMES
@@ -55,8 +56,8 @@ class GPT2TokenizationTest(CommonTestCases.CommonTokenizerTester):
def test_full_tokenizer(self): def test_full_tokenizer(self):
tokenizer = GPT2Tokenizer(self.vocab_file, self.merges_file, **self.special_tokens_map) tokenizer = GPT2Tokenizer(self.vocab_file, self.merges_file, **self.special_tokens_map)
text = "lower" text = "lower newer"
bpe_tokens = ["\u0120low", "er"] bpe_tokens = ["\u0120low", "er", "\u0120newer"]
tokens = tokenizer.tokenize(text) tokens = tokenizer.tokenize(text)
self.assertListEqual(tokens, bpe_tokens) self.assertListEqual(tokens, bpe_tokens)

1
pytorch_transformers/tests/tokenization_roberta_test.py
View File

@@ -17,6 +17,7 @@ from __future__ import absolute_import, division, print_function, unicode_litera
import os import os
import json import json
import unittest import unittest
from io import open
from pytorch_transformers.tokenization_roberta import RobertaTokenizer, VOCAB_FILES_NAMES from pytorch_transformers.tokenization_roberta import RobertaTokenizer, VOCAB_FILES_NAMES
from .tokenization_tests_commons import CommonTestCases from .tokenization_tests_commons import CommonTestCases