SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Fix slow gpu tests lysandre (#4487)

Browse Source 
* There is one missing key in BERT

* Correct device for CamemBERT model

* RoBERTa tokenization adding prefix space

* Style
This commit is contained in:
Lysandre Debut
2020-05-20 11:59:45 -04:00
committed by GitHub
parent 6dc52c78d8
commit 14cb5b35fa
3 changed files with 8 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
tests/test_tokenization_roberta.py
View File

@@ -100,9 +100,11 @@ class RobertaTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
text = tokenizer.encode("sequence builders", add_special_tokens=False)
text_2 = tokenizer.encode("multi-sequence build", add_special_tokens=False)
encoded_text_from_decode = tokenizer.encode("sequence builders", add_special_tokens=True)
encoded_text_from_decode = tokenizer.encode(
"sequence builders", add_special_tokens=True, add_prefix_space=False
)
encoded_pair_from_decode = tokenizer.encode(
"sequence builders", "multi-sequence build", add_special_tokens=True
"sequence builders", "multi-sequence build", add_special_tokens=True, add_prefix_space=False
)
encoded_sentence = tokenizer.build_inputs_with_special_tokens(text)