SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

fixes

Browse Source
This commit is contained in:
thomwolf
2019-10-04 17:59:44 -04:00
parent 6c1d0bc066
commit 78ef1a9930
4 changed files with 29 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
transformers/tests/tokenization_tests_commons.py
View File

@@ -249,10 +249,10 @@ class CommonTestCases:
)
information = tokenizer.encode_plus(seq_0, seq_1, max_length=len(sequence) - 2, add_special_tokens=True,
stride=stride, truncate_first_sequence=False)
stride=stride, truncation_strategy='only_second')
information_first_truncated = tokenizer.encode_plus(seq_0, seq_1, max_length=len(sequence) - 2,
add_special_tokens=True, stride=stride,
truncate_first_sequence=True)
truncation_strategy='only_first')
truncated_sequence = information["input_ids"]
overflowing_tokens = information["overflowing_tokens"]