SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Tokenizers API developments (#5103)

Browse Source 
* Add return lengths

* make pad a bit more flexible so it can be used as collate_fn

* check all kwargs sent to encoding method are known

* fixing kwargs in encodings

* New AddedToken class in python

This class let you specify specifique tokenization behaviors for some special tokens. Used in particular for GPT2 and Roberta, to control how white spaces are stripped around special tokens.

* style and quality

* switched to hugginface tokenizers library for AddedTokens

* up to tokenizer 0.8.0-rc3 - update API to use AddedToken state

* style and quality

* do not raise an error on additional or unused kwargs for tokenize() but only a warning

* transfo-xl pretrained model requires torch

* Update src/transformers/tokenization_utils.py

Co-authored-by: Lysandre Debut <lysandre@huggingface.co>

Co-authored-by: Lysandre Debut <lysandre@huggingface.co>
This commit is contained in:
Thomas Wolf
2020-06-23 13:36:57 +02:00
committed by GitHub
parent 1ae132a07d
commit 11fdde0271
11 changed files with 414 additions and 230 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
tests/test_tokenization_common.py
View File

@@ -390,7 +390,7 @@ class TokenizerTesterMixin:
seq_1 = "With these inputs."
sequences = tokenizer.encode(seq_0, seq_1, add_special_tokens=False)
attached_sequences = tokenizer.encode(seq_0, seq_1, add_special_tokens=True, add_prefix_space=False)
attached_sequences = tokenizer.encode(seq_0, seq_1, add_special_tokens=True)
# Method is implemented (e.g. not GPT-2)
if len(attached_sequences) != 2:
@@ -416,7 +416,7 @@ class TokenizerTesterMixin:
stride=stride,
truncation="longest_first",
return_overflowing_tokens=True,
add_prefix_space=False,
# add_prefix_space=False,
)
# Overflowing tokens are handled quite differently in slow and fast tokenizers
@@ -468,7 +468,7 @@ class TokenizerTesterMixin:
# We are not using the special tokens - a bit too hard to test all the tokenizers with this
# TODO try this again later
sequence = tokenizer.encode(seq_0, seq_1, add_special_tokens=False, add_prefix_space=False)
sequence = tokenizer.encode(seq_0, seq_1, add_special_tokens=False) # , add_prefix_space=False)
truncated_first_sequence = tokenizer.encode(seq_0, add_special_tokens=False)[:-2] + tokenizer.encode(
seq_1, add_special_tokens=False
)
@@ -499,7 +499,7 @@ class TokenizerTesterMixin:
stride=stride,
truncation="longest_first",
return_overflowing_tokens=True,
add_prefix_space=False,
# add_prefix_space=False,
)
# Overflowing tokens are handled quite differently in slow and fast tokenizers
if isinstance(tokenizer, PreTrainedTokenizerFast):
@@ -531,7 +531,7 @@ class TokenizerTesterMixin:
stride=stride,
truncation=True,
return_overflowing_tokens=True,
add_prefix_space=False,
# add_prefix_space=False,
)
# Overflowing tokens are handled quite differently in slow and fast tokenizers
if isinstance(tokenizer, PreTrainedTokenizerFast):
@@ -562,7 +562,7 @@ class TokenizerTesterMixin:
stride=stride,
truncation="only_second",
return_overflowing_tokens=True,
add_prefix_space=False,
# add_prefix_space=False,
)
# Overflowing tokens are handled quite differently in slow and fast tokenizers
if isinstance(tokenizer, PreTrainedTokenizerFast):
@@ -638,7 +638,7 @@ class TokenizerTesterMixin:
# Testing single inputs
encoded_sequence = tokenizer.encode(sequence_0, add_special_tokens=False)
encoded_sequence_dict = tokenizer.encode_plus(
sequence_0, add_special_tokens=True, return_special_tokens_mask=True, add_prefix_space=False
sequence_0, add_special_tokens=True, return_special_tokens_mask=True # , add_prefix_space=False
)
encoded_sequence_w_special = encoded_sequence_dict["input_ids"]
special_tokens_mask = encoded_sequence_dict["special_tokens_mask"]
@@ -660,7 +660,7 @@ class TokenizerTesterMixin:
sequence_1,
add_special_tokens=True,
return_special_tokens_mask=True,
add_prefix_space=False,
# add_prefix_space=False,
)
encoded_sequence_w_special = encoded_sequence_dict["input_ids"]
special_tokens_mask = encoded_sequence_dict["special_tokens_mask"]
@@ -1042,7 +1042,7 @@ class TokenizerTesterMixin:
def test_pretokenized_inputs(self):
# Test when inputs are pretokenized
tokenizers = self.get_tokenizers(do_lower_case=False, add_prefix_space=True)
tokenizers = self.get_tokenizers(do_lower_case=False) # , add_prefix_space=True)
for tokenizer in tokenizers:
with self.subTest(f"{tokenizer.__class__.__name__}"):