SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Fix + Test (#8049)

Browse Source
This commit is contained in:
Lysandre Debut
2020-10-26 16:32:27 +00:00
committed by GitHub
parent 664c7ec453
commit cbad90d86d
2 changed files with 12 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/transformers/tokenization_blenderbot.py
View File

@@ -166,6 +166,9 @@ class BlenderbotSmallTokenizer(PreTrainedTokenizer):
tokens = token.split(" ") tokens = token.split(" ")
words = [] words = []
for token in tokens: for token in tokens:
if not len(token):
continue
token = token.lower() token = token.lower()
word = tuple(token) word = tuple(token)
word = tuple(list(word[:-1]) + [word[-1] + "</w>"]) word = tuple(list(word[:-1]) + [word[-1] + "</w>"])

9
tests/test_tokenization_blenderbot.py
View File

@@ -75,6 +75,15 @@ class BlenderbotSmallTokenizerTest(TokenizerTesterMixin, unittest.TestCase):
assert src_text != decoded # I wish it did! assert src_text != decoded # I wish it did!
assert decoded == "i am a small frog ." assert decoded == "i am a small frog ."
def test_empty_word_small_tok(self):
tok = BlenderbotSmallTokenizer.from_pretrained("facebook/blenderbot-90M")
src_text = "I am a small frog ."
src_text_dot = "."
encoded = tok(src_text)["input_ids"]
encoded_dot = tok(src_text_dot)["input_ids"]
assert encoded[-1] == encoded_dot[0]
class Blenderbot3BTokenizerTests(unittest.TestCase): class Blenderbot3BTokenizerTests(unittest.TestCase):
@cached_property @cached_property