SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

fix #1920

Browse Source
This commit is contained in:
thomwolf
2019-12-05 11:18:43 +01:00
parent d425a4d60b
commit 8b388827b5
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
transformers/tokenization_ctrl.py
View File

@@ -192,9 +192,9 @@ class CTRLTokenizer(PreTrainedTokenizer):
""" """
split_tokens = [] split_tokens = []
text = text.split(' ') words = re.findall(r'\S+\n?', text)
for token in text: for token in words:
split_tokens.extend([t for t in self.bpe(token).split(' ')]) split_tokens.extend([t for t in self.bpe(token).split(' ')])
return split_tokens return split_tokens