SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

gpt-2 tokenizer

Browse Source
This commit is contained in:
thomwolf
2019-05-01 11:40:48 +02:00
parent cd110835a0
commit db98a4a48b
1 changed files with 4 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
pytorch_pretrained_bert/tokenization_gpt2.py
View File

@@ -221,7 +221,10 @@ class GPT2Tokenizer(object):
""" Tokenize a string. """ """ Tokenize a string. """
bpe_tokens = [] bpe_tokens = []
for token in re.findall(self.pat, text): for token in re.findall(self.pat, text):
if sys.version_info[0] == 2:
token = ''.join(self.byte_encoder[ord(b)] for b in token) token = ''.join(self.byte_encoder[ord(b)] for b in token)
else:
token = ''.join(self.byte_encoder[b] for b in token.encode('utf-8'))
bpe_tokens.extend(bpe_token for bpe_token in self.bpe(token).split(' ')) bpe_tokens.extend(bpe_token for bpe_token in self.bpe(token).split(' '))
return bpe_tokens return bpe_tokens