SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Merge pull request #42 from weiyumou/master

Browse Source 
Fixed UnicodeDecodeError: 'ascii' codec can't decode byte 0xc2
This commit is contained in:
Thomas Wolf
2018-11-20 10:09:50 +01:00
committed by GitHub
parent eed255a58d 9ff2b7d86d
commit fd32ebed81
2 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@@ -99,7 +99,7 @@ from pytorch_pretrained_bert import BertTokenizer, BertModel, BertForMaskedLM
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
# Tokenized input # Tokenized input
tokenized_text = "Who was Jim Henson ? Jim Henson was a puppeteer" text = "Who was Jim Henson ? Jim Henson was a puppeteer"
tokenized_text = tokenizer.tokenize(text) tokenized_text = tokenizer.tokenize(text)
# Mask a token that we will try to predict back with `BertForMaskedLM` # Mask a token that we will try to predict back with `BertForMaskedLM`

2
pytorch_pretrained_bert/tokenization.py
View File

@@ -65,7 +65,7 @@ def load_vocab(vocab_file):
"""Loads a vocabulary file into a dictionary.""" """Loads a vocabulary file into a dictionary."""
vocab = collections.OrderedDict() vocab = collections.OrderedDict()
index = 0 index = 0
with open(vocab_file, "r") as reader: with open(vocab_file, "r", encoding="utf8") as reader:
while True: while True:
token = convert_to_unicode(reader.readline()) token = convert_to_unicode(reader.readline())
if not token: if not token: