From 37b6c9b21b96447a2712a4559a045d79c4f9c40e Mon Sep 17 00:00:00 2001 From: weiyumou Date: Mon, 19 Nov 2018 23:01:28 -0500 Subject: [PATCH 1/2] Fixed UnicodeDecodeError: 'ascii' codec can't decode byte 0xc2 in position 3793: ordinal not in range(128) --- pytorch_pretrained_bert/tokenization.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_pretrained_bert/tokenization.py b/pytorch_pretrained_bert/tokenization.py index fab7b0049c..5c9369eb4f 100644 --- a/pytorch_pretrained_bert/tokenization.py +++ b/pytorch_pretrained_bert/tokenization.py @@ -65,7 +65,7 @@ def load_vocab(vocab_file): """Loads a vocabulary file into a dictionary.""" vocab = collections.OrderedDict() index = 0 - with open(vocab_file, "r") as reader: + with open(vocab_file, "r", encoding="utf8") as reader: while True: token = convert_to_unicode(reader.readline()) if not token: From 9ff2b7d86de6bb8fd8f258ef4daddd1e9543b0c0 Mon Sep 17 00:00:00 2001 From: weiyumou Date: Mon, 19 Nov 2018 23:13:10 -0500 Subject: [PATCH 2/2] Fixed README typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 707818b445..17dc882dc2 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,7 @@ from pytorch_pretrained_bert import BertTokenizer, BertModel, BertForMaskedLM tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') # Tokenized input -tokenized_text = "Who was Jim Henson ? Jim Henson was a puppeteer" +text = "Who was Jim Henson ? Jim Henson was a puppeteer" tokenized_text = tokenizer.tokenize(text) # Mask a token that we will try to predict back with `BertForMaskedLM`