From 37b6c9b21b96447a2712a4559a045d79c4f9c40e Mon Sep 17 00:00:00 2001 From: weiyumou Date: Mon, 19 Nov 2018 23:01:28 -0500 Subject: [PATCH] Fixed UnicodeDecodeError: 'ascii' codec can't decode byte 0xc2 in position 3793: ordinal not in range(128) --- pytorch_pretrained_bert/tokenization.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_pretrained_bert/tokenization.py b/pytorch_pretrained_bert/tokenization.py index fab7b0049c..5c9369eb4f 100644 --- a/pytorch_pretrained_bert/tokenization.py +++ b/pytorch_pretrained_bert/tokenization.py @@ -65,7 +65,7 @@ def load_vocab(vocab_file): """Loads a vocabulary file into a dictionary.""" vocab = collections.OrderedDict() index = 0 - with open(vocab_file, "r") as reader: + with open(vocab_file, "r", encoding="utf8") as reader: while True: token = convert_to_unicode(reader.readline()) if not token: