Fixed UnicodeDecodeError: 'ascii' codec can't decode byte 0xc2 in position 3793: ordinal not in range(128)

2018-11-19 23:01:28 -05:00
parent da73925f6a
commit 37b6c9b21b
1 changed files with 1 additions and 1 deletions
--- a/pytorch_pretrained_bert/tokenization.py
+++ b/pytorch_pretrained_bert/tokenization.py
@@ -65,7 +65,7 @@ def load_vocab(vocab_file):
    """Loads a vocabulary file into a dictionary."""
    vocab = collections.OrderedDict()
    index = 0
-    with open(vocab_file, "r") as reader:
+    with open(vocab_file, "r", encoding="utf8") as reader:
        while True:
            token = convert_to_unicode(reader.readline())
            if not token: