SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Merge pull request #860 from Yiqing-Zhou/patch-1

Browse Source 
read().splitlines() -> readlines()
This commit is contained in:
Thomas Wolf
2019-07-23 15:24:25 +02:00
committed by GitHub
parent ba52fe69d5 b1019d2a8e
commit c4bc66886d
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
pytorch_transformers/tokenization_bert.py
View File

@@ -67,10 +67,10 @@ def load_vocab(vocab_file):
"""Loads a vocabulary file into a dictionary.""" """Loads a vocabulary file into a dictionary."""
vocab = collections.OrderedDict() vocab = collections.OrderedDict()
with open(vocab_file, "r", encoding="utf-8") as reader: with open(vocab_file, "r", encoding="utf-8") as reader:
tokens = reader.read().splitlines() tokens = reader.readlines()
for index, token in enumerate(tokens): for index, token in enumerate(tokens):
token = token.rstrip('\n')
vocab[token] = index vocab[token] = index
index += 1
return vocab return vocab