SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

tokenizer white space: revert to previous behavior

Browse Source
This commit is contained in:
Julien Chaumond
2019-11-05 15:48:00 +00:00
parent 0b77d66a6d
commit d5faa74cd6
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
examples/run_pplm.py
View File

@@ -373,7 +373,7 @@ def get_bag_of_words_indices(bag_of_words_ids_or_paths: List[str]) -> List[List[
filepath = id_or_path filepath = id_or_path
with open(filepath, "r") as f: with open(filepath, "r") as f:
words = f.read().split("\n") words = f.read().split("\n")
bow_indices.append([TOKENIZER.encode(word) for word in words]) bow_indices.append([TOKENIZER.encode(word, add_prefix_space=True) for word in words])
return bow_indices return bow_indices