SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Removed dataset limit

Browse Source
This commit is contained in:
LysandreJik
2019-08-06 14:57:07 -04:00
parent 3e3e145497
commit 5c18825a18
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
examples/utils_lm.py
View File

@@ -14,7 +14,7 @@ class WikiTextDataset(Dataset):
with open(os.path.join(directory, f"wiki.{file}.raw"), encoding="utf-8") as f: with open(os.path.join(directory, f"wiki.{file}.raw"), encoding="utf-8") as f:
text = f.read() text = f.read()
spans = list(filter(lambda item: len(item) > 120, text.split("\n")[:20])) spans = list(filter(lambda item: len(item) > 120, text.split("\n")))
for span in spans: for span in spans:
span = tokenizer.encode(span) span = tokenizer.encode(span)