From 5c18825a1850ad59021ea9a914e638256dd372f6 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Tue, 6 Aug 2019 14:57:07 -0400 Subject: [PATCH] Removed dataset limit --- examples/utils_lm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/utils_lm.py b/examples/utils_lm.py index 4a3bafb789..2944cdc9ea 100644 --- a/examples/utils_lm.py +++ b/examples/utils_lm.py @@ -14,7 +14,7 @@ class WikiTextDataset(Dataset): with open(os.path.join(directory, f"wiki.{file}.raw"), encoding="utf-8") as f: text = f.read() - spans = list(filter(lambda item: len(item) > 120, text.split("\n")[:20])) + spans = list(filter(lambda item: len(item) > 120, text.split("\n"))) for span in spans: span = tokenizer.encode(span)