Removed dataset limit

2019-08-06 14:57:07 -04:00
parent 3e3e145497
commit 5c18825a18
1 changed files with 1 additions and 1 deletions
--- a/examples/utils_lm.py
+++ b/examples/utils_lm.py
@@ -14,7 +14,7 @@ class WikiTextDataset(Dataset):

 		with open(os.path.join(directory, f"wiki.{file}.raw"), encoding="utf-8") as f:
 			text = f.read()
-			spans = list(filter(lambda item: len(item) > 120, text.split("\n")[:20]))
+			spans = list(filter(lambda item: len(item) > 120, text.split("\n")))

 			for span in spans:
 				span = tokenizer.encode(span)