This commit is contained in:
Yongbo Wang
2019-02-20 21:11:06 +08:00
committed by GitHub
parent 813e4d18ba
commit 2fdab323d1

View File

@@ -292,7 +292,7 @@ class TransfoXLTokenizer(object):
return "".join(output) return "".join(output)
def whitespace_tokenize(self, text): def whitespace_tokenize(self, text):
"""Runs basic whitespace cleaning and splitting on a peice of text.""" """Runs basic whitespace cleaning and splitting on a piece of text."""
text = text.strip() text = text.strip()
if not text: if not text:
return [] return []