typo
This commit is contained in:
@@ -292,7 +292,7 @@ class TransfoXLTokenizer(object):
|
|||||||
return "".join(output)
|
return "".join(output)
|
||||||
|
|
||||||
def whitespace_tokenize(self, text):
|
def whitespace_tokenize(self, text):
|
||||||
"""Runs basic whitespace cleaning and splitting on a peice of text."""
|
"""Runs basic whitespace cleaning and splitting on a piece of text."""
|
||||||
text = text.strip()
|
text = text.strip()
|
||||||
if not text:
|
if not text:
|
||||||
return []
|
return []
|
||||||
|
|||||||
Reference in New Issue
Block a user