SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

[s2s] fix nltk pytest race condition with FileLock (#7515)

Browse Source
This commit is contained in:
Sam Shleifer
2020-10-01 12:51:09 -04:00
committed by GitHub
parent 72d363d979
commit 2a358f45ef
1 changed files with 5 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
examples/seq2seq/sentence_splitter.py
View File

@@ -1,5 +1,7 @@
import re import re
from filelock import FileLock
try: try:
import nltk import nltk
@@ -9,13 +11,12 @@ except (ImportError, ModuleNotFoundError):
NLTK_AVAILABLE = False NLTK_AVAILABLE = False
if NLTK_AVAILABLE: if NLTK_AVAILABLE:
try: with FileLock("a_random_string") as lock:
nltk.download("punkt", quiet=True) nltk.download("punkt", quiet=True)
except FileExistsError: # multiprocessing race condition
pass
def add_newline_to_end_of_each_sentence(x: str) -> str: def add_newline_to_end_of_each_sentence(x: str) -> str:
"""This was added to get rougeLsum scores matching published rougeL scores for BART and PEGASUS."""
re.sub("<n>", "", x) # remove pegasus newline char re.sub("<n>", "", x) # remove pegasus newline char
assert NLTK_AVAILABLE, "nltk must be installed to separate newlines betwee sentences. (pip install nltk)" assert NLTK_AVAILABLE, "nltk must be installed to separate newlines between sentences. (pip install nltk)"
return "\n".join(nltk.sent_tokenize(x)) return "\n".join(nltk.sent_tokenize(x))