SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

[MarianTokenizer] Switch to sacremoses for punc normalization (#5092)

Browse Source
This commit is contained in:
Sam Shleifer
2020-06-17 16:31:05 -04:00
committed by GitHub
parent 049e14f0e3
commit 90c833870c
1 changed files with 4 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/transformers/tokenization_marian.py
View File

@@ -82,11 +82,11 @@ class MarianTokenizer(PreTrainedTokenizer):
def _setup_normalizer(self): def _setup_normalizer(self):
try: try:
from mosestokenizer import MosesPunctuationNormalizer from sacremoses import MosesPunctNormalizer
self.punc_normalizer = MosesPunctuationNormalizer(self.source_lang) self.punc_normalizer = MosesPunctNormalizer(self.source_lang).normalize
except ImportError: except (ImportError, FileNotFoundError):
warnings.warn("Recommended: pip install mosestokenizer") warnings.warn("Recommended: pip install sacremoses.")
self.punc_normalizer = lambda x: x self.punc_normalizer = lambda x: x
def normalize(self, x: str) -> str: def normalize(self, x: str) -> str: