SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

[t5 tokenizer] add info logs (#9897)

Browse Source 
* save fast tokenizer + add info logs

* fix tests

* remove the saving of fast tokenizer
This commit is contained in:
Stas Bekman
2021-02-13 06:10:22 -08:00
committed by GitHub
parent 803498318c
commit 8fae93ca19
4 changed files with 6 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/transformers/models/t5/tokenization_t5.py
View File

@@ -295,5 +295,6 @@ class T5Tokenizer(PreTrainedTokenizer):
if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file):
copyfile(self.vocab_file, out_vocab_file)
logger.info(f"Copy vocab file to {out_vocab_file}")
return (out_vocab_file,)

1
src/transformers/models/t5/tokenization_t5_fast.py
View File

@@ -160,6 +160,7 @@ class T5TokenizerFast(PreTrainedTokenizerFast):
if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file):
copyfile(self.vocab_file, out_vocab_file)
logger.info(f"Copy vocab file to {out_vocab_file}")
return (out_vocab_file,)