SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

[t5 tokenizer] add info logs (#9897)

Browse Source 
* save fast tokenizer + add info logs

* fix tests

* remove the saving of fast tokenizer
This commit is contained in:
Stas Bekman
2021-02-13 06:10:22 -08:00
committed by GitHub
parent 803498318c
commit 8fae93ca19
4 changed files with 6 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/transformers/configuration_utils.py
View File

@@ -308,7 +308,7 @@ class PretrainedConfig(object):
output_config_file = os.path.join(save_directory, CONFIG_NAME) output_config_file = os.path.join(save_directory, CONFIG_NAME)
self.to_json_file(output_config_file, use_diff=True) self.to_json_file(output_config_file, use_diff=True)
logger.info("Configuration saved in {}".format(output_config_file)) logger.info(f"Configuration saved in {output_config_file}")
@classmethod @classmethod
def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig": def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":

1
src/transformers/models/t5/tokenization_t5.py
View File

@@ -295,5 +295,6 @@ class T5Tokenizer(PreTrainedTokenizer):
if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file): if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file):
copyfile(self.vocab_file, out_vocab_file) copyfile(self.vocab_file, out_vocab_file)
logger.info(f"Copy vocab file to {out_vocab_file}")
return (out_vocab_file,) return (out_vocab_file,)

1
src/transformers/models/t5/tokenization_t5_fast.py
View File

@@ -160,6 +160,7 @@ class T5TokenizerFast(PreTrainedTokenizerFast):
if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file): if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file):
copyfile(self.vocab_file, out_vocab_file) copyfile(self.vocab_file, out_vocab_file)
logger.info(f"Copy vocab file to {out_vocab_file}")
return (out_vocab_file,) return (out_vocab_file,)

3
src/transformers/tokenization_utils_base.py
View File

@@ -1977,11 +1977,13 @@ class PreTrainedTokenizerBase(SpecialTokensMixin):
tokenizer_config = convert_added_tokens(tokenizer_config, add_type_field=True) tokenizer_config = convert_added_tokens(tokenizer_config, add_type_field=True)
with open(tokenizer_config_file, "w", encoding="utf-8") as f: with open(tokenizer_config_file, "w", encoding="utf-8") as f:
f.write(json.dumps(tokenizer_config, ensure_ascii=False)) f.write(json.dumps(tokenizer_config, ensure_ascii=False))
logger.info(f"tokenizer config file saved in {tokenizer_config_file}")
# Sanitize AddedTokens in special_tokens_map # Sanitize AddedTokens in special_tokens_map
write_dict = convert_added_tokens(self.special_tokens_map_extended, add_type_field=False) write_dict = convert_added_tokens(self.special_tokens_map_extended, add_type_field=False)
with open(special_tokens_map_file, "w", encoding="utf-8") as f: with open(special_tokens_map_file, "w", encoding="utf-8") as f:
f.write(json.dumps(write_dict, ensure_ascii=False)) f.write(json.dumps(write_dict, ensure_ascii=False))
logger.info(f"Special tokens file saved in {special_tokens_map_file}")
file_names = (tokenizer_config_file, special_tokens_map_file) file_names = (tokenizer_config_file, special_tokens_map_file)
@@ -2020,6 +2022,7 @@ class PreTrainedTokenizerBase(SpecialTokensMixin):
with open(added_tokens_file, "w", encoding="utf-8") as f: with open(added_tokens_file, "w", encoding="utf-8") as f:
out_str = json.dumps(added_vocab, ensure_ascii=False) out_str = json.dumps(added_vocab, ensure_ascii=False)
f.write(out_str) f.write(out_str)
logger.info(f"added tokens file saved in {added_tokens_file}")
vocab_files = self.save_vocabulary(save_directory, filename_prefix=filename_prefix) vocab_files = self.save_vocabulary(save_directory, filename_prefix=filename_prefix)