SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

up (#14079)

Browse Source
This commit is contained in:
Patrick von Platen
2021-10-20 13:01:42 +02:00
committed by GitHub
parent 0bc2e54f00
commit 53dc39d821
1 changed files with 6 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
examples/pytorch/speech-recognition/run_speech_recognition_ctc.py
View File

@@ -410,10 +410,15 @@ def main():
# load config # load config
config = AutoConfig.from_pretrained(model_args.model_name_or_path, cache_dir=model_args.cache_dir) config = AutoConfig.from_pretrained(model_args.model_name_or_path, cache_dir=model_args.cache_dir)
# tokenizer is defined by `tokenizer_class` if present in config else by `model_type`
config_for_tokenizer = config if config.tokenizer_class is not None else None
tokenizer_type = config.model_type if config.tokenizer_class is None else None
# load feature_extractor, tokenizer and create processor # load feature_extractor, tokenizer and create processor
tokenizer = AutoTokenizer.from_pretrained( tokenizer = AutoTokenizer.from_pretrained(
training_args.output_dir, training_args.output_dir,
tokenizer_type=config.model_type, config=config_for_tokenizer,
tokenizer_type=tokenizer_type,
unk_token="[UNK]", unk_token="[UNK]",
pad_token="[PAD]", pad_token="[PAD]",
word_delimiter_token="|", word_delimiter_token="|",