SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Expose missing mappings (see #3415)

Browse Source
This commit is contained in:
Julien Chaumond
2020-03-24 17:46:25 -04:00
parent d0c36a7b72
commit f8823bad9a
2 changed files with 10 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
examples/run_language_modeling.py
View File

@@ -38,7 +38,6 @@ from torch.utils.data.distributed import DistributedSampler
from tqdm import tqdm, trange
from transformers import (
CONFIG_MAPPING,
MODEL_WITH_LM_HEAD_MAPPING,
WEIGHTS_NAME,
AdamW,
@@ -679,7 +678,12 @@ def main():
elif args.model_name_or_path:
config = AutoConfig.from_pretrained(args.model_name_or_path, cache_dir=args.cache_dir)
else:
config = CONFIG_MAPPING[args.model_type]()
# When we release a pip version exposing CONFIG_MAPPING,
# we can do `config = CONFIG_MAPPING[args.model_type]()`.
raise ValueError(
"You are instantiating a new config instance from scratch. This is not supported, but you can do it from another script, save it,"
"and load it from here, using --config_name"
)
if args.tokenizer_name:
tokenizer = AutoTokenizer.from_pretrained(args.tokenizer_name, cache_dir=args.cache_dir)
@@ -687,8 +691,8 @@ def main():
tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path, cache_dir=args.cache_dir)
else:
raise ValueError(
"You are instantiating a new {} tokenizer. This is not supported, but you can do it from another script, save it,"
"and load it from here, using --tokenizer_name".format(AutoTokenizer.__name__)
"You are instantiating a new tokenizer from scratch. This is not supported, but you can do it from another script, save it,"
"and load it from here, using --tokenizer_name"
)
if args.block_size <= 0: