From 2e4db64cab198dc241e18221ef088908f2587c61 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Mon, 7 Jan 2019 13:06:42 +0100 Subject: [PATCH] add do_lower_case tokenizer loading optino in run_squad and ine_tuning examples --- examples/run_lm_finetuning.py | 2 +- examples/run_squad.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/run_lm_finetuning.py b/examples/run_lm_finetuning.py index 2c64f67b5f..39df2e99f8 100644 --- a/examples/run_lm_finetuning.py +++ b/examples/run_lm_finetuning.py @@ -515,7 +515,7 @@ def main(): raise ValueError("Output directory ({}) already exists and is not empty.".format(args.output_dir)) os.makedirs(args.output_dir, exist_ok=True) - tokenizer = BertTokenizer.from_pretrained(args.bert_model) + tokenizer = BertTokenizer.from_pretrained(args.bert_model, do_lower_case=args.do_lower_case) #train_examples = None num_train_steps = None diff --git a/examples/run_squad.py b/examples/run_squad.py index 88ea590936..245aee0ff2 100644 --- a/examples/run_squad.py +++ b/examples/run_squad.py @@ -786,7 +786,7 @@ def main(): raise ValueError("Output directory () already exists and is not empty.") os.makedirs(args.output_dir, exist_ok=True) - tokenizer = BertTokenizer.from_pretrained(args.bert_model) + tokenizer = BertTokenizer.from_pretrained(args.bert_model, do_lower_case=args.do_lower_case) train_examples = None num_train_steps = None