From 6e2c28a14a3d171e8c4d3838429abb1d69456df5 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 16 Jan 2020 13:59:26 -0500 Subject: [PATCH] Run SQuAD warning when the doc stride may be too high --- examples/run_squad.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/examples/run_squad.py b/examples/run_squad.py index 0a621f9ee0..b713219d20 100644 --- a/examples/run_squad.py +++ b/examples/run_squad.py @@ -671,6 +671,13 @@ def main(): parser.add_argument("--threads", type=int, default=1, help="multiple threads for converting example to features") args = parser.parse_args() + if args.doc_stride >= args.max_seq_length - args.max_query_length: + logger.warning( + "WARNING - You've set a doc stride which may be superior to the document length in some " + "examples. This could result in errors when building features from the examples. Please reduce the doc " + "stride or increase the maximum length to ensure the features are correctly built." + ) + if ( os.path.exists(args.output_dir) and os.listdir(args.output_dir)