SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Run SQuAD warning when the doc stride may be too high

Browse Source
This commit is contained in:
Lysandre
2020-01-16 13:59:26 -05:00
parent b8f43cb273
commit 6e2c28a14a
1 changed files with 7 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
examples/run_squad.py
View File

@@ -671,6 +671,13 @@ def main():
parser.add_argument("--threads", type=int, default=1, help="multiple threads for converting example to features") parser.add_argument("--threads", type=int, default=1, help="multiple threads for converting example to features")
args = parser.parse_args() args = parser.parse_args()
if args.doc_stride >= args.max_seq_length - args.max_query_length:
logger.warning(
"WARNING - You've set a doc stride which may be superior to the document length in some "
"examples. This could result in errors when building features from the examples. Please reduce the doc "
"stride or increase the maximum length to ensure the features are correctly built."
)
if ( if (
os.path.exists(args.output_dir) os.path.exists(args.output_dir)
and os.listdir(args.output_dir) and os.listdir(args.output_dir)