From 454455c695ff38df1ed3670a43677fdd1abcedf3 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Wed, 20 Nov 2019 09:42:48 -0500 Subject: [PATCH] fix #1879 --- examples/utils_squad.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/utils_squad.py b/examples/utils_squad.py index c04dacf6d3..4f1c581588 100644 --- a/examples/utils_squad.py +++ b/examples/utils_squad.py @@ -240,6 +240,7 @@ def convert_examples_to_features(examples, tokenizer, max_seq_length, # The -3 accounts for [CLS], [SEP] and [SEP] max_tokens_for_doc = max_seq_length - len(query_tokens) - 3 + assert max_tokens_for_doc > 0 # We can have documents that are longer than the maximum sequence length. # To deal with this we do a sliding window approach, where we take chunks