From edf9ac11d473ad45974b4383bc16ceca62bfa7bd Mon Sep 17 00:00:00 2001 From: Lysandre Debut Date: Fri, 15 May 2020 09:49:11 -0400 Subject: [PATCH] Should return overflowing information for the log (#4385) --- examples/multiple-choice/utils_multiple_choice.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/examples/multiple-choice/utils_multiple_choice.py b/examples/multiple-choice/utils_multiple_choice.py index 05b808ad9e..1e86880ecc 100644 --- a/examples/multiple-choice/utils_multiple_choice.py +++ b/examples/multiple-choice/utils_multiple_choice.py @@ -535,7 +535,12 @@ def convert_examples_to_features( text_b = example.question + " " + ending inputs = tokenizer.encode_plus( - text_a, text_b, add_special_tokens=True, max_length=max_length, pad_to_max_length=True, + text_a, + text_b, + add_special_tokens=True, + max_length=max_length, + pad_to_max_length=True, + return_overflowing_tokens=True, ) if "num_truncated_tokens" in inputs and inputs["num_truncated_tokens"] > 0: logger.info(