Fix list index out of range when padding nested empty lists (#13876)

* Fix index out of range when padding

* Apply suggestions from code review

* Style
This commit is contained in:
Li-Huai (Allan) Lin
2021-11-11 04:34:52 +08:00
committed by GitHub
parent bec02ff209
commit 9e37c5cdf8
2 changed files with 17 additions and 14 deletions

View File

@@ -2733,11 +2733,10 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
first_element = required_input[0]
if isinstance(first_element, (list, tuple)):
# first_element might be an empty list/tuple in some edge cases so we grab the first non empty element.
index = 0
while len(required_input[index]) == 0:
index += 1
if index < len(required_input):
first_element = required_input[index][0]
for item in required_input:
if len(item) != 0:
first_element = item[0]
break
# At this state, if `first_element` is still a list/tuple, it's an empty one so there is nothing to do.
if not isinstance(first_element, (int, list, tuple)):
if is_tf_available() and _is_tensorflow(first_element):