Check if text is set to avoid IndexError (#4209)
Fix for https://github.com/huggingface/transformers/issues/3809
This commit is contained in:
@@ -236,7 +236,7 @@ class RobertaTokenizer(GPT2Tokenizer):
|
|||||||
add_prefix_space = kwargs["add_prefix_space"]
|
add_prefix_space = kwargs["add_prefix_space"]
|
||||||
else:
|
else:
|
||||||
add_prefix_space = add_special_tokens
|
add_prefix_space = add_special_tokens
|
||||||
if add_prefix_space and not text[0].isspace():
|
if add_prefix_space and len(text) > 0 and not text[0].isspace():
|
||||||
text = " " + text
|
text = " " + text
|
||||||
return text
|
return text
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user