fix deberta 2 tokenizer integration test (#12017)

This commit is contained in:
Philip May
2021-06-07 10:55:55 +02:00
committed by GitHub
parent 20b6f3b80c
commit 3857f2b4e3
2 changed files with 4 additions and 1 deletions

View File

@@ -221,6 +221,9 @@ class TokenizerTesterMixin:
"The quick brown fox jumps over the lazy dog.",
]
if self.test_sentencepiece_ignore_case:
sequences = [sequence.lower() for sequence in sequences]
tokenizer_classes = [self.tokenizer_class]
if self.test_rust_tokenizer:
tokenizer_classes.append(self.rust_tokenizer_class)