Revert error back into warning for byte fallback conversion. (#22607)

This commit is contained in:
Nicolas Patry
2023-04-06 14:00:29 +02:00
committed by GitHub
parent 1670be4bde
commit 0aa1153ffb
2 changed files with 6 additions and 3 deletions

View File

@@ -24,10 +24,12 @@ class ConvertSlowTokenizerTest(unittest.TestCase):
original_tokenizer_with_bytefallback = FakeOriginalTokenizer(vocab_file=spm_model_file_with_bytefallback)
with self.assertRaises(RuntimeError) as cm:
with warnings.catch_warnings(record=True) as w:
_ = SpmConverter(original_tokenizer_with_bytefallback)
self.assertEqual(len(w), 1)
self.assertIn(
"The sentencepiece tokenizer that you are converting to a fast tokenizer uses the byte fallback option"
" which is not implemented in the fast tokenizers.",
str(cm.exception),
str(w[0].message),
)