Revert error back into warning for byte fallback conversion. (#22607)
This commit is contained in:
@@ -24,10 +24,12 @@ class ConvertSlowTokenizerTest(unittest.TestCase):
|
||||
|
||||
original_tokenizer_with_bytefallback = FakeOriginalTokenizer(vocab_file=spm_model_file_with_bytefallback)
|
||||
|
||||
with self.assertRaises(RuntimeError) as cm:
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
_ = SpmConverter(original_tokenizer_with_bytefallback)
|
||||
self.assertEqual(len(w), 1)
|
||||
|
||||
self.assertIn(
|
||||
"The sentencepiece tokenizer that you are converting to a fast tokenizer uses the byte fallback option"
|
||||
" which is not implemented in the fast tokenizers.",
|
||||
str(cm.exception),
|
||||
str(w[0].message),
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user