SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

fix: Replace add_prefix_space in get_prompt_ids with manual space for FastTokenizer compatibility (#23796)

Browse Source 
* add ' ' replacement for add_prefix_space

* add fast tokenizer test
This commit is contained in:
Connor Henderson
2023-05-31 10:52:35 -04:00
committed by GitHub
parent 84bac652f3
commit 7adce8b532
3 changed files with 12 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
tests/models/whisper/test_tokenization_whisper.py
View File

@@ -213,6 +213,16 @@ class WhisperTokenizerTest(TokenizerTesterMixin, unittest.TestCase):
rust_tokenizer.decode(encoded_input, skip_special_tokens=True), expected_without_special_tokens
)
def test_fast_tokenizer_get_prompt_ids(self):
tokenizer = self.get_tokenizer()
rust_tokenizer = self.get_rust_tokenizer()
prompt = "This is test prompt text."
tokenizer_prompt_ids = tokenizer.get_prompt_ids(prompt)
fast_tokenizer_prompt_ids = rust_tokenizer.get_prompt_ids(prompt)
self.assertListEqual(tokenizer_prompt_ids.tolist(), fast_tokenizer_prompt_ids.tolist())
class SpeechToTextTokenizerMultilinguialTest(unittest.TestCase):
checkpoint_name = "openai/whisper-small.en"