SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

fix: Replace add_prefix_space in get_prompt_ids with manual space for FastTokenizer compatibility (#23796)

Browse Source 
* add ' ' replacement for add_prefix_space

* add fast tokenizer test
This commit is contained in:
Connor Henderson
2023-05-31 10:52:35 -04:00
committed by GitHub
parent 84bac652f3
commit 7adce8b532
3 changed files with 12 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/transformers/models/whisper/tokenization_whisper.py
View File

@@ -721,7 +721,7 @@ class WhisperTokenizer(PreTrainedTokenizer):
def get_prompt_ids(self, text: str, return_tensors="np"): def get_prompt_ids(self, text: str, return_tensors="np"):
"""Converts prompt text to IDs that can be passed to [`~WhisperForConditionalGeneration.generate`].""" """Converts prompt text to IDs that can be passed to [`~WhisperForConditionalGeneration.generate`]."""
batch_encoding = self("<|startofprev|>", text.strip(), add_prefix_space=True, add_special_tokens=False) batch_encoding = self("<|startofprev|>", " " + text.strip(), add_special_tokens=False)
# Check for special tokens # Check for special tokens
prompt_text_ids = batch_encoding["input_ids"][1:] prompt_text_ids = batch_encoding["input_ids"][1:]

2
src/transformers/models/whisper/tokenization_whisper_fast.py
View File

@@ -494,7 +494,7 @@ class WhisperTokenizerFast(PreTrainedTokenizerFast):
# Copied from transformers.models.whisper.tokenization_whisper.WhisperTokenizer.get_prompt_ids # Copied from transformers.models.whisper.tokenization_whisper.WhisperTokenizer.get_prompt_ids
def get_prompt_ids(self, text: str, return_tensors="np"): def get_prompt_ids(self, text: str, return_tensors="np"):
"""Converts prompt text to IDs that can be passed to [`~WhisperForConditionalGeneration.generate`].""" """Converts prompt text to IDs that can be passed to [`~WhisperForConditionalGeneration.generate`]."""
batch_encoding = self("<|startofprev|>", text.strip(), add_prefix_space=True, add_special_tokens=False) batch_encoding = self("<|startofprev|>", " " + text.strip(), add_special_tokens=False)
# Check for special tokens # Check for special tokens
prompt_text_ids = batch_encoding["input_ids"][1:] prompt_text_ids = batch_encoding["input_ids"][1:]

10
tests/models/whisper/test_tokenization_whisper.py
View File

@@ -213,6 +213,16 @@ class WhisperTokenizerTest(TokenizerTesterMixin, unittest.TestCase):
rust_tokenizer.decode(encoded_input, skip_special_tokens=True), expected_without_special_tokens rust_tokenizer.decode(encoded_input, skip_special_tokens=True), expected_without_special_tokens
) )
def test_fast_tokenizer_get_prompt_ids(self):
tokenizer = self.get_tokenizer()
rust_tokenizer = self.get_rust_tokenizer()
prompt = "This is test prompt text."
tokenizer_prompt_ids = tokenizer.get_prompt_ids(prompt)
fast_tokenizer_prompt_ids = rust_tokenizer.get_prompt_ids(prompt)
self.assertListEqual(tokenizer_prompt_ids.tolist(), fast_tokenizer_prompt_ids.tolist())
class SpeechToTextTokenizerMultilinguialTest(unittest.TestCase): class SpeechToTextTokenizerMultilinguialTest(unittest.TestCase):
checkpoint_name = "openai/whisper-small.en" checkpoint_name = "openai/whisper-small.en"