From a7738f5a89640ea2d8e012fba746e3d5242a40d6 Mon Sep 17 00:00:00 2001 From: Mohamed Mekkouri <93391238+MekkCyber@users.noreply.github.com> Date: Wed, 22 Jan 2025 12:28:40 +0100 Subject: [PATCH] Fix : Nemotron tokenizer for GGUF format (#35836) fix nemotron gguf --- src/transformers/models/auto/tokenization_auto.py | 1 + tests/quantization/ggml/test_ggml.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py index 9ce9edd06c..99a5c0034d 100644 --- a/src/transformers/models/auto/tokenization_auto.py +++ b/src/transformers/models/auto/tokenization_auto.py @@ -339,6 +339,7 @@ else: ("musicgen_melody", ("T5Tokenizer", "T5TokenizerFast" if is_tokenizers_available() else None)), ("mvp", ("MvpTokenizer", "MvpTokenizerFast" if is_tokenizers_available() else None)), ("myt5", ("MyT5Tokenizer", None)), + ("nemotron", (None, "PreTrainedTokenizerFast" if is_tokenizers_available() else None)), ("nezha", ("BertTokenizer", "BertTokenizerFast" if is_tokenizers_available() else None)), ( "nllb", diff --git a/tests/quantization/ggml/test_ggml.py b/tests/quantization/ggml/test_ggml.py index 08e87ba44a..12648a8aac 100644 --- a/tests/quantization/ggml/test_ggml.py +++ b/tests/quantization/ggml/test_ggml.py @@ -835,9 +835,9 @@ class GgufIntegrationTests(unittest.TestCase): tokenizer = AutoTokenizer.from_pretrained(self.nemotron_model_id, gguf_file=self.q6_k_nemotron_model_id) text = tokenizer(self.example_text, return_tensors="pt")["input_ids"] - out = model.generate(text, max_new_tokens=10) + out = model.generate(text, max_new_tokens=16) - EXPECTED_TEXT = "'Hello. hotmail.com.'" + EXPECTED_TEXT = "Hello.ā–hotmail.com" self.assertEqual(tokenizer.decode(out[0], skip_special_tokens=True), EXPECTED_TEXT) def test_gemma2_q3_k(self):