Release: v4.22.1

* Fix tokenizer load from one file

* Add a test

* Style

Co-authored-by: Lysandre <lysandre.debut@reseau.eseo.fr>

README.md

@@ -249,6 +249,8 @@ conda install -c huggingface transformers
 
 Follow the installation pages of Flax, PyTorch or TensorFlow to see how to install them with conda.
 
+> **_NOTE:_**  On Windows, you may be prompted to activate Developer Mode in order to benefit from caching. If this is not an option for you, please let us know in [this issue](https://github.com/huggingface/huggingface_hub/issues/1062).
+
 ## Model architectures
 
 **[All the model checkpoints](https://huggingface.co/models)** provided by 🤗 Transformers are seamlessly integrated from the huggingface.co [model hub](https://huggingface.co) where they are uploaded directly by [users](https://huggingface.co/users) and [organizations](https://huggingface.co/organizations).

setup.py

@@ -400,7 +400,7 @@ install_requires = [
 
 setup(
     name="transformers",
-    version="4.22.0",  # expected format is one of x.y.z.dev0, or x.y.z.rc1 or x.y.z (no to dashes, yes to dots)
+    version="4.22.1",  # expected format is one of x.y.z.dev0, or x.y.z.rc1 or x.y.z (no to dashes, yes to dots)
     author="The Hugging Face team (past and future) with the help of all our contributors (https://github.com/huggingface/transformers/graphs/contributors)",
     author_email="transformers@huggingface.co",
     description="State-of-the-art Machine Learning for JAX, PyTorch and TensorFlow",

src/transformers/__init__.py

@@ -22,7 +22,7 @@
 # to defer the actual importing for when the objects are requested. This way `import transformers` provides the names
 # in the namespace without actually importing anything (and especially none of the backends).
 
-__version__ = "4.22.0"
+__version__ = "4.22.1"
 
 from typing import TYPE_CHECKING
 

src/transformers/convert_slow_tokenizer.py

@@ -282,8 +282,20 @@ class GPT2Converter(Converter):
 
         tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=self.original_tokenizer.add_prefix_space)
         tokenizer.decoder = decoders.ByteLevel()
-        tokenizer.post_processor = processors.ByteLevel(trim_offsets=False)
-
+        if self.original_tokenizer.add_bos_token:
+            bos = self.original_tokenizer.bos_token
+            bos_token_id = self.original_tokenizer.bos_token_id
+            tokenizer.post_processor = processors.TemplateProcessing(
+                single=f"{bos}:0 $A:0",  # token_type_id is 2 for Funnel transformer
+                pair=f"{bos}:0 $A:0 $B:1",
+                special_tokens=[
+                    (bos, bos_token_id),
+                ],
+            )
+        else:
+            # XXX trim_offsets=False actually means this post_processor doesn't
+            # really do anything.
+            tokenizer.post_processor = processors.ByteLevel(trim_offsets=False)
         return tokenizer
 
 

src/transformers/modeling_flax_utils.py

@@ -680,7 +680,7 @@ class FlaxPreTrainedModel(PushToHubMixin, FlaxGenerationMixin):
                 archive_file = pretrained_model_name_or_path
                 is_local = True
             elif is_remote_url(pretrained_model_name_or_path):
-                archive_file = pretrained_model_name_or_path
+                filename = pretrained_model_name_or_path
                 resolved_archive_file = download_url(pretrained_model_name_or_path)
             else:
                 filename = WEIGHTS_NAME if from_pt else FLAX_WEIGHTS_NAME

src/transformers/modeling_tf_utils.py

@@ -2348,7 +2348,7 @@ class TFPreTrainedModel(tf.keras.Model, TFModelUtilsMixin, TFGenerationMixin, Pu
                 archive_file = pretrained_model_name_or_path + ".index"
                 is_local = True
             elif is_remote_url(pretrained_model_name_or_path):
-                archive_file = pretrained_model_name_or_path
+                filename = pretrained_model_name_or_path
                 resolved_archive_file = download_url(pretrained_model_name_or_path)
             else:
                 # set correct filename

src/transformers/modeling_utils.py

@@ -2001,7 +2001,7 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin, GenerationMixin, PushToHubMix
                 archive_file = os.path.join(subfolder, pretrained_model_name_or_path + ".index")
                 is_local = True
             elif is_remote_url(pretrained_model_name_or_path):
-                archive_file = pretrained_model_name_or_path
+                filename = pretrained_model_name_or_path
                 resolved_archive_file = download_url(pretrained_model_name_or_path)
             else:
                 # set correct filename

src/transformers/models/gpt2/tokenization_gpt2_fast.py

@@ -146,16 +146,7 @@ class GPT2TokenizerFast(PreTrainedTokenizerFast):
             **kwargs,
         )
 
-        if kwargs.pop("add_bos_token", False):
-            model_id = kwargs.pop("name_or_path", "")
-            raise ValueError(
-                "Currenty GPT2's fast tokenizer does NOT support adding a BOS token."
-                "Instead you should use GPT2's slow tokenizer class `GPT2Tokenizer` as follows: \n"
-                f"`GPT2Tokenizer.from_pretrained('{model_id}')`\nor\n"
-                f"`AutoTokenizer.from_pretrained('{model_id}', use_fast=False)`\n"
-                "This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005."
-                " so that the fast tokenizer works correctly."
-            )
+        self.add_bos_token = kwargs.pop("add_bos_token", False)
 
         pre_tok_state = json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__())
         if pre_tok_state.get("add_prefix_space", add_prefix_space) != add_prefix_space:

src/transformers/tokenization_utils_base.py

@@ -1670,7 +1670,7 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
         init_configuration = {}
 
         is_local = os.path.isdir(pretrained_model_name_or_path)
-        if os.path.isfile(pretrained_model_name_or_path):
+        if os.path.isfile(pretrained_model_name_or_path) or is_remote_url(pretrained_model_name_or_path):
             if len(cls.vocab_files_names) > 1:
                 raise ValueError(
                     f"Calling {cls.__name__}.from_pretrained() with the path to a single file or url is not "
@@ -1726,6 +1726,8 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
         for file_id, file_path in vocab_files.items():
             if file_path is None:
                 resolved_vocab_files[file_id] = None
+            elif os.path.isfile(file_path):
+                resolved_vocab_files[file_id] = file_path
             elif is_remote_url(file_path):
                 resolved_vocab_files[file_id] = download_url(file_path, proxies=proxies)
             else:

src/transformers/utils/hub.py

@@ -1128,9 +1128,9 @@ if cache_version < 1:
     except Exception as e:
         trace = "\n".join(traceback.format_tb(e.__traceback__))
         logger.error(
-            f"There was a problem when trying to move your cache:\n\n{trace}\n\nPlease file an issue at "
-            "https://github.com/huggingface/transformers/issues/new/choose and copy paste this whole message and we "
-            "will do our best to help."
+            f"There was a problem when trying to move your cache:\n\n{trace}\n{e.__class__.__name__}: {e}\n\nPlease "
+            "file an issue at https://github.com/huggingface/transformers/issues/new/choose and copy paste this whole "
+            "message and we will do our best to help."
         )
 
     try:

tests/models/gpt2/test_tokenization_gpt2.py

@@ -18,7 +18,7 @@ import json
 import os
 import unittest
 
-from transformers import GPT2Tokenizer, GPT2TokenizerFast
+from transformers import AutoTokenizer, GPT2Tokenizer, GPT2TokenizerFast
 from transformers.models.gpt2.tokenization_gpt2 import VOCAB_FILES_NAMES
 from transformers.testing_utils import require_tokenizers
 
@@ -275,3 +275,57 @@ class GPT2TokenizationTest(TokenizerTesterMixin, unittest.TestCase):
                 ]
                 filtered_sequence = [x for x in filtered_sequence if x is not None]
                 self.assertEqual(encoded_sequence, filtered_sequence)
+
+
+@require_tokenizers
+class OPTTokenizationTest(unittest.TestCase):
+    def test_serialize_deserialize_fast_opt(self):
+        # More context:
+        # https://huggingface.co/wjmcat/opt-350m-paddle/discussions/1
+        # https://huggingface.slack.com/archives/C01N44FJDHT/p1653511495183519
+        # https://github.com/huggingface/transformers/pull/17088#discussion_r871246439
+
+        tokenizer = AutoTokenizer.from_pretrained("facebook/opt-350m", from_slow=True)
+        text = "A photo of a cat"
+
+        tokens_ids = tokenizer.encode(
+            text,
+        )
+        self.assertEqual(tokens_ids, [2, 250, 1345, 9, 10, 4758])
+        tokenizer.save_pretrained("test_opt")
+
+        tokenizer = AutoTokenizer.from_pretrained("./test_opt")
+        tokens_ids = tokenizer.encode(
+            text,
+        )
+        self.assertEqual(tokens_ids, [2, 250, 1345, 9, 10, 4758])
+
+    def test_fast_slow_equivalence(self):
+        tokenizer = AutoTokenizer.from_pretrained("facebook/opt-350m", use_slow=True)
+        text = "A photo of a cat"
+
+        tokens_ids = tokenizer.encode(
+            text,
+        )
+        # Same as above
+        self.assertEqual(tokens_ids, [2, 250, 1345, 9, 10, 4758])
+
+    def test_users_can_modify_bos(self):
+        tokenizer = AutoTokenizer.from_pretrained("facebook/opt-350m", from_slow=True)
+
+        tokenizer.bos_token = "bos"
+        tokenizer.bos_token_id = tokenizer.get_vocab()["bos"]
+
+        text = "A photo of a cat"
+        tokens_ids = tokenizer.encode(
+            text,
+        )
+        # We changed the bos token
+        self.assertEqual(tokens_ids, [31957, 250, 1345, 9, 10, 4758])
+        tokenizer.save_pretrained("./tok")
+        tokenizer = AutoTokenizer.from_pretrained("./tok")
+        self.assertTrue(tokenizer.is_fast)
+        tokens_ids = tokenizer.encode(
+            text,
+        )
+        self.assertEqual(tokens_ids, [31957, 250, 1345, 9, 10, 4758])

tests/test_configuration_common.py

@@ -360,6 +360,12 @@ class ConfigTestUtils(unittest.TestCase):
             # This check we did call the fake head request
             mock_head.assert_called()
 
+    def test_legacy_load_from_url(self):
+        # This test is for deprecated behavior and can be removed in v5
+        _ = BertConfig.from_pretrained(
+            "https://huggingface.co/hf-internal-testing/tiny-random-bert/resolve/main/config.json"
+        )
+
 
 class ConfigurationVersioningTest(unittest.TestCase):
     def test_local_versioning(self):

tests/test_feature_extraction_common.py

@@ -182,6 +182,12 @@ class FeatureExtractorUtilTester(unittest.TestCase):
             # This check we did call the fake head request
             mock_head.assert_called()
 
+    def test_legacy_load_from_url(self):
+        # This test is for deprecated behavior and can be removed in v5
+        _ = Wav2Vec2FeatureExtractor.from_pretrained(
+            "https://huggingface.co/hf-internal-testing/tiny-random-wav2vec2/resolve/main/preprocessor_config.json"
+        )
+
 
 @is_staging_test
 class FeatureExtractorPushToHubTester(unittest.TestCase):

tests/test_modeling_common.py

@@ -33,6 +33,7 @@ import numpy as np
 
 import transformers
 from huggingface_hub import HfFolder, delete_repo, set_access_token
+from huggingface_hub.file_download import http_get
 from requests.exceptions import HTTPError
 from transformers import (
     AutoConfig,
@@ -2949,6 +2950,26 @@ class ModelUtilsTest(TestCasePlus):
             # This check we did call the fake head request
             mock_head.assert_called()
 
+    def test_load_from_one_file(self):
+        try:
+            tmp_file = tempfile.mktemp()
+            with open(tmp_file, "wb") as f:
+                http_get(
+                    "https://huggingface.co/hf-internal-testing/tiny-random-bert/resolve/main/pytorch_model.bin", f
+                )
+
+            config = BertConfig.from_pretrained("hf-internal-testing/tiny-random-bert")
+            _ = BertModel.from_pretrained(tmp_file, config=config)
+        finally:
+            os.remove(tmp_file)
+
+    def test_legacy_load_from_url(self):
+        # This test is for deprecated behavior and can be removed in v5
+        config = BertConfig.from_pretrained("hf-internal-testing/tiny-random-bert")
+        _ = BertModel.from_pretrained(
+            "https://huggingface.co/hf-internal-testing/tiny-random-bert/resolve/main/pytorch_model.bin", config=config
+        )
+
 
 @require_torch
 @is_staging_test

tests/test_modeling_tf_common.py

@@ -30,6 +30,7 @@ from typing import List, Tuple, get_type_hints
 from datasets import Dataset
 
 from huggingface_hub import HfFolder, Repository, delete_repo, set_access_token
+from huggingface_hub.file_download import http_get
 from requests.exceptions import HTTPError
 from transformers import is_tf_available, is_torch_available
 from transformers.configuration_utils import PretrainedConfig
@@ -1868,6 +1869,24 @@ class UtilsFunctionsTest(unittest.TestCase):
             # This check we did call the fake head request
             mock_head.assert_called()
 
+    def test_load_from_one_file(self):
+        try:
+            tmp_file = tempfile.mktemp()
+            with open(tmp_file, "wb") as f:
+                http_get("https://huggingface.co/hf-internal-testing/tiny-random-bert/resolve/main/tf_model.h5", f)
+
+            config = BertConfig.from_pretrained("hf-internal-testing/tiny-random-bert")
+            _ = TFBertModel.from_pretrained(tmp_file, config=config)
+        finally:
+            os.remove(tmp_file)
+
+    def test_legacy_load_from_url(self):
+        # This test is for deprecated behavior and can be removed in v5
+        config = BertConfig.from_pretrained("hf-internal-testing/tiny-random-bert")
+        _ = TFBertModel.from_pretrained(
+            "https://huggingface.co/hf-internal-testing/tiny-random-bert/resolve/main/tf_model.h5", config=config
+        )
+
     # tests whether the unpack_inputs function behaves as expected
     def test_unpack_inputs(self):
         class DummyModel:

tests/test_tokenization_common.py

@@ -31,6 +31,7 @@ from pathlib import Path
 from typing import TYPE_CHECKING, Any, Dict, List, Tuple, Union
 
 from huggingface_hub import HfFolder, delete_repo, set_access_token
+from huggingface_hub.file_download import http_get
 from parameterized import parameterized
 from requests.exceptions import HTTPError
 from transformers import (
@@ -3886,6 +3887,21 @@ class TokenizerUtilTester(unittest.TestCase):
             # This check we did call the fake head request
             mock_head.assert_called()
 
+    def test_legacy_load_from_one_file(self):
+        # This test is for deprecated behavior and can be removed in v5
+        try:
+            tmp_file = tempfile.mktemp()
+            with open(tmp_file, "wb") as f:
+                http_get("https://huggingface.co/albert-base-v1/resolve/main/spiece.model", f)
+
+            _ = AlbertTokenizer.from_pretrained(tmp_file)
+        finally:
+            os.remove(tmp_file)
+
+    def test_legacy_load_from_url(self):
+        # This test is for deprecated behavior and can be removed in v5
+        _ = AlbertTokenizer.from_pretrained("https://huggingface.co/albert-base-v1/resolve/main/spiece.model")
+
 
 @is_staging_test
 class TokenizerPushToHubTester(unittest.TestCase):