Avoid using get_list_of_files (#15287)

* Avoid using get_list_of_files in config

* Wip, change tokenizer file getter

* Remove call in tokenizer files

* Remove last call to get_list_model_files

* Better tests

* Unit tests for new function

* Document bad API

tests/test_configuration_common.py

@@ -313,6 +313,7 @@ class ConfigTestUtils(unittest.TestCase):
 class ConfigurationVersioningTest(unittest.TestCase):
     def test_local_versioning(self):
         configuration = AutoConfig.from_pretrained("bert-base-cased")
+        configuration.configuration_files = ["config.4.0.0.json"]
 
         with tempfile.TemporaryDirectory() as tmp_dir:
             configuration.save_pretrained(tmp_dir)
@@ -325,23 +326,26 @@ class ConfigurationVersioningTest(unittest.TestCase):
 
             # Will need to be adjusted if we reach v42 and this test is still here.
             # Should pick the old configuration file as the version of Transformers is < 4.42.0
+            configuration.configuration_files = ["config.42.0.0.json"]
+            configuration.hidden_size = 768
+            configuration.save_pretrained(tmp_dir)
             shutil.move(os.path.join(tmp_dir, "config.4.0.0.json"), os.path.join(tmp_dir, "config.42.0.0.json"))
             new_configuration = AutoConfig.from_pretrained(tmp_dir)
             self.assertEqual(new_configuration.hidden_size, 768)
 
     def test_repo_versioning_before(self):
-        # This repo has two configuration files, one for v5.0.0 and above with an added token, one for versions lower.
-        repo = "microsoft/layoutxlm-base"
+        # This repo has two configuration files, one for v4.0.0 and above with a different hidden size.
+        repo = "hf-internal-testing/test-two-configs"
 
         import transformers as new_transformers
 
-        new_transformers.configuration_utils.__version__ = "v5.0.0"
+        new_transformers.configuration_utils.__version__ = "v4.0.0"
         new_configuration = new_transformers.models.auto.AutoConfig.from_pretrained(repo)
-        self.assertEqual(new_configuration.tokenizer_class, None)
+        self.assertEqual(new_configuration.hidden_size, 2)
 
         # Testing an older version by monkey-patching the version in the module it's used.
         import transformers as old_transformers
 
         old_transformers.configuration_utils.__version__ = "v3.0.0"
         old_configuration = old_transformers.models.auto.AutoConfig.from_pretrained(repo)
-        self.assertEqual(old_configuration.tokenizer_class, "XLMRobertaTokenizer")
+        self.assertEqual(old_configuration.hidden_size, 768)

tests/test_file_utils.py

@@ -15,7 +15,10 @@
 import contextlib
 import importlib
 import io
+import json
+import tempfile
 import unittest
+from pathlib import Path
 
 import transformers
 
@@ -31,6 +34,7 @@ from transformers.file_utils import (
     RepositoryNotFoundError,
     RevisionNotFoundError,
     filename_to_url,
+    get_file_from_repo,
     get_from_cache,
     has_file,
     hf_bucket_url,
@@ -128,6 +132,31 @@ class GetFromCacheTests(unittest.TestCase):
         self.assertFalse(has_file("hf-internal-testing/tiny-bert-pt-only", TF2_WEIGHTS_NAME))
         self.assertFalse(has_file("hf-internal-testing/tiny-bert-pt-only", FLAX_WEIGHTS_NAME))
 
+    def test_get_file_from_repo_distant(self):
+        # `get_file_from_repo` returns None if the file does not exist
+        self.assertIsNone(get_file_from_repo("bert-base-cased", "ahah.txt"))
+
+        # The function raises if the repository does not exist.
+        with self.assertRaisesRegex(EnvironmentError, "is not a valid model identifier"):
+            get_file_from_repo("bert-base-case", "config.json")
+
+        # The function raises if the revision does not exist.
+        with self.assertRaisesRegex(EnvironmentError, "is not a valid git identifier"):
+            get_file_from_repo("bert-base-cased", "config.json", revision="ahaha")
+
+        resolved_file = get_file_from_repo("bert-base-cased", "config.json")
+        # The name is the cached name which is not very easy to test, so instead we load the content.
+        config = json.loads(open(resolved_file, "r").read())
+        self.assertEqual(config["hidden_size"], 768)
+
+    def test_get_file_from_repo_local(self):
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            filename = Path(tmp_dir) / "a.txt"
+            filename.touch()
+            self.assertEqual(get_file_from_repo(tmp_dir, "a.txt"), str(filename))
+
+            self.assertIsNone(get_file_from_repo(tmp_dir, "b.txt"))
+
 
 class ContextManagerTests(unittest.TestCase):
     @unittest.mock.patch("sys.stdout", new_callable=io.StringIO)

tests/test_tokenization_fast.py

@@ -108,6 +108,8 @@ class TokenizerVersioningTest(unittest.TestCase):
         json_tokenizer["model"]["vocab"]["huggingface"] = len(tokenizer)
 
         with tempfile.TemporaryDirectory() as tmp_dir:
+            # Hack to save this in the tokenizer_config.json
+            tokenizer.init_kwargs["fast_tokenizer_files"] = ["tokenizer.4.0.0.json"]
             tokenizer.save_pretrained(tmp_dir)
             json.dump(json_tokenizer, open(os.path.join(tmp_dir, "tokenizer.4.0.0.json"), "w"))
 
@@ -120,6 +122,8 @@ class TokenizerVersioningTest(unittest.TestCase):
             # Will need to be adjusted if we reach v42 and this test is still here.
             # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
             shutil.move(os.path.join(tmp_dir, "tokenizer.4.0.0.json"), os.path.join(tmp_dir, "tokenizer.42.0.0.json"))
+            tokenizer.init_kwargs["fast_tokenizer_files"] = ["tokenizer.42.0.0.json"]
+            tokenizer.save_pretrained(tmp_dir)
             new_tokenizer = AutoTokenizer.from_pretrained(tmp_dir)
             self.assertEqual(len(new_tokenizer), len(tokenizer))
             json_tokenizer = json.loads(new_tokenizer._tokenizer.to_str())
@@ -127,7 +131,7 @@ class TokenizerVersioningTest(unittest.TestCase):
 
     def test_repo_versioning(self):
         # This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
-        repo = "sgugger/finetuned-bert-mrpc"
+        repo = "hf-internal-testing/test-two-tokenizers"
 
         # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
         tokenizer = AutoTokenizer.from_pretrained(repo)