[ci] Load pretrained models into the default (long-lived) cache

There's an inconsistency right now where:
- we load some models into CACHE_DIR
- and some models in the default cache
- and often, in both for the same models

When running the RUN_SLOW tests, this takes a lot of disk space, time, and bandwidth.

I'd rather always use the default cache

tests/test_modeling_tf_t5.py

@@ -20,7 +20,7 @@ from transformers import T5Config, is_tf_available
 
 from .test_configuration_common import ConfigTester
 from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
-from .utils import CACHE_DIR, require_tf, slow
+from .utils import require_tf, slow
 
 
 if is_tf_available():
@@ -260,7 +260,7 @@ class TFT5ModelTest(TFModelTesterMixin, unittest.TestCase):
     @slow
     def test_model_from_pretrained(self):
         for model_name in ["t5-small"]:
-            model = TFT5Model.from_pretrained(model_name, cache_dir=CACHE_DIR)
+            model = TFT5Model.from_pretrained(model_name)
             self.assertIsNotNone(model)