Further reduce the number of alls to head for cached objects (#18871)

* Further reduce the number of alls to head for cached models/tokenizers/pipelines

* Fix tests

* Address review comments

tests/models/auto/test_modeling_tf_auto.py

@@ -303,6 +303,5 @@ class TFAutoModelTest(unittest.TestCase):
         with RequestCounter() as counter:
             _ = TFAutoModel.from_pretrained("ArthurZ/tiny-random-bert-sharded")
             self.assertEqual(counter.get_request_count, 0)
-            # There is no pytorch_model.bin so we still get one call for this one.
-            self.assertEqual(counter.head_request_count, 2)
+            self.assertEqual(counter.head_request_count, 1)
             self.assertEqual(counter.other_request_count, 0)