Further reduce the number of alls to head for cached objects (#18871)

* Further reduce the number of alls to head for cached models/tokenizers/pipelines

* Fix tests

* Address review comments

tests/models/auto/test_modeling_auto.py

@@ -370,6 +370,5 @@ class AutoModelTest(unittest.TestCase):
         with RequestCounter() as counter:
             _ = AutoModel.from_pretrained("hf-internal-testing/tiny-random-bert-sharded")
             self.assertEqual(counter.get_request_count, 0)
-            # There is no pytorch_model.bin so we still get one call for this one.
-            self.assertEqual(counter.head_request_count, 2)
+            self.assertEqual(counter.head_request_count, 1)
             self.assertEqual(counter.other_request_count, 0)