Further reduce the number of alls to head for cached objects (#18871)

* Further reduce the number of alls to head for cached models/tokenizers/pipelines

* Fix tests

* Address review comments

tests/pipelines/test_pipelines_common.py

@@ -884,8 +884,7 @@ class CustomPipelineTest(unittest.TestCase):
         with RequestCounter() as counter:
             _ = pipeline("text-classification", model="hf-internal-testing/tiny-random-bert")
             self.assertEqual(counter.get_request_count, 0)
-            # We still have one extra call because the model does not have a added_tokens.json file
-            self.assertEqual(counter.head_request_count, 2)
+            self.assertEqual(counter.head_request_count, 1)
             self.assertEqual(counter.other_request_count, 0)