enable finegrained_fp8 and granite_speech cases on XPU (#38036)

* enable finegrained_fp8 cases on XPU

Signed-off-by: Yao Matrix <matrix.yao@intel.com>

* fix style

Signed-off-by: Yao Matrix <matrix.yao@intel.com>

* change back to auto

Signed-off-by: Yao Matrix <matrix.yao@intel.com>

* rename per comments

Signed-off-by: Matrix Yao <matrix.yao@intel.com>

---------

Signed-off-by: Yao Matrix <matrix.yao@intel.com>
Signed-off-by: Matrix Yao <matrix.yao@intel.com>
Co-authored-by: Marc Sun <57196510+SunMarc@users.noreply.github.com>

tests/models/granite_speech/test_processor_granite_speech.py

@@ -23,8 +23,9 @@ from parameterized import parameterized
 from transformers import AutoTokenizer, GPT2TokenizerFast
 from transformers.testing_utils import (
     require_torch,
-    require_torch_gpu,
+    require_torch_accelerator,
     require_torchaudio,
+    torch_device,
 )
 from transformers.utils import is_torchaudio_available
 
@@ -195,7 +196,7 @@ class GraniteSpeechProcessorTest(unittest.TestCase):
         assert num_calculated_features == [90, 171]
         assert sum(num_expected_features) == num_audio_tokens
 
-    @require_torch_gpu
+    @require_torch_accelerator
     def test_device_override(self):
         """Ensure that we regardless of the processing device, the tensors
         produced are on the CPU.
@@ -214,7 +215,7 @@ class GraniteSpeechProcessorTest(unittest.TestCase):
             text=f"{processor.audio_token} Can you transcribe this audio?",
             audio=wav,
             return_tensors="pt",
-            device="cuda",
+            device=torch_device,
         )
 
         assert inputs["input_features"].device.type == "cpu"