Fix Cohere CI (#31263)

* [run-slow] cohere

* [run-slow] cohere

* [run-slow] cohere

---------

Co-authored-by: ydshieh <ydshieh@users.noreply.github.com>
This commit is contained in:
Yih-Dar
2024-06-10 15:16:58 +02:00
committed by GitHub
parent dc6eb44841
commit 8fff07ded0
4 changed files with 46 additions and 4 deletions

View File

@@ -16,7 +16,7 @@
import unittest
from transformers import CohereTokenizerFast
from transformers.testing_utils import require_jinja, require_tokenizers
from transformers.testing_utils import require_jinja, require_tokenizers, require_torch_multi_gpu
from ...test_tokenization_common import TokenizerTesterMixin
@@ -46,6 +46,11 @@ class CohereTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
kwargs.update(self.special_tokens_map)
return CohereTokenizerFast.from_pretrained(self.tmpdirname, **kwargs)
# This gives CPU OOM on a single-gpu runner (~60G RAM). On multi-gpu runner, it has ~180G RAM which is enough.
@require_torch_multi_gpu
def test_torch_encode_plus_sent_to_model(self):
super().test_torch_encode_plus_sent_to_model()
@unittest.skip("This needs a slow tokenizer. Cohere does not have one!")
def test_encode_decode_with_spaces(self):
return