SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Fix broken CI on release branch due to missing conversion files (#36275)

Browse Source 
* fix

* fix

---------

Co-authored-by: ydshieh <ydshieh@users.noreply.github.com>
This commit is contained in:
Yih-Dar
2025-02-20 13:22:10 +01:00
committed by GitHub
parent 5e2183f344
commit e8531a0e33
1 changed files with 0 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
tests/models/marian/test_modeling_marian.py
View File

@@ -17,8 +17,6 @@
import tempfile import tempfile
import unittest import unittest
from huggingface_hub.hf_api import list_models
from transformers import MarianConfig, is_torch_available from transformers import MarianConfig, is_torch_available
from transformers.testing_utils import ( from transformers.testing_utils import (
require_sentencepiece, require_sentencepiece,
@@ -47,11 +45,6 @@ if is_torch_available():
MarianMTModel, MarianMTModel,
TranslationPipeline, TranslationPipeline,
) )
from transformers.models.marian.convert_marian_to_pytorch import (
ORG_NAME,
convert_hf_name_to_opus_name,
convert_opus_name_to_hf_name,
)
from transformers.models.marian.modeling_marian import ( from transformers.models.marian.modeling_marian import (
MarianDecoder, MarianDecoder,
MarianEncoder, MarianEncoder,
@@ -395,17 +388,6 @@ def _long_tensor(tok_lst):
return torch.tensor(tok_lst, dtype=torch.long, device=torch_device) return torch.tensor(tok_lst, dtype=torch.long, device=torch_device)
class ModelManagementTests(unittest.TestCase):
@slow
@require_torch
def test_model_names(self):
model_list = list_models()
model_ids = [x.id for x in model_list if x.id.startswith(ORG_NAME)]
bad_model_ids = [mid for mid in model_ids if "+" in model_ids]
self.assertListEqual([], bad_model_ids)
self.assertGreater(len(model_ids), 500)
@require_torch @require_torch
@require_sentencepiece @require_sentencepiece
@require_tokenizers @require_tokenizers
@@ -654,30 +636,6 @@ class TestMarian_FI_EN_V2(MarianIntegrationTest):
self._assert_generated_batch_equal_expected() self._assert_generated_batch_equal_expected()
@require_torch
class TestConversionUtils(unittest.TestCase):
def test_renaming_multilingual(self):
old_names = [
"opus-mt-cmn+cn+yue+ze_zh+zh_cn+zh_CN+zh_HK+zh_tw+zh_TW+zh_yue+zhs+zht+zh-fi",
"opus-mt-cmn+cn-fi", # no group
"opus-mt-en-de", # standard name
"opus-mt-en-de", # standard name
]
expected = ["opus-mt-ZH-fi", "opus-mt-cmn_cn-fi", "opus-mt-en-de", "opus-mt-en-de"]
self.assertListEqual(expected, [convert_opus_name_to_hf_name(x) for x in old_names])
def test_undoing_renaming(self):
hf_names = ["opus-mt-ZH-fi", "opus-mt-cmn_cn-fi", "opus-mt-en-de", "opus-mt-en-de"]
converted_opus_names = [convert_hf_name_to_opus_name(x) for x in hf_names]
expected_opus_names = [
"cmn+cn+yue+ze_zh+zh_cn+zh_CN+zh_HK+zh_tw+zh_TW+zh_yue+zhs+zht+zh-fi",
"cmn+cn-fi",
"en-de", # standard name
"en-de",
]
self.assertListEqual(expected_opus_names, converted_opus_names)
class MarianStandaloneDecoderModelTester: class MarianStandaloneDecoderModelTester:
def __init__( def __init__(
self, self,