SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

[tests] remove TF tests (uses of require_tf) (#38944)

Browse Source 
* remove uses of require_tf

* remove redundant import guards

* this class has no tests

* nits

* del tf rng comment
This commit is contained in:
Joao Gante
2025-06-25 18:29:10 +01:00
committed by GitHub
parent d37f751797
commit 1d45d90e5d
44 changed files with 21 additions and 2504 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
tests/test_tokenization_common.py
View File

@@ -53,7 +53,6 @@ from transformers.testing_utils import (
get_tests_dir,
require_jinja,
require_read_token,
require_tf,
require_tokenizers,
require_torch,
run_test_in_subprocess,
@@ -3106,40 +3105,6 @@ class TokenizerTesterMixin:
# model(**encoded_sequence_fast)
# model(**batch_encoded_sequence_fast)
@require_tf
@slow
def test_tf_encode_plus_sent_to_model(self):
from transformers import TF_MODEL_MAPPING, TOKENIZER_MAPPING
MODEL_TOKENIZER_MAPPING = merge_model_tokenizer_mappings(TF_MODEL_MAPPING, TOKENIZER_MAPPING)
tokenizers = self.get_tokenizers(do_lower_case=False)
for tokenizer in tokenizers:
with self.subTest(f"{tokenizer.__class__.__name__}"):
if tokenizer.__class__ not in MODEL_TOKENIZER_MAPPING:
self.skipTest(f"{tokenizer.__class__.__name__} is not in the MODEL_TOKENIZER_MAPPING")
config_class, model_class = MODEL_TOKENIZER_MAPPING[tokenizer.__class__]
config = config_class()
if config.is_encoder_decoder or config.pad_token_id is None:
self.skipTest(reason="Model is not an encoder-decoder model or has no set pad token id")
model = model_class(config)
# Make sure the model contains at least the full vocabulary size in its embedding matrix
self.assertGreaterEqual(model.config.vocab_size, len(tokenizer))
# Build sequence
first_ten_tokens = list(tokenizer.get_vocab().keys())[:10]
sequence = " ".join(first_ten_tokens)
encoded_sequence = tokenizer.encode_plus(sequence, return_tensors="tf")
batch_encoded_sequence = tokenizer.batch_encode_plus([sequence, sequence], return_tensors="tf")
# This should not fail
model(encoded_sequence)
model(batch_encoded_sequence)
# TODO: Check if require_torch is the best to test for numpy here ... Maybe move to require_flax when available
@require_torch
@slow