Delete deprecated stuff (#38838)
* delete deprecated stuff * fix copies * remove unused tests * fix modernbert and fuyu * Update src/transformers/cache_utils.py Co-authored-by: Joao Gante <joaofranciscocardosogante@gmail.com> * bye bye `seen_tokens` * address comments * update typings * ecnoder decoder models follow same pattern as whisper * fix copies * why is it set to False? * fix switch transformers * fix encoder decoder models shared weight * fix copies and RAG * remove `next_cache` * fix gptj/git * fix copies * fix copies * style... * another forgotten docsrting --------- Co-authored-by: Joao Gante <joaofranciscocardosogante@gmail.com>
This commit is contained in:
Raushan Turganbay
committed by
GitHub
GitHub
parent
c6ee0b1da8
commit
bc161d5d06
@@ -2,8 +2,6 @@ import shutil
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
import pytest
|
||||
|
||||
from transformers import Owlv2Processor
|
||||
from transformers.testing_utils import require_scipy
|
||||
|
||||
@@ -23,18 +21,3 @@ class Owlv2ProcessorTest(ProcessorTesterMixin, unittest.TestCase):
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
shutil.rmtree(cls.tmpdirname, ignore_errors=True)
|
||||
|
||||
def test_processor_query_images_positional(self):
|
||||
processor_components = self.prepare_components()
|
||||
processor = Owlv2Processor(**processor_components)
|
||||
|
||||
image_input = self.prepare_image_inputs()
|
||||
query_images = self.prepare_image_inputs()
|
||||
|
||||
inputs = processor(None, image_input, query_images)
|
||||
|
||||
self.assertListEqual(list(inputs.keys()), ["query_pixel_values", "pixel_values"])
|
||||
|
||||
# test if it raises when no input is passed
|
||||
with pytest.raises(ValueError):
|
||||
processor()
|
||||
|
||||
@@ -232,21 +232,6 @@ class OwlViTProcessorTest(ProcessorTesterMixin, unittest.TestCase):
|
||||
with pytest.raises(ValueError):
|
||||
processor()
|
||||
|
||||
def test_processor_query_images_positional(self):
|
||||
processor_components = self.prepare_components()
|
||||
processor = OwlViTProcessor(**processor_components)
|
||||
|
||||
image_input = self.prepare_image_inputs()
|
||||
query_images = self.prepare_image_inputs()
|
||||
|
||||
inputs = processor(None, image_input, query_images)
|
||||
|
||||
self.assertListEqual(list(inputs.keys()), ["query_pixel_values", "pixel_values"])
|
||||
|
||||
# test if it raises when no input is passed
|
||||
with pytest.raises(ValueError):
|
||||
processor()
|
||||
|
||||
def test_tokenizer_decode(self):
|
||||
image_processor = self.get_image_processor()
|
||||
tokenizer = self.get_tokenizer()
|
||||
|
||||
@@ -77,58 +77,6 @@ class RopeTest(unittest.TestCase):
|
||||
self.assertEqual(len(logs.output), 1)
|
||||
self.assertIn(model_specific_kwarg, logs.output[0])
|
||||
|
||||
def test_default_rope_function_bc(self):
|
||||
config = LlamaConfig()
|
||||
device = torch_device
|
||||
|
||||
rope_kwargs = {
|
||||
"rope_type": "default",
|
||||
"dim": config.hidden_size // config.num_attention_heads,
|
||||
"max_position_embeddings": config.max_position_embeddings,
|
||||
"base": config.rope_theta,
|
||||
}
|
||||
|
||||
rope_fn = ROPE_INIT_FUNCTIONS["default"]
|
||||
config_freqs = rope_fn(config=config, device=device)[0]
|
||||
kwargs_freqs = rope_fn(**rope_kwargs, device=device)[0]
|
||||
torch.testing.assert_close(config_freqs, kwargs_freqs)
|
||||
|
||||
def test_linear_rope_function_bc(self):
|
||||
config = LlamaConfig()
|
||||
config.rope_scaling = {"rope_type": "linear", "factor": 10.0}
|
||||
device = torch_device
|
||||
|
||||
rope_kwargs = {
|
||||
"rope_type": "linear",
|
||||
"dim": config.hidden_size // config.num_attention_heads,
|
||||
"max_position_embeddings": config.max_position_embeddings,
|
||||
"base": config.rope_theta,
|
||||
"factor": 10.0,
|
||||
}
|
||||
|
||||
rope_fn = ROPE_INIT_FUNCTIONS["linear"]
|
||||
config_freqs = rope_fn(config=config, device=device)[0]
|
||||
kwargs_freqs = rope_fn(**rope_kwargs, device=device)[0]
|
||||
torch.testing.assert_close(config_freqs, kwargs_freqs)
|
||||
|
||||
def test_dynamic_rope_function_bc(self):
|
||||
config = LlamaConfig()
|
||||
config.rope_scaling = {"rope_type": "dynamic", "factor": 10.0}
|
||||
device = torch_device
|
||||
|
||||
rope_kwargs = {
|
||||
"rope_type": "dynamic",
|
||||
"dim": config.hidden_size // config.num_attention_heads,
|
||||
"max_position_embeddings": config.max_position_embeddings,
|
||||
"base": config.rope_theta,
|
||||
"factor": 10.0,
|
||||
}
|
||||
|
||||
rope_fn = ROPE_INIT_FUNCTIONS["dynamic"]
|
||||
config_freqs = rope_fn(config=config, device=device)[0]
|
||||
kwargs_freqs = rope_fn(**rope_kwargs, device=device)[0]
|
||||
torch.testing.assert_close(config_freqs, kwargs_freqs)
|
||||
|
||||
def test_default_rope_numerically(self):
|
||||
# Note: some RoPE scaling methods start off by calling the default RoPE frequencies. If this test fails, then
|
||||
# multiple RoPE strategies will fail.
|
||||
|
||||
@@ -1,175 +0,0 @@
|
||||
# Copyright 2024 HuggingFace Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import unittest
|
||||
|
||||
import numpy as np
|
||||
|
||||
from transformers import is_torch_available, is_vision_available
|
||||
from transformers.processing_utils import _validate_images_text_input_order
|
||||
from transformers.testing_utils import require_torch, require_vision
|
||||
|
||||
|
||||
if is_vision_available():
|
||||
import PIL
|
||||
|
||||
if is_torch_available():
|
||||
import torch
|
||||
|
||||
|
||||
@require_vision
|
||||
class ProcessingUtilTester(unittest.TestCase):
|
||||
def test_validate_images_text_input_order(self):
|
||||
# text string and PIL images inputs
|
||||
images = PIL.Image.new("RGB", (224, 224))
|
||||
text = "text"
|
||||
# test correct text and images order
|
||||
valid_images, valid_text = _validate_images_text_input_order(images=images, text=text)
|
||||
self.assertEqual(valid_images, images)
|
||||
self.assertEqual(valid_text, text)
|
||||
# test incorrect text and images order
|
||||
valid_images, valid_text = _validate_images_text_input_order(images=text, text=images)
|
||||
self.assertEqual(valid_images, images)
|
||||
self.assertEqual(valid_text, text)
|
||||
|
||||
# text list of string and numpy images inputs
|
||||
images = np.random.rand(224, 224, 3)
|
||||
text = ["text1", "text2"]
|
||||
# test correct text and images order
|
||||
valid_images, valid_text = _validate_images_text_input_order(images=images, text=text)
|
||||
self.assertTrue(np.array_equal(valid_images, images))
|
||||
self.assertEqual(valid_text, text)
|
||||
# test incorrect text and images order
|
||||
valid_images, valid_text = _validate_images_text_input_order(images=text, text=images)
|
||||
self.assertTrue(np.array_equal(valid_images, images))
|
||||
self.assertEqual(valid_text, text)
|
||||
|
||||
# text nested list of string and list of pil images inputs
|
||||
images = [PIL.Image.new("RGB", (224, 224)), PIL.Image.new("RGB", (224, 224))]
|
||||
text = [["text1", "text2, text3"], ["text3", "text4"]]
|
||||
# test correct text and images order
|
||||
valid_images, valid_text = _validate_images_text_input_order(images=images, text=text)
|
||||
self.assertEqual(valid_images, images)
|
||||
self.assertEqual(valid_text, text)
|
||||
# test incorrect text and images order
|
||||
valid_images, valid_text = _validate_images_text_input_order(images=text, text=images)
|
||||
self.assertEqual(valid_images, images)
|
||||
self.assertEqual(valid_text, text)
|
||||
|
||||
# list of strings and list of numpy images inputs
|
||||
images = [np.random.rand(224, 224, 3), np.random.rand(224, 224, 3)]
|
||||
text = ["text1", "text2"]
|
||||
# test correct text and images order
|
||||
valid_images, valid_text = _validate_images_text_input_order(images=images, text=text)
|
||||
self.assertTrue(np.array_equal(valid_images[0], images[0]))
|
||||
self.assertEqual(valid_text, text)
|
||||
# test incorrect text and images order
|
||||
valid_images, valid_text = _validate_images_text_input_order(images=text, text=images)
|
||||
self.assertTrue(np.array_equal(valid_images[0], images[0]))
|
||||
self.assertEqual(valid_text, text)
|
||||
|
||||
# list of strings and list of url images inputs
|
||||
images = ["https://url1", "https://url2"]
|
||||
text = ["text1", "text2"]
|
||||
# test correct text and images order
|
||||
valid_images, valid_text = _validate_images_text_input_order(images=images, text=text)
|
||||
self.assertEqual(valid_images, images)
|
||||
self.assertEqual(valid_text, text)
|
||||
# test incorrect text and images order
|
||||
valid_images, valid_text = _validate_images_text_input_order(images=text, text=images)
|
||||
self.assertEqual(valid_images, images)
|
||||
self.assertEqual(valid_text, text)
|
||||
|
||||
# list of strings and nested list of numpy images inputs
|
||||
images = [[np.random.rand(224, 224, 3), np.random.rand(224, 224, 3)], [np.random.rand(224, 224, 3)]]
|
||||
text = ["text1", "text2"]
|
||||
# test correct text and images order
|
||||
valid_images, valid_text = _validate_images_text_input_order(images=images, text=text)
|
||||
self.assertTrue(np.array_equal(valid_images[0][0], images[0][0]))
|
||||
self.assertEqual(valid_text, text)
|
||||
# test incorrect text and images order
|
||||
valid_images, valid_text = _validate_images_text_input_order(images=text, text=images)
|
||||
self.assertTrue(np.array_equal(valid_images[0][0], images[0][0]))
|
||||
self.assertEqual(valid_text, text)
|
||||
|
||||
# nested list of strings and nested list of PIL images inputs
|
||||
images = [
|
||||
[PIL.Image.new("RGB", (224, 224)), PIL.Image.new("RGB", (224, 224))],
|
||||
[PIL.Image.new("RGB", (224, 224))],
|
||||
]
|
||||
text = [["text1", "text2, text3"], ["text3", "text4"]]
|
||||
# test correct text and images order
|
||||
valid_images, valid_text = _validate_images_text_input_order(images=images, text=text)
|
||||
self.assertEqual(valid_images, images)
|
||||
self.assertEqual(valid_text, text)
|
||||
# test incorrect text and images order
|
||||
valid_images, valid_text = _validate_images_text_input_order(images=text, text=images)
|
||||
self.assertEqual(valid_images, images)
|
||||
self.assertEqual(valid_text, text)
|
||||
|
||||
# None images
|
||||
images = None
|
||||
text = "text"
|
||||
# test correct text and images order
|
||||
valid_images, valid_text = _validate_images_text_input_order(images=images, text=text)
|
||||
self.assertEqual(images, None)
|
||||
self.assertEqual(text, text)
|
||||
# test incorrect text and images order
|
||||
valid_images, valid_text = _validate_images_text_input_order(images=text, text=images)
|
||||
self.assertEqual(images, None)
|
||||
self.assertEqual(text, text)
|
||||
|
||||
# None text
|
||||
images = PIL.Image.new("RGB", (224, 224))
|
||||
text = None
|
||||
# test correct text and images order
|
||||
valid_images, valid_text = _validate_images_text_input_order(images=images, text=text)
|
||||
self.assertEqual(images, images)
|
||||
self.assertEqual(text, None)
|
||||
# test incorrect text and images order
|
||||
valid_images, valid_text = _validate_images_text_input_order(images=text, text=images)
|
||||
self.assertEqual(images, images)
|
||||
self.assertEqual(text, None)
|
||||
|
||||
# incorrect inputs
|
||||
images = "text"
|
||||
text = "text"
|
||||
with self.assertRaises(ValueError):
|
||||
_validate_images_text_input_order(images=images, text=text)
|
||||
|
||||
@require_torch
|
||||
def test_validate_images_text_input_order_torch(self):
|
||||
# text string and torch images inputs
|
||||
images = torch.rand(224, 224, 3)
|
||||
text = "text"
|
||||
# test correct text and images order
|
||||
valid_images, valid_text = _validate_images_text_input_order(images=images, text=text)
|
||||
self.assertTrue(torch.equal(valid_images, images))
|
||||
self.assertEqual(valid_text, text)
|
||||
# test incorrect text and images order
|
||||
valid_images, valid_text = _validate_images_text_input_order(images=text, text=images)
|
||||
self.assertTrue(torch.equal(valid_images, images))
|
||||
self.assertEqual(valid_text, text)
|
||||
|
||||
# text list of string and list of torch images inputs
|
||||
images = [torch.rand(224, 224, 3), torch.rand(224, 224, 3)]
|
||||
text = ["text1", "text2"]
|
||||
# test correct text and images order
|
||||
valid_images, valid_text = _validate_images_text_input_order(images=images, text=text)
|
||||
self.assertTrue(torch.equal(valid_images[0], images[0]))
|
||||
self.assertEqual(valid_text, text)
|
||||
# test incorrect text and images order
|
||||
valid_images, valid_text = _validate_images_text_input_order(images=text, text=images)
|
||||
self.assertTrue(torch.equal(valid_images[0], images[0]))
|
||||
self.assertEqual(valid_text, text)
|
||||
Reference in New Issue
Block a user