🔴 Video processors as a separate class (#35206)
* initial design * update all video processors * add tests * need to add qwen2-vl (not tested yet) * add qwen2-vl in auto map * fix copies * isort * resolve confilicts kinda * nit: * qwen2-vl is happy now * qwen2-5 happy * other models are happy * fix copies * fix tests * add docs * CI green now? * add more tests * even more changes + tests * doc builder fail * nit * Update src/transformers/models/auto/processing_auto.py Co-authored-by: Pavel Iakubovskii <qubvel@gmail.com> * small update * imports correctly * dump, otherwise this is getting unmanagebale T-T * dump * update * another update * update * tests * move * modular * docs * test * another update * init * remove flakiness in tests * fixup * clean up and remove commented lines * docs * skip this one! * last fix after rebasing * run fixup * delete slow files * remove unnecessary tests + clean up a bit * small fixes * fix tests * more updates * docs * fix tests * update * style * fix qwen2-5-vl * fixup * fixup * unflatten batch when preparing * dump, come back soon * add docs and fix some tests * how to guard this with new dummies? * chat templates in qwen * address some comments * remove `Fast` suffix * fixup * oops should be imported from transforms * typo in requires dummies * new model added with video support * fixup once more * last fixup I hope * revert image processor name + comments * oh, this is why fetch test is failing * fix tests * fix more tests * fixup * add new models: internvl, smolvlm * update docs * imprt once * fix failing tests * do we need to guard it here again, why? * new model was added, update it * remove testcase from tester * fix tests * make style * not related CI fail, lets' just fix here * mark flaky for now, filas 15 out of 100 * style * maybe we can do this way? * don't download images in setup class --------- Co-authored-by: Pavel Iakubovskii <qubvel@gmail.com>
This commit is contained in:
Raushan Turganbay
committed by
GitHub
GitHub
parent
716819b830
commit
a31fa218ad
@@ -30,7 +30,6 @@ from transformers import is_torch_available, is_vision_available
|
||||
from transformers.image_utils import (
|
||||
ChannelDimension,
|
||||
get_channel_dimension_axis,
|
||||
make_batched_videos,
|
||||
make_flat_list_of_images,
|
||||
make_list_of_images,
|
||||
make_nested_list_of_images,
|
||||
@@ -396,133 +395,6 @@ class ImageFeatureExtractionTester(unittest.TestCase):
|
||||
self.assertEqual(len(images_list[0]), 4)
|
||||
self.assertTrue(np.array_equal(images_list[0][0], images[0][0]))
|
||||
|
||||
def test_make_batched_videos_pil(self):
|
||||
# Test a single image is converted to a list of 1 video with 1 frame
|
||||
pil_image = get_random_image(16, 32)
|
||||
videos_list = make_batched_videos(pil_image)
|
||||
self.assertIsInstance(videos_list[0], list)
|
||||
self.assertEqual(len(videos_list[0]), 1)
|
||||
self.assertIsInstance(videos_list[0][0], PIL.Image.Image)
|
||||
|
||||
# Test a list of images is converted to a list of 1 video
|
||||
images = [get_random_image(16, 32) for _ in range(4)]
|
||||
videos_list = make_batched_videos(images)
|
||||
self.assertIsInstance(videos_list[0], list)
|
||||
self.assertEqual(len(videos_list), 1)
|
||||
self.assertEqual(len(videos_list[0]), 4)
|
||||
self.assertIsInstance(videos_list[0][0], PIL.Image.Image)
|
||||
|
||||
# Test a nested list of images is not modified
|
||||
images = [[get_random_image(16, 32) for _ in range(2)] for _ in range(2)]
|
||||
videos_list = make_nested_list_of_images(images)
|
||||
self.assertIsInstance(videos_list[0], list)
|
||||
self.assertEqual(len(videos_list), 2)
|
||||
self.assertEqual(len(videos_list[0]), 2)
|
||||
self.assertIsInstance(videos_list[0][0], PIL.Image.Image)
|
||||
|
||||
def test_make_batched_videos_numpy(self):
|
||||
# Test a single image is converted to a list of 1 video with 1 frame
|
||||
images = np.random.randint(0, 256, (16, 32, 3))
|
||||
videos_list = make_batched_videos(images)
|
||||
self.assertIsInstance(videos_list[0], list)
|
||||
self.assertEqual(len(videos_list), 1)
|
||||
self.assertTrue(np.array_equal(videos_list[0][0], images))
|
||||
|
||||
# Test a 4d array of images is converted to a list of 1 video
|
||||
images = np.random.randint(0, 256, (4, 16, 32, 3))
|
||||
videos_list = make_batched_videos(images)
|
||||
self.assertIsInstance(videos_list[0], list)
|
||||
self.assertIsInstance(videos_list[0][0], np.ndarray)
|
||||
self.assertEqual(len(videos_list), 1)
|
||||
self.assertEqual(len(videos_list[0]), 4)
|
||||
self.assertTrue(np.array_equal(videos_list[0][0], images[0]))
|
||||
|
||||
# Test a list of images is converted to a list of videos
|
||||
images = [np.random.randint(0, 256, (16, 32, 3)) for _ in range(4)]
|
||||
videos_list = make_batched_videos(images)
|
||||
self.assertIsInstance(videos_list[0], list)
|
||||
self.assertEqual(len(videos_list), 1)
|
||||
self.assertEqual(len(videos_list[0]), 4)
|
||||
self.assertTrue(np.array_equal(videos_list[0][0], images[0]))
|
||||
|
||||
# Test a nested list of images is left unchanged
|
||||
images = [[np.random.randint(0, 256, (16, 32, 3)) for _ in range(2)] for _ in range(2)]
|
||||
videos_list = make_batched_videos(images)
|
||||
self.assertIsInstance(videos_list[0], list)
|
||||
self.assertEqual(len(videos_list), 2)
|
||||
self.assertEqual(len(videos_list[0]), 2)
|
||||
self.assertTrue(np.array_equal(videos_list[0][0], images[0][0]))
|
||||
|
||||
# Test a list of 4d array images is converted to a list of videos
|
||||
images = [np.random.randint(0, 256, (4, 16, 32, 3)) for _ in range(2)]
|
||||
videos_list = make_batched_videos(images)
|
||||
self.assertIsInstance(videos_list[0], list)
|
||||
self.assertIsInstance(videos_list[0][0], np.ndarray)
|
||||
self.assertEqual(len(videos_list), 2)
|
||||
self.assertEqual(len(videos_list[0]), 4)
|
||||
self.assertTrue(np.array_equal(videos_list[0][0], images[0][0]))
|
||||
|
||||
# Test a batch of list of 4d array images is converted to a list of videos
|
||||
images = [[np.random.randint(0, 256, (4, 16, 32, 3)) for _ in range(2)] for _ in range(2)]
|
||||
videos_list = make_batched_videos(images)
|
||||
self.assertIsInstance(videos_list[0], list)
|
||||
self.assertIsInstance(videos_list[0][0], np.ndarray)
|
||||
self.assertEqual(len(videos_list), 2)
|
||||
self.assertEqual(len(videos_list[0]), 8)
|
||||
self.assertTrue(np.array_equal(videos_list[0][0], images[0][0][0]))
|
||||
|
||||
@require_torch
|
||||
def test_make_batched_videos_torch(self):
|
||||
# Test a single image is converted to a list of 1 video with 1 frame
|
||||
images = torch.randint(0, 256, (16, 32, 3))
|
||||
videos_list = make_batched_videos(images)
|
||||
self.assertIsInstance(videos_list[0], list)
|
||||
self.assertEqual(len(videos_list[0]), 1)
|
||||
self.assertTrue(np.array_equal(videos_list[0][0], images))
|
||||
|
||||
# Test a 4d tensor of images is converted to a list of 1 video
|
||||
images = torch.randint(0, 256, (4, 16, 32, 3))
|
||||
videos_list = make_batched_videos(images)
|
||||
self.assertIsInstance(videos_list[0], list)
|
||||
self.assertIsInstance(videos_list[0][0], torch.Tensor)
|
||||
self.assertEqual(len(videos_list), 1)
|
||||
self.assertEqual(len(videos_list[0]), 4)
|
||||
self.assertTrue(np.array_equal(videos_list[0][0], images[0]))
|
||||
|
||||
# Test a list of images is converted to a list of videos
|
||||
images = [torch.randint(0, 256, (16, 32, 3)) for _ in range(4)]
|
||||
videos_list = make_batched_videos(images)
|
||||
self.assertIsInstance(videos_list[0], list)
|
||||
self.assertEqual(len(videos_list), 1)
|
||||
self.assertEqual(len(videos_list[0]), 4)
|
||||
self.assertTrue(np.array_equal(videos_list[0][0], images[0]))
|
||||
|
||||
# Test a nested list of images is left unchanged
|
||||
images = [[torch.randint(0, 256, (16, 32, 3)) for _ in range(2)] for _ in range(2)]
|
||||
videos_list = make_batched_videos(images)
|
||||
self.assertIsInstance(videos_list[0], list)
|
||||
self.assertEqual(len(videos_list), 2)
|
||||
self.assertEqual(len(videos_list[0]), 2)
|
||||
self.assertTrue(np.array_equal(videos_list[0][0], images[0][0]))
|
||||
|
||||
# Test a list of 4d tensor images is converted to a list of videos
|
||||
images = [torch.randint(0, 256, (4, 16, 32, 3)) for _ in range(2)]
|
||||
videos_list = make_batched_videos(images)
|
||||
self.assertIsInstance(videos_list[0], list)
|
||||
self.assertIsInstance(videos_list[0][0], torch.Tensor)
|
||||
self.assertEqual(len(videos_list), 2)
|
||||
self.assertEqual(len(videos_list[0]), 4)
|
||||
self.assertTrue(np.array_equal(videos_list[0][0], images[0][0]))
|
||||
|
||||
# Test a batch of list of 4d tensor images is converted to a list of videos
|
||||
images = [[torch.randint(0, 256, (4, 16, 32, 3)) for _ in range(2)] for _ in range(2)]
|
||||
videos_list = make_batched_videos(images)
|
||||
self.assertIsInstance(videos_list[0], list)
|
||||
self.assertIsInstance(videos_list[0][0], torch.Tensor)
|
||||
self.assertEqual(len(videos_list), 2)
|
||||
self.assertEqual(len(videos_list[0]), 8)
|
||||
self.assertTrue(np.array_equal(videos_list[0][0], images[0][0][0]))
|
||||
|
||||
@require_torch
|
||||
def test_conversion_torch_to_array(self):
|
||||
feature_extractor = ImageFeatureExtractionMixin()
|
||||
|
||||
Reference in New Issue
Block a user