SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Update VisionEncoderDecoder to use an image processor (#20137)

Browse Source 
* TrOCR processor uses an image processor

* Update VisionEncoderDecoder

* Add feature_extractor_class property
This commit is contained in:
amyeroberts
2022-11-09 16:31:05 +00:00
committed by GitHub
parent a44985b41c
commit f3d99e49d4
7 changed files with 124 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
tests/models/vision_text_dual_encoder/test_processor_vision_text_dual_encoder.py
View File

@@ -23,13 +23,13 @@ import numpy as np
from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES, BertTokenizer
from transformers.testing_utils import require_tokenizers, require_vision
from transformers.utils import FEATURE_EXTRACTOR_NAME, is_vision_available
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available
if is_vision_available():
from PIL import Image
from transformers import VisionTextDualEncoderProcessor, ViTFeatureExtractor
from transformers import VisionTextDualEncoderProcessor, ViTImageProcessor
@require_tokenizers
@@ -45,22 +45,22 @@ class VisionTextDualEncoderProcessorTest(unittest.TestCase):
with open(self.vocab_file, "w", encoding="utf-8") as vocab_writer:
vocab_writer.write("".join([x + "\n" for x in vocab_tokens]))
feature_extractor_map = {
image_processor_map = {
"do_resize": True,
"size": 18,
"size": {"height": 18, "width": 18},
"do_normalize": True,
"image_mean": [0.5, 0.5, 0.5],
"image_std": [0.5, 0.5, 0.5],
}
self.feature_extractor_file = os.path.join(self.tmpdirname, FEATURE_EXTRACTOR_NAME)
with open(self.feature_extractor_file, "w", encoding="utf-8") as fp:
json.dump(feature_extractor_map, fp)
self.image_processor_file = os.path.join(self.tmpdirname, IMAGE_PROCESSOR_NAME)
with open(self.image_processor_file, "w", encoding="utf-8") as fp:
json.dump(image_processor_map, fp)
def get_tokenizer(self, **kwargs):
return BertTokenizer.from_pretrained(self.tmpdirname, **kwargs)
def get_feature_extractor(self, **kwargs):
return ViTFeatureExtractor.from_pretrained(self.tmpdirname, **kwargs)
def get_image_processor(self, **kwargs):
return ViTImageProcessor.from_pretrained(self.tmpdirname, **kwargs)
def tearDown(self):
shutil.rmtree(self.tmpdirname)
@@ -76,13 +76,11 @@ class VisionTextDualEncoderProcessorTest(unittest.TestCase):
return image_inputs
# TODO (Amy): fix me
@unittest.skip("An issue introduced in PR #19796 will be fixed by `AutoImageProcessor`")
def test_save_load_pretrained_default(self):
tokenizer = self.get_tokenizer()
feature_extractor = self.get_feature_extractor()
image_processor = self.get_image_processor()
processor = VisionTextDualEncoderProcessor(tokenizer=tokenizer, feature_extractor=feature_extractor)
processor = VisionTextDualEncoderProcessor(tokenizer=tokenizer, image_processor=image_processor)
processor.save_pretrained(self.tmpdirname)
processor = VisionTextDualEncoderProcessor.from_pretrained(self.tmpdirname)
@@ -90,19 +88,17 @@ class VisionTextDualEncoderProcessorTest(unittest.TestCase):
self.assertEqual(processor.tokenizer.get_vocab(), tokenizer.get_vocab())
self.assertIsInstance(processor.tokenizer, (BertTokenizer, BertTokenizerFast))
self.assertEqual(processor.feature_extractor.to_json_string(), feature_extractor.to_json_string())
self.assertIsInstance(processor.feature_extractor, ViTFeatureExtractor)
self.assertEqual(processor.image_processor.to_json_string(), image_processor.to_json_string())
self.assertIsInstance(processor.image_processor, ViTImageProcessor)
# TODO (Amy): fix me
@unittest.skip("An issue introduced in PR #19796 will be fixed by `AutoImageProcessor`")
def test_save_load_pretrained_additional_features(self):
processor = VisionTextDualEncoderProcessor(
tokenizer=self.get_tokenizer(), feature_extractor=self.get_feature_extractor()
tokenizer=self.get_tokenizer(), image_processor=self.get_image_processor()
)
processor.save_pretrained(self.tmpdirname)
tokenizer_add_kwargs = self.get_tokenizer(bos_token="(BOS)", eos_token="(EOS)")
feature_extractor_add_kwargs = self.get_feature_extractor(do_normalize=False, padding_value=1.0)
image_processor_add_kwargs = self.get_image_processor(do_normalize=False, padding_value=1.0)
processor = VisionTextDualEncoderProcessor.from_pretrained(
self.tmpdirname, bos_token="(BOS)", eos_token="(EOS)", do_normalize=False, padding_value=1.0
@@ -111,28 +107,28 @@ class VisionTextDualEncoderProcessorTest(unittest.TestCase):
self.assertEqual(processor.tokenizer.get_vocab(), tokenizer_add_kwargs.get_vocab())
self.assertIsInstance(processor.tokenizer, (BertTokenizer, BertTokenizerFast))
self.assertEqual(processor.feature_extractor.to_json_string(), feature_extractor_add_kwargs.to_json_string())
self.assertIsInstance(processor.feature_extractor, ViTFeatureExtractor)
self.assertEqual(processor.image_processor.to_json_string(), image_processor_add_kwargs.to_json_string())
self.assertIsInstance(processor.image_processor, ViTImageProcessor)
def test_feature_extractor(self):
feature_extractor = self.get_feature_extractor()
def test_image_processor(self):
image_processor = self.get_image_processor()
tokenizer = self.get_tokenizer()
processor = VisionTextDualEncoderProcessor(tokenizer=tokenizer, feature_extractor=feature_extractor)
processor = VisionTextDualEncoderProcessor(tokenizer=tokenizer, image_processor=image_processor)
image_input = self.prepare_image_inputs()
input_feat_extract = feature_extractor(image_input, return_tensors="np")
input_feat_extract = image_processor(image_input, return_tensors="np")
input_processor = processor(images=image_input, return_tensors="np")
for key in input_feat_extract.keys():
self.assertAlmostEqual(input_feat_extract[key].sum(), input_processor[key].sum(), delta=1e-2)
def test_tokenizer(self):
feature_extractor = self.get_feature_extractor()
image_processor = self.get_image_processor()
tokenizer = self.get_tokenizer()
processor = VisionTextDualEncoderProcessor(tokenizer=tokenizer, feature_extractor=feature_extractor)
processor = VisionTextDualEncoderProcessor(tokenizer=tokenizer, image_processor=image_processor)
input_str = "lower newer"
@@ -144,10 +140,10 @@ class VisionTextDualEncoderProcessorTest(unittest.TestCase):
self.assertListEqual(encoded_tok[key], encoded_processor[key])
def test_processor(self):
feature_extractor = self.get_feature_extractor()
image_processor = self.get_image_processor()
tokenizer = self.get_tokenizer()
processor = VisionTextDualEncoderProcessor(tokenizer=tokenizer, feature_extractor=feature_extractor)
processor = VisionTextDualEncoderProcessor(tokenizer=tokenizer, image_processor=image_processor)
input_str = "lower newer"
image_input = self.prepare_image_inputs()
@@ -161,10 +157,10 @@ class VisionTextDualEncoderProcessorTest(unittest.TestCase):
processor()
def test_tokenizer_decode(self):
feature_extractor = self.get_feature_extractor()
image_processor = self.get_image_processor()
tokenizer = self.get_tokenizer()
processor = VisionTextDualEncoderProcessor(tokenizer=tokenizer, feature_extractor=feature_extractor)
processor = VisionTextDualEncoderProcessor(tokenizer=tokenizer, image_processor=image_processor)
predicted_ids = [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]