From 56a7745704261919dd8117e3a8aa4fb43fade30e Mon Sep 17 00:00:00 2001 From: NielsRogge <48327001+NielsRogge@users.noreply.github.com> Date: Fri, 19 Jul 2024 10:20:03 +0200 Subject: [PATCH] [Chameleon, Hiera] Improve docs (#32038) * Improve docs * Fix docs * Fix code snippet --- docs/source/en/_toctree.yml | 4 ++-- docs/source/en/model_doc/chameleon.md | 14 +++++++------- docs/source/en/model_doc/dinov2.md | 2 +- docs/source/en/model_doc/hiera.md | 14 ++++++++++++++ .../chameleon/convert_chameleon_weights_to_hf.py | 2 +- 5 files changed, 25 insertions(+), 11 deletions(-) diff --git a/docs/source/en/_toctree.yml b/docs/source/en/_toctree.yml index cc6ff752c7..59ebf83670 100644 --- a/docs/source/en/_toctree.yml +++ b/docs/source/en/_toctree.yml @@ -326,8 +326,6 @@ title: CamemBERT - local: model_doc/canine title: CANINE - - local: model_doc/chameleon - title: chameleon - local: model_doc/codegen title: CodeGen - local: model_doc/code_llama @@ -760,6 +758,8 @@ title: BridgeTower - local: model_doc/bros title: BROS + - local: model_doc/chameleon + title: chameleon - local: model_doc/chinese_clip title: Chinese-CLIP - local: model_doc/clip diff --git a/docs/source/en/model_doc/chameleon.md b/docs/source/en/model_doc/chameleon.md index fb524b3247..2730fc4272 100644 --- a/docs/source/en/model_doc/chameleon.md +++ b/docs/source/en/model_doc/chameleon.md @@ -69,13 +69,13 @@ import torch from PIL import Image import requests -processor = ChameleonProcessor.from_pretrained("meta-chameleon") -model = ChameleonForConditionalGeneration.from_pretrained("meta-chameleon", torch_dtype=torch.float16, device_map="auto") +processor = ChameleonProcessor.from_pretrained("facebook/chameleon-7b") +model = ChameleonForConditionalGeneration.from_pretrained("facebook/chameleon-7b", torch_dtype=torch.float16, device_map="auto") # prepare image and text prompt -url = "https://bjiujitsu.com/wp-content/uploads/2021/01/jiu_jitsu_belt_white_1.jpg" +url = 'http://images.cocodataset.org/val2017/000000039769.jpg' image = Image.open(requests.get(url, stream=True).raw) -prompt = "What color is the belt in this image?" +prompt = "What do you see in this image?" inputs = processor(prompt, image, return_tensors="pt").to(model.device) @@ -94,8 +94,8 @@ import torch from PIL import Image import requests -processor = ChameleonProcessor.from_pretrained("meta-chameleon") -model = ChameleonForConditionalGeneration.from_pretrained("meta-chameleon", torch_dtype=torch.float16, device_map="auto") +processor = ChameleonProcessor.from_pretrained("facebook/chameleon-7b") +model = ChameleonForConditionalGeneration.from_pretrained("facebook/chameleon-7b", torch_dtype=torch.float16, device_map="auto") # Get three different images url = "https://www.ilankelman.org/stopsigns/australia.jpg" @@ -115,7 +115,7 @@ prompts = [ # We can simply feed images in the order they have to be used in the text prompt # Each "" token uses one image leaving the next for the subsequent "" tokens -inputs = processor(text=prompts, images=[image_stop, image_cats, image_snowman], padding=True, return_tensors="pt").to(model.device) +inputs = processor(text=prompts, images=[image_stop, image_cats, image_snowman], padding=True, return_tensors="pt").to(device="cuda", dtype=torch.float16) # Generate generate_ids = model.generate(**inputs, max_new_tokens=50) diff --git a/docs/source/en/model_doc/dinov2.md b/docs/source/en/model_doc/dinov2.md index dca9478677..e8f7c08cbf 100644 --- a/docs/source/en/model_doc/dinov2.md +++ b/docs/source/en/model_doc/dinov2.md @@ -57,7 +57,7 @@ print((last_hidden_states - traced_outputs[0]).abs().max()) ## Resources -A list of official Hugging Face and community (indicated by 🌎) resources to help you get started with DPT. +A list of official Hugging Face and community (indicated by 🌎) resources to help you get started with DINOv2. - Demo notebooks for DINOv2 can be found [here](https://github.com/NielsRogge/Transformers-Tutorials/tree/master/DINOv2). 🌎 diff --git a/docs/source/en/model_doc/hiera.md b/docs/source/en/model_doc/hiera.md index 24bf1639fe..9bd2816e7a 100644 --- a/docs/source/en/model_doc/hiera.md +++ b/docs/source/en/model_doc/hiera.md @@ -26,8 +26,22 @@ The abstract from the paper is the following: *Modern hierarchical vision transformers have added several vision-specific components in the pursuit of supervised classification performance. While these components lead to effective accuracies and attractive FLOP counts, the added complexity actually makes these transformers slower than their vanilla ViT counterparts. In this paper, we argue that this additional bulk is unnecessary. By pretraining with a strong visual pretext task (MAE), we can strip out all the bells-and-whistles from a state-of-the-art multi-stage vision transformer without losing accuracy. In the process, we create Hiera, an extremely simple hierarchical vision transformer that is more accurate than previous models while being significantly faster both at inference and during training. We evaluate Hiera on a variety of tasks for image and video recognition. Our code and models are available at https://github.com/facebookresearch/hiera.* + + + Hiera architecture. Taken from the original paper. + This model was a joint contibution by [EduardoPacheco](https://huggingface.co/EduardoPacheco) and [namangarg110](https://huggingface.co/namangarg110). The original code can be found [here] (https://github.com/facebookresearch/hiera). +## Resources + +A list of official Hugging Face and community (indicated by 🌎) resources to help you get started with Hiera. If you're interested in submitting a resource to be included here, please feel free to open a Pull Request and we'll review it! The resource should ideally demonstrate something new instead of duplicating an existing resource. + + + +- [`HieraForImageClassification`] is supported by this [example script](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-classification) and [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification.ipynb). +- See also: [Image classification task guide](../tasks/image_classification) + ## HieraConfig [[autodoc]] HieraConfig diff --git a/src/transformers/models/chameleon/convert_chameleon_weights_to_hf.py b/src/transformers/models/chameleon/convert_chameleon_weights_to_hf.py index 2c1f5e89cb..1aebeb0f0b 100644 --- a/src/transformers/models/chameleon/convert_chameleon_weights_to_hf.py +++ b/src/transformers/models/chameleon/convert_chameleon_weights_to_hf.py @@ -444,7 +444,7 @@ def main(): "--model_size", choices=["7B", "30B"], help="" - " models correspond to the finetuned versions, and are specific to the Chameleon official release. For more details on Chameleon, checkout the original repo: https://huggingface.co/meta-chameleon", + " models correspond to the finetuned versions, and are specific to the Chameleon official release. For more details on Chameleon, checkout the original repo: https://github.com/facebookresearch/chameleon", ) parser.add_argument( "--output_dir",