Add Distill Any Depth (#36614)
* Added conversion Script * Update src/transformers/models/depth_anything/convert_distill_any_depth_to_hf.py Co-authored-by: Pavel Iakubovskii <qubvel@gmail.com> * Updated Conversion Script * Update src/transformers/models/depth_anything/convert_distill_any_depth_to_hf.py Co-authored-by: Pavel Iakubovskii <qubvel@gmail.com> --------- Co-authored-by: Pavel Iakubovskii <qubvel@gmail.com>
This commit is contained in:
Parteek
@@ -0,0 +1,246 @@
|
|||||||
|
# coding=utf-8
|
||||||
|
# Copyright 2025 The HuggingFace Inc. team.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
"""Convert Distill Any Depth checkpoints from the original repository. URL:
|
||||||
|
https://github.com/Westlake-AGI-Lab/Distill-Any-Depth"""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import re
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import requests
|
||||||
|
import torch
|
||||||
|
from huggingface_hub import hf_hub_download
|
||||||
|
from PIL import Image
|
||||||
|
from safetensors.torch import load_file
|
||||||
|
|
||||||
|
from transformers import DepthAnythingConfig, DepthAnythingForDepthEstimation, Dinov2Config, DPTImageProcessor
|
||||||
|
from transformers.utils import logging
|
||||||
|
|
||||||
|
|
||||||
|
logging.set_verbosity_info()
|
||||||
|
logger = logging.get_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
ORIGINAL_TO_CONVERTED_KEY_MAPPING = {
|
||||||
|
r"(backbone|pretrained)\.cls_token": r"backbone.embeddings.cls_token",
|
||||||
|
r"(backbone|pretrained)\.mask_token": r"backbone.embeddings.mask_token",
|
||||||
|
r"(backbone|pretrained)\.pos_embed": r"backbone.embeddings.position_embeddings",
|
||||||
|
r"(backbone|pretrained)\.patch_embed\.proj\.(weight|bias)": r"backbone.embeddings.patch_embeddings.projection.\2",
|
||||||
|
r"(backbone|pretrained)\.norm\.(weight|bias)": r"backbone.layernorm.\2",
|
||||||
|
r"(backbone|pretrained)(\.blocks(\.\d+)?)?\.(\d+)\.attn\.proj\.(weight|bias)": r"backbone.encoder.layer.\4.attention.output.dense.\5",
|
||||||
|
r"(backbone|pretrained)(\.blocks(\.\d+)?)?\.(\d+)\.ls(1|2)\.gamma": r"backbone.encoder.layer.\4.layer_scale\5.lambda1",
|
||||||
|
r"(backbone|pretrained)(\.blocks(\.\d+)?)?\.(\d+)\.mlp\.fc(1|2)\.(weight|bias)": r"backbone.encoder.layer.\4.mlp.fc\5.\6",
|
||||||
|
r"(backbone|pretrained)(\.blocks(\.\d+)?)?\.(\d+)\.norm(1|2)\.(weight|bias)": r"backbone.encoder.layer.\4.norm\5.\6",
|
||||||
|
r"depth_head\.projects\.(\d+)\.(weight|bias)": r"neck.reassemble_stage.layers.\1.projection.\2",
|
||||||
|
r"depth_head\.resize_layers\.(?!2)(\d+)\.(weight|bias)": r"neck.reassemble_stage.layers.\1.resize.\2",
|
||||||
|
r"depth_head\.scratch\.layer(\d+)_rn\.weight": lambda m: f"neck.convs.{int(m[1])-1}.weight",
|
||||||
|
r"depth_head\.scratch\.output_conv(\d+)(?:\.(\d+))?\.(weight|bias)": lambda m: (
|
||||||
|
f"head.conv{int(m[1]) + (int(m[2])//2 if m[2] else 0)}.{m[3]}" if m[1] == "2" else f"head.conv{m[1]}.{m[3]}"
|
||||||
|
),
|
||||||
|
r"depth_head\.scratch\.refinenet(\d+)\.out_conv\.(weight|bias)": lambda m: f"neck.fusion_stage.layers.{3 - (int(m[1])-1)}.projection.{m[2]}",
|
||||||
|
r"depth_head\.scratch\.refinenet(\d+)\.resConfUnit(\d+)\.conv(\d+)\.(weight|bias)": lambda m: f"neck.fusion_stage.layers.{3 - (int(m[1])-1)}.residual_layer{m[2]}.convolution{m[3]}.{m[4]}",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def get_dpt_config(model_name):
|
||||||
|
if "small" in model_name:
|
||||||
|
out_indices = [3, 6, 9, 12]
|
||||||
|
backbone_config = Dinov2Config.from_pretrained(
|
||||||
|
"facebook/dinov2-small", out_indices=out_indices, apply_layernorm=True, reshape_hidden_states=False
|
||||||
|
)
|
||||||
|
fusion_hidden_size = 64
|
||||||
|
neck_hidden_sizes = [48, 96, 192, 384]
|
||||||
|
elif "base" in model_name:
|
||||||
|
out_indices = [3, 6, 9, 12]
|
||||||
|
backbone_config = Dinov2Config.from_pretrained(
|
||||||
|
"facebook/dinov2-base", out_indices=out_indices, apply_layernorm=True, reshape_hidden_states=False
|
||||||
|
)
|
||||||
|
fusion_hidden_size = 128
|
||||||
|
neck_hidden_sizes = [96, 192, 384, 768]
|
||||||
|
elif "large" in model_name:
|
||||||
|
out_indices = [5, 12, 18, 24]
|
||||||
|
backbone_config = Dinov2Config.from_pretrained(
|
||||||
|
"facebook/dinov2-large", out_indices=out_indices, apply_layernorm=True, reshape_hidden_states=False
|
||||||
|
)
|
||||||
|
fusion_hidden_size = 256
|
||||||
|
neck_hidden_sizes = [256, 512, 1024, 1024]
|
||||||
|
else:
|
||||||
|
raise NotImplementedError(f"Model not supported: {model_name}")
|
||||||
|
|
||||||
|
depth_estimation_type = "relative"
|
||||||
|
max_depth = None
|
||||||
|
|
||||||
|
config = DepthAnythingConfig(
|
||||||
|
reassemble_hidden_size=backbone_config.hidden_size,
|
||||||
|
patch_size=backbone_config.patch_size,
|
||||||
|
backbone_config=backbone_config,
|
||||||
|
fusion_hidden_size=fusion_hidden_size,
|
||||||
|
neck_hidden_sizes=neck_hidden_sizes,
|
||||||
|
depth_estimation_type=depth_estimation_type,
|
||||||
|
max_depth=max_depth,
|
||||||
|
)
|
||||||
|
|
||||||
|
return config
|
||||||
|
|
||||||
|
|
||||||
|
def convert_key_pattern(key, mapping):
|
||||||
|
for pattern, replacement in mapping.items():
|
||||||
|
match = re.fullmatch(pattern, key)
|
||||||
|
if match:
|
||||||
|
if callable(replacement):
|
||||||
|
return replacement(match)
|
||||||
|
return re.sub(pattern, replacement, key)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def convert_keys(state_dict, config):
|
||||||
|
new_state_dict = {}
|
||||||
|
qkv_pattern = r"(backbone|pretrained)(\.blocks(\.\d+)?)?\.(\d+)\.attn\.qkv\.(weight|bias)"
|
||||||
|
qkv_keys = [k for k in list(state_dict.keys()) if re.match(qkv_pattern, k)]
|
||||||
|
for old_key in qkv_keys:
|
||||||
|
value = state_dict.pop(old_key)
|
||||||
|
match = re.match(qkv_pattern, old_key)
|
||||||
|
_, _, _, layer, attr = match.groups()
|
||||||
|
hidden_size = config.backbone_config.hidden_size
|
||||||
|
q = value[:hidden_size]
|
||||||
|
k = value[hidden_size : hidden_size * 2]
|
||||||
|
v = value[-hidden_size:]
|
||||||
|
|
||||||
|
for proj, tensor in zip(["query", "key", "value"], [q, k, v]):
|
||||||
|
new_key = f"backbone.encoder.layer.{layer}.attention.attention.{proj}.{attr}"
|
||||||
|
new_state_dict[new_key] = tensor
|
||||||
|
|
||||||
|
for old_key in list(state_dict.keys()):
|
||||||
|
value = state_dict.pop(old_key)
|
||||||
|
new_key = convert_key_pattern(old_key, ORIGINAL_TO_CONVERTED_KEY_MAPPING)
|
||||||
|
|
||||||
|
new_state_dict[new_key] = value
|
||||||
|
|
||||||
|
return new_state_dict
|
||||||
|
|
||||||
|
|
||||||
|
def prepare_img():
|
||||||
|
url = "http://images.cocodataset.org/val2017/000000039769.jpg"
|
||||||
|
return Image.open(requests.get(url, stream=True).raw)
|
||||||
|
|
||||||
|
|
||||||
|
name_to_checkpoint = {
|
||||||
|
"distill-any-depth-small": "small/model.safetensors",
|
||||||
|
"distill-any-depth-base": "base/model.safetensors",
|
||||||
|
"distill-any-depth-large": "large/model.safetensors",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@torch.no_grad()
|
||||||
|
def convert_dpt_checkpoint(model_name, pytorch_dump_folder_path, push_to_hub, verify_logits):
|
||||||
|
config = get_dpt_config(model_name)
|
||||||
|
|
||||||
|
repo_id = "xingyang1/Distill-Any-Depth"
|
||||||
|
filepath = hf_hub_download(repo_id=repo_id, filename=name_to_checkpoint[model_name])
|
||||||
|
state_dict = load_file(filepath)
|
||||||
|
|
||||||
|
converted_state_dict = convert_keys(state_dict, config)
|
||||||
|
|
||||||
|
model = DepthAnythingForDepthEstimation(config)
|
||||||
|
model.load_state_dict(converted_state_dict)
|
||||||
|
model.eval()
|
||||||
|
|
||||||
|
processor = DPTImageProcessor(
|
||||||
|
do_resize=True,
|
||||||
|
size={"height": 518, "width": 518},
|
||||||
|
ensure_multiple_of=14,
|
||||||
|
keep_aspect_ratio=True,
|
||||||
|
do_rescale=True,
|
||||||
|
do_normalize=True,
|
||||||
|
image_mean=[0.485, 0.456, 0.406],
|
||||||
|
image_std=[0.229, 0.224, 0.225],
|
||||||
|
)
|
||||||
|
|
||||||
|
url = "http://images.cocodataset.org/val2017/000000039769.jpg"
|
||||||
|
image = Image.open(requests.get(url, stream=True).raw)
|
||||||
|
|
||||||
|
pixel_values = processor(image, return_tensors="pt").pixel_values
|
||||||
|
|
||||||
|
with torch.no_grad():
|
||||||
|
outputs = model(pixel_values)
|
||||||
|
predicted_depth = outputs.predicted_depth
|
||||||
|
|
||||||
|
print("Shape of predicted depth:", predicted_depth.shape)
|
||||||
|
print("First values:", predicted_depth[0, :3, :3])
|
||||||
|
|
||||||
|
if verify_logits:
|
||||||
|
print("Verifying logits...")
|
||||||
|
expected_shape = torch.Size([1, 518, 686])
|
||||||
|
|
||||||
|
if model_name == "distill-any-depth-small":
|
||||||
|
expected_slice = torch.tensor(
|
||||||
|
[[2.5653, 2.5249, 2.5570], [2.4897, 2.5235, 2.5355], [2.5255, 2.5261, 2.5422]]
|
||||||
|
)
|
||||||
|
elif model_name == "distill-any-depth-base":
|
||||||
|
expected_slice = torch.tensor(
|
||||||
|
[[4.8976, 4.9075, 4.9403], [4.8872, 4.8906, 4.9448], [4.8712, 4.8898, 4.8838]]
|
||||||
|
)
|
||||||
|
elif model_name == "distill-any-depth-large":
|
||||||
|
expected_slice = torch.tensor(
|
||||||
|
[[55.1067, 51.1828, 51.6803], [51.9098, 50.7529, 51.4494], [50.1745, 50.5491, 50.8818]]
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
raise ValueError("Not supported")
|
||||||
|
|
||||||
|
assert predicted_depth.shape == torch.Size(expected_shape)
|
||||||
|
assert torch.allclose(predicted_depth[0, :3, :3], expected_slice, atol=1e-4)
|
||||||
|
print("Looks ok!")
|
||||||
|
|
||||||
|
if pytorch_dump_folder_path is not None:
|
||||||
|
Path(pytorch_dump_folder_path).mkdir(exist_ok=True)
|
||||||
|
print(f"Saving model and processor to {pytorch_dump_folder_path}")
|
||||||
|
model.save_pretrained(pytorch_dump_folder_path)
|
||||||
|
processor.save_pretrained(pytorch_dump_folder_path)
|
||||||
|
|
||||||
|
if push_to_hub:
|
||||||
|
print("Pushing model and processor to hub...")
|
||||||
|
model.push_to_hub(repo_id=f"{model_name.title()}-hf")
|
||||||
|
processor.push_to_hub(repo_id=f"{model_name.title()}-hf")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument(
|
||||||
|
"--model_name",
|
||||||
|
default="distill-any-depth-small",
|
||||||
|
type=str,
|
||||||
|
choices=name_to_checkpoint.keys(),
|
||||||
|
help="Name of the model you'd like to convert.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--pytorch_dump_folder_path",
|
||||||
|
default=None,
|
||||||
|
type=str,
|
||||||
|
help="Path to the output PyTorch model directory.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--push_to_hub",
|
||||||
|
action="store_true",
|
||||||
|
help="Whether to push the model to the hub after conversion.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--verify_logits",
|
||||||
|
action="store_true",
|
||||||
|
required=False,
|
||||||
|
help="Whether to verify the logits after conversion.",
|
||||||
|
)
|
||||||
|
|
||||||
|
args = parser.parse_args()
|
||||||
|
convert_dpt_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub, args.verify_logits)
|
||||||
Reference in New Issue
Block a user