Decorators for deprecation and named arguments validation (#30799)

* Fix do_reduce_labels for maskformer image processor * Deprecate reduce_labels in favor to do_reduce_labels * Deprecate reduce_labels in favor to do_reduce_labels (segformer) * Deprecate reduce_labels in favor to do_reduce_labels (oneformer) * Deprecate reduce_labels in favor to do_reduce_labels (maskformer) * Deprecate reduce_labels in favor to do_reduce_labels (mask2former) * Fix typo * Update mask2former test * fixup * Update segmentation examples * Update docs * Fixup * Imports fixup * Add deprecation decorator draft * Add deprecation decorator * Fixup * Add deprecate_kwarg decorator * Validate kwargs decorator * Kwargs validation (beit) * fixup * Kwargs validation (mask2former) * Kwargs validation (maskformer) * Kwargs validation (oneformer) * Kwargs validation (segformer) * Better message * Fix oneformer processor save-load test * Update src/transformers/utils/deprecation.py Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> * Update src/transformers/utils/deprecation.py Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> * Update src/transformers/utils/deprecation.py Co-authored-by: Pablo Montalvo <39954772+molbap@users.noreply.github.com> * Update src/transformers/utils/deprecation.py Co-authored-by: Pablo Montalvo <39954772+molbap@users.noreply.github.com> * Better handle classmethod warning * Fix typo, remove warn * Add header * Docs and `additional_message` * Move to filter decorator ot generic * Proper deprecation for semantic segm scripts * Add to __init__ and update import * Basic tests for filter decorator * Fix doc * Override `to_dict()` to pop depracated `_max_size` * Pop unused parameters * Fix trailing whitespace * Add test for deprecation * Add deprecation warning control parameter * Update generic test * Fixup deprecation tests * Introduce init service kwargs * Revert popping unused params * Revert oneformer test * Allow "metadata" to pass * Better docs * Fix test * Add notion in docstring * Fix notification for both names * Add func name to warning message * Fixup --------- Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> Co-authored-by: Pablo Montalvo <39954772+molbap@users.noreply.github.com>
2024-06-10 12:35:10 +01:00
parent 4fa4dcb2be
commit 517df566f5
28 changed files with 820 additions and 361 deletions
--- a/examples/pytorch/semantic-segmentation/README.md
+++ b/examples/pytorch/semantic-segmentation/README.md
@@ -204,4 +204,4 @@ For visualization of the segmentation maps, we refer to the [example notebook](h

 Some datasets, like [`scene_parse_150`](https://huggingface.co/datasets/scene_parse_150), contain a "background" label that is not part of the classes. The Scene Parse 150 dataset for instance contains labels between 0 and 150, with 0 being the background class, and 1 to 150 being actual class names (like "tree", "person", etc.). For these kind of datasets, one replaces the background label (0) by 255, which is the `ignore_index` of the PyTorch model's loss function, and reduces all labels by 1. This way, the `labels` are PyTorch tensors containing values between 0 and 149, and 255 for all background/padding.

-In case you're training on such a dataset, make sure to set the ``reduce_labels`` flag, which will take care of this.
+In case you're training on such a dataset, make sure to set the ``do_reduce_labels`` flag, which will take care of this.
--- a/examples/pytorch/semantic-segmentation/run_semantic_segmentation.py
+++ b/examples/pytorch/semantic-segmentation/run_semantic_segmentation.py
@@ -17,6 +17,7 @@ import json
 import logging
 import os
 import sys
+import warnings
 from dataclasses import dataclass, field
 from functools import partial
 from typing import Optional
@@ -108,6 +109,10 @@ class DataTrainingArguments:
            )
        },
    )
+    do_reduce_labels: Optional[bool] = field(
+        default=False,
+        metadata={"help": "Whether or not to reduce all labels by 1 and replace background by 255."},
+    )
    reduce_labels: Optional[bool] = field(
        default=False,
        metadata={"help": "Whether or not to reduce all labels by 1 and replace background by 255."},
@@ -118,6 +123,12 @@ class DataTrainingArguments:
            raise ValueError(
                "You must specify either a dataset name from the hub or a train and/or validation directory."
            )
+        if self.reduce_labels:
+            self.do_reduce_labels = self.reduce_labels
+            warnings.warn(
+                "The `reduce_labels` argument is deprecated and will be removed in v4.45. Please use `do_reduce_labels` instead.",
+                FutureWarning,
+            )


@dataclass
@@ -303,14 +314,12 @@ def main():
    )
    image_processor = AutoImageProcessor.from_pretrained(
        model_args.image_processor_name or model_args.model_name_or_path,
+        do_reduce_labels=data_args.do_reduce_labels,
        cache_dir=model_args.cache_dir,
        revision=model_args.model_revision,
        token=model_args.token,
        trust_remote_code=model_args.trust_remote_code,
    )
-    # `reduce_labels` is a property of dataset labels, in case we use image_processor
-    # pretrained on another dataset we should override the default setting
-    image_processor.do_reduce_labels = data_args.reduce_labels

    # Define transforms to be applied to each image and target.
    if "shortest_edge" in image_processor.size:
@@ -322,7 +331,7 @@ def main():
        [
            A.Lambda(
                name="reduce_labels",
-                mask=reduce_labels_transform if data_args.reduce_labels else None,
+                mask=reduce_labels_transform if data_args.do_reduce_labels else None,
                p=1.0,
            ),
            # pad image with 255, because it is ignored by loss
@@ -337,7 +346,7 @@ def main():
        [
            A.Lambda(
                name="reduce_labels",
-                mask=reduce_labels_transform if data_args.reduce_labels else None,
+                mask=reduce_labels_transform if data_args.do_reduce_labels else None,
                p=1.0,
            ),
            A.Resize(height=height, width=width, p=1.0),
--- a/examples/pytorch/semantic-segmentation/run_semantic_segmentation_no_trainer.py
+++ b/examples/pytorch/semantic-segmentation/run_semantic_segmentation_no_trainer.py
@@ -18,6 +18,7 @@ import argparse
 import json
 import math
 import os
+import warnings
 from functools import partial
 from pathlib import Path

@@ -85,6 +86,11 @@ def parse_args():
        help="Name of the dataset on the hub.",
        default="segments/sidewalk-semantic",
    )
+    parser.add_argument(
+        "--do_reduce_labels",
+        action="store_true",
+        help="Whether or not to reduce all labels by 1 and replace background by 255.",
+    )
    parser.add_argument(
        "--reduce_labels",
        action="store_true",
@@ -219,6 +225,14 @@ def parse_args():
                "Need an `output_dir` to create a repo when `--push_to_hub` or `with_tracking` is specified."
            )

+    # Deprecation
+    if args.reduce_labels:
+        args.do_reduce_labels = args.reduce_labels
+        warnings.warn(
+            "The `reduce_labels` argument is deprecated and will be removed in v4.45. Please use `do_reduce_labels` instead.",
+            FutureWarning,
+        )
+
    if args.output_dir is not None:
        os.makedirs(args.output_dir, exist_ok=True)

@@ -315,11 +329,11 @@ def main():
        args.model_name_or_path, trust_remote_code=args.trust_remote_code
    )
    model = AutoModelForSemanticSegmentation.from_pretrained(
-        args.model_name_or_path, config=config, trust_remote_code=args.trust_remote_code
+        args.model_name_or_path,
+        config=config,
+        trust_remote_code=args.trust_remote_code,
+        do_reduce_labels=args.do_reduce_labels,
    )
-    # `reduce_labels` is a property of dataset labels, in case we use image_processor
-    # pretrained on another dataset we should override the default setting
-    image_processor.do_reduce_labels = args.reduce_labels

    # Define transforms to be applied to each image and target.
    if "shortest_edge" in image_processor.size:
@@ -329,7 +343,7 @@ def main():
        height, width = image_processor.size["height"], image_processor.size["width"]
    train_transforms = A.Compose(
        [
-            A.Lambda(name="reduce_labels", mask=reduce_labels_transform if args.reduce_labels else None, p=1.0),
+            A.Lambda(name="reduce_labels", mask=reduce_labels_transform if args.do_reduce_labels else None, p=1.0),
            # pad image with 255, because it is ignored by loss
            A.PadIfNeeded(min_height=height, min_width=width, border_mode=0, value=255, p=1.0),
            A.RandomCrop(height=height, width=width, p=1.0),
@@ -340,7 +354,7 @@ def main():
    )
    val_transforms = A.Compose(
        [
-            A.Lambda(name="reduce_labels", mask=reduce_labels_transform if args.reduce_labels else None, p=1.0),
+            A.Lambda(name="reduce_labels", mask=reduce_labels_transform if args.do_reduce_labels else None, p=1.0),
            A.Resize(height=height, width=width, p=1.0),
            A.Normalize(mean=image_processor.image_mean, std=image_processor.image_std, max_pixel_value=255.0, p=1.0),
            ToTensorV2(),