Add token arugment in example scripts (#25172)

* fix * fix * fix * fix * fix * fix * fix --------- Co-authored-by: ydshieh <ydshieh@users.noreply.github.com>
2023-08-02 11:17:31 +02:00
parent c6a8768dab
commit 149cb0cce2
43 changed files with 987 additions and 420 deletions
--- a/examples/flax/language-modeling/run_bart_dlm_flax.py
+++ b/examples/flax/language-modeling/run_bart_dlm_flax.py
@@ -26,6 +26,7 @@ import math
 import os
 import sys
 import time
+import warnings
 from dataclasses import asdict, dataclass, field
 from enum import Enum
 from itertools import chain
@@ -168,15 +169,21 @@ class ModelArguments:
            )
        },
    )
-    use_auth_token: bool = field(
-        default=False,
+    token: str = field(
+        default=None,
        metadata={
            "help": (
-                "Will use the token generated when running `huggingface-cli login` (necessary to use this script "
-                "with private models)."
+                "The token to use as HTTP bearer authorization for remote files. If not specified, will use the token "
+                "generated when running `huggingface-cli login` (stored in `~/.huggingface`)."
            )
        },
    )
+    use_auth_token: bool = field(
+        default=None,
+        metadata={
+            "help": "The `use_auth_token` argument is deprecated and will be removed in v4.34. Please use `token`."
+        },
+    )


@dataclass
@@ -463,6 +470,12 @@ def main():
    else:
        model_args, data_args, training_args = parser.parse_args_into_dataclasses()

+    if model_args.use_auth_token is not None:
+        warnings.warn("The `use_auth_token` argument is deprecated and will be removed in v4.34.", FutureWarning)
+        if model_args.token is not None:
+            raise ValueError("`token` and `use_auth_token` are both specified. Please set only the argument `token`.")
+        model_args.token = model_args.use_auth_token
+
    # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The
    # information sent is the one passed as arguments along with your Python/PyTorch versions.
    send_example_telemetry("run_bart_dlm", model_args, data_args, framework="flax")
@@ -517,7 +530,7 @@ def main():
            data_args.dataset_name,
            data_args.dataset_config_name,
            cache_dir=model_args.cache_dir,
-            use_auth_token=True if model_args.use_auth_token else None,
+            token=model_args.token,
        )

        if "validation" not in datasets.keys():
@@ -526,14 +539,14 @@ def main():
                data_args.dataset_config_name,
                split=f"train[:{data_args.validation_split_percentage}%]",
                cache_dir=model_args.cache_dir,
-                use_auth_token=True if model_args.use_auth_token else None,
+                token=model_args.token,
            )
            datasets["train"] = load_dataset(
                data_args.dataset_name,
                data_args.dataset_config_name,
                split=f"train[{data_args.validation_split_percentage}%:]",
                cache_dir=model_args.cache_dir,
-                use_auth_token=True if model_args.use_auth_token else None,
+                token=model_args.token,
            )
    else:
        data_files = {}
@@ -548,7 +561,7 @@ def main():
            extension,
            data_files=data_files,
            cache_dir=model_args.cache_dir,
-            use_auth_token=True if model_args.use_auth_token else None,
+            token=model_args.token,
        )

        if "validation" not in datasets.keys():
@@ -557,14 +570,14 @@ def main():
                data_files=data_files,
                split=f"train[:{data_args.validation_split_percentage}%]",
                cache_dir=model_args.cache_dir,
-                use_auth_token=True if model_args.use_auth_token else None,
+                token=model_args.token,
            )
            datasets["train"] = load_dataset(
                extension,
                data_files=data_files,
                split=f"train[{data_args.validation_split_percentage}%:]",
                cache_dir=model_args.cache_dir,
-                use_auth_token=True if model_args.use_auth_token else None,
+                token=model_args.token,
            )
    # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at
    # https://huggingface.co/docs/datasets/loading_datasets.html.
@@ -576,14 +589,14 @@ def main():
            model_args.tokenizer_name,
            cache_dir=model_args.cache_dir,
            use_fast=model_args.use_fast_tokenizer,
-            token=True if model_args.use_auth_token else None,
+            token=model_args.token,
        )
    elif model_args.model_name_or_path:
        tokenizer = AutoTokenizer.from_pretrained(
            model_args.model_name_or_path,
            cache_dir=model_args.cache_dir,
            use_fast=model_args.use_fast_tokenizer,
-            token=True if model_args.use_auth_token else None,
+            token=model_args.token,
        )
    else:
        raise ValueError(
@@ -596,13 +609,13 @@ def main():
            model_args.config_name,
            cache_dir=model_args.cache_dir,
            vocab_size=len(tokenizer),
-            token=True if model_args.use_auth_token else None,
+            token=model_args.token,
        )
    elif model_args.model_name_or_path:
        config = BartConfig.from_pretrained(
            model_args.model_name_or_path,
            cache_dir=model_args.cache_dir,
-            token=True if model_args.use_auth_token else None,
+            token=model_args.token,
        )
    else:
        config = CONFIG_MAPPING[model_args.model_type]()
@@ -707,7 +720,7 @@ def main():
            config=config,
            seed=training_args.seed,
            dtype=getattr(jnp, model_args.dtype),
-            token=True if model_args.use_auth_token else None,
+            token=model_args.token,
        )
    else:
        config.vocab_size = len(tokenizer)
--- a/examples/flax/language-modeling/run_clm_flax.py
+++ b/examples/flax/language-modeling/run_clm_flax.py
@@ -27,6 +27,7 @@ import math
 import os
 import sys
 import time
+import warnings
 from dataclasses import asdict, dataclass, field
 from enum import Enum
 from itertools import chain
@@ -169,15 +170,21 @@ class ModelArguments:
            )
        },
    )
-    use_auth_token: bool = field(
-        default=False,
+    token: str = field(
+        default=None,
        metadata={
            "help": (
-                "Will use the token generated when running `huggingface-cli login` (necessary to use this script "
-                "with private models)."
+                "The token to use as HTTP bearer authorization for remote files. If not specified, will use the token "
+                "generated when running `huggingface-cli login` (stored in `~/.huggingface`)."
            )
        },
    )
+    use_auth_token: bool = field(
+        default=None,
+        metadata={
+            "help": "The `use_auth_token` argument is deprecated and will be removed in v4.34. Please use `token`."
+        },
+    )


@dataclass
@@ -334,6 +341,12 @@ def main():
    else:
        model_args, data_args, training_args = parser.parse_args_into_dataclasses()

+    if model_args.use_auth_token is not None:
+        warnings.warn("The `use_auth_token` argument is deprecated and will be removed in v4.34.", FutureWarning)
+        if model_args.token is not None:
+            raise ValueError("`token` and `use_auth_token` are both specified. Please set only the argument `token`.")
+        model_args.token = model_args.use_auth_token
+
    # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The
    # information sent is the one passed as arguments along with your Python/PyTorch versions.
    send_example_telemetry("run_clm", model_args, data_args, framework="flax")
@@ -397,7 +410,7 @@ def main():
            data_args.dataset_config_name,
            cache_dir=model_args.cache_dir,
            keep_in_memory=False,
-            use_auth_token=True if model_args.use_auth_token else None,
+            token=model_args.token,
        )

        if "validation" not in dataset.keys():
@@ -406,14 +419,14 @@ def main():
                data_args.dataset_config_name,
                split=f"train[:{data_args.validation_split_percentage}%]",
                cache_dir=model_args.cache_dir,
-                use_auth_token=True if model_args.use_auth_token else None,
+                token=model_args.token,
            )
            dataset["train"] = load_dataset(
                data_args.dataset_name,
                data_args.dataset_config_name,
                split=f"train[{data_args.validation_split_percentage}%:]",
                cache_dir=model_args.cache_dir,
-                use_auth_token=True if model_args.use_auth_token else None,
+                token=model_args.token,
            )
    else:
        data_files = {}
@@ -431,7 +444,7 @@ def main():
            data_files=data_files,
            cache_dir=model_args.cache_dir,
            **dataset_args,
-            use_auth_token=True if model_args.use_auth_token else None,
+            token=model_args.token,
        )

        if "validation" not in dataset.keys():
@@ -441,7 +454,7 @@ def main():
                split=f"train[:{data_args.validation_split_percentage}%]",
                cache_dir=model_args.cache_dir,
                **dataset_args,
-                use_auth_token=True if model_args.use_auth_token else None,
+                token=model_args.token,
            )
            dataset["train"] = load_dataset(
                extension,
@@ -449,7 +462,7 @@ def main():
                split=f"train[{data_args.validation_split_percentage}%:]",
                cache_dir=model_args.cache_dir,
                **dataset_args,
-                use_auth_token=True if model_args.use_auth_token else None,
+                token=model_args.token,
            )
    # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at
    # https://huggingface.co/docs/datasets/loading_datasets.html.
@@ -463,13 +476,13 @@ def main():
        config = AutoConfig.from_pretrained(
            model_args.config_name,
            cache_dir=model_args.cache_dir,
-            token=True if model_args.use_auth_token else None,
+            token=model_args.token,
        )
    elif model_args.model_name_or_path:
        config = AutoConfig.from_pretrained(
            model_args.model_name_or_path,
            cache_dir=model_args.cache_dir,
-            token=True if model_args.use_auth_token else None,
+            token=model_args.token,
        )
    else:
        config = CONFIG_MAPPING[model_args.model_type]()
@@ -480,14 +493,14 @@ def main():
            model_args.tokenizer_name,
            cache_dir=model_args.cache_dir,
            use_fast=model_args.use_fast_tokenizer,
-            token=True if model_args.use_auth_token else None,
+            token=model_args.token,
        )
    elif model_args.model_name_or_path:
        tokenizer = AutoTokenizer.from_pretrained(
            model_args.model_name_or_path,
            cache_dir=model_args.cache_dir,
            use_fast=model_args.use_fast_tokenizer,
-            token=True if model_args.use_auth_token else None,
+            token=model_args.token,
        )
    else:
        raise ValueError(
@@ -501,7 +514,7 @@ def main():
            config=config,
            seed=training_args.seed,
            dtype=getattr(jnp, model_args.dtype),
-            token=True if model_args.use_auth_token else None,
+            token=model_args.token,
        )
    else:
        model = FlaxAutoModelForCausalLM.from_config(
--- a/examples/flax/language-modeling/run_mlm_flax.py
+++ b/examples/flax/language-modeling/run_mlm_flax.py
@@ -26,6 +26,7 @@ import math
 import os
 import sys
 import time
+import warnings
 from dataclasses import asdict, dataclass, field
 from enum import Enum
 from itertools import chain
@@ -174,15 +175,21 @@ class ModelArguments:
            )
        },
    )
-    use_auth_token: bool = field(
-        default=False,
+    token: str = field(
+        default=None,
        metadata={
            "help": (
-                "Will use the token generated when running `huggingface-cli login` (necessary to use this script "
-                "with private models)."
+                "The token to use as HTTP bearer authorization for remote files. If not specified, will use the token "
+                "generated when running `huggingface-cli login` (stored in `~/.huggingface`)."
            )
        },
    )
+    use_auth_token: bool = field(
+        default=None,
+        metadata={
+            "help": "The `use_auth_token` argument is deprecated and will be removed in v4.34. Please use `token`."
+        },
+    )


@dataclass
@@ -377,6 +384,12 @@ def main():
    else:
        model_args, data_args, training_args = parser.parse_args_into_dataclasses()

+    if model_args.use_auth_token is not None:
+        warnings.warn("The `use_auth_token` argument is deprecated and will be removed in v4.34.", FutureWarning)
+        if model_args.token is not None:
+            raise ValueError("`token` and `use_auth_token` are both specified. Please set only the argument `token`.")
+        model_args.token = model_args.use_auth_token
+
    # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The
    # information sent is the one passed as arguments along with your Python/PyTorch versions.
    send_example_telemetry("run_mlm", model_args, data_args, framework="flax")
@@ -434,7 +447,7 @@ def main():
            data_args.dataset_name,
            data_args.dataset_config_name,
            cache_dir=model_args.cache_dir,
-            use_auth_token=True if model_args.use_auth_token else None,
+            token=model_args.token,
        )

        if "validation" not in datasets.keys():
@@ -443,14 +456,14 @@ def main():
                data_args.dataset_config_name,
                split=f"train[:{data_args.validation_split_percentage}%]",
                cache_dir=model_args.cache_dir,
-                use_auth_token=True if model_args.use_auth_token else None,
+                token=model_args.token,
            )
            datasets["train"] = load_dataset(
                data_args.dataset_name,
                data_args.dataset_config_name,
                split=f"train[{data_args.validation_split_percentage}%:]",
                cache_dir=model_args.cache_dir,
-                use_auth_token=True if model_args.use_auth_token else None,
+                token=model_args.token,
            )
    else:
        data_files = {}
@@ -465,7 +478,7 @@ def main():
            extension,
            data_files=data_files,
            cache_dir=model_args.cache_dir,
-            use_auth_token=True if model_args.use_auth_token else None,
+            token=model_args.token,
        )

        if "validation" not in datasets.keys():
@@ -474,14 +487,14 @@ def main():
                data_files=data_files,
                split=f"train[:{data_args.validation_split_percentage}%]",
                cache_dir=model_args.cache_dir,
-                use_auth_token=True if model_args.use_auth_token else None,
+                token=model_args.token,
            )
            datasets["train"] = load_dataset(
                extension,
                data_files=data_files,
                split=f"train[{data_args.validation_split_percentage}%:]",
                cache_dir=model_args.cache_dir,
-                use_auth_token=True if model_args.use_auth_token else None,
+                token=model_args.token,
            )
    # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at
    # https://huggingface.co/docs/datasets/loading_datasets.html.
@@ -495,13 +508,13 @@ def main():
        config = AutoConfig.from_pretrained(
            model_args.config_name,
            cache_dir=model_args.cache_dir,
-            token=True if model_args.use_auth_token else None,
+            token=model_args.token,
        )
    elif model_args.model_name_or_path:
        config = AutoConfig.from_pretrained(
            model_args.model_name_or_path,
            cache_dir=model_args.cache_dir,
-            token=True if model_args.use_auth_token else None,
+            token=model_args.token,
        )
    else:
        config = CONFIG_MAPPING[model_args.model_type]()
@@ -512,14 +525,14 @@ def main():
            model_args.tokenizer_name,
            cache_dir=model_args.cache_dir,
            use_fast=model_args.use_fast_tokenizer,
-            token=True if model_args.use_auth_token else None,
+            token=model_args.token,
        )
    elif model_args.model_name_or_path:
        tokenizer = AutoTokenizer.from_pretrained(
            model_args.model_name_or_path,
            cache_dir=model_args.cache_dir,
            use_fast=model_args.use_fast_tokenizer,
-            token=True if model_args.use_auth_token else None,
+            token=model_args.token,
        )
    else:
        raise ValueError(
@@ -638,7 +651,7 @@ def main():
            config=config,
            seed=training_args.seed,
            dtype=getattr(jnp, model_args.dtype),
-            token=True if model_args.use_auth_token else None,
+            token=model_args.token,
        )
    else:
        model = FlaxAutoModelForMaskedLM.from_config(
--- a/examples/flax/language-modeling/run_t5_mlm_flax.py
+++ b/examples/flax/language-modeling/run_t5_mlm_flax.py
@@ -25,6 +25,7 @@ import math
 import os
 import sys
 import time
+import warnings
 from dataclasses import asdict, dataclass, field

 # You can also adapt this script on your own masked language modeling task. Pointers for this are left as comments.
@@ -168,15 +169,21 @@ class ModelArguments:
            )
        },
    )
-    use_auth_token: bool = field(
-        default=False,
+    token: str = field(
+        default=None,
        metadata={
            "help": (
-                "Will use the token generated when running `huggingface-cli login` (necessary to use this script "
-                "with private models)."
+                "The token to use as HTTP bearer authorization for remote files. If not specified, will use the token "
+                "generated when running `huggingface-cli login` (stored in `~/.huggingface`)."
            )
        },
    )
+    use_auth_token: bool = field(
+        default=None,
+        metadata={
+            "help": "The `use_auth_token` argument is deprecated and will be removed in v4.34. Please use `token`."
+        },
+    )


@dataclass
@@ -504,6 +511,12 @@ def main():
    else:
        model_args, data_args, training_args = parser.parse_args_into_dataclasses()

+    if model_args.use_auth_token is not None:
+        warnings.warn("The `use_auth_token` argument is deprecated and will be removed in v4.34.", FutureWarning)
+        if model_args.token is not None:
+            raise ValueError("`token` and `use_auth_token` are both specified. Please set only the argument `token`.")
+        model_args.token = model_args.use_auth_token
+
    # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The
    # information sent is the one passed as arguments along with your Python/PyTorch versions.
    send_example_telemetry("run_t5_mlm", model_args, data_args, framework="flax")
@@ -558,7 +571,7 @@ def main():
            data_args.dataset_name,
            data_args.dataset_config_name,
            cache_dir=model_args.cache_dir,
-            use_auth_token=True if model_args.use_auth_token else None,
+            token=model_args.token,
        )

        if "validation" not in datasets.keys():
@@ -567,14 +580,14 @@ def main():
                data_args.dataset_config_name,
                split=f"train[:{data_args.validation_split_percentage}%]",
                cache_dir=model_args.cache_dir,
-                use_auth_token=True if model_args.use_auth_token else None,
+                token=model_args.token,
            )
            datasets["train"] = load_dataset(
                data_args.dataset_name,
                data_args.dataset_config_name,
                split=f"train[{data_args.validation_split_percentage}%:]",
                cache_dir=model_args.cache_dir,
-                use_auth_token=True if model_args.use_auth_token else None,
+                token=model_args.token,
            )
    else:
        data_files = {}
@@ -589,7 +602,7 @@ def main():
            extension,
            data_files=data_files,
            cache_dir=model_args.cache_dir,
-            use_auth_token=True if model_args.use_auth_token else None,
+            token=model_args.token,
        )

        if "validation" not in datasets.keys():
@@ -598,14 +611,14 @@ def main():
                data_files=data_files,
                split=f"train[:{data_args.validation_split_percentage}%]",
                cache_dir=model_args.cache_dir,
-                use_auth_token=True if model_args.use_auth_token else None,
+                token=model_args.token,
            )
            datasets["train"] = load_dataset(
                extension,
                data_files=data_files,
                split=f"train[{data_args.validation_split_percentage}%:]",
                cache_dir=model_args.cache_dir,
-                use_auth_token=True if model_args.use_auth_token else None,
+                token=model_args.token,
            )
    # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at
    # https://huggingface.co/docs/datasets/loading_datasets.html.
@@ -617,14 +630,14 @@ def main():
            model_args.tokenizer_name,
            cache_dir=model_args.cache_dir,
            use_fast=model_args.use_fast_tokenizer,
-            token=True if model_args.use_auth_token else None,
+            token=model_args.token,
        )
    elif model_args.model_name_or_path:
        tokenizer = AutoTokenizer.from_pretrained(
            model_args.model_name_or_path,
            cache_dir=model_args.cache_dir,
            use_fast=model_args.use_fast_tokenizer,
-            token=True if model_args.use_auth_token else None,
+            token=model_args.token,
        )
    else:
        raise ValueError(
@@ -637,13 +650,13 @@ def main():
            model_args.config_name,
            cache_dir=model_args.cache_dir,
            vocab_size=len(tokenizer),
-            token=True if model_args.use_auth_token else None,
+            token=model_args.token,
        )
    elif model_args.model_name_or_path:
        config = T5Config.from_pretrained(
            model_args.model_name_or_path,
            cache_dir=model_args.cache_dir,
-            token=True if model_args.use_auth_token else None,
+            token=model_args.token,
        )
    else:
        config = CONFIG_MAPPING[model_args.model_type]()
@@ -738,7 +751,7 @@ def main():
            config=config,
            seed=training_args.seed,
            dtype=getattr(jnp, model_args.dtype),
-            token=True if model_args.use_auth_token else None,
+            token=model_args.token,
        )
    else:
        config.vocab_size = len(tokenizer)