From 91f591f7bcad6ef4027b5244cab0eb505a491671 Mon Sep 17 00:00:00 2001 From: Quentin Lhoest <42851186+lhoestq@users.noreply.github.com> Date: Fri, 25 Jul 2025 10:46:22 +0200 Subject: [PATCH] Make pytorch examples UV-compatible (#39635) * update release.py * add uv headers in some pytorch examples * rest of pytorch examples * style --- .../audio-classification/requirements.txt | 4 ++-- .../run_audio_classification.py | 11 +++++++++ .../contrastive-image-text/run_clip.py | 10 ++++++++ .../run_image_classification.py | 12 ++++++++++ .../run_image_classification_no_trainer.py | 13 ++++++++++ examples/pytorch/image-pretraining/run_mae.py | 9 +++++++ examples/pytorch/image-pretraining/run_mim.py | 9 +++++++ .../image-pretraining/run_mim_no_trainer.py | 9 +++++++ .../run_instance_segmentation.py | 11 +++++++++ .../run_instance_segmentation_no_trainer.py | 11 +++++++++ examples/pytorch/language-modeling/run_clm.py | 15 ++++++++++++ .../language-modeling/run_clm_no_trainer.py | 15 ++++++++++++ examples/pytorch/language-modeling/run_fim.py | 15 ++++++++++++ .../language-modeling/run_fim_no_trainer.py | 15 ++++++++++++ examples/pytorch/language-modeling/run_mlm.py | 15 ++++++++++++ .../language-modeling/run_mlm_no_trainer.py | 15 ++++++++++++ examples/pytorch/language-modeling/run_plm.py | 15 ++++++++++++ examples/pytorch/multiple-choice/run_swag.py | 12 ++++++++++ .../multiple-choice/run_swag_no_trainer.py | 12 ++++++++++ .../object-detection/run_object_detection.py | 11 +++++++++ .../run_object_detection_no_trainer.py | 12 ++++++++++ .../run_semantic_segmentation.py | 12 ++++++++++ .../run_semantic_segmentation_no_trainer.py | 13 ++++++++++ .../speech-pretraining/requirements.txt | 2 +- .../run_wav2vec2_pretraining_no_trainer.py | 11 +++++++++ .../speech-recognition/requirements.txt | 2 +- .../run_speech_recognition_ctc.py | 12 ++++++++++ .../run_speech_recognition_ctc_adapter.py | 12 ++++++++++ .../run_speech_recognition_seq2seq.py | 13 ++++++++++ .../summarization/run_summarization.py | 16 +++++++++++++ .../run_summarization_no_trainer.py | 16 +++++++++++++ .../text-classification/run_classification.py | 15 ++++++++++++ .../pytorch/text-classification/run_glue.py | 15 ++++++++++++ .../run_glue_no_trainer.py | 15 ++++++++++++ .../pytorch/text-classification/run_xnli.py | 15 ++++++++++++ .../pytorch/text-generation/run_generation.py | 11 +++++++++ .../run_generation_contrastive_search.py | 11 +++++++++ .../pytorch/token-classification/run_ner.py | 12 ++++++++++ .../run_ner_no_trainer.py | 12 ++++++++++ .../pytorch/translation/run_translation.py | 15 ++++++++++++ .../translation/run_translation_no_trainer.py | 15 ++++++++++++ utils/release.py | 24 +++++++++++++++---- 42 files changed, 511 insertions(+), 9 deletions(-) diff --git a/examples/pytorch/audio-classification/requirements.txt b/examples/pytorch/audio-classification/requirements.txt index acf058d4cf..d23b9b86f1 100644 --- a/examples/pytorch/audio-classification/requirements.txt +++ b/examples/pytorch/audio-classification/requirements.txt @@ -1,5 +1,5 @@ -datasets>=1.14.0 +datasets[audio]>=1.14.0 evaluate librosa torchaudio -torch>=1.6 \ No newline at end of file +torch>=1.6 diff --git a/examples/pytorch/audio-classification/run_audio_classification.py b/examples/pytorch/audio-classification/run_audio_classification.py index a06f334bdc..bb5651ab25 100644 --- a/examples/pytorch/audio-classification/run_audio_classification.py +++ b/examples/pytorch/audio-classification/run_audio_classification.py @@ -13,6 +13,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +# /// script +# dependencies = [ +# "transformers @ git+https://github.com/huggingface/transformers.git", +# "datasets[audio]>=1.14.0", +# "evaluate", +# "librosa", +# "torchaudio", +# "torch>=1.6", +# ] +# /// + import logging import os import sys diff --git a/examples/pytorch/contrastive-image-text/run_clip.py b/examples/pytorch/contrastive-image-text/run_clip.py index 58dba1083c..229c97dd4f 100644 --- a/examples/pytorch/contrastive-image-text/run_clip.py +++ b/examples/pytorch/contrastive-image-text/run_clip.py @@ -12,6 +12,16 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +# /// script +# dependencies = [ +# "transformers @ git+https://github.com/huggingface/transformers.git", +# "torch>=1.5.0", +# "torchvision>=0.6.0", +# "datasets>=1.8.0", +# ] +# /// + """ Training a CLIP like dual encoder models using text and vision encoders in the library. diff --git a/examples/pytorch/image-classification/run_image_classification.py b/examples/pytorch/image-classification/run_image_classification.py index ff2e8873ba..baf8f15d92 100755 --- a/examples/pytorch/image-classification/run_image_classification.py +++ b/examples/pytorch/image-classification/run_image_classification.py @@ -12,6 +12,18 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and +# /// script +# dependencies = [ +# "transformers @ git+https://github.com/huggingface/transformers.git", +# "accelerate>=0.12.0", +# "torch>=1.5.0", +# "torchvision>=0.6.0", +# "datasets>=2.14.0", +# "evaluate", +# "scikit-learn", +# ] +# /// + import logging import os import sys diff --git a/examples/pytorch/image-classification/run_image_classification_no_trainer.py b/examples/pytorch/image-classification/run_image_classification_no_trainer.py index 49fa4bd0fd..30a65e6b6e 100644 --- a/examples/pytorch/image-classification/run_image_classification_no_trainer.py +++ b/examples/pytorch/image-classification/run_image_classification_no_trainer.py @@ -11,6 +11,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +# /// script +# dependencies = [ +# "transformers @ git+https://github.com/huggingface/transformers.git", +# "accelerate>=0.12.0", +# "torch>=1.5.0", +# "torchvision>=0.6.0", +# "datasets>=2.14.0", +# "evaluate", +# "scikit-learn", +# ] +# /// + """Finetuning any 🤗 Transformers model for image classification leveraging 🤗 Accelerate.""" import argparse diff --git a/examples/pytorch/image-pretraining/run_mae.py b/examples/pytorch/image-pretraining/run_mae.py index 514a4314ad..20ae2b5316 100644 --- a/examples/pytorch/image-pretraining/run_mae.py +++ b/examples/pytorch/image-pretraining/run_mae.py @@ -12,6 +12,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and +# /// script +# dependencies = [ +# "transformers @ git+https://github.com/huggingface/transformers.git", +# "torch>=1.5.0", +# "torchvision>=0.6.0", +# "datasets>=1.8.0", +# ] +# /// + import logging import os import sys diff --git a/examples/pytorch/image-pretraining/run_mim.py b/examples/pytorch/image-pretraining/run_mim.py index a45dc6b757..8d6b1ba6d5 100644 --- a/examples/pytorch/image-pretraining/run_mim.py +++ b/examples/pytorch/image-pretraining/run_mim.py @@ -12,6 +12,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and +# /// script +# dependencies = [ +# "transformers @ git+https://github.com/huggingface/transformers.git", +# "torch>=1.5.0", +# "torchvision>=0.6.0", +# "datasets>=1.8.0", +# ] +# /// + import logging import os import sys diff --git a/examples/pytorch/image-pretraining/run_mim_no_trainer.py b/examples/pytorch/image-pretraining/run_mim_no_trainer.py index ff58130a0b..97b5e95778 100644 --- a/examples/pytorch/image-pretraining/run_mim_no_trainer.py +++ b/examples/pytorch/image-pretraining/run_mim_no_trainer.py @@ -12,6 +12,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and +# /// script +# dependencies = [ +# "transformers @ git+https://github.com/huggingface/transformers.git", +# "torch>=1.5.0", +# "torchvision>=0.6.0", +# "datasets>=1.8.0", +# ] +# /// + import argparse import logging import math diff --git a/examples/pytorch/instance-segmentation/run_instance_segmentation.py b/examples/pytorch/instance-segmentation/run_instance_segmentation.py index 357a020822..6c277e49ca 100644 --- a/examples/pytorch/instance-segmentation/run_instance_segmentation.py +++ b/examples/pytorch/instance-segmentation/run_instance_segmentation.py @@ -12,6 +12,17 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and +# /// script +# dependencies = [ +# "transformers @ git+https://github.com/huggingface/transformers.git", +# "albumentations >= 1.4.16", +# "timm", +# "datasets", +# "torchmetrics", +# "pycocotools", +# ] +# /// + """Finetuning 🤗 Transformers model for instance segmentation leveraging the Trainer API.""" import logging diff --git a/examples/pytorch/instance-segmentation/run_instance_segmentation_no_trainer.py b/examples/pytorch/instance-segmentation/run_instance_segmentation_no_trainer.py index 92a5a1537b..38a712bf0c 100644 --- a/examples/pytorch/instance-segmentation/run_instance_segmentation_no_trainer.py +++ b/examples/pytorch/instance-segmentation/run_instance_segmentation_no_trainer.py @@ -12,6 +12,17 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and +# /// script +# dependencies = [ +# "transformers @ git+https://github.com/huggingface/transformers.git", +# "albumentations >= 1.4.16", +# "timm", +# "datasets", +# "torchmetrics", +# "pycocotools", +# ] +# /// + """Finetuning 🤗 Transformers model for instance segmentation with Accelerate 🚀.""" import argparse diff --git a/examples/pytorch/language-modeling/run_clm.py b/examples/pytorch/language-modeling/run_clm.py index dad24cd7ef..c8447cd4c4 100755 --- a/examples/pytorch/language-modeling/run_clm.py +++ b/examples/pytorch/language-modeling/run_clm.py @@ -12,6 +12,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +# /// script +# dependencies = [ +# "transformers @ git+https://github.com/huggingface/transformers.git", +# "albumentations >= 1.4.16", +# "accelerate >= 0.12.0", +# "torch >= 1.3", +# "datasets >= 2.14.0", +# "sentencepiece != 0.1.92", +# "protobuf", +# "evaluate", +# "scikit-learn", +# ] +# /// + """ Fine-tuning the library models for causal language modeling (GPT, GPT-2, CTRL, ...) on a text file or a dataset. diff --git a/examples/pytorch/language-modeling/run_clm_no_trainer.py b/examples/pytorch/language-modeling/run_clm_no_trainer.py index 554174ab1b..f0ef6a6acb 100755 --- a/examples/pytorch/language-modeling/run_clm_no_trainer.py +++ b/examples/pytorch/language-modeling/run_clm_no_trainer.py @@ -12,6 +12,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +# /// script +# dependencies = [ +# "transformers @ git+https://github.com/huggingface/transformers.git", +# "albumentations >= 1.4.16", +# "accelerate >= 0.12.0", +# "torch >= 1.3", +# "datasets >= 2.14.0", +# "sentencepiece != 0.1.92", +# "protobuf", +# "evaluate", +# "scikit-learn", +# ] +# /// + """ Fine-tuning the library models for causal language modeling (GPT, GPT-2, CTRL, ...) on a text file or a dataset without using HuggingFace Trainer. diff --git a/examples/pytorch/language-modeling/run_fim.py b/examples/pytorch/language-modeling/run_fim.py index b77922c15a..dd2b5ab6b8 100644 --- a/examples/pytorch/language-modeling/run_fim.py +++ b/examples/pytorch/language-modeling/run_fim.py @@ -12,6 +12,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +# /// script +# dependencies = [ +# "transformers @ git+https://github.com/huggingface/transformers.git", +# "albumentations >= 1.4.16", +# "accelerate >= 0.12.0", +# "torch >= 1.3", +# "datasets >= 2.14.0", +# "sentencepiece != 0.1.92", +# "protobuf", +# "evaluate", +# "scikit-learn", +# ] +# /// + """ Fine-tuning the library models for causal language modeling using Fill-in-the middle (FIM) objective on a text file or a dataset. diff --git a/examples/pytorch/language-modeling/run_fim_no_trainer.py b/examples/pytorch/language-modeling/run_fim_no_trainer.py index bb3ea1db29..d63bbd04a7 100644 --- a/examples/pytorch/language-modeling/run_fim_no_trainer.py +++ b/examples/pytorch/language-modeling/run_fim_no_trainer.py @@ -12,6 +12,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +# /// script +# dependencies = [ +# "transformers @ git+https://github.com/huggingface/transformers.git", +# "albumentations >= 1.4.16", +# "accelerate >= 0.12.0", +# "torch >= 1.3", +# "datasets >= 2.14.0", +# "sentencepiece != 0.1.92", +# "protobuf", +# "evaluate", +# "scikit-learn", +# ] +# /// + """ Fine-tuning the library models for causal language modeling using Fill-in-the middle (FIM) objective on a text file or a dataset without using HuggingFace Trainer. diff --git a/examples/pytorch/language-modeling/run_mlm.py b/examples/pytorch/language-modeling/run_mlm.py index 67a574f44d..57b2dbc713 100755 --- a/examples/pytorch/language-modeling/run_mlm.py +++ b/examples/pytorch/language-modeling/run_mlm.py @@ -12,6 +12,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +# /// script +# dependencies = [ +# "transformers @ git+https://github.com/huggingface/transformers.git", +# "albumentations >= 1.4.16", +# "accelerate >= 0.12.0", +# "torch >= 1.3", +# "datasets >= 2.14.0", +# "sentencepiece != 0.1.92", +# "protobuf", +# "evaluate", +# "scikit-learn", +# ] +# /// + """ Fine-tuning the library models for masked language modeling (BERT, ALBERT, RoBERTa...) on a text file or a dataset. diff --git a/examples/pytorch/language-modeling/run_mlm_no_trainer.py b/examples/pytorch/language-modeling/run_mlm_no_trainer.py index 42384d9e1e..5e64f27b6c 100755 --- a/examples/pytorch/language-modeling/run_mlm_no_trainer.py +++ b/examples/pytorch/language-modeling/run_mlm_no_trainer.py @@ -12,6 +12,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +# /// script +# dependencies = [ +# "transformers @ git+https://github.com/huggingface/transformers.git", +# "albumentations >= 1.4.16", +# "accelerate >= 0.12.0", +# "torch >= 1.3", +# "datasets >= 2.14.0", +# "sentencepiece != 0.1.92", +# "protobuf", +# "evaluate", +# "scikit-learn", +# ] +# /// + """ Fine-tuning the library models for masked language modeling (BERT, ALBERT, RoBERTa...) on a text file or a dataset without using HuggingFace Trainer. diff --git a/examples/pytorch/language-modeling/run_plm.py b/examples/pytorch/language-modeling/run_plm.py index 1c35de8d03..844350af1f 100755 --- a/examples/pytorch/language-modeling/run_plm.py +++ b/examples/pytorch/language-modeling/run_plm.py @@ -12,6 +12,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +# /// script +# dependencies = [ +# "transformers @ git+https://github.com/huggingface/transformers.git", +# "albumentations >= 1.4.16", +# "accelerate >= 0.12.0", +# "torch >= 1.3", +# "datasets >= 2.14.0", +# "sentencepiece != 0.1.92", +# "protobuf", +# "evaluate", +# "scikit-learn", +# ] +# /// + """ Fine-tuning the library models for permutation language modeling. """ diff --git a/examples/pytorch/multiple-choice/run_swag.py b/examples/pytorch/multiple-choice/run_swag.py index 927d439335..b1487cdbe9 100755 --- a/examples/pytorch/multiple-choice/run_swag.py +++ b/examples/pytorch/multiple-choice/run_swag.py @@ -12,6 +12,18 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +# /// script +# dependencies = [ +# "transformers @ git+https://github.com/huggingface/transformers.git", +# "accelerate >= 0.12.0", +# "sentencepiece != 0.1.92", +# "protobuf", +# "torch >= 1.3", +# "evaluate", +# ] +# /// + """ Fine-tuning the library models for multiple choice. """ diff --git a/examples/pytorch/multiple-choice/run_swag_no_trainer.py b/examples/pytorch/multiple-choice/run_swag_no_trainer.py index d0581c5b6c..fab10a263a 100755 --- a/examples/pytorch/multiple-choice/run_swag_no_trainer.py +++ b/examples/pytorch/multiple-choice/run_swag_no_trainer.py @@ -12,6 +12,18 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +# /// script +# dependencies = [ +# "transformers @ git+https://github.com/huggingface/transformers.git", +# "accelerate >= 0.12.0", +# "sentencepiece != 0.1.92", +# "protobuf", +# "torch >= 1.3", +# "evaluate", +# ] +# /// + """ Fine-tuning a 🤗 Transformers model on multiple choice relying on the accelerate library without using a Trainer. """ diff --git a/examples/pytorch/object-detection/run_object_detection.py b/examples/pytorch/object-detection/run_object_detection.py index f534d9a323..51f5d6ddca 100644 --- a/examples/pytorch/object-detection/run_object_detection.py +++ b/examples/pytorch/object-detection/run_object_detection.py @@ -12,6 +12,17 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and +# /// script +# dependencies = [ +# "transformers @ git+https://github.com/huggingface/transformers.git", +# "albumentations >= 1.4.16", +# "timm", +# "datasets>=4.0", +# "torchmetrics", +# "pycocotools", +# ] +# /// + """Finetuning any 🤗 Transformers model supported by AutoModelForObjectDetection for object detection leveraging the Trainer API.""" import logging diff --git a/examples/pytorch/object-detection/run_object_detection_no_trainer.py b/examples/pytorch/object-detection/run_object_detection_no_trainer.py index 1133435d7b..522b99dda1 100644 --- a/examples/pytorch/object-detection/run_object_detection_no_trainer.py +++ b/examples/pytorch/object-detection/run_object_detection_no_trainer.py @@ -11,6 +11,18 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +# /// script +# dependencies = [ +# "transformers @ git+https://github.com/huggingface/transformers.git", +# "albumentations >= 1.4.16", +# "timm", +# "datasets>=4.0", +# "torchmetrics", +# "pycocotools", +# ] +# /// + """Finetuning 🤗 Transformers model for object detection with Accelerate.""" import argparse diff --git a/examples/pytorch/semantic-segmentation/run_semantic_segmentation.py b/examples/pytorch/semantic-segmentation/run_semantic_segmentation.py index 3e43383ca4..cf38403ca8 100644 --- a/examples/pytorch/semantic-segmentation/run_semantic_segmentation.py +++ b/examples/pytorch/semantic-segmentation/run_semantic_segmentation.py @@ -12,6 +12,18 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and +# /// script +# dependencies = [ +# "transformers @ git+https://github.com/huggingface/transformers.git", +# "datasets >= 2.0.0", +# "torch >= 1.3", +# "accelerate", +# "evaluate"" +# "Pillow", +# "albumentations >= 1.4.16", +# ] +# /// + import json import logging import os diff --git a/examples/pytorch/semantic-segmentation/run_semantic_segmentation_no_trainer.py b/examples/pytorch/semantic-segmentation/run_semantic_segmentation_no_trainer.py index fca89a6428..9dfe745738 100644 --- a/examples/pytorch/semantic-segmentation/run_semantic_segmentation_no_trainer.py +++ b/examples/pytorch/semantic-segmentation/run_semantic_segmentation_no_trainer.py @@ -11,6 +11,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +# /// script +# dependencies = [ +# "transformers @ git+https://github.com/huggingface/transformers.git", +# "datasets >= 2.0.0", +# "torch >= 1.3", +# "accelerate", +# "evaluate"" +# "Pillow", +# "albumentations >= 1.4.16", +# ] +# /// + """Finetuning any 🤗 Transformers model supported by AutoModelForSemanticSegmentation for semantic segmentation.""" import argparse diff --git a/examples/pytorch/speech-pretraining/requirements.txt b/examples/pytorch/speech-pretraining/requirements.txt index c270b3a565..57851d2a09 100644 --- a/examples/pytorch/speech-pretraining/requirements.txt +++ b/examples/pytorch/speech-pretraining/requirements.txt @@ -1,4 +1,4 @@ -datasets >= 1.12.0 +datasets[audio] >= 1.12.0 torch >= 1.5 torchaudio accelerate >= 0.12.0 diff --git a/examples/pytorch/speech-pretraining/run_wav2vec2_pretraining_no_trainer.py b/examples/pytorch/speech-pretraining/run_wav2vec2_pretraining_no_trainer.py index 111706412a..f30fd1676a 100755 --- a/examples/pytorch/speech-pretraining/run_wav2vec2_pretraining_no_trainer.py +++ b/examples/pytorch/speech-pretraining/run_wav2vec2_pretraining_no_trainer.py @@ -12,6 +12,17 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and +# /// script +# dependencies = [ +# "transformers @ git+https://github.com/huggingface/transformers.git", +# "datasets[audio] >= 1.12.0", +# "torch >= 1.5", +# "torchaudio", +# "accelerate >= 0.12.0", +# "librosa", +# ] +# /// + """Pre-Training a 🤗 Wav2Vec2 model on unlabeled audio data""" import argparse diff --git a/examples/pytorch/speech-recognition/requirements.txt b/examples/pytorch/speech-recognition/requirements.txt index a16697b038..4137ea02ad 100644 --- a/examples/pytorch/speech-recognition/requirements.txt +++ b/examples/pytorch/speech-recognition/requirements.txt @@ -1,4 +1,4 @@ -datasets >= 1.18.0 +datasets[audio] >= 1.18.0 torch >= 1.5 torchaudio librosa diff --git a/examples/pytorch/speech-recognition/run_speech_recognition_ctc.py b/examples/pytorch/speech-recognition/run_speech_recognition_ctc.py index 879f3320c1..e4cf938561 100755 --- a/examples/pytorch/speech-recognition/run_speech_recognition_ctc.py +++ b/examples/pytorch/speech-recognition/run_speech_recognition_ctc.py @@ -13,6 +13,18 @@ # See the License for the specific language governing permissions and # limitations under the License. +# /// script +# dependencies = [ +# "transformers @ git+https://github.com/huggingface/transformers.git", +# "datasets[audio] >= 1.18.0", +# "torch >= 1.5", +# "torchaudio", +# "librosa", +# "jiwer", +# "evaluate", +# ] +# /// + """Fine-tuning a 🤗 Transformers CTC model for automatic speech recognition""" import functools diff --git a/examples/pytorch/speech-recognition/run_speech_recognition_ctc_adapter.py b/examples/pytorch/speech-recognition/run_speech_recognition_ctc_adapter.py index 8f3b61ea43..c7b5df009a 100755 --- a/examples/pytorch/speech-recognition/run_speech_recognition_ctc_adapter.py +++ b/examples/pytorch/speech-recognition/run_speech_recognition_ctc_adapter.py @@ -13,6 +13,18 @@ # See the License for the specific language governing permissions and # limitations under the License. +# /// script +# dependencies = [ +# "transformers @ git+https://github.com/huggingface/transformers.git", +# "datasets[audio] >= 1.18.0", +# "torch >= 1.5", +# "torchaudio", +# "librosa", +# "jiwer", +# "evaluate", +# ] +# /// + """Fine-tuning a 🤗 Transformers CTC adapter model for automatic speech recognition""" import functools diff --git a/examples/pytorch/speech-recognition/run_speech_recognition_seq2seq.py b/examples/pytorch/speech-recognition/run_speech_recognition_seq2seq.py index 76e8325548..d68f97d280 100755 --- a/examples/pytorch/speech-recognition/run_speech_recognition_seq2seq.py +++ b/examples/pytorch/speech-recognition/run_speech_recognition_seq2seq.py @@ -12,6 +12,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +# /// script +# dependencies = [ +# "transformers @ git+https://github.com/huggingface/transformers.git", +# "datasets[audio] >= 1.18.0", +# "torch >= 1.5", +# "torchaudio", +# "librosa", +# "jiwer", +# "evaluate", +# ] +# /// + """ Fine-tuning the library models for sequence to sequence speech recognition. """ diff --git a/examples/pytorch/summarization/run_summarization.py b/examples/pytorch/summarization/run_summarization.py index d22d01479d..bca7ef6dc8 100755 --- a/examples/pytorch/summarization/run_summarization.py +++ b/examples/pytorch/summarization/run_summarization.py @@ -12,6 +12,22 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +# /// script +# dependencies = [ +# "transformers @ git+https://github.com/huggingface/transformers.git", +# "accelerate >= 0.12.0", +# "datasets >= 1.8.0", +# "sentencepiece != 0.1.92", +# "protobuf", +# "rouge-score", +# "nltk", +# "py7zr", +# "torch >= 1.3", +# "evaluate", +# ] +# /// + """ Fine-tuning the library models for sequence to sequence. """ diff --git a/examples/pytorch/summarization/run_summarization_no_trainer.py b/examples/pytorch/summarization/run_summarization_no_trainer.py index e769017eb1..31b18308f6 100644 --- a/examples/pytorch/summarization/run_summarization_no_trainer.py +++ b/examples/pytorch/summarization/run_summarization_no_trainer.py @@ -12,6 +12,22 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +# /// script +# dependencies = [ +# "transformers @ git+https://github.com/huggingface/transformers.git", +# "accelerate >= 0.12.0", +# "datasets >= 1.8.0", +# "sentencepiece != 0.1.92", +# "protobuf", +# "rouge-score", +# "nltk", +# "py7zr", +# "torch >= 1.3", +# "evaluate", +# ] +# /// + """ Fine-tuning a 🤗 Transformers model on summarization. """ diff --git a/examples/pytorch/text-classification/run_classification.py b/examples/pytorch/text-classification/run_classification.py index 8de9e117ca..9e5344d69a 100755 --- a/examples/pytorch/text-classification/run_classification.py +++ b/examples/pytorch/text-classification/run_classification.py @@ -12,6 +12,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +# /// script +# dependencies = [ +# "transformers @ git+https://github.com/huggingface/transformers.git", +# "accelerate >= 0.12.0", +# "datasets >= 1.8.0", +# "sentencepiece != 0.1.92", +# "scipy", +# "scikit-learn", +# "protobuf", +# "torch >= 1.3", +# "evaluate", +# ] +# /// + """Finetuning the library models for text classification.""" # You can also adapt this script on your own text classification task. Pointers for this are left as comments. diff --git a/examples/pytorch/text-classification/run_glue.py b/examples/pytorch/text-classification/run_glue.py index 76f4e30ce8..6072433b58 100755 --- a/examples/pytorch/text-classification/run_glue.py +++ b/examples/pytorch/text-classification/run_glue.py @@ -12,6 +12,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +# /// script +# dependencies = [ +# "transformers @ git+https://github.com/huggingface/transformers.git", +# "accelerate >= 0.12.0", +# "datasets >= 1.8.0", +# "sentencepiece != 0.1.92", +# "scipy", +# "scikit-learn", +# "protobuf", +# "torch >= 1.3", +# "evaluate", +# ] +# /// + """Finetuning the library models for sequence classification on GLUE.""" # You can also adapt this script on your own text classification task. Pointers for this are left as comments. diff --git a/examples/pytorch/text-classification/run_glue_no_trainer.py b/examples/pytorch/text-classification/run_glue_no_trainer.py index 5bab946d38..88ffc3034a 100644 --- a/examples/pytorch/text-classification/run_glue_no_trainer.py +++ b/examples/pytorch/text-classification/run_glue_no_trainer.py @@ -11,6 +11,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +# /// script +# dependencies = [ +# "transformers @ git+https://github.com/huggingface/transformers.git", +# "accelerate >= 0.12.0", +# "datasets >= 1.8.0", +# "sentencepiece != 0.1.92", +# "scipy", +# "scikit-learn", +# "protobuf", +# "torch >= 1.3", +# "evaluate", +# ] +# /// + """Finetuning a 🤗 Transformers model for sequence classification on GLUE.""" import argparse diff --git a/examples/pytorch/text-classification/run_xnli.py b/examples/pytorch/text-classification/run_xnli.py index f2f7088ac4..f62b0ee248 100755 --- a/examples/pytorch/text-classification/run_xnli.py +++ b/examples/pytorch/text-classification/run_xnli.py @@ -13,6 +13,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +# /// script +# dependencies = [ +# "transformers @ git+https://github.com/huggingface/transformers.git", +# "accelerate >= 0.12.0", +# "datasets >= 1.8.0", +# "sentencepiece != 0.1.92", +# "scipy", +# "scikit-learn", +# "protobuf", +# "torch >= 1.3", +# "evaluate", +# ] +# /// + """Finetuning multi-lingual models on XNLI (e.g. Bert, DistilBERT, XLM). Adapted from `examples/text-classification/run_glue.py`""" diff --git a/examples/pytorch/text-generation/run_generation.py b/examples/pytorch/text-generation/run_generation.py index 4b50fbd07f..cea4c881ea 100755 --- a/examples/pytorch/text-generation/run_generation.py +++ b/examples/pytorch/text-generation/run_generation.py @@ -13,6 +13,17 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +# /// script +# dependencies = [ +# "transformers @ git+https://github.com/huggingface/transformers.git", +# "accelerate >= 0.21.0", +# "sentencepiece != 0.1.92", +# "protobuf", +# "torch >= 1.3", +# ] +# /// + """Conditional text generation with the auto-regressive models of the library (GPT/GPT-2/CTRL/Transformer-XL/XLNet)""" import argparse diff --git a/examples/pytorch/text-generation/run_generation_contrastive_search.py b/examples/pytorch/text-generation/run_generation_contrastive_search.py index 5610dfb7f5..879229c062 100755 --- a/examples/pytorch/text-generation/run_generation_contrastive_search.py +++ b/examples/pytorch/text-generation/run_generation_contrastive_search.py @@ -12,6 +12,17 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +# /// script +# dependencies = [ +# "transformers @ git+https://github.com/huggingface/transformers.git", +# "accelerate >= 0.21.0", +# "sentencepiece != 0.1.92", +# "protobuf", +# "torch >= 1.3", +# ] +# /// + """The examples of running contrastive search on the auto-APIs; Running this example: diff --git a/examples/pytorch/token-classification/run_ner.py b/examples/pytorch/token-classification/run_ner.py index e6b9ee8bbe..609f1f57dd 100755 --- a/examples/pytorch/token-classification/run_ner.py +++ b/examples/pytorch/token-classification/run_ner.py @@ -12,6 +12,18 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +# /// script +# dependencies = [ +# "transformers @ git+https://github.com/huggingface/transformers.git", +# "accelerate >= 0.12.0", +# "seqeval", +# "datasets >= 1.8.0", +# "torch >= 1.3", +# "evaluate", +# ] +# /// + """ Fine-tuning the library models for token classification. """ diff --git a/examples/pytorch/token-classification/run_ner_no_trainer.py b/examples/pytorch/token-classification/run_ner_no_trainer.py index b2eb75c287..8155ab58d2 100755 --- a/examples/pytorch/token-classification/run_ner_no_trainer.py +++ b/examples/pytorch/token-classification/run_ner_no_trainer.py @@ -12,6 +12,18 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +# /// script +# dependencies = [ +# "transformers @ git+https://github.com/huggingface/transformers.git", +# "accelerate >= 0.12.0", +# "seqeval", +# "datasets >= 1.8.0", +# "torch >= 1.3", +# "evaluate", +# ] +# /// + """ Fine-tuning a 🤗 Transformers model on token classification tasks (NER, POS, CHUNKS) relying on the accelerate library without using a Trainer. diff --git a/examples/pytorch/translation/run_translation.py b/examples/pytorch/translation/run_translation.py index eba71deb64..671cda3340 100755 --- a/examples/pytorch/translation/run_translation.py +++ b/examples/pytorch/translation/run_translation.py @@ -12,6 +12,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +# /// script +# dependencies = [ +# "transformers @ git+https://github.com/huggingface/transformers.git", +# "accelerate >= 0.12.0", +# "datasets >= 1.8.0", +# "sentencepiece != 0.1.92", +# "protobuf", +# "sacrebleu >= 1.4.12", +# "py7zr", +# "torch >= 1.3", +# "evaluate", +# ] +# /// + """ Fine-tuning the library models for sequence to sequence. """ diff --git a/examples/pytorch/translation/run_translation_no_trainer.py b/examples/pytorch/translation/run_translation_no_trainer.py index 64fed716ff..2788624e17 100644 --- a/examples/pytorch/translation/run_translation_no_trainer.py +++ b/examples/pytorch/translation/run_translation_no_trainer.py @@ -12,6 +12,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +# /// script +# dependencies = [ +# "transformers @ git+https://github.com/huggingface/transformers.git", +# "accelerate >= 0.12.0", +# "datasets >= 1.8.0", +# "sentencepiece != 0.1.92", +# "protobuf", +# "sacrebleu >= 1.4.12", +# "py7zr", +# "torch >= 1.3", +# "evaluate", +# ] +# /// + """ Fine-tuning a 🤗 Transformers model on text translation. """ diff --git a/utils/release.py b/utils/release.py index e4e79cec15..d8fee77733 100644 --- a/utils/release.py +++ b/utils/release.py @@ -59,6 +59,14 @@ REPLACE_PATTERNS = { "examples": (re.compile(r'^check_min_version\("[^"]+"\)\s*$', re.MULTILINE), 'check_min_version("VERSION")\n'), "init": (re.compile(r'^__version__\s+=\s+"([^"]+)"\s*$', re.MULTILINE), '__version__ = "VERSION"\n'), "setup": (re.compile(r'^(\s*)version\s*=\s*"[^"]+",', re.MULTILINE), r'\1version="VERSION",'), + "uv_script_release": ( + re.compile(r'^# "transformers(\[.+\])?.*$', re.MULTILINE), + r'# "transformers\g<1>==VERSION",', + ), + "uv_script_dev": ( + re.compile(r'^# "transformers(\[.+\])?.*$', re.MULTILINE), + r'# "transformers\g<1> @ git+https://github.com/huggingface/transformers.git",', + ), } # This maps a type of file to its path in Transformers REPLACE_FILES = { @@ -66,6 +74,7 @@ REPLACE_FILES = { "setup": "setup.py", } README_FILE = "README.md" +UV_SCRIPT_MARKER = "# /// script" def update_version_in_file(fname: str, version: str, file_type: str): @@ -86,12 +95,13 @@ def update_version_in_file(fname: str, version: str, file_type: str): f.write(code) -def update_version_in_examples(version: str): +def update_version_in_examples(version: str, patch: bool = False): """ Update the version in all examples files. Args: version (`str`): The new version to set in the examples. + patch (`bool`, *optional*, defaults to `False`): Whether or not this is a patch release. """ for folder, directories, fnames in os.walk(PATH_TO_EXAMPLES): # Removing some of the folders with non-actively maintained examples from the walk @@ -99,7 +109,13 @@ def update_version_in_examples(version: str): directories.remove("legacy") for fname in fnames: if fname.endswith(".py"): - update_version_in_file(os.path.join(folder, fname), version, file_type="examples") + if UV_SCRIPT_MARKER in Path(folder, fname).read_text(): + # Update the depdendencies in UV scripts + uv_script_file_type = "uv_script_dev" if ".dev" in version else "uv_script_release" + update_version_in_file(os.path.join(folder, fname), version, file_type=uv_script_file_type) + if not patch: + # We don't update the version in the examples for patch releases. + update_version_in_file(os.path.join(folder, fname), version, file_type="examples") def global_version_update(version: str, patch: bool = False): @@ -112,9 +128,7 @@ def global_version_update(version: str, patch: bool = False): """ for pattern, fname in REPLACE_FILES.items(): update_version_in_file(fname, version, pattern) - if not patch: - # We don't update the version in the examples for patch releases. - update_version_in_examples(version) + update_version_in_examples(version, patch=patch) def remove_conversion_scripts():