diff --git a/.circleci/config.yml b/.circleci/config.yml index 826057339a..097a6dea80 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -78,7 +78,7 @@ jobs: keys: - v0.4-torch_and_tf-{{ checksum "setup.py" }} - v0.4-{{ checksum "setup.py" }} - - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev + - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng - run: pip install --upgrade pip - run: pip install .[sklearn,tf-cpu,torch,testing,sentencepiece,torch-speech,vision] - run: pip install torch-scatter -f https://pytorch-geometric.com/whl/torch-1.10.0+cpu.html @@ -116,7 +116,7 @@ jobs: keys: - v0.4-torch_and_tf-{{ checksum "setup.py" }} - v0.4-{{ checksum "setup.py" }} - - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev + - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng - run: pip install --upgrade pip - run: pip install .[sklearn,tf-cpu,torch,testing,sentencepiece,torch-speech,vision] - run: pip install torch-scatter -f https://pytorch-geometric.com/whl/torch-1.10.0+cpu.html @@ -149,7 +149,7 @@ jobs: keys: - v0.4-torch_and_flax-{{ checksum "setup.py" }} - v0.4-{{ checksum "setup.py" }} - - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev + - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng - run: pip install --upgrade pip - run: pip install .[sklearn,flax,torch,testing,sentencepiece,torch-speech,vision] - run: pip install torch-scatter -f https://pytorch-geometric.com/whl/torch-1.10.0+cpu.html @@ -186,7 +186,7 @@ jobs: keys: - v0.4-torch_and_flax-{{ checksum "setup.py" }} - v0.4-{{ checksum "setup.py" }} - - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev + - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng - run: pip install --upgrade pip - run: pip install .[sklearn,flax,torch,testing,sentencepiece,torch-speech,vision] - run: pip install torch-scatter -f https://pytorch-geometric.com/whl/torch-1.10.0+cpu.html @@ -217,7 +217,7 @@ jobs: keys: - v0.4-torch-{{ checksum "setup.py" }} - v0.4-{{ checksum "setup.py" }} - - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev + - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng - run: pip install --upgrade pip - run: pip install .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm] - run: pip install torch-scatter -f https://pytorch-geometric.com/whl/torch-1.10.0+cpu.html @@ -253,7 +253,7 @@ jobs: keys: - v0.4-torch-{{ checksum "setup.py" }} - v0.4-{{ checksum "setup.py" }} - - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev + - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng - run: pip install --upgrade pip - run: pip install .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm] - run: pip install torch-scatter -f https://pytorch-geometric.com/whl/torch-1.10.0+cpu.html @@ -284,7 +284,7 @@ jobs: keys: - v0.4-tf-{{ checksum "setup.py" }} - v0.4-{{ checksum "setup.py" }} - - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev + - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng - run: pip install --upgrade pip - run: pip install .[sklearn,tf-cpu,testing,sentencepiece,tf-speech,vision] - run: pip install tensorflow_probability @@ -320,7 +320,7 @@ jobs: keys: - v0.4-tf-{{ checksum "setup.py" }} - v0.4-{{ checksum "setup.py" }} - - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev + - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng - run: pip install --upgrade pip - run: pip install .[sklearn,tf-cpu,testing,sentencepiece,tf-speech,vision] - run: pip install tensorflow_probability @@ -351,7 +351,7 @@ jobs: keys: - v0.4-flax-{{ checksum "setup.py" }} - v0.4-{{ checksum "setup.py" }} - - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev + - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng - run: pip install --upgrade pip - run: pip install .[flax,testing,sentencepiece,flax-speech,vision] - run: pip install https://github.com/kpu/kenlm/archive/master.zip @@ -386,7 +386,7 @@ jobs: keys: - v0.4-flax-{{ checksum "setup.py" }} - v0.4-{{ checksum "setup.py" }} - - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev + - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng - run: pip install --upgrade pip - run: pip install .[flax,testing,sentencepiece,vision,flax-speech] - run: pip install https://github.com/kpu/kenlm/archive/master.zip @@ -417,7 +417,7 @@ jobs: keys: - v0.4-torch-{{ checksum "setup.py" }} - v0.4-{{ checksum "setup.py" }} - - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev + - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng - run: pip install --upgrade pip - run: pip install .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm] - run: pip install torch-scatter -f https://pytorch-geometric.com/whl/torch-1.10.0+cpu.html @@ -454,7 +454,7 @@ jobs: keys: - v0.4-torch-{{ checksum "setup.py" }} - v0.4-{{ checksum "setup.py" }} - - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev + - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng - run: pip install --upgrade pip - run: pip install .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm] - run: pip install torch-scatter -f https://pytorch-geometric.com/whl/torch-1.10.0+cpu.html @@ -579,7 +579,7 @@ jobs: keys: - v0.4-torch_examples-{{ checksum "setup.py" }} - v0.4-{{ checksum "setup.py" }} - - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev + - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng - run: pip install --upgrade pip - run: pip install .[sklearn,torch,sentencepiece,testing,torch-speech] - run: pip install -r examples/pytorch/_tests_requirements.txt @@ -614,7 +614,7 @@ jobs: keys: - v0.4-torch_examples-{{ checksum "setup.py" }} - v0.4-{{ checksum "setup.py" }} - - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev + - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng - run: pip install --upgrade pip - run: pip install .[sklearn,torch,sentencepiece,testing,torch-speech] - run: pip install -r examples/pytorch/_tests_requirements.txt diff --git a/.github/workflows/self-push.yml b/.github/workflows/self-push.yml index b997f1db08..d0072ff483 100644 --- a/.github/workflows/self-push.yml +++ b/.github/workflows/self-push.yml @@ -31,7 +31,7 @@ jobs: - name: Install dependencies run: | apt -y update && apt install -y software-properties-common && apt -y update && add-apt-repository -y ppa:git-core/ppa && apt -y update && apt install -y git - apt install -y libsndfile1-dev + apt install -y libsndfile1-dev espeak-ng pip install --upgrade pip pip install .[sklearn,testing,onnxruntime,sentencepiece,torch-speech,vision,timm] pip install https://github.com/kpu/kenlm/archive/master.zip @@ -84,7 +84,7 @@ jobs: steps: - name: Install dependencies run: | - apt -y update && apt install -y software-properties-common && apt -y update && add-apt-repository -y ppa:git-core/ppa && apt -y update && apt install -y git + apt -y update && apt install -y software-properties-common && apt -y update && add-apt-repository -y ppa:git-core/ppa && apt -y update && apt install -y git espeak-ng pip install --upgrade "jax[cuda111]" -f https://storage.googleapis.com/jax-releases/jax_releases.html pip install --upgrade pip pip install .[sklearn,testing,sentencepiece,flax,flax-speech,vision] @@ -141,7 +141,7 @@ jobs: # steps: # - name: Install dependencies # run: | -# apt -y update && apt install -y software-properties-common && apt -y update && add-apt-repository -y ppa:git-core/ppa && apt -y update && apt install -y git +# apt -y update && apt install -y software-properties-common && apt -y update && add-apt-repository -y ppa:git-core/ppa && apt -y update && apt install -y git espeak-ng # pip install --upgrade pip # pip install .[sklearn,testing,onnxruntime,sentencepiece,tf-speech] # pip install https://github.com/kpu/kenlm/archive/master.zip @@ -199,8 +199,8 @@ jobs: steps: - name: Install dependencies run: | - apt -y update && apt install -y software-properties-common && apt -y update && add-apt-repository -y ppa:git-core/ppa && apt -y update && apt install -y git - apt install -y libsndfile1-dev + apt -y update && apt install -y software-properties-common && apt -y update && add-apt-repository -y ppa:git-core/ppa && apt -y update && apt install -y git espeak-ng + apt install -y libsndfile1-dev espeak-ng pip install --upgrade pip pip install .[sklearn,testing,onnxruntime,sentencepiece,torch-speech,vision,timm] pip install https://github.com/kpu/kenlm/archive/master.zip @@ -255,7 +255,7 @@ jobs: # steps: # - name: Install dependencies # run: | -# apt -y update && apt install -y software-properties-common && apt -y update && add-apt-repository -y ppa:git-core/ppa && apt -y update && apt install -y git +# apt -y update && apt install -y software-properties-common && apt -y update && add-apt-repository -y ppa:git-core/ppa && apt -y update && apt install -y git espeak-ng # pip install --upgrade "jax[cuda111]" -f https://storage.googleapis.com/jax-releases/jax_releases.html # pip install --upgrade pip # pip install .[sklearn,testing,sentencepiece,flax,flax-speech,vision] @@ -312,7 +312,7 @@ jobs: # steps: # - name: Install dependencies # run: | -# apt -y update && apt install -y software-properties-common && apt -y update && add-apt-repository -y ppa:git-core/ppa && apt -y update && apt install -y git +# apt -y update && apt install -y software-properties-common && apt -y update && add-apt-repository -y ppa:git-core/ppa && apt -y update && apt install -y git espeak-ng # pip install --upgrade pip # pip install .[sklearn,testing,onnxruntime,sentencepiece,tf-speech] # pip install https://github.com/kpu/kenlm/archive/master.zip diff --git a/.github/workflows/self-scheduled.yml b/.github/workflows/self-scheduled.yml index 72a4b8a3d3..d9be815128 100644 --- a/.github/workflows/self-scheduled.yml +++ b/.github/workflows/self-scheduled.yml @@ -33,7 +33,7 @@ jobs: - name: Install dependencies run: | - apt -y update && apt install -y libsndfile1-dev git + apt -y update && apt install -y libsndfile1-dev git espeak-ng pip install --upgrade pip pip install .[integrations,sklearn,testing,onnxruntime,sentencepiece,torch-speech,vision,timm] pip install https://github.com/kpu/kenlm/archive/master.zip @@ -140,7 +140,7 @@ jobs: - name: Install dependencies run: | - apt -y update && apt install -y libsndfile1-dev git + apt -y update && apt install -y libsndfile1-dev git espeak-ng pip install --upgrade pip pip install .[sklearn,testing,onnx,sentencepiece,tf-speech,vision] pip install https://github.com/kpu/kenlm/archive/master.zip @@ -237,7 +237,7 @@ jobs: - name: Install dependencies run: | - apt -y update && apt install -y libsndfile1-dev git + apt -y update && apt install -y libsndfile1-dev git espeak-ng pip install --upgrade pip pip install .[integrations,sklearn,testing,onnxruntime,sentencepiece,torch-speech,vision,timm] pip install https://github.com/kpu/kenlm/archive/master.zip diff --git a/README.md b/README.md index 620d2d5796..13a2b31a5c 100644 --- a/README.md +++ b/README.md @@ -313,11 +313,13 @@ AWARE PRE-TRAINING](https://arxiv.org/abs/2110.05752) by Sanyuan Chen, Yu Wu, Ch 1. **[VisualBERT](https://huggingface.co/docs/transformers/model_doc/visual_bert)** (from UCLA NLP) released with the paper [VisualBERT: A Simple and Performant Baseline for Vision and Language](https://arxiv.org/pdf/1908.03557) by Liunian Harold Li, Mark Yatskar, Da Yin, Cho-Jui Hsieh, Kai-Wei Chang. 1. **[WavLM](https://huggingface.co/docs/transformers/master/model_doc/wavlm)** (from Microsoft Research) released with the paper [WavLM: Large-Scale Self-Supervised Pre-Training for Full Stack Speech Processing](https://arxiv.org/abs/2110.13900) by Sanyuan Chen, Chengyi Wang, Zhengyang Chen, Yu Wu, Shujie Liu, Zhuo Chen, Jinyu Li, Naoyuki Kanda, Takuya Yoshioka, Xiong Xiao, Jian Wu, Long Zhou, Shuo Ren, Yanmin Qian, Yao Qian, Jian Wu, Michael Zeng, Furu Wei. 1. **[Wav2Vec2](https://huggingface.co/docs/transformers/model_doc/wav2vec2)** (from Facebook AI) released with the paper [wav2vec 2.0: A Framework for Self-Supervised Learning of Speech Representations](https://arxiv.org/abs/2006.11477) by Alexei Baevski, Henry Zhou, Abdelrahman Mohamed, Michael Auli. +1. **[Wav2Vec2Phoneme](https://huggingface.co/docs/master/transformers/model_doc/wav2vec2_phoneme)** (from Facebook AI) released with the paper [Simple and Effective Zero-shot Cross-lingual Phoneme Recognition](https://arxiv.org/abs/2109.11680) by Qiantong Xu, Alexei Baevski, Michael Auli. 1. **[XLM](https://huggingface.co/docs/transformers/model_doc/xlm)** (from Facebook) released together with the paper [Cross-lingual Language Model Pretraining](https://arxiv.org/abs/1901.07291) by Guillaume Lample and Alexis Conneau. 1. **[XLM-ProphetNet](https://huggingface.co/docs/transformers/model_doc/xlmprophetnet)** (from Microsoft Research) released with the paper [ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training](https://arxiv.org/abs/2001.04063) by Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou. 1. **[XLM-RoBERTa](https://huggingface.co/docs/transformers/model_doc/xlmroberta)** (from Facebook AI), released together with the paper [Unsupervised Cross-lingual Representation Learning at Scale](https://arxiv.org/abs/1911.02116) by Alexis Conneau*, Kartikay Khandelwal*, Naman Goyal, Vishrav Chaudhary, Guillaume Wenzek, Francisco Guzmán, Edouard Grave, Myle Ott, Luke Zettlemoyer and Veselin Stoyanov. 1. **[XLNet](https://huggingface.co/docs/transformers/model_doc/xlnet)** (from Google/CMU) released with the paper [​XLNet: Generalized Autoregressive Pretraining for Language Understanding](https://arxiv.org/abs/1906.08237) by Zhilin Yang*, Zihang Dai*, Yiming Yang, Jaime Carbonell, Ruslan Salakhutdinov, Quoc V. Le. 1. **[XLSR-Wav2Vec2](https://huggingface.co/docs/transformers/model_doc/xlsr_wav2vec2)** (from Facebook AI) released with the paper [Unsupervised Cross-Lingual Representation Learning For Speech Recognition](https://arxiv.org/abs/2006.13979) by Alexis Conneau, Alexei Baevski, Ronan Collobert, Abdelrahman Mohamed, Michael Auli. +1. **[XLS-R](https://huggingface.co/docs/master/transformers/model_doc/xls_r)** (from Facebook AI) released with the paper [XLS-R: Self-supervised Cross-lingual Speech Representation Learning at Scale](https://arxiv.org/abs/2111.09296) by Arun Babu, Changhan Wang, Andros Tjandra, Kushal Lakhotia, Qiantong Xu, Naman Goyal, Kritika Singh, Patrick von Platen, Yatharth Saraf, Juan Pino, Alexei Baevski, Alexis Conneau, Michael Auli. 1. Want to contribute a new model? We have added a **detailed guide and templates** to guide you in the process of adding a new model. You can find them in the [`templates`](./templates) folder of the repository. Be sure to check the [contributing guidelines](./CONTRIBUTING.md) and contact the maintainers or open an issue to collect feedbacks before starting your PR. To check if each model has an implementation in Flax, PyTorch or TensorFlow, or has an associated tokenizer backed by the 🤗 Tokenizers library, refer to [this table](https://huggingface.co/docs/transformers/index#supported-frameworks). diff --git a/README_ko.md b/README_ko.md index dc888dcd85..5d001c6c11 100644 --- a/README_ko.md +++ b/README_ko.md @@ -290,11 +290,13 @@ Flax, PyTorch, TensorFlow 설치 페이지에서 이들을 conda로 설치하는 1. **[Vision Transformer (ViT)](https://huggingface.co/docs/transformers/model_doc/vit)** (from Google AI) released with the paper [An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale](https://arxiv.org/abs/2010.11929) by Alexey Dosovitskiy, Lucas Beyer, Alexander Kolesnikov, Dirk Weissenborn, Xiaohua Zhai, Thomas Unterthiner, Mostafa Dehghani, Matthias Minderer, Georg Heigold, Sylvain Gelly, Jakob Uszkoreit, Neil Houlsby. 1. **[VisualBERT](https://huggingface.co/docs/transformers/model_doc/visual_bert)** (from UCLA NLP) released with the paper [VisualBERT: A Simple and Performant Baseline for Vision and Language](https://arxiv.org/pdf/1908.03557) by Liunian Harold Li, Mark Yatskar, Da Yin, Cho-Jui Hsieh, Kai-Wei Chang. 1. **[Wav2Vec2](https://huggingface.co/docs/transformers/model_doc/wav2vec2)** (from Facebook AI) released with the paper [wav2vec 2.0: A Framework for Self-Supervised Learning of Speech Representations](https://arxiv.org/abs/2006.11477) by Alexei Baevski, Henry Zhou, Abdelrahman Mohamed, Michael Auli. +1. **[Wav2Vec2Phoneme](https://huggingface.co/docs/master/transformers/model_doc/wav2vec2_phoneme)** (from Facebook AI) released with the paper [Simple and Effective Zero-shot Cross-lingual Phoneme Recognition](https://arxiv.org/abs/2109.11680) by Qiantong Xu, Alexei Baevski, Michael Auli. 1. **[WavLM](https://huggingface.co/docs/transformers/master/model_doc/wavlm)** (from Microsoft Research) released with the paper [WavLM: Large-Scale Self-Supervised Pre-Training for Full Stack Speech Processing](https://arxiv.org/abs/2110.13900) by Sanyuan Chen, Chengyi Wang, Zhengyang Chen, Yu Wu, Shujie Liu, Zhuo Chen, Jinyu Li, Naoyuki Kanda, Takuya Yoshioka, Xiong Xiao, Jian Wu, Long Zhou, Shuo Ren, Yanmin Qian, Yao Qian, Jian Wu, Michael Zeng, Furu Wei. 1. **[XLM](https://huggingface.co/docs/transformers/model_doc/xlm)** (from Facebook) released together with the paper [Cross-lingual Language Model Pretraining](https://arxiv.org/abs/1901.07291) by Guillaume Lample and Alexis Conneau. 1. **[XLM-ProphetNet](https://huggingface.co/docs/transformers/model_doc/xlmprophetnet)** (from Microsoft Research) released with the paper [ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training](https://arxiv.org/abs/2001.04063) by Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou. 1. **[XLM-RoBERTa](https://huggingface.co/docs/transformers/model_doc/xlmroberta)** (from Facebook AI), released together with the paper [Unsupervised Cross-lingual Representation Learning at Scale](https://arxiv.org/abs/1911.02116) by Alexis Conneau*, Kartikay Khandelwal*, Naman Goyal, Vishrav Chaudhary, Guillaume Wenzek, Francisco Guzmán, Edouard Grave, Myle Ott, Luke Zettlemoyer and Veselin Stoyanov. 1. **[XLNet](https://huggingface.co/docs/transformers/model_doc/xlnet)** (from Google/CMU) released with the paper [​XLNet: Generalized Autoregressive Pretraining for Language Understanding](https://arxiv.org/abs/1906.08237) by Zhilin Yang*, Zihang Dai*, Yiming Yang, Jaime Carbonell, Ruslan Salakhutdinov, Quoc V. Le. +1. **[XLS-R](https://huggingface.co/docs/master/transformers/model_doc/xls_r)** (from Facebook AI) released with the paper [XLS-R: Self-supervised Cross-lingual Speech Representation Learning at Scale](https://arxiv.org/abs/2111.09296) by Arun Babu, Changhan Wang, Andros Tjandra, Kushal Lakhotia, Qiantong Xu, Naman Goyal, Kritika Singh, Patrick von Platen, Yatharth Saraf, Juan Pino, Alexei Baevski, Alexis Conneau, Michael Auli. 1. **[XLSR-Wav2Vec2](https://huggingface.co/docs/transformers/model_doc/xlsr_wav2vec2)** (from Facebook AI) released with the paper [Unsupervised Cross-Lingual Representation Learning For Speech Recognition](https://arxiv.org/abs/2006.13979) by Alexis Conneau, Alexei Baevski, Ronan Collobert, Abdelrahman Mohamed, Michael Auli. 1. 새로운 모델을 올리고 싶나요? 우리가 **상세한 가이드와 템플릿** 으로 새로운 모델을 올리도록 도와드릴게요. 가이드와 템플릿은 이 저장소의 [`templates`](./templates) 폴더에서 확인하실 수 있습니다. [컨트리뷰션 가이드라인](./CONTRIBUTING.md)을 꼭 확인해주시고, PR을 올리기 전에 메인테이너에게 연락하거나 이슈를 오픈해 피드백을 받으시길 바랍니다. diff --git a/README_zh-hans.md b/README_zh-hans.md index 9ecd8c1a64..ad4dd26f47 100644 --- a/README_zh-hans.md +++ b/README_zh-hans.md @@ -314,11 +314,13 @@ conda install -c huggingface transformers 1. **[Vision Transformer (ViT)](https://huggingface.co/docs/transformers/model_doc/vit)** (来自 Google AI) 伴随论文 [An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale](https://arxiv.org/abs/2010.11929) 由 Alexey Dosovitskiy, Lucas Beyer, Alexander Kolesnikov, Dirk Weissenborn, Xiaohua Zhai, Thomas Unterthiner, Mostafa Dehghani, Matthias Minderer, Georg Heigold, Sylvain Gelly, Jakob Uszkoreit, Neil Houlsby 发布。 1. **[VisualBERT](https://huggingface.co/docs/transformers/model_doc/visual_bert)** (来自 UCLA NLP) 伴随论文 [VisualBERT: A Simple and Performant Baseline for Vision and Language](https://arxiv.org/pdf/1908.03557) 由 Liunian Harold Li, Mark Yatskar, Da Yin, Cho-Jui Hsieh, Kai-Wei Chang 发布。 1. **[Wav2Vec2](https://huggingface.co/docs/transformers/model_doc/wav2vec2)** (来自 Facebook AI) 伴随论文 [wav2vec 2.0: A Framework for Self-Supervised Learning of Speech Representations](https://arxiv.org/abs/2006.11477) 由 Alexei Baevski, Henry Zhou, Abdelrahman Mohamed, Michael Auli 发布。 +1. **[Wav2Vec2Phoneme](https://huggingface.co/docs/master/transformers/model_doc/wav2vec2_phoneme)** (来自 Facebook AI) 伴随论文 [Simple and Effective Zero-shot Cross-lingual Phoneme Recognition](https://arxiv.org/abs/2109.11680) 由 Qiantong Xu, Alexei Baevski, Michael Auli 发布。 1. **[WavLM](https://huggingface.co/docs/transformers/master/model_doc/wavlm)** (from Microsoft Research) released with the paper [WavLM: Large-Scale Self-Supervised Pre-Training for Full Stack Speech Processing](https://arxiv.org/abs/2110.13900) by Sanyuan Chen, Chengyi Wang, Zhengyang Chen, Yu Wu, Shujie Liu, Zhuo Chen, Jinyu Li, Naoyuki Kanda, Takuya Yoshioka, Xiong Xiao, Jian Wu, Long Zhou, Shuo Ren, Yanmin Qian, Yao Qian, Jian Wu, Michael Zeng, Furu Wei. 1. **[XLM](https://huggingface.co/docs/transformers/model_doc/xlm)** (来自 Facebook) 伴随论文 [Cross-lingual Language Model Pretraining](https://arxiv.org/abs/1901.07291) 由 Guillaume Lample and Alexis Conneau 发布。 1. **[XLM-ProphetNet](https://huggingface.co/docs/transformers/model_doc/xlmprophetnet)** (来自 Microsoft Research) 伴随论文 [ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training](https://arxiv.org/abs/2001.04063) 由 Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou 发布。 1. **[XLM-RoBERTa](https://huggingface.co/docs/transformers/model_doc/xlmroberta)** (来自 Facebook AI), 伴随论文 [Unsupervised Cross-lingual Representation Learning at Scale](https://arxiv.org/abs/1911.02116) 由 Alexis Conneau*, Kartikay Khandelwal*, Naman Goyal, Vishrav Chaudhary, Guillaume Wenzek, Francisco Guzmán, Edouard Grave, Myle Ott, Luke Zettlemoyer and Veselin Stoyanov 发布。 1. **[XLNet](https://huggingface.co/docs/transformers/model_doc/xlnet)** (来自 Google/CMU) 伴随论文 [XLNet: Generalized Autoregressive Pretraining for Language Understanding](https://arxiv.org/abs/1906.08237) 由 Zhilin Yang*, Zihang Dai*, Yiming Yang, Jaime Carbonell, Ruslan Salakhutdinov, Quoc V. Le 发布。 +1. **[XLS-R](https://huggingface.co/docs/master/transformers/model_doc/xls_r)** (来自 Facebook AI) 伴随论文 [XLS-R: Self-supervised Cross-lingual Speech Representation Learning at Scale](https://arxiv.org/abs/2111.09296) 由 Arun Babu, Changhan Wang, Andros Tjandra, Kushal Lakhotia, Qiantong Xu, Naman Goyal, Kritika Singh, Patrick von Platen, Yatharth Saraf, Juan Pino, Alexei Baevski, Alexis Conneau, Michael Auli 发布。 1. **[XLSR-Wav2Vec2](https://huggingface.co/docs/transformers/model_doc/xlsr_wav2vec2)** (来自 Facebook AI) 伴随论文 [Unsupervised Cross-Lingual Representation Learning For Speech Recognition](https://arxiv.org/abs/2006.13979) 由 Alexis Conneau, Alexei Baevski, Ronan Collobert, Abdelrahman Mohamed, Michael Auli 发布。 1. 想要贡献新的模型？我们这里有一份**详细指引和模板**来引导你添加新的模型。你可以在 [`templates`](./templates) 目录中找到他们。记得查看 [贡献指南](./CONTRIBUTING.md) 并在开始写 PR 前联系维护人员或开一个新的 issue 来获得反馈。 diff --git a/README_zh-hant.md b/README_zh-hant.md index 6ae9509cfa..99bb5e63dd 100644 --- a/README_zh-hant.md +++ b/README_zh-hant.md @@ -326,11 +326,13 @@ conda install -c huggingface transformers 1. **[Vision Transformer (ViT)](https://huggingface.co/docs/transformers/model_doc/vit)** (from Google AI) released with the paper [An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale](https://arxiv.org/abs/2010.11929) by Alexey Dosovitskiy, Lucas Beyer, Alexander Kolesnikov, Dirk Weissenborn, Xiaohua Zhai, Thomas Unterthiner, Mostafa Dehghani, Matthias Minderer, Georg Heigold, Sylvain Gelly, Jakob Uszkoreit, Neil Houlsby. 1. **[VisualBERT](https://huggingface.co/docs/transformers/model_doc/visual_bert)** (from UCLA NLP) released with the paper [VisualBERT: A Simple and Performant Baseline for Vision and Language](https://arxiv.org/pdf/1908.03557) by Liunian Harold Li, Mark Yatskar, Da Yin, Cho-Jui Hsieh, Kai-Wei Chang. 1. **[Wav2Vec2](https://huggingface.co/docs/transformers/model_doc/wav2vec2)** (from Facebook AI) released with the paper [wav2vec 2.0: A Framework for Self-Supervised Learning of Speech Representations](https://arxiv.org/abs/2006.11477) by Alexei Baevski, Henry Zhou, Abdelrahman Mohamed, Michael Auli. +1. **[Wav2Vec2Phoneme](https://huggingface.co/docs/master/transformers/model_doc/wav2vec2_phoneme)** (from Facebook AI) released with the paper [Simple and Effective Zero-shot Cross-lingual Phoneme Recognition](https://arxiv.org/abs/2109.11680) by Qiantong Xu, Alexei Baevski, Michael Auli. 1. **[WavLM](https://huggingface.co/docs/transformers/master/model_doc/wavlm)** (from Microsoft Research) released with the paper [WavLM: Large-Scale Self-Supervised Pre-Training for Full Stack Speech Processing](https://arxiv.org/abs/2110.13900) by Sanyuan Chen, Chengyi Wang, Zhengyang Chen, Yu Wu, Shujie Liu, Zhuo Chen, Jinyu Li, Naoyuki Kanda, Takuya Yoshioka, Xiong Xiao, Jian Wu, Long Zhou, Shuo Ren, Yanmin Qian, Yao Qian, Jian Wu, Michael Zeng, Furu Wei. 1. **[XLM](https://huggingface.co/docs/transformers/model_doc/xlm)** (from Facebook) released together with the paper [Cross-lingual Language Model Pretraining](https://arxiv.org/abs/1901.07291) by Guillaume Lample and Alexis Conneau. 1. **[XLM-ProphetNet](https://huggingface.co/docs/transformers/model_doc/xlmprophetnet)** (from Microsoft Research) released with the paper [ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training](https://arxiv.org/abs/2001.04063) by Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou. 1. **[XLM-RoBERTa](https://huggingface.co/docs/transformers/model_doc/xlmroberta)** (from Facebook AI), released together with the paper [Unsupervised Cross-lingual Representation Learning at Scale](https://arxiv.org/abs/1911.02116) by Alexis Conneau*, Kartikay Khandelwal*, Naman Goyal, Vishrav Chaudhary, Guillaume Wenzek, Francisco Guzmán, Edouard Grave, Myle Ott, Luke Zettlemoyer and Veselin Stoyanov. 1. **[XLNet](https://huggingface.co/docs/transformers/model_doc/xlnet)** (from Google/CMU) released with the paper [​XLNet: Generalized Autoregressive Pretraining for Language Understanding](https://arxiv.org/abs/1906.08237) by Zhilin Yang*, Zihang Dai*, Yiming Yang, Jaime Carbonell, Ruslan Salakhutdinov, Quoc V. Le. +1. **[XLS-R](https://huggingface.co/docs/master/transformers/model_doc/xls_r)** (from Facebook AI) released with the paper [XLS-R: Self-supervised Cross-lingual Speech Representation Learning at Scale](https://arxiv.org/abs/2111.09296) by Arun Babu, Changhan Wang, Andros Tjandra, Kushal Lakhotia, Qiantong Xu, Naman Goyal, Kritika Singh, Patrick von Platen, Yatharth Saraf, Juan Pino, Alexei Baevski, Alexis Conneau, Michael Auli. 1. **[XLSR-Wav2Vec2](https://huggingface.co/docs/transformers/model_doc/xlsr_wav2vec2)** (from Facebook AI) released with the paper [Unsupervised Cross-Lingual Representation Learning For Speech Recognition](https://arxiv.org/abs/2006.13979) by Alexis Conneau, Alexei Baevski, Ronan Collobert, Abdelrahman Mohamed, Michael Auli. 1. 想要貢獻新的模型？我們這裡有一份**詳細指引和模板**來引導你加入新的模型。你可以在 [`templates`](./templates) 目錄中找到它們。記得查看[貢獻指引](./CONTRIBUTING.md)並在開始寫 PR 前聯繫維護人員或開一個新的 issue 來獲得 feedbacks。 diff --git a/docs/source/_toctree.yml b/docs/source/_toctree.yml index 2afd514ea2..32113dc5f4 100644 --- a/docs/source/_toctree.yml +++ b/docs/source/_toctree.yml @@ -284,6 +284,8 @@ title: VisualBERT - local: model_doc/wav2vec2 title: Wav2Vec2 + - local: model_doc/wav2vec2_phoneme + title: Wav2Vec2Phoneme - local: model_doc/wavlm title: WavLM - local: model_doc/xlm @@ -296,6 +298,8 @@ title: XLNet - local: model_doc/xlsr_wav2vec2 title: XLSR-Wav2Vec2 + - local: model_doc/xls_r + title: XLS-R title: Models - sections: - local: internal/modeling_utils diff --git a/docs/source/index.mdx b/docs/source/index.mdx index 444ba3e840..1609228aa0 100644 --- a/docs/source/index.mdx +++ b/docs/source/index.mdx @@ -172,11 +172,13 @@ conversion utilities for the following models. 1. **[VisualBERT](model_doc/visual_bert)** (from UCLA NLP) released with the paper [VisualBERT: A Simple and Performant Baseline for Vision and Language](https://arxiv.org/pdf/1908.03557) by Liunian Harold Li, Mark Yatskar, Da Yin, Cho-Jui Hsieh, Kai-Wei Chang. 1. **[WavLM](model_doc/wavlm)** (from Microsoft Research) released with the paper [WavLM: Large-Scale Self-Supervised Pre-Training for Full Stack Speech Processing](https://arxiv.org/abs/2110.13900) by Sanyuan Chen, Chengyi Wang, Zhengyang Chen, Yu Wu, Shujie Liu, Zhuo Chen, Jinyu Li, Naoyuki Kanda, Takuya Yoshioka, Xiong Xiao, Jian Wu, Long Zhou, Shuo Ren, Yanmin Qian, Yao Qian, Jian Wu, Michael Zeng, Furu Wei. 1. **[Wav2Vec2](model_doc/wav2vec2)** (from Facebook AI) released with the paper [wav2vec 2.0: A Framework for Self-Supervised Learning of Speech Representations](https://arxiv.org/abs/2006.11477) by Alexei Baevski, Henry Zhou, Abdelrahman Mohamed, Michael Auli. +1. **[Wav2Vec2Phoneme](https://huggingface.co/docs/master/transformers/model_doc/wav2vec2_phoneme)** (from Facebook AI) released with the paper [Simple and Effective Zero-shot Cross-lingual Phoneme Recognition](https://arxiv.org/abs/2109.11680) by Qiantong Xu, Alexei Baevski, Michael Auli. 1. **[XLM](model_doc/xlm)** (from Facebook) released together with the paper [Cross-lingual Language Model Pretraining](https://arxiv.org/abs/1901.07291) by Guillaume Lample and Alexis Conneau. 1. **[XLM-ProphetNet](model_doc/xlmprophetnet)** (from Microsoft Research) released with the paper [ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training](https://arxiv.org/abs/2001.04063) by Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou. 1. **[XLM-RoBERTa](model_doc/xlmroberta)** (from Facebook AI), released together with the paper [Unsupervised Cross-lingual Representation Learning at Scale](https://arxiv.org/abs/1911.02116) by Alexis Conneau*, Kartikay Khandelwal*, Naman Goyal, Vishrav Chaudhary, Guillaume Wenzek, Francisco Guzmán, Edouard Grave, Myle Ott, Luke Zettlemoyer and Veselin Stoyanov. 1. **[XLNet](model_doc/xlnet)** (from Google/CMU) released with the paper [​XLNet: Generalized Autoregressive Pretraining for Language Understanding](https://arxiv.org/abs/1906.08237) by Zhilin Yang*, Zihang Dai*, Yiming Yang, Jaime Carbonell, Ruslan Salakhutdinov, Quoc V. Le. 1. **[XLSR-Wav2Vec2](model_doc/xlsr_wav2vec2)** (from Facebook AI) released with the paper [Unsupervised Cross-Lingual Representation Learning For Speech Recognition](https://arxiv.org/abs/2006.13979) by Alexis Conneau, Alexei Baevski, Ronan Collobert, Abdelrahman Mohamed, Michael Auli. +1. **[XLS-R](https://huggingface.co/docs/master/transformers/model_doc/xls_r)** (from Facebook AI) released with the paper [XLS-R: Self-supervised Cross-lingual Speech Representation Learning at Scale](https://arxiv.org/abs/2111.09296) by Arun Babu, Changhan Wang, Andros Tjandra, Kushal Lakhotia, Qiantong Xu, Naman Goyal, Kritika Singh, Patrick von Platen, Yatharth Saraf, Juan Pino, Alexei Baevski, Alexis Conneau, Michael Auli. ### Supported frameworks diff --git a/docs/source/model_doc/wav2vec2_phoneme.mdx b/docs/source/model_doc/wav2vec2_phoneme.mdx new file mode 100644 index 0000000000..b39cf66ce1 --- /dev/null +++ b/docs/source/model_doc/wav2vec2_phoneme.mdx @@ -0,0 +1,56 @@ + + +# Wav2Vec2Phoneme + +## Overview + +The Wav2Vec2Phoneme model was proposed in [Simple and Effective Zero-shot Cross-lingual Phoneme Recognition (Xu et al., +2021](https://arxiv.org/abs/2109.11680) by Qiantong Xu, Alexei Baevski, Michael Auli. + +The abstract from the paper is the following: + +*Recent progress in self-training, self-supervised pretraining and unsupervised learning enabled well performing speech +recognition systems without any labeled data. However, in many cases there is labeled data available for related +languages which is not utilized by these methods. This paper extends previous work on zero-shot cross-lingual transfer +learning by fine-tuning a multilingually pretrained wav2vec 2.0 model to transcribe unseen languages. This is done by +mapping phonemes of the training languages to the target language using articulatory features. Experiments show that +this simple method significantly outperforms prior work which introduced task-specific architectures and used only part +of a monolingually pretrained model.* + +Tips: + +- Wav2Vec2Phoneme uses the exact same architecture as Wav2Vec2 +- Wav2Vec2Phoneme is a speech model that accepts a float array corresponding to the raw waveform of the speech signal. +- Wav2Vec2Phoneme model was trained using connectionist temporal classification (CTC) so the model output has to be + decoded using [`Wav2Vec2PhonemeCTCTokenizer`]. +- Wav2Vec2Phoneme can be fine-tuned on multiple language at once and decode unseen languages in a single forward pass + to a sequence of phonemes +- By default the model outputs a sequence of phonemes. In order to transform the phonemes to a sequence of words one + should make use of a dictionary and language model. + +Relevant checkpoints can be found under https://huggingface.co/models?other=phoneme-recognition. + +This model was contributed by [patrickvonplaten](https://huggingface.co/patrickvonplaten) + +The original code can be found [here](https://github.com/pytorch/fairseq/tree/master/fairseq/models/wav2vec). + +Wav2Vec2Phoneme's architecture is based on the Wav2Vec2 model, so one can refer to [`Wav2Vec2`]'s documentation page except for the tokenizer. + + +## Wav2Vec2PhonemeCTCTokenizer + +[[autodoc]] Wav2Vec2PhonemeCTCTokenizer + - __call__ + - batch_decode + - decode + - phonemize diff --git a/docs/source/model_doc/xls_r.rst b/docs/source/model_doc/xls_r.rst new file mode 100644 index 0000000000..889243541a --- /dev/null +++ b/docs/source/model_doc/xls_r.rst @@ -0,0 +1,47 @@ +.. + Copyright 2021 The HuggingFace Team. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + specific language governing permissions and limitations under the License. + +XLS-R +----------------------------------------------------------------------------------------------------------------------- + +Overview +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The XLS-R model was proposed in `XLS-R: Self-supervised Cross-lingual Speech Representation Learning at Scale +`__ by Arun Babu, Changhan Wang, Andros Tjandra, Kushal Lakhotia, Qiantong Xu, Naman +Goyal, Kritika Singh, Patrick von Platen, Yatharth Saraf, Juan Pino, Alexei Baevski, Alexis Conneau, Michael Auli. + +The abstract from the paper is the following: + +*This paper presents XLS-R, a large-scale model for cross-lingual speech representation learning based on wav2vec 2.0. +We train models with up to 2B parameters on nearly half a million hours of publicly available speech audio in 128 +languages, an order of magnitude more public data than the largest known prior work. Our evaluation covers a wide range +of tasks, domains, data regimes and languages, both high and low-resource. On the CoVoST-2 speech translation +benchmark, we improve the previous state of the art by an average of 7.4 BLEU over 21 translation directions into +English. For speech recognition, XLS-R improves over the best known prior work on BABEL, MLS, CommonVoice as well as +VoxPopuli, lowering error rates by 14-34% relative on average. XLS-R also sets a new state of the art on VoxLingua107 +language identification. Moreover, we show that with sufficient model size, cross-lingual pretraining can outperform +English-only pretraining when translating English speech into other languages, a setting which favors monolingual +pretraining. We hope XLS-R can help to improve speech processing tasks for many more languages of the world.* + +Tips: + +- XLS-R is a speech model that accepts a float array corresponding to the raw waveform of the speech signal. +- XLS-R model was trained using connectionist temporal classification (CTC) so the model output has to be decoded using + :class:`~transformers.Wav2Vec2CTCTokenizer`. + +Relevant checkpoints can be found under https://huggingface.co/models?other=xls_r. + +XLS-R's architecture is based on the Wav2Vec2 model, so one can refer to :doc:`Wav2Vec2's documentation page +`. + +The original code can be found `here `__. diff --git a/examples/pytorch/speech-recognition/run_speech_recognition_ctc.py b/examples/pytorch/speech-recognition/run_speech_recognition_ctc.py index 3a7747e791..1329e16fc3 100755 --- a/examples/pytorch/speech-recognition/run_speech_recognition_ctc.py +++ b/examples/pytorch/speech-recognition/run_speech_recognition_ctc.py @@ -21,6 +21,7 @@ import logging import os import re import sys +import warnings from dataclasses import dataclass, field from typing import Dict, List, Optional, Union @@ -34,6 +35,7 @@ from transformers import ( AutoConfig, AutoFeatureExtractor, AutoModelForCTC, + AutoProcessor, AutoTokenizer, HfArgumentParser, Trainer, @@ -68,6 +70,10 @@ class ModelArguments: model_name_or_path: str = field( metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"} ) + tokenizer_name_or_path: Optional[str] = field( + default=None, + metadata={"help": "Path to pretrained tokenizer or tokenizer identifier from huggingface.co/models"}, + ) cache_dir: Optional[str] = field( default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"}, @@ -191,7 +197,7 @@ class DataTrainingArguments: max_duration_in_seconds: Optional[float] = field( default=20.0, metadata={ - "help": "Truncate audio files that are longer than `max_duration_in_seconds` seconds to 'max_duration_in_seconds`" + "help": "Filter audio files that are longer than `max_duration_in_seconds` seconds to 'max_duration_in_seconds`" }, ) min_duration_in_seconds: Optional[float] = field( @@ -210,7 +216,28 @@ class DataTrainingArguments: default=False, metadata={ "help": "If :obj:`True`, will use the token generated when running" - ":obj:`transformers-cli logiin as HTTP bearer authorization for remote files." + ":obj:`transformers-cli login` as HTTP bearer authorization for remote files." + }, + ) + unk_token: Optional[str] = field( + default="[UNK]", + metadata={"help": "The unk token for the tokenizer"}, + ) + pad_token: Optional[str] = field( + default="[PAD]", + metadata={"help": "The padding token for the tokenizer"}, + ) + word_delimiter_token: Optional[str] = field( + default="|", + metadata={"help": "The word delimiter token for the tokenizer"}, + ) + phoneme_language: Optional[str] = field( + default=None, + metadata={ + "help": "The target language that should be used be" + " passed to the tokenizer for tokenization. Note that" + " this is only relevant if the model classifies the" + " input audio to a sequence of phoneme sequences." }, ) @@ -220,7 +247,7 @@ class DataCollatorCTCWithPadding: """ Data collator that will dynamically pad the inputs received. Args: - processor (:class:`~transformers.Wav2Vec2Processor`) + processor (:class:`~transformers.AutoProcessor`) The processor used for proccessing the data. padding (:obj:`bool`, :obj:`str` or :class:`~transformers.tokenization_utils_base.PaddingStrategy`, `optional`, defaults to :obj:`True`): Select a strategy to pad the returned sequences (according to the model's padding side and padding index) @@ -241,7 +268,7 @@ class DataCollatorCTCWithPadding: 7.5 (Volta). """ - processor: Wav2Vec2Processor + processor: AutoProcessor padding: Union[bool, str] = "longest" pad_to_multiple_of: Optional[int] = None pad_to_multiple_of_labels: Optional[int] = None @@ -275,7 +302,12 @@ class DataCollatorCTCWithPadding: return batch -def create_vocabulary_from_data(datasets: DatasetDict): +def create_vocabulary_from_data( + datasets: DatasetDict, + word_delimiter_token: Optional[str] = None, + unk_token: Optional[str] = None, + pad_token: Optional[str] = None, +): # Given training and test labels create vocabulary def extract_all_chars(batch): all_text = " ".join(batch["target_text"]) @@ -298,12 +330,16 @@ def create_vocabulary_from_data(datasets: DatasetDict): vocab_dict = {v: k for k, v in enumerate(sorted(list(vocab_set)))} # replace white space with delimiter token - vocab_dict["|"] = vocab_dict[" "] - del vocab_dict[" "] + if word_delimiter_token is not None: + vocab_dict[word_delimiter_token] = vocab_dict[" "] + del vocab_dict[" "] # add unk and pad token - vocab_dict["[UNK]"] = len(vocab_dict) - vocab_dict["[PAD]"] = len(vocab_dict) + if unk_token is not None: + vocab_dict[unk_token] = len(vocab_dict) + + if pad_token is not None: + vocab_dict[pad_token] = len(vocab_dict) return vocab_dict @@ -359,103 +395,126 @@ def main(): # 1. First, let's load the dataset raw_datasets = DatasetDict() - raw_datasets["train"] = load_dataset( - data_args.dataset_name, data_args.dataset_config_name, split=data_args.train_split_name - ) - raw_datasets["eval"] = load_dataset( - data_args.dataset_name, data_args.dataset_config_name, split=data_args.eval_split_name - ) - if data_args.audio_column_name not in raw_datasets["train"].column_names: - raise ValueError( - f"--audio_column_name '{data_args.audio_column_name}' not found in dataset '{data_args.dataset_name}'. " - "Make sure to set `--audio_column_name` to the correct audio column - one of " - f"{', '.join(raw_datasets['train'].column_names)}." + if training_args.do_train: + raw_datasets["train"] = load_dataset( + data_args.dataset_name, data_args.dataset_config_name, split=data_args.train_split_name ) - if data_args.text_column_name not in raw_datasets["train"].column_names: - raise ValueError( - f"--text_column_name {data_args.text_column_name} not found in dataset '{data_args.dataset_name}'. " - "Make sure to set `--text_column_name` to the correct text column - one of " - f"{', '.join(raw_datasets['train'].column_names)}." + if data_args.audio_column_name not in raw_datasets["train"].column_names: + raise ValueError( + f"--audio_column_name '{data_args.audio_column_name}' not found in dataset '{data_args.dataset_name}'. " + "Make sure to set `--audio_column_name` to the correct audio column - one of " + f"{', '.join(raw_datasets['train'].column_names)}." + ) + + if data_args.text_column_name not in raw_datasets["train"].column_names: + raise ValueError( + f"--text_column_name {data_args.text_column_name} not found in dataset '{data_args.dataset_name}'. " + "Make sure to set `--text_column_name` to the correct text column - one of " + f"{', '.join(raw_datasets['train'].column_names)}." + ) + + if data_args.max_train_samples is not None: + raw_datasets["train"] = raw_datasets["train"].select(range(data_args.max_train_samples)) + + if training_args.do_eval: + raw_datasets["eval"] = load_dataset( + data_args.dataset_name, data_args.dataset_config_name, split=data_args.eval_split_name ) - # prepare dataset - if data_args.max_train_samples is not None: - raw_datasets["train"] = raw_datasets["train"].select(range(data_args.max_train_samples)) - - if data_args.max_eval_samples is not None: - raw_datasets["eval"] = raw_datasets["eval"].select(range(data_args.max_eval_samples)) + if data_args.max_eval_samples is not None: + raw_datasets["eval"] = raw_datasets["eval"].select(range(data_args.max_eval_samples)) # 2. We remove some special characters from the datasets # that make training complicated and do not help in transcribing the speech # E.g. characters, such as `,` and `.` do not really have an acoustic characteristic # that could be easily picked up by the model - chars_to_ignore_regex = ( f'[{"".join(data_args.chars_to_ignore)}]' if data_args.chars_to_ignore is not None else None ) + text_column_name = data_args.text_column_name def remove_special_characters(batch): if chars_to_ignore_regex is not None: - batch["target_text"] = re.sub(chars_to_ignore_regex, "", batch[data_args.text_column_name]).lower() + " " + batch["target_text"] = re.sub(chars_to_ignore_regex, "", batch[text_column_name]).lower() + " " else: - batch["target_text"] = batch[data_args.text_column_name].lower() + " " + batch["target_text"] = batch[text_column_name].lower() + " " return batch with training_args.main_process_first(desc="dataset map special characters removal"): raw_datasets = raw_datasets.map( remove_special_characters, - remove_columns=[data_args.text_column_name], + remove_columns=[text_column_name], desc="remove special characters from datasets", ) - # 3. Next, we create the vocabulary of the model by extracting all unique characters from - # the training and evaluation datasets - # We need to make sure that only first rank saves vocabulary - # make sure all processes wait until vocab is created - vocab_file = os.path.join(training_args.output_dir, "vocab.json") - - with training_args.main_process_first(): - if training_args.overwrite_output_dir and os.path.isfile(vocab_file): - os.remove(vocab_file) - - with training_args.main_process_first(desc="dataset map vocabulary creation"): - if not os.path.isfile(vocab_file): - os.makedirs(training_args.output_dir, exist_ok=True) - vocab_dict = create_vocabulary_from_data(raw_datasets) - - # save vocab dict to be loaded into tokenizer - with open(vocab_file, "w") as file: - json.dump(vocab_dict, file) - - # 4. Now we can instantiate the configuration, feature extractor, tokenizer and model - # Note for distributed training, the .from_pretrained methods guarantee that only - # one local process can concurrently download model & vocab. + # save special tokens for tokenizer + word_delimiter_token = data_args.word_delimiter_token + unk_token = data_args.unk_token + pad_token = data_args.pad_token + # 3. Next, let's load the config as we might need it to create + # the tokenizer # load config config = AutoConfig.from_pretrained( model_args.model_name_or_path, cache_dir=model_args.cache_dir, use_auth_token=data_args.use_auth_token ) - # tokenizer is defined by `tokenizer_class` if present in config else by `model_type` - config_for_tokenizer = config if config.tokenizer_class is not None else None - tokenizer_type = config.model_type if config.tokenizer_class is None else None + # 4. Next, if no tokenizer file is defined, + # we create the vocabulary of the model by extracting all unique characters from + # the training and evaluation datasets + # We need to make sure that only first rank saves vocabulary + # make sure all processes wait until vocab is created + tokenizer_name_or_path = model_args.tokenizer_name_or_path + tokenizer_type_hints = {} + if tokenizer_name_or_path is None: + # save vocab in training output dir + tokenizer_name_or_path = training_args.output_dir - # load feature_extractor, tokenizer and create processor + vocab_file = os.path.join(tokenizer_name_or_path, "vocab.json") + + with training_args.main_process_first(): + if training_args.overwrite_output_dir and os.path.isfile(vocab_file): + os.remove(vocab_file) + + with training_args.main_process_first(desc="dataset map vocabulary creation"): + if not os.path.isfile(vocab_file): + os.makedirs(tokenizer_name_or_path, exist_ok=True) + vocab_dict = create_vocabulary_from_data( + raw_datasets, + word_delimiter_token=word_delimiter_token, + unk_token=unk_token, + pad_token=pad_token, + ) + + # save vocab dict to be loaded into tokenizer + with open(vocab_file, "w") as file: + json.dump(vocab_dict, file) + + # if tokenizer has just been created + # it is defined by `tokenizer_class` if present in config else by `model_type` + tokenizer_type_hints = { + "config": config if config.tokenizer_class is not None else None, + "tokenizer_type": config.model_type if config.tokenizer_class is None else None, + } + + # 5. Now we can instantiate the feature extractor, tokenizer and model + # Note for distributed training, the .from_pretrained methods guarantee that only + # one local process can concurrently download model & vocab. + + # load feature_extractor and tokenizer tokenizer = AutoTokenizer.from_pretrained( - training_args.output_dir, - config=config_for_tokenizer, - tokenizer_type=tokenizer_type, - unk_token="[UNK]", - pad_token="[PAD]", - word_delimiter_token="|", + tokenizer_name_or_path, + unk_token=unk_token, + pad_token=pad_token, + word_delimiter_token=word_delimiter_token, use_auth_token=data_args.use_auth_token, + **tokenizer_type_hints, ) feature_extractor = AutoFeatureExtractor.from_pretrained( model_args.model_name_or_path, cache_dir=model_args.cache_dir, use_auth_token=data_args.use_auth_token ) - processor = Wav2Vec2Processor(feature_extractor=feature_extractor, tokenizer=tokenizer) # adapt config config.update( @@ -471,8 +530,8 @@ def main(): "gradient_checkpointing": training_args.gradient_checkpointing, "layerdrop": model_args.layerdrop, "ctc_loss_reduction": model_args.ctc_loss_reduction, - "pad_token_id": processor.tokenizer.pad_token_id, - "vocab_size": len(processor.tokenizer), + "pad_token_id": tokenizer.pad_token_id, + "vocab_size": len(tokenizer), "activation_dropout": model_args.activation_dropout, } ) @@ -489,55 +548,64 @@ def main(): if model_args.freeze_feature_extractor: model.freeze_feature_extractor() - # 5. Now we preprocess the datasets including loading the audio, resampling and normalization + # 6. Now we preprocess the datasets including loading the audio, resampling and normalization # Thankfully, `datasets` takes care of automatically loading and resampling the audio, # so that we just need to set the correct target sampling rate and normalize the input # via the `feature_extractor` # make sure that dataset decodes audio with correct sampling rate - raw_datasets = raw_datasets.cast_column( - data_args.audio_column_name, datasets.features.Audio(sampling_rate=feature_extractor.sampling_rate) - ) + dataset_sampling_rate = next(iter(raw_datasets.values())).features[data_args.audio_column_name].sampling_rate + if dataset_sampling_rate != feature_extractor.sampling_rate: + raw_datasets = raw_datasets.cast_column( + data_args.audio_column_name, datasets.features.Audio(sampling_rate=feature_extractor.sampling_rate) + ) # derive max & min input length for sample rate & max duration - max_input_length = data_args.max_duration_in_seconds * processor.feature_extractor.sampling_rate - min_input_length = data_args.min_duration_in_seconds * processor.feature_extractor.sampling_rate + max_input_length = data_args.max_duration_in_seconds * feature_extractor.sampling_rate + min_input_length = data_args.min_duration_in_seconds * feature_extractor.sampling_rate + audio_column_name = data_args.audio_column_name + num_workers = data_args.preprocessing_num_workers + + # `phoneme_language` is only relevant if the model is fine-tuned on phoneme classification + phoneme_language = data_args.phoneme_language # Preprocessing the datasets. # We need to read the audio files as arrays and tokenize the targets. def prepare_dataset(batch): # load audio - sample = batch[data_args.audio_column_name] + sample = batch[audio_column_name] - batch["input_values"] = processor( - sample["array"], sampling_rate=sample["sampling_rate"], truncate=True, max_length=max_input_length - ).input_values[0] + inputs = feature_extractor(sample["array"], sampling_rate=sample["sampling_rate"]) + batch["input_values"] = inputs.input_values[0] batch["input_length"] = len(batch["input_values"]) - # Setup the processor for targets - with processor.as_target_processor(): - batch["labels"] = processor(batch["target_text"]).input_ids + # encode targets + additional_kwargs = {} + if phoneme_language is not None: + additional_kwargs["phonemizer_lang"] = phoneme_language + + batch["labels"] = tokenizer(batch["target_text"], **additional_kwargs).input_ids return batch with training_args.main_process_first(desc="dataset map preprocessing"): vectorized_datasets = raw_datasets.map( prepare_dataset, - remove_columns=raw_datasets["train"].column_names, - num_proc=data_args.preprocessing_num_workers, + remove_columns=next(iter(raw_datasets.values())).column_names, + num_proc=num_workers, desc="preprocess datasets", ) - if min_input_length > 0.0: - # filter data that is shorter than min_input_length - vectorized_datasets = vectorized_datasets.filter( - lambda x: x > min_input_length, - num_proc=data_args.preprocessing_num_workers, - input_columns=["input_length"], - ) + def is_audio_in_length_range(length): + return length > min_input_length and length < max_input_length - vectorized_datasets = vectorized_datasets.remove_columns("input_length") + # filter data that is shorter than min_input_length + vectorized_datasets = vectorized_datasets.filter( + is_audio_in_length_range, + num_proc=num_workers, + input_columns=["input_length"], + ) - # 6. Next, we can prepare the training. + # 7. Next, we can prepare the training. # Let's use word error rate (WER) as our evaluation metric, # instantiate a data collator and the trainer @@ -557,16 +625,36 @@ def main(): pred_logits = pred.predictions pred_ids = np.argmax(pred_logits, axis=-1) - pred.label_ids[pred.label_ids == -100] = processor.tokenizer.pad_token_id + pred.label_ids[pred.label_ids == -100] = tokenizer.pad_token_id - pred_str = processor.batch_decode(pred_ids) + pred_str = tokenizer.batch_decode(pred_ids) # we do not want to group tokens when computing the metrics - label_str = processor.batch_decode(pred.label_ids, group_tokens=False) + label_str = tokenizer.batch_decode(pred.label_ids, group_tokens=False) metrics = {k: v.compute(predictions=pred_str, references=label_str) for k, v in eval_metrics.items()} return metrics + # Now create a single processor + if is_main_process(training_args.local_rank): + # save feature extractor, tokenizer and config + feature_extractor.save_pretrained(training_args.output_dir) + tokenizer.save_pretrained(training_args.output_dir) + config.save_pretrained(training_args.output_dir) + + # load processor + try: + processor = AutoProcessor.from_pretrained(training_args.output_dir) + except (OSError, KeyError): + warnings.warn( + "Loading a processor from a feature extractor config that does not" + " include a `processor_class` attribute is deprecated and will be removed in v5. Please add the following " + " attribute to your `preprocessor_config.json` file to suppress this warning: " + " `'processor_class': 'Wav2Vec2Processor'`", + FutureWarning, + ) + processor = Wav2Vec2Processor.from_pretrained(training_args.output_dir) + # Instantiate custom data collator data_collator = DataCollatorCTCWithPadding(processor=processor) @@ -578,10 +666,10 @@ def main(): compute_metrics=compute_metrics, train_dataset=vectorized_datasets["train"] if training_args.do_train else None, eval_dataset=vectorized_datasets["eval"] if training_args.do_eval else None, - tokenizer=processor.feature_extractor, + tokenizer=feature_extractor, ) - # 7. Finally, we can start training + # 8. Finally, we can start training # Training if training_args.do_train: @@ -594,10 +682,6 @@ def main(): else: checkpoint = None - # Save the feature_extractor and the tokenizer - if is_main_process(training_args.local_rank): - processor.save_pretrained(training_args.output_dir) - train_result = trainer.train(resume_from_checkpoint=checkpoint) trainer.save_model() diff --git a/setup.py b/setup.py index 6acb8aef1d..a7c3c34a37 100644 --- a/setup.py +++ b/setup.py @@ -123,6 +123,7 @@ _deps = [ "optax>=0.0.8", "packaging>=20.0", "parameterized", + "phonemizer", "protobuf", "psutil", "pyyaml>=5.1", @@ -254,7 +255,7 @@ extras["sigopt"] = deps_list("sigopt") extras["integrations"] = extras["optuna"] + extras["ray"] + extras["sigopt"] extras["serving"] = deps_list("pydantic", "uvicorn", "fastapi", "starlette") -extras["audio"] = deps_list("librosa", "pyctcdecode") +extras["audio"] = deps_list("librosa", "pyctcdecode", "phonemizer") extras["speech"] = deps_list("torchaudio") + extras["audio"] # `pip install ".[speech]"` is deprecated and `pip install ".[torch-speech]"` should be used instead extras["torch-speech"] = deps_list("torchaudio") + extras["audio"] extras["tf-speech"] = extras["audio"] diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index 8a23fac8dd..0fe7eea849 100755 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -116,6 +116,7 @@ _import_structure = { "is_datasets_available", "is_faiss_available", "is_flax_available", + "is_phonemizer_available", "is_psutil_available", "is_py3nvml_available", "is_pyctcdecode_available", @@ -313,6 +314,7 @@ _import_structure = { "Wav2Vec2Processor", "Wav2Vec2Tokenizer", ], + "models.wav2vec2_phoneme": ["Wav2Vec2PhonemeCTCTokenizer"], "models.wav2vec2_with_lm": ["Wav2Vec2ProcessorWithLM"], "models.wavlm": [ "WAVLM_PRETRAINED_CONFIG_ARCHIVE_MAP", @@ -2158,6 +2160,7 @@ if TYPE_CHECKING: is_datasets_available, is_faiss_available, is_flax_available, + is_phonemizer_available, is_psutil_available, is_py3nvml_available, is_pyctcdecode_available, @@ -2339,6 +2342,7 @@ if TYPE_CHECKING: Wav2Vec2Processor, Wav2Vec2Tokenizer, ) + from .models.wav2vec2_phoneme import Wav2Vec2PhonemeCTCTokenizer from .models.wav2vec2_with_lm import Wav2Vec2ProcessorWithLM from .models.wavlm import WAVLM_PRETRAINED_CONFIG_ARCHIVE_MAP, WavLMConfig from .models.xlm import XLM_PRETRAINED_CONFIG_ARCHIVE_MAP, XLMConfig, XLMTokenizer diff --git a/src/transformers/dependency_versions_table.py b/src/transformers/dependency_versions_table.py index 7fcef460a3..a83325bfde 100644 --- a/src/transformers/dependency_versions_table.py +++ b/src/transformers/dependency_versions_table.py @@ -34,6 +34,7 @@ deps = { "optax": "optax>=0.0.8", "packaging": "packaging>=20.0", "parameterized": "parameterized", + "phonemizer": "phonemizer", "protobuf": "protobuf", "psutil": "psutil", "pyyaml": "pyyaml>=5.1", diff --git a/src/transformers/file_utils.py b/src/transformers/file_utils.py index b8dcd70f1e..6d6490bb38 100644 --- a/src/transformers/file_utils.py +++ b/src/transformers/file_utils.py @@ -237,6 +237,14 @@ except importlib_metadata.PackageNotFoundError: _torchaudio_available = False +_phonemizer_available = importlib.util.find_spec("phonemizer") is not None +try: + _phonemizer_version = importlib_metadata.version("phonemizer") + logger.debug(f"Successfully imported phonemizer version {_phonemizer_version}") +except importlib_metadata.PackageNotFoundError: + _phonemizer_available = False + + _pyctcdecode_available = importlib.util.find_spec("pyctcdecode") is not None try: _pyctcdecode_version = importlib_metadata.version("pyctcdecode") @@ -592,6 +600,10 @@ def is_speech_available(): return _torchaudio_available +def is_phonemizer_available(): + return _phonemizer_available + + def torch_only_method(fn): def wrapper(*args, **kwargs): if not _torch_available: @@ -728,6 +740,13 @@ explained here: https://pandas.pydata.org/pandas-docs/stable/getting_started/ins """ +# docstyle-ignore +PHONEMIZER_IMPORT_ERROR = """ +{0} requires the phonemizer library but it was not found in your environment. You can install it with pip: +`pip install phonemizer` +""" + + # docstyle-ignore SCIPY_IMPORT_ERROR = """ {0} requires the scipy library but it was not found in your environment. You can install it with pip: @@ -774,6 +793,7 @@ BACKENDS_MAPPING = OrderedDict( ("faiss", (is_faiss_available, FAISS_IMPORT_ERROR)), ("flax", (is_flax_available, FLAX_IMPORT_ERROR)), ("pandas", (is_pandas_available, PANDAS_IMPORT_ERROR)), + ("phonemizer", (is_phonemizer_available, PHONEMIZER_IMPORT_ERROR)), ("protobuf", (is_protobuf_available, PROTOBUF_IMPORT_ERROR)), ("pyctcdecode", (is_pyctcdecode_available, PYCTCDECODE_IMPORT_ERROR)), ("pytesseract", (is_pytesseract_available, PYTESSERACT_IMPORT_ERROR)), diff --git a/src/transformers/models/__init__.py b/src/transformers/models/__init__.py index 1c1aef5aaf..db36d12966 100644 --- a/src/transformers/models/__init__.py +++ b/src/transformers/models/__init__.py @@ -107,6 +107,7 @@ from . import ( visual_bert, vit, wav2vec2, + wav2vec2_phoneme, wav2vec2_with_lm, wavlm, xlm, diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py index 8b9934e458..ca7103f238 100644 --- a/src/transformers/models/auto/tokenization_auto.py +++ b/src/transformers/models/auto/tokenization_auto.py @@ -223,6 +223,7 @@ else: "CLIPTokenizerFast" if is_tokenizers_available() else None, ), ), + ("wav2vec2_phoneme", ("Wav2Vec2PhonemeCTCTokenizer", None)), ( "perceiver", ( diff --git a/src/transformers/models/unispeech/convert_unispeech_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/models/unispeech/convert_unispeech_original_pytorch_checkpoint_to_pytorch.py index 967cdb28bb..83f051627c 100644 --- a/src/transformers/models/unispeech/convert_unispeech_original_pytorch_checkpoint_to_pytorch.py +++ b/src/transformers/models/unispeech/convert_unispeech_original_pytorch_checkpoint_to_pytorch.py @@ -16,11 +16,22 @@ import argparse +import json +import os import fairseq import torch +from fairseq.data import Dictionary -from transformers import UniSpeechConfig, UniSpeechForPreTraining, logging +from transformers import ( + UniSpeechConfig, + UniSpeechForCTC, + UniSpeechForPreTraining, + Wav2Vec2FeatureExtractor, + Wav2Vec2PhonemeCTCTokenizer, + Wav2Vec2Processor, + logging, +) logging.set_verbosity_info() @@ -55,8 +66,17 @@ TOP_LEVEL_KEYS = [ ] -def set_recursively(hf_pointer, key, value, full_name, weight_type): +def set_recursively(hf_pointer, key, value, full_name, weight_type, is_finetuned): for attribute in key.split("."): + if is_finetuned: + if attribute in ["quantizer", "project_q", "project_hid"]: + # those layers are only relevant for pretraining and should be dropped + return + + if attribute == "ctc_proj": + # we should rename `ctc_proj` to `lm_head` for fine-tuned phoneme models + attribute = "lm_head" + hf_pointer = getattr(hf_pointer, attribute) if weight_type is not None: @@ -82,7 +102,7 @@ def set_recursively(hf_pointer, key, value, full_name, weight_type): logger.info(f"{key + '.' + weight_type if weight_type is not None else ''} was initialized from {full_name}.") -def recursively_load_weights(fairseq_model, hf_model): +def recursively_load_weights(fairseq_model, hf_model, is_finetuned): unused_weights = [] fairseq_dict = fairseq_model.state_dict() @@ -118,7 +138,7 @@ def recursively_load_weights(fairseq_model, hf_model): weight_type = "weight" else: weight_type = None - set_recursively(hf_model, mapped_key, value, name, weight_type) + set_recursively(hf_model, mapped_key, value, name, weight_type, is_finetuned) continue if not is_used: unused_weights.append(name) @@ -163,7 +183,9 @@ def load_conv_layer(full_name, value, feature_extractor, unused_weights, use_gro @torch.no_grad() -def convert_unispeech_checkpoint(checkpoint_path, pytorch_dump_folder_path, config_path=None): +def convert_unispeech_checkpoint( + checkpoint_path, pytorch_dump_folder_path, config_path=None, dict_path=None, is_finetuned=True +): """ Copy/paste/tweak model's weights to transformers design. """ @@ -172,12 +194,62 @@ def convert_unispeech_checkpoint(checkpoint_path, pytorch_dump_folder_path, conf else: config = UniSpeechConfig() - hf_unispeech = UniSpeechForPreTraining(config) + if is_finetuned: + if dict_path: + target_dict = Dictionary.load_from_json(dict_path) + + # important change bos & pad token id since CTC symbol is and + # not as in fairseq + config.bos_token_id = target_dict.pad_index + config.pad_token_id = target_dict.bos_index + config.eos_token_id = target_dict.eos_index + config.vocab_size = len(target_dict.symbols) + vocab_path = os.path.join(pytorch_dump_folder_path, "vocab.json") + if not os.path.isdir(pytorch_dump_folder_path): + logger.error("--pytorch_dump_folder_path ({}) should be a directory".format(pytorch_dump_folder_path)) + return + os.makedirs(pytorch_dump_folder_path, exist_ok=True) + vocab_dict = target_dict.indices + + # fairseq has the and switched + vocab_dict[""] = 42 + vocab_dict[""] = 43 + with open(vocab_path, "w", encoding="utf-8") as vocab_handle: + json.dump(vocab_dict, vocab_handle) + tokenizer = Wav2Vec2PhonemeCTCTokenizer( + vocab_path, + unk_token=target_dict.unk_word, + pad_token=target_dict.pad_word, + bos_token=target_dict.bos_word, + eos_token=target_dict.eos_word, + word_delimiter_token="|", + do_lower_case=False, + ) + return_attention_mask = True if config.feat_extract_norm == "layer" else False + feature_extractor = Wav2Vec2FeatureExtractor( + feature_size=1, + sampling_rate=16000, + padding_value=0, + do_normalize=True, + return_attention_mask=return_attention_mask, + ) + processor = Wav2Vec2Processor(feature_extractor=feature_extractor, tokenizer=tokenizer) + processor.save_pretrained(pytorch_dump_folder_path) + + hf_unispeech = UniSpeechForCTC(config) + else: + hf_unispeech = UniSpeechForPreTraining(config) + + if is_finetuned: + model, _, _ = fairseq.checkpoint_utils.load_model_ensemble_and_task( + [checkpoint_path], arg_overrides={"data": "/".join(dict_path.split("/")[:-1]), "w2v_path": checkpoint_path} + ) + else: + model, _, _ = fairseq.checkpoint_utils.load_model_ensemble_and_task([checkpoint_path]) - model, _, _ = fairseq.checkpoint_utils.load_model_ensemble_and_task([checkpoint_path]) model = model[0].eval() - recursively_load_weights(model, hf_unispeech) + recursively_load_weights(model, hf_unispeech, is_finetuned) hf_unispeech.save_pretrained(pytorch_dump_folder_path) @@ -186,6 +258,12 @@ if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--pytorch_dump_folder_path", default=None, type=str, help="Path to the output PyTorch model.") parser.add_argument("--checkpoint_path", default=None, type=str, help="Path to fairseq checkpoint") + parser.add_argument("--dict_path", default=None, type=str, help="Path to dict of fine-tuned model") parser.add_argument("--config_path", default=None, type=str, help="Path to hf config.json of model to convert") + parser.add_argument( + "--not_finetuned", action="store_true", help="Whether the model to convert is a fine-tuned model or not" + ) args = parser.parse_args() - convert_unispeech_checkpoint(args.checkpoint_path, args.pytorch_dump_folder_path, args.config_path) + convert_unispeech_checkpoint( + args.checkpoint_path, args.pytorch_dump_folder_path, args.config_path, args.dict_path, not args.not_finetuned + ) diff --git a/src/transformers/models/wav2vec2/convert_wav2vec2_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/models/wav2vec2/convert_wav2vec2_original_pytorch_checkpoint_to_pytorch.py index b86771d68b..db77a9ea16 100644 --- a/src/transformers/models/wav2vec2/convert_wav2vec2_original_pytorch_checkpoint_to_pytorch.py +++ b/src/transformers/models/wav2vec2/convert_wav2vec2_original_pytorch_checkpoint_to_pytorch.py @@ -205,8 +205,13 @@ def convert_wav2vec2_checkpoint( logger.error("--pytorch_dump_folder_path ({}) should be a directory".format(pytorch_dump_folder_path)) return os.makedirs(pytorch_dump_folder_path, exist_ok=True) + vocab_dict = target_dict.indices + + # fairseq has the and switched + vocab_dict[""] = 0 + vocab_dict[""] = 1 with open(vocab_path, "w", encoding="utf-8") as vocab_handle: - json.dump(target_dict.indices, vocab_handle) + json.dump(vocab_dict, vocab_handle) tokenizer = Wav2Vec2CTCTokenizer( vocab_path, unk_token=target_dict.unk_word, diff --git a/src/transformers/models/wav2vec2/processing_wav2vec2.py b/src/transformers/models/wav2vec2/processing_wav2vec2.py index 04f9233fdf..2f8dcb4ad0 100644 --- a/src/transformers/models/wav2vec2/processing_wav2vec2.py +++ b/src/transformers/models/wav2vec2/processing_wav2vec2.py @@ -15,8 +15,11 @@ """ Speech processor class for Wav2Vec2 """ +import warnings from contextlib import contextmanager +from ...tokenization_utils import PreTrainedTokenizer +from ..auto.tokenization_auto import AutoTokenizer from .feature_extraction_wav2vec2 import Wav2Vec2FeatureExtractor from .tokenization_wav2vec2 import Wav2Vec2CTCTokenizer @@ -27,15 +30,15 @@ class Wav2Vec2Processor: processor. :class:`~transformers.Wav2Vec2Processor` offers all the functionalities of - :class:`~transformers.Wav2Vec2FeatureExtractor` and :class:`~transformers.Wav2Vec2CTCTokenizer`. See the docstring + :class:`~transformers.Wav2Vec2FeatureExtractor` and :class:`~transformers.PreTrainedTokenizer`. See the docstring of :meth:`~transformers.Wav2Vec2Processor.__call__` and :meth:`~transformers.Wav2Vec2Processor.decode` for more information. Args: feature_extractor (:obj:`Wav2Vec2FeatureExtractor`): An instance of :class:`~transformers.Wav2Vec2FeatureExtractor`. The feature extractor is a required input. - tokenizer (:obj:`Wav2Vec2CTCTokenizer`): - An instance of :class:`~transformers.Wav2Vec2CTCTokenizer`. The tokenizer is a required input. + tokenizer (:class:`~transformers.PreTrainedTokenizer`): + An instance of :class:`~transformers.PreTrainedTokenizer`. The tokenizer is a required input. """ def __init__(self, feature_extractor, tokenizer): @@ -43,9 +46,9 @@ class Wav2Vec2Processor: raise ValueError( f"`feature_extractor` has to be of type {Wav2Vec2FeatureExtractor.__class__}, but is {type(feature_extractor)}" ) - if not isinstance(tokenizer, Wav2Vec2CTCTokenizer): + if not isinstance(tokenizer, PreTrainedTokenizer): raise ValueError( - f"`tokenizer` has to be of type {Wav2Vec2CTCTokenizer.__class__}, but is {type(tokenizer)}" + f"`tokenizer` has to be of type {PreTrainedTokenizer.__class__}, but is {type(tokenizer)}" ) self.feature_extractor = feature_extractor @@ -82,7 +85,7 @@ class Wav2Vec2Processor: This class method is simply calling Wav2Vec2FeatureExtractor's :meth:`~transformers.feature_extraction_utils.FeatureExtractionMixin.from_pretrained` and - Wav2Vec2CTCTokenizer's :meth:`~transformers.tokenization_utils_base.PreTrainedTokenizer.from_pretrained`. + PreTrainedTokenizer's :meth:`~transformers.tokenization_utils_base.PreTrainedTokenizer.from_pretrained`. Please refer to the docstrings of the methods above for more information. Args: @@ -102,7 +105,23 @@ class Wav2Vec2Processor: :class:`~transformers.PreTrainedTokenizer` """ feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(pretrained_model_name_or_path, **kwargs) - tokenizer = Wav2Vec2CTCTokenizer.from_pretrained(pretrained_model_name_or_path, **kwargs) + + # load generic `AutoTokenizer` + # need fallback here for backward compatibility in case processor is + # loaded from just a tokenizer file that does not have a `tokenizer_class` attribute + # behavior should be deprecated in major future release + try: + tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path, **kwargs) + except OSError: + warnings.warn( + f"Loading a tokenizer inside {cls.__name__} from a config that does not" + " include a `tokenizer_class` attribute is deprecated and will be " + "removed in v5. Please add `'tokenizer_class': 'Wav2Vec2CTCTokenizer'`" + " attribute to either your `config.json` or `tokenizer_config.json` " + "file to suppress this warning: ", + FutureWarning, + ) + tokenizer = Wav2Vec2CTCTokenizer.from_pretrained(pretrained_model_name_or_path, **kwargs) return cls(feature_extractor=feature_extractor, tokenizer=tokenizer) @@ -111,8 +130,8 @@ class Wav2Vec2Processor: When used in normal mode, this method forwards all its arguments to Wav2Vec2FeatureExtractor's :meth:`~transformers.Wav2Vec2FeatureExtractor.__call__` and returns its output. If used in the context :meth:`~transformers.Wav2Vec2Processor.as_target_processor` this method forwards all its arguments to - Wav2Vec2CTCTokenizer's :meth:`~transformers.Wav2Vec2CTCTokenizer.__call__`. Please refer to the docstring of - the above two methods for more information. + PreTrainedTokenizer's :meth:`~transformers.PreTrainedTokenizer.__call__`. Please refer to the docstring of the + above two methods for more information. """ return self.current_processor(*args, **kwargs) @@ -121,14 +140,14 @@ class Wav2Vec2Processor: When used in normal mode, this method forwards all its arguments to Wav2Vec2FeatureExtractor's :meth:`~transformers.Wav2Vec2FeatureExtractor.pad` and returns its output. If used in the context :meth:`~transformers.Wav2Vec2Processor.as_target_processor` this method forwards all its arguments to - Wav2Vec2CTCTokenizer's :meth:`~transformers.Wav2Vec2CTCTokenizer.pad`. Please refer to the docstring of the - above two methods for more information. + PreTrainedTokenizer's :meth:`~transformers.PreTrainedTokenizer.pad`. Please refer to the docstring of the above + two methods for more information. """ return self.current_processor.pad(*args, **kwargs) def batch_decode(self, *args, **kwargs): """ - This method forwards all its arguments to Wav2Vec2CTCTokenizer's + This method forwards all its arguments to PreTrainedTokenizer's :meth:`~transformers.PreTrainedTokenizer.batch_decode`. Please refer to the docstring of this method for more information. """ @@ -136,7 +155,7 @@ class Wav2Vec2Processor: def decode(self, *args, **kwargs): """ - This method forwards all its arguments to Wav2Vec2CTCTokenizer's + This method forwards all its arguments to PreTrainedTokenizer's :meth:`~transformers.PreTrainedTokenizer.decode`. Please refer to the docstring of this method for more information. """ diff --git a/src/transformers/models/wav2vec2_phoneme/__init__.py b/src/transformers/models/wav2vec2_phoneme/__init__.py new file mode 100644 index 0000000000..4ac68aa740 --- /dev/null +++ b/src/transformers/models/wav2vec2_phoneme/__init__.py @@ -0,0 +1,35 @@ +# flake8: noqa +# There's no way to ignore "F401 '...' imported but unused" warnings in this +# module, but to preserve other warnings. So, don't check this module at all. + +# Copyright 2021 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import TYPE_CHECKING + +from ...file_utils import _LazyModule + + +# fmt: off +_import_structure = { + "tokenization_wav2vec2_phoneme": ["Wav2Vec2PhonemeCTCTokenizer"] +} +# fmt: on + + +if TYPE_CHECKING: + from .tokenization_wav2vec2_phoneme import Wav2Vec2PhonemeCTCTokenizer +else: + import sys + + sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure) diff --git a/src/transformers/models/wav2vec2_phoneme/tokenization_wav2vec2_phoneme.py b/src/transformers/models/wav2vec2_phoneme/tokenization_wav2vec2_phoneme.py new file mode 100644 index 0000000000..b295837fa5 --- /dev/null +++ b/src/transformers/models/wav2vec2_phoneme/tokenization_wav2vec2_phoneme.py @@ -0,0 +1,408 @@ +# coding=utf-8 +# Copyright 2021 The Facebook Inc. and The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tokenization class for Wav2Vec2Phoneme.""" + +import json +import os +import sys +from itertools import groupby +from typing import Any, Dict, List, Optional, Tuple, Union + +from ...file_utils import requires_backends +from ...tokenization_utils import PreTrainedTokenizer, _insert_one_token_to_ordered_list +from ...tokenization_utils_base import AddedToken +from ...utils import logging + + +logger = logging.get_logger(__name__) + + +VOCAB_FILES_NAMES = { + "vocab_file": "vocab.json", + "tokenizer_config_file": "tokenizer_config.json", +} + +PRETRAINED_VOCAB_FILES_MAP = { + "vocab_file": { + "facebook/wav2vec2-lv-60-espeak-cv-ft": "https://huggingface.co/facebook/wav2vec2-lv-60-espeak-cv-ft/resolve/main/vocab.json", + }, + "tokenizer_config_file": { + "facebook/wav2vec2-lv-60-espeak-cv-ft": "https://huggingface.co/facebook/wav2vec2-lv-60-espeak-cv-ft/resolve/main/tokenizer_config.json", + }, +} + +# Wav2Vec2Phoneme has no max input length +PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {"facebook/wav2vec2-lv-60-espeak-cv-ft": sys.maxsize} + + +class Wav2Vec2PhonemeCTCTokenizer(PreTrainedTokenizer): + + """ + Constructs a Wav2Vec2PhonemeCTC tokenizer. + + This tokenizer inherits from :class:`~transformers.PreTrainedTokenizer` which contains some of the main methods. + Users should refer to the superclass for more information regarding such methods. + + Args: + vocab_file (:obj:`str`): + File containing the vocabulary. + bos_token (:obj:`str`, `optional`, defaults to :obj:`""`): + The beginning of sentence token. + eos_token (:obj:`str`, `optional`, defaults to :obj:`""`): + The end of sentence token. + unk_token (:obj:`str`, `optional`, defaults to :obj:`""`): + The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this + token instead. + pad_token (:obj:`str`, `optional`, defaults to :obj:`""`): + The token used for padding, for example when batching sequences of different lengths. + do_phonemize (:obj:`bool`, `optional`, defaults to :obj:`True`): + Whether the tokenizer should phonetize the input or not. Only if a sequence of phonemes is passed to the + tokenizer, :obj:`do_phonemize` should be set to ``False``. + phonemizer_lang (:obj:`str`, `optional`, defaults to :obj:`"en-us"`): + The language of the phoneme set to which the tokenizer should phonetize the input text to. + phonemizer_backend (:obj:`str`, `optional`. defaults to :obj:`"espeak"`): + The backend phonetization library that shall be used by the phonemizer library. Defaults to ``espeak-ng``. + See the `phonemizer package `_. for more information. + + **kwargs + Additional keyword arguments passed along to :class:`~transformers.PreTrainedTokenizer` + """ + + vocab_files_names = VOCAB_FILES_NAMES + pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP + max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES + model_input_names = ["input_ids", "attention_mask"] + + def __init__( + self, + vocab_file, + bos_token="", + eos_token="", + unk_token="", + pad_token="", + phone_delimiter_token=" ", + word_delimiter_token=None, + do_phonemize=True, + phonemizer_lang="en-us", + phonemizer_backend="espeak", + **kwargs + ): + super().__init__( + unk_token=unk_token, + bos_token=bos_token, + eos_token=eos_token, + pad_token=pad_token, + word_delimiter_token=word_delimiter_token, + phone_delimiter_token=phone_delimiter_token, + do_phonemize=do_phonemize, + phonemizer_lang=phonemizer_lang, + phonemizer_backend=phonemizer_backend, + **kwargs, + ) + + self._word_delimiter_token = word_delimiter_token + self._phone_delimiter_token = phone_delimiter_token + self.do_phonemize = do_phonemize + self.phonemizer_lang = phonemizer_lang + self.phonemizer_backend = phonemizer_backend + + with open(vocab_file, encoding="utf-8") as vocab_handle: + self.encoder = json.load(vocab_handle) + self.decoder = {v: k for k, v in self.encoder.items()} + + @property + def vocab_size(self) -> int: + return len(self.decoder) + + def get_vocab(self) -> Dict: + return dict(self.encoder, **self.added_tokens_encoder) + + def prepare_for_tokenization( + self, + text: str, + is_split_into_words: bool = False, + phonemizer_lang: Optional[str] = None, + do_phonemize: Optional[bool] = None, + ) -> Tuple[str, Dict[str, Any]]: + """ + Performs any necessary transformations before tokenization. + + This method should pop the arguments from kwargs and return the remaining :obj:`kwargs` as well. We test the + :obj:`kwargs` at the end of the encoding process to be sure all the arguments have been used. + + Args: + text (:obj:`str`): + The text to prepare. + is_split_into_words (:obj:`bool`, `optional`, defaults to :obj:`False`): + Whether or not the input is already pre-tokenized (e.g., split into words). If set to :obj:`True`, the + tokenizer assumes the input is already split into words (for instance, by splitting it on whitespace) + which it will tokenize. This is useful for NER or token classification. + phonemizer_lang (:obj:`str`, `optional`): + The language of the phoneme set to which the tokenizer should phonetize the input text to. + do_phonemize (:obj:`bool`, `optional`): + Whether the tokenizer should phonetize the input text or not. Only if a sequence of phonemes is passed + to the tokenizer, :obj:`do_phonemize` should be set to ``False``. + + + Returns: + :obj:`Tuple[str, Dict[str, Any]]`: The prepared text and the unused kwargs. + """ + if is_split_into_words: + text = " " + text + + # set whether tokenizer should phonemize or not + if do_phonemize is not None: + self.do_phonemize = do_phonemize + + # set the correct phonemizer language + if phonemizer_lang is not None: + self.phonemizer_lang = phonemizer_lang + + return (text, {}) + + def _tokenize(self, text, **kwargs): + """ + Converts a string in a sequence of tokens (string), using the tokenizer. + """ + + # make sure whitespace is stripped to prevent + text = text.strip() + + # phonemize + if self.do_phonemize: + text = text.lower() + + # create list of phonemes + text = self.phonemize(text, self.phonemizer_lang) + + # make sure ' ' is between phonemes + tokens = text.split(" ") + + tokens = list(filter(lambda p: p.strip() != "", tokens)) + return tokens + + def phonemize(self, text: str, phonemizer_lang: Optional[str] = None) -> str: + requires_backends(self, "phonemizer") + + from phonemizer import phonemize + from phonemizer.separator import Separator + + word_delimiter = self.word_delimiter_token + " " if self.word_delimiter_token is not None else "" + + separator = Separator(phone=self.phone_delimiter_token, word=word_delimiter, syllable="") + phonemes = phonemize( + text, + language=phonemizer_lang, + backend=self.phonemizer_backend, + separator=separator, + language_switch="remove-flags", + ) + phonemes = phonemes.strip() + + return phonemes + + @property + def word_delimiter_token(self) -> str: + """ + :obj:`str`: Word delimiter token. Log an error if used while not having been set. + """ + if self._word_delimiter_token is None and self.verbose: + return None + return str(self._word_delimiter_token) + + @property + def word_delimiter_token_id(self) -> Optional[int]: + """ + :obj:`Optional[int]`: Id of the word_delimiter_token in the vocabulary. Returns :obj:`None` if the token has + not been set. + """ + if self._word_delimiter_token is None: + return None + return self.convert_tokens_to_ids(self.word_delimiter_token) + + @word_delimiter_token.setter + def word_delimiter_token(self, value): + self._word_delimiter_token = value + + @word_delimiter_token_id.setter + def word_delimiter_token_id(self, value): + self._word_delimiter_token = self.convert_tokens_to_ids(value) + + @property + def phone_delimiter_token(self) -> str: + """ + :obj:`str`: Word delimiter token. Log an error if used while not having been set. + """ + if self._phone_delimiter_token is None and self.verbose: + logger.error("Using phone_delimiter_token, but it is not set yet.") + return None + return str(self._phone_delimiter_token) + + @property + def phone_delimiter_token_id(self) -> Optional[int]: + """ + :obj:`Optional[int]`: Id of the phone_delimiter_token in the vocabulary. Returns :obj:`None` if the token has + not been set. + """ + if self._phone_delimiter_token is None: + return None + return self.convert_tokens_to_ids(self.phone_delimiter_token) + + @phone_delimiter_token.setter + def phone_delimiter_token(self, value): + self._phone_delimiter_token = value + + @phone_delimiter_token_id.setter + def phone_delimiter_token_id(self, value): + self._phone_delimiter_token = self.convert_tokens_to_ids(value) + + def _convert_token_to_id(self, token: str) -> int: + """Converts a token (str) in an index (integer) using the vocab.""" + return self.encoder.get(token, self.encoder.get(self.unk_token)) + + def _convert_id_to_token(self, index: int) -> str: + """Converts an index (integer) in a token (str) using the vocab.""" + result = self.decoder.get(index, self.unk_token) + return result + + def convert_tokens_to_string( + self, + tokens: List[str], + group_tokens: bool = True, + spaces_between_special_tokens: bool = False, + filter_word_delimiter_token: bool = True, + ) -> str: + """ + Converts a connectionist-temporal-classification (CTC) output tokens into a single string. + """ + # group same tokens into non-repeating tokens in CTC style decoding + if group_tokens: + tokens = [token_group[0] for token_group in groupby(tokens)] + + # filter self.pad_token which is used as CTC-blank token + filtered_tokens = list(filter(lambda token: token != self.pad_token, tokens)) + + # also filter self.word_delimiter_token if not not + if filter_word_delimiter_token and self.word_delimiter_token is not None: + filtered_tokens = list(filter(lambda token: token != self.word_delimiter_token, filtered_tokens)) + + string = " ".join(filtered_tokens).strip() + + return string + + def _decode( + self, + token_ids: List[int], + skip_special_tokens: bool = False, + clean_up_tokenization_spaces: bool = True, + group_tokens: bool = True, + filter_word_delimiter_token: bool = True, + spaces_between_special_tokens: bool = False, + ) -> str: + """ + special _decode function is needed for Wav2Vec2PhonemeTokenizer because added tokens should be treated exactly + the same as tokens of the base vocabulary and therefore the function `convert_tokens_to_string` has to be + called on the whole token list and not individually on added tokens + """ + filtered_tokens = self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens) + + result = [] + for token in filtered_tokens: + if skip_special_tokens and token in self.all_special_ids: + continue + result.append(token) + + text = self.convert_tokens_to_string( + result, + group_tokens=group_tokens, + spaces_between_special_tokens=spaces_between_special_tokens, + filter_word_delimiter_token=filter_word_delimiter_token, + ) + + if clean_up_tokenization_spaces: + clean_text = self.clean_up_tokenization(text) + return clean_text + else: + return text + + def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]: + if not os.path.isdir(save_directory): + logger.error(f"Vocabulary path ({save_directory}) should be a directory") + return + vocab_file = os.path.join( + save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"] + ) + + with open(vocab_file, "w", encoding="utf-8") as f: + f.write(json.dumps(self.encoder, ensure_ascii=False)) + + return (vocab_file,) + + def _add_tokens(self, new_tokens: Union[List[str], List[AddedToken]], special_tokens: bool = False) -> int: + """ + Add a list of new tokens to the tokenizer class. If the new tokens are not in the vocabulary, they are added to + it with indices starting from length of the current vocabulary. + + Args: + new_tokens (:obj:`List[str]`or :obj:`List[tokenizers.AddedToken]`): + Token(s) to add in vocabulary. A token is only added if it's not already in the vocabulary (tested by + checking if the tokenizer assign the index of the ``unk_token`` to them). + special_tokens (:obj:`bool`, `optional`, defaults to :obj:`False`): + Whether or not the tokens should be added as special tokens. + + Returns: + :obj:`int`: The number of tokens actually added to the vocabulary. + + Examples:: + + # Let's see how to increase the vocabulary of Bert model and tokenizer + tokenizer = Wav2Vec2PhonemeCTCTokenizer.from_pretrained('facebook/wav2vec2-lv-60-espeak-cv-ft') + model = Wav2Vec2PhonemeForCTC.from_pretrained('facebook/wav2vec2-lv-60-espeak-cv-ft') + + num_added_toks = tokenizer.add_tokens(['new_tok1', 'my_new-tok2']) + print('We have added', num_added_toks, 'tokens') + # Note: resize_token_embeddings expects to receive the full size of the new vocabulary, i.e. the length of the tokenizer. + model.resize_token_embeddings(len(tokenizer)) + """ + new_tokens = [str(tok) for tok in new_tokens] + + tokens_to_add = [] + for token in new_tokens: + if not isinstance(token, str): + raise ValueError(f"Token {token} has to be of type string, but is " f"of type {type(token)}.") + assert isinstance(token, str) + if ( + token != self.unk_token + and self.convert_tokens_to_ids(token) == self.convert_tokens_to_ids(self.unk_token) + and token not in tokens_to_add + ): + tokens_to_add.append(token) + if self.verbose: + logger.info(f"Adding {token} to the vocabulary") + + added_tok_encoder = dict((tok, len(self) + i) for i, tok in enumerate(tokens_to_add)) + added_tok_decoder = {v: k for k, v in added_tok_encoder.items()} + self.added_tokens_encoder.update(added_tok_encoder) + self.added_tokens_decoder.update(added_tok_decoder) + + # Make sure we don't split on any special tokens (even they were already in the vocab before) + for token in tokens_to_add: + if len(token) > 1: + self._additional_special_tokens.append(AddedToken(token)) + _insert_one_token_to_ordered_list(self.unique_no_split_tokens, token) + + self._create_trie(self.unique_no_split_tokens) + + return len(tokens_to_add) diff --git a/src/transformers/testing_utils.py b/src/transformers/testing_utils.py index 52a5563a76..37ef221472 100644 --- a/src/transformers/testing_utils.py +++ b/src/transformers/testing_utils.py @@ -38,6 +38,7 @@ from .file_utils import ( is_librosa_available, is_onnx_available, is_pandas_available, + is_phonemizer_available, is_pyctcdecode_available, is_pytesseract_available, is_pytorch_quantization_available, @@ -590,6 +591,16 @@ def require_deepspeed(test_case): return test_case +def require_phonemizer(test_case): + """ + Decorator marking a test that requires phonemizer + """ + if not is_phonemizer_available(): + return unittest.skip("test requires phonemizer")(test_case) + else: + return test_case + + def require_pyctcdecode(test_case): """ Decorator marking a test that requires pyctcdecode diff --git a/tests/test_modeling_wav2vec2.py b/tests/test_modeling_wav2vec2.py index ef83c07633..552f7a2992 100644 --- a/tests/test_modeling_wav2vec2.py +++ b/tests/test_modeling_wav2vec2.py @@ -1399,6 +1399,38 @@ class Wav2Vec2ModelIntegrationTest(unittest.TestCase): self.assertListEqual(predicted_ids.tolist(), expected_labels) self.assertTrue(torch.allclose(predicted_logits, expected_logits, atol=1e-2)) + def test_phoneme_recognition(self): + model = Wav2Vec2ForCTC.from_pretrained("facebook/wav2vec2-lv-60-espeak-cv-ft").to(torch_device) + processor = Wav2Vec2Processor.from_pretrained("facebook/wav2vec2-lv-60-espeak-cv-ft") + + input_speech = self._load_datasamples(4) + + inputs = processor(input_speech, return_tensors="pt", padding=True) + + input_values = inputs.input_values.to(torch_device) + attention_mask = inputs.attention_mask.to(torch_device) + + with torch.no_grad(): + logits = model(input_values, attention_mask=attention_mask).logits + + predicted_ids = torch.argmax(logits, dim=-1) + predicted_trans = processor.batch_decode(predicted_ids) + + EXPECTED_TRANSCRIPTIONS = [ + "ɐ m æ n s ɛ d t ə ð ə j uː n ɪ v ɚ s s ɚ aɪ ɛ ɡ z ɪ s t", + "s w ɛ t k ʌ v ɚ d b ɹ iː ɔ n z b ɑː d i t ɹ ɪ k l ɪ ŋ ɪ n t ə ð ə t aɪ t l oɪ n k l ɑː θ ð æ w ʌ z ð ɪ oʊ n l i ɡ ɑːɹ m ə n t h iː w ɔːɹ", + "ð ə k aɪ t ɔ n h ɪ z tʃ ɛ s t s t ɪ l d ɹ ɪ p ɪ ŋ b l ʌ d ð ɪ eɪ k ʌ v h ɪ z oʊ v ɚ s t ɹ eɪ n d aɪ z iː v ə n ð ə s ɔːɹ ɹ ɪ ŋ ɐ ɹ iː n ɐ ɚ ɹ aʊ n d h ɪ m w ɪ ð ə θ aʊ z ə n d z ʌ v s p ɛ k t eɪ ɾ ɚ z w ɜː t ɹ ɪ v ɪ æ l ᵻ ɾ i z n ɑː t w ɜː θ θ ɪ ŋ k ɪ ŋ ɐ b aʊ t", + "h ɪ z ɪ n s t ə n t v p æ n ɪ k w ʌ z f ɑː l oʊ d b aɪ ɐ s m ɔː l ʃ ɑːɹ p b l oʊ h aɪ ɔ n h ɪ z tʃ ɛ s t", + ] + # should correspond to =>: + # [ + # "a man said to the universe sir i exist", + # "sweat covered brion's body trickling into the tight loin cloth that was the only garment he wore", + # "the cut on his chest still dripping blood the ache of his overstrained eyes even the soaring arena around him with the thousands of spectators were trivialities not worth thinking about", + # "his instant panic was followed by a small sharp blow high on his chest", + # ] + self.assertListEqual(predicted_trans, EXPECTED_TRANSCRIPTIONS) + @require_pyctcdecode @require_torchaudio def test_wav2vec2_with_lm(self): diff --git a/tests/test_tokenization_wav2vec2_phoneme.py b/tests/test_tokenization_wav2vec2_phoneme.py new file mode 100644 index 0000000000..7c479d0733 --- /dev/null +++ b/tests/test_tokenization_wav2vec2_phoneme.py @@ -0,0 +1,328 @@ +# coding=utf-8 +# Copyright 2021 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for the Wav2Vec2Phoneme tokenizer.""" +import json +import os +import unittest +from typing import Tuple + +from transformers import Wav2Vec2PhonemeCTCTokenizer +from transformers.models.wav2vec2.tokenization_wav2vec2 import VOCAB_FILES_NAMES +from transformers.testing_utils import require_phonemizer + +from .test_tokenization_common import TokenizerTesterMixin + + +@require_phonemizer +class Wav2Vec2PhonemeCTCTokenizerTest(TokenizerTesterMixin, unittest.TestCase): + tokenizer_class = Wav2Vec2PhonemeCTCTokenizer + test_rust_tokenizer = False + + def setUp(self): + super().setUp() + + vocab = ( + " n s t ə l a i k d m ɛ ɾ e ɪ p o ɐ z ð f j v b ɹ ʁ ʊ iː r w ʌ u ɡ æ aɪ ʃ h ɔ ɑː " + "ŋ ɚ eɪ β uː y ɑ̃ oʊ ᵻ eː θ aʊ ts oː ɔ̃ ɣ ɜ ɑ dʒ əl x ɜː ç ʒ tʃ ɔː ɑːɹ ɛ̃ ʎ ɔːɹ ʋ aː ɕ œ ø oːɹ ɲ yː " + "ʔ iə i5 s. tɕ ?? nʲ ɛː œ̃ ɭ ɔø ʑ tʲ ɨ ɛɹ ts. rʲ ɪɹ ɭʲ i.5 ɔɪ q sʲ u5 ʊɹ iɜ a5 iɛ5 øː ʕ ja əɜ th ɑ5 " + "oɪ dʲ ə5 tɕh ts.h mʲ ɯ dʑ vʲ e̞ tʃʲ ei5 o5 onɡ5 ɑu5 iɑ5 ai5 aɪɚ kh ə1 ʐ i2 ʉ ħ t[ aɪə ʲ ju ə2 u2 oɜ " + "pː iɛɜ ou5 y5 uɜ tː uo5 d[ uoɜ tsh ɑɜ ɵ i̪5 uei5 ɟ aɜ ɑɨ i.ɜ eʊ o2 ɐ̃ ä pʲ kʲ n̩ ɒ ph ɑu2 uɨ əɪ ɫ ɬ " + "yɜ bʲ ɑ2 s̪ aiɜ χ ɐ̃ʊ̃ 1 ə4 yæɜ a2 ɨː t̪ iouɜ ũ onɡɜ aɨ iɛ2 ɔɨ ɑuɜ o̞ ei2 iou2 c kː y2 ɖ oe dˤ yɛɜ " + 'əʊ S ɡʲ onɡ2 u" eiɜ ʈ ɯᵝ iou5 dZ r̝̊ i.2 tS s^ ʝ yə5 iɑɜ uə5 pf ɨu iɑ2 ou2 ər2 fʲ ai2 r̝ uəɜ ɳ əɨ ' + "ua5 uɪ ɽ bː yu5 uo2 yɛ5 l̩ ɻ ərɜ ʂ i̪2 ouɜ uaɜ a. a.ː yæ5 dː r̩ ee ɪu ər5 i̪ ɜ æi u: i.ː t^ o1 ɪ^ " + "ai ueiɜ æː ɛɪ eə i. ɴ ie ua2 ɑ1 o4 tʃː o: ɑ: u1 N i̪1 au yæ2 u. qː yəɜ y: kʰ tʃʰ iʊ sx õ uo tʰ " + "uai5 bʰ u.ː uə2 ʊə d^ s̪ː yiɜ dʰ r. oe: i1 ɟː yu2 nʲʲ i̪4 uei2 tsʲ ɸ ĩ ɑ4 t̪ː eɑ u4 e: tsː ʈʰ ɡʰ " + "ɯɯ dʒʲ ʂʲ X ɵː uaiɜ tɕʲ ã t^ː ẽː yɛ2 cː i.1 ɛʊ dˤdˤ dʒː i4 ɡː yi ɕʲ ɟʰ pʰ dʑʲ yuɜ ua1 ua4 æiː ɐɐ " + "ui iou1 ʊː a1 iou4 cʰ iɛ1 yə2 ɖʰ ẽ ʒʲ ää ər4 iːː ɪː iɑ1 ər1 œː øi ɪuː cʰcʰ əː1 iː1 ũ kʰː o̞o̞ xʲ " + "ou1 iɛ4 e̞e̞ y1 dzː dʲʲ dʰː ɯᵝɯᵝ lː uo1 i.4 i: yɛ5ʲ a4" + ).split(" ") + vocab_tokens = dict(zip(vocab, range(len(vocab)))) + + self.special_tokens_map = {"pad_token": "", "unk_token": "", "bos_token": "", "eos_token": ""} + + self.vocab_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES["vocab_file"]) + with open(self.vocab_file, "w", encoding="utf-8") as fp: + fp.write(json.dumps(vocab_tokens) + "\n") + + # overwrite since phonemes require specific creation + def get_clean_sequence(self, tokenizer, with_prefix_space=False, max_length=20, min_length=5) -> Tuple[str, list]: + toks = [(i, tokenizer.decode([i], clean_up_tokenization_spaces=False)) for i in range(len(tokenizer))] + toks = list(filter(lambda t: [t[0]] == tokenizer.encode(t[1], do_phonemize=False), toks)) + if max_length is not None and len(toks) > max_length: + toks = toks[:max_length] + if min_length is not None and len(toks) < min_length and len(toks) > 0: + while len(toks) < min_length: + toks = toks + toks + # toks_str = [t[1] for t in toks] + toks_ids = [t[0] for t in toks] + + # Ensure consistency + output_txt = tokenizer.decode(toks_ids, clean_up_tokenization_spaces=False) + if " " not in output_txt and len(toks_ids) > 1: + output_txt = ( + tokenizer.decode([toks_ids[0]], clean_up_tokenization_spaces=False) + + " " + + tokenizer.decode(toks_ids[1:], clean_up_tokenization_spaces=False) + ) + if with_prefix_space: + output_txt = " " + output_txt + output_ids = tokenizer.encode(output_txt, add_special_tokens=False) + return output_txt, output_ids + + def get_tokenizer(self, **kwargs): + kwargs.update(self.special_tokens_map) + return Wav2Vec2PhonemeCTCTokenizer.from_pretrained(self.tmpdirname, **kwargs) + + def test_tokenizer_add_new_tokens(self): + tokenizer = self.tokenizer_class.from_pretrained("facebook/wav2vec2-lv-60-espeak-cv-ft") + + # check adding a single token + tokenizer.add_tokens("xxx") + token_ids = tokenizer("m xxx ɪ", do_phonemize=False).input_ids + self.assertEqual(token_ids, [13, 392, 17]) # xxx should be last token + + tokenizer.add_tokens(["aaa", "bbb", "ccc"]) + token_ids = tokenizer("m aaa ɪ ccc", do_phonemize=False).input_ids + self.assertEqual(token_ids, [13, 393, 17, 395]) # aaa and ccc should be after xxx and 2 after aaa + + token_ids = tokenizer("maɪ c", do_phonemize=False).input_ids + self.assertEqual(token_ids, [3, 200]) # mai should be (=3) + + def test_phonemize(self): + tokenizer = self.tokenizer_class.from_pretrained("facebook/wav2vec2-lv-60-espeak-cv-ft") + + input_text = "Hello how are you" + phonemes = tokenizer.phonemize(input_text, phonemizer_lang="en-us") + self.assertEqual(phonemes, "h ə l oʊ h aʊ ɑːɹ j uː") + + def test_encode(self): + tokenizer = self.tokenizer_class.from_pretrained("facebook/wav2vec2-lv-60-espeak-cv-ft") + + input_text = "Hello how are you" + phonemes = tokenizer.phonemize(input_text, phonemizer_lang="en-us") + self.assertEqual(tokenizer(input_text).input_ids, tokenizer(phonemes, do_phonemize=False).input_ids) + + def test_encode_decode(self): + tokenizer = self.tokenizer_class.from_pretrained("facebook/wav2vec2-lv-60-espeak-cv-ft") + input_text = "Hello how are you" + phonemes = tokenizer.phonemize(input_text, phonemizer_lang="en-us") + + phonemes_enc_dec = tokenizer.decode(tokenizer(input_text).input_ids) + + self.assertEqual(phonemes, phonemes_enc_dec) + + def test_decode(self): + tokenizer = self.tokenizer_class.from_pretrained("facebook/wav2vec2-lv-60-espeak-cv-ft") + + sample_ids = [ + [11, 5, 15, tokenizer.pad_token_id, 15, 8, 98], + [24, 22, 5, 24, 22, 5, 77], + ] + tokens = tokenizer.decode(sample_ids[0]) + batch_tokens = tokenizer.batch_decode(sample_ids) + self.assertEqual(tokens, batch_tokens[0]) + self.assertEqual(batch_tokens, ["k s ɾ ɾ l ɭʲ", "j ð s j ð s oːɹ"]) + + def test_phonemize_with_word_del(self): + tokenizer = self.tokenizer_class.from_pretrained( + "facebook/wav2vec2-lv-60-espeak-cv-ft", word_delimiter_token="|" + ) + tokenizer.add_tokens("|") + + input_text = "Hello how are you" + phonemes = tokenizer.phonemize(input_text, phonemizer_lang="en-us") + self.assertEqual(phonemes, "h ə l oʊ | h aʊ | ɑːɹ | j uː |") + + def test_encode_with_del(self): + tokenizer = self.tokenizer_class.from_pretrained( + "facebook/wav2vec2-lv-60-espeak-cv-ft", word_delimiter_token="|" + ) + tokenizer.add_tokens("|") + + input_text = "Hello how are you" + phonemes = tokenizer.phonemize(input_text, phonemizer_lang="en-us") + self.assertEqual(tokenizer(input_text).input_ids, tokenizer(phonemes, do_phonemize=False).input_ids) + + def test_decode_with_del(self): + tokenizer = self.tokenizer_class.from_pretrained( + "facebook/wav2vec2-lv-60-espeak-cv-ft", word_delimiter_token="|" + ) + tokenizer.add_tokens("|") + + # fmt: off + sample_ids = [ + [11, 5, 15, tokenizer.pad_token_id, tokenizer.word_delimiter_token_id, 15, 8, tokenizer.word_delimiter_token_id, 98], + [tokenizer.word_delimiter_token_id, 24, 22, tokenizer.word_delimiter_token_id, 5, 24, 22, 5, 77], + ] + # fmt: on + + # decode with word_del_token filter + tokens = tokenizer.decode(sample_ids[0]) + batch_tokens = tokenizer.batch_decode(sample_ids) + self.assertEqual(tokens, batch_tokens[0]) + self.assertEqual(batch_tokens, ["k s ɾ ɾ l ɭʲ", "j ð s j ð s oːɹ"]) + + # decode with no word_del_token filter + tokens = tokenizer.decode(sample_ids[0], filter_word_delimiter_token=False) + batch_tokens = tokenizer.batch_decode(sample_ids, filter_word_delimiter_token=False) + self.assertEqual(tokens, batch_tokens[0]) + self.assertEqual(batch_tokens, ["k s ɾ | ɾ l | ɭʲ", "| j ð | s j ð s oːɹ"]) + + def test_encode_decode_with_del(self): + tokenizer = self.tokenizer_class.from_pretrained( + "facebook/wav2vec2-lv-60-espeak-cv-ft", word_delimiter_token="|" + ) + tokenizer.add_tokens("|") + + input_text = "Hello how are you" + phonemes = tokenizer.phonemize(input_text, phonemizer_lang="en-us") + + phonemes_enc_dec = tokenizer.decode(tokenizer(input_text).input_ids, filter_word_delimiter_token=False) + + self.assertEqual(phonemes, phonemes_enc_dec) + + def test_encode_decode_with_del_filter(self): + tokenizer = self.tokenizer_class.from_pretrained( + "facebook/wav2vec2-lv-60-espeak-cv-ft", word_delimiter_token="|" + ) + tokenizer.add_tokens("|") + + input_text = "Hello how are you" + phonemes = tokenizer.phonemize(input_text, phonemizer_lang="en-us") + + phonemes_enc_dec = tokenizer.decode(tokenizer(input_text).input_ids, filter_word_delimiter_token=True) + + self.assertEqual(" ".join([p.strip() for p in phonemes.split(" |")]).strip(), phonemes_enc_dec) + + def test_change_phonemizer_lang(self): + tokenizer = self.tokenizer_class.from_pretrained( + "facebook/wav2vec2-lv-60-espeak-cv-ft", word_delimiter_token=None + ) + input_text = "Hello how are you" + + input_ids_en = tokenizer(input_text, phonemizer_lang="en-us").input_ids + input_ids_fr = tokenizer(input_text, phonemizer_lang="fr-fr").input_ids + + self.assertNotEqual(input_ids_en, input_ids_fr) + + text_en = tokenizer.decode(input_ids_en) + text_fr = tokenizer.decode(input_ids_fr) + + self.assertEqual(text_en, "h ə l oʊ h aʊ ɑːɹ j uː") + self.assertEqual(text_fr, "ɛ l o h aʊ a ʁ j u") + + def test_case_insensitive(self): + tokenizer = self.tokenizer_class.from_pretrained("facebook/wav2vec2-lv-60-espeak-cv-ft") + input_text_up = "Hello how Are you" + input_text_low = "hello how are you" + + input_ids_up = tokenizer(input_text_up).input_ids + input_ids_low = tokenizer(input_text_low).input_ids + + self.assertEqual(input_ids_up, input_ids_low) + + def test_tokenizer_decode_added_tokens(self): + tokenizer = self.tokenizer_class.from_pretrained("facebook/wav2vec2-lv-60-espeak-cv-ft") + tokenizer.add_tokens(["!", "?"]) + tokenizer.add_special_tokens({"cls_token": "$$$"}) + + # fmt: off + sample_ids = [ + [11, 5, 15, tokenizer.pad_token_id, 15, 8, 98, 392, 392, 393, 392, 392, 393, 394, 394], + [24, 22, 5, 24, 22, 5, 77, tokenizer.pad_token_id, 394, 394], + ] + # fmt: on + + batch_tokens = tokenizer.batch_decode(sample_ids) + self.assertEqual(batch_tokens, ["k s ɾ ɾ l ɭʲ!?!? $$$", "j ð s j ð s oːɹ $$$"]) + + # overwrite common test + def test_added_tokens_do_lower_case(self): + # Wav2Vec2PhonemeTokenizer always lower cases letters to correctly map to phonemes + pass + + # overwrite common test + def test_encode_decode_with_spaces(self): + # Wav2Vec2PhonemeTokenizer always puts spaces between phonemes + pass + + # overwrite common test + def test_internal_consistency(self): + # encodes to text to ids, but decodes ids to phonemes -> not possible to have internal consistency + pass + + def test_pretrained_model_lists(self): + # Wav2Vec2PhonemeModel has no max model length => no testing + pass + + # overwrite common + def test_add_tokens_tokenizer(self): + tokenizers = self.get_tokenizers(do_lower_case=False) + for tokenizer in tokenizers: + with self.subTest(f"{tokenizer.__class__.__name__}"): + vocab_size = tokenizer.vocab_size + all_size = len(tokenizer) + + self.assertNotEqual(vocab_size, 0) + + # We usually have added tokens from the start in tests because our vocab fixtures are + # smaller than the original vocabs - let's not assert this + # self.assertEqual(vocab_size, all_size) + + new_toks = ["aaaaa bbbbbb", "cccccccccdddddddd"] + added_toks = tokenizer.add_tokens(new_toks) + vocab_size_2 = tokenizer.vocab_size + all_size_2 = len(tokenizer) + + self.assertNotEqual(vocab_size_2, 0) + self.assertEqual(vocab_size, vocab_size_2) + self.assertEqual(added_toks, len(new_toks)) + self.assertEqual(all_size_2, all_size + len(new_toks)) + + tokens = tokenizer.encode("aaaaa bbbbbb low cccccccccdddddddd l", add_special_tokens=False) + + self.assertGreaterEqual(len(tokens), 4) + self.assertGreater(tokens[0], tokenizer.vocab_size - 1) + self.assertGreater(tokens[-3], tokenizer.vocab_size - 1) + + new_toks_2 = {"eos_token": ">>>>|||<||<<|<<", "pad_token": "<<<<<|||>|>>>>|>"} + added_toks_2 = tokenizer.add_special_tokens(new_toks_2) + vocab_size_3 = tokenizer.vocab_size + all_size_3 = len(tokenizer) + + self.assertNotEqual(vocab_size_3, 0) + self.assertEqual(vocab_size, vocab_size_3) + self.assertEqual(added_toks_2, len(new_toks_2)) + self.assertEqual(all_size_3, all_size_2 + len(new_toks_2)) + + tokens = tokenizer.encode( + ">>>>|||<||<<|<< aaaaabbbbbb low cccccccccdddddddd <<<<<|||>|>>>>|> l", add_special_tokens=False + ) + + self.assertGreaterEqual(len(tokens), 6) + self.assertGreater(tokens[0], tokenizer.vocab_size - 1) + self.assertGreater(tokens[0], tokens[1]) + self.assertGreater(tokens[-3], tokenizer.vocab_size - 1) + self.assertGreater(tokens[-3], tokens[-4]) + self.assertEqual(tokens[0], tokenizer.eos_token_id) + self.assertEqual(tokens[-3], tokenizer.pad_token_id) + + @unittest.skip("The tokenizer shouldn't be used to encode input IDs (except for labels), only to decode.") + def test_tf_encode_plus_sent_to_model(self): + pass + + @unittest.skip("The tokenizer shouldn't be used to encode input IDs (except for labels), only to decode.") + def test_torch_encode_plus_sent_to_model(self): + pass