From d1f74b9aff71a1d5a949b0bc48dbe9894cc6e40b Mon Sep 17 00:00:00 2001 From: HuYong <1030614418@vip.jiangnan.edu.cn> Date: Thu, 16 Jul 2020 11:03:05 +0800 Subject: [PATCH] ADD ERNIE model (#5763) * ERNIE model card * Update Readme.md * Update Readme.md * Update Readme.md * Rename Readme.md to README.md * Update README.md * Update Readme.md * Update README.md * Rename Readme.md to README.md * Update Readme.md * Update Readme.md * Rename Readme.md to README.md * Update and rename Readme.md to README.md Co-authored-by: Kevin Canwen Xu --- model_cards/nghuyong/ernie-1.0/README.md | 44 +++++++++++++++++++ model_cards/nghuyong/ernie-2.0-en/README.md | 43 ++++++++++++++++++ .../nghuyong/ernie-2.0-large-en/README.md | 39 ++++++++++++++++ model_cards/nghuyong/ernie-tiny/README.md | 42 ++++++++++++++++++ 4 files changed, 168 insertions(+) create mode 100644 model_cards/nghuyong/ernie-1.0/README.md create mode 100644 model_cards/nghuyong/ernie-2.0-en/README.md create mode 100644 model_cards/nghuyong/ernie-2.0-large-en/README.md create mode 100644 model_cards/nghuyong/ernie-tiny/README.md diff --git a/model_cards/nghuyong/ernie-1.0/README.md b/model_cards/nghuyong/ernie-1.0/README.md new file mode 100644 index 0000000000..fc6b601ad5 --- /dev/null +++ b/model_cards/nghuyong/ernie-1.0/README.md @@ -0,0 +1,44 @@ +--- +language: zh +--- + +# ERNIE-1.0 + +## Introduction + +ERNIE (Enhanced Representation through kNowledge IntEgration) is proposed by Baidu in 2019, +which is designed to learn language representation enhanced by knowledge masking strategies i.e. entity-level masking and phrase-level masking. +Experimental results show that ERNIE achieve state-of-the-art results on five Chinese natural language processing tasks including natural language inference, +semantic similarity, named entity recognition, sentiment analysis and question answering. + +More detail: https://arxiv.org/abs/1904.09223 + +## Released Model Info + +|Model Name|Language|Model Structure| +|:---:|:---:|:---:| +|ernie-1.0| Chinese |Layer:12, Hidden:768, Heads:12| + +This released pytorch model is converted from the officially released PaddlePaddle ERNIE model and +a series of experiments have been conducted to check the accuracy of the conversion. + +- Official PaddlePaddle ERNIE repo: https://github.com/PaddlePaddle/ERNIE +- Pytorch Conversion repo: https://github.com/nghuyong/ERNIE-Pytorch + +## How to use +```Python +from transformers import AutoTokenizer, AutoModel +tokenizer = AutoTokenizer.from_pretrained("nghuyong/ernie-1.0") +model = AutoModel.from_pretrained("nghuyong/ernie-1.0") +``` + +## Citation + +```bibtex +@article{sun2019ernie, + title={Ernie: Enhanced representation through knowledge integration}, + author={Sun, Yu and Wang, Shuohuan and Li, Yukun and Feng, Shikun and Chen, Xuyi and Zhang, Han and Tian, Xin and Zhu, Danxiang and Tian, Hao and Wu, Hua}, + journal={arXiv preprint arXiv:1904.09223}, + year={2019} +} +``` diff --git a/model_cards/nghuyong/ernie-2.0-en/README.md b/model_cards/nghuyong/ernie-2.0-en/README.md new file mode 100644 index 0000000000..1b0c7dcf38 --- /dev/null +++ b/model_cards/nghuyong/ernie-2.0-en/README.md @@ -0,0 +1,43 @@ +--- +language: en +--- + +# ERNIE-2.0 + +## Introduction + +ERNIE 2.0 is a continual pre-training framework proposed by Baidu in 2019, +which builds and learns incrementally pre-training tasks through constant multi-task learning. +Experimental results demonstrate that ERNIE 2.0 outperforms BERT and XLNet on 16 tasks including English tasks on GLUE benchmarks and several common tasks in Chinese. + +More detail: https://arxiv.org/abs/1907.12412 + +## Released Model Info + +|Model Name|Language|Model Structure| +|:---:|:---:|:---:| +|ernie-2.0-en| English |Layer:12, Hidden:768, Heads:12| + +This released pytorch model is converted from the officially released PaddlePaddle ERNIE model and +a series of experiments have been conducted to check the accuracy of the conversion. + +- Official PaddlePaddle ERNIE repo: https://github.com/PaddlePaddle/ERNIE +- Pytorch Conversion repo: https://github.com/nghuyong/ERNIE-Pytorch + +## How to use +```Python +from transformers import AutoTokenizer, AutoModel +tokenizer = AutoTokenizer.from_pretrained("nghuyong/ernie-2.0-en") +model = AutoModel.from_pretrained("nghuyong/ernie-2.0-en") +``` + +## Citation + +```bibtex +@article{sun2019ernie20, + title={ERNIE 2.0: A Continual Pre-training Framework for Language Understanding}, + author={Sun, Yu and Wang, Shuohuan and Li, Yukun and Feng, Shikun and Tian, Hao and Wu, Hua and Wang, Haifeng}, + journal={arXiv preprint arXiv:1907.12412}, + year={2019} +} +``` diff --git a/model_cards/nghuyong/ernie-2.0-large-en/README.md b/model_cards/nghuyong/ernie-2.0-large-en/README.md new file mode 100644 index 0000000000..5158524050 --- /dev/null +++ b/model_cards/nghuyong/ernie-2.0-large-en/README.md @@ -0,0 +1,39 @@ +# ERNIE-2.0-large + +## Introduction + +ERNIE 2.0 is a continual pre-training framework proposed by Baidu in 2019, +which builds and learns incrementally pre-training tasks through constant multi-task learning. +Experimental results demonstrate that ERNIE 2.0 outperforms BERT and XLNet on 16 tasks including English tasks on GLUE benchmarks and several common tasks in Chinese. + +More detail: https://arxiv.org/abs/1907.12412 + +## Released Model Info + +|Model Name|Language|Model Structure| +|:---:|:---:|:---:| +|ernie-2.0-large-en| English |Layer:24, Hidden:1024, Heads:16| + +This released pytorch model is converted from the officially released PaddlePaddle ERNIE model and +a series of experiments have been conducted to check the accuracy of the conversion. + +- Official PaddlePaddle ERNIE repo: https://github.com/PaddlePaddle/ERNIE +- Pytorch Conversion repo: https://github.com/nghuyong/ERNIE-Pytorch + +## How to use +```Python +from transformers import AutoTokenizer, AutoModel +tokenizer = AutoTokenizer.from_pretrained("nghuyong/ernie-2.0-large-en") +model = AutoModel.from_pretrained("nghuyong/ernie-2.0-large-en") +``` + +## Citation + +```bibtex +@article{sun2019ernie20, + title={ERNIE 2.0: A Continual Pre-training Framework for Language Understanding}, + author={Sun, Yu and Wang, Shuohuan and Li, Yukun and Feng, Shikun and Tian, Hao and Wu, Hua and Wang, Haifeng}, + journal={arXiv preprint arXiv:1907.12412}, + year={2019} +} +``` diff --git a/model_cards/nghuyong/ernie-tiny/README.md b/model_cards/nghuyong/ernie-tiny/README.md new file mode 100644 index 0000000000..56992328a1 --- /dev/null +++ b/model_cards/nghuyong/ernie-tiny/README.md @@ -0,0 +1,42 @@ +--- +language: en +--- + +# ERNIE-tiny + +## Introduction +ERNIE-tiny is a compressed model from [ERNIE 2.0](../ernie-2.0-en) base model through model structure compression and model distillation. +Through compression, the performance of the ERNIE-tiny only decreases by an average of 2.37% compared to ERNIE 2.0 base, +but it outperforms Google BERT by 8.35%, and the speed increases by 4.3 times. + +More details: https://github.com/PaddlePaddle/ERNIE/blob/develop/distill/README.md + +## Released Model Info + +|Model Name|Language|Model Structure| +|:---:|:---:|:---:| +|ernie-tiny| English |Layer:3, Hidden:1024, Heads:16| + +This released pytorch model is converted from the officially released PaddlePaddle ERNIE model and +a series of experiments have been conducted to check the accuracy of the conversion. + +- Official PaddlePaddle ERNIE repo: https://github.com/PaddlePaddle/ERNIE +- Pytorch Conversion repo: https://github.com/nghuyong/ERNIE-Pytorch + +## How to use +```Python +from transformers import AutoTokenizer, AutoModel +tokenizer = AutoTokenizer.from_pretrained("nghuyong/ernie-tiny") +model = AutoModel.from_pretrained("nghuyong/ernie-tiny") +``` + +## Citation + +```bibtex +@article{sun2019ernie20, + title={ERNIE 2.0: A Continual Pre-training Framework for Language Understanding}, + author={Sun, Yu and Wang, Shuohuan and Li, Yukun and Feng, Shikun and Tian, Hao and Wu, Hua and Wang, Haifeng}, + journal={arXiv preprint arXiv:1907.12412}, + year={2019} +} +```