diff --git a/.circleci/config.yml b/.circleci/config.yml index e2a758e6eb..820e8951e5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -81,6 +81,7 @@ jobs: - checkout - run: sudo pip install --progress-bar off -r docs/requirements.txt - run: sudo pip install --progress-bar off -r requirements.txt + - run: cd docs/source && ln -s ../../examples/README.md examples.md && cd - - run: cd docs && make clean && make html && scp -r -oStrictHostKeyChecking=no _build/html/* $doc:$dir workflow_filters: &workflow_filters filters: diff --git a/.gitignore b/.gitignore index bbc738b931..e673ce5f47 100644 --- a/.gitignore +++ b/.gitignore @@ -130,4 +130,5 @@ runs examples/runs # data -data \ No newline at end of file +/data +serialization_dir \ No newline at end of file diff --git a/README.md b/README.md index c473111f2b..9187250c19 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ These implementations have been tested on several datasets (see the example scri | Section | Description | |-|-| | [Installation](#installation) | How to install the package | +| [Online demo](#online-demo) | Experimenting with this repo’s text generation capabilities | | [Quick tour: Usage](#quick-tour) | Tokenizers & models usage: Bert and GPT-2 | | [Quick tour: Fine-tuning/usage scripts](#quick-tour-of-the-fine-tuningusage-scripts) | Using provided scripts: GLUE, SQuAD and Text generation | | [Migrating from pytorch-pretrained-bert to pytorch-transformers](#Migrating-from-pytorch-pretrained-bert-to-pytorch-transformers) | Migrating your code from pytorch-pretrained-bert to pytorch-transformers | @@ -68,6 +69,14 @@ It contains an example of a conversion script from a Pytorch trained Transformer At some point in the future, you'll be able to seamlessly move from pre-training or fine-tuning models in PyTorch to productizing them in CoreML, or prototype a model or an app in CoreML then research its hyperparameters or architecture from PyTorch. Super exciting! +## Online demo + +**[Write With Transformer](https://transformer.huggingface.co)**, built by the Hugging Face team at transformer.huggingface.co, is the official demo of this repo’s text generation capabilities. +You can use it to experiment with completions generated by `GPT2Model`, `TransfoXLModel`, and `XLNetModel`. + +> “🦄 Write with transformer is to writing what calculators are to calculus.” + +![write_with_transformer](https://transformer.huggingface.co/front/assets/thumbnail-large.png) ## Quick tour @@ -279,7 +288,7 @@ This is the model provided as `bert-large-uncased-whole-word-masking-finetuned-s ### `run_generation.py`: Text generation with GPT, GPT-2, Transformer-XL and XLNet A conditional generation script is also included to generate text from a prompt. -The generation script includes the [tricks](https://github.com/rusiaaman/XLNet-gen#methodology) proposed by by Aman Rusia to get high quality generation with memory models like Transformer-XL and XLNet (include a predefined text to make short inputs longer). +The generation script includes the [tricks](https://github.com/rusiaaman/XLNet-gen#methodology) proposed by Aman Rusia to get high quality generation with memory models like Transformer-XL and XLNet (include a predefined text to make short inputs longer). Here is how to run the script with the small version of OpenAI GPT-2 model: diff --git a/docs/README.md b/docs/README.md index 1b3c1feade..6804a22e69 100644 --- a/docs/README.md +++ b/docs/README.md @@ -34,6 +34,13 @@ pip install recommonmark ## Building the documentation +Make sure that there is a symlink from the `example` file (in /examples) inside the source folder. Run the followig +command to generate it: + +```bash +ln -s ../../examples/README.md source/examples.md +``` + Once you have setup `sphinx`, you can build the documentation by running the following command in the `/docs` folder: ```bash diff --git a/docs/requirements.txt b/docs/requirements.txt index 112beb3f72..0c2a31c09a 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -26,3 +26,4 @@ sphinxcontrib-jsmath==1.0.1 sphinxcontrib-qthelp==1.0.2 sphinxcontrib-serializinghtml==1.1.3 urllib3==1.25.3 +sphinx-markdown-tables==0.0.9 \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py index cdca1d82d0..c847dee806 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -43,7 +43,8 @@ extensions = [ 'sphinx.ext.coverage', 'sphinx.ext.napoleon', 'recommonmark', - 'sphinx.ext.viewcode' + 'sphinx.ext.viewcode', + 'sphinx_markdown_tables' ] # Add any paths that contain templates here, relative to this directory. diff --git a/docs/source/examples.rst b/docs/source/examples.rst deleted file mode 100644 index d978451438..0000000000 --- a/docs/source/examples.rst +++ /dev/null @@ -1,686 +0,0 @@ -examples.rst - -Examples -================================================ - -.. list-table:: - :header-rows: 1 - - * - Sub-section - - Description - * - `Training large models: introduction, tools and examples <#introduction>`_ - - How to use gradient-accumulation, multi-gpu training, distributed training, optimize on CPU and 16-bits training to train Bert models - * - `Fine-tuning with BERT: running the examples <#fine-tuning-bert-examples>`_ - - Running the examples in `examples `_\ : ``extract_classif.py``\ , ``run_bert_classifier.py``\ , ``run_bert_squad.py`` and ``run_lm_finetuning.py`` - * - `Fine-tuning with OpenAI GPT, Transformer-XL, GPT-2 as well as BERT and RoBERTa <#fine-tuning>`_ - - Running the examples in `examples `_\ : ``run_openai_gpt.py``\ , ``run_transfo_xl.py``, ``run_gpt2.py`` and ``run_lm_finetuning.py`` - * - `Fine-tuning BERT-large on GPUs <#fine-tuning-bert-large>`_ - - How to fine tune ``BERT large`` - - -.. _introduction: - -Training large models: introduction, tools and examples -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -BERT-base and BERT-large are respectively 110M and 340M parameters models and it can be difficult to fine-tune them on a single GPU with the recommended batch size for good performance (in most case a batch size of 32). - -To help with fine-tuning these models, we have included several techniques that you can activate in the fine-tuning scripts `run_bert_classifier.py `_ and `run_bert_squad.py `_\ : gradient-accumulation, multi-gpu training, distributed training and 16-bits training . For more details on how to use these techniques you can read `the tips on training large batches in PyTorch `_ that I published earlier this year. - -Here is how to use these techniques in our scripts: - - -* **Gradient Accumulation**\ : Gradient accumulation can be used by supplying a integer greater than 1 to the ``--gradient_accumulation_steps`` argument. The batch at each step will be divided by this integer and gradient will be accumulated over ``gradient_accumulation_steps`` steps. -* **Multi-GPU**\ : Multi-GPU is automatically activated when several GPUs are detected and the batches are splitted over the GPUs. -* **Distributed training**\ : Distributed training can be activated by supplying an integer greater or equal to 0 to the ``--local_rank`` argument (see below). -* **16-bits training**\ : 16-bits training, also called mixed-precision training, can reduce the memory requirement of your model on the GPU by using half-precision training, basically allowing to double the batch size. If you have a recent GPU (starting from NVIDIA Volta architecture) you should see no decrease in speed. A good introduction to Mixed precision training can be found `here `__ and a full documentation is `here `__. In our scripts, this option can be activated by setting the ``--fp16`` flag and you can play with loss scaling using the ``--loss_scale`` flag (see the previously linked documentation for details on loss scaling). The loss scale can be zero in which case the scale is dynamically adjusted or a positive power of two in which case the scaling is static. - -To use 16-bits training and distributed training, you need to install NVIDIA's apex extension `as detailed here `__. You will find more information regarding the internals of ``apex`` and how to use ``apex`` in `the doc and the associated repository `_. The results of the tests performed on pytorch-BERT by the NVIDIA team (and my trials at reproducing them) can be consulted in `the relevant PR of the present repository `_. - -Note: To use *Distributed Training*\ , you will need to run one training script on each of your machines. This can be done for example by running the following command on each server (see `the above mentioned blog post `_\ ) for more details): - -.. code-block:: bash - - python -m torch.distributed.launch \ - --nproc_per_node=4 \ - --nnodes=2 \ - --node_rank=$THIS_MACHINE_INDEX \ - --master_addr="192.168.1.1" \ - --master_port=1234 run_bert_classifier.py \ - (--arg1 --arg2 --arg3 and all other arguments of the run_classifier script) - -Where ``$THIS_MACHINE_INDEX`` is an sequential index assigned to each of your machine (0, 1, 2...) and the machine with rank 0 has an IP address ``192.168.1.1`` and an open port ``1234``. - -.. _fine-tuning-bert-examples: - -Fine-tuning with BERT: running the examples -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -We showcase several fine-tuning examples based on (and extended from) `the original implementation `_\ : - - -* a *sequence-level classifier* on nine different GLUE tasks, -* a *token-level classifier* on the question answering dataset SQuAD, and -* a *sequence-level multiple-choice classifier* on the SWAG classification corpus. -* a *BERT language model* on another target corpus - -GLUE results on dev set -~~~~~~~~~~~~~~~~~~~~~~~ - -We get the following results on the dev set of GLUE benchmark with an uncased BERT base -model (`bert-base-uncased`). All experiments ran on 8 V100 GPUs with a total train batch size of 24. Some of -these tasks have a small dataset and training can lead to high variance in the results between different runs. -We report the median on 5 runs (with different seeds) for each of the metrics. - -.. list-table:: - :header-rows: 1 - - * - Task - - Metric - - Result - * - CoLA - - Matthew's corr. - - 55.75 - * - SST-2 - - accuracy - - 92.09 - * - MRPC - - F1/accuracy - - 90.48/86.27 - * - STS-B - - Pearson/Spearman corr. - - 89.03/88.64 - * - QQP - - accuracy/F1 - - 90.92/87.72 - * - MNLI - - matched acc./mismatched acc. - - 83.74/84.06 - * - QNLI - - accuracy - - 91.07 - * - RTE - - accuracy - - 68.59 - * - WNLI - - accuracy - - 43.66 - - -Some of these results are significantly different from the ones reported on the test set -of GLUE benchmark on the website. For QQP and WNLI, please refer to `FAQ #12 `_ on the webite. - -Before running anyone of these GLUE tasks you should download the -`GLUE data `_ by running -`this script `_ -and unpack it to some directory ``$GLUE_DIR``. - -.. code-block:: shell - - export GLUE_DIR=/path/to/glue - export TASK_NAME=MRPC - - python run_bert_classifier.py \ - --task_name $TASK_NAME \ - --do_train \ - --do_eval \ - --do_lower_case \ - --data_dir $GLUE_DIR/$TASK_NAME \ - --bert_model bert-base-uncased \ - --max_seq_length 128 \ - --train_batch_size 32 \ - --learning_rate 2e-5 \ - --num_train_epochs 3.0 \ - --output_dir /tmp/$TASK_NAME/ - -where task name can be one of CoLA, SST-2, MRPC, STS-B, QQP, MNLI, QNLI, RTE, WNLI. - -The dev set results will be present within the text file 'eval_results.txt' in the specified output_dir. In case of MNLI, since there are two separate dev sets, matched and mismatched, there will be a separate output folder called '/tmp/MNLI-MM/' in addition to '/tmp/MNLI/'. - -The code has not been tested with half-precision training with apex on any GLUE task apart from MRPC, MNLI, CoLA, SST-2. The following section provides details on how to run half-precision training with MRPC. With that being said, there shouldn't be any issues in running half-precision training with the remaining GLUE tasks as well, since the data processor for each task inherits from the base class DataProcessor. - -MRPC -~~~~ - -This example code fine-tunes BERT on the Microsoft Research Paraphrase -Corpus (MRPC) corpus and runs in less than 10 minutes on a single K-80 and in 27 seconds (!) on single tesla V100 16GB with apex installed. - -Before running this example you should download the -`GLUE data `_ by running -`this script `_ -and unpack it to some directory ``$GLUE_DIR``. - -.. code-block:: shell - - export GLUE_DIR=/path/to/glue - - python run_bert_classifier.py \ - --task_name MRPC \ - --do_train \ - --do_eval \ - --do_lower_case \ - --data_dir $GLUE_DIR/MRPC/ \ - --bert_model bert-base-uncased \ - --max_seq_length 128 \ - --train_batch_size 32 \ - --learning_rate 2e-5 \ - --num_train_epochs 3.0 \ - --output_dir /tmp/mrpc_output/ - -Our test ran on a few seeds with `the original implementation hyper-parameters `__ gave evaluation results between 84% and 88%. - -**Fast run with apex and 16 bit precision: fine-tuning on MRPC in 27 seconds!** -First install apex as indicated `here `__. -Then run - -.. code-block:: shell - - export GLUE_DIR=/path/to/glue - - python run_bert_classifier.py \ - --task_name MRPC \ - --do_train \ - --do_eval \ - --do_lower_case \ - --data_dir $GLUE_DIR/MRPC/ \ - --bert_model bert-base-uncased \ - --max_seq_length 128 \ - --train_batch_size 32 \ - --learning_rate 2e-5 \ - --num_train_epochs 3.0 \ - --output_dir /tmp/mrpc_output/ \ - --fp16 - -**Distributed training** -Here is an example using distributed training on 8 V100 GPUs and Bert Whole Word Masking model to reach a F1 > 92 on MRPC: - -.. code-block:: bash - - python -m torch.distributed.launch \ - --nproc_per_node 8 run_bert_classifier.py \ - --bert_model bert-large-uncased-whole-word-masking \ - --task_name MRPC \ - --do_train \ - --do_eval \ - --do_lower_case \ - --data_dir $GLUE_DIR/MRPC/ \ - --max_seq_length 128 \ - --train_batch_size 8 \ - --learning_rate 2e-5 \ - --num_train_epochs 3.0 \ - --output_dir /tmp/mrpc_output/ - -Training with these hyper-parameters gave us the following results: - -.. code-block:: bash - - acc = 0.8823529411764706 - acc_and_f1 = 0.901702786377709 - eval_loss = 0.3418912578906332 - f1 = 0.9210526315789473 - global_step = 174 - loss = 0.07231863956341798 - -Here is an example on MNLI: - -.. code-block:: bash - - python -m torch.distributed.launch \ - --nproc_per_node 8 run_bert_classifier.py \ - --bert_model bert-large-uncased-whole-word-masking \ - --task_name mnli \ - --do_train \ - --do_eval \ - --do_lower_case \ - --data_dir /datadrive/bert_data/glue_data//MNLI/ \ - --max_seq_length 128 \ - --train_batch_size 8 \ - --learning_rate 2e-5 \ - --num_train_epochs 3.0 \ - --output_dir ../models/wwm-uncased-finetuned-mnli/ \ - --overwrite_output_dir - -.. code-block:: bash - - ***** Eval results ***** - acc = 0.8679706601466992 - eval_loss = 0.4911287787382479 - global_step = 18408 - loss = 0.04755385363816904 - - ***** Eval results ***** - acc = 0.8747965825874695 - eval_loss = 0.45516540421714036 - global_step = 18408 - loss = 0.04755385363816904 - -This is the example of the ``bert-large-uncased-whole-word-masking-finetuned-mnli`` model - -SQuAD -~~~~~ - -This example code fine-tunes BERT on the SQuAD dataset. It runs in 24 min (with BERT-base) or 68 min (with BERT-large) on a single tesla V100 16GB. - -The data for SQuAD can be downloaded with the following links and should be saved in a ``$SQUAD_DIR`` directory. - - -* `train-v1.1.json `_ -* `dev-v1.1.json `_ -* `evaluate-v1.1.py `_ - -.. code-block:: shell - - export SQUAD_DIR=/path/to/SQUAD - - python run_bert_squad.py \ - --bert_model bert-base-uncased \ - --do_train \ - --do_predict \ - --do_lower_case \ - --train_file $SQUAD_DIR/train-v1.1.json \ - --predict_file $SQUAD_DIR/dev-v1.1.json \ - --train_batch_size 12 \ - --learning_rate 3e-5 \ - --num_train_epochs 2.0 \ - --max_seq_length 384 \ - --doc_stride 128 \ - --output_dir /tmp/debug_squad/ - -Training with the previous hyper-parameters gave us the following results: - -.. code-block:: bash - - python $SQUAD_DIR/evaluate-v1.1.py $SQUAD_DIR/dev-v1.1.json /tmp/debug_squad/predictions.json - {"f1": 88.52381567990474, "exact_match": 81.22043519394512} - -**distributed training** - -Here is an example using distributed training on 8 V100 GPUs and Bert Whole Word Masking uncased model to reach a F1 > 93 on SQuAD: - -.. code-block:: bash - - python -m torch.distributed.launch --nproc_per_node=8 \ - run_bert_squad.py \ - --bert_model bert-large-uncased-whole-word-masking \ - --do_train \ - --do_predict \ - --do_lower_case \ - --train_file $SQUAD_DIR/train-v1.1.json \ - --predict_file $SQUAD_DIR/dev-v1.1.json \ - --learning_rate 3e-5 \ - --num_train_epochs 2 \ - --max_seq_length 384 \ - --doc_stride 128 \ - --output_dir ../models/wwm_uncased_finetuned_squad/ \ - --train_batch_size 24 \ - --gradient_accumulation_steps 12 - -Training with these hyper-parameters gave us the following results: - -.. code-block:: bash - - python $SQUAD_DIR/evaluate-v1.1.py $SQUAD_DIR/dev-v1.1.json ../models/wwm_uncased_finetuned_squad/predictions.json - {"exact_match": 86.91579943235573, "f1": 93.1532499015869} - -This is the model provided as ``bert-large-uncased-whole-word-masking-finetuned-squad``. - -And here is the model provided as ``bert-large-cased-whole-word-masking-finetuned-squad``\ : - -.. code-block:: bash - - python -m torch.distributed.launch --nproc_per_node=8 run_bert_squad.py \ - --bert_model bert-large-cased-whole-word-masking \ - --do_train \ - --do_predict \ - --do_lower_case \ - --train_file $SQUAD_DIR/train-v1.1.json \ - --predict_file $SQUAD_DIR/dev-v1.1.json \ - --learning_rate 3e-5 \ - --num_train_epochs 2 \ - --max_seq_length 384 \ - --doc_stride 128 \ - --output_dir ../models/wwm_cased_finetuned_squad/ \ - --train_batch_size 24 \ - --gradient_accumulation_steps 12 - -Training with these hyper-parameters gave us the following results: - -.. code-block:: bash - - python $SQUAD_DIR/evaluate-v1.1.py $SQUAD_DIR/dev-v1.1.json ../models/wwm_uncased_finetuned_squad/predictions.json - {"exact_match": 84.18164616840113, "f1": 91.58645594850135} - -SWAG -~~~~ - -The data for SWAG can be downloaded by cloning the following `repository `_ - -.. code-block:: shell - - export SWAG_DIR=/path/to/SWAG - - python run_bert_swag.py \ - --bert_model bert-base-uncased \ - --do_train \ - --do_lower_case \ - --do_eval \ - --data_dir $SWAG_DIR/data \ - --train_batch_size 16 \ - --learning_rate 2e-5 \ - --num_train_epochs 3.0 \ - --max_seq_length 80 \ - --output_dir /tmp/swag_output/ \ - --gradient_accumulation_steps 4 - -Training with the previous hyper-parameters on a single GPU gave us the following results: - -.. code-block:: - - eval_accuracy = 0.8062081375587323 - eval_loss = 0.5966546792367169 - global_step = 13788 - loss = 0.06423990014260186 - -LM Fine-tuning -~~~~~~~~~~~~~~ - -The data should be a text file in the same format as `sample_text.txt <./pytorch_transformers/tests/fixtures/sample_text.txt/sample_text.txt>`_ (one sentence per line, docs separated by empty line). -You can download an `exemplary training corpus `_ generated from wikipedia articles and split into ~500k sentences with spaCy. -Training one epoch on this corpus takes about 1:20h on 4 x NVIDIA Tesla P100 with ``train_batch_size=200`` and ``max_seq_length=128``\ : - -Thank to the work of @Rocketknight1 and @tholor there are now **several scripts** that can be used to fine-tune BERT using the pretraining objective (combination of masked-language modeling and next sentence prediction loss). These scripts are detailed in the `README `_ of the `examples/lm_finetuning/ `_ folder. - -.. _fine-tuning: - -OpenAI GPT, Transformer-XL and GPT-2: running the examples -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -We provide three examples of scripts for OpenAI GPT, Transformer-XL, OpenAI GPT-2, BERT and RoBERTa based on (and extended from) the respective original implementations: - - -* fine-tuning OpenAI GPT on the ROCStories dataset -* evaluating Transformer-XL on Wikitext 103 -* unconditional and conditional generation from a pre-trained OpenAI GPT-2 model -* fine-tuning GPT/GPT-2 on a causal language modeling task and BERT/RoBERTa on a masked language modeling task - -Fine-tuning OpenAI GPT on the RocStories dataset -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -This example code fine-tunes OpenAI GPT on the RocStories dataset. - -Before running this example you should download the -`RocStories dataset `_ and unpack it to some directory ``$ROC_STORIES_DIR``. - -.. code-block:: shell - - export ROC_STORIES_DIR=/path/to/RocStories - - python run_openai_gpt.py \ - --model_name openai-gpt \ - --do_train \ - --do_eval \ - --train_dataset $ROC_STORIES_DIR/cloze_test_val__spring2016\ -\ cloze_test_ALL_val.csv \ - --eval_dataset $ROC_STORIES_DIR/cloze_test_test__spring2016\ -\ cloze_test_ALL_test.csv \ - --output_dir ../log \ - --train_batch_size 16 \ - -This command runs in about 10 min on a single K-80 an gives an evaluation accuracy of about 87.7% (the authors report a median accuracy with the TensorFlow code of 85.8% and the OpenAI GPT paper reports a best single run accuracy of 86.5%). - -Evaluating the pre-trained Transformer-XL on the WikiText 103 dataset -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -This example code evaluate the pre-trained Transformer-XL on the WikiText 103 dataset. -This command will download a pre-processed version of the WikiText 103 dataset in which the vocabulary has been computed. - -.. code-block:: shell - - python run_transfo_xl.py --work_dir ../log - -This command runs in about 1 min on a V100 and gives an evaluation perplexity of 18.22 on WikiText-103 (the authors report a perplexity of about 18.3 on this dataset with the TensorFlow code). - -Unconditional and conditional generation from OpenAI's GPT-2 model -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -This example code is identical to the original unconditional and conditional generation codes. - -Conditional generation: - -.. code-block:: shell - - python run_gpt2.py - -Unconditional generation: - -.. code-block:: shell - - python run_gpt2.py --unconditional - -The same option as in the original scripts are provided, please refer to the code of the example and the original repository of OpenAI. - - -Causal LM fine-tuning on GPT/GPT-2, Masked LM fine-tuning on BERT/RoBERTa -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Before running the following examples you should download the `WikiText-2 dataset `__ and unpack it to some directory `$WIKITEXT_2_DATASET` -The following results were obtained using the `raw` WikiText-2 (no tokens were replaced before the tokenization). - -This example fine-tunes GPT-2 on the WikiText-2 dataset. The loss function is a causal language modeling loss (perplexity). - -.. code-block:: bash - - - export WIKITEXT_2_DATASET=/path/to/wikitext_dataset - - python run_lm_finetuning.py - --output_dir=output - --model_type=gpt2 - --model_name_or_path=gpt2 - --do_train - --train_data_file=$WIKITEXT_2_DATASET/wiki.train.raw - --do_eval - --eval_data_file=$WIKITEXT_2_DATASET/wiki.test.raw - -This takes about half an hour to train on a single K80 GPU and about one minute for the evaluation to run. -It reaches a score of about 20 perplexity once fine-tuned on the dataset. - -This example fine-tunes RoBERTa on the WikiText-2 dataset. The loss function is a masked language modeling loss (masked perplexity). -The `--mlm` flag is necessary to fine-tune BERT/RoBERTa on masked language modeling. - -.. code-block:: bash - - - export WIKITEXT_2_DATASET=/path/to/wikitext_dataset - - python run_lm_finetuning.py - --output_dir=output - --model_type=roberta - --model_name_or_path=roberta-base - --do_train - --train_data_file=$WIKITEXT_2_DATASET/wiki.train.raw - --do_eval - --eval_data_file=$WIKITEXT_2_DATASET/wiki.test.raw - --mlm - -.. _fine-tuning-BERT-large: - -Fine-tuning BERT-large on GPUs ------------------------------- - -The options we list above allow to fine-tune BERT-large rather easily on GPU(s) instead of the TPU used by the original implementation. - -For example, fine-tuning BERT-large on SQuAD can be done on a server with 4 k-80 (these are pretty old now) in 18 hours. Our results are similar to the TensorFlow implementation results (actually slightly higher): - -.. code-block:: bash - - {"exact_match": 84.56953642384106, "f1": 91.04028647786927} - -To get these results we used a combination of: - - -* multi-GPU training (automatically activated on a multi-GPU server), -* 2 steps of gradient accumulation and -* perform the optimization step on CPU to store Adam's averages in RAM. - -Here is the full list of hyper-parameters for this run: - -.. code-block:: bash - - export SQUAD_DIR=/path/to/SQUAD - - python ./run_bert_squad.py \ - --bert_model bert-large-uncased \ - --do_train \ - --do_predict \ - --do_lower_case \ - --train_file $SQUAD_DIR/train-v1.1.json \ - --predict_file $SQUAD_DIR/dev-v1.1.json \ - --learning_rate 3e-5 \ - --num_train_epochs 2 \ - --max_seq_length 384 \ - --doc_stride 128 \ - --output_dir /tmp/debug_squad/ \ - --train_batch_size 24 \ - --gradient_accumulation_steps 2 - -If you have a recent GPU (starting from NVIDIA Volta series), you should try **16-bit fine-tuning** (FP16). - -Here is an example of hyper-parameters for a FP16 run we tried: - -.. code-block:: bash - - export SQUAD_DIR=/path/to/SQUAD - - python ./run_bert_squad.py \ - --bert_model bert-large-uncased \ - --do_train \ - --do_predict \ - --do_lower_case \ - --train_file $SQUAD_DIR/train-v1.1.json \ - --predict_file $SQUAD_DIR/dev-v1.1.json \ - --learning_rate 3e-5 \ - --num_train_epochs 2 \ - --max_seq_length 384 \ - --doc_stride 128 \ - --output_dir /tmp/debug_squad/ \ - --train_batch_size 24 \ - --fp16 \ - --loss_scale 128 - -The results were similar to the above FP32 results (actually slightly higher): - -.. code-block:: bash - - {"exact_match": 84.65468306527909, "f1": 91.238669287002} - -Here is an example with the recent ``bert-large-uncased-whole-word-masking``\ : - -.. code-block:: bash - - python -m torch.distributed.launch --nproc_per_node=8 \ - run_bert_squad.py \ - --bert_model bert-large-uncased-whole-word-masking \ - --do_train \ - --do_predict \ - --do_lower_case \ - --train_file $SQUAD_DIR/train-v1.1.json \ - --predict_file $SQUAD_DIR/dev-v1.1.json \ - --learning_rate 3e-5 \ - --num_train_epochs 2 \ - --max_seq_length 384 \ - --doc_stride 128 \ - --output_dir /tmp/debug_squad/ \ - --train_batch_size 24 \ - --gradient_accumulation_steps 2 - -Fine-tuning XLNet ------------------ - -STS-B -~~~~~ - -This example code fine-tunes XLNet on the STS-B corpus. - -Before running this example you should download the -`GLUE data `_ by running -`this script `_ -and unpack it to some directory ``$GLUE_DIR``. - -.. code-block:: shell - - export GLUE_DIR=/path/to/glue - - python run_xlnet_classifier.py \ - --task_name STS-B \ - --do_train \ - --do_eval \ - --data_dir $GLUE_DIR/STS-B/ \ - --max_seq_length 128 \ - --train_batch_size 8 \ - --gradient_accumulation_steps 1 \ - --learning_rate 5e-5 \ - --num_train_epochs 3.0 \ - --output_dir /tmp/mrpc_output/ - -Our test ran on a few seeds with `the original implementation hyper-parameters `__ gave evaluation results between 84% and 88%. - -**Distributed training** -Here is an example using distributed training on 8 V100 GPUs to reach XXXX: - -.. code-block:: bash - - python -m torch.distributed.launch --nproc_per_node 8 \ - run_xlnet_classifier.py \ - --task_name STS-B \ - --do_train \ - --do_eval \ - --data_dir $GLUE_DIR/STS-B/ \ - --max_seq_length 128 \ - --train_batch_size 8 \ - --gradient_accumulation_steps 1 \ - --learning_rate 5e-5 \ - --num_train_epochs 3.0 \ - --output_dir /tmp/mrpc_output/ - -Training with these hyper-parameters gave us the following results: - -.. code-block:: bash - - acc = 0.8823529411764706 - acc_and_f1 = 0.901702786377709 - eval_loss = 0.3418912578906332 - f1 = 0.9210526315789473 - global_step = 174 - loss = 0.07231863956341798 - -Here is an example on MNLI: - -.. code-block:: bash - - python -m torch.distributed.launch --nproc_per_node 8 run_bert_classifier.py \ - --bert_model bert-large-uncased-whole-word-masking \ - --task_name mnli \ - --do_train \ - --do_eval \ - --data_dir /datadrive/bert_data/glue_data//MNLI/ \ - --max_seq_length 128 \ - --train_batch_size 8 \ - --learning_rate 2e-5 \ - --num_train_epochs 3.0 \ - --output_dir ../models/wwm-uncased-finetuned-mnli/ \ - --overwrite_output_dir - -.. code-block:: bash - - ***** Eval results ***** - acc = 0.8679706601466992 - eval_loss = 0.4911287787382479 - global_step = 18408 - loss = 0.04755385363816904 - - ***** Eval results ***** - acc = 0.8747965825874695 - eval_loss = 0.45516540421714036 - global_step = 18408 - loss = 0.04755385363816904 - -This is the example of the ``bert-large-uncased-whole-word-masking-finetuned-mnli`` model. diff --git a/docs/source/pretrained_models.rst b/docs/source/pretrained_models.rst index 4222ee32cf..d6e273797f 100644 --- a/docs/source/pretrained_models.rst +++ b/docs/source/pretrained_models.rst @@ -79,10 +79,10 @@ Here is the full list of the currently provided pretrained models together with | | | | XLM English model | | +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``xlm-mlm-ende-1024`` | | 6-layer, 1024-hidden, 8-heads | -| | | | XLM English-German Multi-language model | +| | | | XLM English-German model trained on the concatenation of English and German wikipedia | | +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``xlm-mlm-enfr-1024`` | | 6-layer, 1024-hidden, 8-heads | -| | | | XLM English-French Multi-language model | +| | | | XLM English-French model trained on the concatenation of English and French wikipedia | | +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``xlm-mlm-enro-1024`` | | 6-layer, 1024-hidden, 8-heads | | | | | XLM English-Romanian Multi-language model | @@ -93,11 +93,11 @@ Here is the full list of the currently provided pretrained models together with | | ``xlm-mlm-tlm-xnli15-1024`` | | 12-layer, 1024-hidden, 8-heads | | | | | XLM Model pre-trained with MLM + TLM on the `15 XNLI languages `__. | | +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ -| | ``xlm-clm-enfr-1024`` | | 12-layer, 1024-hidden, 8-heads | -| | | | XLM English model trained with CLM (Causal Language Modeling) | +| | ``xlm-clm-enfr-1024`` | | 6-layer, 1024-hidden, 8-heads | +| | | | XLM English-French model trained with CLM (Causal Language Modeling) on the concatenation of English and French wikipedia | | +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``xlm-clm-ende-1024`` | | 6-layer, 1024-hidden, 8-heads | -| | | | XLM English-German Multi-language model trained with CLM (Causal Language Modeling) | +| | | | XLM English-German model trained with CLM (Causal Language Modeling) on the concatenation of English and German wikipedia | +-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | RoBERTa | ``roberta-base`` | | 12-layer, 768-hidden, 12-heads, 125M parameters | | | | | RoBERTa using the BERT-base architecture | diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000000..3253e5481c --- /dev/null +++ b/examples/README.md @@ -0,0 +1,392 @@ +# Examples + +In this section a few examples are put together. All of these examples work for several models, making use of the very +similar API between the different models. + +| Section | Description | +|----------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------| +| [Language Model fine-tuning](#language-model-fine-tuning) | Fine-tuning the library models for language modeling on a text dataset. Causal language modeling for GPT/GPT-2, masked language modeling for BERT/RoBERTa. | +| [Language Generation](#language-generation) | Conditional text generation using the auto-regressive models of the library: GPT, GPT-2, Transformer-XL and XLNet. | +| [GLUE](#glue) | Examples running BERT/XLM/XLNet/RoBERTa on the 9 GLUE tasks. Examples feature distributed training as well as half-precision. | +| [SQuAD](#squad) | Using BERT for question answering, examples with distributed training. | +| [Multiple Choice](#multiple choice) | Examples running BERT/XLNet/RoBERTa on the SWAG/RACE/ARC tasks. + +## Language model fine-tuning + +Based on the script [`run_lm_finetuning.py`](https://github.com/huggingface/pytorch-transformers/blob/master/examples/run_lm_finetuning.py). + +Fine-tuning the library models for language modeling on a text dataset for GPT, GPT-2, BERT and RoBERTa (DistilBERT +to be added soon). GPT and GPT-2 are fine-tuned using a causal language modeling (CLM) loss while BERT and RoBERTa +are fine-tuned using a masked language modeling (MLM) loss. + +Before running the following example, you should get a file that contains text on which the language model will be +fine-tuned. A good example of such text is the [WikiText-2 dataset](https://blog.einstein.ai/the-wikitext-long-term-dependency-language-modeling-dataset/). + +We will refer to two different files: `$TRAIN_FILE`, which contains text for training, and `$TEST_FILE`, which contains +text that will be used for evaluation. + +### GPT-2/GPT and causal language modeling + +The following example fine-tunes GPT-2 on WikiText-2. We're using the raw WikiText-2 (no tokens were replaced before +the tokenization). The loss here is that of causal language modeling. + +```bash +export TRAIN_FILE=/path/to/dataset/wiki.train.raw +export TEST_FILE=/path/to/dataset/wiki.test.raw + +python run_lm_finetuning.py \ + --output_dir=output \ + --model_type=gpt2 \ + --model_name_or_path=gpt2 \ + --do_train \ + --train_data_file=$TRAIN_FILE \ + --do_eval \ + --eval_data_file=$TEST_FILE +``` + +This takes about half an hour to train on a single K80 GPU and about one minute for the evaluation to run. It reaches +a score of ~20 perplexity once fine-tuned on the dataset. + +### RoBERTa/BERT and masked language modeling + +The following example fine-tunes RoBERTa on WikiText-2. Here too, we're using the raw WikiText-2. The loss is different +as BERT/RoBERTa have a bidirectional mechanism; we're therefore using the same loss that was used during their +pre-training: masked language modeling. + +In accordance to the RoBERTa paper, we use dynamic masking rather than static masking. The model may, therefore, converge +slightly slower (over-fitting takes more epochs). + +We use the `--mlm` flag so that the script may change its loss function. + +```bash +export TRAIN_FILE=/path/to/dataset/wiki.train.raw +export TEST_FILE=/path/to/dataset/wiki.test.raw + +python run_lm_finetuning.py \ + --output_dir=output \ + --model_type=roberta \ + --model_name_or_path=roberta-base \ + --do_train \ + --train_data_file=$TRAIN_FILE \ + --do_eval \ + --eval_data_file=$TEST_FILE \ + --mlm +``` + +## Language generation + +Based on the script [`run_generation.py`](https://github.com/huggingface/pytorch-transformers/blob/master/examples/run_generation.py). + +Conditional text generation using the auto-regressive models of the library: GPT, GPT-2, Transformer-XL and XLNet. +A similar script is used for our official demo [Write With Transfomer](https://transformer.huggingface.co), where you +can try out the different models available in the library. + +Example usage: + +```bash +python run_generation.py \ + --model_type=gpt2 \ + --model_name_or_path=gpt2 +``` + +## GLUE + +Based on the script [`run_glue.py`](https://github.com/huggingface/pytorch-transformers/blob/master/examples/run_glue.py). + +Fine-tuning the library models for sequence classification on the GLUE benchmark: [General Language Understanding +Evaluation](https://gluebenchmark.com/). This script can fine-tune the following models: BERT, XLM, XLNet and RoBERTa. + +GLUE is made up of a total of 9 different tasks. We get the following results on the dev set of the benchmark with an +uncased BERT base model (the checkpoint `bert-base-uncased`). All experiments ran on 8 V100 GPUs with a total train +batch size of 24. Some of these tasks have a small dataset and training can lead to high variance in the results +between different runs. We report the median on 5 runs (with different seeds) for each of the metrics. + +| Task | Metric | Result | +|-------|------------------------------|-------------| +| CoLA | Matthew's corr | 55.75 | +| SST-2 | Accuracy | 92.09 | +| MRPC | F1/Accuracy | 90.48/86.27 | +| STS-B | Person/Spearman corr. | 89.03/88.64 | +| QQP | Accuracy/F1 | 90.92/87.72 | +| MNLI | Matched acc./Mismatched acc. | 83.74/84.06 | +| QNLI | Accuracy | 91.07 | +| RTE | Accuracy | 68.59 | +| WNLI | Accuracy | 43.66 | + +Some of these results are significantly different from the ones reported on the test set +of GLUE benchmark on the website. For QQP and WNLI, please refer to [FAQ #12](https://gluebenchmark.com/faq) on the webite. + +Before running anyone of these GLUE tasks you should download the +[GLUE data](https://gluebenchmark.com/tasks) by running +[this script](https://gist.github.com/W4ngatang/60c2bdb54d156a41194446737ce03e2e) +and unpack it to some directory `$GLUE_DIR`. + +```bash +export GLUE_DIR=/path/to/glue +export TASK_NAME=MRPC + +python run_glue.py \ + --model_type bert \ + --model_name_or_path bert-base-cased \ + --task_name $TASK_NAME \ + --do_train \ + --do_eval \ + --do_lower_case \ + --data_dir $GLUE_DIR/$TASK_NAME \ + --max_seq_length 128 \ + --per_gpu_train_batch_size 32 \ + --learning_rate 2e-5 \ + --num_train_epochs 3.0 \ + --output_dir /tmp/$TASK_NAME/ +``` + +where task name can be one of CoLA, SST-2, MRPC, STS-B, QQP, MNLI, QNLI, RTE, WNLI. + +The dev set results will be present within the text file `eval_results.txt` in the specified output_dir. +In case of MNLI, since there are two separate dev sets (matched and mismatched), there will be a separate +output folder called `/tmp/MNLI-MM/` in addition to `/tmp/MNLI/`. + +The code has not been tested with half-precision training with apex on any GLUE task apart from MRPC, MNLI, +CoLA, SST-2. The following section provides details on how to run half-precision training with MRPC. With that being +said, there shouldn’t be any issues in running half-precision training with the remaining GLUE tasks as well, +since the data processor for each task inherits from the base class DataProcessor. + +### MRPC + +#### Fine-tuning example + +The following examples fine-tune BERT on the Microsoft Research Paraphrase Corpus (MRPC) corpus and runs in less +than 10 minutes on a single K-80 and in 27 seconds (!) on single tesla V100 16GB with apex installed. + +Before running anyone of these GLUE tasks you should download the +[GLUE data](https://gluebenchmark.com/tasks) by running +[this script](https://gist.github.com/W4ngatang/60c2bdb54d156a41194446737ce03e2e) +and unpack it to some directory `$GLUE_DIR`. + +```bash +export GLUE_DIR=/path/to/glue + +python run_glue.py \ + --model_type bert \ + --model_name_or_path bert-base-cased \ + --task_name MRPC \ + --do_train \ + --do_eval \ + --do_lower_case \ + --data_dir $GLUE_DIR/MRPC/ \ + --max_seq_length 128 \ + --per_gpu_train_batch_size 32 \ + --learning_rate 2e-5 \ + --num_train_epochs 3.0 \ + --output_dir /tmp/mrpc_output/ +``` + +Our test ran on a few seeds with [the original implementation hyper- +parameters](https://github.com/google-research/bert#sentence-and-sentence-pair-classification-tasks) gave evaluation +results between 84% and 88%. + +#### Using Apex and mixed-precision + +Using Apex and 16 bit precision, the fine-tuning on MRPC only takes 27 seconds. First install +[apex](https://github.com/NVIDIA/apex), then run the following example: + +```bash +export GLUE_DIR=/path/to/glue + +python run_glue.py \ + --model_type bert \ + --model_name_or_path bert-base-cased \ + --task_name MRPC \ + --do_train \ + --do_eval \ + --do_lower_case \ + --data_dir $GLUE_DIR/MRPC/ \ + --max_seq_length 128 \ + --per_gpu_train_batch_size 32 \ + --learning_rate 2e-5 \ + --num_train_epochs 3.0 \ + --output_dir /tmp/mrpc_output/ \ + --fp16 +``` + +#### Distributed training + +Here is an example using distributed training on 8 V100 GPUs. The model used is the BERT whole-word-masking and it +reaches F1 > 92 on MRPC. + +```bash +export GLUE_DIR=/path/to/glue + +python -m torch.distributed.launch \ + --nproc_per_node 8 run_glue.py \ + --model_type bert \ + --model_name_or_path bert-base-cased \ + --task_name MRPC \ + --do_train \ + --do_eval \ + --do_lower_case \ + --data_dir $GLUE_DIR/MRPC/ \ + --max_seq_length 128 \ + --per_gpu_train_batch_size 8 \ + --learning_rate 2e-5 \ + --num_train_epochs 3.0 \ + --output_dir /tmp/mrpc_output/ +``` + +Training with these hyper-parameters gave us the following results: + +```bash +acc = 0.8823529411764706 +acc_and_f1 = 0.901702786377709 +eval_loss = 0.3418912578906332 +f1 = 0.9210526315789473 +global_step = 174 +loss = 0.07231863956341798 +``` + +### MNLI + +The following example uses the BERT-large, uncased, whole-word-masking model and fine-tunes it on the MNLI task. + +```bash +export GLUE_DIR=/path/to/glue + +python -m torch.distributed.launch \ + --nproc_per_node 8 run_glue.py \ + --model_type bert \ + --model_name_or_path bert-base-cased \ + --task_name mnli \ + --do_train \ + --do_eval \ + --do_lower_case \ + --data_dir $GLUE_DIR/MNLI/ \ + --max_seq_length 128 \ + --per_gpu_train_batch_size 8 \ + --learning_rate 2e-5 \ + --num_train_epochs 3.0 \ + --output_dir output_dir \ +``` + +The results are the following: + +```bash +***** Eval results ***** + acc = 0.8679706601466992 + eval_loss = 0.4911287787382479 + global_step = 18408 + loss = 0.04755385363816904 + +***** Eval results ***** + acc = 0.8747965825874695 + eval_loss = 0.45516540421714036 + global_step = 18408 + loss = 0.04755385363816904 +``` + +##Multiple Choice + +Based on the script [`run_multiple_choice.py`](). + +#### Fine-tuning on SWAG +Download [swag](https://github.com/rowanz/swagaf/tree/master/data) data + +``` +#training on 4 tesla V100(16GB) GPUS +export SWAG_DIR=/path/to/swag_data_dir +python ./examples/single_model_scripts/run_multiple_choice.py \ +--model_type roberta \ +--task_name swag \ +--model_name_or_path roberta-base \ +--do_train \ +--do_eval \ +--do_lower_case \ +--data_dir $SWAG_DIR \ +--learning_rate 5e-5 \ +--num_train_epochs 3 \ +--max_seq_length 80 \ +--output_dir models_bert/swag_base \ +--per_gpu_eval_batch_size=16 \ +--per_gpu_train_batch_size=16 \ +--gradient_accumulation_steps 2 \ +--overwrite_output +``` +Training with the defined hyper-parameters yields the following results: +``` +***** Eval results ***** +eval_acc = 0.8338998300509847 +eval_loss = 0.44457291918821606 +``` + +## SQuAD + +Based on the script [`run_squad.py`](https://github.com/huggingface/pytorch-transformers/blob/master/examples/run_squad.py). + +#### Fine-tuning on SQuAD + +This example code fine-tunes BERT on the SQuAD dataset. It runs in 24 min (with BERT-base) or 68 min (with BERT-large) +on a single tesla V100 16GB. The data for SQuAD can be downloaded with the following links and should be saved in a +$SQUAD_DIR directory. + +* [train-v1.1.json](https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v1.1.json) +* [dev-v1.1.json](https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json) +* [evaluate-v1.1.py](https://github.com/allenai/bi-att-flow/blob/master/squad/evaluate-v1.1.py) + +```bash +export SQUAD_DIR=/path/to/SQUAD + +python run_squad.py \ + --model_type bert \ + --model_name_or_path bert-base-cased \ + --do_train \ + --do_eval \ + --do_lower_case \ + --train_file $SQUAD_DIR/train-v1.1.json \ + --predict_file $SQUAD_DIR/dev-v1.1.json \ + --per_gpu_train_batch_size 12 \ + --learning_rate 3e-5 \ + --num_train_epochs 2.0 \ + --max_seq_length 384 \ + --doc_stride 128 \ + --output_dir /tmp/debug_squad/ +``` + +Training with the previously defined hyper-parameters yields the following results: + +```bash +f1 = 88.52 +exact_match = 81.22 +``` + +#### Distributed training + + +Here is an example using distributed training on 8 V100 GPUs and Bert Whole Word Masking uncased model to reach a F1 > 93 on SQuAD: + +```bash +python -m torch.distributed.launch --nproc_per_node=8 run_squad.py \ + --model_type bert \ + --model_name_or_path bert-base-cased \ + --do_train \ + --do_eval \ + --do_lower_case \ + --train_file $SQUAD_DIR/train-v1.1.json \ + --predict_file $SQUAD_DIR/dev-v1.1.json \ + --learning_rate 3e-5 \ + --num_train_epochs 2 \ + --max_seq_length 384 \ + --doc_stride 128 \ + --output_dir ../models/wwm_uncased_finetuned_squad/ \ + --per_gpu_train_batch_size 24 \ + --gradient_accumulation_steps 12 +``` + +Training with the previously defined hyper-parameters yields the following results: + +```bash +f1 = 93.15 +exact_match = 86.91 +``` + +This fine-tuneds model is available as a checkpoint under the reference +`bert-large-uncased-whole-word-masking-finetuned-squad`. + diff --git a/examples/contrib/README.md b/examples/contrib/README.md new file mode 100644 index 0000000000..f2d0616e62 --- /dev/null +++ b/examples/contrib/README.md @@ -0,0 +1,5 @@ +# Community contributed examples + +This folder contains examples which are not actively maintained (mostly contributed by the community). + +Using these examples together with a recent version of the library usually requires to make small (sometimes big) adaptations to get the scripts working. diff --git a/examples/single_model_scripts/run_openai_gpt.py b/examples/contrib/run_openai_gpt.py similarity index 97% rename from examples/single_model_scripts/run_openai_gpt.py rename to examples/contrib/run_openai_gpt.py index 479c08782d..1c9fba8ee8 100644 --- a/examples/single_model_scripts/run_openai_gpt.py +++ b/examples/contrib/run_openai_gpt.py @@ -153,9 +153,11 @@ def main(): # This loading functions also add new tokens and embeddings called `special tokens` # These new embeddings will be fine-tuned on the RocStories dataset special_tokens = ['_start_', '_delimiter_', '_classify_'] - tokenizer = OpenAIGPTTokenizer.from_pretrained(args.model_name, special_tokens=special_tokens) - special_tokens_ids = list(tokenizer.convert_tokens_to_ids(token) for token in special_tokens) - model = OpenAIGPTDoubleHeadsModel.from_pretrained(args.model_name, num_special_tokens=len(special_tokens)) + tokenizer = OpenAIGPTTokenizer.from_pretrained(args.model_name) + tokenizer.add_tokens(special_tokens) + special_tokens_ids = tokenizer.convert_tokens_to_ids(special_tokens) + model = OpenAIGPTDoubleHeadsModel.from_pretrained(args.model_name) + model.resize_token_embeddings(len(tokenizer)) model.to(device) # Load and encode the datasets @@ -221,7 +223,7 @@ def main(): for step, batch in enumerate(tqdm_bar): batch = tuple(t.to(device) for t in batch) input_ids, mc_token_ids, lm_labels, mc_labels = batch - losses = model(input_ids, mc_token_ids, lm_labels, mc_labels) + losses = model(input_ids, mc_token_ids=mc_token_ids, lm_labels=lm_labels, mc_labels=mc_labels) loss = args.lm_coef * losses[0] + losses[1] loss.backward() scheduler.step() @@ -258,7 +260,7 @@ def main(): batch = tuple(t.to(device) for t in batch) input_ids, mc_token_ids, lm_labels, mc_labels = batch with torch.no_grad(): - _, mc_loss, _, mc_logits = model(input_ids, mc_token_ids, lm_labels, mc_labels) + _, mc_loss, _, mc_logits = model(input_ids, mc_token_ids=mc_token_ids, lm_labels=lm_labels, mc_labels=mc_labels) mc_logits = mc_logits.detach().cpu().numpy() mc_labels = mc_labels.to('cpu').numpy() diff --git a/examples/contrib/run_swag.py b/examples/contrib/run_swag.py new file mode 100644 index 0000000000..495f40cec9 --- /dev/null +++ b/examples/contrib/run_swag.py @@ -0,0 +1,673 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""BERT finetuning runner. + Finetuning the library models for multiple choice on SWAG (Bert). +""" +from __future__ import absolute_import, division, print_function + +import argparse +import logging +import csv +import os +import random +import sys +import glob + +import numpy as np +import torch +from torch.utils.data import (DataLoader, RandomSampler, SequentialSampler, + TensorDataset) +from torch.utils.data.distributed import DistributedSampler +from tqdm import tqdm, trange + +from tensorboardX import SummaryWriter + +from pytorch_transformers import (WEIGHTS_NAME, BertConfig, + BertForMultipleChoice, BertTokenizer) + +from pytorch_transformers import AdamW, WarmupLinearSchedule + +logger = logging.getLogger(__name__) + +ALL_MODELS = sum((tuple(conf.pretrained_config_archive_map.keys()) \ + for conf in [BertConfig]), ()) + +MODEL_CLASSES = { + 'bert': (BertConfig, BertForMultipleChoice, BertTokenizer), +} + +class SwagExample(object): + """A single training/test example for the SWAG dataset.""" + def __init__(self, + swag_id, + context_sentence, + start_ending, + ending_0, + ending_1, + ending_2, + ending_3, + label = None): + self.swag_id = swag_id + self.context_sentence = context_sentence + self.start_ending = start_ending + self.endings = [ + ending_0, + ending_1, + ending_2, + ending_3, + ] + self.label = label + + def __str__(self): + return self.__repr__() + + def __repr__(self): + l = [ + "swag_id: {}".format(self.swag_id), + "context_sentence: {}".format(self.context_sentence), + "start_ending: {}".format(self.start_ending), + "ending_0: {}".format(self.endings[0]), + "ending_1: {}".format(self.endings[1]), + "ending_2: {}".format(self.endings[2]), + "ending_3: {}".format(self.endings[3]), + ] + + if self.label is not None: + l.append("label: {}".format(self.label)) + + return ", ".join(l) + +class InputFeatures(object): + def __init__(self, + example_id, + choices_features, + label + + ): + self.example_id = example_id + self.choices_features = [ + { + 'input_ids': input_ids, + 'input_mask': input_mask, + 'segment_ids': segment_ids + } + for _, input_ids, input_mask, segment_ids in choices_features + ] + self.label = label + +def read_swag_examples(input_file, is_training=True): + with open(input_file, 'r', encoding='utf-8') as f: + reader = csv.reader(f) + lines = [] + for line in reader: + if sys.version_info[0] == 2: + line = list(unicode(cell, 'utf-8') for cell in line) + lines.append(line) + + if is_training and lines[0][-1] != 'label': + raise ValueError( + "For training, the input file must contain a label column." + ) + + examples = [ + SwagExample( + swag_id = line[2], + context_sentence = line[4], + start_ending = line[5], # in the swag dataset, the + # common beginning of each + # choice is stored in "sent2". + ending_0 = line[7], + ending_1 = line[8], + ending_2 = line[9], + ending_3 = line[10], + label = int(line[11]) if is_training else None + ) for line in lines[1:] # we skip the line with the column names + ] + + return examples + +def convert_examples_to_features(examples, tokenizer, max_seq_length, + is_training): + """Loads a data file into a list of `InputBatch`s.""" + + # Swag is a multiple choice task. To perform this task using Bert, + # we will use the formatting proposed in "Improving Language + # Understanding by Generative Pre-Training" and suggested by + # @jacobdevlin-google in this issue + # https://github.com/google-research/bert/issues/38. + # + # Each choice will correspond to a sample on which we run the + # inference. For a given Swag example, we will create the 4 + # following inputs: + # - [CLS] context [SEP] choice_1 [SEP] + # - [CLS] context [SEP] choice_2 [SEP] + # - [CLS] context [SEP] choice_3 [SEP] + # - [CLS] context [SEP] choice_4 [SEP] + # The model will output a single value for each input. To get the + # final decision of the model, we will run a softmax over these 4 + # outputs. + features = [] + for example_index, example in tqdm(enumerate(examples)): + context_tokens = tokenizer.tokenize(example.context_sentence) + start_ending_tokens = tokenizer.tokenize(example.start_ending) + + choices_features = [] + for ending_index, ending in enumerate(example.endings): + # We create a copy of the context tokens in order to be + # able to shrink it according to ending_tokens + context_tokens_choice = context_tokens[:] + ending_tokens = start_ending_tokens + tokenizer.tokenize(ending) + # Modifies `context_tokens_choice` and `ending_tokens` in + # place so that the total length is less than the + # specified length. Account for [CLS], [SEP], [SEP] with + # "- 3" + _truncate_seq_pair(context_tokens_choice, ending_tokens, max_seq_length - 3) + + tokens = ["[CLS]"] + context_tokens_choice + ["[SEP]"] + ending_tokens + ["[SEP]"] + segment_ids = [0] * (len(context_tokens_choice) + 2) + [1] * (len(ending_tokens) + 1) + + input_ids = tokenizer.convert_tokens_to_ids(tokens) + input_mask = [1] * len(input_ids) + + # Zero-pad up to the sequence length. + padding = [0] * (max_seq_length - len(input_ids)) + input_ids += padding + input_mask += padding + segment_ids += padding + + assert len(input_ids) == max_seq_length + assert len(input_mask) == max_seq_length + assert len(segment_ids) == max_seq_length + + choices_features.append((tokens, input_ids, input_mask, segment_ids)) + + label = example.label + if example_index < 5: + logger.info("*** Example ***") + logger.info("swag_id: {}".format(example.swag_id)) + for choice_idx, (tokens, input_ids, input_mask, segment_ids) in enumerate(choices_features): + logger.info("choice: {}".format(choice_idx)) + logger.info("tokens: {}".format(' '.join(tokens))) + logger.info("input_ids: {}".format(' '.join(map(str, input_ids)))) + logger.info("input_mask: {}".format(' '.join(map(str, input_mask)))) + logger.info("segment_ids: {}".format(' '.join(map(str, segment_ids)))) + if is_training: + logger.info("label: {}".format(label)) + + features.append( + InputFeatures( + example_id = example.swag_id, + choices_features = choices_features, + label = label + ) + ) + + return features + +def _truncate_seq_pair(tokens_a, tokens_b, max_length): + """Truncates a sequence pair in place to the maximum length.""" + + # This is a simple heuristic which will always truncate the longer sequence + # one token at a time. This makes more sense than truncating an equal percent + # of tokens from each, since if one sequence is very short then each token + # that's truncated likely contains more information than a longer sequence. + while True: + total_length = len(tokens_a) + len(tokens_b) + if total_length <= max_length: + break + if len(tokens_a) > len(tokens_b): + tokens_a.pop() + else: + tokens_b.pop() + +def accuracy(out, labels): + outputs = np.argmax(out, axis=1) + return np.sum(outputs == labels) + +def select_field(features, field): + return [ + [ + choice[field] + for choice in feature.choices_features + ] + for feature in features + ] + + +def set_seed(args): + random.seed(args.seed) + np.random.seed(args.seed) + torch.manual_seed(args.seed) + if args.n_gpu > 0: + torch.cuda.manual_seed_all(args.seed) + +def load_and_cache_examples(args, tokenizer, evaluate=False, output_examples=False): + if args.local_rank not in [-1, 0]: + torch.distributed.barrier() # Make sure only the first process in distributed training process the dataset, and the others will use the cache + + # Load data features from cache or dataset file + input_file = args.predict_file if evaluate else args.train_file + cached_features_file = os.path.join(os.path.dirname(input_file), 'cached_{}_{}_{}'.format( + 'dev' if evaluate else 'train', + list(filter(None, args.model_name_or_path.split('/'))).pop(), + str(args.max_seq_length))) + if os.path.exists(cached_features_file) and not args.overwrite_cache and not output_examples: + logger.info("Loading features from cached file %s", cached_features_file) + features = torch.load(cached_features_file) + else: + logger.info("Creating features from dataset file at %s", input_file) + examples = read_swag_examples(input_file) + features = convert_examples_to_features( + examples, tokenizer, args.max_seq_length, not evaluate) + + if args.local_rank in [-1, 0]: + logger.info("Saving features into cached file %s", cached_features_file) + torch.save(features, cached_features_file) + + if args.local_rank == 0: + torch.distributed.barrier() # Make sure only the first process in distributed training process the dataset, and the others will use the cache + + # Convert to Tensors and build dataset + all_input_ids = torch.tensor(select_field(features, 'input_ids'), dtype=torch.long) + all_input_mask = torch.tensor(select_field(features, 'input_mask'), dtype=torch.long) + all_segment_ids = torch.tensor(select_field(features, 'segment_ids'), dtype=torch.long) + all_label = torch.tensor([f.label for f in features], dtype=torch.long) + + if evaluate: + dataset = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, + all_label) + else: + dataset = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, + all_label) + + if output_examples: + return dataset, examples, features + return dataset +def train(args, train_dataset, model, tokenizer): + """ Train the model """ + if args.local_rank in [-1, 0]: + tb_writer = SummaryWriter() + + args.train_batch_size = args.per_gpu_train_batch_size * max(1, args.n_gpu) + train_sampler = RandomSampler(train_dataset) if args.local_rank == -1 else DistributedSampler(train_dataset) + train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=args.train_batch_size) + + if args.max_steps > 0: + t_total = args.max_steps + args.num_train_epochs = args.max_steps // (len(train_dataloader) // args.gradient_accumulation_steps) + 1 + else: + t_total = len(train_dataloader) // args.gradient_accumulation_steps * args.num_train_epochs + + # Prepare optimizer and schedule (linear warmup and decay) + no_decay = ['bias', 'LayerNorm.weight'] + optimizer_grouped_parameters = [ + {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)], 'weight_decay': args.weight_decay}, + {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0} + ] + optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon) + scheduler = WarmupLinearSchedule(optimizer, warmup_steps=args.warmup_steps, t_total=t_total) + if args.fp16: + try: + from apex import amp + except ImportError: + raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use fp16 training.") + model, optimizer = amp.initialize(model, optimizer, opt_level=args.fp16_opt_level) + + # multi-gpu training (should be after apex fp16 initialization) + if args.n_gpu > 1: + model = torch.nn.DataParallel(model) + + # Distributed training (should be after apex fp16 initialization) + if args.local_rank != -1: + model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.local_rank], + output_device=args.local_rank, + find_unused_parameters=True) + + # Train! + logger.info("***** Running training *****") + logger.info(" Num examples = %d", len(train_dataset)) + logger.info(" Num Epochs = %d", args.num_train_epochs) + logger.info(" Instantaneous batch size per GPU = %d", args.per_gpu_train_batch_size) + logger.info(" Total train batch size (w. parallel, distributed & accumulation) = %d", + args.train_batch_size * args.gradient_accumulation_steps * (torch.distributed.get_world_size() if args.local_rank != -1 else 1)) + logger.info(" Gradient Accumulation steps = %d", args.gradient_accumulation_steps) + logger.info(" Total optimization steps = %d", t_total) + + global_step = 0 + tr_loss, logging_loss = 0.0, 0.0 + model.zero_grad() + train_iterator = trange(int(args.num_train_epochs), desc="Epoch", disable=args.local_rank not in [-1, 0]) + set_seed(args) # Added here for reproductibility (even between python 2 and 3) + for _ in train_iterator: + epoch_iterator = tqdm(train_dataloader, desc="Iteration", disable=args.local_rank not in [-1, 0]) + for step, batch in enumerate(epoch_iterator): + model.train() + batch = tuple(t.to(args.device) for t in batch) + inputs = {'input_ids': batch[0], + 'attention_mask': batch[1], + #'token_type_ids': None if args.model_type == 'xlm' else batch[2], + 'token_type_ids': batch[2], + 'labels': batch[3]} + # if args.model_type in ['xlnet', 'xlm']: + # inputs.update({'cls_index': batch[5], + # 'p_mask': batch[6]}) + outputs = model(**inputs) + loss = outputs[0] # model outputs are always tuple in pytorch-transformers (see doc) + + if args.n_gpu > 1: + loss = loss.mean() # mean() to average on multi-gpu parallel (not distributed) training + if args.gradient_accumulation_steps > 1: + loss = loss / args.gradient_accumulation_steps + + if args.fp16: + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args.max_grad_norm) + else: + loss.backward() + torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm) + + tr_loss += loss.item() + if (step + 1) % args.gradient_accumulation_steps == 0: + optimizer.step() + scheduler.step() # Update learning rate schedule + model.zero_grad() + global_step += 1 + + if args.local_rank in [-1, 0] and args.logging_steps > 0 and global_step % args.logging_steps == 0: + # Log metrics + if args.local_rank == -1 and args.evaluate_during_training: # Only evaluate when single GPU otherwise metrics may not average well + results = evaluate(args, model, tokenizer) + for key, value in results.items(): + tb_writer.add_scalar('eval_{}'.format(key), value, global_step) + tb_writer.add_scalar('lr', scheduler.get_lr()[0], global_step) + tb_writer.add_scalar('loss', (tr_loss - logging_loss)/args.logging_steps, global_step) + logging_loss = tr_loss + + if args.local_rank in [-1, 0] and args.save_steps > 0 and global_step % args.save_steps == 0: + # Save model checkpoint + output_dir = os.path.join(args.output_dir, 'checkpoint-{}'.format(global_step)) + if not os.path.exists(output_dir): + os.makedirs(output_dir) + model_to_save = model.module if hasattr(model, 'module') else model # Take care of distributed/parallel training + model_to_save.save_pretrained(output_dir) + tokenizer.save_vocabulary(output_dir) + torch.save(args, os.path.join(output_dir, 'training_args.bin')) + logger.info("Saving model checkpoint to %s", output_dir) + + if args.max_steps > 0 and global_step > args.max_steps: + epoch_iterator.close() + break + if args.max_steps > 0 and global_step > args.max_steps: + train_iterator.close() + break + + if args.local_rank in [-1, 0]: + tb_writer.close() + + return global_step, tr_loss / global_step + +def evaluate(args, model, tokenizer, prefix=""): + dataset, examples, features = load_and_cache_examples(args, tokenizer, evaluate=True, output_examples=True) + + if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]: + os.makedirs(args.output_dir) + + args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu) + # Note that DistributedSampler samples randomly + eval_sampler = SequentialSampler(dataset) if args.local_rank == -1 else DistributedSampler(dataset) + eval_dataloader = DataLoader(dataset, sampler=eval_sampler, batch_size=args.eval_batch_size) + + # Eval! + logger.info("***** Running evaluation {} *****".format(prefix)) + logger.info(" Num examples = %d", len(dataset)) + logger.info(" Batch size = %d", args.eval_batch_size) + + + eval_loss, eval_accuracy = 0, 0 + nb_eval_steps, nb_eval_examples = 0, 0 + + for batch in tqdm(eval_dataloader, desc="Evaluating"): + model.eval() + batch = tuple(t.to(args.device) for t in batch) + with torch.no_grad(): + inputs = {'input_ids': batch[0], + 'attention_mask': batch[1], + # 'token_type_ids': None if args.model_type == 'xlm' else batch[2] # XLM don't use segment_ids + 'token_type_ids': batch[2], + 'labels': batch[3]} + + # if args.model_type in ['xlnet', 'xlm']: + # inputs.update({'cls_index': batch[4], + # 'p_mask': batch[5]}) + outputs = model(**inputs) + tmp_eval_loss, logits = outputs[:2] + eval_loss += tmp_eval_loss.mean().item() + + logits = logits.detach().cpu().numpy() + label_ids = inputs['labels'].to('cpu').numpy() + tmp_eval_accuracy = accuracy(logits, label_ids) + eval_accuracy += tmp_eval_accuracy + + nb_eval_steps += 1 + nb_eval_examples += inputs['input_ids'].size(0) + + eval_loss = eval_loss / nb_eval_steps + eval_accuracy = eval_accuracy / nb_eval_examples + result = {'eval_loss': eval_loss, + 'eval_accuracy': eval_accuracy} + + output_eval_file = os.path.join(args.output_dir, "eval_results.txt") + with open(output_eval_file, "w") as writer: + logger.info("***** Eval results *****") + for key in sorted(result.keys()): + logger.info("%s = %s", key, str(result[key])) + writer.write("%s = %s\n" % (key, str(result[key]))) + + return result + +def main(): + parser = argparse.ArgumentParser() + + ## Required parameters + parser.add_argument("--train_file", default=None, type=str, required=True, + help="SWAG csv for training. E.g., train.csv") + parser.add_argument("--predict_file", default=None, type=str, required=True, + help="SWAG csv for predictions. E.g., val.csv or test.csv") + parser.add_argument("--model_type", default=None, type=str, required=True, + help="Model type selected in the list: " + ", ".join(MODEL_CLASSES.keys())) + parser.add_argument("--model_name_or_path", default=None, type=str, required=True, + help="Path to pre-trained model or shortcut name selected in the list: " + ", ".join(ALL_MODELS)) + parser.add_argument("--output_dir", default=None, type=str, required=True, + help="The output directory where the model checkpoints and predictions will be written.") + + ## Other parameters + parser.add_argument("--config_name", default="", type=str, + help="Pretrained config name or path if not the same as model_name") + parser.add_argument("--tokenizer_name", default="", type=str, + help="Pretrained tokenizer name or path if not the same as model_name") + parser.add_argument("--max_seq_length", default=384, type=int, + help="The maximum total input sequence length after tokenization. Sequences " + "longer than this will be truncated, and sequences shorter than this will be padded.") + parser.add_argument("--do_train", action='store_true', + help="Whether to run training.") + parser.add_argument("--do_eval", action='store_true', + help="Whether to run eval on the dev set.") + parser.add_argument("--evaluate_during_training", action='store_true', + help="Rul evaluation during training at each logging step.") + parser.add_argument("--do_lower_case", action='store_true', + help="Set this flag if you are using an uncased model.") + + parser.add_argument("--per_gpu_train_batch_size", default=8, type=int, + help="Batch size per GPU/CPU for training.") + parser.add_argument("--per_gpu_eval_batch_size", default=8, type=int, + help="Batch size per GPU/CPU for evaluation.") + parser.add_argument("--learning_rate", default=5e-5, type=float, + help="The initial learning rate for Adam.") + parser.add_argument('--gradient_accumulation_steps', type=int, default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.") + parser.add_argument("--weight_decay", default=0.0, type=float, + help="Weight deay if we apply some.") + parser.add_argument("--adam_epsilon", default=1e-8, type=float, + help="Epsilon for Adam optimizer.") + parser.add_argument("--max_grad_norm", default=1.0, type=float, + help="Max gradient norm.") + parser.add_argument("--num_train_epochs", default=3.0, type=float, + help="Total number of training epochs to perform.") + parser.add_argument("--max_steps", default=-1, type=int, + help="If > 0: set total number of training steps to perform. Override num_train_epochs.") + parser.add_argument("--warmup_steps", default=0, type=int, + help="Linear warmup over warmup_steps.") + + parser.add_argument('--logging_steps', type=int, default=50, + help="Log every X updates steps.") + parser.add_argument('--save_steps', type=int, default=50, + help="Save checkpoint every X updates steps.") + parser.add_argument("--eval_all_checkpoints", action='store_true', + help="Evaluate all checkpoints starting with the same prefix as model_name ending and ending with step number") + parser.add_argument("--no_cuda", action='store_true', + help="Whether not to use CUDA when available") + parser.add_argument('--overwrite_output_dir', action='store_true', + help="Overwrite the content of the output directory") + parser.add_argument('--overwrite_cache', action='store_true', + help="Overwrite the cached training and evaluation sets") + parser.add_argument('--seed', type=int, default=42, + help="random seed for initialization") + + parser.add_argument("--local_rank", type=int, default=-1, + help="local_rank for distributed training on gpus") + parser.add_argument('--fp16', action='store_true', + help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit") + parser.add_argument('--fp16_opt_level', type=str, default='O1', + help="For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']." + "See details at https://nvidia.github.io/apex/amp.html") + parser.add_argument('--server_ip', type=str, default='', help="Can be used for distant debugging.") + parser.add_argument('--server_port', type=str, default='', help="Can be used for distant debugging.") + args = parser.parse_args() + + if os.path.exists(args.output_dir) and os.listdir(args.output_dir) and args.do_train and not args.overwrite_output_dir: + raise ValueError("Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome.".format(args.output_dir)) + + # Setup distant debugging if needed + if args.server_ip and args.server_port: + # Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script + import ptvsd + print("Waiting for debugger attach") + ptvsd.enable_attach(address=(args.server_ip, args.server_port), redirect_output=True) + ptvsd.wait_for_attach() + + # Setup CUDA, GPU & distributed training + if args.local_rank == -1 or args.no_cuda: + device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu") + args.n_gpu = torch.cuda.device_count() + else: # Initializes the distributed backend which will take care of sychronizing nodes/GPUs + torch.cuda.set_device(args.local_rank) + device = torch.device("cuda", args.local_rank) + torch.distributed.init_process_group(backend='nccl') + args.n_gpu = 1 + args.device = device + + # Setup logging + logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', + datefmt = '%m/%d/%Y %H:%M:%S', + level = logging.INFO if args.local_rank in [-1, 0] else logging.WARN) + logger.warning("Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s", + args.local_rank, device, args.n_gpu, bool(args.local_rank != -1), args.fp16) + + # Set seed + set_seed(args) + + # Load pretrained model and tokenizer + if args.local_rank not in [-1, 0]: + torch.distributed.barrier() # Make sure only the first process in distributed training will download model & vocab + + args.model_type = args.model_type.lower() + config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type] + config = config_class.from_pretrained(args.config_name if args.config_name else args.model_name_or_path) + tokenizer = tokenizer_class.from_pretrained(args.tokenizer_name if args.tokenizer_name else args.model_name_or_path, do_lower_case=args.do_lower_case) + model = model_class.from_pretrained(args.model_name_or_path, from_tf=bool('.ckpt' in args.model_name_or_path), config=config) + + if args.local_rank == 0: + torch.distributed.barrier() # Make sure only the first process in distributed training will download model & vocab + + model.to(args.device) + + logger.info("Training/evaluation parameters %s", args) + + # Training + if args.do_train: + train_dataset = load_and_cache_examples(args, tokenizer, evaluate=False, output_examples=False) + global_step, tr_loss = train(args, train_dataset, model, tokenizer) + logger.info(" global_step = %s, average loss = %s", global_step, tr_loss) + + + # Save the trained model and the tokenizer + if args.local_rank == -1 or torch.distributed.get_rank() == 0: + # Create output directory if needed + if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]: + os.makedirs(args.output_dir) + + logger.info("Saving model checkpoint to %s", args.output_dir) + # Save a trained model, configuration and tokenizer using `save_pretrained()`. + # They can then be reloaded using `from_pretrained()` + model_to_save = model.module if hasattr(model, 'module') else model # Take care of distributed/parallel training + model_to_save.save_pretrained(args.output_dir) + tokenizer.save_pretrained(args.output_dir) + + # Good practice: save your training arguments together with the trained model + torch.save(args, os.path.join(args.output_dir, 'training_args.bin')) + + # Load a trained model and vocabulary that you have fine-tuned + model = model_class.from_pretrained(args.output_dir) + tokenizer = tokenizer_class.from_pretrained(args.output_dir) + model.to(args.device) + + + # Evaluation - we can ask to evaluate all the checkpoints (sub-directories) in a directory + results = {} + if args.do_eval and args.local_rank in [-1, 0]: + if args.do_train: + checkpoints = [args.output_dir] + else: + # if do_train is False and do_eval is true, load model directly from pretrained. + checkpoints = [args.model_name_or_path] + + if args.eval_all_checkpoints: + checkpoints = list(os.path.dirname(c) for c in sorted(glob.glob(args.output_dir + '/**/' + WEIGHTS_NAME, recursive=True))) + logging.getLogger("pytorch_transformers.modeling_utils").setLevel(logging.WARN) # Reduce model loading logs + + logger.info("Evaluate the following checkpoints: %s", checkpoints) + + for checkpoint in checkpoints: + # Reload the model + global_step = checkpoint.split('-')[-1] if len(checkpoints) > 1 else "" + model = model_class.from_pretrained(checkpoint) + tokenizer = tokenizer_class.from_pretrained(checkpoint) + model.to(args.device) + + # Evaluate + result = evaluate(args, model, tokenizer, prefix=global_step) + + result = dict((k + ('_{}'.format(global_step) if global_step else ''), v) for k, v in result.items()) + results.update(result) + + logger.info("Results: {}".format(results)) + + return results + + +if __name__ == "__main__": + main() diff --git a/examples/single_model_scripts/run_transfo_xl.py b/examples/contrib/run_transfo_xl.py similarity index 99% rename from examples/single_model_scripts/run_transfo_xl.py rename to examples/contrib/run_transfo_xl.py index 95efbb8855..4c99777b98 100644 --- a/examples/single_model_scripts/run_transfo_xl.py +++ b/examples/contrib/run_transfo_xl.py @@ -113,7 +113,7 @@ def main(): with torch.no_grad(): mems = None for idx, (data, target, seq_len) in enumerate(eval_iter): - ret = model(data, target, mems) + ret = model(data, lm_labels=target, mems=mems) loss, _, mems = ret loss = loss.mean() total_loss += seq_len * loss.item() diff --git a/examples/distillation/README.md b/examples/distillation/README.md index bb919385f1..73e0cc0655 100644 --- a/examples/distillation/README.md +++ b/examples/distillation/README.md @@ -9,6 +9,12 @@ DistilBERT stands for Distillated-BERT. DistilBERT is a small, fast, cheap and l For more information on DistilBERT, please refer to our [detailed blog post](https://medium.com/huggingface/smaller-faster-cheaper-lighter-introducing-distilbert-a-distilled-version-of-bert-8cf3380435b5 ). +## Setup + +This part of the library has only be tested with Python3.6+. There are few specific dependencies to install before launching a distillation, you can install them with the command `pip install -r requirements.txt`. + +**Important note:** The training scripts have been updated to support PyTorch v1.2.0 (there are breakings changes compared to v1.1.0). It is important to note that there is a small internal bug in the current version of PyTorch available on pip that causes a memory leak in our training/distillation. It has been recently fixed and will likely be integrated into the next release. For the moment, we recommend to [compile PyTorch from source](https://github.com/pytorch/pytorch#from-source). Please refer to [issue 1179](https://github.com/huggingface/pytorch-transformers/issues/1179) for more details. + ## How to use DistilBERT PyTorch-Transformers includes two pre-trained DistilBERT models, currently only provided for English (we are investigating the possibility to train and release a multilingual version of DistilBERT): diff --git a/examples/distillation/dataset.py b/examples/distillation/dataset.py index cdc16b94f3..89e3f1187f 100644 --- a/examples/distillation/dataset.py +++ b/examples/distillation/dataset.py @@ -77,7 +77,7 @@ class Dataset: if sub_s[0] != cls_id: sub_s = np.insert(sub_s, 0, cls_id) if sub_s[-1] != sep_id: - sub_s = np.insert(sub_s, len(sub_s), cls_id) + sub_s = np.insert(sub_s, len(sub_s), sep_id) assert len(sub_s) <= max_len sub_seqs.append(sub_s) diff --git a/examples/distillation/distiller.py b/examples/distillation/distiller.py index 38769c4b0e..93135e292c 100644 --- a/examples/distillation/distiller.py +++ b/examples/distillation/distiller.py @@ -17,6 +17,7 @@ """ import os import math +import psutil from tensorboardX import SummaryWriter from tqdm import trange, tqdm import numpy as np @@ -192,7 +193,7 @@ class Distiller: x_prob = self.token_probs[token_ids.flatten()] n_tgt = math.ceil(self.mlm_mask_prop * lengths.sum().item()) tgt_ids = torch.multinomial(x_prob / x_prob.sum(), n_tgt, replacement=False) - pred_mask = torch.zeros(bs * max_seq_len, dtype=torch.uint8, device=token_ids.device) + pred_mask = torch.zeros(bs * max_seq_len, dtype=torch.bool, device=token_ids.device) # previously `dtype=torch.uint8`, cf pytorch 1.2.0 compatibility pred_mask[tgt_ids] = 1 pred_mask = pred_mask.view(bs, max_seq_len) @@ -216,7 +217,7 @@ class Distiller: _token_ids = _token_ids_mask * (probs == 0).long() + _token_ids_real * (probs == 1).long() + _token_ids_rand * (probs == 2).long() token_ids = token_ids.masked_scatter(pred_mask, _token_ids) - mlm_labels[1-pred_mask] = -1 + mlm_labels[~pred_mask] = -1 # previously `mlm_labels[1-pred_mask] = -1`, cf pytorch 1.2.0 compatibility return token_ids, attn_mask, mlm_labels @@ -294,7 +295,10 @@ class Distiller: if self.is_master: logger.info(f'--- Ending epoch {self.epoch}/{self.params.n_epoch-1}') self.end_epoch() - if self.is_master: logger.info('Training is finished') + if self.is_master: + logger.info(f'Save very last checkpoint as `pytorch_model.bin`.') + self.save_checkpoint(checkpoint_name=f'pytorch_model.bin') + logger.info('Training is finished') def step(self, input_ids: torch.tensor, @@ -379,9 +383,9 @@ class Distiller: torch.nn.utils.clip_grad_norm_(amp.master_params(self.optimizer), self.params.max_grad_norm) else: torch.nn.utils.clip_grad_norm_(self.student.parameters(), self.params.max_grad_norm) - self.scheduler.step() self.optimizer.step() self.optimizer.zero_grad() + self.scheduler.step() def iter(self): """ @@ -418,6 +422,8 @@ class Distiller: if self.alpha_mse > 0.: self.tensorboard.add_scalar(tag="losses/loss_mse", scalar_value=self.last_loss_mse, global_step=self.n_total_iter) self.tensorboard.add_scalar(tag="learning_rate/lr", scalar_value=self.scheduler.get_lr()[0], global_step=self.n_total_iter) + + self.tensorboard.add_scalar(tag="global/memory_usage", scalar_value=psutil.virtual_memory()._asdict()['used']/1_000_000, global_step=self.n_total_iter) def end_epoch(self): """ diff --git a/examples/distillation/requirements.txt b/examples/distillation/requirements.txt index efb369dc43..18146239eb 100644 --- a/examples/distillation/requirements.txt +++ b/examples/distillation/requirements.txt @@ -1 +1,4 @@ gitpython==3.0.2 +tensorboard>=1.14.0 +tensorboardX==1.8 +psutil==5.6.3 diff --git a/examples/distillation/scripts/binarized_data.py b/examples/distillation/scripts/binarized_data.py index 792a5692e4..51be8fd0be 100644 --- a/examples/distillation/scripts/binarized_data.py +++ b/examples/distillation/scripts/binarized_data.py @@ -21,8 +21,12 @@ import random import time import numpy as np from pytorch_transformers import BertTokenizer +import logging -from examples.distillation.utils import logger +logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', + datefmt = '%m/%d/%Y %H:%M:%S', + level = logging.INFO) +logger = logging.getLogger(__name__) def main(): parser = argparse.ArgumentParser(description="Preprocess the data to avoid re-doing it several times by (tokenization + token_to_ids).") @@ -74,4 +78,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/examples/distillation/scripts/token_counts.py b/examples/distillation/scripts/token_counts.py index d791c66be3..a484a6f51b 100644 --- a/examples/distillation/scripts/token_counts.py +++ b/examples/distillation/scripts/token_counts.py @@ -18,8 +18,12 @@ Preprocessing script before training DistilBERT. from collections import Counter import argparse import pickle +import logging -from examples.distillation.utils import logger +logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', + datefmt = '%m/%d/%Y %H:%M:%S', + level = logging.INFO) +logger = logging.getLogger(__name__) if __name__ == '__main__': parser = argparse.ArgumentParser(description="Token Counts for smoothing the masking probabilities in MLM (cf XLM/word2vec)") diff --git a/examples/lm_finetuning/README.md b/examples/lm_finetuning/README.md deleted file mode 100644 index 0f3e186745..0000000000 --- a/examples/lm_finetuning/README.md +++ /dev/null @@ -1,64 +0,0 @@ -# BERT Model Finetuning using Masked Language Modeling objective - -## Introduction - -The three example scripts in this folder can be used to **fine-tune** a pre-trained BERT model using the pretraining objective (combination of masked language modeling and next sentence prediction loss). In general, pretrained models like BERT are first trained with a pretraining objective (masked language modeling and next sentence prediction for BERT) on a large and general natural language corpus. A classifier head is then added on top of the pre-trained architecture and the model is quickly fine-tuned on a target task, while still (hopefully) retaining its general language understanding. This greatly reduces overfitting and yields state-of-the-art results, especially when training data for the target task are limited. - -The [ULMFiT paper](https://arxiv.org/abs/1801.06146) took a slightly different approach, however, and added an intermediate step in which the model is fine-tuned on text **from the same domain as the target task and using the pretraining objective** before the final stage in which the classifier head is added and the model is trained on the target task itself. This paper reported significantly improved results from this step, and found that they could get high-quality classifications even with only tiny numbers (<1000) of labelled training examples, as long as they had a lot of unlabelled data from the target domain. - -Although this wasn't covered in the original BERT paper, domain-specific fine-tuning of Transformer models has [recently been reported by other authors](https://arxiv.org/pdf/1905.05583.pdf), and they report performance improvements as well. - -## Input format - -The scripts in this folder expect a single file as input, consisting of untokenized text, with one **sentence** per line, and one blank line between documents. The reason for the sentence splitting is that part of BERT's training involves a _next sentence_ objective in which the model must predict whether two sequences of text are contiguous text from the same document or not, and to avoid making the task _too easy_, the split point between the sequences is always at the end of a sentence. The linebreaks in the file are therefore necessary to mark the points where the text can be split. - -## Usage - -There are two ways to fine-tune a language model using these scripts. The first _quick_ approach is to use [`simple_lm_finetuning.py`](./simple_lm_finetuning.py). This script does everything in a single script, but generates training instances that consist of just two sentences. This is quite different from the BERT paper, where (confusingly) the NextSentence task concatenated sentences together from each document to form two long multi-sentences, which the paper just referred to as _sentences_. The difference between this simple approach and the original paper approach can have a significant effect for long sequences since two sentences will be much shorter than the max sequence length. In this case, most of each training example will just consist of blank padding characters, which wastes a lot of computation and results in a model that isn't really training on long sequences. - -As such, the preferred approach (assuming you have documents containing multiple contiguous sentences from your target domain) is to use [`pregenerate_training_data.py`](./pregenerate_training_data.py) to pre-process your data into training examples following the methodology used for LM training in the original BERT paper and repository. Since there is a significant random component to training data generation for BERT, this script includes an option to generate multiple _epochs_ of pre-processed data, to avoid training on the same random splits each epoch. Generating an epoch of data for each training epoch should result a better final model, and so we recommend doing so. - -You can then train on the pregenerated data using [`finetune_on_pregenerated.py`](./finetune_on_pregenerated.py), and pointing it to the folder created by [`pregenerate_training_data.py`](./pregenerate_training_data.py). Note that you should use the same `bert_model` and case options for both! Also note that `max_seq_len` does not need to be specified for the [`finetune_on_pregenerated.py`](./finetune_on_pregenerated.py) script, as it is inferred from the training examples. - -There are various options that can be tweaked, but they are mostly set to the values from the BERT paper/repository and default values should make sense. The most relevant ones are: - -- `--max_seq_len`: Controls the length of training examples (in wordpiece tokens) seen by the model. Defaults to 128 but can be set as high as 512. Higher values may yield stronger language models at the cost of slower and more memory-intensive training. -- `--fp16`: Enables fast half-precision training on recent GPUs. - -In addition, if memory usage is an issue, especially when training on a single GPU, reducing `--train_batch_size` from the default 32 to a lower number (4-16) can be helpful, or leaving `--train_batch_size` at the default and increasing `--gradient_accumulation_steps` to 2-8. Changing `--gradient_accumulation_steps` may be preferable as alterations to the batch size may require corresponding changes in the learning rate to compensate. There is also a `--reduce_memory` option for both the `pregenerate_training_data.py` and `finetune_on_pregenerated.py` scripts that spills data to disc in shelf objects or numpy memmaps rather than retaining it in memory, which significantly reduces memory usage with little performance impact. - -## Examples - -### Simple fine-tuning - -``` -python3 simple_lm_finetuning.py ---train_corpus my_corpus.txt ---bert_model bert-base-uncased ---do_lower_case ---output_dir finetuned_lm/ ---do_train -``` - -### Pregenerating training data - -``` -python3 pregenerate_training_data.py ---train_corpus my_corpus.txt ---bert_model bert-base-uncased ---do_lower_case ---output_dir training/ ---epochs_to_generate 3 ---max_seq_len 256 -``` - -### Training on pregenerated data - -``` -python3 finetune_on_pregenerated.py ---pregenerated_data training/ ---bert_model bert-base-uncased ---do_lower_case ---output_dir finetuned_lm/ ---epochs 3 -``` diff --git a/examples/lm_finetuning/finetune_on_pregenerated.py b/examples/lm_finetuning/finetune_on_pregenerated.py deleted file mode 100644 index 10721c7dcd..0000000000 --- a/examples/lm_finetuning/finetune_on_pregenerated.py +++ /dev/null @@ -1,346 +0,0 @@ -from argparse import ArgumentParser -from pathlib import Path -import os -import torch -import logging -import json -import random -import numpy as np -from collections import namedtuple -from tempfile import TemporaryDirectory - -from torch.utils.data import DataLoader, Dataset, RandomSampler -from torch.utils.data.distributed import DistributedSampler -from tqdm import tqdm - -from pytorch_transformers import WEIGHTS_NAME, CONFIG_NAME -from pytorch_transformers.modeling_bert import BertForPreTraining -from pytorch_transformers.tokenization_bert import BertTokenizer -from pytorch_transformers.optimization import AdamW, WarmupLinearSchedule - -InputFeatures = namedtuple("InputFeatures", "input_ids input_mask segment_ids lm_label_ids is_next") - -log_format = '%(asctime)-10s: %(message)s' -logging.basicConfig(level=logging.INFO, format=log_format) - - -def convert_example_to_features(example, tokenizer, max_seq_length): - tokens = example["tokens"] - segment_ids = example["segment_ids"] - is_random_next = example["is_random_next"] - masked_lm_positions = example["masked_lm_positions"] - masked_lm_labels = example["masked_lm_labels"] - - assert len(tokens) == len(segment_ids) <= max_seq_length # The preprocessed data should be already truncated - input_ids = tokenizer.convert_tokens_to_ids(tokens) - masked_label_ids = tokenizer.convert_tokens_to_ids(masked_lm_labels) - - input_array = np.zeros(max_seq_length, dtype=np.int) - input_array[:len(input_ids)] = input_ids - - mask_array = np.zeros(max_seq_length, dtype=np.bool) - mask_array[:len(input_ids)] = 1 - - segment_array = np.zeros(max_seq_length, dtype=np.bool) - segment_array[:len(segment_ids)] = segment_ids - - lm_label_array = np.full(max_seq_length, dtype=np.int, fill_value=-1) - lm_label_array[masked_lm_positions] = masked_label_ids - - features = InputFeatures(input_ids=input_array, - input_mask=mask_array, - segment_ids=segment_array, - lm_label_ids=lm_label_array, - is_next=is_random_next) - return features - - -class PregeneratedDataset(Dataset): - def __init__(self, training_path, epoch, tokenizer, num_data_epochs, reduce_memory=False): - self.vocab = tokenizer.vocab - self.tokenizer = tokenizer - self.epoch = epoch - self.data_epoch = epoch % num_data_epochs - data_file = training_path / f"epoch_{self.data_epoch}.json" - metrics_file = training_path / f"epoch_{self.data_epoch}_metrics.json" - assert data_file.is_file() and metrics_file.is_file() - metrics = json.loads(metrics_file.read_text()) - num_samples = metrics['num_training_examples'] - seq_len = metrics['max_seq_len'] - self.temp_dir = None - self.working_dir = None - if reduce_memory: - self.temp_dir = TemporaryDirectory() - self.working_dir = Path(self.temp_dir.name) - input_ids = np.memmap(filename=self.working_dir/'input_ids.memmap', - mode='w+', dtype=np.int32, shape=(num_samples, seq_len)) - input_masks = np.memmap(filename=self.working_dir/'input_masks.memmap', - shape=(num_samples, seq_len), mode='w+', dtype=np.bool) - segment_ids = np.memmap(filename=self.working_dir/'segment_ids.memmap', - shape=(num_samples, seq_len), mode='w+', dtype=np.bool) - lm_label_ids = np.memmap(filename=self.working_dir/'lm_label_ids.memmap', - shape=(num_samples, seq_len), mode='w+', dtype=np.int32) - lm_label_ids[:] = -1 - is_nexts = np.memmap(filename=self.working_dir/'is_nexts.memmap', - shape=(num_samples,), mode='w+', dtype=np.bool) - else: - input_ids = np.zeros(shape=(num_samples, seq_len), dtype=np.int32) - input_masks = np.zeros(shape=(num_samples, seq_len), dtype=np.bool) - segment_ids = np.zeros(shape=(num_samples, seq_len), dtype=np.bool) - lm_label_ids = np.full(shape=(num_samples, seq_len), dtype=np.int32, fill_value=-1) - is_nexts = np.zeros(shape=(num_samples,), dtype=np.bool) - logging.info(f"Loading training examples for epoch {epoch}") - with data_file.open() as f: - for i, line in enumerate(tqdm(f, total=num_samples, desc="Training examples")): - line = line.strip() - example = json.loads(line) - features = convert_example_to_features(example, tokenizer, seq_len) - input_ids[i] = features.input_ids - segment_ids[i] = features.segment_ids - input_masks[i] = features.input_mask - lm_label_ids[i] = features.lm_label_ids - is_nexts[i] = features.is_next - assert i == num_samples - 1 # Assert that the sample count metric was true - logging.info("Loading complete!") - self.num_samples = num_samples - self.seq_len = seq_len - self.input_ids = input_ids - self.input_masks = input_masks - self.segment_ids = segment_ids - self.lm_label_ids = lm_label_ids - self.is_nexts = is_nexts - - def __len__(self): - return self.num_samples - - def __getitem__(self, item): - return (torch.tensor(self.input_ids[item].astype(np.int64)), - torch.tensor(self.input_masks[item].astype(np.int64)), - torch.tensor(self.segment_ids[item].astype(np.int64)), - torch.tensor(self.lm_label_ids[item].astype(np.int64)), - torch.tensor(self.is_nexts[item].astype(np.int64))) - - -def main(): - parser = ArgumentParser() - parser.add_argument('--pregenerated_data', type=Path, required=True) - parser.add_argument('--output_dir', type=Path, required=True) - parser.add_argument("--bert_model", type=str, required=True, help="Bert pre-trained model selected in the list: bert-base-uncased, " - "bert-large-uncased, bert-base-cased, bert-base-multilingual, bert-base-chinese.") - parser.add_argument("--do_lower_case", action="store_true") - parser.add_argument("--reduce_memory", action="store_true", - help="Store training data as on-disc memmaps to massively reduce memory usage") - - parser.add_argument("--epochs", type=int, default=3, help="Number of epochs to train for") - parser.add_argument("--local_rank", - type=int, - default=-1, - help="local_rank for distributed training on gpus") - parser.add_argument("--no_cuda", - action='store_true', - help="Whether not to use CUDA when available") - parser.add_argument('--gradient_accumulation_steps', - type=int, - default=1, - help="Number of updates steps to accumulate before performing a backward/update pass.") - parser.add_argument("--train_batch_size", - default=32, - type=int, - help="Total batch size for training.") - parser.add_argument('--fp16', - action='store_true', - help="Whether to use 16-bit float precision instead of 32-bit") - parser.add_argument('--loss_scale', - type=float, default=0, - help="Loss scaling to improve fp16 numeric stability. Only used when fp16 set to True.\n" - "0 (default value): dynamic loss scaling.\n" - "Positive power of 2: static loss scaling value.\n") - parser.add_argument("--warmup_steps", - default=0, - type=int, - help="Linear warmup over warmup_steps.") - parser.add_argument("--adam_epsilon", - default=1e-8, - type=float, - help="Epsilon for Adam optimizer.") - parser.add_argument("--learning_rate", - default=3e-5, - type=float, - help="The initial learning rate for Adam.") - parser.add_argument('--seed', - type=int, - default=42, - help="random seed for initialization") - args = parser.parse_args() - - assert args.pregenerated_data.is_dir(), \ - "--pregenerated_data should point to the folder of files made by pregenerate_training_data.py!" - - samples_per_epoch = [] - for i in range(args.epochs): - epoch_file = args.pregenerated_data / f"epoch_{i}.json" - metrics_file = args.pregenerated_data / f"epoch_{i}_metrics.json" - if epoch_file.is_file() and metrics_file.is_file(): - metrics = json.loads(metrics_file.read_text()) - samples_per_epoch.append(metrics['num_training_examples']) - else: - if i == 0: - exit("No training data was found!") - print(f"Warning! There are fewer epochs of pregenerated data ({i}) than training epochs ({args.epochs}).") - print("This script will loop over the available data, but training diversity may be negatively impacted.") - num_data_epochs = i - break - else: - num_data_epochs = args.epochs - - if args.local_rank == -1 or args.no_cuda: - device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu") - n_gpu = torch.cuda.device_count() - else: - torch.cuda.set_device(args.local_rank) - device = torch.device("cuda", args.local_rank) - n_gpu = 1 - # Initializes the distributed backend which will take care of sychronizing nodes/GPUs - torch.distributed.init_process_group(backend='nccl') - logging.info("device: {} n_gpu: {}, distributed training: {}, 16-bits training: {}".format( - device, n_gpu, bool(args.local_rank != -1), args.fp16)) - - if args.gradient_accumulation_steps < 1: - raise ValueError("Invalid gradient_accumulation_steps parameter: {}, should be >= 1".format( - args.gradient_accumulation_steps)) - - args.train_batch_size = args.train_batch_size // args.gradient_accumulation_steps - - random.seed(args.seed) - np.random.seed(args.seed) - torch.manual_seed(args.seed) - if n_gpu > 0: - torch.cuda.manual_seed_all(args.seed) - - if args.output_dir.is_dir() and list(args.output_dir.iterdir()): - logging.warning(f"Output directory ({args.output_dir}) already exists and is not empty!") - args.output_dir.mkdir(parents=True, exist_ok=True) - - tokenizer = BertTokenizer.from_pretrained(args.bert_model, do_lower_case=args.do_lower_case) - - total_train_examples = 0 - for i in range(args.epochs): - # The modulo takes into account the fact that we may loop over limited epochs of data - total_train_examples += samples_per_epoch[i % len(samples_per_epoch)] - - num_train_optimization_steps = int( - total_train_examples / args.train_batch_size / args.gradient_accumulation_steps) - if args.local_rank != -1: - num_train_optimization_steps = num_train_optimization_steps // torch.distributed.get_world_size() - - # Prepare model - model = BertForPreTraining.from_pretrained(args.bert_model) - # We don't need to manually call model.half() following Apex's recommend - # if args.fp16: - # model.half() - model.to(device) - if args.local_rank != -1: - try: - from apex.parallel import DistributedDataParallel as DDP - except ImportError: - raise ImportError( - "Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training.") - model = DDP(model) - elif n_gpu > 1: - model = torch.nn.DataParallel(model) - - # Prepare optimizer - param_optimizer = list(model.named_parameters()) - no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight'] - optimizer_grouped_parameters = [ - {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], - 'weight_decay': 0.01}, - {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0} - ] - - optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon) - scheduler = WarmupLinearSchedule(optimizer, warmup_steps=args.warmup_steps, - t_total=num_train_optimization_steps) - - if args.fp16: - try: - # from apex.optimizers import FP16_Optimizer - # from apex.optimizers import FusedAdam - from apex import amp - except ImportError: - raise ImportError( - "Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training.") - - # This below line of code is the main upgrade of Apex Fp16 implementation. I chose opt_leve="01" - # because it's recommended for typical use by Apex. We can make it configured - model, optimizer = amp.initialize(model, optimizer, opt_level="O1") - - # We don't need to use FP16_Optimizer wrapping over FusedAdam as well. Now Apex supports all Pytorch Optimizer - - # optimizer = FusedAdam(optimizer_grouped_parameters, - # lr=args.learning_rate, - # bias_correction=False, - # max_grad_norm=1.0) - # if args.loss_scale == 0: - # optimizer = FP16_Optimizer(optimizer, dynamic_loss_scale=True) - # else: - # optimizer = FP16_Optimizer(optimizer, static_loss_scale=args.loss_scale) - # else: - # optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon) - # scheduler = WarmupLinearSchedule(optimizer, warmup_steps=args.warmup_steps, t_total=num_train_optimization_steps) - - global_step = 0 - logging.info("***** Running training *****") - logging.info(f" Num examples = {total_train_examples}") - logging.info(" Batch size = %d", args.train_batch_size) - logging.info(" Num steps = %d", num_train_optimization_steps) - model.train() - for epoch in range(args.epochs): - epoch_dataset = PregeneratedDataset(epoch=epoch, training_path=args.pregenerated_data, tokenizer=tokenizer, - num_data_epochs=num_data_epochs, reduce_memory=args.reduce_memory) - if args.local_rank == -1: - train_sampler = RandomSampler(epoch_dataset) - else: - train_sampler = DistributedSampler(epoch_dataset) - train_dataloader = DataLoader(epoch_dataset, sampler=train_sampler, batch_size=args.train_batch_size) - tr_loss = 0 - nb_tr_examples, nb_tr_steps = 0, 0 - with tqdm(total=len(train_dataloader), desc=f"Epoch {epoch}") as pbar: - for step, batch in enumerate(train_dataloader): - batch = tuple(t.to(device) for t in batch) - input_ids, input_mask, segment_ids, lm_label_ids, is_next = batch - outputs = model(input_ids, segment_ids, input_mask, lm_label_ids, is_next) - loss = outputs[0] - if n_gpu > 1: - loss = loss.mean() # mean() to average on multi-gpu. - if args.gradient_accumulation_steps > 1: - loss = loss / args.gradient_accumulation_steps - if args.fp16: - # I depricate FP16_Optimizer's backward func and replace as Apex document - # optimizer.backward(loss) - with amp.scale_loss(loss, optimizer) as scaled_loss: - scaled_loss.backward() - else: - loss.backward() - tr_loss += loss.item() - nb_tr_examples += input_ids.size(0) - nb_tr_steps += 1 - pbar.update(1) - mean_loss = tr_loss * args.gradient_accumulation_steps / nb_tr_steps - pbar.set_postfix_str(f"Loss: {mean_loss:.5f}") - if (step + 1) % args.gradient_accumulation_steps == 0: - optimizer.step() - scheduler.step() # Update learning rate schedule - optimizer.zero_grad() - global_step += 1 - - # Save a trained model - if args.local_rank == -1 or torch.distributed.get_rank() == 0: - logging.info("** ** * Saving fine-tuned model ** ** * ") - model_to_save = model.module if hasattr(model, 'module') else model # Take care of distributed/parallel training - model_to_save.save_pretrained(args.output_dir) - tokenizer.save_pretrained(args.output_dir) - - -if __name__ == '__main__': - main() diff --git a/examples/lm_finetuning/pregenerate_training_data.py b/examples/lm_finetuning/pregenerate_training_data.py deleted file mode 100644 index ff40d95f75..0000000000 --- a/examples/lm_finetuning/pregenerate_training_data.py +++ /dev/null @@ -1,354 +0,0 @@ -from argparse import ArgumentParser -from pathlib import Path -from tqdm import tqdm, trange -from tempfile import TemporaryDirectory -import shelve -from multiprocessing import Pool - -from random import random, randrange, randint, shuffle, choice -from pytorch_transformers.tokenization_bert import BertTokenizer -import numpy as np -import json -import collections - -class DocumentDatabase: - def __init__(self, reduce_memory=False): - if reduce_memory: - self.temp_dir = TemporaryDirectory() - self.working_dir = Path(self.temp_dir.name) - self.document_shelf_filepath = self.working_dir / 'shelf.db' - self.document_shelf = shelve.open(str(self.document_shelf_filepath), - flag='n', protocol=-1) - self.documents = None - else: - self.documents = [] - self.document_shelf = None - self.document_shelf_filepath = None - self.temp_dir = None - self.doc_lengths = [] - self.doc_cumsum = None - self.cumsum_max = None - self.reduce_memory = reduce_memory - - def add_document(self, document): - if not document: - return - if self.reduce_memory: - current_idx = len(self.doc_lengths) - self.document_shelf[str(current_idx)] = document - else: - self.documents.append(document) - self.doc_lengths.append(len(document)) - - def _precalculate_doc_weights(self): - self.doc_cumsum = np.cumsum(self.doc_lengths) - self.cumsum_max = self.doc_cumsum[-1] - - def sample_doc(self, current_idx, sentence_weighted=True): - # Uses the current iteration counter to ensure we don't sample the same doc twice - if sentence_weighted: - # With sentence weighting, we sample docs proportionally to their sentence length - if self.doc_cumsum is None or len(self.doc_cumsum) != len(self.doc_lengths): - self._precalculate_doc_weights() - rand_start = self.doc_cumsum[current_idx] - rand_end = rand_start + self.cumsum_max - self.doc_lengths[current_idx] - sentence_index = randrange(rand_start, rand_end) % self.cumsum_max - sampled_doc_index = np.searchsorted(self.doc_cumsum, sentence_index, side='right') - else: - # If we don't use sentence weighting, then every doc has an equal chance to be chosen - sampled_doc_index = (current_idx + randrange(1, len(self.doc_lengths))) % len(self.doc_lengths) - assert sampled_doc_index != current_idx - if self.reduce_memory: - return self.document_shelf[str(sampled_doc_index)] - else: - return self.documents[sampled_doc_index] - - def __len__(self): - return len(self.doc_lengths) - - def __getitem__(self, item): - if self.reduce_memory: - return self.document_shelf[str(item)] - else: - return self.documents[item] - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, traceback): - if self.document_shelf is not None: - self.document_shelf.close() - if self.temp_dir is not None: - self.temp_dir.cleanup() - - -def truncate_seq_pair(tokens_a, tokens_b, max_num_tokens): - """Truncates a pair of sequences to a maximum sequence length. Lifted from Google's BERT repo.""" - while True: - total_length = len(tokens_a) + len(tokens_b) - if total_length <= max_num_tokens: - break - - trunc_tokens = tokens_a if len(tokens_a) > len(tokens_b) else tokens_b - assert len(trunc_tokens) >= 1 - - # We want to sometimes truncate from the front and sometimes from the - # back to add more randomness and avoid biases. - if random() < 0.5: - del trunc_tokens[0] - else: - trunc_tokens.pop() - -MaskedLmInstance = collections.namedtuple("MaskedLmInstance", - ["index", "label"]) - -def create_masked_lm_predictions(tokens, masked_lm_prob, max_predictions_per_seq, whole_word_mask, vocab_list): - """Creates the predictions for the masked LM objective. This is mostly copied from the Google BERT repo, but - with several refactors to clean it up and remove a lot of unnecessary variables.""" - cand_indices = [] - for (i, token) in enumerate(tokens): - if token == "[CLS]" or token == "[SEP]": - continue - # Whole Word Masking means that if we mask all of the wordpieces - # corresponding to an original word. When a word has been split into - # WordPieces, the first token does not have any marker and any subsequence - # tokens are prefixed with ##. So whenever we see the ## token, we - # append it to the previous set of word indexes. - # - # Note that Whole Word Masking does *not* change the training code - # at all -- we still predict each WordPiece independently, softmaxed - # over the entire vocabulary. - if (whole_word_mask and len(cand_indices) >= 1 and token.startswith("##")): - cand_indices[-1].append(i) - else: - cand_indices.append([i]) - - num_to_mask = min(max_predictions_per_seq, - max(1, int(round(len(tokens) * masked_lm_prob)))) - shuffle(cand_indices) - masked_lms = [] - covered_indexes = set() - for index_set in cand_indices: - if len(masked_lms) >= num_to_mask: - break - # If adding a whole-word mask would exceed the maximum number of - # predictions, then just skip this candidate. - if len(masked_lms) + len(index_set) > num_to_mask: - continue - is_any_index_covered = False - for index in index_set: - if index in covered_indexes: - is_any_index_covered = True - break - if is_any_index_covered: - continue - for index in index_set: - covered_indexes.add(index) - - masked_token = None - # 80% of the time, replace with [MASK] - if random() < 0.8: - masked_token = "[MASK]" - else: - # 10% of the time, keep original - if random() < 0.5: - masked_token = tokens[index] - # 10% of the time, replace with random word - else: - masked_token = choice(vocab_list) - masked_lms.append(MaskedLmInstance(index=index, label=tokens[index])) - tokens[index] = masked_token - - assert len(masked_lms) <= num_to_mask - masked_lms = sorted(masked_lms, key=lambda x: x.index) - mask_indices = [p.index for p in masked_lms] - masked_token_labels = [p.label for p in masked_lms] - - return tokens, mask_indices, masked_token_labels - - -def create_instances_from_document( - doc_database, doc_idx, max_seq_length, short_seq_prob, - masked_lm_prob, max_predictions_per_seq, whole_word_mask, vocab_list): - """This code is mostly a duplicate of the equivalent function from Google BERT's repo. - However, we make some changes and improvements. Sampling is improved and no longer requires a loop in this function. - Also, documents are sampled proportionally to the number of sentences they contain, which means each sentence - (rather than each document) has an equal chance of being sampled as a false example for the NextSentence task.""" - document = doc_database[doc_idx] - # Account for [CLS], [SEP], [SEP] - max_num_tokens = max_seq_length - 3 - - # We *usually* want to fill up the entire sequence since we are padding - # to `max_seq_length` anyways, so short sequences are generally wasted - # computation. However, we *sometimes* - # (i.e., short_seq_prob == 0.1 == 10% of the time) want to use shorter - # sequences to minimize the mismatch between pre-training and fine-tuning. - # The `target_seq_length` is just a rough target however, whereas - # `max_seq_length` is a hard limit. - target_seq_length = max_num_tokens - if random() < short_seq_prob: - target_seq_length = randint(2, max_num_tokens) - - # We DON'T just concatenate all of the tokens from a document into a long - # sequence and choose an arbitrary split point because this would make the - # next sentence prediction task too easy. Instead, we split the input into - # segments "A" and "B" based on the actual "sentences" provided by the user - # input. - instances = [] - current_chunk = [] - current_length = 0 - i = 0 - while i < len(document): - segment = document[i] - current_chunk.append(segment) - current_length += len(segment) - if i == len(document) - 1 or current_length >= target_seq_length: - if current_chunk: - # `a_end` is how many segments from `current_chunk` go into the `A` - # (first) sentence. - a_end = 1 - if len(current_chunk) >= 2: - a_end = randrange(1, len(current_chunk)) - - tokens_a = [] - for j in range(a_end): - tokens_a.extend(current_chunk[j]) - - tokens_b = [] - - # Random next - if len(current_chunk) == 1 or random() < 0.5: - is_random_next = True - target_b_length = target_seq_length - len(tokens_a) - - # Sample a random document, with longer docs being sampled more frequently - random_document = doc_database.sample_doc(current_idx=doc_idx, sentence_weighted=True) - - random_start = randrange(0, len(random_document)) - for j in range(random_start, len(random_document)): - tokens_b.extend(random_document[j]) - if len(tokens_b) >= target_b_length: - break - # We didn't actually use these segments so we "put them back" so - # they don't go to waste. - num_unused_segments = len(current_chunk) - a_end - i -= num_unused_segments - # Actual next - else: - is_random_next = False - for j in range(a_end, len(current_chunk)): - tokens_b.extend(current_chunk[j]) - truncate_seq_pair(tokens_a, tokens_b, max_num_tokens) - - assert len(tokens_a) >= 1 - assert len(tokens_b) >= 1 - - tokens = ["[CLS]"] + tokens_a + ["[SEP]"] + tokens_b + ["[SEP]"] - # The segment IDs are 0 for the [CLS] token, the A tokens and the first [SEP] - # They are 1 for the B tokens and the final [SEP] - segment_ids = [0 for _ in range(len(tokens_a) + 2)] + [1 for _ in range(len(tokens_b) + 1)] - - tokens, masked_lm_positions, masked_lm_labels = create_masked_lm_predictions( - tokens, masked_lm_prob, max_predictions_per_seq, whole_word_mask, vocab_list) - - instance = { - "tokens": tokens, - "segment_ids": segment_ids, - "is_random_next": is_random_next, - "masked_lm_positions": masked_lm_positions, - "masked_lm_labels": masked_lm_labels} - instances.append(instance) - current_chunk = [] - current_length = 0 - i += 1 - - return instances - - -def create_training_file(docs, vocab_list, args, epoch_num): - epoch_filename = args.output_dir / "epoch_{}.json".format(epoch_num) - num_instances = 0 - with epoch_filename.open('w') as epoch_file: - for doc_idx in trange(len(docs), desc="Document"): - doc_instances = create_instances_from_document( - docs, doc_idx, max_seq_length=args.max_seq_len, short_seq_prob=args.short_seq_prob, - masked_lm_prob=args.masked_lm_prob, max_predictions_per_seq=args.max_predictions_per_seq, - whole_word_mask=args.do_whole_word_mask, vocab_list=vocab_list) - doc_instances = [json.dumps(instance) for instance in doc_instances] - for instance in doc_instances: - epoch_file.write(instance + '\n') - num_instances += 1 - metrics_file = args.output_dir / "epoch_{}_metrics.json".format(epoch_num) - with metrics_file.open('w') as metrics_file: - metrics = { - "num_training_examples": num_instances, - "max_seq_len": args.max_seq_len - } - metrics_file.write(json.dumps(metrics)) - - -def main(): - parser = ArgumentParser() - parser.add_argument('--train_corpus', type=Path, required=True) - parser.add_argument("--output_dir", type=Path, required=True) - parser.add_argument("--bert_model", type=str, required=True, - choices=["bert-base-uncased", "bert-large-uncased", "bert-base-cased", - "bert-base-multilingual-uncased", "bert-base-chinese", "bert-base-multilingual-cased"]) - parser.add_argument("--do_lower_case", action="store_true") - parser.add_argument("--do_whole_word_mask", action="store_true", - help="Whether to use whole word masking rather than per-WordPiece masking.") - parser.add_argument("--reduce_memory", action="store_true", - help="Reduce memory usage for large datasets by keeping data on disc rather than in memory") - - parser.add_argument("--num_workers", type=int, default=1, - help="The number of workers to use to write the files") - parser.add_argument("--epochs_to_generate", type=int, default=3, - help="Number of epochs of data to pregenerate") - parser.add_argument("--max_seq_len", type=int, default=128) - parser.add_argument("--short_seq_prob", type=float, default=0.1, - help="Probability of making a short sentence as a training example") - parser.add_argument("--masked_lm_prob", type=float, default=0.15, - help="Probability of masking each token for the LM task") - parser.add_argument("--max_predictions_per_seq", type=int, default=20, - help="Maximum number of tokens to mask in each sequence") - - args = parser.parse_args() - - if args.num_workers > 1 and args.reduce_memory: - raise ValueError("Cannot use multiple workers while reducing memory") - - tokenizer = BertTokenizer.from_pretrained(args.bert_model, do_lower_case=args.do_lower_case) - vocab_list = list(tokenizer.vocab.keys()) - with DocumentDatabase(reduce_memory=args.reduce_memory) as docs: - with args.train_corpus.open() as f: - doc = [] - for line in tqdm(f, desc="Loading Dataset", unit=" lines"): - line = line.strip() - if line == "": - docs.add_document(doc) - doc = [] - else: - tokens = tokenizer.tokenize(line) - doc.append(tokens) - if doc: - docs.add_document(doc) # If the last doc didn't end on a newline, make sure it still gets added - if len(docs) <= 1: - exit("ERROR: No document breaks were found in the input file! These are necessary to allow the script to " - "ensure that random NextSentences are not sampled from the same document. Please add blank lines to " - "indicate breaks between documents in your input file. If your dataset does not contain multiple " - "documents, blank lines can be inserted at any natural boundary, such as the ends of chapters, " - "sections or paragraphs.") - - args.output_dir.mkdir(exist_ok=True) - - if args.num_workers > 1: - writer_workers = Pool(min(args.num_workers, args.epochs_to_generate)) - arguments = [(docs, vocab_list, args, idx) for idx in range(args.epochs_to_generate)] - writer_workers.starmap(create_training_file, arguments) - else: - for epoch in trange(args.epochs_to_generate, desc="Epoch"): - create_training_file(docs, vocab_list, args, epoch) - - -if __name__ == '__main__': - main() diff --git a/examples/lm_finetuning/simple_lm_finetuning.py b/examples/lm_finetuning/simple_lm_finetuning.py deleted file mode 100644 index 8c5d5bca74..0000000000 --- a/examples/lm_finetuning/simple_lm_finetuning.py +++ /dev/null @@ -1,641 +0,0 @@ -# coding=utf-8 -# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. -# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""BERT finetuning runner.""" - -from __future__ import absolute_import, division, print_function, unicode_literals - -import argparse -import logging -import os -import random -from io import open - -import numpy as np -import torch -from torch.utils.data import DataLoader, Dataset, RandomSampler -from torch.utils.data.distributed import DistributedSampler -from tqdm import tqdm, trange - -from pytorch_transformers import WEIGHTS_NAME, CONFIG_NAME -from pytorch_transformers.modeling_bert import BertForPreTraining -from pytorch_transformers.tokenization_bert import BertTokenizer -from pytorch_transformers.optimization import AdamW, WarmupLinearSchedule - -logging.basicConfig(format='%(asctime)s - %(levelname)s - %(name)s - %(message)s', - datefmt='%m/%d/%Y %H:%M:%S', - level=logging.INFO) -logger = logging.getLogger(__name__) - - -class BERTDataset(Dataset): - def __init__(self, corpus_path, tokenizer, seq_len, encoding="utf-8", corpus_lines=None, on_memory=True): - self.vocab = tokenizer.vocab - self.tokenizer = tokenizer - self.seq_len = seq_len - self.on_memory = on_memory - self.corpus_lines = corpus_lines # number of non-empty lines in input corpus - self.corpus_path = corpus_path - self.encoding = encoding - self.current_doc = 0 # to avoid random sentence from same doc - - # for loading samples directly from file - self.sample_counter = 0 # used to keep track of full epochs on file - self.line_buffer = None # keep second sentence of a pair in memory and use as first sentence in next pair - - # for loading samples in memory - self.current_random_doc = 0 - self.num_docs = 0 - self.sample_to_doc = [] # map sample index to doc and line - - # load samples into memory - if on_memory: - self.all_docs = [] - doc = [] - self.corpus_lines = 0 - with open(corpus_path, "r", encoding=encoding) as f: - for line in tqdm(f, desc="Loading Dataset", total=corpus_lines): - line = line.strip() - if line == "": - self.all_docs.append(doc) - doc = [] - #remove last added sample because there won't be a subsequent line anymore in the doc - self.sample_to_doc.pop() - else: - #store as one sample - sample = {"doc_id": len(self.all_docs), - "line": len(doc)} - self.sample_to_doc.append(sample) - doc.append(line) - self.corpus_lines = self.corpus_lines + 1 - - # if last row in file is not empty - if self.all_docs[-1] != doc: - self.all_docs.append(doc) - self.sample_to_doc.pop() - - self.num_docs = len(self.all_docs) - - # load samples later lazily from disk - else: - if self.corpus_lines is None: - with open(corpus_path, "r", encoding=encoding) as f: - self.corpus_lines = 0 - for line in tqdm(f, desc="Loading Dataset", total=corpus_lines): - if line.strip() == "": - self.num_docs += 1 - else: - self.corpus_lines += 1 - - # if doc does not end with empty line - if line.strip() != "": - self.num_docs += 1 - - self.file = open(corpus_path, "r", encoding=encoding) - self.random_file = open(corpus_path, "r", encoding=encoding) - - def __len__(self): - # last line of doc won't be used, because there's no "nextSentence". Additionally, we start counting at 0. - return self.corpus_lines - self.num_docs - 1 - - def __getitem__(self, item): - cur_id = self.sample_counter - self.sample_counter += 1 - if not self.on_memory: - # after one epoch we start again from beginning of file - if cur_id != 0 and (cur_id % len(self) == 0): - self.file.close() - self.file = open(self.corpus_path, "r", encoding=self.encoding) - - t1, t2, is_next_label = self.random_sent(item) - - # tokenize - tokens_a = self.tokenizer.tokenize(t1) - tokens_b = self.tokenizer.tokenize(t2) - - # combine to one sample - cur_example = InputExample(guid=cur_id, tokens_a=tokens_a, tokens_b=tokens_b, is_next=is_next_label) - - # transform sample to features - cur_features = convert_example_to_features(cur_example, self.seq_len, self.tokenizer) - - cur_tensors = (torch.tensor(cur_features.input_ids), - torch.tensor(cur_features.input_mask), - torch.tensor(cur_features.segment_ids), - torch.tensor(cur_features.lm_label_ids), - torch.tensor(cur_features.is_next)) - - return cur_tensors - - def random_sent(self, index): - """ - Get one sample from corpus consisting of two sentences. With prob. 50% these are two subsequent sentences - from one doc. With 50% the second sentence will be a random one from another doc. - :param index: int, index of sample. - :return: (str, str, int), sentence 1, sentence 2, isNextSentence Label - """ - t1, t2 = self.get_corpus_line(index) - if random.random() > 0.5: - label = 0 - else: - t2 = self.get_random_line() - label = 1 - - assert len(t1) > 0 - assert len(t2) > 0 - return t1, t2, label - - def get_corpus_line(self, item): - """ - Get one sample from corpus consisting of a pair of two subsequent lines from the same doc. - :param item: int, index of sample. - :return: (str, str), two subsequent sentences from corpus - """ - t1 = "" - t2 = "" - assert item < self.corpus_lines - if self.on_memory: - sample = self.sample_to_doc[item] - t1 = self.all_docs[sample["doc_id"]][sample["line"]] - t2 = self.all_docs[sample["doc_id"]][sample["line"]+1] - # used later to avoid random nextSentence from same doc - self.current_doc = sample["doc_id"] - return t1, t2 - else: - if self.line_buffer is None: - # read first non-empty line of file - while t1 == "" : - t1 = next(self.file).strip() - t2 = next(self.file).strip() - else: - # use t2 from previous iteration as new t1 - t1 = self.line_buffer - t2 = next(self.file).strip() - # skip empty rows that are used for separating documents and keep track of current doc id - while t2 == "" or t1 == "": - t1 = next(self.file).strip() - t2 = next(self.file).strip() - self.current_doc = self.current_doc+1 - self.line_buffer = t2 - - assert t1 != "" - assert t2 != "" - return t1, t2 - - def get_random_line(self): - """ - Get random line from another document for nextSentence task. - :return: str, content of one line - """ - # Similar to original tf repo: This outer loop should rarely go for more than one iteration for large - # corpora. However, just to be careful, we try to make sure that - # the random document is not the same as the document we're processing. - for _ in range(10): - if self.on_memory: - rand_doc_idx = random.randint(0, len(self.all_docs)-1) - rand_doc = self.all_docs[rand_doc_idx] - line = rand_doc[random.randrange(len(rand_doc))] - else: - rand_index = random.randint(1, self.corpus_lines if self.corpus_lines < 1000 else 1000) - #pick random line - for _ in range(rand_index): - line = self.get_next_line() - #check if our picked random line is really from another doc like we want it to be - if self.current_random_doc != self.current_doc: - break - return line - - def get_next_line(self): - """ Gets next line of random_file and starts over when reaching end of file""" - try: - line = next(self.random_file).strip() - #keep track of which document we are currently looking at to later avoid having the same doc as t1 - if line == "": - self.current_random_doc = self.current_random_doc + 1 - line = next(self.random_file).strip() - except StopIteration: - self.random_file.close() - self.random_file = open(self.corpus_path, "r", encoding=self.encoding) - line = next(self.random_file).strip() - return line - - -class InputExample(object): - """A single training/test example for the language model.""" - - def __init__(self, guid, tokens_a, tokens_b=None, is_next=None, lm_labels=None): - """Constructs a InputExample. - - Args: - guid: Unique id for the example. - tokens_a: string. The untokenized text of the first sequence. For single - sequence tasks, only this sequence must be specified. - tokens_b: (Optional) string. The untokenized text of the second sequence. - Only must be specified for sequence pair tasks. - label: (Optional) string. The label of the example. This should be - specified for train and dev examples, but not for test examples. - """ - self.guid = guid - self.tokens_a = tokens_a - self.tokens_b = tokens_b - self.is_next = is_next # nextSentence - self.lm_labels = lm_labels # masked words for language model - - -class InputFeatures(object): - """A single set of features of data.""" - - def __init__(self, input_ids, input_mask, segment_ids, is_next, lm_label_ids): - self.input_ids = input_ids - self.input_mask = input_mask - self.segment_ids = segment_ids - self.is_next = is_next - self.lm_label_ids = lm_label_ids - - -def random_word(tokens, tokenizer): - """ - Masking some random tokens for Language Model task with probabilities as in the original BERT paper. - :param tokens: list of str, tokenized sentence. - :param tokenizer: Tokenizer, object used for tokenization (we need it's vocab here) - :return: (list of str, list of int), masked tokens and related labels for LM prediction - """ - output_label = [] - - for i, token in enumerate(tokens): - prob = random.random() - # mask token with 15% probability - if prob < 0.15: - prob /= 0.15 - - # 80% randomly change token to mask token - if prob < 0.8: - tokens[i] = "[MASK]" - - # 10% randomly change token to random token - elif prob < 0.9: - tokens[i] = random.choice(list(tokenizer.vocab.items()))[0] - - # -> rest 10% randomly keep current token - - # append current token to output (we will predict these later) - try: - output_label.append(tokenizer.vocab[token]) - except KeyError: - # For unknown words (should not occur with BPE vocab) - output_label.append(tokenizer.vocab["[UNK]"]) - logger.warning("Cannot find token '{}' in vocab. Using [UNK] insetad".format(token)) - else: - # no masking token (will be ignored by loss function later) - output_label.append(-1) - - return tokens, output_label - - -def convert_example_to_features(example, max_seq_length, tokenizer): - """ - Convert a raw sample (pair of sentences as tokenized strings) into a proper training sample with - IDs, LM labels, input_mask, CLS and SEP tokens etc. - :param example: InputExample, containing sentence input as strings and is_next label - :param max_seq_length: int, maximum length of sequence. - :param tokenizer: Tokenizer - :return: InputFeatures, containing all inputs and labels of one sample as IDs (as used for model training) - """ - tokens_a = example.tokens_a - tokens_b = example.tokens_b - # Modifies `tokens_a` and `tokens_b` in place so that the total - # length is less than the specified length. - # Account for [CLS], [SEP], [SEP] with "- 3" - _truncate_seq_pair(tokens_a, tokens_b, max_seq_length - 3) - - tokens_a, t1_label = random_word(tokens_a, tokenizer) - tokens_b, t2_label = random_word(tokens_b, tokenizer) - # concatenate lm labels and account for CLS, SEP, SEP - lm_label_ids = ([-1] + t1_label + [-1] + t2_label + [-1]) - - # The convention in BERT is: - # (a) For sequence pairs: - # tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP] - # type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1 - # (b) For single sequences: - # tokens: [CLS] the dog is hairy . [SEP] - # type_ids: 0 0 0 0 0 0 0 - # - # Where "type_ids" are used to indicate whether this is the first - # sequence or the second sequence. The embedding vectors for `type=0` and - # `type=1` were learned during pre-training and are added to the wordpiece - # embedding vector (and position vector). This is not *strictly* necessary - # since the [SEP] token unambigiously separates the sequences, but it makes - # it easier for the model to learn the concept of sequences. - # - # For classification tasks, the first vector (corresponding to [CLS]) is - # used as as the "sentence vector". Note that this only makes sense because - # the entire model is fine-tuned. - tokens = [] - segment_ids = [] - tokens.append("[CLS]") - segment_ids.append(0) - for token in tokens_a: - tokens.append(token) - segment_ids.append(0) - tokens.append("[SEP]") - segment_ids.append(0) - - assert len(tokens_b) > 0 - for token in tokens_b: - tokens.append(token) - segment_ids.append(1) - tokens.append("[SEP]") - segment_ids.append(1) - - input_ids = tokenizer.convert_tokens_to_ids(tokens) - - # The mask has 1 for real tokens and 0 for padding tokens. Only real - # tokens are attended to. - input_mask = [1] * len(input_ids) - - # Zero-pad up to the sequence length. - while len(input_ids) < max_seq_length: - input_ids.append(0) - input_mask.append(0) - segment_ids.append(0) - lm_label_ids.append(-1) - - assert len(input_ids) == max_seq_length - assert len(input_mask) == max_seq_length - assert len(segment_ids) == max_seq_length - assert len(lm_label_ids) == max_seq_length - - if example.guid < 5: - logger.info("*** Example ***") - logger.info("guid: %s" % (example.guid)) - logger.info("tokens: %s" % " ".join( - [str(x) for x in tokens])) - logger.info("input_ids: %s" % " ".join([str(x) for x in input_ids])) - logger.info("input_mask: %s" % " ".join([str(x) for x in input_mask])) - logger.info( - "segment_ids: %s" % " ".join([str(x) for x in segment_ids])) - logger.info("LM label: %s " % (lm_label_ids)) - logger.info("Is next sentence label: %s " % (example.is_next)) - - features = InputFeatures(input_ids=input_ids, - input_mask=input_mask, - segment_ids=segment_ids, - lm_label_ids=lm_label_ids, - is_next=example.is_next) - return features - - -def main(): - parser = argparse.ArgumentParser() - - ## Required parameters - parser.add_argument("--train_corpus", - default=None, - type=str, - required=True, - help="The input train corpus.") - parser.add_argument("--bert_model", default=None, type=str, required=True, - help="Bert pre-trained model selected in the list: bert-base-uncased, " - "bert-large-uncased, bert-base-cased, bert-base-multilingual, bert-base-chinese.") - parser.add_argument("--output_dir", - default=None, - type=str, - required=True, - help="The output directory where the model checkpoints will be written.") - - ## Other parameters - parser.add_argument("--max_seq_length", - default=128, - type=int, - help="The maximum total input sequence length after WordPiece tokenization. \n" - "Sequences longer than this will be truncated, and sequences shorter \n" - "than this will be padded.") - parser.add_argument("--do_train", - action='store_true', - help="Whether to run training.") - parser.add_argument("--train_batch_size", - default=32, - type=int, - help="Total batch size for training.") - parser.add_argument("--learning_rate", - default=3e-5, - type=float, - help="The initial learning rate for Adam.") - parser.add_argument("--adam_epsilon", - default=1e-8, - type=float, - help="Epsilon for Adam optimizer.") - parser.add_argument("--num_train_epochs", - default=3.0, - type=float, - help="Total number of training epochs to perform.") - parser.add_argument("--warmup_steps", - default=0, - type=int, - help="Linear warmup over warmup_steps.") - parser.add_argument("--no_cuda", - action='store_true', - help="Whether not to use CUDA when available") - parser.add_argument("--on_memory", - action='store_true', - help="Whether to load train samples into memory or use disk") - parser.add_argument("--do_lower_case", - action='store_true', - help="Whether to lower case the input text. True for uncased models, False for cased models.") - parser.add_argument("--local_rank", - type=int, - default=-1, - help="local_rank for distributed training on gpus") - parser.add_argument('--seed', - type=int, - default=42, - help="random seed for initialization") - parser.add_argument('--gradient_accumulation_steps', - type=int, - default=1, - help="Number of updates steps to accumualte before performing a backward/update pass.") - parser.add_argument('--fp16', - action='store_true', - help="Whether to use 16-bit float precision instead of 32-bit") - parser.add_argument('--loss_scale', - type = float, default = 0, - help = "Loss scaling to improve fp16 numeric stability. Only used when fp16 set to True.\n" - "0 (default value): dynamic loss scaling.\n" - "Positive power of 2: static loss scaling value.\n") - - args = parser.parse_args() - - if args.local_rank == -1 or args.no_cuda: - device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu") - n_gpu = torch.cuda.device_count() - else: - torch.cuda.set_device(args.local_rank) - device = torch.device("cuda", args.local_rank) - n_gpu = 1 - # Initializes the distributed backend which will take care of sychronizing nodes/GPUs - torch.distributed.init_process_group(backend='nccl') - logger.info("device: {} n_gpu: {}, distributed training: {}, 16-bits training: {}".format( - device, n_gpu, bool(args.local_rank != -1), args.fp16)) - - if args.gradient_accumulation_steps < 1: - raise ValueError("Invalid gradient_accumulation_steps parameter: {}, should be >= 1".format( - args.gradient_accumulation_steps)) - - args.train_batch_size = args.train_batch_size // args.gradient_accumulation_steps - - random.seed(args.seed) - np.random.seed(args.seed) - torch.manual_seed(args.seed) - if n_gpu > 0: - torch.cuda.manual_seed_all(args.seed) - - if not args.do_train: - raise ValueError("Training is currently the only implemented execution option. Please set `do_train`.") - - if os.path.exists(args.output_dir) and os.listdir(args.output_dir): - raise ValueError("Output directory ({}) already exists and is not empty.".format(args.output_dir)) - if not os.path.exists(args.output_dir) and (args.local_rank == -1 or torch.distributed.get_rank() == 0): - os.makedirs(args.output_dir) - - tokenizer = BertTokenizer.from_pretrained(args.bert_model, do_lower_case=args.do_lower_case) - - #train_examples = None - num_train_optimization_steps = None - if args.do_train: - print("Loading Train Dataset", args.train_corpus) - train_dataset = BERTDataset(args.train_corpus, tokenizer, seq_len=args.max_seq_length, - corpus_lines=None, on_memory=args.on_memory) - num_train_optimization_steps = int( - len(train_dataset) / args.train_batch_size / args.gradient_accumulation_steps) * args.num_train_epochs - if args.local_rank != -1: - num_train_optimization_steps = num_train_optimization_steps // torch.distributed.get_world_size() - - # Prepare model - model = BertForPreTraining.from_pretrained(args.bert_model) - if args.fp16: - model.half() - model.to(device) - if args.local_rank != -1: - try: - from apex.parallel import DistributedDataParallel as DDP - except ImportError: - raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training.") - model = DDP(model) - elif n_gpu > 1: - model = torch.nn.DataParallel(model) - - # Prepare optimizer - if args.do_train: - param_optimizer = list(model.named_parameters()) - no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight'] - optimizer_grouped_parameters = [ - {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01}, - {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0} - ] - - if args.fp16: - try: - from apex.optimizers import FP16_Optimizer - from apex.optimizers import FusedAdam - except ImportError: - raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training.") - - optimizer = FusedAdam(optimizer_grouped_parameters, - lr=args.learning_rate, - bias_correction=False, - max_grad_norm=1.0) - if args.loss_scale == 0: - optimizer = FP16_Optimizer(optimizer, dynamic_loss_scale=True) - else: - optimizer = FP16_Optimizer(optimizer, static_loss_scale=args.loss_scale) - - else: - optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon) - scheduler = WarmupLinearSchedule(optimizer, warmup_steps=args.warmup_steps, t_total=num_train_optimization_steps) - - global_step = 0 - if args.do_train: - logger.info("***** Running training *****") - logger.info(" Num examples = %d", len(train_dataset)) - logger.info(" Batch size = %d", args.train_batch_size) - logger.info(" Num steps = %d", num_train_optimization_steps) - - if args.local_rank == -1: - train_sampler = RandomSampler(train_dataset) - else: - #TODO: check if this works with current data generator from disk that relies on next(file) - # (it doesn't return item back by index) - train_sampler = DistributedSampler(train_dataset) - train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=args.train_batch_size) - - model.train() - for _ in trange(int(args.num_train_epochs), desc="Epoch"): - tr_loss = 0 - nb_tr_examples, nb_tr_steps = 0, 0 - for step, batch in enumerate(tqdm(train_dataloader, desc="Iteration")): - batch = tuple(t.to(device) for t in batch) - input_ids, input_mask, segment_ids, lm_label_ids, is_next = batch - outputs = model(input_ids, segment_ids, input_mask, lm_label_ids, is_next) - loss = outputs[0] - if n_gpu > 1: - loss = loss.mean() # mean() to average on multi-gpu. - if args.gradient_accumulation_steps > 1: - loss = loss / args.gradient_accumulation_steps - if args.fp16: - optimizer.backward(loss) - else: - loss.backward() - tr_loss += loss.item() - nb_tr_examples += input_ids.size(0) - nb_tr_steps += 1 - if (step + 1) % args.gradient_accumulation_steps == 0: - optimizer.step() - scheduler.step() # Update learning rate schedule - optimizer.zero_grad() - global_step += 1 - - # Save a trained model - if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0): - logger.info("** ** * Saving fine - tuned model ** ** * ") - model_to_save = model.module if hasattr(model, 'module') else model # Take care of distributed/parallel training - model_to_save.save_pretrained(args.output_dir) - tokenizer.save_pretrained(args.output_dir) - - -def _truncate_seq_pair(tokens_a, tokens_b, max_length): - """Truncates a sequence pair in place to the maximum length.""" - - # This is a simple heuristic which will always truncate the longer sequence - # one token at a time. This makes more sense than truncating an equal percent - # of tokens from each, since if one sequence is very short then each token - # that's truncated likely contains more information than a longer sequence. - while True: - total_length = len(tokens_a) + len(tokens_b) - if total_length <= max_length: - break - if len(tokens_a) > len(tokens_b): - tokens_a.pop() - else: - tokens_b.pop() - - -def accuracy(out, labels): - outputs = np.argmax(out, axis=1) - return np.sum(outputs == labels) - - -if __name__ == "__main__": - main() diff --git a/examples/run_glue.py b/examples/run_glue.py index e20f6d84c4..b39c6bf054 100644 --- a/examples/run_glue.py +++ b/examples/run_glue.py @@ -39,12 +39,17 @@ from pytorch_transformers import (WEIGHTS_NAME, BertConfig, XLMConfig, XLMForSequenceClassification, XLMTokenizer, XLNetConfig, XLNetForSequenceClassification, - XLNetTokenizer) + XLNetTokenizer, + DistilBertConfig, + DistilBertForSequenceClassification, + DistilBertTokenizer) from pytorch_transformers import AdamW, WarmupLinearSchedule -from utils_glue import (compute_metrics, convert_examples_to_features, - output_modes, processors) +from pytorch_transformers import glue_compute_metrics as compute_metrics +from pytorch_transformers import glue_output_modes as output_modes +from pytorch_transformers import glue_processors as processors +from pytorch_transformers import glue_convert_examples_to_features as convert_examples_to_features logger = logging.getLogger(__name__) @@ -55,6 +60,7 @@ MODEL_CLASSES = { 'xlnet': (XLNetConfig, XLNetForSequenceClassification, XLNetTokenizer), 'xlm': (XLMConfig, XLMForSequenceClassification, XLMTokenizer), 'roberta': (RobertaConfig, RobertaForSequenceClassification, RobertaTokenizer), + 'distilbert': (DistilBertConfig, DistilBertForSequenceClassification, DistilBertTokenizer) } @@ -128,7 +134,7 @@ def train(args, train_dataset, model, tokenizer): batch = tuple(t.to(args.device) for t in batch) inputs = {'input_ids': batch[0], 'attention_mask': batch[1], - 'token_type_ids': batch[2] if args.model_type in ['bert', 'xlnet'] else None, # XLM and RoBERTa don't use segment_ids + 'token_type_ids': batch[2] if args.model_type in ['bert', 'xlnet'] else None, # XLM, DistilBERT and RoBERTa don't use segment_ids 'labels': batch[3]} outputs = model(**inputs) loss = outputs[0] # model outputs are always tuple in pytorch-transformers (see doc) @@ -218,7 +224,7 @@ def evaluate(args, model, tokenizer, prefix=""): with torch.no_grad(): inputs = {'input_ids': batch[0], 'attention_mask': batch[1], - 'token_type_ids': batch[2] if args.model_type in ['bert', 'xlnet'] else None, # XLM and RoBERTa don't use segment_ids + 'token_type_ids': batch[2] if args.model_type in ['bert', 'xlnet'] else None, # XLM, DistilBERT and RoBERTa don't use segment_ids 'labels': batch[3]} outputs = model(**inputs) tmp_eval_loss, logits = outputs[:2] @@ -273,11 +279,6 @@ def load_and_cache_examples(args, task, tokenizer, evaluate=False): label_list[1], label_list[2] = label_list[2], label_list[1] examples = processor.get_dev_examples(args.data_dir) if evaluate else processor.get_train_examples(args.data_dir) features = convert_examples_to_features(examples, label_list, args.max_seq_length, tokenizer, output_mode, - cls_token_at_end=bool(args.model_type in ['xlnet']), # xlnet has a cls token at the end - cls_token=tokenizer.cls_token, - cls_token_segment_id=2 if args.model_type in ['xlnet'] else 0, - sep_token=tokenizer.sep_token, - sep_token_extra=bool(args.model_type in ['roberta']), # roberta uses an extra separator b/w pairs of sentences, cf. github.com/pytorch/fairseq/commit/1684e166e3da03f5b600dbb7855cb98ddfcd0805 pad_on_left=bool(args.model_type in ['xlnet']), # pad on the left for xlnet pad_token=tokenizer.convert_tokens_to_ids([tokenizer.pad_token])[0], pad_token_segment_id=4 if args.model_type in ['xlnet'] else 0, diff --git a/examples/run_lm_finetuning.py b/examples/run_lm_finetuning.py index d37f7a443a..556cce6753 100644 --- a/examples/run_lm_finetuning.py +++ b/examples/run_lm_finetuning.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """ -Fine-tuning the library models for language modeling on WikiText-2 (GPT, GPT-2, BERT, RoBERTa). +Fine-tuning the library models for language modeling on a text file (GPT, GPT-2, BERT, RoBERTa). GPT and GPT-2 are fine-tuned using a causal language modeling (CLM) loss while BERT and RoBERTa are fine-tuned using a masked language modeling (MLM) loss. """ @@ -39,7 +39,8 @@ from pytorch_transformers import (WEIGHTS_NAME, AdamW, WarmupLinearSchedule, BertConfig, BertForMaskedLM, BertTokenizer, GPT2Config, GPT2LMHeadModel, GPT2Tokenizer, OpenAIGPTConfig, OpenAIGPTLMHeadModel, OpenAIGPTTokenizer, - RobertaConfig, RobertaForMaskedLM, RobertaTokenizer) + RobertaConfig, RobertaForMaskedLM, RobertaTokenizer, + DistilBertConfig, DistilBertForMaskedLM, DistilBertTokenizer) logger = logging.getLogger(__name__) @@ -49,7 +50,8 @@ MODEL_CLASSES = { 'gpt2': (GPT2Config, GPT2LMHeadModel, GPT2Tokenizer), 'openai-gpt': (OpenAIGPTConfig, OpenAIGPTLMHeadModel, OpenAIGPTTokenizer), 'bert': (BertConfig, BertForMaskedLM, BertTokenizer), - 'roberta': (RobertaConfig, RobertaForMaskedLM, RobertaTokenizer) + 'roberta': (RobertaConfig, RobertaForMaskedLM, RobertaTokenizer), + 'distilbert': (DistilBertConfig, DistilBertForMaskedLM, DistilBertTokenizer) } @@ -73,7 +75,7 @@ class TextDataset(Dataset): tokenized_text = tokenizer.convert_tokens_to_ids(tokenizer.tokenize(text)) while len(tokenized_text) >= block_size: # Truncate in block of block_size - self.examples.append(tokenizer.add_special_tokens_single_sentence(tokenized_text[:block_size])) + self.examples.append(tokenizer.add_special_tokens_single_sequence(tokenized_text[:block_size])) tokenized_text = tokenized_text[block_size:] # Note that we are loosing the last truncated example here for the sake of simplicity (no padding) # If your dataset is small, first you should loook for a bigger one :-) and second you @@ -247,7 +249,6 @@ def evaluate(args, model, tokenizer, prefix=""): # Loop to handle MNLI double evaluation (matched, mis-matched) eval_output_dir = args.output_dir - results = {} eval_dataset = load_and_cache_examples(args, tokenizer, evaluate=True) if not os.path.exists(eval_output_dir) and args.local_rank in [-1, 0]: @@ -289,7 +290,7 @@ def evaluate(args, model, tokenizer, prefix=""): logger.info(" %s = %s", key, str(result[key])) writer.write("%s = %s\n" % (key, str(result[key]))) - return results + return result def main(): @@ -381,7 +382,7 @@ def main(): parser.add_argument('--server_port', type=str, default='', help="For distant debugging.") args = parser.parse_args() - if args.model_type in ["bert", "roberta"] and not args.mlm: + if args.model_type in ["bert", "roberta", "distilbert"] and not args.mlm: raise ValueError("BERT and RoBERTa do not have LM heads but masked LM heads. They must be run using the --mlm " "flag (masked language modeling).") if args.eval_data_file is None and args.do_eval: diff --git a/examples/run_multiple_choice.py b/examples/run_multiple_choice.py new file mode 100644 index 0000000000..05f9a48f50 --- /dev/null +++ b/examples/run_multiple_choice.py @@ -0,0 +1,542 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Finetuning the library models for multiple choice (Bert, Roberta, XLNet).""" + +from __future__ import absolute_import, division, print_function + +import argparse +import glob +import logging +import os +import random + + +import numpy as np +import torch +from torch.utils.data import (DataLoader, RandomSampler, SequentialSampler, + TensorDataset) +from torch.utils.data.distributed import DistributedSampler +from tensorboardX import SummaryWriter +from tqdm import tqdm, trange + +from pytorch_transformers import (WEIGHTS_NAME, BertConfig, + BertForMultipleChoice, BertTokenizer, + XLNetConfig, XLNetForMultipleChoice, + XLNetTokenizer, RobertaConfig, + RobertaForMultipleChoice, RobertaTokenizer) + +from pytorch_transformers import AdamW, WarmupLinearSchedule + +from utils_multiple_choice import (convert_examples_to_features, processors) + +logger = logging.getLogger(__name__) + +ALL_MODELS = sum((tuple(conf.pretrained_config_archive_map.keys()) for conf in (BertConfig, XLNetConfig, RobertaConfig)), ()) + +MODEL_CLASSES = { + 'bert': (BertConfig, BertForMultipleChoice, BertTokenizer), + 'xlnet': (XLNetConfig, XLNetForMultipleChoice, XLNetTokenizer), + 'roberta': (RobertaConfig, RobertaForMultipleChoice, RobertaTokenizer) +} + +def select_field(features, field): + return [ + [ + choice[field] + for choice in feature.choices_features + ] + for feature in features + ] + + +def simple_accuracy(preds, labels): + return (preds == labels).mean() + + +def set_seed(args): + random.seed(args.seed) + np.random.seed(args.seed) + torch.manual_seed(args.seed) + if args.n_gpu > 0: + torch.cuda.manual_seed_all(args.seed) + + +def train(args, train_dataset, model, tokenizer): + """ Train the model """ + if args.local_rank in [-1, 0]: + tb_writer = SummaryWriter() + + args.train_batch_size = args.per_gpu_train_batch_size * max(1, args.n_gpu) + train_sampler = RandomSampler(train_dataset) if args.local_rank == -1 else DistributedSampler(train_dataset) + train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=args.train_batch_size) + + if args.max_steps > 0: + t_total = args.max_steps + args.num_train_epochs = args.max_steps // (len(train_dataloader) // args.gradient_accumulation_steps) + 1 + else: + t_total = len(train_dataloader) // args.gradient_accumulation_steps * args.num_train_epochs + + # Prepare optimizer and schedule (linear warmup and decay) + no_decay = ['bias', 'LayerNorm.weight'] + optimizer_grouped_parameters = [ + {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)], 'weight_decay': args.weight_decay}, + {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0} + ] + optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon) + scheduler = WarmupLinearSchedule(optimizer, warmup_steps=args.warmup_steps, t_total=t_total) + if args.fp16: + try: + from apex import amp + except ImportError: + raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use fp16 training.") + model, optimizer = amp.initialize(model, optimizer, opt_level=args.fp16_opt_level) + + # multi-gpu training (should be after apex fp16 initialization) + if args.n_gpu > 1: + model = torch.nn.DataParallel(model) + + # Distributed training (should be after apex fp16 initialization) + if args.local_rank != -1: + model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.local_rank], + output_device=args.local_rank, + find_unused_parameters=True) + + # Train! + logger.info("***** Running training *****") + logger.info(" Num examples = %d", len(train_dataset)) + logger.info(" Num Epochs = %d", args.num_train_epochs) + logger.info(" Instantaneous batch size per GPU = %d", args.per_gpu_train_batch_size) + logger.info(" Total train batch size (w. parallel, distributed & accumulation) = %d", + args.train_batch_size * args.gradient_accumulation_steps * (torch.distributed.get_world_size() if args.local_rank != -1 else 1)) + logger.info(" Gradient Accumulation steps = %d", args.gradient_accumulation_steps) + logger.info(" Total optimization steps = %d", t_total) + + global_step = 0 + tr_loss, logging_loss = 0.0, 0.0 + best_dev_acc, best_dev_loss = 0.0, 99999999999.0 + best_steps = 0 + model.zero_grad() + train_iterator = trange(int(args.num_train_epochs), desc="Epoch", disable=args.local_rank not in [-1, 0]) + set_seed(args) # Added here for reproductibility (even between python 2 and 3) + for _ in train_iterator: + epoch_iterator = tqdm(train_dataloader, desc="Iteration", disable=args.local_rank not in [-1, 0]) + for step, batch in enumerate(epoch_iterator): + model.train() + batch = tuple(t.to(args.device) for t in batch) + inputs = {'input_ids': batch[0], + 'attention_mask': batch[1], + 'token_type_ids': batch[2] if args.model_type in ['bert', 'xlnet'] else None, # XLM don't use segment_ids + 'labels': batch[3]} + outputs = model(**inputs) + loss = outputs[0] # model outputs are always tuple in pytorch-transformers (see doc) + + if args.n_gpu > 1: + loss = loss.mean() # mean() to average on multi-gpu parallel training + if args.gradient_accumulation_steps > 1: + loss = loss / args.gradient_accumulation_steps + + if args.fp16: + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args.max_grad_norm) + else: + loss.backward() + torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm) + + tr_loss += loss.item() + if (step + 1) % args.gradient_accumulation_steps == 0: + + optimizer.step() + scheduler.step() # Update learning rate schedule + model.zero_grad() + global_step += 1 + + if args.local_rank in [-1, 0] and args.logging_steps > 0 and global_step % args.logging_steps == 0: + # Log metrics + if args.local_rank == -1 and args.evaluate_during_training: # Only evaluate when single GPU otherwise metrics may not average well + results = evaluate(args, model, tokenizer) + for key, value in results.items(): + tb_writer.add_scalar('eval_{}'.format(key), value, global_step) + if results["eval_acc"] > best_dev_acc: + best_dev_acc = results["eval_acc"] + best_dev_loss = results["eval_loss"] + best_steps = global_step + if args.do_test: + results_test = evaluate(args, model, tokenizer, test=True) + for key, value in results_test.items(): + tb_writer.add_scalar('test_{}'.format(key), value, global_step) + logger.info("test acc: %s, loss: %s, global steps: %s", str(results_test['eval_acc']), str(results_test['eval_loss']), str(global_step)) + tb_writer.add_scalar('lr', scheduler.get_lr()[0], global_step) + tb_writer.add_scalar('loss', (tr_loss - logging_loss)/args.logging_steps, global_step) + logger.info("Average loss: %s at global step: %s", str((tr_loss - logging_loss)/args.logging_steps), str(global_step)) + logging_loss = tr_loss + + if args.local_rank in [-1, 0] and args.save_steps > 0 and global_step % args.save_steps == 0: + # Save model checkpoint + output_dir = os.path.join(args.output_dir, 'checkpoint-{}'.format(global_step)) + if not os.path.exists(output_dir): + os.makedirs(output_dir) + model_to_save = model.module if hasattr(model, 'module') else model # Take care of distributed/parallel training + model_to_save.save_pretrained(output_dir) + tokenizer.save_vocabulary(output_dir) + torch.save(args, os.path.join(output_dir, 'training_args.bin')) + logger.info("Saving model checkpoint to %s", output_dir) + + if args.max_steps > 0 and global_step > args.max_steps: + epoch_iterator.close() + break + if args.max_steps > 0 and global_step > args.max_steps: + train_iterator.close() + break + + if args.local_rank in [-1, 0]: + tb_writer.close() + + return global_step, tr_loss / global_step, best_steps + + +def evaluate(args, model, tokenizer, prefix="", test=False): + eval_task_names = (args.task_name,) + eval_outputs_dirs = (args.output_dir,) + + results = {} + for eval_task, eval_output_dir in zip(eval_task_names, eval_outputs_dirs): + eval_dataset = load_and_cache_examples(args, eval_task, tokenizer, evaluate=not test, test=test) + + if not os.path.exists(eval_output_dir) and args.local_rank in [-1, 0]: + os.makedirs(eval_output_dir) + + args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu) + # Note that DistributedSampler samples randomly + eval_sampler = SequentialSampler(eval_dataset) if args.local_rank == -1 else DistributedSampler(eval_dataset) + eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size) + + # Eval! + logger.info("***** Running evaluation {} *****".format(prefix)) + logger.info(" Num examples = %d", len(eval_dataset)) + logger.info(" Batch size = %d", args.eval_batch_size) + eval_loss = 0.0 + nb_eval_steps = 0 + preds = None + out_label_ids = None + for batch in tqdm(eval_dataloader, desc="Evaluating"): + model.eval() + batch = tuple(t.to(args.device) for t in batch) + + with torch.no_grad(): + inputs = {'input_ids': batch[0], + 'attention_mask': batch[1], + 'token_type_ids': batch[2] if args.model_type in ['bert', 'xlnet'] else None, # XLM don't use segment_ids + 'labels': batch[3]} + outputs = model(**inputs) + tmp_eval_loss, logits = outputs[:2] + + eval_loss += tmp_eval_loss.mean().item() + nb_eval_steps += 1 + if preds is None: + preds = logits.detach().cpu().numpy() + out_label_ids = inputs['labels'].detach().cpu().numpy() + else: + preds = np.append(preds, logits.detach().cpu().numpy(), axis=0) + out_label_ids = np.append(out_label_ids, inputs['labels'].detach().cpu().numpy(), axis=0) + + eval_loss = eval_loss / nb_eval_steps + preds = np.argmax(preds, axis=1) + acc = simple_accuracy(preds, out_label_ids) + result = {"eval_acc": acc, "eval_loss": eval_loss} + results.update(result) + + output_eval_file = os.path.join(eval_output_dir, "is_test_" + str(test).lower() + "_eval_results.txt") + + with open(output_eval_file, "w") as writer: + logger.info("***** Eval results {} *****".format(str(prefix) + " is test:" + str(test))) + writer.write("model =%s\n" % str(args.model_name_or_path)) + writer.write("total batch size=%d\n" % (args.per_gpu_train_batch_size * args.gradient_accumulation_steps * + (torch.distributed.get_world_size() if args.local_rank != -1 else 1))) + writer.write("train num epochs=%d\n" % args.num_train_epochs) + writer.write("fp16 =%s\n" % args.fp16) + writer.write("max seq length =%d\n" % args.max_seq_length) + for key in sorted(result.keys()): + logger.info(" %s = %s", key, str(result[key])) + writer.write("%s = %s\n" % (key, str(result[key]))) + return results + + +def load_and_cache_examples(args, task, tokenizer, evaluate=False, test=False): + if args.local_rank not in [-1, 0]: + torch.distributed.barrier() # Make sure only the first process in distributed training process the dataset, and the others will use the cache + + processor = processors[task]() + # Load data features from cache or dataset file + if evaluate: + cached_mode = 'dev' + elif test: + cached_mode = 'test' + else: + cached_mode = 'train' + assert (evaluate == True and test == True) == False + cached_features_file = os.path.join(args.data_dir, 'cached_{}_{}_{}_{}'.format( + cached_mode, + list(filter(None, args.model_name_or_path.split('/'))).pop(), + str(args.max_seq_length), + str(task))) + if os.path.exists(cached_features_file): + logger.info("Loading features from cached file %s", cached_features_file) + features = torch.load(cached_features_file) + else: + logger.info("Creating features from dataset file at %s", args.data_dir) + label_list = processor.get_labels() + if evaluate: + examples = processor.get_dev_examples(args.data_dir) + elif test: + examples = processor.get_test_examples(args.data_dir) + else: + examples = processor.get_train_examples(args.data_dir) + logger.info("Training number: %s", str(len(examples))) + features = convert_examples_to_features(examples, label_list, args.max_seq_length, tokenizer, + cls_token_at_end=bool(args.model_type in ['xlnet']), # xlnet has a cls token at the end + cls_token=tokenizer.cls_token, + sep_token=tokenizer.sep_token, + sep_token_extra=bool(args.model_type in ['roberta']), + cls_token_segment_id=2 if args.model_type in ['xlnet'] else 0, + pad_on_left=bool(args.model_type in ['xlnet']), # pad on the left for xlnet + pad_token_segment_id=4 if args.model_type in ['xlnet'] else 0) + if args.local_rank in [-1, 0]: + logger.info("Saving features into cached file %s", cached_features_file) + torch.save(features, cached_features_file) + + if args.local_rank == 0: + torch.distributed.barrier() # Make sure only the first process in distributed training process the dataset, and the others will use the cache + + # Convert to Tensors and build dataset + all_input_ids = torch.tensor(select_field(features, 'input_ids'), dtype=torch.long) + all_input_mask = torch.tensor(select_field(features, 'input_mask'), dtype=torch.long) + all_segment_ids = torch.tensor(select_field(features, 'segment_ids'), dtype=torch.long) + all_label_ids = torch.tensor([f.label for f in features], dtype=torch.long) + + dataset = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids) + return dataset + + +def main(): + parser = argparse.ArgumentParser() + + ## Required parameters + parser.add_argument("--data_dir", default=None, type=str, required=True, + help="The input data dir. Should contain the .tsv files (or other data files) for the task.") + parser.add_argument("--model_type", default=None, type=str, required=True, + help="Model type selected in the list: " + ", ".join(MODEL_CLASSES.keys())) + parser.add_argument("--model_name_or_path", default=None, type=str, required=True, + help="Path to pre-trained model or shortcut name selected in the list: " + ", ".join(ALL_MODELS)) + parser.add_argument("--task_name", default=None, type=str, required=True, + help="The name of the task to train selected in the list: " + ", ".join(processors.keys())) + parser.add_argument("--output_dir", default=None, type=str, required=True, + help="The output directory where the model predictions and checkpoints will be written.") + + ## Other parameters + parser.add_argument("--config_name", default="", type=str, + help="Pretrained config name or path if not the same as model_name") + parser.add_argument("--tokenizer_name", default="", type=str, + help="Pretrained tokenizer name or path if not the same as model_name") + parser.add_argument("--cache_dir", default="", type=str, + help="Where do you want to store the pre-trained models downloaded from s3") + parser.add_argument("--max_seq_length", default=128, type=int, + help="The maximum total input sequence length after tokenization. Sequences longer " + "than this will be truncated, sequences shorter will be padded.") + parser.add_argument("--do_train", action='store_true', + help="Whether to run training.") + parser.add_argument("--do_eval", action='store_true', + help="Whether to run eval on the dev set.") + parser.add_argument("--do_test", action='store_true', help='Whether to run test on the test set') + parser.add_argument("--evaluate_during_training", action='store_true', + help="Rul evaluation during training at each logging step.") + parser.add_argument("--do_lower_case", action='store_true', + help="Set this flag if you are using an uncased model.") + + parser.add_argument("--per_gpu_train_batch_size", default=8, type=int, + help="Batch size per GPU/CPU for training.") + parser.add_argument("--per_gpu_eval_batch_size", default=8, type=int, + help="Batch size per GPU/CPU for evaluation.") + parser.add_argument('--gradient_accumulation_steps', type=int, default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.") + parser.add_argument("--learning_rate", default=5e-5, type=float, + help="The initial learning rate for Adam.") + parser.add_argument("--weight_decay", default=0.0, type=float, + help="Weight deay if we apply some.") + parser.add_argument("--adam_epsilon", default=1e-8, type=float, + help="Epsilon for Adam optimizer.") + parser.add_argument("--max_grad_norm", default=1.0, type=float, + help="Max gradient norm.") + parser.add_argument("--num_train_epochs", default=3.0, type=float, + help="Total number of training epochs to perform.") + parser.add_argument("--max_steps", default=-1, type=int, + help="If > 0: set total number of training steps to perform. Override num_train_epochs.") + parser.add_argument("--warmup_steps", default=0, type=int, + help="Linear warmup over warmup_steps.") + + parser.add_argument('--logging_steps', type=int, default=50, + help="Log every X updates steps.") + parser.add_argument('--save_steps', type=int, default=50, + help="Save checkpoint every X updates steps.") + parser.add_argument("--eval_all_checkpoints", action='store_true', + help="Evaluate all checkpoints starting with the same prefix as model_name ending and ending with step number") + parser.add_argument("--no_cuda", action='store_true', + help="Avoid using CUDA when available") + parser.add_argument('--overwrite_output_dir', action='store_true', + help="Overwrite the content of the output directory") + parser.add_argument('--overwrite_cache', action='store_true', + help="Overwrite the cached training and evaluation sets") + parser.add_argument('--seed', type=int, default=42, + help="random seed for initialization") + + parser.add_argument('--fp16', action='store_true', + help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit") + parser.add_argument('--fp16_opt_level', type=str, default='O1', + help="For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']." + "See details at https://nvidia.github.io/apex/amp.html") + parser.add_argument("--local_rank", type=int, default=-1, + help="For distributed training: local_rank") + parser.add_argument('--server_ip', type=str, default='', help="For distant debugging.") + parser.add_argument('--server_port', type=str, default='', help="For distant debugging.") + args = parser.parse_args() + + if os.path.exists(args.output_dir) and os.listdir(args.output_dir) and args.do_train and not args.overwrite_output_dir: + raise ValueError("Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome.".format(args.output_dir)) + + # Setup distant debugging if needed + if args.server_ip and args.server_port: + # Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script + import ptvsd + print("Waiting for debugger attach") + ptvsd.enable_attach(address=(args.server_ip, args.server_port), redirect_output=True) + ptvsd.wait_for_attach() + + # Setup CUDA, GPU & distributed training + if args.local_rank == -1 or args.no_cuda: + device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu") + args.n_gpu = torch.cuda.device_count() + else: # Initializes the distributed backend which will take care of sychronizing nodes/GPUs + torch.cuda.set_device(args.local_rank) + device = torch.device("cuda", args.local_rank) + torch.distributed.init_process_group(backend='nccl') + args.n_gpu = 1 + args.device = device + + # Setup logging + logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', + datefmt = '%m/%d/%Y %H:%M:%S', + level = logging.INFO if args.local_rank in [-1, 0] else logging.WARN) + logger.warning("Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s", + args.local_rank, device, args.n_gpu, bool(args.local_rank != -1), args.fp16) + + # Set seed + set_seed(args) + + # Prepare GLUE task + args.task_name = args.task_name.lower() + if args.task_name not in processors: + raise ValueError("Task not found: %s" % (args.task_name)) + processor = processors[args.task_name]() + label_list = processor.get_labels() + num_labels = len(label_list) + + # Load pretrained model and tokenizer + if args.local_rank not in [-1, 0]: + torch.distributed.barrier() # Make sure only the first process in distributed training will download model & vocab + + args.model_type = args.model_type.lower() + config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type] + config = config_class.from_pretrained(args.config_name if args.config_name else args.model_name_or_path, num_labels=num_labels, finetuning_task=args.task_name) + tokenizer = tokenizer_class.from_pretrained(args.tokenizer_name if args.tokenizer_name else args.model_name_or_path, do_lower_case=args.do_lower_case) + model = model_class.from_pretrained(args.model_name_or_path, from_tf=bool('.ckpt' in args.model_name_or_path), config=config) + + if args.local_rank == 0: + torch.distributed.barrier() # Make sure only the first process in distributed training will download model & vocab + + model.to(args.device) + + logger.info("Training/evaluation parameters %s", args) + best_steps = 0 + + # Training + if args.do_train: + train_dataset = load_and_cache_examples(args, args.task_name, tokenizer, evaluate=False) + global_step, tr_loss, best_steps = train(args, train_dataset, model, tokenizer) + logger.info(" global_step = %s, average loss = %s", global_step, tr_loss) + + + # Saving best-practices: if you use defaults names for the model, you can reload it using from_pretrained() + if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0): + # Create output directory if needed + if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]: + os.makedirs(args.output_dir) + + logger.info("Saving model checkpoint to %s", args.output_dir) + # Save a trained model, configuration and tokenizer using `save_pretrained()`. + # They can then be reloaded using `from_pretrained()` + model_to_save = model.module if hasattr(model, 'module') else model # Take care of distributed/parallel training + model_to_save.save_pretrained(args.output_dir) + tokenizer.save_pretrained(args.output_dir) + + # Good practice: save your training arguments together with the trained model + torch.save(args, os.path.join(args.output_dir, 'training_args.bin')) + + # Load a trained model and vocabulary that you have fine-tuned + model = model_class.from_pretrained(args.output_dir) + tokenizer = tokenizer_class.from_pretrained(args.output_dir) + model.to(args.device) + + + # Evaluation + results = {} + if args.do_eval and args.local_rank in [-1, 0]: + if not args.do_train: + args.output_dir = args.model_name_or_path + checkpoints = [args.output_dir] + if args.eval_all_checkpoints: + checkpoints = list(os.path.dirname(c) for c in sorted(glob.glob(args.output_dir + '/**/' + WEIGHTS_NAME, recursive=True))) + logging.getLogger("pytorch_transformers.modeling_utils").setLevel(logging.WARN) # Reduce logging + logger.info("Evaluate the following checkpoints: %s", checkpoints) + for checkpoint in checkpoints: + global_step = checkpoint.split('-')[-1] if len(checkpoints) > 1 else "" + model = model_class.from_pretrained(checkpoint) + model.to(args.device) + result = evaluate(args, model, tokenizer, prefix=global_step) + result = dict((k + '_{}'.format(global_step), v) for k, v in result.items()) + results.update(result) + + if args.do_test and args.local_rank in [-1, 0]: + if not args.do_train: + args.output_dir = args.model_name_or_path + checkpoints = [args.output_dir] + # if args.eval_all_checkpoints: # can not use this to do test!! + # checkpoints = list(os.path.dirname(c) for c in sorted(glob.glob(args.output_dir + '/**/' + WEIGHTS_NAME, recursive=True))) + # logging.getLogger("pytorch_transformers.modeling_utils").setLevel(logging.WARN) # Reduce logging + logger.info("Evaluate the following checkpoints: %s", checkpoints) + for checkpoint in checkpoints: + global_step = checkpoint.split('-')[-1] if len(checkpoints) > 1 else "" + model = model_class.from_pretrained(checkpoint) + model.to(args.device) + result = evaluate(args, model, tokenizer, prefix=global_step, test=True) + result = dict((k + '_{}'.format(global_step), v) for k, v in result.items()) + results.update(result) + if best_steps: + logger.info("best steps of eval acc is the following checkpoints: %s", best_steps) + return results + + +if __name__ == "__main__": + main() diff --git a/examples/run_squad.py b/examples/run_squad.py index cc4eda306c..affef90ca9 100644 --- a/examples/run_squad.py +++ b/examples/run_squad.py @@ -37,7 +37,8 @@ from pytorch_transformers import (WEIGHTS_NAME, BertConfig, XLMConfig, XLMForQuestionAnswering, XLMTokenizer, XLNetConfig, XLNetForQuestionAnswering, - XLNetTokenizer) + XLNetTokenizer, + DistilBertConfig, DistilBertForQuestionAnswering, DistilBertTokenizer) from pytorch_transformers import AdamW, WarmupLinearSchedule @@ -59,6 +60,7 @@ MODEL_CLASSES = { 'bert': (BertConfig, BertForQuestionAnswering, BertTokenizer), 'xlnet': (XLNetConfig, XLNetForQuestionAnswering, XLNetTokenizer), 'xlm': (XLMConfig, XLMForQuestionAnswering, XLMTokenizer), + 'distilbert': (DistilBertConfig, DistilBertForQuestionAnswering, DistilBertTokenizer) } def set_seed(args): diff --git a/examples/single_model_scripts/run_swag.py b/examples/single_model_scripts/run_swag.py deleted file mode 100644 index fdda56e40b..0000000000 --- a/examples/single_model_scripts/run_swag.py +++ /dev/null @@ -1,555 +0,0 @@ -# coding=utf-8 -# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. -# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""BERT finetuning runner.""" - -from __future__ import absolute_import - -import argparse -import csv -import logging -import os -import random -import sys -from io import open - -import numpy as np -import torch -from torch.utils.data import (DataLoader, RandomSampler, SequentialSampler, - TensorDataset) -from torch.utils.data.distributed import DistributedSampler -from tqdm import tqdm, trange - -from pytorch_transformers.file_utils import PYTORCH_PRETRAINED_BERT_CACHE, WEIGHTS_NAME, CONFIG_NAME -from pytorch_transformers.modeling_bert import BertForMultipleChoice, BertConfig -from pytorch_transformers.optimization import AdamW, WarmupLinearSchedule -from pytorch_transformers.tokenization_bert import BertTokenizer - -logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', - datefmt = '%m/%d/%Y %H:%M:%S', - level = logging.INFO) -logger = logging.getLogger(__name__) - - -class SwagExample(object): - """A single training/test example for the SWAG dataset.""" - def __init__(self, - swag_id, - context_sentence, - start_ending, - ending_0, - ending_1, - ending_2, - ending_3, - label = None): - self.swag_id = swag_id - self.context_sentence = context_sentence - self.start_ending = start_ending - self.endings = [ - ending_0, - ending_1, - ending_2, - ending_3, - ] - self.label = label - - def __str__(self): - return self.__repr__() - - def __repr__(self): - l = [ - "swag_id: {}".format(self.swag_id), - "context_sentence: {}".format(self.context_sentence), - "start_ending: {}".format(self.start_ending), - "ending_0: {}".format(self.endings[0]), - "ending_1: {}".format(self.endings[1]), - "ending_2: {}".format(self.endings[2]), - "ending_3: {}".format(self.endings[3]), - ] - - if self.label is not None: - l.append("label: {}".format(self.label)) - - return ", ".join(l) - - -class InputFeatures(object): - def __init__(self, - example_id, - choices_features, - label - - ): - self.example_id = example_id - self.choices_features = [ - { - 'input_ids': input_ids, - 'input_mask': input_mask, - 'segment_ids': segment_ids - } - for _, input_ids, input_mask, segment_ids in choices_features - ] - self.label = label - - -def read_swag_examples(input_file, is_training): - with open(input_file, 'r', encoding='utf-8') as f: - reader = csv.reader(f) - lines = [] - for line in reader: - if sys.version_info[0] == 2: - line = list(unicode(cell, 'utf-8') for cell in line) - lines.append(line) - - if is_training and lines[0][-1] != 'label': - raise ValueError( - "For training, the input file must contain a label column." - ) - - examples = [ - SwagExample( - swag_id = line[2], - context_sentence = line[4], - start_ending = line[5], # in the swag dataset, the - # common beginning of each - # choice is stored in "sent2". - ending_0 = line[7], - ending_1 = line[8], - ending_2 = line[9], - ending_3 = line[10], - label = int(line[11]) if is_training else None - ) for line in lines[1:] # we skip the line with the column names - ] - - return examples - -def convert_examples_to_features(examples, tokenizer, max_seq_length, - is_training): - """Loads a data file into a list of `InputBatch`s.""" - - # Swag is a multiple choice task. To perform this task using Bert, - # we will use the formatting proposed in "Improving Language - # Understanding by Generative Pre-Training" and suggested by - # @jacobdevlin-google in this issue - # https://github.com/google-research/bert/issues/38. - # - # Each choice will correspond to a sample on which we run the - # inference. For a given Swag example, we will create the 4 - # following inputs: - # - [CLS] context [SEP] choice_1 [SEP] - # - [CLS] context [SEP] choice_2 [SEP] - # - [CLS] context [SEP] choice_3 [SEP] - # - [CLS] context [SEP] choice_4 [SEP] - # The model will output a single value for each input. To get the - # final decision of the model, we will run a softmax over these 4 - # outputs. - features = [] - for example_index, example in enumerate(examples): - context_tokens = tokenizer.tokenize(example.context_sentence) - start_ending_tokens = tokenizer.tokenize(example.start_ending) - - choices_features = [] - for ending_index, ending in enumerate(example.endings): - # We create a copy of the context tokens in order to be - # able to shrink it according to ending_tokens - context_tokens_choice = context_tokens[:] - ending_tokens = start_ending_tokens + tokenizer.tokenize(ending) - # Modifies `context_tokens_choice` and `ending_tokens` in - # place so that the total length is less than the - # specified length. Account for [CLS], [SEP], [SEP] with - # "- 3" - _truncate_seq_pair(context_tokens_choice, ending_tokens, max_seq_length - 3) - - tokens = ["[CLS]"] + context_tokens_choice + ["[SEP]"] + ending_tokens + ["[SEP]"] - segment_ids = [0] * (len(context_tokens_choice) + 2) + [1] * (len(ending_tokens) + 1) - - input_ids = tokenizer.convert_tokens_to_ids(tokens) - input_mask = [1] * len(input_ids) - - # Zero-pad up to the sequence length. - padding = [0] * (max_seq_length - len(input_ids)) - input_ids += padding - input_mask += padding - segment_ids += padding - - assert len(input_ids) == max_seq_length - assert len(input_mask) == max_seq_length - assert len(segment_ids) == max_seq_length - - choices_features.append((tokens, input_ids, input_mask, segment_ids)) - - label = example.label - if example_index < 5: - logger.info("*** Example ***") - logger.info("swag_id: {}".format(example.swag_id)) - for choice_idx, (tokens, input_ids, input_mask, segment_ids) in enumerate(choices_features): - logger.info("choice: {}".format(choice_idx)) - logger.info("tokens: {}".format(' '.join(tokens))) - logger.info("input_ids: {}".format(' '.join(map(str, input_ids)))) - logger.info("input_mask: {}".format(' '.join(map(str, input_mask)))) - logger.info("segment_ids: {}".format(' '.join(map(str, segment_ids)))) - if is_training: - logger.info("label: {}".format(label)) - - features.append( - InputFeatures( - example_id = example.swag_id, - choices_features = choices_features, - label = label - ) - ) - - return features - -def _truncate_seq_pair(tokens_a, tokens_b, max_length): - """Truncates a sequence pair in place to the maximum length.""" - - # This is a simple heuristic which will always truncate the longer sequence - # one token at a time. This makes more sense than truncating an equal percent - # of tokens from each, since if one sequence is very short then each token - # that's truncated likely contains more information than a longer sequence. - while True: - total_length = len(tokens_a) + len(tokens_b) - if total_length <= max_length: - break - if len(tokens_a) > len(tokens_b): - tokens_a.pop() - else: - tokens_b.pop() - -def accuracy(out, labels): - outputs = np.argmax(out, axis=1) - return np.sum(outputs == labels) - -def select_field(features, field): - return [ - [ - choice[field] - for choice in feature.choices_features - ] - for feature in features - ] - -def main(): - parser = argparse.ArgumentParser() - - ## Required parameters - parser.add_argument("--data_dir", - default=None, - type=str, - required=True, - help="The input data dir. Should contain the .csv files (or other data files) for the task.") - parser.add_argument("--bert_model", default=None, type=str, required=True, - help="Bert pre-trained model selected in the list: bert-base-uncased, " - "bert-large-uncased, bert-base-cased, bert-large-cased, bert-base-multilingual-uncased, " - "bert-base-multilingual-cased, bert-base-chinese.") - parser.add_argument("--output_dir", - default=None, - type=str, - required=True, - help="The output directory where the model checkpoints will be written.") - - ## Other parameters - parser.add_argument("--max_seq_length", - default=128, - type=int, - help="The maximum total input sequence length after WordPiece tokenization. \n" - "Sequences longer than this will be truncated, and sequences shorter \n" - "than this will be padded.") - parser.add_argument("--do_train", - action='store_true', - help="Whether to run training.") - parser.add_argument("--do_eval", - action='store_true', - help="Whether to run eval on the dev set.") - parser.add_argument("--do_lower_case", - action='store_true', - help="Set this flag if you are using an uncased model.") - parser.add_argument("--train_batch_size", - default=32, - type=int, - help="Total batch size for training.") - parser.add_argument("--eval_batch_size", - default=8, - type=int, - help="Total batch size for eval.") - parser.add_argument("--learning_rate", - default=5e-5, - type=float, - help="The initial learning rate for Adam.") - parser.add_argument("--num_train_epochs", - default=3.0, - type=float, - help="Total number of training epochs to perform.") - parser.add_argument("--warmup_proportion", - default=0.1, - type=float, - help="Proportion of training to perform linear learning rate warmup for. " - "E.g., 0.1 = 10%% of training.") - parser.add_argument("--no_cuda", - action='store_true', - help="Whether not to use CUDA when available") - parser.add_argument("--local_rank", - type=int, - default=-1, - help="local_rank for distributed training on gpus") - parser.add_argument('--seed', - type=int, - default=42, - help="random seed for initialization") - parser.add_argument('--gradient_accumulation_steps', - type=int, - default=1, - help="Number of updates steps to accumulate before performing a backward/update pass.") - parser.add_argument('--fp16', - action='store_true', - help="Whether to use 16-bit float precision instead of 32-bit") - parser.add_argument('--loss_scale', - type=float, default=0, - help="Loss scaling to improve fp16 numeric stability. Only used when fp16 set to True.\n" - "0 (default value): dynamic loss scaling.\n" - "Positive power of 2: static loss scaling value.\n") - - args = parser.parse_args() - - if args.local_rank == -1 or args.no_cuda: - device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu") - n_gpu = torch.cuda.device_count() - else: - torch.cuda.set_device(args.local_rank) - device = torch.device("cuda", args.local_rank) - n_gpu = 1 - # Initializes the distributed backend which will take care of sychronizing nodes/GPUs - torch.distributed.init_process_group(backend='nccl') - logger.info("device: {} n_gpu: {}, distributed training: {}, 16-bits training: {}".format( - device, n_gpu, bool(args.local_rank != -1), args.fp16)) - - if args.gradient_accumulation_steps < 1: - raise ValueError("Invalid gradient_accumulation_steps parameter: {}, should be >= 1".format( - args.gradient_accumulation_steps)) - - args.train_batch_size = args.train_batch_size // args.gradient_accumulation_steps - - random.seed(args.seed) - np.random.seed(args.seed) - torch.manual_seed(args.seed) - if n_gpu > 0: - torch.cuda.manual_seed_all(args.seed) - - if not args.do_train and not args.do_eval: - raise ValueError("At least one of `do_train` or `do_eval` must be True.") - - if os.path.exists(args.output_dir) and os.listdir(args.output_dir): - raise ValueError("Output directory ({}) already exists and is not empty.".format(args.output_dir)) - if not os.path.exists(args.output_dir): - os.makedirs(args.output_dir) - - tokenizer = BertTokenizer.from_pretrained(args.bert_model, do_lower_case=args.do_lower_case) - - # Prepare model - model = BertForMultipleChoice.from_pretrained(args.bert_model, - cache_dir=os.path.join(str(PYTORCH_PRETRAINED_BERT_CACHE), 'distributed_{}'.format(args.local_rank)), - num_choices=4) - if args.fp16: - model.half() - model.to(device) - if args.local_rank != -1: - try: - from apex.parallel import DistributedDataParallel as DDP - except ImportError: - raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training.") - - model = DDP(model) - elif n_gpu > 1: - model = torch.nn.DataParallel(model) - - if args.do_train: - - # Prepare data loader - - train_examples = read_swag_examples(os.path.join(args.data_dir, 'train.csv'), is_training = True) - train_features = convert_examples_to_features( - train_examples, tokenizer, args.max_seq_length, True) - all_input_ids = torch.tensor(select_field(train_features, 'input_ids'), dtype=torch.long) - all_input_mask = torch.tensor(select_field(train_features, 'input_mask'), dtype=torch.long) - all_segment_ids = torch.tensor(select_field(train_features, 'segment_ids'), dtype=torch.long) - all_label = torch.tensor([f.label for f in train_features], dtype=torch.long) - train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label) - if args.local_rank == -1: - train_sampler = RandomSampler(train_data) - else: - train_sampler = DistributedSampler(train_data) - train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=args.train_batch_size) - - num_train_optimization_steps = len(train_dataloader) // args.gradient_accumulation_steps * args.num_train_epochs - if args.local_rank != -1: - num_train_optimization_steps = num_train_optimization_steps // torch.distributed.get_world_size() - - # Prepare optimizer - - param_optimizer = list(model.named_parameters()) - - # hack to remove pooler, which is not used - # thus it produce None grad that break apex - param_optimizer = [n for n in param_optimizer] - - no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight'] - optimizer_grouped_parameters = [ - {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01}, - {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0} - ] - if args.fp16: - try: - from apex.optimizers import FP16_Optimizer - from apex.optimizers import FusedAdam - except ImportError: - raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training.") - - optimizer = FusedAdam(optimizer_grouped_parameters, - lr=args.learning_rate, - bias_correction=False, - max_grad_norm=1.0) - if args.loss_scale == 0: - optimizer = FP16_Optimizer(optimizer, dynamic_loss_scale=True) - else: - optimizer = FP16_Optimizer(optimizer, static_loss_scale=args.loss_scale) - warmup_linear = WarmupLinearSchedule(warmup=args.warmup_proportion, - t_total=num_train_optimization_steps) - else: - optimizer = BertAdam(optimizer_grouped_parameters, - lr=args.learning_rate, - warmup=args.warmup_proportion, - t_total=num_train_optimization_steps) - - global_step = 0 - - logger.info("***** Running training *****") - logger.info(" Num examples = %d", len(train_examples)) - logger.info(" Batch size = %d", args.train_batch_size) - logger.info(" Num steps = %d", num_train_optimization_steps) - - model.train() - for _ in trange(int(args.num_train_epochs), desc="Epoch"): - tr_loss = 0 - nb_tr_examples, nb_tr_steps = 0, 0 - for step, batch in enumerate(tqdm(train_dataloader, desc="Iteration")): - batch = tuple(t.to(device) for t in batch) - input_ids, input_mask, segment_ids, label_ids = batch - loss = model(input_ids, segment_ids, input_mask, label_ids) - if n_gpu > 1: - loss = loss.mean() # mean() to average on multi-gpu. - if args.fp16 and args.loss_scale != 1.0: - # rescale loss for fp16 training - # see https://docs.nvidia.com/deeplearning/sdk/mixed-precision-training/index.html - loss = loss * args.loss_scale - if args.gradient_accumulation_steps > 1: - loss = loss / args.gradient_accumulation_steps - tr_loss += loss.item() - nb_tr_examples += input_ids.size(0) - nb_tr_steps += 1 - - if args.fp16: - optimizer.backward(loss) - else: - loss.backward() - if (step + 1) % args.gradient_accumulation_steps == 0: - if args.fp16: - # modify learning rate with special warm up BERT uses - # if args.fp16 is False, BertAdam is used that handles this automatically - lr_this_step = args.learning_rate * warmup_linear.get_lr(global_step, args.warmup_proportion) - for param_group in optimizer.param_groups: - param_group['lr'] = lr_this_step - optimizer.step() - optimizer.zero_grad() - global_step += 1 - - - if args.do_train: - # Save a trained model, configuration and tokenizer - model_to_save = model.module if hasattr(model, 'module') else model # Only save the model it-self - - # If we save using the predefined names, we can load using `from_pretrained` - output_model_file = os.path.join(args.output_dir, WEIGHTS_NAME) - output_config_file = os.path.join(args.output_dir, CONFIG_NAME) - - torch.save(model_to_save.state_dict(), output_model_file) - model_to_save.config.to_json_file(output_config_file) - tokenizer.save_vocabulary(args.output_dir) - - # Load a trained model and vocabulary that you have fine-tuned - model = BertForMultipleChoice.from_pretrained(args.output_dir, num_choices=4) - tokenizer = BertTokenizer.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case) - else: - model = BertForMultipleChoice.from_pretrained(args.bert_model, num_choices=4) - model.to(device) - - - if args.do_eval and (args.local_rank == -1 or torch.distributed.get_rank() == 0): - eval_examples = read_swag_examples(os.path.join(args.data_dir, 'val.csv'), is_training = True) - eval_features = convert_examples_to_features( - eval_examples, tokenizer, args.max_seq_length, True) - logger.info("***** Running evaluation *****") - logger.info(" Num examples = %d", len(eval_examples)) - logger.info(" Batch size = %d", args.eval_batch_size) - all_input_ids = torch.tensor(select_field(eval_features, 'input_ids'), dtype=torch.long) - all_input_mask = torch.tensor(select_field(eval_features, 'input_mask'), dtype=torch.long) - all_segment_ids = torch.tensor(select_field(eval_features, 'segment_ids'), dtype=torch.long) - all_label = torch.tensor([f.label for f in eval_features], dtype=torch.long) - eval_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label) - # Run prediction for full data - eval_sampler = SequentialSampler(eval_data) - eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=args.eval_batch_size) - - model.eval() - eval_loss, eval_accuracy = 0, 0 - nb_eval_steps, nb_eval_examples = 0, 0 - for input_ids, input_mask, segment_ids, label_ids in tqdm(eval_dataloader, desc="Evaluating"): - input_ids = input_ids.to(device) - input_mask = input_mask.to(device) - segment_ids = segment_ids.to(device) - label_ids = label_ids.to(device) - - with torch.no_grad(): - tmp_eval_loss = model(input_ids, segment_ids, input_mask, label_ids) - logits = model(input_ids, segment_ids, input_mask) - - logits = logits.detach().cpu().numpy() - label_ids = label_ids.to('cpu').numpy() - tmp_eval_accuracy = accuracy(logits, label_ids) - - eval_loss += tmp_eval_loss.mean().item() - eval_accuracy += tmp_eval_accuracy - - nb_eval_examples += input_ids.size(0) - nb_eval_steps += 1 - - eval_loss = eval_loss / nb_eval_steps - eval_accuracy = eval_accuracy / nb_eval_examples - - result = {'eval_loss': eval_loss, - 'eval_accuracy': eval_accuracy, - 'global_step': global_step, - 'loss': tr_loss/global_step} - - output_eval_file = os.path.join(args.output_dir, "eval_results.txt") - with open(output_eval_file, "w") as writer: - logger.info("***** Eval results *****") - for key in sorted(result.keys()): - logger.info(" %s = %s", key, str(result[key])) - writer.write("%s = %s\n" % (key, str(result[key]))) - - -if __name__ == "__main__": - main() diff --git a/examples/utils_multiple_choice.py b/examples/utils_multiple_choice.py new file mode 100644 index 0000000000..7abcc5e1e9 --- /dev/null +++ b/examples/utils_multiple_choice.py @@ -0,0 +1,463 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" BERT multiple choice fine-tuning: utilities to work with multiple choice tasks of reading comprehension """ + +from __future__ import absolute_import, division, print_function + + +import logging +import os +import sys +from io import open +import json +import csv +import glob +import tqdm + + +logger = logging.getLogger(__name__) + + +class InputExample(object): + """A single training/test example for multiple choice""" + + def __init__(self, example_id, question, contexts, endings, label=None): + """Constructs a InputExample. + + Args: + example_id: Unique id for the example. + contexts: list of str. The untokenized text of the first sequence (context of corresponding question). + question: string. The untokenized text of the second sequence (qustion). + endings: list of str. multiple choice's options. Its length must be equal to contexts' length. + label: (Optional) string. The label of the example. This should be + specified for train and dev examples, but not for test examples. + """ + self.example_id = example_id + self.question = question + self.contexts = contexts + self.endings = endings + self.label = label + + +class InputFeatures(object): + def __init__(self, + example_id, + choices_features, + label + + ): + self.example_id = example_id + self.choices_features = [ + { + 'input_ids': input_ids, + 'input_mask': input_mask, + 'segment_ids': segment_ids + } + for _, input_ids, input_mask, segment_ids in choices_features + ] + self.label = label + + +class DataProcessor(object): + """Base class for data converters for multiple choice data sets.""" + + def get_train_examples(self, data_dir): + """Gets a collection of `InputExample`s for the train set.""" + raise NotImplementedError() + + def get_dev_examples(self, data_dir): + """Gets a collection of `InputExample`s for the dev set.""" + raise NotImplementedError() + + def get_test_examples(self, data_dir): + """Gets a collection of `InputExample`s for the test set.""" + raise NotImplementedError() + + def get_labels(self): + """Gets the list of labels for this data set.""" + raise NotImplementedError() + + +class RaceProcessor(DataProcessor): + """Processor for the RACE data set.""" + + def get_train_examples(self, data_dir): + """See base class.""" + logger.info("LOOKING AT {} train".format(data_dir)) + high = os.path.join(data_dir, 'train/high') + middle = os.path.join(data_dir, 'train/middle') + high = self._read_txt(high) + middle = self._read_txt(middle) + return self._create_examples(high + middle, 'train') + + def get_dev_examples(self, data_dir): + """See base class.""" + logger.info("LOOKING AT {} dev".format(data_dir)) + high = os.path.join(data_dir, 'dev/high') + middle = os.path.join(data_dir, 'dev/middle') + high = self._read_txt(high) + middle = self._read_txt(middle) + return self._create_examples(high + middle, 'dev') + + def get_test_examples(self, data_dir): + """See base class.""" + logger.info("LOOKING AT {} test".format(data_dir)) + high = os.path.join(data_dir, 'test/high') + middle = os.path.join(data_dir, 'test/middle') + high = self._read_txt(high) + middle = self._read_txt(middle) + return self._create_examples(high + middle, 'test') + + def get_labels(self): + """See base class.""" + return ["0", "1", "2", "3"] + + def _read_txt(self, input_dir): + lines = [] + files = glob.glob(input_dir + "/*txt") + for file in tqdm.tqdm(files, desc="read files"): + with open(file, 'r', encoding='utf-8') as fin: + data_raw = json.load(fin) + data_raw["race_id"] = file + lines.append(data_raw) + return lines + + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (_, data_raw) in enumerate(lines): + race_id = "%s-%s" % (set_type, data_raw["race_id"]) + article = data_raw["article"] + for i in range(len(data_raw["answers"])): + truth = str(ord(data_raw['answers'][i]) - ord('A')) + question = data_raw['questions'][i] + options = data_raw['options'][i] + + examples.append( + InputExample( + example_id=race_id, + question=question, + contexts=[article, article, article, article], # this is not efficient but convenient + endings=[options[0], options[1], options[2], options[3]], + label=truth)) + return examples + +class SwagProcessor(DataProcessor): + """Processor for the SWAG data set.""" + + def get_train_examples(self, data_dir): + """See base class.""" + logger.info("LOOKING AT {} train".format(data_dir)) + return self._create_examples(self._read_csv(os.path.join(data_dir, "train.csv")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + logger.info("LOOKING AT {} dev".format(data_dir)) + return self._create_examples(self._read_csv(os.path.join(data_dir, "val.csv")), "dev") + + def get_test_examples(self, data_dir): + """See base class.""" + logger.info("LOOKING AT {} dev".format(data_dir)) + raise ValueError( + "For swag testing, the input file does not contain a label column. It can not be tested in current code" + "setting!" + ) + return self._create_examples(self._read_csv(os.path.join(data_dir, "test.csv")), "test") + def get_labels(self): + """See base class.""" + return ["0", "1", "2", "3"] + + def _read_csv(self, input_file): + with open(input_file, 'r', encoding='utf-8') as f: + reader = csv.reader(f) + lines = [] + for line in reader: + if sys.version_info[0] == 2: + line = list(unicode(cell, 'utf-8') for cell in line) + lines.append(line) + return lines + + + def _create_examples(self, lines, type): + """Creates examples for the training and dev sets.""" + if type == "train" and lines[0][-1] != 'label': + raise ValueError( + "For training, the input file must contain a label column." + ) + + examples = [ + InputExample( + example_id=line[2], + question=line[5], # in the swag dataset, the + # common beginning of each + # choice is stored in "sent2". + contexts = [line[4], line[4], line[4], line[4]], + endings = [line[7], line[8], line[9], line[10]], + label=line[11] + ) for line in lines[1:] # we skip the line with the column names + ] + + return examples + + +class ArcProcessor(DataProcessor): + """Processor for the ARC data set (request from allennlp).""" + + def get_train_examples(self, data_dir): + """See base class.""" + logger.info("LOOKING AT {} train".format(data_dir)) + return self._create_examples(self._read_json(os.path.join(data_dir, "train.jsonl")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + logger.info("LOOKING AT {} dev".format(data_dir)) + return self._create_examples(self._read_json(os.path.join(data_dir, "dev.jsonl")), "dev") + + def get_test_examples(self, data_dir): + logger.info("LOOKING AT {} test".format(data_dir)) + return self._create_examples(self._read_json(os.path.join(data_dir, "test.jsonl")), "test") + + def get_labels(self): + """See base class.""" + return ["0", "1", "2", "3"] + + def _read_json(self, input_file): + with open(input_file, 'r', encoding='utf-8') as fin: + lines = fin.readlines() + return lines + + + def _create_examples(self, lines, type): + """Creates examples for the training and dev sets.""" + + #There are two types of labels. They should be normalized + def normalize(truth): + if truth in "ABCD": + return ord(truth) - ord("A") + elif truth in "1234": + return int(truth) - 1 + else: + logger.info("truth ERROR! %s", str(truth)) + return None + + examples = [] + three_choice = 0 + four_choice = 0 + five_choice = 0 + other_choices = 0 + # we deleted example which has more than or less than four choices + for line in tqdm.tqdm(lines, desc="read arc data"): + data_raw = json.loads(line.strip("\n")) + if len(data_raw["question"]["choices"]) == 3: + three_choice += 1 + continue + elif len(data_raw["question"]["choices"]) == 5: + five_choice += 1 + continue + elif len(data_raw["question"]["choices"]) != 4: + other_choices += 1 + continue + four_choice += 1 + truth = str(normalize(data_raw["answerKey"])) + assert truth != "None" + question_choices = data_raw["question"] + question = question_choices["stem"] + id = data_raw["id"] + options = question_choices["choices"] + if len(options) == 4: + examples.append( + InputExample( + example_id = id, + question=question, + contexts=[options[0]["para"].replace("_", ""), options[1]["para"].replace("_", ""), + options[2]["para"].replace("_", ""), options[3]["para"].replace("_", "")], + endings=[options[0]["text"], options[1]["text"], options[2]["text"], options[3]["text"]], + label=truth)) + + if type == "train": + assert len(examples) > 1 + assert examples[0].label is not None + logger.info("len examples: %s}", str(len(examples))) + logger.info("Three choices: %s", str(three_choice)) + logger.info("Five choices: %s", str(five_choice)) + logger.info("Other choices: %s", str(other_choices)) + logger.info("four choices: %s", str(four_choice)) + + return examples + + +def convert_examples_to_features(examples, label_list, max_seq_length, + tokenizer, + cls_token_at_end=False, + cls_token='[CLS]', + cls_token_segment_id=1, + sep_token='[SEP]', + sequence_a_segment_id=0, + sequence_b_segment_id=1, + sep_token_extra=False, + pad_token_segment_id=0, + pad_on_left=False, + pad_token=0, + mask_padding_with_zero=True): + """ Loads a data file into a list of `InputBatch`s + `cls_token_at_end` define the location of the CLS token: + - False (Default, BERT/XLM pattern): [CLS] + A + [SEP] + B + [SEP] + - True (XLNet/GPT pattern): A + [SEP] + B + [SEP] + [CLS] + `cls_token_segment_id` define the segment id associated to the CLS token (0 for BERT, 2 for XLNet) + """ + + label_map = {label : i for i, label in enumerate(label_list)} + + features = [] + for (ex_index, example) in tqdm.tqdm(enumerate(examples), desc="convert examples to features"): + if ex_index % 10000 == 0: + logger.info("Writing example %d of %d" % (ex_index, len(examples))) + choices_features = [] + for ending_idx, (context, ending) in enumerate(zip(example.contexts, example.endings)): + tokens_a = tokenizer.tokenize(context) + tokens_b = None + if example.question.find("_") != -1: + #this is for cloze question + tokens_b = tokenizer.tokenize(example.question.replace("_", ending)) + else: + tokens_b = tokenizer.tokenize(example.question + " " + ending) + # you can add seq token between quesiotn and ending. This does not make too much difference. + # tokens_b = tokenizer.tokenize(example.question) + # tokens_b += [sep_token] + # if sep_token_extra: + # tokens_b += [sep_token] + # tokens_b += tokenizer.tokenize(ending) + + special_tokens_count = 4 if sep_token_extra else 3 + _truncate_seq_pair(tokens_a, tokens_b, max_seq_length - special_tokens_count) + + # The convention in BERT is: + # (a) For sequence pairs: + # tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP] + # type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1 + # (b) For single sequences: + # tokens: [CLS] the dog is hairy . [SEP] + # type_ids: 0 0 0 0 0 0 0 + # + # Where "type_ids" are used to indicate whether this is the first + # sequence or the second sequence. The embedding vectors for `type=0` and + # `type=1` were learned during pre-training and are added to the wordpiece + # embedding vector (and position vector). This is not *strictly* necessary + # since the [SEP] token unambiguously separates the sequences, but it makes + # it easier for the model to learn the concept of sequences. + # + # For classification tasks, the first vector (corresponding to [CLS]) is + # used as as the "sentence vector". Note that this only makes sense because + # the entire model is fine-tuned. + tokens = tokens_a + [sep_token] + if sep_token_extra: + # roberta uses an extra separator b/w pairs of sentences + tokens += [sep_token] + + segment_ids = [sequence_a_segment_id] * len(tokens) + + if tokens_b: + tokens += tokens_b + [sep_token] + segment_ids += [sequence_b_segment_id] * (len(tokens_b) + 1) + + if cls_token_at_end: + tokens = tokens + [cls_token] + segment_ids = segment_ids + [cls_token_segment_id] + else: + tokens = [cls_token] + tokens + segment_ids = [cls_token_segment_id] + segment_ids + + input_ids = tokenizer.convert_tokens_to_ids(tokens) + + # The mask has 1 for real tokens and 0 for padding tokens. Only real + # tokens are attended to. + input_mask = [1 if mask_padding_with_zero else 0] * len(input_ids) + + # Zero-pad up to the sequence length. + padding_length = max_seq_length - len(input_ids) + if pad_on_left: + input_ids = ([pad_token] * padding_length) + input_ids + input_mask = ([0 if mask_padding_with_zero else 1] * padding_length) + input_mask + segment_ids = ([pad_token_segment_id] * padding_length) + segment_ids + else: + input_ids = input_ids + ([pad_token] * padding_length) + input_mask = input_mask + ([0 if mask_padding_with_zero else 1] * padding_length) + segment_ids = segment_ids + ([pad_token_segment_id] * padding_length) + + assert len(input_ids) == max_seq_length + assert len(input_mask) == max_seq_length + assert len(segment_ids) == max_seq_length + choices_features.append((tokens, input_ids, input_mask, segment_ids)) + label = label_map[example.label] + + if ex_index < 2: + logger.info("*** Example ***") + logger.info("race_id: {}".format(example.example_id)) + for choice_idx, (tokens, input_ids, input_mask, segment_ids) in enumerate(choices_features): + logger.info("choice: {}".format(choice_idx)) + logger.info("tokens: {}".format(' '.join(tokens))) + logger.info("input_ids: {}".format(' '.join(map(str, input_ids)))) + logger.info("input_mask: {}".format(' '.join(map(str, input_mask)))) + logger.info("segment_ids: {}".format(' '.join(map(str, segment_ids)))) + logger.info("label: {}".format(label)) + + features.append( + InputFeatures( + example_id = example.example_id, + choices_features = choices_features, + label = label + ) + ) + + return features + + +def _truncate_seq_pair(tokens_a, tokens_b, max_length): + """Truncates a sequence pair in place to the maximum length.""" + + # This is a simple heuristic which will always truncate the longer sequence + # one token at a time. This makes more sense than truncating an equal percent + # of tokens from each, since if one sequence is very short then each token + # that's truncated likely contains more information than a longer sequence. + + # However, since we'd better not to remove tokens of options and questions, you can choose to use a bigger + # length or only pop from context + while True: + total_length = len(tokens_a) + len(tokens_b) + if total_length <= max_length: + break + if len(tokens_a) > len(tokens_b): + tokens_a.pop() + else: + logger.info('Attention! you are removing from token_b (swag task is ok). ' + 'If you are training ARC and RACE (you are poping question + options), ' + 'you need to try to use a bigger max seq length!') + tokens_b.pop() + + +processors = { + "race": RaceProcessor, + "swag": SwagProcessor, + "arc": ArcProcessor +} + + +GLUE_TASKS_NUM_LABELS = { + "race", 4, + "swag", 4, + "arc", 4 +} diff --git a/hubconf.py b/hubconf.py index 35e7f1eea8..d9aaa6b53a 100644 --- a/hubconf.py +++ b/hubconf.py @@ -1,7 +1,7 @@ from pytorch_transformers import ( AutoTokenizer, AutoConfig, AutoModel, AutoModelWithLMHead, AutoModelForSequenceClassification, AutoModelForQuestionAnswering ) -from pytorch_transformers.modeling_utils import add_start_docstrings +from pytorch_transformers.file_utils import add_start_docstrings dependencies = ['torch', 'tqdm', 'boto3', 'requests', 'regex', 'sentencepiece', 'sacremoses'] diff --git a/pytorch_transformers/__init__.py b/pytorch_transformers/__init__.py index 5efbece795..68f81be085 100644 --- a/pytorch_transformers/__init__.py +++ b/pytorch_transformers/__init__.py @@ -164,4 +164,12 @@ if _tf_available and _torch_available: from .file_utils import (PYTORCH_TRANSFORMERS_CACHE, PYTORCH_PRETRAINED_BERT_CACHE, cached_path, add_start_docstrings, add_end_docstrings, WEIGHTS_NAME, TF2_WEIGHTS_NAME, TF_WEIGHTS_NAME, CONFIG_NAME, - is_tf_available, is_torch_available) \ No newline at end of file + is_tf_available, is_torch_available) + +from .data import (is_sklearn_available, + InputExample, InputFeatures, DataProcessor, + glue_output_modes, glue_convert_examples_to_features, + glue_processors, glue_tasks_num_labels) + +if is_sklearn_available(): + from .data import glue_compute_metrics diff --git a/pytorch_transformers/data/__init__.py b/pytorch_transformers/data/__init__.py new file mode 100644 index 0000000000..e910d6da2e --- /dev/null +++ b/pytorch_transformers/data/__init__.py @@ -0,0 +1,6 @@ +from .processors import InputExample, InputFeatures, DataProcessor +from .processors import glue_output_modes, glue_processors, glue_tasks_num_labels, glue_convert_examples_to_features + +from .metrics import is_sklearn_available +if is_sklearn_available(): + from .metrics import glue_compute_metrics diff --git a/pytorch_transformers/data/metrics/__init__.py b/pytorch_transformers/data/metrics/__init__.py new file mode 100644 index 0000000000..c9ebaac38d --- /dev/null +++ b/pytorch_transformers/data/metrics/__init__.py @@ -0,0 +1,83 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import csv +import sys +import logging + +logger = logging.getLogger(__name__) + +try: + from scipy.stats import pearsonr, spearmanr + from sklearn.metrics import matthews_corrcoef, f1_score + _has_sklearn = True +except (AttributeError, ImportError) as e: + logger.warning("To use data.metrics please install scikit-learn. See https://scikit-learn.org/stable/index.html") + _has_sklearn = False + +def is_sklearn_available(): + return _has_sklearn + +if _has_sklearn: + + def simple_accuracy(preds, labels): + return (preds == labels).mean() + + + def acc_and_f1(preds, labels): + acc = simple_accuracy(preds, labels) + f1 = f1_score(y_true=labels, y_pred=preds) + return { + "acc": acc, + "f1": f1, + "acc_and_f1": (acc + f1) / 2, + } + + + def pearson_and_spearman(preds, labels): + pearson_corr = pearsonr(preds, labels)[0] + spearman_corr = spearmanr(preds, labels)[0] + return { + "pearson": pearson_corr, + "spearmanr": spearman_corr, + "corr": (pearson_corr + spearman_corr) / 2, + } + + + def glue_compute_metrics(task_name, preds, labels): + assert len(preds) == len(labels) + if task_name == "cola": + return {"mcc": matthews_corrcoef(labels, preds)} + elif task_name == "sst-2": + return {"acc": simple_accuracy(preds, labels)} + elif task_name == "mrpc": + return acc_and_f1(preds, labels) + elif task_name == "sts-b": + return pearson_and_spearman(preds, labels) + elif task_name == "qqp": + return acc_and_f1(preds, labels) + elif task_name == "mnli": + return {"acc": simple_accuracy(preds, labels)} + elif task_name == "mnli-mm": + return {"acc": simple_accuracy(preds, labels)} + elif task_name == "qnli": + return {"acc": simple_accuracy(preds, labels)} + elif task_name == "rte": + return {"acc": simple_accuracy(preds, labels)} + elif task_name == "wnli": + return {"acc": simple_accuracy(preds, labels)} + else: + raise KeyError(task_name) diff --git a/pytorch_transformers/data/processors/__init__.py b/pytorch_transformers/data/processors/__init__.py new file mode 100644 index 0000000000..af38c54beb --- /dev/null +++ b/pytorch_transformers/data/processors/__init__.py @@ -0,0 +1,3 @@ +from .utils import InputExample, InputFeatures, DataProcessor +from .glue import glue_output_modes, glue_processors, glue_tasks_num_labels, glue_convert_examples_to_features + diff --git a/examples/utils_glue.py b/pytorch_transformers/data/processors/glue.py similarity index 60% rename from examples/utils_glue.py rename to pytorch_transformers/data/processors/glue.py index 3e3f104672..cb89ccf6c6 100644 --- a/examples/utils_glue.py +++ b/pytorch_transformers/data/processors/glue.py @@ -13,79 +13,81 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" BERT classification fine-tuning: utilities to work with GLUE tasks """ +""" GLUE processors and helpers """ -from __future__ import absolute_import, division, print_function - -import csv import logging import os -import sys -from io import open -from scipy.stats import pearsonr, spearmanr -from sklearn.metrics import matthews_corrcoef, f1_score +from .utils import DataProcessor, InputExample, InputFeatures logger = logging.getLogger(__name__) +def glue_convert_examples_to_features(examples, label_list, max_seq_length, + tokenizer, output_mode, + pad_on_left=False, + pad_token=0, + pad_token_segment_id=0, + mask_padding_with_zero=True): + """ + Loads a data file into a list of `InputBatch`s + """ -class InputExample(object): - """A single training/test example for simple sequence classification.""" + label_map = {label: i for i, label in enumerate(label_list)} - def __init__(self, guid, text_a, text_b=None, label=None): - """Constructs a InputExample. + features = [] + for (ex_index, example) in enumerate(examples): + if ex_index % 10000 == 0: + logger.info("Writing example %d of %d" % (ex_index, len(examples))) - Args: - guid: Unique id for the example. - text_a: string. The untokenized text of the first sequence. For single - sequence tasks, only this sequence must be specified. - text_b: (Optional) string. The untokenized text of the second sequence. - Only must be specified for sequence pair tasks. - label: (Optional) string. The label of the example. This should be - specified for train and dev examples, but not for test examples. - """ - self.guid = guid - self.text_a = text_a - self.text_b = text_b - self.label = label + inputs = tokenizer.encode_plus( + example.text_a, + example.text_b, + add_special_tokens=True, + max_length=max_seq_length, + truncate_first_sequence=True # We're truncating the first sequence as a priority + ) + input_ids, segment_ids = inputs["input_ids"], inputs["token_type_ids"] + # The mask has 1 for real tokens and 0 for padding tokens. Only real + # tokens are attended to. + input_mask = [1 if mask_padding_with_zero else 0] * len(input_ids) -class InputFeatures(object): - """A single set of features of data.""" + # Zero-pad up to the sequence length. + padding_length = max_seq_length - len(input_ids) + if pad_on_left: + input_ids = ([pad_token] * padding_length) + input_ids + input_mask = ([0 if mask_padding_with_zero else 1] * padding_length) + input_mask + segment_ids = ([pad_token_segment_id] * padding_length) + segment_ids + else: + input_ids = input_ids + ([pad_token] * padding_length) + input_mask = input_mask + ([0 if mask_padding_with_zero else 1] * padding_length) + segment_ids = segment_ids + ([pad_token_segment_id] * padding_length) - def __init__(self, input_ids, input_mask, segment_ids, label_id): - self.input_ids = input_ids - self.input_mask = input_mask - self.segment_ids = segment_ids - self.label_id = label_id + assert len(input_ids) == max_seq_length + assert len(input_mask) == max_seq_length + assert len(segment_ids) == max_seq_length + if output_mode == "classification": + label_id = label_map[example.label] + elif output_mode == "regression": + label_id = float(example.label) + else: + raise KeyError(output_mode) -class DataProcessor(object): - """Base class for data converters for sequence classification data sets.""" + if ex_index < 5: + logger.info("*** Example ***") + logger.info("guid: %s" % (example.guid)) + logger.info("input_ids: %s" % " ".join([str(x) for x in input_ids])) + logger.info("input_mask: %s" % " ".join([str(x) for x in input_mask])) + logger.info("segment_ids: %s" % " ".join([str(x) for x in segment_ids])) + logger.info("label: %s (id = %d)" % (example.label, label_id)) - def get_train_examples(self, data_dir): - """Gets a collection of `InputExample`s for the train set.""" - raise NotImplementedError() - - def get_dev_examples(self, data_dir): - """Gets a collection of `InputExample`s for the dev set.""" - raise NotImplementedError() - - def get_labels(self): - """Gets the list of labels for this data set.""" - raise NotImplementedError() - - @classmethod - def _read_tsv(cls, input_file, quotechar=None): - """Reads a tab separated value file.""" - with open(input_file, "r", encoding="utf-8-sig") as f: - reader = csv.reader(f, delimiter="\t", quotechar=quotechar) - lines = [] - for line in reader: - if sys.version_info[0] == 2: - line = list(unicode(cell, 'utf-8') for cell in line) - lines.append(line) - return lines + features.append( + InputFeatures(input_ids=input_ids, + input_mask=input_mask, + segment_ids=segment_ids, + label_id=label_id)) + return features class MrpcProcessor(DataProcessor): @@ -302,7 +304,7 @@ class QnliProcessor(DataProcessor): def get_dev_examples(self, data_dir): """See base class.""" return self._create_examples( - self._read_tsv(os.path.join(data_dir, "dev.tsv")), + self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev_matched") def get_labels(self): @@ -387,198 +389,19 @@ class WnliProcessor(DataProcessor): InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) return examples +glue_tasks_num_labels = { + "cola": 2, + "mnli": 3, + "mrpc": 2, + "sst-2": 2, + "sts-b": 1, + "qqp": 2, + "qnli": 2, + "rte": 2, + "wnli": 2, +} -def convert_examples_to_features(examples, label_list, max_seq_length, - tokenizer, output_mode, - cls_token_at_end=False, - cls_token='[CLS]', - cls_token_segment_id=1, - sep_token='[SEP]', - sep_token_extra=False, - pad_on_left=False, - pad_token=0, - pad_token_segment_id=0, - sequence_a_segment_id=0, - sequence_b_segment_id=1, - mask_padding_with_zero=True): - """ Loads a data file into a list of `InputBatch`s - `cls_token_at_end` define the location of the CLS token: - - False (Default, BERT/XLM pattern): [CLS] + A + [SEP] + B + [SEP] - - True (XLNet/GPT pattern): A + [SEP] + B + [SEP] + [CLS] - `cls_token_segment_id` define the segment id associated to the CLS token (0 for BERT, 2 for XLNet) - """ - - label_map = {label : i for i, label in enumerate(label_list)} - - features = [] - for (ex_index, example) in enumerate(examples): - if ex_index % 10000 == 0: - logger.info("Writing example %d of %d" % (ex_index, len(examples))) - - tokens_a = tokenizer.tokenize(example.text_a) - - tokens_b = None - if example.text_b: - tokens_b = tokenizer.tokenize(example.text_b) - # Modifies `tokens_a` and `tokens_b` in place so that the total - # length is less than the specified length. - # Account for [CLS], [SEP], [SEP] with "- 3". " -4" for RoBERTa. - special_tokens_count = 4 if sep_token_extra else 3 - _truncate_seq_pair(tokens_a, tokens_b, max_seq_length - special_tokens_count) - else: - # Account for [CLS] and [SEP] with "- 2" and with "- 3" for RoBERTa. - special_tokens_count = 3 if sep_token_extra else 2 - if len(tokens_a) > max_seq_length - special_tokens_count: - tokens_a = tokens_a[:(max_seq_length - special_tokens_count)] - - # The convention in BERT is: - # (a) For sequence pairs: - # tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP] - # type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1 - # (b) For single sequences: - # tokens: [CLS] the dog is hairy . [SEP] - # type_ids: 0 0 0 0 0 0 0 - # - # Where "type_ids" are used to indicate whether this is the first - # sequence or the second sequence. The embedding vectors for `type=0` and - # `type=1` were learned during pre-training and are added to the wordpiece - # embedding vector (and position vector). This is not *strictly* necessary - # since the [SEP] token unambiguously separates the sequences, but it makes - # it easier for the model to learn the concept of sequences. - # - # For classification tasks, the first vector (corresponding to [CLS]) is - # used as as the "sentence vector". Note that this only makes sense because - # the entire model is fine-tuned. - tokens = tokens_a + [sep_token] - if sep_token_extra: - # roberta uses an extra separator b/w pairs of sentences - tokens += [sep_token] - segment_ids = [sequence_a_segment_id] * len(tokens) - - if tokens_b: - tokens += tokens_b + [sep_token] - segment_ids += [sequence_b_segment_id] * (len(tokens_b) + 1) - - if cls_token_at_end: - tokens = tokens + [cls_token] - segment_ids = segment_ids + [cls_token_segment_id] - else: - tokens = [cls_token] + tokens - segment_ids = [cls_token_segment_id] + segment_ids - - input_ids = tokenizer.convert_tokens_to_ids(tokens) - - # The mask has 1 for real tokens and 0 for padding tokens. Only real - # tokens are attended to. - input_mask = [1 if mask_padding_with_zero else 0] * len(input_ids) - - # Zero-pad up to the sequence length. - padding_length = max_seq_length - len(input_ids) - if pad_on_left: - input_ids = ([pad_token] * padding_length) + input_ids - input_mask = ([0 if mask_padding_with_zero else 1] * padding_length) + input_mask - segment_ids = ([pad_token_segment_id] * padding_length) + segment_ids - else: - input_ids = input_ids + ([pad_token] * padding_length) - input_mask = input_mask + ([0 if mask_padding_with_zero else 1] * padding_length) - segment_ids = segment_ids + ([pad_token_segment_id] * padding_length) - - assert len(input_ids) == max_seq_length - assert len(input_mask) == max_seq_length - assert len(segment_ids) == max_seq_length - - if output_mode == "classification": - label_id = label_map[example.label] - elif output_mode == "regression": - label_id = float(example.label) - else: - raise KeyError(output_mode) - - if ex_index < 5: - logger.info("*** Example ***") - logger.info("guid: %s" % (example.guid)) - logger.info("tokens: %s" % " ".join( - [str(x) for x in tokens])) - logger.info("input_ids: %s" % " ".join([str(x) for x in input_ids])) - logger.info("input_mask: %s" % " ".join([str(x) for x in input_mask])) - logger.info("segment_ids: %s" % " ".join([str(x) for x in segment_ids])) - logger.info("label: %s (id = %d)" % (example.label, label_id)) - - features.append( - InputFeatures(input_ids=input_ids, - input_mask=input_mask, - segment_ids=segment_ids, - label_id=label_id)) - return features - - -def _truncate_seq_pair(tokens_a, tokens_b, max_length): - """Truncates a sequence pair in place to the maximum length.""" - - # This is a simple heuristic which will always truncate the longer sequence - # one token at a time. This makes more sense than truncating an equal percent - # of tokens from each, since if one sequence is very short then each token - # that's truncated likely contains more information than a longer sequence. - while True: - total_length = len(tokens_a) + len(tokens_b) - if total_length <= max_length: - break - if len(tokens_a) > len(tokens_b): - tokens_a.pop() - else: - tokens_b.pop() - - -def simple_accuracy(preds, labels): - return (preds == labels).mean() - - -def acc_and_f1(preds, labels): - acc = simple_accuracy(preds, labels) - f1 = f1_score(y_true=labels, y_pred=preds) - return { - "acc": acc, - "f1": f1, - "acc_and_f1": (acc + f1) / 2, - } - - -def pearson_and_spearman(preds, labels): - pearson_corr = pearsonr(preds, labels)[0] - spearman_corr = spearmanr(preds, labels)[0] - return { - "pearson": pearson_corr, - "spearmanr": spearman_corr, - "corr": (pearson_corr + spearman_corr) / 2, - } - - -def compute_metrics(task_name, preds, labels): - assert len(preds) == len(labels) - if task_name == "cola": - return {"mcc": matthews_corrcoef(labels, preds)} - elif task_name == "sst-2": - return {"acc": simple_accuracy(preds, labels)} - elif task_name == "mrpc": - return acc_and_f1(preds, labels) - elif task_name == "sts-b": - return pearson_and_spearman(preds, labels) - elif task_name == "qqp": - return acc_and_f1(preds, labels) - elif task_name == "mnli": - return {"acc": simple_accuracy(preds, labels)} - elif task_name == "mnli-mm": - return {"acc": simple_accuracy(preds, labels)} - elif task_name == "qnli": - return {"acc": simple_accuracy(preds, labels)} - elif task_name == "rte": - return {"acc": simple_accuracy(preds, labels)} - elif task_name == "wnli": - return {"acc": simple_accuracy(preds, labels)} - else: - raise KeyError(task_name) - -processors = { +glue_processors = { "cola": ColaProcessor, "mnli": MnliProcessor, "mnli-mm": MnliMismatchedProcessor, @@ -591,7 +414,7 @@ processors = { "wnli": WnliProcessor, } -output_modes = { +glue_output_modes = { "cola": "classification", "mnli": "classification", "mnli-mm": "classification", @@ -603,15 +426,3 @@ output_modes = { "rte": "classification", "wnli": "classification", } - -GLUE_TASKS_NUM_LABELS = { - "cola": 2, - "mnli": 3, - "mrpc": 2, - "sst-2": 2, - "sts-b": 1, - "qqp": 2, - "qnli": 2, - "rte": 2, - "wnli": 2, -} diff --git a/pytorch_transformers/data/processors/utils.py b/pytorch_transformers/data/processors/utils.py new file mode 100644 index 0000000000..af90e7a47c --- /dev/null +++ b/pytorch_transformers/data/processors/utils.py @@ -0,0 +1,75 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import csv +import sys + +class InputExample(object): + """A single training/test example for simple sequence classification.""" + def __init__(self, guid, text_a, text_b=None, label=None): + """Constructs a InputExample. + + Args: + guid: Unique id for the example. + text_a: string. The untokenized text of the first sequence. For single + sequence tasks, only this sequence must be specified. + text_b: (Optional) string. The untokenized text of the second sequence. + Only must be specified for sequence pair tasks. + label: (Optional) string. The label of the example. This should be + specified for train and dev examples, but not for test examples. + """ + self.guid = guid + self.text_a = text_a + self.text_b = text_b + self.label = label + + +class InputFeatures(object): + """A single set of features of data.""" + + def __init__(self, input_ids, input_mask, segment_ids, label_id): + self.input_ids = input_ids + self.input_mask = input_mask + self.segment_ids = segment_ids + self.label_id = label_id + + +class DataProcessor(object): + """Base class for data converters for sequence classification data sets.""" + + def get_train_examples(self, data_dir): + """Gets a collection of `InputExample`s for the train set.""" + raise NotImplementedError() + + def get_dev_examples(self, data_dir): + """Gets a collection of `InputExample`s for the dev set.""" + raise NotImplementedError() + + def get_labels(self): + """Gets the list of labels for this data set.""" + raise NotImplementedError() + + @classmethod + def _read_tsv(cls, input_file, quotechar=None): + """Reads a tab separated value file.""" + with open(input_file, "r", encoding="utf-8-sig") as f: + reader = csv.reader(f, delimiter="\t", quotechar=quotechar) + lines = [] + for line in reader: + if sys.version_info[0] == 2: + line = list(unicode(cell, 'utf-8') for cell in line) + lines.append(line) + return lines diff --git a/pytorch_transformers/modeling_bert.py b/pytorch_transformers/modeling_bert.py index 64ea5f947c..4a61a01641 100644 --- a/pytorch_transformers/modeling_bert.py +++ b/pytorch_transformers/modeling_bert.py @@ -533,7 +533,7 @@ BERT_INPUTS_DOCSTRING = r""" ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. """ -@add_start_docstrings("The bare Bert Model transformer outputing raw hidden-states without any specific head on top.", +@add_start_docstrings("The bare Bert Model transformer outputting raw hidden-states without any specific head on top.", BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) class BertModel(BertPreTrainedModel): r""" diff --git a/pytorch_transformers/modeling_distilbert.py b/pytorch_transformers/modeling_distilbert.py index 29ecbfb846..c5cc44be75 100644 --- a/pytorch_transformers/modeling_distilbert.py +++ b/pytorch_transformers/modeling_distilbert.py @@ -394,7 +394,7 @@ DISTILBERT_INPUTS_DOCSTRING = r""" ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. """ -@add_start_docstrings("The bare DistilBERT encoder/transformer outputing raw hidden-states without any specific head on top.", +@add_start_docstrings("The bare DistilBERT encoder/transformer outputting raw hidden-states without any specific head on top.", DISTILBERT_START_DOCSTRING, DISTILBERT_INPUTS_DOCSTRING) class DistilBertModel(DistilBertPreTrainedModel): r""" diff --git a/pytorch_transformers/modeling_gpt2.py b/pytorch_transformers/modeling_gpt2.py index 324d020fc3..ee246f1731 100644 --- a/pytorch_transformers/modeling_gpt2.py +++ b/pytorch_transformers/modeling_gpt2.py @@ -290,7 +290,7 @@ GPT2_INPUTS_DOCSTRING = r""" Inputs: Indices of input sequence tokens in the vocabulary. GPT-2 is a model with absolute position embeddings so it's usually advised to pad the inputs on the right rather than the left. - Indices can be obtained using :class:`pytorch_transformers.BPT2Tokenizer`. + Indices can be obtained using :class:`pytorch_transformers.GPT2Tokenizer`. See :func:`pytorch_transformers.PreTrainedTokenizer.encode` and :func:`pytorch_transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. **past**: @@ -314,7 +314,7 @@ GPT2_INPUTS_DOCSTRING = r""" Inputs: ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. """ -@add_start_docstrings("The bare GPT2 Model transformer outputing raw hidden-states without any specific head on top.", +@add_start_docstrings("The bare GPT2 Model transformer outputting raw hidden-states without any specific head on top.", GPT2_START_DOCSTRING, GPT2_INPUTS_DOCSTRING) class GPT2Model(GPT2PreTrainedModel): r""" diff --git a/pytorch_transformers/modeling_openai.py b/pytorch_transformers/modeling_openai.py index 9936e72030..4b02baf2f4 100644 --- a/pytorch_transformers/modeling_openai.py +++ b/pytorch_transformers/modeling_openai.py @@ -324,7 +324,7 @@ OPENAI_GPT_INPUTS_DOCSTRING = r""" Inputs: ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. """ -@add_start_docstrings("The bare OpenAI GPT transformer model outputing raw hidden-states without any specific head on top.", +@add_start_docstrings("The bare OpenAI GPT transformer model outputting raw hidden-states without any specific head on top.", OPENAI_GPT_START_DOCSTRING, OPENAI_GPT_INPUTS_DOCSTRING) class OpenAIGPTModel(OpenAIGPTPreTrainedModel): r""" diff --git a/pytorch_transformers/modeling_roberta.py b/pytorch_transformers/modeling_roberta.py index 41027330a8..9b30bcd4be 100644 --- a/pytorch_transformers/modeling_roberta.py +++ b/pytorch_transformers/modeling_roberta.py @@ -124,7 +124,7 @@ ROBERTA_INPUTS_DOCSTRING = r""" ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. """ -@add_start_docstrings("The bare RoBERTa Model transformer outputing raw hidden-states without any specific head on top.", +@add_start_docstrings("The bare RoBERTa Model transformer outputting raw hidden-states without any specific head on top.", ROBERTA_START_DOCSTRING, ROBERTA_INPUTS_DOCSTRING) class RobertaModel(BertModel): r""" @@ -296,7 +296,7 @@ class RobertaForSequenceClassification(BertPreTrainedModel): Examples:: - tokenizer = RoertaTokenizer.from_pretrained('roberta-base') + tokenizer = RobertaTokenizer.from_pretrained('roberta-base') model = RobertaForSequenceClassification.from_pretrained('roberta-base') input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 labels = torch.tensor([1]).unsqueeze(0) # Batch size 1 @@ -338,6 +338,113 @@ class RobertaForSequenceClassification(BertPreTrainedModel): return outputs # (loss), logits, (hidden_states), (attentions) +@add_start_docstrings("""Roberta Model with a multiple choice classification head on top (a linear layer on top of + the pooled output and a softmax) e.g. for RocStories/SWAG tasks. """, + ROBERTA_START_DOCSTRING, ROBERTA_INPUTS_DOCSTRING) +class RobertaForMultipleChoice(BertPreTrainedModel): + r""" + Inputs: + **input_ids**: ``torch.LongTensor`` of shape ``(batch_size, num_choices, sequence_length)``: + Indices of input sequence tokens in the vocabulary. + The second dimension of the input (`num_choices`) indicates the number of choices to score. + To match pre-training, RoBerta input sequence should be formatted with [CLS] and [SEP] tokens as follows: + + (a) For sequence pairs: + + ``tokens: [CLS] is this jack ##son ##ville ? [SEP] [SEP] no it is not . [SEP]`` + + ``token_type_ids: 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1`` + + (b) For single sequences: + + ``tokens: [CLS] the dog is hairy . [SEP]`` + + ``token_type_ids: 0 0 0 0 0 0 0`` + + Indices can be obtained using :class:`pytorch_transformers.BertTokenizer`. + See :func:`pytorch_transformers.PreTrainedTokenizer.encode` and + :func:`pytorch_transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + **token_type_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, num_choices, sequence_length)``: + Segment token indices to indicate first and second portions of the inputs. + The second dimension of the input (`num_choices`) indicates the number of choices to score. + Indices are selected in ``[0, 1]``: ``0`` corresponds to a `sentence A` token, ``1`` + **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, num_choices, sequence_length)``: + Mask to avoid performing attention on padding token indices. + The second dimension of the input (`num_choices`) indicates the number of choices to score. + Mask values selected in ``[0, 1]``: + ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. + **head_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: + Mask to nullify selected heads of the self-attention modules. + Mask values selected in ``[0, 1]``: + ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. + **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for computing the multiple choice classification loss. + Indices should be in ``[0, ..., num_choices]`` where `num_choices` is the size of the second dimension + of the input tensors. (see `input_ids` above) + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Classification loss. + **classification_scores**: ``torch.FloatTensor`` of shape ``(batch_size, num_choices)`` where `num_choices` is the size of the second dimension + of the input tensors. (see `input_ids` above). + Classification scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = RobertaTokenizer.from_pretrained('roberta-base') + model = RobertaForMultipleChoice.from_pretrained('roberta-base') + choices = ["Hello, my dog is cute", "Hello, my cat is amazing"] + input_ids = torch.tensor([tokenizer.encode(s, add_special_tokens=True) for s in choices]).unsqueeze(0) # Batch size 1, 2 choices + labels = torch.tensor(1).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, labels=labels) + loss, classification_scores = outputs[:2] + + """ + config_class = RobertaConfig + pretrained_model_archive_map = ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP + base_model_prefix = "roberta" + + def __init__(self, config): + super(RobertaForMultipleChoice, self).__init__(config) + + self.roberta = RobertaModel(config) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + self.classifier = nn.Linear(config.hidden_size, 1) + + self.init_weights() + + def forward(self, input_ids, token_type_ids=None, attention_mask=None, labels=None, + position_ids=None, head_mask=None): + num_choices = input_ids.shape[1] + + flat_input_ids = input_ids.view(-1, input_ids.size(-1)) + flat_position_ids = position_ids.view(-1, position_ids.size(-1)) if position_ids is not None else None + flat_token_type_ids = token_type_ids.view(-1, token_type_ids.size(-1)) if token_type_ids is not None else None + flat_attention_mask = attention_mask.view(-1, attention_mask.size(-1)) if attention_mask is not None else None + outputs = self.roberta(flat_input_ids, position_ids=flat_position_ids, token_type_ids=flat_token_type_ids, + attention_mask=flat_attention_mask, head_mask=head_mask) + pooled_output = outputs[1] + + pooled_output = self.dropout(pooled_output) + logits = self.classifier(pooled_output) + reshaped_logits = logits.view(-1, num_choices) + + outputs = (reshaped_logits,) + outputs[2:] # add hidden states and attention if they are here + + if labels is not None: + loss_fct = CrossEntropyLoss() + loss = loss_fct(reshaped_logits, labels) + outputs = (loss,) + outputs + + return outputs # (loss), reshaped_logits, (hidden_states), (attentions) + class RobertaClassificationHead(nn.Module): diff --git a/pytorch_transformers/modeling_transfo_xl.py b/pytorch_transformers/modeling_transfo_xl.py index c0919c8543..8925d01c50 100644 --- a/pytorch_transformers/modeling_transfo_xl.py +++ b/pytorch_transformers/modeling_transfo_xl.py @@ -321,11 +321,19 @@ class RelPartialLearnableMultiHeadAttn(nn.Module): if attn_mask is not None and torch.sum(attn_mask).item(): attn_mask = (attn_mask == 1) # Switch to bool if attn_mask.dim() == 2: - attn_score = attn_score.float().masked_fill( - attn_mask[None,:,:,None], -1e30).type_as(attn_score) + if next(self.parameters()).dtype == torch.float16: + attn_score = attn_score.float().masked_fill( + attn_mask[None,:,:,None], -65000).type_as(attn_score) + else: + attn_score = attn_score.float().masked_fill( + attn_mask[None,:,:,None], -1e30).type_as(attn_score) elif attn_mask.dim() == 3: - attn_score = attn_score.float().masked_fill( - attn_mask[:,:,:,None], -1e30).type_as(attn_score) + if next(self.parameters()).dtype == torch.float16: + attn_score = attn_score.float().masked_fill( + attn_mask[:,:,:,None], -65000).type_as(attn_score) + else: + attn_score = attn_score.float().masked_fill( + attn_mask[:,:,:,None], -1e30).type_as(attn_score) # [qlen x klen x bsz x n_head] attn_prob = F.softmax(attn_score, dim=1) @@ -383,7 +391,7 @@ class RelPartialLearnableDecoderLayer(nn.Module): class AdaptiveEmbedding(nn.Module): - def __init__(self, n_token, d_embed, d_proj, cutoffs, div_val=1, + def __init__(self, n_token, d_embed, d_proj, cutoffs, div_val=1, sample_softmax=False): super(AdaptiveEmbedding, self).__init__() @@ -421,7 +429,7 @@ class AdaptiveEmbedding(nn.Module): else: param = next(self.parameters()) inp_flat = inp.view(-1) - emb_flat = torch.zeros([inp_flat.size(0), self.d_proj], + emb_flat = torch.zeros([inp_flat.size(0), self.d_proj], dtype=param.dtype, device=param.device) for i in range(len(self.cutoffs)): l_idx, r_idx = self.cutoff_ends[i], self.cutoff_ends[i + 1] @@ -547,7 +555,7 @@ TRANSFO_XL_INPUTS_DOCSTRING = r""" ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. """ -@add_start_docstrings("The bare Bert Model transformer outputing raw hidden-states without any specific head on top.", +@add_start_docstrings("The bare Bert Model transformer outputting raw hidden-states without any specific head on top.", TRANSFO_XL_START_DOCSTRING, TRANSFO_XL_INPUTS_DOCSTRING) class TransfoXLModel(TransfoXLPreTrainedModel): r""" @@ -587,7 +595,7 @@ class TransfoXLModel(TransfoXLPreTrainedModel): self.n_head = config.n_head self.d_head = config.d_head - self.word_emb = AdaptiveEmbedding(config.n_token, config.d_embed, config.d_model, config.cutoffs, + self.word_emb = AdaptiveEmbedding(config.n_token, config.d_embed, config.d_model, config.cutoffs, div_val=config.div_val) self.drop = nn.Dropout(config.dropout) @@ -727,7 +735,7 @@ class TransfoXLModel(TransfoXLPreTrainedModel): hids = [] attentions = [] if self.attn_type == 0: # default - pos_seq = torch.arange(klen-1, -1, -1.0, device=word_emb.device, + pos_seq = torch.arange(klen-1, -1, -1.0, device=word_emb.device, dtype=word_emb.dtype) if self.clamp_len > 0: pos_seq.clamp_(max=self.clamp_len) @@ -815,7 +823,7 @@ class TransfoXLLMHeadModel(TransfoXLPreTrainedModel): self.sampler = LogUniformSampler(config.n_token, config.sample_softmax) # use adaptive softmax (including standard softmax) else: - self.crit = ProjectedAdaptiveLogSoftmax(config.n_token, config.d_embed, config.d_model, + self.crit = ProjectedAdaptiveLogSoftmax(config.n_token, config.d_embed, config.d_model, config.cutoffs, div_val=config.div_val) self.init_weights() self.tie_weights() diff --git a/pytorch_transformers/modeling_utils.py b/pytorch_transformers/modeling_utils.py index 790d4dcd54..af33c22d6e 100644 --- a/pytorch_transformers/modeling_utils.py +++ b/pytorch_transformers/modeling_utils.py @@ -140,7 +140,7 @@ class PreTrainedModel(nn.Module): Arguments: new_num_tokens: (`optional`) int: - New number of tokens in the embedding matrix. Increasing the size will add newly initialized vectors at the end. Reducing the size will remove vectors from the end. + New number of tokens in the embedding matrix. Increasing the size will add newly initialized vectors at the end. Reducing the size will remove vectors from the end. If not provided or None: does nothing and just returns a pointer to the input tokens ``torch.nn.Embeddings`` Module of the model. Return: ``torch.nn.Embeddings`` @@ -457,7 +457,10 @@ class PoolerStartLogits(nn.Module): x = self.dense(hidden_states).squeeze(-1) if p_mask is not None: - x = x * (1 - p_mask) - 1e30 * p_mask + if next(self.parameters()).dtype == torch.float16: + x = x * (1 - p_mask) - 65500 * p_mask + else: + x = x * (1 - p_mask) - 1e30 * p_mask return x diff --git a/pytorch_transformers/modeling_xlm.py b/pytorch_transformers/modeling_xlm.py index 95629ba535..782e2265ce 100644 --- a/pytorch_transformers/modeling_xlm.py +++ b/pytorch_transformers/modeling_xlm.py @@ -313,7 +313,7 @@ XLM_INPUTS_DOCSTRING = r""" ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. """ -@add_start_docstrings("The bare XLM Model transformer outputing raw hidden-states without any specific head on top.", +@add_start_docstrings("The bare XLM Model transformer outputting raw hidden-states without any specific head on top.", XLM_START_DOCSTRING, XLM_INPUTS_DOCSTRING) class XLMModel(XLMPreTrainedModel): r""" diff --git a/pytorch_transformers/modeling_xlnet.py b/pytorch_transformers/modeling_xlnet.py index c8e55d2107..f9960d4945 100644 --- a/pytorch_transformers/modeling_xlnet.py +++ b/pytorch_transformers/modeling_xlnet.py @@ -502,6 +502,12 @@ XLNET_INPUTS_DOCSTRING = r""" Indices can be obtained using :class:`pytorch_transformers.XLNetTokenizer`. See :func:`pytorch_transformers.PreTrainedTokenizer.encode` and :func:`pytorch_transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + **token_type_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + A parallel sequence of tokens (can be used to indicate various portions of the inputs). + The type indices in XLNet are NOT selected in the vocabulary, they can be arbitrary numbers and + the important thing is that they should be different for tokens which belong to different segments. + The model will compute relative segment differences from the given type indices: + 0 if the segment id of two tokens are the same, 1 if not. **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``: Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``: @@ -542,7 +548,7 @@ XLNET_INPUTS_DOCSTRING = r""" ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. """ -@add_start_docstrings("The bare XLNet Model transformer outputing raw hidden-states without any specific head on top.", +@add_start_docstrings("The bare XLNet Model transformer outputting raw hidden-states without any specific head on top.", XLNET_START_DOCSTRING, XLNET_INPUTS_DOCSTRING) class XLNetModel(XLNetPreTrainedModel): r""" @@ -739,8 +745,9 @@ class XLNetModel(XLNetPreTrainedModel): if data_mask is not None: # all mems can be attended to - mems_mask = torch.zeros([data_mask.shape[0], mlen, bsz]).to(data_mask) - data_mask = torch.cat([mems_mask, data_mask], dim=1) + if mlen > 0: + mems_mask = torch.zeros([data_mask.shape[0], mlen, bsz]).to(data_mask) + data_mask = torch.cat([mems_mask, data_mask], dim=1) if attn_mask is None: attn_mask = data_mask[:, :, :, None] else: @@ -751,7 +758,8 @@ class XLNetModel(XLNetPreTrainedModel): if attn_mask is not None: non_tgt_mask = -torch.eye(qlen).to(attn_mask) - non_tgt_mask = torch.cat([torch.zeros([qlen, mlen]).to(attn_mask), non_tgt_mask], dim=-1) + if mlen > 0: + non_tgt_mask = torch.cat([torch.zeros([qlen, mlen]).to(attn_mask), non_tgt_mask], dim=-1) non_tgt_mask = ((attn_mask + non_tgt_mask[:, :, None, None]) > 0).to(attn_mask) else: non_tgt_mask = None @@ -771,8 +779,11 @@ class XLNetModel(XLNetPreTrainedModel): ##### Segment embedding if token_type_ids is not None: # Convert `token_type_ids` to one-hot `seg_mat` - mem_pad = torch.zeros([mlen, bsz], dtype=torch.long, device=device) - cat_ids = torch.cat([mem_pad, token_type_ids], dim=0) + if mlen > 0: + mem_pad = torch.zeros([mlen, bsz], dtype=torch.long, device=device) + cat_ids = torch.cat([mem_pad, token_type_ids], dim=0) + else: + cat_ids = token_type_ids # `1` indicates not in the same segment [qlen x klen x bsz] seg_mat = (token_type_ids[:, None] != cat_ids[None, :]).long() @@ -1002,6 +1013,97 @@ class XLNetForSequenceClassification(XLNetPreTrainedModel): return outputs # return (loss), logits, mems, (hidden states), (attentions) +@add_start_docstrings("""XLNet Model with a multiple choice classification head on top (a linear layer on top of + the pooled output and a softmax) e.g. for RACE/SWAG tasks. """, + XLNET_START_DOCSTRING, XLNET_INPUTS_DOCSTRING) +class XLNetForMultipleChoice(XLNetPreTrainedModel): + r""" + Inputs: + **input_ids**: ``torch.LongTensor`` of shape ``(batch_size, num_choices, sequence_length)``: + Indices of input sequence tokens in the vocabulary. + The second dimension of the input (`num_choices`) indicates the number of choices to scores. + **token_type_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, num_choices, sequence_length)``: + Segment token indices to indicate first and second portions of the inputs. + The second dimension of the input (`num_choices`) indicates the number of choices to score. + Indices are selected in ``[0, 1]``: ``0`` corresponds to a `sentence A` token, ``1`` + **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, num_choices, sequence_length)``: + Mask to avoid performing attention on padding token indices. + The second dimension of the input (`num_choices`) indicates the number of choices to score. + Mask values selected in ``[0, 1]``: + ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. + **head_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: + Mask to nullify selected heads of the self-attention modules. + Mask values selected in ``[0, 1]``: + ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. + **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for computing the multiple choice classification loss. + Indices should be in ``[0, ..., num_choices]`` where `num_choices` is the size of the second dimension + of the input tensors. (see `input_ids` above) + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Classification loss. + **classification_scores**: ``torch.FloatTensor`` of shape ``(batch_size, num_choices)`` where `num_choices` is the size of the second dimension + of the input tensors. (see `input_ids` above). + Classification scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = XLNetTokenizer.from_pretrained('xlnet-base-cased') + model = XLNetForMultipleChoice.from_pretrained('xlnet-base-cased') + choices = ["Hello, my dog is cute", "Hello, my cat is amazing"] + input_ids = torch.tensor([tokenizer.encode(s) for s in choices]).unsqueeze(0) # Batch size 1, 2 choices + labels = torch.tensor(1).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, labels=labels) + loss, classification_scores = outputs[:2] + + """ + def __init__(self, config): + super(XLNetForMultipleChoice, self).__init__(config) + + self.transformer = XLNetModel(config) + self.sequence_summary = SequenceSummary(config) + self.logits_proj = nn.Linear(config.d_model, 1) + + self.init_weights() + + def forward(self, input_ids, token_type_ids=None, input_mask=None, attention_mask=None, + mems=None, perm_mask=None, target_mapping=None, + labels=None, head_mask=None): + num_choices = input_ids.shape[1] + + flat_input_ids = input_ids.view(-1, input_ids.size(-1)) + flat_token_type_ids = token_type_ids.view(-1, token_type_ids.size(-1)) if token_type_ids is not None else None + flat_attention_mask = attention_mask.view(-1, attention_mask.size(-1)) if attention_mask is not None else None + flat_input_mask = input_mask.view(-1, input_mask.size(-1)) if input_mask is not None else None + + transformer_outputs = self.transformer(flat_input_ids, token_type_ids=flat_token_type_ids, + input_mask=flat_input_mask, attention_mask=flat_attention_mask, + mems=mems, perm_mask=perm_mask, target_mapping=target_mapping, + head_mask=head_mask) + + + output = transformer_outputs[0] + + output = self.sequence_summary(output) + logits = self.logits_proj(output) + reshaped_logits = logits.view(-1, num_choices) + outputs = (reshaped_logits,) + transformer_outputs[1:] # Keep mems, hidden states, attentions if there are in it + + if labels is not None: + loss_fct = CrossEntropyLoss() + loss = loss_fct(reshaped_logits, labels.view(-1)) + outputs = (loss,) + outputs + + return outputs # return (loss), logits, mems, (hidden states), (attentions) + @add_start_docstrings("""XLNet Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`). """, @@ -1152,7 +1254,7 @@ class XLNetForQuestionAnswering(XLNetPreTrainedModel): Examples:: - tokenizer = XLMTokenizer.from_pretrained('xlm-mlm-en-2048') + tokenizer = XLNetTokenizer.from_pretrained('xlnet-large-cased') model = XLMForQuestionAnswering.from_pretrained('xlnet-large-cased') input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 start_positions = torch.tensor([1]) diff --git a/pytorch_transformers/tests/modeling_common_test.py b/pytorch_transformers/tests/modeling_common_test.py index 5e30cd1e32..0762a7a634 100644 --- a/pytorch_transformers/tests/modeling_common_test.py +++ b/pytorch_transformers/tests/modeling_common_test.py @@ -179,8 +179,9 @@ class CommonTestCases: if not self.test_head_masking: return - torch.manual_seed(42) + global_rng.seed(42) config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + global_rng.seed() config.output_attentions = True config.output_hidden_states = True @@ -190,7 +191,7 @@ class CommonTestCases: model.eval() # Prepare head_mask - # Set require_grad after having prepared the tensor to avoid error (leaf variable has been moved into the graph interior) + # Set require_grad after having prepared the tensor to avoid error (leaf variable has been moved into the graph interior) head_mask = torch.ones(self.model_tester.num_hidden_layers, self.model_tester.num_attention_heads) head_mask[0, 0] = 0 head_mask[-1, :-1] = 0 @@ -684,12 +685,13 @@ class ConfigTester(object): self.create_and_test_config_to_json_file() +global_rng = random.Random() def ids_tensor(shape, vocab_size, rng=None, name=None): """Creates a random int32 tensor of the shape within the vocab size.""" if rng is None: - rng = random.Random() + rng = global_rng total_dims = 1 for dim in shape: diff --git a/pytorch_transformers/tests/tokenization_bert_test.py b/pytorch_transformers/tests/tokenization_bert_test.py index 1111683ecc..4cfdfed136 100644 --- a/pytorch_transformers/tests/tokenization_bert_test.py +++ b/pytorch_transformers/tests/tokenization_bert_test.py @@ -131,8 +131,8 @@ class BertTokenizationTest(CommonTestCases.CommonTokenizerTester): text = tokenizer.encode("sequence builders") text_2 = tokenizer.encode("multi-sequence build") - encoded_sentence = tokenizer.add_special_tokens_single_sentence(text) - encoded_pair = tokenizer.add_special_tokens_sentences_pair(text, text_2) + encoded_sentence = tokenizer.add_special_tokens_single_sequence(text) + encoded_pair = tokenizer.add_special_tokens_sequence_pair(text, text_2) assert encoded_sentence == [101] + text + [102] assert encoded_pair == [101] + text + [102] + text_2 + [102] diff --git a/pytorch_transformers/tests/tokenization_dilbert_test.py b/pytorch_transformers/tests/tokenization_distilbert_test.py similarity index 81% rename from pytorch_transformers/tests/tokenization_dilbert_test.py rename to pytorch_transformers/tests/tokenization_distilbert_test.py index 42f8060998..674c78d104 100644 --- a/pytorch_transformers/tests/tokenization_dilbert_test.py +++ b/pytorch_transformers/tests/tokenization_distilbert_test.py @@ -36,11 +36,13 @@ class DistilBertTokenizationTest(BertTokenizationTest): text = tokenizer.encode("sequence builders") text_2 = tokenizer.encode("multi-sequence build") - encoded_sentence = tokenizer.add_special_tokens_single_sentence(text) - encoded_pair = tokenizer.add_special_tokens_sentences_pair(text, text_2) + encoded_sentence = tokenizer.add_special_tokens_single_sequence(text) + encoded_pair = tokenizer.add_special_tokens_sequence_pair(text, text_2) + + assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + \ + text_2 + [tokenizer.sep_token_id] - assert encoded_sentence == [101] + text + [102] - assert encoded_pair == [101] + text + [102] + text_2 + [102] if __name__ == '__main__': unittest.main() diff --git a/pytorch_transformers/tests/tokenization_roberta_test.py b/pytorch_transformers/tests/tokenization_roberta_test.py index 8add2529a5..ba2651c7f2 100644 --- a/pytorch_transformers/tests/tokenization_roberta_test.py +++ b/pytorch_transformers/tests/tokenization_roberta_test.py @@ -87,8 +87,8 @@ class RobertaTokenizationTest(CommonTestCases.CommonTokenizerTester): encoded_text_from_decode = tokenizer.encode("sequence builders", add_special_tokens=True) encoded_pair_from_decode = tokenizer.encode("sequence builders", "multi-sequence build", add_special_tokens=True) - encoded_sentence = tokenizer.add_special_tokens_single_sentence(text) - encoded_pair = tokenizer.add_special_tokens_sentences_pair(text, text_2) + encoded_sentence = tokenizer.add_special_tokens_single_sequence(text) + encoded_pair = tokenizer.add_special_tokens_sequence_pair(text, text_2) assert encoded_sentence == encoded_text_from_decode assert encoded_pair == encoded_pair_from_decode diff --git a/pytorch_transformers/tests/tokenization_tests_commons.py b/pytorch_transformers/tests/tokenization_tests_commons.py index 65f45c496c..b71ba44436 100644 --- a/pytorch_transformers/tests/tokenization_tests_commons.py +++ b/pytorch_transformers/tests/tokenization_tests_commons.py @@ -55,6 +55,22 @@ class CommonTestCases: def get_input_output_texts(self): raise NotImplementedError + def test_tokenizers_common_properties(self): + tokenizer = self.get_tokenizer() + attributes_list = ["bos_token", "eos_token", "unk_token", "sep_token", + "pad_token", "cls_token", "mask_token"] + for attr in attributes_list: + self.assertTrue(hasattr(tokenizer, attr)) + self.assertTrue(hasattr(tokenizer, attr + "_id")) + + self.assertTrue(hasattr(tokenizer, "additional_special_tokens")) + self.assertTrue(hasattr(tokenizer, 'additional_special_tokens_ids')) + + attributes_list = ["max_len", "init_inputs", "init_kwargs", "added_tokens_encoder", + "added_tokens_decoder"] + for attr in attributes_list: + self.assertTrue(hasattr(tokenizer, attr)) + def test_save_and_load_tokenizer(self): # safety check on max_len default value so we are sure the test works tokenizer = self.get_tokenizer() @@ -170,3 +186,92 @@ class CommonTestCases: for weights_list_2 in weights_lists_2: self.assertListEqual(weights_list, weights_list_2) + + def test_mask_output(self): + if sys.version_info <= (3, 0): + return + + tokenizer = self.get_tokenizer() + + if tokenizer.add_special_tokens_sequence_pair.__qualname__.split('.')[0] != "PreTrainedTokenizer": + seq_0 = "Test this method." + seq_1 = "With these inputs." + information = tokenizer.encode_plus(seq_0, seq_1, add_special_tokens=True) + sequences, mask = information["input_ids"], information["token_type_ids"] + assert len(sequences) == len(mask) + + def test_number_of_added_tokens(self): + tokenizer = self.get_tokenizer() + + seq_0 = "Test this method." + seq_1 = "With these inputs." + + sequences = tokenizer.encode(seq_0, seq_1) + attached_sequences = tokenizer.encode(seq_0, seq_1, add_special_tokens=True) + + # Method is implemented (e.g. not GPT-2) + if len(attached_sequences) != 2: + assert tokenizer.num_added_tokens(pair=True) == len(attached_sequences) - len(sequences) + + def test_maximum_encoding_length_single_input(self): + tokenizer = self.get_tokenizer() + + seq_0 = "This is a sentence to be encoded." + stride = 2 + + sequence = tokenizer.encode(seq_0) + num_added_tokens = tokenizer.num_added_tokens() + total_length = len(sequence) + num_added_tokens + information = tokenizer.encode_plus(seq_0, max_length=total_length - 2, add_special_tokens=True, stride=stride) + + truncated_sequence = information["input_ids"] + overflowing_tokens = information["overflowing_tokens"] + + assert len(overflowing_tokens) == 2 + stride + assert overflowing_tokens == sequence[-(2 + stride):] + assert len(truncated_sequence) == total_length - 2 + assert truncated_sequence == tokenizer.add_special_tokens_single_sequence(sequence[:-2]) + + def test_maximum_encoding_length_pair_input(self): + tokenizer = self.get_tokenizer() + + seq_0 = "This is a sentence to be encoded." + seq_1 = "This is another sentence to be encoded." + stride = 2 + + sequence_0_no_special_tokens = tokenizer.encode(seq_0) + sequence_1_no_special_tokens = tokenizer.encode(seq_1) + + sequence = tokenizer.encode(seq_0, seq_1, add_special_tokens=True) + truncated_second_sequence = tokenizer.add_special_tokens_sequence_pair( + tokenizer.encode(seq_0), + tokenizer.encode(seq_1)[:-2] + ) + + information = tokenizer.encode_plus(seq_0, seq_1, max_length=len(sequence) - 2, add_special_tokens=True, + stride=stride, truncate_first_sequence=False) + information_first_truncated = tokenizer.encode_plus(seq_0, seq_1, max_length=len(sequence) - 2, + add_special_tokens=True, stride=stride, + truncate_first_sequence=True) + + truncated_sequence = information["input_ids"] + overflowing_tokens = information["overflowing_tokens"] + overflowing_tokens_first_truncated = information_first_truncated["overflowing_tokens"] + + assert len(overflowing_tokens) == 2 + stride + assert overflowing_tokens == sequence_1_no_special_tokens[-(2 + stride):] + assert overflowing_tokens_first_truncated == sequence_0_no_special_tokens[-(2 + stride):] + assert len(truncated_sequence) == len(sequence) - 2 + assert truncated_sequence == truncated_second_sequence + + def test_encode_input_type(self): + tokenizer = self.get_tokenizer() + + sequence = "Let's encode this sequence" + + tokens = tokenizer.tokenize(sequence) + input_ids = tokenizer.convert_tokens_to_ids(tokens) + formatted_input = tokenizer.encode(sequence, add_special_tokens=True) + + assert tokenizer.encode(tokens, add_special_tokens=True) == formatted_input + assert tokenizer.encode(input_ids, add_special_tokens=True) == formatted_input diff --git a/pytorch_transformers/tests/tokenization_xlm_test.py b/pytorch_transformers/tests/tokenization_xlm_test.py index 43f1e0c5dd..13fdb4a8bb 100644 --- a/pytorch_transformers/tests/tokenization_xlm_test.py +++ b/pytorch_transformers/tests/tokenization_xlm_test.py @@ -72,8 +72,8 @@ class XLMTokenizationTest(CommonTestCases.CommonTokenizerTester): text = tokenizer.encode("sequence builders") text_2 = tokenizer.encode("multi-sequence build") - encoded_sentence = tokenizer.add_special_tokens_single_sentence(text) - encoded_pair = tokenizer.add_special_tokens_sentences_pair(text, text_2) + encoded_sentence = tokenizer.add_special_tokens_single_sequence(text) + encoded_pair = tokenizer.add_special_tokens_sequence_pair(text, text_2) assert encoded_sentence == [1] + text + [1] assert encoded_pair == [1] + text + [1] + text_2 + [1] diff --git a/pytorch_transformers/tests/tokenization_xlnet_test.py b/pytorch_transformers/tests/tokenization_xlnet_test.py index c603ce55f9..a6e9f23fe7 100644 --- a/pytorch_transformers/tests/tokenization_xlnet_test.py +++ b/pytorch_transformers/tests/tokenization_xlnet_test.py @@ -95,8 +95,8 @@ class XLNetTokenizationTest(CommonTestCases.CommonTokenizerTester): text = tokenizer.encode("sequence builders") text_2 = tokenizer.encode("multi-sequence build") - encoded_sentence = tokenizer.add_special_tokens_single_sentence(text) - encoded_pair = tokenizer.add_special_tokens_sentences_pair(text, text_2) + encoded_sentence = tokenizer.add_special_tokens_single_sequence(text) + encoded_pair = tokenizer.add_special_tokens_sequence_pair(text, text_2) assert encoded_sentence == text + [4, 3] assert encoded_pair == text + [4] + text_2 + [4, 3] diff --git a/pytorch_transformers/tokenization_bert.py b/pytorch_transformers/tokenization_bert.py index b85a4ccf9c..225152e065 100644 --- a/pytorch_transformers/tokenization_bert.py +++ b/pytorch_transformers/tokenization_bert.py @@ -187,22 +187,35 @@ class BertTokenizer(PreTrainedTokenizer): out_string = ' '.join(tokens).replace(' ##', '').strip() return out_string - def add_special_tokens_single_sentence(self, token_ids): + def add_special_tokens_single_sequence(self, token_ids): """ Adds special tokens to the a sequence for sequence classification tasks. A BERT sequence has the following format: [CLS] X [SEP] """ return [self.cls_token_id] + token_ids + [self.sep_token_id] - def add_special_tokens_sentences_pair(self, token_ids_0, token_ids_1): + def add_special_tokens_sequence_pair(self, token_ids_0, token_ids_1): """ Adds special tokens to a sequence pair for sequence classification tasks. A BERT sequence pair has the following format: [CLS] A [SEP] B [SEP] """ sep = [self.sep_token_id] cls = [self.cls_token_id] + return cls + token_ids_0 + sep + token_ids_1 + sep + def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1): + """ + Creates a mask from the two sequences passed to be used in a sequence-pair classification task. + A BERT sequence pair mask has the following format: + 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 + | first sequence | second sequence + """ + sep = [self.sep_token_id] + cls = [self.cls_token_id] + + return len(cls + token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1] + def save_vocabulary(self, vocab_path): """Save the tokenizer vocabulary to a directory or file.""" index = 0 diff --git a/pytorch_transformers/tokenization_roberta.py b/pytorch_transformers/tokenization_roberta.py index 67808752d5..ee8e97d6bf 100644 --- a/pytorch_transformers/tokenization_roberta.py +++ b/pytorch_transformers/tokenization_roberta.py @@ -81,14 +81,14 @@ class RobertaTokenizer(GPT2Tokenizer): sep_token=sep_token, cls_token=cls_token, pad_token=pad_token, mask_token=mask_token, **kwargs) - def add_special_tokens_single_sentence(self, token_ids): + def add_special_tokens_single_sequence(self, token_ids): """ Adds special tokens to a sequence for sequence classification tasks. A RoBERTa sequence has the following format: X """ return [self.cls_token_id] + token_ids + [self.sep_token_id] - def add_special_tokens_sentences_pair(self, token_ids_0, token_ids_1): + def add_special_tokens_sequence_pair(self, token_ids_0, token_ids_1): """ Adds special tokens to a sequence pair for sequence classification tasks. A RoBERTa sequence pair has the following format: A B @@ -96,3 +96,15 @@ class RobertaTokenizer(GPT2Tokenizer): sep = [self.sep_token_id] cls = [self.cls_token_id] return cls + token_ids_0 + sep + sep + token_ids_1 + sep + + def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1): + """ + Creates a mask from the two sequences passed to be used in a sequence-pair classification task. + A RoBERTa sequence pair mask has the following format: + 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 + | first sequence | second sequence + """ + sep = [self.sep_token_id] + cls = [self.cls_token_id] + + return len(cls + token_ids_0 + sep + sep) * [0] + len(token_ids_1 + sep) * [1] \ No newline at end of file diff --git a/pytorch_transformers/tokenization_utils.py b/pytorch_transformers/tokenization_utils.py index 5a307c5979..01f7532386 100644 --- a/pytorch_transformers/tokenization_utils.py +++ b/pytorch_transformers/tokenization_utils.py @@ -165,58 +165,42 @@ class PreTrainedTokenizer(object): @property def bos_token_id(self): """ Id of the beginning of sentence token in the vocabulary. Log an error if used while not having been set. """ - if self._bos_token is None: - logger.error("Using bos_token, but it is not set yet.") - return self.convert_tokens_to_ids(self._bos_token) + return self.convert_tokens_to_ids(self.bos_token) @property def eos_token_id(self): """ Id of the end of sentence token in the vocabulary. Log an error if used while not having been set. """ - if self._eos_token is None: - logger.error("Using eos_token, but it is not set yet.") - return self.convert_tokens_to_ids(self._eos_token) + return self.convert_tokens_to_ids(self.eos_token) @property - def unk_token_is(self): + def unk_token_id(self): """ Id of the unknown token in the vocabulary. Log an error if used while not having been set. """ - if self._unk_token is None: - logger.error("Using unk_token, but it is not set yet.") - return self.convert_tokens_to_ids(self._unk_token) + return self.convert_tokens_to_ids(self.unk_token) @property def sep_token_id(self): """ Id of the separation token in the vocabulary. E.g. separate context and query in an input sequence. Log an error if used while not having been set. """ - if self._sep_token is None: - logger.error("Using sep_token, but it is not set yet.") - return self.convert_tokens_to_ids(self._sep_token) + return self.convert_tokens_to_ids(self.sep_token) @property def pad_token_id(self): """ Id of the padding token in the vocabulary. Log an error if used while not having been set. """ - if self._pad_token is None: - logger.error("Using pad_token, but it is not set yet.") - return self.convert_tokens_to_ids(self._pad_token) + return self.convert_tokens_to_ids(self.pad_token) @property def cls_token_id(self): """ Id of the classification token in the vocabulary. E.g. to extract a summary of an input sequence leveraging self-attention along the full depth of the model. Log an error if used while not having been set. """ - if self._cls_token is None: - logger.error("Using cls_token, but it is not set yet.") - return self.convert_tokens_to_ids(self._cls_token) + return self.convert_tokens_to_ids(self.cls_token) @property def mask_token_id(self): """ Id of the mask token in the vocabulary. E.g. when training a model with masked-language modeling. Log an error if used while not having been set. """ - if self._mask_token is None: - logger.error("Using mask_token, but it is not set yet.") - return self.convert_tokens_to_ids(self._mask_token) + return self.convert_tokens_to_ids(self.mask_token) @property def additional_special_tokens_ids(self): """ Ids of all the additional special tokens in the vocabulary (list of integers). Log an error if used while not having been set. """ - if self._additional_special_tokens is None: - logger.error("Using additional_special_tokens, but it is not set yet.") - return self.convert_tokens_to_ids(self._additional_special_tokens) + return self.convert_tokens_to_ids(self.additional_special_tokens) def __init__(self, max_len=None, **kwargs): self._bos_token = None @@ -537,6 +521,30 @@ class PreTrainedTokenizer(object): return len(to_add_tokens) + def num_added_tokens(self, pair=False): + """ + Returns the number of added tokens when encoding a sequence with special tokens. + + Note: + This encodes inputs and checks the number of added tokens, and is therefore not efficient. Do not put this + inside your training loop. + + Args: + pair: Returns the number of added tokens in the case of a sequence pair if set to True, returns the + number of added tokens in the case of a single sequence if set to False. + + Returns: + Number of tokens added to sequences + """ + + if pair: + initial_tokens_len = len(self.encode("This is a sequence") + self.encode("This is another")) + final_tokens_len = len(self.encode("This is a sequence", "This is another", add_special_tokens=True)) + else: + initial_tokens_len = len(self.encode("This is a sequence")) + final_tokens_len = len(self.encode("This is a sequence", add_special_tokens=True)) + + return final_tokens_len - initial_tokens_len def add_special_tokens(self, special_tokens_dict): """ @@ -656,6 +664,9 @@ class PreTrainedTokenizer(object): """ Converts a single token, or a sequence of tokens, (str/unicode) in a single integer id (resp. a sequence of ids), using the vocabulary. """ + if tokens is None: + return None + if isinstance(tokens, str) or (six.PY2 and isinstance(tokens, unicode)): return self._convert_token_to_id_with_added_voc(tokens) @@ -669,6 +680,9 @@ class PreTrainedTokenizer(object): return ids def _convert_token_to_id_with_added_voc(self, token): + if token is None: + return None + if token in self.added_tokens_encoder: return self.added_tokens_encoder[token] return self._convert_token_to_id(token) @@ -679,48 +693,143 @@ class PreTrainedTokenizer(object): def encode(self, text, text_pair=None, add_special_tokens=False, **kwargs): """ Converts a string in a sequence of ids (integer), using the tokenizer and vocabulary. - + Same as doing ``self.convert_tokens_to_ids(self.tokenize(text))``. Args: - text: The first sequence to be encoded. - text_pair: Optional second sequence to be encoded. + text: The first sequence to be encoded. This can be a string, a list of strings (tokenized string using + the `tokenize` method) or a list of integers (tokenized string ids using the `convert_tokens_to_ids` + method) + text_pair: Optional second sequence to be encoded. This can be a string, a list of strings (tokenized + string using the `tokenize` method) or a list of integers (tokenized string ids using the + `convert_tokens_to_ids` method) add_special_tokens: if set to ``True``, the sequences will be encoded with the special tokens relative to their model. **kwargs: passed to the `self.tokenize()` method """ - if is_tf_available(): - is_tf_tensor = False - if isinstance(text, tf.Tensor): - text = text.numpy() - is_tf_tensor = True - if isinstance(text, bytes): - text = text.decode('utf-8') + encoded_inputs = self.encode_plus(text, text_pair=text_pair, add_special_tokens=add_special_tokens, **kwargs) - if text_pair is None: - if add_special_tokens: - output = self.add_special_tokens_single_sentence(self.convert_tokens_to_ids(self.tokenize(text, **kwargs))) + return encoded_inputs["input_ids"] + + def encode_plus(self, + text, + text_pair=None, + add_special_tokens=False, + max_length=None, + stride=0, + truncate_first_sequence=True, + **kwargs): + """ + Returns a dictionary containing the encoded sequence or sequence pair. Other values can be returned by this + method: the mask for sequence classification and the overflowing elements if a ``max_length`` is specified. + + Args: + text: The first sequence to be encoded. This can be a string, a list of strings (tokenized string using + the `tokenize` method) or a list of integers (tokenized string ids using the `convert_tokens_to_ids` + method) + text_pair: Optional second sequence to be encoded. This can be a string, a list of strings (tokenized + string using the `tokenize` method) or a list of integers (tokenized string ids using the + `convert_tokens_to_ids` method) + add_special_tokens: if set to ``True``, the sequences will be encoded with the special tokens relative + to their model. + max_length: if set to a number, will limit the total sequence returned so that it has a maximum length. + If there are overflowing tokens, those will be added to the returned dictionary + stride: if set to a number along with max_length, the overflowing tokens returned will contain some tokens + from the main sequence returned. The value of this argument defined the number of additional tokens. + truncate_first_sequence: if there is a specified max_length, this flag will choose which sequence + will be truncated. + **kwargs: passed to the `self.tokenize()` method + """ + + def get_input_ids(text): + if isinstance(text, six.string_types): + return self.convert_tokens_to_ids(self.tokenize(text, **kwargs)) + elif isinstance(text, (list, tuple)) and len(text) > 0 and isinstance(text[0], six.string_types): + return self.convert_tokens_to_ids(text) + elif isinstance(text, (list, tuple)) and len(text) > 0 and isinstance(text[0], int): + return text else: - output = self.convert_tokens_to_ids(self.tokenize(text, **kwargs)) + raise ValueError("Input is not valid. Should be a string, a list/tuple of strings or a list/tuple of integers.") + + first_ids = get_input_ids(text) + second_ids = get_input_ids(text_pair) if text_pair is not None else None + + return self.prepare_for_model(first_ids, + pair_ids=second_ids, + max_length=max_length, + add_special_tokens=add_special_tokens, + stride=stride, + truncate_first_sequence=truncate_first_sequence) + + + def prepare_for_model(self, ids, pair_ids=None, max_length=None, add_special_tokens=False, stride=0, truncate_first_sequence=True): + """ + Prepares a sequence of input id, or a pair of sequences of inputs ids so that it can be used by the model. + It adds special tokens, truncates + sequences if overflowing while taking into account the special tokens and manages a window stride for + overflowing tokens + + Args: + ids: list of tokenized input ids. Can be obtained from a string by chaining the + `tokenize` and `convert_tokens_to_ids` methods. + pair_ids: Optional second list of input ids. Can be obtained from a string by chaining the + `tokenize` and `convert_tokens_to_ids` methods. + max_length: maximum length of the returned list. Will truncate by taking into account the special tokens. + add_special_tokens: if set to ``True``, the sequences will be encoded with the special tokens relative + to their model. + stride: window stride for overflowing tokens. Can be useful for edge effect removal when using sequential + list of inputs. + truncate_first_sequence: if set to `True` and an optional second list of input ids is provided, + alongside a specified `max_length`, will truncate the first sequence if the total size is superior + than the specified `max_length`. If set to `False`, will truncate the second sequence instead. + + Return: + a dictionary containing the `input_ids` as well as the `overflowing_tokens` if a `max_length` was given. + """ + pair = bool(pair_ids is not None) + len_ids = len(ids) + len_pair_ids = len(pair_ids) if pair else 0 + + encoded_inputs = {} + if max_length: + n_added_tokens = self.num_added_tokens(pair=pair) if add_special_tokens else 0 + if pair and n_added_tokens + (len_pair_ids if truncate_first_sequence else len_ids) >= max_length: + logger.warning( + "You supplied a pair of sequence in which the sequence that will not be truncated is longer than the maximum specified length." + "This pair of sequences will not be truncated.") + else: + if n_added_tokens + len_ids + len_pair_ids > max_length: + if truncate_first_sequence or not pair: + encoded_inputs["overflowing_tokens"] = ids[max_length - len_pair_ids - n_added_tokens - stride:] + ids = ids[:max_length - len_pair_ids - n_added_tokens] + elif not truncate_first_sequence and pair: + encoded_inputs["overflowing_tokens"] = pair_ids[max_length - len_ids - n_added_tokens - stride:] + pair_ids = pair_ids[:max_length - len_ids - n_added_tokens] + else: + logger.warning( + "Cannot truncate second sequence as it is not provided. No truncation.") + + if add_special_tokens: + sequence = self.add_special_tokens_sequence_pair(ids, pair_ids) if pair else self.add_special_tokens_single_sequence(ids) + token_type_ids = self.create_token_type_ids_from_sequences(ids, pair_ids) if pair else [0] * len(sequence) else: - first_sentence_tokens = [self._convert_token_to_id(token) for token in self.tokenize(text, **kwargs)] - second_sentence_tokens = [self._convert_token_to_id(token) for token in self.tokenize(text_pair, **kwargs)] + sequence = ids + pair_ids if pair else ids + token_type_ids = [0] * len(ids) + ([1] * len(pair_ids) if pair else []) - if add_special_tokens: - output = self.add_special_tokens_sentences_pair(first_sentence_tokens, second_sentence_tokens) - else: - output = first_sentence_tokens, second_sentence_tokens + encoded_inputs["input_ids"] = sequence + encoded_inputs["token_type_ids"] = token_type_ids - if is_tf_available() and is_tf_tensor: - output = tf.constant(output) + return encoded_inputs - return output + def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1): + logger.warning("This tokenizer does not make use of special tokens.") + return [0] * len(token_ids_0) + [1] * len(token_ids_1) - def add_special_tokens_single_sentence(self, token_ids): + def add_special_tokens_single_sequence(self, token_ids): logger.warning("This tokenizer does not make use of special tokens. The sequence has been returned with no modification.") return token_ids - def add_special_tokens_sentences_pair(self, token_ids_0, token_ids_1): + def add_special_tokens_sequence_pair(self, token_ids_0, token_ids_1): logger.warning("This tokenizer does not make use of special tokens. The two sequences have been concatenated.") return token_ids_0 + token_ids_1 diff --git a/pytorch_transformers/tokenization_xlm.py b/pytorch_transformers/tokenization_xlm.py index f7231384b3..f1e49416a4 100644 --- a/pytorch_transformers/tokenization_xlm.py +++ b/pytorch_transformers/tokenization_xlm.py @@ -754,14 +754,14 @@ class XLMTokenizer(PreTrainedTokenizer): out_string = ''.join(tokens).replace('', ' ').strip() return out_string - def add_special_tokens_single_sentence(self, token_ids): + def add_special_tokens_single_sequence(self, token_ids): """ Adds special tokens to a sequence for sequence classification tasks. An XLM sequence has the following format: [CLS] X [SEP] """ return [self.cls_token_id] + token_ids + [self.sep_token_id] - def add_special_tokens_sentences_pair(self, token_ids_0, token_ids_1): + def add_special_tokens_sequence_pair(self, token_ids_0, token_ids_1): """ Adds special tokens to a sequence pair for sequence classification tasks. An XLM sequence pair has the following format: [CLS] A [SEP] B [SEP] @@ -770,6 +770,18 @@ class XLMTokenizer(PreTrainedTokenizer): cls = [self.cls_token_id] return cls + token_ids_0 + sep + token_ids_1 + sep + def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1): + """ + Creates a mask from the two sequences passed to be used in a sequence-pair classification task. + An XLM sequence pair mask has the following format: + 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 + | first sequence | second sequence + """ + sep = [self.sep_token_id] + cls = [self.cls_token_id] + + return len(cls + token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1] + def save_vocabulary(self, save_directory): """Save the tokenizer vocabulary and merge files to a directory.""" if not os.path.isdir(save_directory): diff --git a/pytorch_transformers/tokenization_xlnet.py b/pytorch_transformers/tokenization_xlnet.py index 230095daa9..941c6c5bc3 100644 --- a/pytorch_transformers/tokenization_xlnet.py +++ b/pytorch_transformers/tokenization_xlnet.py @@ -181,7 +181,7 @@ class XLNetTokenizer(PreTrainedTokenizer): out_string = ''.join(tokens).replace(SPIECE_UNDERLINE, ' ').strip() return out_string - def add_special_tokens_single_sentence(self, token_ids): + def add_special_tokens_single_sequence(self, token_ids): """ Adds special tokens to a sequence pair for sequence classification tasks. An XLNet sequence pair has the following format: A [SEP] B [SEP][CLS] @@ -190,15 +190,29 @@ class XLNetTokenizer(PreTrainedTokenizer): cls = [self.cls_token_id] return token_ids + sep + cls - def add_special_tokens_sentences_pair(self, token_ids_0, token_ids_1): + def add_special_tokens_sequence_pair(self, token_ids_0, token_ids_1): """ Adds special tokens to a sequence for sequence classification tasks. An XLNet sequence has the following format: X [SEP][CLS] """ + sep = [self.sep_token_id] cls = [self.cls_token_id] return token_ids_0 + sep + token_ids_1 + sep + cls + def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1): + """ + Creates a mask from the two sequences passed to be used in a sequence-pair classification task. + A BERT sequence pair mask has the following format: + 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 2 + | first sequence | second sequence | CLS segment ID + """ + sep = [self.sep_token_id] + cls = [self.cls_token_id] + cls_segment_id = [2] + + return len(token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1] + cls_segment_id + def save_vocabulary(self, save_directory): """ Save the sentencepiece vocabulary (copy original file) and special tokens file to a directory.