From 11fae9e636d9d0ae45d578e05193c52ff40f6e30 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 5 Sep 2019 02:27:39 +0200 Subject: [PATCH 001/219] add tf bert files --- pytorch_transformers/file_utils.py | 6 + pytorch_transformers/modeling_tf_bert.py | 832 ++++++++++++++++++ pytorch_transformers/modeling_tf_utils.py | 255 ++++++ .../tests/modeling_tf_common_test.py | 308 +++++++ .../tests/modeling_tf_test.py | 327 +++++++ 5 files changed, 1728 insertions(+) create mode 100644 pytorch_transformers/modeling_tf_bert.py create mode 100644 pytorch_transformers/modeling_tf_utils.py create mode 100644 pytorch_transformers/tests/modeling_tf_common_test.py create mode 100644 pytorch_transformers/tests/modeling_tf_test.py diff --git a/pytorch_transformers/file_utils.py b/pytorch_transformers/file_utils.py index 37ebc57fb3..94ada93d91 100644 --- a/pytorch_transformers/file_utils.py +++ b/pytorch_transformers/file_utils.py @@ -79,6 +79,9 @@ def url_to_filename(url, etag=None): Convert `url` into a hashed filename in a repeatable way. If `etag` is specified, append its hash to the url's, delimited by a period. + If the url ends with .h5 (Keras HDF5 weights) ands '.h5' to the name + so that TF 2.0 can identify it as a HDF5 file + (see https://github.com/tensorflow/tensorflow/blob/00fad90125b18b80fe054de1055770cfb8fe4ba3/tensorflow/python/keras/engine/network.py#L1380) """ url_bytes = url.encode('utf-8') url_hash = sha256(url_bytes) @@ -89,6 +92,9 @@ def url_to_filename(url, etag=None): etag_hash = sha256(etag_bytes) filename += '.' + etag_hash.hexdigest() + if url.endswith('.h5'): + filename += '.h5' + return filename diff --git a/pytorch_transformers/modeling_tf_bert.py b/pytorch_transformers/modeling_tf_bert.py new file mode 100644 index 0000000000..dda713fe7d --- /dev/null +++ b/pytorch_transformers/modeling_tf_bert.py @@ -0,0 +1,832 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" TF 2.0 BERT model. """ + +from __future__ import absolute_import, division, print_function, unicode_literals + +import json +import logging +import math +import os +import sys +from io import open + +import numpy as np +import tensorflow as tf + +from .configuration_bert import BertConfig +from .modeling_tf_utils import TFPreTrainedModel +from .file_utils import add_start_docstrings + +logger = logging.getLogger(__name__) + + +TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP = { + 'bert-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-tf_model.h5", + 'bert-large-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-tf_model.h5", + 'bert-base-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-tf_model.h5", + 'bert-large-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-tf_model.h5", + 'bert-base-multilingual-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-uncased-tf_model.h5", + 'bert-base-multilingual-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-cased-tf_model.h5", + 'bert-base-chinese': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-chinese-tf_model.h5", + 'bert-base-german-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-german-cased-tf_model.h5", + 'bert-large-uncased-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-whole-word-masking-tf_model.h5", + 'bert-large-cased-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-whole-word-masking-tf_model.h5", + 'bert-large-uncased-whole-word-masking-finetuned-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-whole-word-masking-finetuned-squad-tf_model.h5", + 'bert-large-cased-whole-word-masking-finetuned-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-whole-word-masking-finetuned-squad-tf_model.h5", + 'bert-base-cased-finetuned-mrpc': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-finetuned-mrpc-tf_model.h5", +} + + +def load_pt_weights_in_bert(tf_model, config, pytorch_checkpoint_path): + """ Load pytorch checkpoints in a TF 2.0 model and save it using HDF5 format + We use HDF5 to easily do transfer learning + (see https://github.com/tensorflow/tensorflow/blob/ee16fcac960ae660e0e4496658a366e2f745e1f0/tensorflow/python/keras/engine/network.py#L1352-L1357). + """ + try: + import re + import torch + import numpy + from tensorflow.python.keras import backend as K + except ImportError: + logger.error("Loading a PyTorch model in TensorFlow, requires PyTorch to be installed. Please see " + "https://pytorch.org/ for installation instructions.") + raise + + pt_path = os.path.abspath(pytorch_checkpoint_path) + logger.info("Loading PyTorch weights from {}".format(pt_path)) + # Load pytorch model + state_dict = torch.load(pt_path, map_location='cpu') + + inputs_list = [[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]] + tf_inputs = tf.constant(inputs_list) + tfo = tf_model(tf_inputs, training=False) # build the network + + symbolic_weights = tf_model.trainable_weights + tf_model.non_trainable_weights + weight_value_tuples = [] + for symbolic_weight in symbolic_weights: + name = symbolic_weight.name + name = name.replace('cls_mlm', 'cls') # We had to split this layer in two in the TF model to be + name = name.replace('cls_nsp', 'cls') # able to do transfer learning (Keras only allow to remove full layers) + name = name.replace(':0', '') + name = name.replace('layer_', 'layer/') + name = name.split('/') + name = name[1:] + + transpose = bool(name[-1] == 'kernel') + if name[-1] == 'kernel' or name[-1] == 'embeddings': + name[-1] = 'weight' + + name = '.'.join(name) + assert name in state_dict + array = state_dict[name].numpy() + + if transpose: + array = numpy.transpose(array) + + try: + assert list(symbolic_weight.shape) == list(array.shape) + except AssertionError as e: + e.args += (symbolic_weight.shape, array.shape) + raise e + + logger.info("Initialize TF weight {}".format(symbolic_weight.name)) + + weight_value_tuples.append((symbolic_weight, array)) + + K.batch_set_value(weight_value_tuples) + + tfo = tf_model(tf_inputs, training=False) # Make sure restore ops are run + return tf_model + + +def gelu(x): + """Gaussian Error Linear Unit. + This is a smoother version of the RELU. + Original paper: https://arxiv.org/abs/1606.08415 + Args: + x: float Tensor to perform activation. + Returns: + `x` with the GELU activation applied. + """ + cdf = 0.5 * (1.0 + tf.tanh( + (np.sqrt(2 / np.pi) * (x + 0.044715 * tf.pow(x, 3))))) + return x * cdf + + +def swish(x): + return x * tf.sigmoid(x) + + +ACT2FN = {"gelu": tf.keras.layers.Activation(gelu), + "relu": tf.keras.activations.relu, + "swish": tf.keras.layers.Activation(swish)} + + +class TFBertEmbeddings(tf.keras.layers.Layer): + """Construct the embeddings from word, position and token_type embeddings. + """ + def __init__(self, config, **kwargs): + super(TFBertEmbeddings, self).__init__(**kwargs) + self.word_embeddings = tf.keras.layers.Embedding(config.vocab_size, config.hidden_size, name='word_embeddings') + self.position_embeddings = tf.keras.layers.Embedding(config.max_position_embeddings, config.hidden_size, name='position_embeddings') + self.token_type_embeddings = tf.keras.layers.Embedding(config.type_vocab_size, config.hidden_size, name='token_type_embeddings') + + # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load + # any TensorFlow checkpoint file + self.LayerNorm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name='LayerNorm') + self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) + + def call(self, inputs, training=False): + input_ids, position_ids, token_type_ids = inputs + + seq_length = tf.shape(input_ids)[1] + if position_ids is None: + position_ids = tf.range(seq_length, dtype=tf.int32)[tf.newaxis, :] + if token_type_ids is None: + token_type_ids = tf.fill(tf.shape(input_ids), 0) + + words_embeddings = self.word_embeddings(input_ids) + position_embeddings = self.position_embeddings(position_ids) + token_type_embeddings = self.token_type_embeddings(token_type_ids) + + embeddings = words_embeddings + position_embeddings + token_type_embeddings + embeddings = self.LayerNorm(embeddings) + if training: + embeddings = self.dropout(embeddings) + return embeddings + + +class TFBertSelfAttention(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFBertSelfAttention, self).__init__(**kwargs) + if config.hidden_size % config.num_attention_heads != 0: + raise ValueError( + "The hidden size (%d) is not a multiple of the number of attention " + "heads (%d)" % (config.hidden_size, config.num_attention_heads)) + self.output_attentions = config.output_attentions + + self.num_attention_heads = config.num_attention_heads + assert config.hidden_size % config.num_attention_heads == 0 + self.attention_head_size = int(config.hidden_size / config.num_attention_heads) + self.all_head_size = self.num_attention_heads * self.attention_head_size + + self.query = tf.keras.layers.Dense(self.all_head_size, name='query') + self.key = tf.keras.layers.Dense(self.all_head_size, name='key') + self.value = tf.keras.layers.Dense(self.all_head_size, name='value') + + self.dropout = tf.keras.layers.Dropout(config.attention_probs_dropout_prob) + + def transpose_for_scores(self, x, batch_size): + x = tf.reshape(x, (batch_size, -1, self.num_attention_heads, self.attention_head_size)) + return tf.transpose(x, perm=[0, 2, 1, 3]) + + def call(self, inputs, training=False): + hidden_states, attention_mask, head_mask = inputs + + batch_size = tf.shape(hidden_states)[0] + mixed_query_layer = self.query(hidden_states) + mixed_key_layer = self.key(hidden_states) + mixed_value_layer = self.value(hidden_states) + + query_layer = self.transpose_for_scores(mixed_query_layer, batch_size) + key_layer = self.transpose_for_scores(mixed_key_layer, batch_size) + value_layer = self.transpose_for_scores(mixed_value_layer, batch_size) + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = tf.matmul(query_layer, key_layer, transpose_b=True) # (batch size, num_heads, seq_len_q, seq_len_k) + dk = tf.cast(tf.shape(key_layer)[-1], tf.float32) # scale attention_scores + attention_scores = attention_scores / tf.math.sqrt(dk) + # Apply the attention mask is (precomputed for all layers in TFBertModel call() function) + attention_scores = attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs = tf.nn.softmax(attention_scores, axis=-1) + + if training: + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.dropout(attention_probs) + + # Mask heads if we want to + if head_mask is not None: + attention_probs = attention_probs * head_mask + + context_layer = tf.matmul(attention_probs, value_layer) + + context_layer = tf.transpose(context_layer, perm=[0, 2, 1, 3]) + context_layer = tf.reshape(context_layer, + (batch_size, -1, self.all_head_size)) # (batch_size, seq_len_q, all_head_size) + + outputs = (context_layer, attention_probs) if self.output_attentions else (context_layer,) + return outputs + + +class TFBertSelfOutput(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFBertSelfOutput, self).__init__(**kwargs) + self.dense = tf.keras.layers.Dense(config.hidden_size, name='dense') + self.LayerNorm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name='LayerNorm') + self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) + + def call(self, inputs, training=False): + hidden_states, input_tensor = inputs + + hidden_states = self.dense(hidden_states) + if training: + hidden_states = self.dropout(hidden_states) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +class TFBertAttention(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFBertAttention, self).__init__(**kwargs) + self.self_attention = TFBertSelfAttention(config, name='self') + self.dense_output = TFBertSelfOutput(config, name='output') + + def prune_heads(self, heads): + raise NotImplementedError + + def call(self, inputs, training=False): + input_tensor, attention_mask, head_mask = inputs + + self_outputs = self.self_attention([input_tensor, attention_mask, head_mask], training=training) + attention_output = self.dense_output([self_outputs[0], input_tensor], training=training) + outputs = (attention_output,) + self_outputs[1:] # add attentions if we output them + return outputs + + +class TFBertIntermediate(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFBertIntermediate, self).__init__(**kwargs) + self.dense = tf.keras.layers.Dense(config.intermediate_size, name='dense') + if isinstance(config.hidden_act, str) or (sys.version_info[0] == 2 and isinstance(config.hidden_act, unicode)): + self.intermediate_act_fn = ACT2FN[config.hidden_act] + else: + self.intermediate_act_fn = config.hidden_act + + def call(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.intermediate_act_fn(hidden_states) + return hidden_states + + +class TFBertOutput(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFBertOutput, self).__init__(**kwargs) + self.dense = tf.keras.layers.Dense(config.hidden_size, name='dense') + self.LayerNorm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name='LayerNorm') + self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) + + def call(self, inputs, training=False): + hidden_states, input_tensor = inputs + + hidden_states = self.dense(hidden_states) + if training: + hidden_states = self.dropout(hidden_states) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +class TFBertLayer(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFBertLayer, self).__init__(**kwargs) + self.attention = TFBertAttention(config, name='attention') + self.intermediate = TFBertIntermediate(config, name='intermediate') + self.bert_output = TFBertOutput(config, name='output') + + def call(self, inputs, training=False): + hidden_states, attention_mask, head_mask = inputs + + attention_outputs = self.attention([hidden_states, attention_mask, head_mask], training=training) + attention_output = attention_outputs[0] + intermediate_output = self.intermediate(attention_output) + layer_output = self.bert_output([intermediate_output, attention_output], training=training) + outputs = (layer_output,) + attention_outputs[1:] # add attentions if we output them + return outputs + + +class TFBertEncoder(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFBertEncoder, self).__init__(**kwargs) + self.output_attentions = config.output_attentions + self.output_hidden_states = config.output_hidden_states + self.layer = [TFBertLayer(config, name='layer_{}'.format(i)) for i in range(config.num_hidden_layers)] + + def call(self, inputs, training=False): + hidden_states, attention_mask, head_mask = inputs + + all_hidden_states = () + all_attentions = () + for i, layer_module in enumerate(self.layer): + if self.output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + layer_outputs = layer_module([hidden_states, attention_mask, head_mask[i]], training=training) + hidden_states = layer_outputs[0] + + if self.output_attentions: + all_attentions = all_attentions + (layer_outputs[1],) + + # Add last layer + if self.output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + outputs = (hidden_states,) + if self.output_hidden_states: + outputs = outputs + (all_hidden_states,) + if self.output_attentions: + outputs = outputs + (all_attentions,) + return outputs # outputs, (hidden states), (attentions) + + +class TFBertPooler(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFBertPooler, self).__init__(**kwargs) + self.dense = tf.keras.layers.Dense(config.hidden_size, activation='tanh', name='dense') + + def call(self, hidden_states): + # We "pool" the model by simply taking the hidden state corresponding + # to the first token. + first_token_tensor = hidden_states[:, 0] + pooled_output = self.dense(first_token_tensor) + return pooled_output + + +class TFBertPredictionHeadTransform(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFBertPredictionHeadTransform, self).__init__(**kwargs) + self.dense = tf.keras.layers.Dense(config.hidden_size, name='dense') + if isinstance(config.hidden_act, str) or (sys.version_info[0] == 2 and isinstance(config.hidden_act, unicode)): + self.transform_act_fn = ACT2FN[config.hidden_act] + else: + self.transform_act_fn = config.hidden_act + self.LayerNorm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name='LayerNorm') + + def call(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.transform_act_fn(hidden_states) + hidden_states = self.LayerNorm(hidden_states) + return hidden_states + + +class TFBertLMPredictionHead(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFBertLMPredictionHead, self).__init__(**kwargs) + self.vocab_size = config.vocab_size + self.transform = TFBertPredictionHeadTransform(config, name='transform') + + # The output weights are the same as the input embeddings, but there is + # an output-only bias for each token. + self.decoder = tf.keras.layers.Dense(config.vocab_size, use_bias=False, name='decoder') + + def build(self, input_shape): + self.bias = self.add_weight(shape=(self.vocab_size,), + initializer='zeros', + trainable=True, + name='bias') + + def call(self, hidden_states): + hidden_states = self.transform(hidden_states) + hidden_states = self.decoder(hidden_states) + self.bias + return hidden_states + + +class TFBertMLMHead(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFBertMLMHead, self).__init__(**kwargs) + self.predictions = TFBertLMPredictionHead(config, name='predictions') + + def call(self, sequence_output): + prediction_scores = self.predictions(sequence_output) + return prediction_scores + + +class TFBertNSPHead(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFBertNSPHead, self).__init__(**kwargs) + self.seq_relationship = tf.keras.layers.Dense(2, name='seq_relationship') + + def call(self, pooled_output): + seq_relationship_score = self.seq_relationship(pooled_output) + return seq_relationship_score + + +class TFBertMainLayer(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFBertMainLayer, self).__init__(**kwargs) + self.num_hidden_layers = config.num_hidden_layers + + self.embeddings = TFBertEmbeddings(config, name='embeddings') + self.encoder = TFBertEncoder(config, name='encoder') + self.pooler = TFBertPooler(config, name='pooler') + + # self.apply(self.init_weights) # TODO check weights initialization + + def _resize_token_embeddings(self, new_num_tokens): + raise NotImplementedError + + def _prune_heads(self, heads_to_prune): + """ Prunes heads of the model. + heads_to_prune: dict of {layer_num: list of heads to prune in this layer} + See base class PreTrainedModel + """ + raise NotImplementedError + + def call(self, inputs, training=False): + if not isinstance(inputs, (dict, tuple, list)): + input_ids = inputs + attention_mask, head_mask, position_ids, token_type_ids = None, None, None, None + elif isinstance(inputs, (tuple, list)): + input_ids = inputs[0] + attention_mask = inputs[1] if len(inputs) > 1 else None + token_type_ids = inputs[2] if len(inputs) > 2 else None + position_ids = inputs[3] if len(inputs) > 3 else None + head_mask = inputs[4] if len(inputs) > 4 else None + assert len(inputs) <= 5, "Too many inputs." + else: + input_ids = inputs.pop('input_ids') + attention_mask = inputs.pop('attention_mask', None) + token_type_ids = inputs.pop('token_type_ids', None) + position_ids = inputs.pop('position_ids', None) + head_mask = inputs.pop('head_mask', None) + assert len(inputs) == 0, "Unexpected inputs detected: {}. Check inputs dict key names.".format(list(inputs.keys())) + + if attention_mask is None: + attention_mask = tf.fill(tf.shape(input_ids), 1) + if token_type_ids is None: + token_type_ids = tf.fill(tf.shape(input_ids), 0) + + # We create a 3D attention mask from a 2D tensor mask. + # Sizes are [batch_size, 1, 1, to_seq_length] + # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length] + # this attention mask is more simple than the triangular masking of causal attention + # used in OpenAI GPT, we just need to prepare the broadcast dimension here. + extended_attention_mask = attention_mask[:, tf.newaxis, tf.newaxis, :] + + # Since attention_mask is 1.0 for positions we want to attend and 0.0 for + # masked positions, this operation will create a tensor which is 0.0 for + # positions we want to attend and -10000.0 for masked positions. + # Since we are adding it to the raw scores before the softmax, this is + # effectively the same as removing these entirely. + + extended_attention_mask = tf.cast(extended_attention_mask, tf.float32) + extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0 + + # Prepare head mask if needed + # 1.0 in head_mask indicate we keep the head + # attention_probs has shape bsz x n_heads x N x N + # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] + # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length] + if not head_mask is None: + raise NotImplementedError + else: + head_mask = [None] * self.num_hidden_layers + # head_mask = tf.constant([0] * self.num_hidden_layers) + + embedding_output = self.embeddings([input_ids, position_ids, token_type_ids], training=training) + encoder_outputs = self.encoder([embedding_output, extended_attention_mask, head_mask], training=training) + + sequence_output = encoder_outputs[0] + pooled_output = self.pooler(sequence_output) + + outputs = (sequence_output, pooled_output,) + encoder_outputs[1:] # add hidden_states and attentions if they are here + return outputs # sequence_output, pooled_output, (hidden_states), (attentions) + +class TFBertPreTrainedModel(TFPreTrainedModel): + """ An abstract class to handle weights initialization and + a simple interface for dowloading and loading pretrained models. + """ + config_class = BertConfig + pretrained_model_archive_map = TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP + load_pt_weights = load_pt_weights_in_bert + base_model_prefix = "bert" + + def __init__(self, *inputs, **kwargs): + super(TFBertPreTrainedModel, self).__init__(*inputs, **kwargs) + + def init_weights(self, module): + """ Initialize the weights. + """ + raise NotImplementedError + + +BERT_START_DOCSTRING = r""" The BERT model was proposed in + `BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ + by Jacob Devlin, Ming-Wei Chang, Kenton Lee and Kristina Toutanova. It's a bidirectional transformer + pre-trained using a combination of masked language modeling objective and next sentence prediction + on a large corpus comprising the Toronto Book Corpus and Wikipedia. + + This model is a tf.keras.Model `tf.keras.Model`_ sub-class. Use it as a regular TF 2.0 Keras Model and + refer to the TF 2.0 documentation for all matter related to general usage and behavior. + + .. _`BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`: + https://arxiv.org/abs/1810.04805 + + .. _`tf.keras.Model`: + https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/Model + + Important note on the model inputs: + The inputs of the TF 2.0 models are slightly different from the PyTorch ones since + TF 2.0 Keras doesn't accept named arguments with defaults values for input Tensor. + More precisely, input Tensors are gathered in the first arguments of the model call function: `model(inputs)`. + There are three possibilities to gather and feed the inputs to the model: + + - a single Tensor with input_ids only and nothing else: `model(inputs_ids) + - a list of varying length with one or several input Tensors IN THE ORDER given in the docstring: + `model([input_ids, attention_mask])` or `model([input_ids, attention_mask, token_type_ids])` + - a dictionary with one or several input Tensors associaed to the input names given in the docstring: + `model({'input_ids': input_ids, 'token_type_ids': token_type_ids})` + + Parameters: + config (:class:`~pytorch_transformers.BertConfig`): Model configuration class with all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the configuration. + Check out the :meth:`~pytorch_transformers.PreTrainedModel.from_pretrained` method to load the model weights. +""" + +BERT_INPUTS_DOCSTRING = r""" + Inputs: + **input_ids**: ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Indices of input sequence tokens in the vocabulary. + To match pre-training, BERT input sequence should be formatted with [CLS] and [SEP] tokens as follows: + + (a) For sequence pairs: + + ``tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]`` + + ``token_type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1`` + + (b) For single sequences: + + ``tokens: [CLS] the dog is hairy . [SEP]`` + + ``token_type_ids: 0 0 0 0 0 0 0`` + + Bert is a model with absolute position embeddings so it's usually advised to pad the inputs on + the right rather than the left. + + Indices can be obtained using :class:`pytorch_transformers.BertTokenizer`. + See :func:`pytorch_transformers.PreTrainedTokenizer.encode` and + :func:`pytorch_transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``: + Mask to avoid performing attention on padding token indices. + Mask values selected in ``[0, 1]``: + ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. + **token_type_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Segment token indices to indicate first and second portions of the inputs. + Indices are selected in ``[0, 1]``: ``0`` corresponds to a `sentence A` token, ``1`` + corresponds to a `sentence B` token + (see `BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ for more details). + **position_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Indices of positions of each input sequence tokens in the position embeddings. + Selected in the range ``[0, config.max_position_embeddings - 1]``. + **head_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: + Mask to nullify selected heads of the self-attention modules. + Mask values selected in ``[0, 1]``: + ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. +""" + +@add_start_docstrings("The bare Bert Model transformer outputing raw hidden-states without any specific head on top.", + BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) +class TFBertModel(TFBertPreTrainedModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)`` + Sequence of hidden-states at the output of the last layer of the model. + **pooler_output**: ``torch.FloatTensor`` of shape ``(batch_size, hidden_size)`` + Last layer hidden-state of the first token of the sequence (classification token) + further processed by a Linear layer and a Tanh activation function. The Linear + layer weights are trained from the next sentence prediction (classification) + objective during Bert pretraining. This output is usually *not* a good summary + of the semantic content of the input, you're often better with averaging or pooling + the sequence of hidden-states for the whole input sequence. + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + model = TFBertModel.from_pretrained('bert-base-uncased') + input_ids = tf.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids) + last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple + + """ + def __init__(self, config): + super(TFBertModel, self).__init__(config) + self.bert = TFBertMainLayer(config, name='bert') + + def call(self, inputs, training=False): + outputs = self.bert(inputs, training=training) + return outputs + + +@add_start_docstrings("""Bert Model with two heads on top as done during the pre-training: + a `masked language modeling` head and a `next sentence prediction (classification)` head. """, + BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) +class TFBertForPreTraining(TFBertPreTrainedModel): + r""" + **masked_lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Labels for computing the masked language modeling loss. + Indices should be in ``[-1, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring) + Tokens with indices set to ``-1`` are ignored (masked), the loss is only computed for the tokens with labels + in ``[0, ..., config.vocab_size]`` + **next_sentence_label**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair (see ``input_ids`` docstring) + Indices should be in ``[0, 1]``. + ``0`` indicates sequence B is a continuation of sequence A, + ``1`` indicates sequence B is a random sequence. + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when both ``masked_lm_labels`` and ``next_sentence_label`` are provided) ``torch.FloatTensor`` of shape ``(1,)``: + Total loss as the sum of the masked language modeling loss and the next sequence prediction (classification) loss. + **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + **seq_relationship_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, 2)`` + Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + model = TFBertForPreTraining.from_pretrained('bert-base-uncased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids) + prediction_scores, seq_relationship_scores = outputs[:2] + + """ + def __init__(self, config): + super(TFBertForPreTraining, self).__init__(config) + + self.bert = TFBertMainLayer(config, name='bert') + self.cls_mlm = TFBertMLMHead(config, name='cls_mlm') + self.cls_nsp = TFBertNSPHead(config, name='cls_nsp') + + # self.apply(self.init_weights) # TODO check added weights initialization + self.tie_weights() + + def tie_weights(self): + """ Make sure we are sharing the input and output embeddings. + """ + pass # TODO add weights tying + + def call(self, inputs, training=False): + outputs = self.bert(inputs, training=training) + + sequence_output, pooled_output = outputs[:2] + prediction_scores = self.cls_mlm(sequence_output) + seq_relationship_score = self.cls_nsp(pooled_output) + + outputs = (prediction_scores, seq_relationship_score,) + outputs[2:] # add hidden states and attention if they are here + + # if masked_lm_labels is not None and next_sentence_label is not None: + # loss_fct = CrossEntropyLoss(ignore_index=-1) + # masked_lm_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), masked_lm_labels.view(-1)) + # next_sentence_loss = loss_fct(seq_relationship_score.view(-1, 2), next_sentence_label.view(-1)) + # total_loss = masked_lm_loss + next_sentence_loss + # outputs = (total_loss,) + outputs + # TODO add example with losses using model.compile and a dictionary of losses (give names to the output layers) + + return outputs # prediction_scores, seq_relationship_score, (hidden_states), (attentions) + + +@add_start_docstrings("""Bert Model with a `language modeling` head on top. """, + BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) +class TFBertForMaskedLM(TFBertPreTrainedModel): + r""" + **masked_lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Labels for computing the masked language modeling loss. + Indices should be in ``[-1, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring) + Tokens with indices set to ``-1`` are ignored (masked), the loss is only computed for the tokens with labels + in ``[0, ..., config.vocab_size]`` + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``masked_lm_labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Masked language modeling loss. + **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + model = TFBertForMaskedLM.from_pretrained('bert-base-uncased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, masked_lm_labels=input_ids) + loss, prediction_scores = outputs[:2] + + """ + def __init__(self, config): + super(TFBertForMaskedLM, self).__init__(config) + + self.bert = TFBertMainLayer(config, name='bert') + self.cls_mlm = TFBertMLMHead(config, name='cls_mlm') + + # self.apply(self.init_weights) + self.tie_weights() + + def tie_weights(self): + """ Make sure we are sharing the input and output embeddings. + """ + pass # TODO add weights tying + + def call(self, inputs, training=False): + outputs = self.bert(inputs, training=training) + + sequence_output = outputs[0] + prediction_scores = self.cls_mlm(sequence_output) + + outputs = (prediction_scores,) + outputs[2:] # Add hidden states and attention if they are here + # if masked_lm_labels is not None: + # loss_fct = CrossEntropyLoss(ignore_index=-1) + # masked_lm_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), masked_lm_labels.view(-1)) + # outputs = (masked_lm_loss,) + outputs + # TODO example with losses + + return outputs # prediction_scores, (hidden_states), (attentions) + + +@add_start_docstrings("""Bert Model with a `next sentence prediction (classification)` head on top. """, + BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) +class TFBertForNextSentencePrediction(TFBertPreTrainedModel): + r""" + **next_sentence_label**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair (see ``input_ids`` docstring) + Indices should be in ``[0, 1]``. + ``0`` indicates sequence B is a continuation of sequence A, + ``1`` indicates sequence B is a random sequence. + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``next_sentence_label`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Next sequence prediction (classification) loss. + **seq_relationship_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, 2)`` + Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + model = TFBertForNextSentencePrediction.from_pretrained('bert-base-uncased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids) + seq_relationship_scores = outputs[0] + + """ + def __init__(self, config): + super(TFBertForNextSentencePrediction, self).__init__(config) + + self.bert = TFBertMainLayer(config, name='bert') + self.cls_nsp = TFBertNSPHead(config, name='cls_nsp') + + # self.apply(self.init_weights) + + def call(self, inputs, training=False): + outputs = self.bert(inputs, training=training) + + pooled_output = outputs[1] + seq_relationship_score = self.cls_nsp(pooled_output) + + outputs = (seq_relationship_score,) + outputs[2:] # add hidden states and attention if they are here + # if next_sentence_label is not None: + # loss_fct = CrossEntropyLoss(ignore_index=-1) + # next_sentence_loss = loss_fct(seq_relationship_score.view(-1, 2), next_sentence_label.view(-1)) + # outputs = (next_sentence_loss,) + outputs + + return outputs # seq_relationship_score, (hidden_states), (attentions) diff --git a/pytorch_transformers/modeling_tf_utils.py b/pytorch_transformers/modeling_tf_utils.py new file mode 100644 index 0000000000..442d9dbe44 --- /dev/null +++ b/pytorch_transformers/modeling_tf_utils.py @@ -0,0 +1,255 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""TF general model utils.""" + +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +import logging +import os + +import tensorflow as tf + +from .configuration_utils import PretrainedConfig +from .file_utils import cached_path, WEIGHTS_NAME, TF_WEIGHTS_NAME + +logger = logging.getLogger(__name__) + + +class TFPreTrainedModel(tf.keras.Model): + r""" Base class for all TF models. + + :class:`~pytorch_transformers.TFPreTrainedModel` takes care of storing the configuration of the models and handles methods for loading/downloading/saving models + as well as a few methods commons to all models to (i) resize the input embeddings and (ii) prune heads in the self-attention heads. + + Class attributes (overridden by derived classes): + - ``config_class``: a class derived from :class:`~pytorch_transformers.PretrainedConfig` to use as configuration class for this model architecture. + - ``pretrained_model_archive_map``: a python ``dict`` of with `short-cut-names` (string) as keys and `url` (string) of associated pretrained weights as values. + - ``load_tf_weights``: a python ``method`` for loading a TensorFlow checkpoint in a PyTorch model, taking as arguments: + + - ``model``: an instance of the relevant subclass of :class:`~pytorch_transformers.PreTrainedModel`, + - ``config``: an instance of the relevant subclass of :class:`~pytorch_transformers.PretrainedConfig`, + - ``path``: a path (string) to the TensorFlow checkpoint. + + - ``base_model_prefix``: a string indicating the attribute associated to the base model in derived classes of the same architecture adding modules on top of the base model. + """ + config_class = None + pretrained_model_archive_map = {} + load_pt_weights = lambda model, config, path: None + base_model_prefix = "" + + def __init__(self, config, *inputs, **kwargs): + super(TFPreTrainedModel, self).__init__() + if not isinstance(config, PretrainedConfig): + raise ValueError( + "Parameter config in `{}(config)` should be an instance of class `PretrainedConfig`. " + "To create a model from a pretrained model use " + "`model = {}.from_pretrained(PRETRAINED_MODEL_NAME)`".format( + self.__class__.__name__, self.__class__.__name__ + )) + # Save config in model + self.config = config + + def _get_resized_embeddings(self, old_embeddings, new_num_tokens=None): + """ Build a resized Embedding Module from a provided token Embedding Module. + Increasing the size will add newly initialized vectors at the end + Reducing the size will remove vectors from the end + + Args: + new_num_tokens: (`optional`) int + New number of tokens in the embedding matrix. + Increasing the size will add newly initialized vectors at the end + Reducing the size will remove vectors from the end + If not provided or None: return the provided token Embedding Module. + Return: ``torch.nn.Embeddings`` + Pointer to the resized Embedding Module or the old Embedding Module if new_num_tokens is None + """ + raise NotImplementedError + + def _tie_or_clone_weights(self, first_module, second_module): + """ Tie or clone module weights depending of weither we are using TorchScript or not + """ + raise NotImplementedError + + def resize_token_embeddings(self, new_num_tokens=None): + """ Resize input token embeddings matrix of the model if new_num_tokens != config.vocab_size. + Take care of tying weights embeddings afterwards if the model class has a `tie_weights()` method. + + Arguments: + + new_num_tokens: (`optional`) int: + New number of tokens in the embedding matrix. Increasing the size will add newly initialized vectors at the end. Reducing the size will remove vectors from the end. + If not provided or None: does nothing and just returns a pointer to the input tokens ``torch.nn.Embeddings`` Module of the model. + + Return: ``torch.nn.Embeddings`` + Pointer to the input tokens Embeddings Module of the model + """ + raise NotImplementedError + + def prune_heads(self, heads_to_prune): + """ Prunes heads of the base model. + + Arguments: + + heads_to_prune: dict with keys being selected layer indices (`int`) and associated values being the list of heads to prune in said layer (list of `int`). + """ + raise NotImplementedError + + def save_pretrained(self, save_directory): + """ Save a model and its configuration file to a directory, so that it + can be re-loaded using the `:func:`~pytorch_transformers.PreTrainedModel.from_pretrained`` class method. + """ + raise NotImplementedError + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): + r"""Instantiate a pretrained pytorch model from a pre-trained model configuration. + + The model is set in evaluation mode by default using ``model.eval()`` (Dropout modules are deactivated) + To train the model, you should first set it back in training mode with ``model.train()`` + + The warning ``Weights from XXX not initialized from pretrained model`` means that the weights of XXX do not come pre-trained with the rest of the model. + It is up to you to train those weights with a downstream fine-tuning task. + + The warning ``Weights from XXX not used in YYY`` means that the layer XXX is not used by YYY, therefore those weights are discarded. + + Parameters: + pretrained_model_name_or_path: either: + + - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. + - a path to a `directory` containing model weights saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. + - a path or url to a `PyTorch state_dict save file` (e.g. `./pt_model/pytorch_model.bin`). In this case, ``from_pt`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the PyTorch checkpoint in a TensorFlow model using the provided conversion scripts and loading the TensorFlow model afterwards. + + model_args: (`optional`) Sequence of positional arguments: + All remaning positional arguments will be passed to the underlying model's ``__init__`` method + + config: (`optional`) instance of a class derived from :class:`~pytorch_transformers.PretrainedConfig`: + Configuration for the model to use instead of an automatically loaded configuation. Configuration can be automatically loaded when: + + - the model is a model provided by the library (loaded with the ``shortcut-name`` string of a pretrained model), or + - the model was saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained` and is reloaded by suppling the save directory. + - the model is loaded by suppling a local directory as ``pretrained_model_name_or_path`` and a configuration JSON file named `config.json` is found in the directory. + + from_pt: (`optional`) boolean, default False: + Load the model weights from a PyTorch state_dict save file (see docstring of pretrained_model_name_or_path argument). + + cache_dir: (`optional`) string: + Path to a directory in which a downloaded pre-trained model + configuration should be cached if the standard cache should not be used. + + force_download: (`optional`) boolean, default False: + Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + + proxies: (`optional`) dict, default None: + A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. + The proxies are used on each request. + + output_loading_info: (`optional`) boolean: + Set to ``True`` to also return a dictionnary containing missing keys, unexpected keys and error messages. + + kwargs: (`optional`) Remaining dictionary of keyword arguments: + Can be used to update the configuration object (after it being loaded) and initiate the model. (e.g. ``output_attention=True``). Behave differently depending on whether a `config` is provided or automatically loaded: + + - If a configuration is provided with ``config``, ``**kwargs`` will be directly passed to the underlying model's ``__init__`` method (we assume all relevant updates to the configuration have already been done) + - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~pytorch_transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function. + + Examples:: + + model = BertModel.from_pretrained('bert-base-uncased') # Download model and configuration from S3 and cache. + model = BertModel.from_pretrained('./test/saved_model/') # E.g. model was saved using `save_pretrained('./test/saved_model/')` + model = BertModel.from_pretrained('bert-base-uncased', output_attention=True) # Update configuration during loading + assert model.config.output_attention == True + # Loading from a TF checkpoint file instead of a PyTorch model (slower) + config = BertConfig.from_json_file('./tf_model/my_tf_model_config.json') + model = BertModel.from_pretrained('./tf_model/my_tf_checkpoint.ckpt.index', from_pt=True, config=config) + + """ + config = kwargs.pop('config', None) + cache_dir = kwargs.pop('cache_dir', None) + from_pt = kwargs.pop('from_pt', False) + force_download = kwargs.pop('force_download', False) + proxies = kwargs.pop('proxies', None) + output_loading_info = kwargs.pop('output_loading_info', False) + + # Load config + if config is None: + config, model_kwargs = cls.config_class.from_pretrained( + pretrained_model_name_or_path, *model_args, + cache_dir=cache_dir, return_unused_kwargs=True, + force_download=force_download, + **kwargs + ) + else: + model_kwargs = kwargs + + # Load model + if pretrained_model_name_or_path in cls.pretrained_model_archive_map: + archive_file = cls.pretrained_model_archive_map[pretrained_model_name_or_path] + elif os.path.isdir(pretrained_model_name_or_path): + if from_pt: + # Load from a PyTorch checkpoint + archive_file = os.path.join(pretrained_model_name_or_path, WEIGHTS_NAME) + else: + archive_file = os.path.join(pretrained_model_name_or_path, TF_WEIGHTS_NAME) + else: + archive_file = pretrained_model_name_or_path + # redirect to the cache, if necessary + try: + resolved_archive_file = cached_path(archive_file, cache_dir=cache_dir, force_download=force_download, proxies=proxies) + except EnvironmentError: + if pretrained_model_name_or_path in cls.pretrained_model_archive_map: + logger.error( + "Couldn't reach server at '{}' to download pretrained weights.".format( + archive_file)) + else: + logger.error( + "Model name '{}' was not found in model name list ({}). " + "We assumed '{}' was a path or url but couldn't find any file " + "associated to this path or url.".format( + pretrained_model_name_or_path, + ', '.join(cls.pretrained_model_archive_map.keys()), + archive_file)) + return None + if resolved_archive_file == archive_file: + logger.info("loading weights file {}".format(archive_file)) + else: + logger.info("loading weights file {} from cache at {}".format( + archive_file, resolved_archive_file)) + + # Instantiate model. + model = cls(config, *model_args, **model_kwargs) + + if from_pt: + # Load from a PyTorch checkpoint + return cls.load_pt_weights(model, config, resolved_archive_file) + + inputs = tf.constant([[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]]) + ret = model(inputs, training=False) # build the network with dummy inputs + + # 'by_name' allow us to do transfer learning by skipping/adding layers + # see https://github.com/tensorflow/tensorflow/blob/00fad90125b18b80fe054de1055770cfb8fe4ba3/tensorflow/python/keras/engine/network.py#L1339-L1357 + model.load_weights(resolved_archive_file, by_name=True) + + ret = model(inputs, training=False) # Make sure restore ops are run + + # if hasattr(model, 'tie_weights'): + # model.tie_weights() # TODO make sure word embedding weights are still tied + + if output_loading_info: + loading_info = {"missing_keys": missing_keys, "unexpected_keys": unexpected_keys, "error_msgs": error_msgs} + return model, loading_info + + return model diff --git a/pytorch_transformers/tests/modeling_tf_common_test.py b/pytorch_transformers/tests/modeling_tf_common_test.py new file mode 100644 index 0000000000..d432ab5fdf --- /dev/null +++ b/pytorch_transformers/tests/modeling_tf_common_test.py @@ -0,0 +1,308 @@ +# coding=utf-8 +# Copyright 2019 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import copy +import os +import shutil +import json +import random +import uuid + +import unittest +import logging + +import tensorflow as tf + +from pytorch_transformers import TFPreTrainedModel +# from pytorch_transformers.modeling_bert import BertModel, BertConfig, BERT_PRETRAINED_MODEL_ARCHIVE_MAP + + +def _config_zero_init(config): + configs_no_init = copy.deepcopy(config) + for key in configs_no_init.__dict__.keys(): + if '_range' in key or '_std' in key: + setattr(configs_no_init, key, 0.0) + return configs_no_init + +class TFCommonTestCases: + + class TFCommonModelTester(unittest.TestCase): + + model_tester = None + all_model_classes = () + test_torchscript = True + test_pruning = True + test_resize_embeddings = True + + def test_initialization(self): + pass + # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + # configs_no_init = _config_zero_init(config) + # for model_class in self.all_model_classes: + # model = model_class(config=configs_no_init) + # for name, param in model.named_parameters(): + # if param.requires_grad: + # self.assertIn(param.data.mean().item(), [0.0, 1.0], + # msg="Parameter {} of model {} seems not properly initialized".format(name, model_class)) + + + def test_attention_outputs(self): + pass + # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + # for model_class in self.all_model_classes: + # config.output_attentions = True + # config.output_hidden_states = False + # model = model_class(config) + # model.eval() + # outputs = model(**inputs_dict) + # attentions = outputs[-1] + # self.assertEqual(model.config.output_attentions, True) + # self.assertEqual(model.config.output_hidden_states, False) + # self.assertEqual(len(attentions), self.model_tester.num_hidden_layers) + # self.assertListEqual( + # list(attentions[0].shape[-3:]), + # [self.model_tester.num_attention_heads, + # self.model_tester.seq_length, + # self.model_tester.key_len if hasattr(self.model_tester, 'key_len') else self.model_tester.seq_length]) + # out_len = len(outputs) + + # # Check attention is always last and order is fine + # config.output_attentions = True + # config.output_hidden_states = True + # model = model_class(config) + # model.eval() + # outputs = model(**inputs_dict) + # self.assertEqual(out_len+1, len(outputs)) + # self.assertEqual(model.config.output_attentions, True) + # self.assertEqual(model.config.output_hidden_states, True) + + # attentions = outputs[-1] + # self.assertEqual(len(attentions), self.model_tester.num_hidden_layers) + # self.assertListEqual( + # list(attentions[0].shape[-3:]), + # [self.model_tester.num_attention_heads, + # self.model_tester.seq_length, + # self.model_tester.key_len if hasattr(self.model_tester, 'key_len') else self.model_tester.seq_length]) + + + def test_headmasking(self): + pass + # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + # config.output_attentions = True + # config.output_hidden_states = True + # configs_no_init = _config_zero_init(config) # To be sure we have no Nan + # for model_class in self.all_model_classes: + # model = model_class(config=configs_no_init) + # model.eval() + + # # Prepare head_mask + # # Set require_grad after having prepared the tensor to avoid error (leaf variable has been moved into the graph interior) + # head_mask = torch.ones(self.model_tester.num_hidden_layers, self.model_tester.num_attention_heads) + # head_mask[0, 0] = 0 + # head_mask[-1, :-1] = 0 + # head_mask.requires_grad_(requires_grad=True) + # inputs = inputs_dict.copy() + # inputs['head_mask'] = head_mask + + # outputs = model(**inputs) + + # # Test that we can get a gradient back for importance score computation + # output = sum(t.sum() for t in outputs[0]) + # output = output.sum() + # output.backward() + # multihead_outputs = head_mask.grad + + # attentions = outputs[-1] + # hidden_states = outputs[-2] + + # # Remove Nan + + # self.assertIsNotNone(multihead_outputs) + # self.assertEqual(len(multihead_outputs), self.model_tester.num_hidden_layers) + # self.assertAlmostEqual( + # attentions[0][..., 0, :, :].flatten().sum().item(), 0.0) + # self.assertNotEqual( + # attentions[0][..., -1, :, :].flatten().sum().item(), 0.0) + # self.assertNotEqual( + # attentions[1][..., 0, :, :].flatten().sum().item(), 0.0) + # self.assertAlmostEqual( + # attentions[-1][..., -2, :, :].flatten().sum().item(), 0.0) + # self.assertNotEqual( + # attentions[-1][..., -1, :, :].flatten().sum().item(), 0.0) + + + def test_head_pruning(self): + pass + # if not self.test_pruning: + # return + + # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + # for model_class in self.all_model_classes: + # config.output_attentions = True + # config.output_hidden_states = False + # model = model_class(config=config) + # model.eval() + # heads_to_prune = {0: list(range(1, self.model_tester.num_attention_heads)), + # -1: [0]} + # model.prune_heads(heads_to_prune) + # outputs = model(**inputs_dict) + + # attentions = outputs[-1] + + # self.assertEqual( + # attentions[0].shape[-3], 1) + # self.assertEqual( + # attentions[1].shape[-3], self.model_tester.num_attention_heads) + # self.assertEqual( + # attentions[-1].shape[-3], self.model_tester.num_attention_heads - 1) + + + def test_hidden_states_output(self): + pass + # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + # for model_class in self.all_model_classes: + # config.output_hidden_states = True + # config.output_attentions = False + # model = model_class(config) + # model.eval() + # outputs = model(**inputs_dict) + # hidden_states = outputs[-1] + # self.assertEqual(model.config.output_attentions, False) + # self.assertEqual(model.config.output_hidden_states, True) + # self.assertEqual(len(hidden_states), self.model_tester.num_hidden_layers + 1) + # self.assertListEqual( + # list(hidden_states[0].shape[-2:]), + # [self.model_tester.seq_length, self.model_tester.hidden_size]) + + + def test_resize_tokens_embeddings(self): + pass + # original_config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + # if not self.test_resize_embeddings: + # return + + # for model_class in self.all_model_classes: + # config = copy.deepcopy(original_config) + # model = model_class(config) + + # model_vocab_size = config.vocab_size + # # Retrieve the embeddings and clone theme + # model_embed = model.resize_token_embeddings(model_vocab_size) + # cloned_embeddings = model_embed.weight.clone() + + # # Check that resizing the token embeddings with a larger vocab size increases the model's vocab size + # model_embed = model.resize_token_embeddings(model_vocab_size + 10) + # self.assertEqual(model.config.vocab_size, model_vocab_size + 10) + # # Check that it actually resizes the embeddings matrix + # self.assertEqual(model_embed.weight.shape[0], cloned_embeddings.shape[0] + 10) + + # # Check that resizing the token embeddings with a smaller vocab size decreases the model's vocab size + # model_embed = model.resize_token_embeddings(model_vocab_size - 15) + # self.assertEqual(model.config.vocab_size, model_vocab_size - 15) + # # Check that it actually resizes the embeddings matrix + # self.assertEqual(model_embed.weight.shape[0], cloned_embeddings.shape[0] - 15) + + # # Check that adding and removing tokens has not modified the first part of the embedding matrix. + # models_equal = True + # for p1, p2 in zip(cloned_embeddings, model_embed.weight): + # if p1.data.ne(p2.data).sum() > 0: + # models_equal = False + + # self.assertTrue(models_equal) + + + def test_tie_model_weights(self): + pass + # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + # def check_same_values(layer_1, layer_2): + # equal = True + # for p1, p2 in zip(layer_1.weight, layer_2.weight): + # if p1.data.ne(p2.data).sum() > 0: + # equal = False + # return equal + + # for model_class in self.all_model_classes: + # if not hasattr(model_class, 'tie_weights'): + # continue + + # config.torchscript = True + # model_not_tied = model_class(config) + # params_not_tied = list(model_not_tied.parameters()) + + # config_tied = copy.deepcopy(config) + # config_tied.torchscript = False + # model_tied = model_class(config_tied) + # params_tied = list(model_tied.parameters()) + + # # Check that the embedding layer and decoding layer are the same in size and in value + # self.assertGreater(len(params_not_tied), len(params_tied)) + + # # Check that after resize they remain tied. + # model_tied.resize_token_embeddings(config.vocab_size + 10) + # params_tied_2 = list(model_tied.parameters()) + # self.assertGreater(len(params_not_tied), len(params_tied)) + # self.assertEqual(len(params_tied_2), len(params_tied)) + + +def ids_tensor(shape, vocab_size, rng=None, name=None): + """Creates a random int32 tensor of the shape within the vocab size.""" + if rng is None: + rng = random.Random() + + total_dims = 1 + for dim in shape: + total_dims *= dim + + values = [] + for _ in range(total_dims): + values.append(rng.randint(0, vocab_size - 1)) + + return tf.constant(values, shape=shape) + + +class TFModelUtilsTest(unittest.TestCase): + def test_model_from_pretrained(self): + pass + # logging.basicConfig(level=logging.INFO) + # for model_name in list(BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: + # config = BertConfig.from_pretrained(model_name) + # self.assertIsNotNone(config) + # self.assertIsInstance(config, PretrainedConfig) + + # model = BertModel.from_pretrained(model_name) + # model, loading_info = BertModel.from_pretrained(model_name, output_loading_info=True) + # self.assertIsNotNone(model) + # self.assertIsInstance(model, PreTrainedModel) + # for value in loading_info.values(): + # self.assertEqual(len(value), 0) + + # config = BertConfig.from_pretrained(model_name, output_attentions=True, output_hidden_states=True) + # model = BertModel.from_pretrained(model_name, output_attentions=True, output_hidden_states=True) + # self.assertEqual(model.config.output_attentions, True) + # self.assertEqual(model.config.output_hidden_states, True) + # self.assertEqual(model.config, config) + + +if __name__ == "__main__": + unittest.main() diff --git a/pytorch_transformers/tests/modeling_tf_test.py b/pytorch_transformers/tests/modeling_tf_test.py new file mode 100644 index 0000000000..ee2580bd9e --- /dev/null +++ b/pytorch_transformers/tests/modeling_tf_test.py @@ -0,0 +1,327 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import unittest +import shutil +import pytest + +import tensorflow as tf + +from pytorch_transformers import (BertConfig) +from pytorch_transformers.modeling_tf_bert import TFBertModel, TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP + +from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) +from .configuration_common_test import ConfigTester + + +class TFBertModelTest(TFCommonTestCases.TFCommonModelTester): + + all_model_classes = (TFBertModel,) + # BertForMaskedLM, BertForNextSentencePrediction, + # BertForPreTraining, BertForQuestionAnswering, BertForSequenceClassification, + # BertForTokenClassification) + + class TFBertModelTester(object): + + def __init__(self, + parent, + batch_size=13, + seq_length=7, + is_training=True, + use_input_mask=True, + use_token_type_ids=True, + use_labels=True, + vocab_size=99, + hidden_size=32, + num_hidden_layers=5, + num_attention_heads=4, + intermediate_size=37, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + type_sequence_label_size=2, + initializer_range=0.02, + num_labels=3, + num_choices=4, + scope=None, + ): + self.parent = parent + self.batch_size = batch_size + self.seq_length = seq_length + self.is_training = is_training + self.use_input_mask = use_input_mask + self.use_token_type_ids = use_token_type_ids + self.use_labels = use_labels + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.intermediate_size = intermediate_size + self.hidden_act = hidden_act + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.type_sequence_label_size = type_sequence_label_size + self.initializer_range = initializer_range + self.num_labels = num_labels + self.num_choices = num_choices + self.scope = scope + + def prepare_config_and_inputs(self): + input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) + + input_mask = None + if self.use_input_mask: + input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2) + + token_type_ids = None + if self.use_token_type_ids: + token_type_ids = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size) + + sequence_labels = None + token_labels = None + choice_labels = None + if self.use_labels: + sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size) + token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels) + choice_labels = ids_tensor([self.batch_size], self.num_choices) + + config = BertConfig( + vocab_size_or_config_json_file=self.vocab_size, + hidden_size=self.hidden_size, + num_hidden_layers=self.num_hidden_layers, + num_attention_heads=self.num_attention_heads, + intermediate_size=self.intermediate_size, + hidden_act=self.hidden_act, + hidden_dropout_prob=self.hidden_dropout_prob, + attention_probs_dropout_prob=self.attention_probs_dropout_prob, + max_position_embeddings=self.max_position_embeddings, + type_vocab_size=self.type_vocab_size, + initializer_range=self.initializer_range) + + return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels + + def check_loss_output(self, result): + self.parent.assertListEqual( + list(result["loss"].size()), + []) + + def create_and_check_bert_model(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + model = TFBertModel(config=config) + # model.eval() + inputs = {'input_ids': input_ids, + 'attention_mask': input_mask, + 'token_type_ids': token_type_ids} + sequence_output, pooled_output = model(inputs) + + inputs = [input_ids, input_mask] + sequence_output, pooled_output = model(inputs) + + sequence_output, pooled_output = model(input_ids) + + result = { + "sequence_output": sequence_output.numpy(), + "pooled_output": pooled_output.numpy(), + } + self.parent.assertListEqual( + list(result["sequence_output"].shape), + [self.batch_size, self.seq_length, self.hidden_size]) + self.parent.assertListEqual(list(result["pooled_output"].shape), [self.batch_size, self.hidden_size]) + + + def create_and_check_bert_for_masked_lm(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + pass + # model = BertForMaskedLM(config=config) + # model.eval() + # loss, prediction_scores = model(input_ids, token_type_ids, input_mask, token_labels) + # result = { + # "loss": loss, + # "prediction_scores": prediction_scores, + # } + # self.parent.assertListEqual( + # list(result["prediction_scores"].size()), + # [self.batch_size, self.seq_length, self.vocab_size]) + # self.check_loss_output(result) + + + def create_and_check_bert_for_next_sequence_prediction(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + pass + # model = BertForNextSentencePrediction(config=config) + # model.eval() + # loss, seq_relationship_score = model(input_ids, token_type_ids, input_mask, sequence_labels) + # result = { + # "loss": loss, + # "seq_relationship_score": seq_relationship_score, + # } + # self.parent.assertListEqual( + # list(result["seq_relationship_score"].size()), + # [self.batch_size, 2]) + # self.check_loss_output(result) + + + def create_and_check_bert_for_pretraining(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + pass + # model = BertForPreTraining(config=config) + # model.eval() + # loss, prediction_scores, seq_relationship_score = model(input_ids, token_type_ids, input_mask, token_labels, sequence_labels) + # result = { + # "loss": loss, + # "prediction_scores": prediction_scores, + # "seq_relationship_score": seq_relationship_score, + # } + # self.parent.assertListEqual( + # list(result["prediction_scores"].size()), + # [self.batch_size, self.seq_length, self.vocab_size]) + # self.parent.assertListEqual( + # list(result["seq_relationship_score"].size()), + # [self.batch_size, 2]) + # self.check_loss_output(result) + + + def create_and_check_bert_for_question_answering(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + pass + # model = BertForQuestionAnswering(config=config) + # model.eval() + # loss, start_logits, end_logits = model(input_ids, token_type_ids, input_mask, sequence_labels, sequence_labels) + # result = { + # "loss": loss, + # "start_logits": start_logits, + # "end_logits": end_logits, + # } + # self.parent.assertListEqual( + # list(result["start_logits"].size()), + # [self.batch_size, self.seq_length]) + # self.parent.assertListEqual( + # list(result["end_logits"].size()), + # [self.batch_size, self.seq_length]) + # self.check_loss_output(result) + + + def create_and_check_bert_for_sequence_classification(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + pass + # config.num_labels = self.num_labels + # model = BertForSequenceClassification(config) + # model.eval() + # loss, logits = model(input_ids, token_type_ids, input_mask, sequence_labels) + # result = { + # "loss": loss, + # "logits": logits, + # } + # self.parent.assertListEqual( + # list(result["logits"].size()), + # [self.batch_size, self.num_labels]) + # self.check_loss_output(result) + + + def create_and_check_bert_for_token_classification(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + pass + # config.num_labels = self.num_labels + # model = BertForTokenClassification(config=config) + # model.eval() + # loss, logits = model(input_ids, token_type_ids, input_mask, token_labels) + # result = { + # "loss": loss, + # "logits": logits, + # } + # self.parent.assertListEqual( + # list(result["logits"].size()), + # [self.batch_size, self.seq_length, self.num_labels]) + # self.check_loss_output(result) + + + def create_and_check_bert_for_multiple_choice(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + pass + # config.num_choices = self.num_choices + # model = BertForMultipleChoice(config=config) + # model.eval() + # multiple_choice_inputs_ids = input_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous() + # multiple_choice_token_type_ids = token_type_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous() + # multiple_choice_input_mask = input_mask.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous() + # loss, logits = model(multiple_choice_inputs_ids, + # multiple_choice_token_type_ids, + # multiple_choice_input_mask, + # choice_labels) + # result = { + # "loss": loss, + # "logits": logits, + # } + # self.parent.assertListEqual( + # list(result["logits"].size()), + # [self.batch_size, self.num_choices]) + # self.check_loss_output(result) + + + def prepare_config_and_inputs_for_common(self): + config_and_inputs = self.prepare_config_and_inputs() + (config, input_ids, token_type_ids, input_mask, + sequence_labels, token_labels, choice_labels) = config_and_inputs + inputs_dict = {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask} + return config, inputs_dict + + def setUp(self): + self.model_tester = TFBertModelTest.TFBertModelTester(self) + self.config_tester = ConfigTester(self, config_class=BertConfig, hidden_size=37) + + def test_config(self): + self.config_tester.run_common_tests() + + def test_bert_model(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_bert_model(*config_and_inputs) + + def test_for_masked_lm(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_bert_for_masked_lm(*config_and_inputs) + + def test_for_multiple_choice(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_bert_for_multiple_choice(*config_and_inputs) + + def test_for_next_sequence_prediction(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_bert_for_next_sequence_prediction(*config_and_inputs) + + def test_for_pretraining(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_bert_for_pretraining(*config_and_inputs) + + def test_for_question_answering(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_bert_for_question_answering(*config_and_inputs) + + def test_for_sequence_classification(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_bert_for_sequence_classification(*config_and_inputs) + + def test_for_token_classification(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_bert_for_token_classification(*config_and_inputs) + + @pytest.mark.slow + def test_model_from_pretrained(self): + cache_dir = "/tmp/pytorch_transformers_test/" + for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: + model = TFBertModel.from_pretrained(model_name, cache_dir=cache_dir) + shutil.rmtree(cache_dir) + self.assertIsNotNone(model) + +if __name__ == "__main__": + unittest.main() From bffd17a43d74b228670c2146c284023f250fc85f Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 5 Sep 2019 02:27:39 +0200 Subject: [PATCH 002/219] add tf bert files --- pytorch_transformers/file_utils.py | 6 + pytorch_transformers/modeling_tf_bert.py | 832 ++++++++++++++++++ pytorch_transformers/modeling_tf_utils.py | 255 ++++++ .../tests/modeling_tf_common_test.py | 308 +++++++ .../tests/modeling_tf_test.py | 327 +++++++ 5 files changed, 1728 insertions(+) create mode 100644 pytorch_transformers/modeling_tf_bert.py create mode 100644 pytorch_transformers/modeling_tf_utils.py create mode 100644 pytorch_transformers/tests/modeling_tf_common_test.py create mode 100644 pytorch_transformers/tests/modeling_tf_test.py diff --git a/pytorch_transformers/file_utils.py b/pytorch_transformers/file_utils.py index 3fe7fa891d..a656e757b5 100644 --- a/pytorch_transformers/file_utils.py +++ b/pytorch_transformers/file_utils.py @@ -83,6 +83,9 @@ def url_to_filename(url, etag=None): Convert `url` into a hashed filename in a repeatable way. If `etag` is specified, append its hash to the url's, delimited by a period. + If the url ends with .h5 (Keras HDF5 weights) ands '.h5' to the name + so that TF 2.0 can identify it as a HDF5 file + (see https://github.com/tensorflow/tensorflow/blob/00fad90125b18b80fe054de1055770cfb8fe4ba3/tensorflow/python/keras/engine/network.py#L1380) """ url_bytes = url.encode('utf-8') url_hash = sha256(url_bytes) @@ -93,6 +96,9 @@ def url_to_filename(url, etag=None): etag_hash = sha256(etag_bytes) filename += '.' + etag_hash.hexdigest() + if url.endswith('.h5'): + filename += '.h5' + return filename diff --git a/pytorch_transformers/modeling_tf_bert.py b/pytorch_transformers/modeling_tf_bert.py new file mode 100644 index 0000000000..dda713fe7d --- /dev/null +++ b/pytorch_transformers/modeling_tf_bert.py @@ -0,0 +1,832 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" TF 2.0 BERT model. """ + +from __future__ import absolute_import, division, print_function, unicode_literals + +import json +import logging +import math +import os +import sys +from io import open + +import numpy as np +import tensorflow as tf + +from .configuration_bert import BertConfig +from .modeling_tf_utils import TFPreTrainedModel +from .file_utils import add_start_docstrings + +logger = logging.getLogger(__name__) + + +TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP = { + 'bert-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-tf_model.h5", + 'bert-large-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-tf_model.h5", + 'bert-base-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-tf_model.h5", + 'bert-large-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-tf_model.h5", + 'bert-base-multilingual-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-uncased-tf_model.h5", + 'bert-base-multilingual-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-cased-tf_model.h5", + 'bert-base-chinese': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-chinese-tf_model.h5", + 'bert-base-german-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-german-cased-tf_model.h5", + 'bert-large-uncased-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-whole-word-masking-tf_model.h5", + 'bert-large-cased-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-whole-word-masking-tf_model.h5", + 'bert-large-uncased-whole-word-masking-finetuned-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-whole-word-masking-finetuned-squad-tf_model.h5", + 'bert-large-cased-whole-word-masking-finetuned-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-whole-word-masking-finetuned-squad-tf_model.h5", + 'bert-base-cased-finetuned-mrpc': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-finetuned-mrpc-tf_model.h5", +} + + +def load_pt_weights_in_bert(tf_model, config, pytorch_checkpoint_path): + """ Load pytorch checkpoints in a TF 2.0 model and save it using HDF5 format + We use HDF5 to easily do transfer learning + (see https://github.com/tensorflow/tensorflow/blob/ee16fcac960ae660e0e4496658a366e2f745e1f0/tensorflow/python/keras/engine/network.py#L1352-L1357). + """ + try: + import re + import torch + import numpy + from tensorflow.python.keras import backend as K + except ImportError: + logger.error("Loading a PyTorch model in TensorFlow, requires PyTorch to be installed. Please see " + "https://pytorch.org/ for installation instructions.") + raise + + pt_path = os.path.abspath(pytorch_checkpoint_path) + logger.info("Loading PyTorch weights from {}".format(pt_path)) + # Load pytorch model + state_dict = torch.load(pt_path, map_location='cpu') + + inputs_list = [[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]] + tf_inputs = tf.constant(inputs_list) + tfo = tf_model(tf_inputs, training=False) # build the network + + symbolic_weights = tf_model.trainable_weights + tf_model.non_trainable_weights + weight_value_tuples = [] + for symbolic_weight in symbolic_weights: + name = symbolic_weight.name + name = name.replace('cls_mlm', 'cls') # We had to split this layer in two in the TF model to be + name = name.replace('cls_nsp', 'cls') # able to do transfer learning (Keras only allow to remove full layers) + name = name.replace(':0', '') + name = name.replace('layer_', 'layer/') + name = name.split('/') + name = name[1:] + + transpose = bool(name[-1] == 'kernel') + if name[-1] == 'kernel' or name[-1] == 'embeddings': + name[-1] = 'weight' + + name = '.'.join(name) + assert name in state_dict + array = state_dict[name].numpy() + + if transpose: + array = numpy.transpose(array) + + try: + assert list(symbolic_weight.shape) == list(array.shape) + except AssertionError as e: + e.args += (symbolic_weight.shape, array.shape) + raise e + + logger.info("Initialize TF weight {}".format(symbolic_weight.name)) + + weight_value_tuples.append((symbolic_weight, array)) + + K.batch_set_value(weight_value_tuples) + + tfo = tf_model(tf_inputs, training=False) # Make sure restore ops are run + return tf_model + + +def gelu(x): + """Gaussian Error Linear Unit. + This is a smoother version of the RELU. + Original paper: https://arxiv.org/abs/1606.08415 + Args: + x: float Tensor to perform activation. + Returns: + `x` with the GELU activation applied. + """ + cdf = 0.5 * (1.0 + tf.tanh( + (np.sqrt(2 / np.pi) * (x + 0.044715 * tf.pow(x, 3))))) + return x * cdf + + +def swish(x): + return x * tf.sigmoid(x) + + +ACT2FN = {"gelu": tf.keras.layers.Activation(gelu), + "relu": tf.keras.activations.relu, + "swish": tf.keras.layers.Activation(swish)} + + +class TFBertEmbeddings(tf.keras.layers.Layer): + """Construct the embeddings from word, position and token_type embeddings. + """ + def __init__(self, config, **kwargs): + super(TFBertEmbeddings, self).__init__(**kwargs) + self.word_embeddings = tf.keras.layers.Embedding(config.vocab_size, config.hidden_size, name='word_embeddings') + self.position_embeddings = tf.keras.layers.Embedding(config.max_position_embeddings, config.hidden_size, name='position_embeddings') + self.token_type_embeddings = tf.keras.layers.Embedding(config.type_vocab_size, config.hidden_size, name='token_type_embeddings') + + # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load + # any TensorFlow checkpoint file + self.LayerNorm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name='LayerNorm') + self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) + + def call(self, inputs, training=False): + input_ids, position_ids, token_type_ids = inputs + + seq_length = tf.shape(input_ids)[1] + if position_ids is None: + position_ids = tf.range(seq_length, dtype=tf.int32)[tf.newaxis, :] + if token_type_ids is None: + token_type_ids = tf.fill(tf.shape(input_ids), 0) + + words_embeddings = self.word_embeddings(input_ids) + position_embeddings = self.position_embeddings(position_ids) + token_type_embeddings = self.token_type_embeddings(token_type_ids) + + embeddings = words_embeddings + position_embeddings + token_type_embeddings + embeddings = self.LayerNorm(embeddings) + if training: + embeddings = self.dropout(embeddings) + return embeddings + + +class TFBertSelfAttention(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFBertSelfAttention, self).__init__(**kwargs) + if config.hidden_size % config.num_attention_heads != 0: + raise ValueError( + "The hidden size (%d) is not a multiple of the number of attention " + "heads (%d)" % (config.hidden_size, config.num_attention_heads)) + self.output_attentions = config.output_attentions + + self.num_attention_heads = config.num_attention_heads + assert config.hidden_size % config.num_attention_heads == 0 + self.attention_head_size = int(config.hidden_size / config.num_attention_heads) + self.all_head_size = self.num_attention_heads * self.attention_head_size + + self.query = tf.keras.layers.Dense(self.all_head_size, name='query') + self.key = tf.keras.layers.Dense(self.all_head_size, name='key') + self.value = tf.keras.layers.Dense(self.all_head_size, name='value') + + self.dropout = tf.keras.layers.Dropout(config.attention_probs_dropout_prob) + + def transpose_for_scores(self, x, batch_size): + x = tf.reshape(x, (batch_size, -1, self.num_attention_heads, self.attention_head_size)) + return tf.transpose(x, perm=[0, 2, 1, 3]) + + def call(self, inputs, training=False): + hidden_states, attention_mask, head_mask = inputs + + batch_size = tf.shape(hidden_states)[0] + mixed_query_layer = self.query(hidden_states) + mixed_key_layer = self.key(hidden_states) + mixed_value_layer = self.value(hidden_states) + + query_layer = self.transpose_for_scores(mixed_query_layer, batch_size) + key_layer = self.transpose_for_scores(mixed_key_layer, batch_size) + value_layer = self.transpose_for_scores(mixed_value_layer, batch_size) + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = tf.matmul(query_layer, key_layer, transpose_b=True) # (batch size, num_heads, seq_len_q, seq_len_k) + dk = tf.cast(tf.shape(key_layer)[-1], tf.float32) # scale attention_scores + attention_scores = attention_scores / tf.math.sqrt(dk) + # Apply the attention mask is (precomputed for all layers in TFBertModel call() function) + attention_scores = attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs = tf.nn.softmax(attention_scores, axis=-1) + + if training: + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.dropout(attention_probs) + + # Mask heads if we want to + if head_mask is not None: + attention_probs = attention_probs * head_mask + + context_layer = tf.matmul(attention_probs, value_layer) + + context_layer = tf.transpose(context_layer, perm=[0, 2, 1, 3]) + context_layer = tf.reshape(context_layer, + (batch_size, -1, self.all_head_size)) # (batch_size, seq_len_q, all_head_size) + + outputs = (context_layer, attention_probs) if self.output_attentions else (context_layer,) + return outputs + + +class TFBertSelfOutput(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFBertSelfOutput, self).__init__(**kwargs) + self.dense = tf.keras.layers.Dense(config.hidden_size, name='dense') + self.LayerNorm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name='LayerNorm') + self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) + + def call(self, inputs, training=False): + hidden_states, input_tensor = inputs + + hidden_states = self.dense(hidden_states) + if training: + hidden_states = self.dropout(hidden_states) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +class TFBertAttention(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFBertAttention, self).__init__(**kwargs) + self.self_attention = TFBertSelfAttention(config, name='self') + self.dense_output = TFBertSelfOutput(config, name='output') + + def prune_heads(self, heads): + raise NotImplementedError + + def call(self, inputs, training=False): + input_tensor, attention_mask, head_mask = inputs + + self_outputs = self.self_attention([input_tensor, attention_mask, head_mask], training=training) + attention_output = self.dense_output([self_outputs[0], input_tensor], training=training) + outputs = (attention_output,) + self_outputs[1:] # add attentions if we output them + return outputs + + +class TFBertIntermediate(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFBertIntermediate, self).__init__(**kwargs) + self.dense = tf.keras.layers.Dense(config.intermediate_size, name='dense') + if isinstance(config.hidden_act, str) or (sys.version_info[0] == 2 and isinstance(config.hidden_act, unicode)): + self.intermediate_act_fn = ACT2FN[config.hidden_act] + else: + self.intermediate_act_fn = config.hidden_act + + def call(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.intermediate_act_fn(hidden_states) + return hidden_states + + +class TFBertOutput(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFBertOutput, self).__init__(**kwargs) + self.dense = tf.keras.layers.Dense(config.hidden_size, name='dense') + self.LayerNorm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name='LayerNorm') + self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) + + def call(self, inputs, training=False): + hidden_states, input_tensor = inputs + + hidden_states = self.dense(hidden_states) + if training: + hidden_states = self.dropout(hidden_states) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +class TFBertLayer(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFBertLayer, self).__init__(**kwargs) + self.attention = TFBertAttention(config, name='attention') + self.intermediate = TFBertIntermediate(config, name='intermediate') + self.bert_output = TFBertOutput(config, name='output') + + def call(self, inputs, training=False): + hidden_states, attention_mask, head_mask = inputs + + attention_outputs = self.attention([hidden_states, attention_mask, head_mask], training=training) + attention_output = attention_outputs[0] + intermediate_output = self.intermediate(attention_output) + layer_output = self.bert_output([intermediate_output, attention_output], training=training) + outputs = (layer_output,) + attention_outputs[1:] # add attentions if we output them + return outputs + + +class TFBertEncoder(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFBertEncoder, self).__init__(**kwargs) + self.output_attentions = config.output_attentions + self.output_hidden_states = config.output_hidden_states + self.layer = [TFBertLayer(config, name='layer_{}'.format(i)) for i in range(config.num_hidden_layers)] + + def call(self, inputs, training=False): + hidden_states, attention_mask, head_mask = inputs + + all_hidden_states = () + all_attentions = () + for i, layer_module in enumerate(self.layer): + if self.output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + layer_outputs = layer_module([hidden_states, attention_mask, head_mask[i]], training=training) + hidden_states = layer_outputs[0] + + if self.output_attentions: + all_attentions = all_attentions + (layer_outputs[1],) + + # Add last layer + if self.output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + outputs = (hidden_states,) + if self.output_hidden_states: + outputs = outputs + (all_hidden_states,) + if self.output_attentions: + outputs = outputs + (all_attentions,) + return outputs # outputs, (hidden states), (attentions) + + +class TFBertPooler(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFBertPooler, self).__init__(**kwargs) + self.dense = tf.keras.layers.Dense(config.hidden_size, activation='tanh', name='dense') + + def call(self, hidden_states): + # We "pool" the model by simply taking the hidden state corresponding + # to the first token. + first_token_tensor = hidden_states[:, 0] + pooled_output = self.dense(first_token_tensor) + return pooled_output + + +class TFBertPredictionHeadTransform(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFBertPredictionHeadTransform, self).__init__(**kwargs) + self.dense = tf.keras.layers.Dense(config.hidden_size, name='dense') + if isinstance(config.hidden_act, str) or (sys.version_info[0] == 2 and isinstance(config.hidden_act, unicode)): + self.transform_act_fn = ACT2FN[config.hidden_act] + else: + self.transform_act_fn = config.hidden_act + self.LayerNorm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name='LayerNorm') + + def call(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.transform_act_fn(hidden_states) + hidden_states = self.LayerNorm(hidden_states) + return hidden_states + + +class TFBertLMPredictionHead(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFBertLMPredictionHead, self).__init__(**kwargs) + self.vocab_size = config.vocab_size + self.transform = TFBertPredictionHeadTransform(config, name='transform') + + # The output weights are the same as the input embeddings, but there is + # an output-only bias for each token. + self.decoder = tf.keras.layers.Dense(config.vocab_size, use_bias=False, name='decoder') + + def build(self, input_shape): + self.bias = self.add_weight(shape=(self.vocab_size,), + initializer='zeros', + trainable=True, + name='bias') + + def call(self, hidden_states): + hidden_states = self.transform(hidden_states) + hidden_states = self.decoder(hidden_states) + self.bias + return hidden_states + + +class TFBertMLMHead(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFBertMLMHead, self).__init__(**kwargs) + self.predictions = TFBertLMPredictionHead(config, name='predictions') + + def call(self, sequence_output): + prediction_scores = self.predictions(sequence_output) + return prediction_scores + + +class TFBertNSPHead(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFBertNSPHead, self).__init__(**kwargs) + self.seq_relationship = tf.keras.layers.Dense(2, name='seq_relationship') + + def call(self, pooled_output): + seq_relationship_score = self.seq_relationship(pooled_output) + return seq_relationship_score + + +class TFBertMainLayer(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFBertMainLayer, self).__init__(**kwargs) + self.num_hidden_layers = config.num_hidden_layers + + self.embeddings = TFBertEmbeddings(config, name='embeddings') + self.encoder = TFBertEncoder(config, name='encoder') + self.pooler = TFBertPooler(config, name='pooler') + + # self.apply(self.init_weights) # TODO check weights initialization + + def _resize_token_embeddings(self, new_num_tokens): + raise NotImplementedError + + def _prune_heads(self, heads_to_prune): + """ Prunes heads of the model. + heads_to_prune: dict of {layer_num: list of heads to prune in this layer} + See base class PreTrainedModel + """ + raise NotImplementedError + + def call(self, inputs, training=False): + if not isinstance(inputs, (dict, tuple, list)): + input_ids = inputs + attention_mask, head_mask, position_ids, token_type_ids = None, None, None, None + elif isinstance(inputs, (tuple, list)): + input_ids = inputs[0] + attention_mask = inputs[1] if len(inputs) > 1 else None + token_type_ids = inputs[2] if len(inputs) > 2 else None + position_ids = inputs[3] if len(inputs) > 3 else None + head_mask = inputs[4] if len(inputs) > 4 else None + assert len(inputs) <= 5, "Too many inputs." + else: + input_ids = inputs.pop('input_ids') + attention_mask = inputs.pop('attention_mask', None) + token_type_ids = inputs.pop('token_type_ids', None) + position_ids = inputs.pop('position_ids', None) + head_mask = inputs.pop('head_mask', None) + assert len(inputs) == 0, "Unexpected inputs detected: {}. Check inputs dict key names.".format(list(inputs.keys())) + + if attention_mask is None: + attention_mask = tf.fill(tf.shape(input_ids), 1) + if token_type_ids is None: + token_type_ids = tf.fill(tf.shape(input_ids), 0) + + # We create a 3D attention mask from a 2D tensor mask. + # Sizes are [batch_size, 1, 1, to_seq_length] + # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length] + # this attention mask is more simple than the triangular masking of causal attention + # used in OpenAI GPT, we just need to prepare the broadcast dimension here. + extended_attention_mask = attention_mask[:, tf.newaxis, tf.newaxis, :] + + # Since attention_mask is 1.0 for positions we want to attend and 0.0 for + # masked positions, this operation will create a tensor which is 0.0 for + # positions we want to attend and -10000.0 for masked positions. + # Since we are adding it to the raw scores before the softmax, this is + # effectively the same as removing these entirely. + + extended_attention_mask = tf.cast(extended_attention_mask, tf.float32) + extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0 + + # Prepare head mask if needed + # 1.0 in head_mask indicate we keep the head + # attention_probs has shape bsz x n_heads x N x N + # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] + # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length] + if not head_mask is None: + raise NotImplementedError + else: + head_mask = [None] * self.num_hidden_layers + # head_mask = tf.constant([0] * self.num_hidden_layers) + + embedding_output = self.embeddings([input_ids, position_ids, token_type_ids], training=training) + encoder_outputs = self.encoder([embedding_output, extended_attention_mask, head_mask], training=training) + + sequence_output = encoder_outputs[0] + pooled_output = self.pooler(sequence_output) + + outputs = (sequence_output, pooled_output,) + encoder_outputs[1:] # add hidden_states and attentions if they are here + return outputs # sequence_output, pooled_output, (hidden_states), (attentions) + +class TFBertPreTrainedModel(TFPreTrainedModel): + """ An abstract class to handle weights initialization and + a simple interface for dowloading and loading pretrained models. + """ + config_class = BertConfig + pretrained_model_archive_map = TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP + load_pt_weights = load_pt_weights_in_bert + base_model_prefix = "bert" + + def __init__(self, *inputs, **kwargs): + super(TFBertPreTrainedModel, self).__init__(*inputs, **kwargs) + + def init_weights(self, module): + """ Initialize the weights. + """ + raise NotImplementedError + + +BERT_START_DOCSTRING = r""" The BERT model was proposed in + `BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ + by Jacob Devlin, Ming-Wei Chang, Kenton Lee and Kristina Toutanova. It's a bidirectional transformer + pre-trained using a combination of masked language modeling objective and next sentence prediction + on a large corpus comprising the Toronto Book Corpus and Wikipedia. + + This model is a tf.keras.Model `tf.keras.Model`_ sub-class. Use it as a regular TF 2.0 Keras Model and + refer to the TF 2.0 documentation for all matter related to general usage and behavior. + + .. _`BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`: + https://arxiv.org/abs/1810.04805 + + .. _`tf.keras.Model`: + https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/Model + + Important note on the model inputs: + The inputs of the TF 2.0 models are slightly different from the PyTorch ones since + TF 2.0 Keras doesn't accept named arguments with defaults values for input Tensor. + More precisely, input Tensors are gathered in the first arguments of the model call function: `model(inputs)`. + There are three possibilities to gather and feed the inputs to the model: + + - a single Tensor with input_ids only and nothing else: `model(inputs_ids) + - a list of varying length with one or several input Tensors IN THE ORDER given in the docstring: + `model([input_ids, attention_mask])` or `model([input_ids, attention_mask, token_type_ids])` + - a dictionary with one or several input Tensors associaed to the input names given in the docstring: + `model({'input_ids': input_ids, 'token_type_ids': token_type_ids})` + + Parameters: + config (:class:`~pytorch_transformers.BertConfig`): Model configuration class with all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the configuration. + Check out the :meth:`~pytorch_transformers.PreTrainedModel.from_pretrained` method to load the model weights. +""" + +BERT_INPUTS_DOCSTRING = r""" + Inputs: + **input_ids**: ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Indices of input sequence tokens in the vocabulary. + To match pre-training, BERT input sequence should be formatted with [CLS] and [SEP] tokens as follows: + + (a) For sequence pairs: + + ``tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]`` + + ``token_type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1`` + + (b) For single sequences: + + ``tokens: [CLS] the dog is hairy . [SEP]`` + + ``token_type_ids: 0 0 0 0 0 0 0`` + + Bert is a model with absolute position embeddings so it's usually advised to pad the inputs on + the right rather than the left. + + Indices can be obtained using :class:`pytorch_transformers.BertTokenizer`. + See :func:`pytorch_transformers.PreTrainedTokenizer.encode` and + :func:`pytorch_transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``: + Mask to avoid performing attention on padding token indices. + Mask values selected in ``[0, 1]``: + ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. + **token_type_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Segment token indices to indicate first and second portions of the inputs. + Indices are selected in ``[0, 1]``: ``0`` corresponds to a `sentence A` token, ``1`` + corresponds to a `sentence B` token + (see `BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ for more details). + **position_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Indices of positions of each input sequence tokens in the position embeddings. + Selected in the range ``[0, config.max_position_embeddings - 1]``. + **head_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: + Mask to nullify selected heads of the self-attention modules. + Mask values selected in ``[0, 1]``: + ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. +""" + +@add_start_docstrings("The bare Bert Model transformer outputing raw hidden-states without any specific head on top.", + BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) +class TFBertModel(TFBertPreTrainedModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)`` + Sequence of hidden-states at the output of the last layer of the model. + **pooler_output**: ``torch.FloatTensor`` of shape ``(batch_size, hidden_size)`` + Last layer hidden-state of the first token of the sequence (classification token) + further processed by a Linear layer and a Tanh activation function. The Linear + layer weights are trained from the next sentence prediction (classification) + objective during Bert pretraining. This output is usually *not* a good summary + of the semantic content of the input, you're often better with averaging or pooling + the sequence of hidden-states for the whole input sequence. + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + model = TFBertModel.from_pretrained('bert-base-uncased') + input_ids = tf.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids) + last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple + + """ + def __init__(self, config): + super(TFBertModel, self).__init__(config) + self.bert = TFBertMainLayer(config, name='bert') + + def call(self, inputs, training=False): + outputs = self.bert(inputs, training=training) + return outputs + + +@add_start_docstrings("""Bert Model with two heads on top as done during the pre-training: + a `masked language modeling` head and a `next sentence prediction (classification)` head. """, + BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) +class TFBertForPreTraining(TFBertPreTrainedModel): + r""" + **masked_lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Labels for computing the masked language modeling loss. + Indices should be in ``[-1, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring) + Tokens with indices set to ``-1`` are ignored (masked), the loss is only computed for the tokens with labels + in ``[0, ..., config.vocab_size]`` + **next_sentence_label**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair (see ``input_ids`` docstring) + Indices should be in ``[0, 1]``. + ``0`` indicates sequence B is a continuation of sequence A, + ``1`` indicates sequence B is a random sequence. + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when both ``masked_lm_labels`` and ``next_sentence_label`` are provided) ``torch.FloatTensor`` of shape ``(1,)``: + Total loss as the sum of the masked language modeling loss and the next sequence prediction (classification) loss. + **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + **seq_relationship_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, 2)`` + Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + model = TFBertForPreTraining.from_pretrained('bert-base-uncased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids) + prediction_scores, seq_relationship_scores = outputs[:2] + + """ + def __init__(self, config): + super(TFBertForPreTraining, self).__init__(config) + + self.bert = TFBertMainLayer(config, name='bert') + self.cls_mlm = TFBertMLMHead(config, name='cls_mlm') + self.cls_nsp = TFBertNSPHead(config, name='cls_nsp') + + # self.apply(self.init_weights) # TODO check added weights initialization + self.tie_weights() + + def tie_weights(self): + """ Make sure we are sharing the input and output embeddings. + """ + pass # TODO add weights tying + + def call(self, inputs, training=False): + outputs = self.bert(inputs, training=training) + + sequence_output, pooled_output = outputs[:2] + prediction_scores = self.cls_mlm(sequence_output) + seq_relationship_score = self.cls_nsp(pooled_output) + + outputs = (prediction_scores, seq_relationship_score,) + outputs[2:] # add hidden states and attention if they are here + + # if masked_lm_labels is not None and next_sentence_label is not None: + # loss_fct = CrossEntropyLoss(ignore_index=-1) + # masked_lm_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), masked_lm_labels.view(-1)) + # next_sentence_loss = loss_fct(seq_relationship_score.view(-1, 2), next_sentence_label.view(-1)) + # total_loss = masked_lm_loss + next_sentence_loss + # outputs = (total_loss,) + outputs + # TODO add example with losses using model.compile and a dictionary of losses (give names to the output layers) + + return outputs # prediction_scores, seq_relationship_score, (hidden_states), (attentions) + + +@add_start_docstrings("""Bert Model with a `language modeling` head on top. """, + BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) +class TFBertForMaskedLM(TFBertPreTrainedModel): + r""" + **masked_lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Labels for computing the masked language modeling loss. + Indices should be in ``[-1, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring) + Tokens with indices set to ``-1`` are ignored (masked), the loss is only computed for the tokens with labels + in ``[0, ..., config.vocab_size]`` + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``masked_lm_labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Masked language modeling loss. + **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + model = TFBertForMaskedLM.from_pretrained('bert-base-uncased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, masked_lm_labels=input_ids) + loss, prediction_scores = outputs[:2] + + """ + def __init__(self, config): + super(TFBertForMaskedLM, self).__init__(config) + + self.bert = TFBertMainLayer(config, name='bert') + self.cls_mlm = TFBertMLMHead(config, name='cls_mlm') + + # self.apply(self.init_weights) + self.tie_weights() + + def tie_weights(self): + """ Make sure we are sharing the input and output embeddings. + """ + pass # TODO add weights tying + + def call(self, inputs, training=False): + outputs = self.bert(inputs, training=training) + + sequence_output = outputs[0] + prediction_scores = self.cls_mlm(sequence_output) + + outputs = (prediction_scores,) + outputs[2:] # Add hidden states and attention if they are here + # if masked_lm_labels is not None: + # loss_fct = CrossEntropyLoss(ignore_index=-1) + # masked_lm_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), masked_lm_labels.view(-1)) + # outputs = (masked_lm_loss,) + outputs + # TODO example with losses + + return outputs # prediction_scores, (hidden_states), (attentions) + + +@add_start_docstrings("""Bert Model with a `next sentence prediction (classification)` head on top. """, + BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) +class TFBertForNextSentencePrediction(TFBertPreTrainedModel): + r""" + **next_sentence_label**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair (see ``input_ids`` docstring) + Indices should be in ``[0, 1]``. + ``0`` indicates sequence B is a continuation of sequence A, + ``1`` indicates sequence B is a random sequence. + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``next_sentence_label`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Next sequence prediction (classification) loss. + **seq_relationship_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, 2)`` + Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + model = TFBertForNextSentencePrediction.from_pretrained('bert-base-uncased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids) + seq_relationship_scores = outputs[0] + + """ + def __init__(self, config): + super(TFBertForNextSentencePrediction, self).__init__(config) + + self.bert = TFBertMainLayer(config, name='bert') + self.cls_nsp = TFBertNSPHead(config, name='cls_nsp') + + # self.apply(self.init_weights) + + def call(self, inputs, training=False): + outputs = self.bert(inputs, training=training) + + pooled_output = outputs[1] + seq_relationship_score = self.cls_nsp(pooled_output) + + outputs = (seq_relationship_score,) + outputs[2:] # add hidden states and attention if they are here + # if next_sentence_label is not None: + # loss_fct = CrossEntropyLoss(ignore_index=-1) + # next_sentence_loss = loss_fct(seq_relationship_score.view(-1, 2), next_sentence_label.view(-1)) + # outputs = (next_sentence_loss,) + outputs + + return outputs # seq_relationship_score, (hidden_states), (attentions) diff --git a/pytorch_transformers/modeling_tf_utils.py b/pytorch_transformers/modeling_tf_utils.py new file mode 100644 index 0000000000..442d9dbe44 --- /dev/null +++ b/pytorch_transformers/modeling_tf_utils.py @@ -0,0 +1,255 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""TF general model utils.""" + +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +import logging +import os + +import tensorflow as tf + +from .configuration_utils import PretrainedConfig +from .file_utils import cached_path, WEIGHTS_NAME, TF_WEIGHTS_NAME + +logger = logging.getLogger(__name__) + + +class TFPreTrainedModel(tf.keras.Model): + r""" Base class for all TF models. + + :class:`~pytorch_transformers.TFPreTrainedModel` takes care of storing the configuration of the models and handles methods for loading/downloading/saving models + as well as a few methods commons to all models to (i) resize the input embeddings and (ii) prune heads in the self-attention heads. + + Class attributes (overridden by derived classes): + - ``config_class``: a class derived from :class:`~pytorch_transformers.PretrainedConfig` to use as configuration class for this model architecture. + - ``pretrained_model_archive_map``: a python ``dict`` of with `short-cut-names` (string) as keys and `url` (string) of associated pretrained weights as values. + - ``load_tf_weights``: a python ``method`` for loading a TensorFlow checkpoint in a PyTorch model, taking as arguments: + + - ``model``: an instance of the relevant subclass of :class:`~pytorch_transformers.PreTrainedModel`, + - ``config``: an instance of the relevant subclass of :class:`~pytorch_transformers.PretrainedConfig`, + - ``path``: a path (string) to the TensorFlow checkpoint. + + - ``base_model_prefix``: a string indicating the attribute associated to the base model in derived classes of the same architecture adding modules on top of the base model. + """ + config_class = None + pretrained_model_archive_map = {} + load_pt_weights = lambda model, config, path: None + base_model_prefix = "" + + def __init__(self, config, *inputs, **kwargs): + super(TFPreTrainedModel, self).__init__() + if not isinstance(config, PretrainedConfig): + raise ValueError( + "Parameter config in `{}(config)` should be an instance of class `PretrainedConfig`. " + "To create a model from a pretrained model use " + "`model = {}.from_pretrained(PRETRAINED_MODEL_NAME)`".format( + self.__class__.__name__, self.__class__.__name__ + )) + # Save config in model + self.config = config + + def _get_resized_embeddings(self, old_embeddings, new_num_tokens=None): + """ Build a resized Embedding Module from a provided token Embedding Module. + Increasing the size will add newly initialized vectors at the end + Reducing the size will remove vectors from the end + + Args: + new_num_tokens: (`optional`) int + New number of tokens in the embedding matrix. + Increasing the size will add newly initialized vectors at the end + Reducing the size will remove vectors from the end + If not provided or None: return the provided token Embedding Module. + Return: ``torch.nn.Embeddings`` + Pointer to the resized Embedding Module or the old Embedding Module if new_num_tokens is None + """ + raise NotImplementedError + + def _tie_or_clone_weights(self, first_module, second_module): + """ Tie or clone module weights depending of weither we are using TorchScript or not + """ + raise NotImplementedError + + def resize_token_embeddings(self, new_num_tokens=None): + """ Resize input token embeddings matrix of the model if new_num_tokens != config.vocab_size. + Take care of tying weights embeddings afterwards if the model class has a `tie_weights()` method. + + Arguments: + + new_num_tokens: (`optional`) int: + New number of tokens in the embedding matrix. Increasing the size will add newly initialized vectors at the end. Reducing the size will remove vectors from the end. + If not provided or None: does nothing and just returns a pointer to the input tokens ``torch.nn.Embeddings`` Module of the model. + + Return: ``torch.nn.Embeddings`` + Pointer to the input tokens Embeddings Module of the model + """ + raise NotImplementedError + + def prune_heads(self, heads_to_prune): + """ Prunes heads of the base model. + + Arguments: + + heads_to_prune: dict with keys being selected layer indices (`int`) and associated values being the list of heads to prune in said layer (list of `int`). + """ + raise NotImplementedError + + def save_pretrained(self, save_directory): + """ Save a model and its configuration file to a directory, so that it + can be re-loaded using the `:func:`~pytorch_transformers.PreTrainedModel.from_pretrained`` class method. + """ + raise NotImplementedError + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): + r"""Instantiate a pretrained pytorch model from a pre-trained model configuration. + + The model is set in evaluation mode by default using ``model.eval()`` (Dropout modules are deactivated) + To train the model, you should first set it back in training mode with ``model.train()`` + + The warning ``Weights from XXX not initialized from pretrained model`` means that the weights of XXX do not come pre-trained with the rest of the model. + It is up to you to train those weights with a downstream fine-tuning task. + + The warning ``Weights from XXX not used in YYY`` means that the layer XXX is not used by YYY, therefore those weights are discarded. + + Parameters: + pretrained_model_name_or_path: either: + + - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. + - a path to a `directory` containing model weights saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. + - a path or url to a `PyTorch state_dict save file` (e.g. `./pt_model/pytorch_model.bin`). In this case, ``from_pt`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the PyTorch checkpoint in a TensorFlow model using the provided conversion scripts and loading the TensorFlow model afterwards. + + model_args: (`optional`) Sequence of positional arguments: + All remaning positional arguments will be passed to the underlying model's ``__init__`` method + + config: (`optional`) instance of a class derived from :class:`~pytorch_transformers.PretrainedConfig`: + Configuration for the model to use instead of an automatically loaded configuation. Configuration can be automatically loaded when: + + - the model is a model provided by the library (loaded with the ``shortcut-name`` string of a pretrained model), or + - the model was saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained` and is reloaded by suppling the save directory. + - the model is loaded by suppling a local directory as ``pretrained_model_name_or_path`` and a configuration JSON file named `config.json` is found in the directory. + + from_pt: (`optional`) boolean, default False: + Load the model weights from a PyTorch state_dict save file (see docstring of pretrained_model_name_or_path argument). + + cache_dir: (`optional`) string: + Path to a directory in which a downloaded pre-trained model + configuration should be cached if the standard cache should not be used. + + force_download: (`optional`) boolean, default False: + Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + + proxies: (`optional`) dict, default None: + A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. + The proxies are used on each request. + + output_loading_info: (`optional`) boolean: + Set to ``True`` to also return a dictionnary containing missing keys, unexpected keys and error messages. + + kwargs: (`optional`) Remaining dictionary of keyword arguments: + Can be used to update the configuration object (after it being loaded) and initiate the model. (e.g. ``output_attention=True``). Behave differently depending on whether a `config` is provided or automatically loaded: + + - If a configuration is provided with ``config``, ``**kwargs`` will be directly passed to the underlying model's ``__init__`` method (we assume all relevant updates to the configuration have already been done) + - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~pytorch_transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function. + + Examples:: + + model = BertModel.from_pretrained('bert-base-uncased') # Download model and configuration from S3 and cache. + model = BertModel.from_pretrained('./test/saved_model/') # E.g. model was saved using `save_pretrained('./test/saved_model/')` + model = BertModel.from_pretrained('bert-base-uncased', output_attention=True) # Update configuration during loading + assert model.config.output_attention == True + # Loading from a TF checkpoint file instead of a PyTorch model (slower) + config = BertConfig.from_json_file('./tf_model/my_tf_model_config.json') + model = BertModel.from_pretrained('./tf_model/my_tf_checkpoint.ckpt.index', from_pt=True, config=config) + + """ + config = kwargs.pop('config', None) + cache_dir = kwargs.pop('cache_dir', None) + from_pt = kwargs.pop('from_pt', False) + force_download = kwargs.pop('force_download', False) + proxies = kwargs.pop('proxies', None) + output_loading_info = kwargs.pop('output_loading_info', False) + + # Load config + if config is None: + config, model_kwargs = cls.config_class.from_pretrained( + pretrained_model_name_or_path, *model_args, + cache_dir=cache_dir, return_unused_kwargs=True, + force_download=force_download, + **kwargs + ) + else: + model_kwargs = kwargs + + # Load model + if pretrained_model_name_or_path in cls.pretrained_model_archive_map: + archive_file = cls.pretrained_model_archive_map[pretrained_model_name_or_path] + elif os.path.isdir(pretrained_model_name_or_path): + if from_pt: + # Load from a PyTorch checkpoint + archive_file = os.path.join(pretrained_model_name_or_path, WEIGHTS_NAME) + else: + archive_file = os.path.join(pretrained_model_name_or_path, TF_WEIGHTS_NAME) + else: + archive_file = pretrained_model_name_or_path + # redirect to the cache, if necessary + try: + resolved_archive_file = cached_path(archive_file, cache_dir=cache_dir, force_download=force_download, proxies=proxies) + except EnvironmentError: + if pretrained_model_name_or_path in cls.pretrained_model_archive_map: + logger.error( + "Couldn't reach server at '{}' to download pretrained weights.".format( + archive_file)) + else: + logger.error( + "Model name '{}' was not found in model name list ({}). " + "We assumed '{}' was a path or url but couldn't find any file " + "associated to this path or url.".format( + pretrained_model_name_or_path, + ', '.join(cls.pretrained_model_archive_map.keys()), + archive_file)) + return None + if resolved_archive_file == archive_file: + logger.info("loading weights file {}".format(archive_file)) + else: + logger.info("loading weights file {} from cache at {}".format( + archive_file, resolved_archive_file)) + + # Instantiate model. + model = cls(config, *model_args, **model_kwargs) + + if from_pt: + # Load from a PyTorch checkpoint + return cls.load_pt_weights(model, config, resolved_archive_file) + + inputs = tf.constant([[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]]) + ret = model(inputs, training=False) # build the network with dummy inputs + + # 'by_name' allow us to do transfer learning by skipping/adding layers + # see https://github.com/tensorflow/tensorflow/blob/00fad90125b18b80fe054de1055770cfb8fe4ba3/tensorflow/python/keras/engine/network.py#L1339-L1357 + model.load_weights(resolved_archive_file, by_name=True) + + ret = model(inputs, training=False) # Make sure restore ops are run + + # if hasattr(model, 'tie_weights'): + # model.tie_weights() # TODO make sure word embedding weights are still tied + + if output_loading_info: + loading_info = {"missing_keys": missing_keys, "unexpected_keys": unexpected_keys, "error_msgs": error_msgs} + return model, loading_info + + return model diff --git a/pytorch_transformers/tests/modeling_tf_common_test.py b/pytorch_transformers/tests/modeling_tf_common_test.py new file mode 100644 index 0000000000..d432ab5fdf --- /dev/null +++ b/pytorch_transformers/tests/modeling_tf_common_test.py @@ -0,0 +1,308 @@ +# coding=utf-8 +# Copyright 2019 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import copy +import os +import shutil +import json +import random +import uuid + +import unittest +import logging + +import tensorflow as tf + +from pytorch_transformers import TFPreTrainedModel +# from pytorch_transformers.modeling_bert import BertModel, BertConfig, BERT_PRETRAINED_MODEL_ARCHIVE_MAP + + +def _config_zero_init(config): + configs_no_init = copy.deepcopy(config) + for key in configs_no_init.__dict__.keys(): + if '_range' in key or '_std' in key: + setattr(configs_no_init, key, 0.0) + return configs_no_init + +class TFCommonTestCases: + + class TFCommonModelTester(unittest.TestCase): + + model_tester = None + all_model_classes = () + test_torchscript = True + test_pruning = True + test_resize_embeddings = True + + def test_initialization(self): + pass + # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + # configs_no_init = _config_zero_init(config) + # for model_class in self.all_model_classes: + # model = model_class(config=configs_no_init) + # for name, param in model.named_parameters(): + # if param.requires_grad: + # self.assertIn(param.data.mean().item(), [0.0, 1.0], + # msg="Parameter {} of model {} seems not properly initialized".format(name, model_class)) + + + def test_attention_outputs(self): + pass + # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + # for model_class in self.all_model_classes: + # config.output_attentions = True + # config.output_hidden_states = False + # model = model_class(config) + # model.eval() + # outputs = model(**inputs_dict) + # attentions = outputs[-1] + # self.assertEqual(model.config.output_attentions, True) + # self.assertEqual(model.config.output_hidden_states, False) + # self.assertEqual(len(attentions), self.model_tester.num_hidden_layers) + # self.assertListEqual( + # list(attentions[0].shape[-3:]), + # [self.model_tester.num_attention_heads, + # self.model_tester.seq_length, + # self.model_tester.key_len if hasattr(self.model_tester, 'key_len') else self.model_tester.seq_length]) + # out_len = len(outputs) + + # # Check attention is always last and order is fine + # config.output_attentions = True + # config.output_hidden_states = True + # model = model_class(config) + # model.eval() + # outputs = model(**inputs_dict) + # self.assertEqual(out_len+1, len(outputs)) + # self.assertEqual(model.config.output_attentions, True) + # self.assertEqual(model.config.output_hidden_states, True) + + # attentions = outputs[-1] + # self.assertEqual(len(attentions), self.model_tester.num_hidden_layers) + # self.assertListEqual( + # list(attentions[0].shape[-3:]), + # [self.model_tester.num_attention_heads, + # self.model_tester.seq_length, + # self.model_tester.key_len if hasattr(self.model_tester, 'key_len') else self.model_tester.seq_length]) + + + def test_headmasking(self): + pass + # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + # config.output_attentions = True + # config.output_hidden_states = True + # configs_no_init = _config_zero_init(config) # To be sure we have no Nan + # for model_class in self.all_model_classes: + # model = model_class(config=configs_no_init) + # model.eval() + + # # Prepare head_mask + # # Set require_grad after having prepared the tensor to avoid error (leaf variable has been moved into the graph interior) + # head_mask = torch.ones(self.model_tester.num_hidden_layers, self.model_tester.num_attention_heads) + # head_mask[0, 0] = 0 + # head_mask[-1, :-1] = 0 + # head_mask.requires_grad_(requires_grad=True) + # inputs = inputs_dict.copy() + # inputs['head_mask'] = head_mask + + # outputs = model(**inputs) + + # # Test that we can get a gradient back for importance score computation + # output = sum(t.sum() for t in outputs[0]) + # output = output.sum() + # output.backward() + # multihead_outputs = head_mask.grad + + # attentions = outputs[-1] + # hidden_states = outputs[-2] + + # # Remove Nan + + # self.assertIsNotNone(multihead_outputs) + # self.assertEqual(len(multihead_outputs), self.model_tester.num_hidden_layers) + # self.assertAlmostEqual( + # attentions[0][..., 0, :, :].flatten().sum().item(), 0.0) + # self.assertNotEqual( + # attentions[0][..., -1, :, :].flatten().sum().item(), 0.0) + # self.assertNotEqual( + # attentions[1][..., 0, :, :].flatten().sum().item(), 0.0) + # self.assertAlmostEqual( + # attentions[-1][..., -2, :, :].flatten().sum().item(), 0.0) + # self.assertNotEqual( + # attentions[-1][..., -1, :, :].flatten().sum().item(), 0.0) + + + def test_head_pruning(self): + pass + # if not self.test_pruning: + # return + + # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + # for model_class in self.all_model_classes: + # config.output_attentions = True + # config.output_hidden_states = False + # model = model_class(config=config) + # model.eval() + # heads_to_prune = {0: list(range(1, self.model_tester.num_attention_heads)), + # -1: [0]} + # model.prune_heads(heads_to_prune) + # outputs = model(**inputs_dict) + + # attentions = outputs[-1] + + # self.assertEqual( + # attentions[0].shape[-3], 1) + # self.assertEqual( + # attentions[1].shape[-3], self.model_tester.num_attention_heads) + # self.assertEqual( + # attentions[-1].shape[-3], self.model_tester.num_attention_heads - 1) + + + def test_hidden_states_output(self): + pass + # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + # for model_class in self.all_model_classes: + # config.output_hidden_states = True + # config.output_attentions = False + # model = model_class(config) + # model.eval() + # outputs = model(**inputs_dict) + # hidden_states = outputs[-1] + # self.assertEqual(model.config.output_attentions, False) + # self.assertEqual(model.config.output_hidden_states, True) + # self.assertEqual(len(hidden_states), self.model_tester.num_hidden_layers + 1) + # self.assertListEqual( + # list(hidden_states[0].shape[-2:]), + # [self.model_tester.seq_length, self.model_tester.hidden_size]) + + + def test_resize_tokens_embeddings(self): + pass + # original_config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + # if not self.test_resize_embeddings: + # return + + # for model_class in self.all_model_classes: + # config = copy.deepcopy(original_config) + # model = model_class(config) + + # model_vocab_size = config.vocab_size + # # Retrieve the embeddings and clone theme + # model_embed = model.resize_token_embeddings(model_vocab_size) + # cloned_embeddings = model_embed.weight.clone() + + # # Check that resizing the token embeddings with a larger vocab size increases the model's vocab size + # model_embed = model.resize_token_embeddings(model_vocab_size + 10) + # self.assertEqual(model.config.vocab_size, model_vocab_size + 10) + # # Check that it actually resizes the embeddings matrix + # self.assertEqual(model_embed.weight.shape[0], cloned_embeddings.shape[0] + 10) + + # # Check that resizing the token embeddings with a smaller vocab size decreases the model's vocab size + # model_embed = model.resize_token_embeddings(model_vocab_size - 15) + # self.assertEqual(model.config.vocab_size, model_vocab_size - 15) + # # Check that it actually resizes the embeddings matrix + # self.assertEqual(model_embed.weight.shape[0], cloned_embeddings.shape[0] - 15) + + # # Check that adding and removing tokens has not modified the first part of the embedding matrix. + # models_equal = True + # for p1, p2 in zip(cloned_embeddings, model_embed.weight): + # if p1.data.ne(p2.data).sum() > 0: + # models_equal = False + + # self.assertTrue(models_equal) + + + def test_tie_model_weights(self): + pass + # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + # def check_same_values(layer_1, layer_2): + # equal = True + # for p1, p2 in zip(layer_1.weight, layer_2.weight): + # if p1.data.ne(p2.data).sum() > 0: + # equal = False + # return equal + + # for model_class in self.all_model_classes: + # if not hasattr(model_class, 'tie_weights'): + # continue + + # config.torchscript = True + # model_not_tied = model_class(config) + # params_not_tied = list(model_not_tied.parameters()) + + # config_tied = copy.deepcopy(config) + # config_tied.torchscript = False + # model_tied = model_class(config_tied) + # params_tied = list(model_tied.parameters()) + + # # Check that the embedding layer and decoding layer are the same in size and in value + # self.assertGreater(len(params_not_tied), len(params_tied)) + + # # Check that after resize they remain tied. + # model_tied.resize_token_embeddings(config.vocab_size + 10) + # params_tied_2 = list(model_tied.parameters()) + # self.assertGreater(len(params_not_tied), len(params_tied)) + # self.assertEqual(len(params_tied_2), len(params_tied)) + + +def ids_tensor(shape, vocab_size, rng=None, name=None): + """Creates a random int32 tensor of the shape within the vocab size.""" + if rng is None: + rng = random.Random() + + total_dims = 1 + for dim in shape: + total_dims *= dim + + values = [] + for _ in range(total_dims): + values.append(rng.randint(0, vocab_size - 1)) + + return tf.constant(values, shape=shape) + + +class TFModelUtilsTest(unittest.TestCase): + def test_model_from_pretrained(self): + pass + # logging.basicConfig(level=logging.INFO) + # for model_name in list(BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: + # config = BertConfig.from_pretrained(model_name) + # self.assertIsNotNone(config) + # self.assertIsInstance(config, PretrainedConfig) + + # model = BertModel.from_pretrained(model_name) + # model, loading_info = BertModel.from_pretrained(model_name, output_loading_info=True) + # self.assertIsNotNone(model) + # self.assertIsInstance(model, PreTrainedModel) + # for value in loading_info.values(): + # self.assertEqual(len(value), 0) + + # config = BertConfig.from_pretrained(model_name, output_attentions=True, output_hidden_states=True) + # model = BertModel.from_pretrained(model_name, output_attentions=True, output_hidden_states=True) + # self.assertEqual(model.config.output_attentions, True) + # self.assertEqual(model.config.output_hidden_states, True) + # self.assertEqual(model.config, config) + + +if __name__ == "__main__": + unittest.main() diff --git a/pytorch_transformers/tests/modeling_tf_test.py b/pytorch_transformers/tests/modeling_tf_test.py new file mode 100644 index 0000000000..ee2580bd9e --- /dev/null +++ b/pytorch_transformers/tests/modeling_tf_test.py @@ -0,0 +1,327 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import unittest +import shutil +import pytest + +import tensorflow as tf + +from pytorch_transformers import (BertConfig) +from pytorch_transformers.modeling_tf_bert import TFBertModel, TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP + +from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) +from .configuration_common_test import ConfigTester + + +class TFBertModelTest(TFCommonTestCases.TFCommonModelTester): + + all_model_classes = (TFBertModel,) + # BertForMaskedLM, BertForNextSentencePrediction, + # BertForPreTraining, BertForQuestionAnswering, BertForSequenceClassification, + # BertForTokenClassification) + + class TFBertModelTester(object): + + def __init__(self, + parent, + batch_size=13, + seq_length=7, + is_training=True, + use_input_mask=True, + use_token_type_ids=True, + use_labels=True, + vocab_size=99, + hidden_size=32, + num_hidden_layers=5, + num_attention_heads=4, + intermediate_size=37, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + type_sequence_label_size=2, + initializer_range=0.02, + num_labels=3, + num_choices=4, + scope=None, + ): + self.parent = parent + self.batch_size = batch_size + self.seq_length = seq_length + self.is_training = is_training + self.use_input_mask = use_input_mask + self.use_token_type_ids = use_token_type_ids + self.use_labels = use_labels + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.intermediate_size = intermediate_size + self.hidden_act = hidden_act + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.type_sequence_label_size = type_sequence_label_size + self.initializer_range = initializer_range + self.num_labels = num_labels + self.num_choices = num_choices + self.scope = scope + + def prepare_config_and_inputs(self): + input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) + + input_mask = None + if self.use_input_mask: + input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2) + + token_type_ids = None + if self.use_token_type_ids: + token_type_ids = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size) + + sequence_labels = None + token_labels = None + choice_labels = None + if self.use_labels: + sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size) + token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels) + choice_labels = ids_tensor([self.batch_size], self.num_choices) + + config = BertConfig( + vocab_size_or_config_json_file=self.vocab_size, + hidden_size=self.hidden_size, + num_hidden_layers=self.num_hidden_layers, + num_attention_heads=self.num_attention_heads, + intermediate_size=self.intermediate_size, + hidden_act=self.hidden_act, + hidden_dropout_prob=self.hidden_dropout_prob, + attention_probs_dropout_prob=self.attention_probs_dropout_prob, + max_position_embeddings=self.max_position_embeddings, + type_vocab_size=self.type_vocab_size, + initializer_range=self.initializer_range) + + return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels + + def check_loss_output(self, result): + self.parent.assertListEqual( + list(result["loss"].size()), + []) + + def create_and_check_bert_model(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + model = TFBertModel(config=config) + # model.eval() + inputs = {'input_ids': input_ids, + 'attention_mask': input_mask, + 'token_type_ids': token_type_ids} + sequence_output, pooled_output = model(inputs) + + inputs = [input_ids, input_mask] + sequence_output, pooled_output = model(inputs) + + sequence_output, pooled_output = model(input_ids) + + result = { + "sequence_output": sequence_output.numpy(), + "pooled_output": pooled_output.numpy(), + } + self.parent.assertListEqual( + list(result["sequence_output"].shape), + [self.batch_size, self.seq_length, self.hidden_size]) + self.parent.assertListEqual(list(result["pooled_output"].shape), [self.batch_size, self.hidden_size]) + + + def create_and_check_bert_for_masked_lm(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + pass + # model = BertForMaskedLM(config=config) + # model.eval() + # loss, prediction_scores = model(input_ids, token_type_ids, input_mask, token_labels) + # result = { + # "loss": loss, + # "prediction_scores": prediction_scores, + # } + # self.parent.assertListEqual( + # list(result["prediction_scores"].size()), + # [self.batch_size, self.seq_length, self.vocab_size]) + # self.check_loss_output(result) + + + def create_and_check_bert_for_next_sequence_prediction(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + pass + # model = BertForNextSentencePrediction(config=config) + # model.eval() + # loss, seq_relationship_score = model(input_ids, token_type_ids, input_mask, sequence_labels) + # result = { + # "loss": loss, + # "seq_relationship_score": seq_relationship_score, + # } + # self.parent.assertListEqual( + # list(result["seq_relationship_score"].size()), + # [self.batch_size, 2]) + # self.check_loss_output(result) + + + def create_and_check_bert_for_pretraining(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + pass + # model = BertForPreTraining(config=config) + # model.eval() + # loss, prediction_scores, seq_relationship_score = model(input_ids, token_type_ids, input_mask, token_labels, sequence_labels) + # result = { + # "loss": loss, + # "prediction_scores": prediction_scores, + # "seq_relationship_score": seq_relationship_score, + # } + # self.parent.assertListEqual( + # list(result["prediction_scores"].size()), + # [self.batch_size, self.seq_length, self.vocab_size]) + # self.parent.assertListEqual( + # list(result["seq_relationship_score"].size()), + # [self.batch_size, 2]) + # self.check_loss_output(result) + + + def create_and_check_bert_for_question_answering(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + pass + # model = BertForQuestionAnswering(config=config) + # model.eval() + # loss, start_logits, end_logits = model(input_ids, token_type_ids, input_mask, sequence_labels, sequence_labels) + # result = { + # "loss": loss, + # "start_logits": start_logits, + # "end_logits": end_logits, + # } + # self.parent.assertListEqual( + # list(result["start_logits"].size()), + # [self.batch_size, self.seq_length]) + # self.parent.assertListEqual( + # list(result["end_logits"].size()), + # [self.batch_size, self.seq_length]) + # self.check_loss_output(result) + + + def create_and_check_bert_for_sequence_classification(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + pass + # config.num_labels = self.num_labels + # model = BertForSequenceClassification(config) + # model.eval() + # loss, logits = model(input_ids, token_type_ids, input_mask, sequence_labels) + # result = { + # "loss": loss, + # "logits": logits, + # } + # self.parent.assertListEqual( + # list(result["logits"].size()), + # [self.batch_size, self.num_labels]) + # self.check_loss_output(result) + + + def create_and_check_bert_for_token_classification(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + pass + # config.num_labels = self.num_labels + # model = BertForTokenClassification(config=config) + # model.eval() + # loss, logits = model(input_ids, token_type_ids, input_mask, token_labels) + # result = { + # "loss": loss, + # "logits": logits, + # } + # self.parent.assertListEqual( + # list(result["logits"].size()), + # [self.batch_size, self.seq_length, self.num_labels]) + # self.check_loss_output(result) + + + def create_and_check_bert_for_multiple_choice(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + pass + # config.num_choices = self.num_choices + # model = BertForMultipleChoice(config=config) + # model.eval() + # multiple_choice_inputs_ids = input_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous() + # multiple_choice_token_type_ids = token_type_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous() + # multiple_choice_input_mask = input_mask.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous() + # loss, logits = model(multiple_choice_inputs_ids, + # multiple_choice_token_type_ids, + # multiple_choice_input_mask, + # choice_labels) + # result = { + # "loss": loss, + # "logits": logits, + # } + # self.parent.assertListEqual( + # list(result["logits"].size()), + # [self.batch_size, self.num_choices]) + # self.check_loss_output(result) + + + def prepare_config_and_inputs_for_common(self): + config_and_inputs = self.prepare_config_and_inputs() + (config, input_ids, token_type_ids, input_mask, + sequence_labels, token_labels, choice_labels) = config_and_inputs + inputs_dict = {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask} + return config, inputs_dict + + def setUp(self): + self.model_tester = TFBertModelTest.TFBertModelTester(self) + self.config_tester = ConfigTester(self, config_class=BertConfig, hidden_size=37) + + def test_config(self): + self.config_tester.run_common_tests() + + def test_bert_model(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_bert_model(*config_and_inputs) + + def test_for_masked_lm(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_bert_for_masked_lm(*config_and_inputs) + + def test_for_multiple_choice(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_bert_for_multiple_choice(*config_and_inputs) + + def test_for_next_sequence_prediction(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_bert_for_next_sequence_prediction(*config_and_inputs) + + def test_for_pretraining(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_bert_for_pretraining(*config_and_inputs) + + def test_for_question_answering(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_bert_for_question_answering(*config_and_inputs) + + def test_for_sequence_classification(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_bert_for_sequence_classification(*config_and_inputs) + + def test_for_token_classification(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_bert_for_token_classification(*config_and_inputs) + + @pytest.mark.slow + def test_model_from_pretrained(self): + cache_dir = "/tmp/pytorch_transformers_test/" + for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: + model = TFBertModel.from_pretrained(model_name, cache_dir=cache_dir) + shutil.rmtree(cache_dir) + self.assertIsNotNone(model) + +if __name__ == "__main__": + unittest.main() From 134847db8145deb051794bd1ee24881bfc23f6fc Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 5 Sep 2019 02:53:52 +0200 Subject: [PATCH 003/219] fix test when tf is not here --- .circleci/config.yml | 2 + pytorch_transformers/__init__.py | 90 ++++++++++++------- ...ng_tf_test.py => modeling_tf_bert_test.py} | 24 +++-- .../tests/modeling_tf_common_test.py | 31 ++++--- 4 files changed, 101 insertions(+), 46 deletions(-) rename pytorch_transformers/tests/{modeling_tf_test.py => modeling_tf_bert_test.py} (94%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 48e80beaeb..e54d92ab95 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,6 +11,7 @@ jobs: - run: sudo pip install --progress-bar off . - run: sudo pip install pytest codecov pytest-cov - run: sudo pip install tensorboardX scikit-learn + - run: sudo pip install tensorflow==2.0.0-rc0 - run: python -m pytest -sv ./pytorch_transformers/tests/ --cov - run: python -m pytest -sv ./examples/ - run: codecov @@ -24,6 +25,7 @@ jobs: - checkout - run: sudo pip install --progress-bar off . - run: sudo pip install pytest codecov pytest-cov + - run: sudo pip install tensorflow==2.0.0-rc0 - run: python -m pytest -sv ./pytorch_transformers/tests/ --cov - run: codecov deploy_doc: diff --git a/pytorch_transformers/__init__.py b/pytorch_transformers/__init__.py index 24ff52e5d4..43b0cb2e07 100644 --- a/pytorch_transformers/__init__.py +++ b/pytorch_transformers/__init__.py @@ -1,4 +1,5 @@ __version__ = "1.2.0" + # Work around to update TensorFlow's absl.logging threshold which alters the # default Python logging output behavior when present. # see: https://github.com/abseil/abseil-py/issues/99 @@ -11,6 +12,10 @@ try: except: pass +import logging + +logger = logging.getLogger(__name__) # pylint: disable=invalid-name + # Tokenizer from .tokenization_utils import (PreTrainedTokenizer) from .tokenization_auto import AutoTokenizer @@ -36,38 +41,63 @@ from .configuration_roberta import RobertaConfig, ROBERTA_PRETRAINED_CONFIG_ARCH from .configuration_distilbert import DistilBertConfig, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP # Modeling -from .modeling_utils import (PreTrainedModel, prune_layer, Conv1D) -from .modeling_auto import (AutoModel, AutoModelForSequenceClassification, AutoModelForQuestionAnswering, - AutoModelWithLMHead) +try: + import torch + torch_available = True # pylint: disable=invalid-name +except ImportError: + torch_available = False # pylint: disable=invalid-name -from .modeling_bert import (BertPreTrainedModel, BertModel, BertForPreTraining, - BertForMaskedLM, BertForNextSentencePrediction, - BertForSequenceClassification, BertForMultipleChoice, - BertForTokenClassification, BertForQuestionAnswering, - load_tf_weights_in_bert, BERT_PRETRAINED_MODEL_ARCHIVE_MAP) -from .modeling_openai import (OpenAIGPTPreTrainedModel, OpenAIGPTModel, - OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel, - load_tf_weights_in_openai_gpt, OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP) -from .modeling_transfo_xl import (TransfoXLPreTrainedModel, TransfoXLModel, TransfoXLLMHeadModel, - load_tf_weights_in_transfo_xl, TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP) -from .modeling_gpt2 import (GPT2PreTrainedModel, GPT2Model, - GPT2LMHeadModel, GPT2DoubleHeadsModel, - load_tf_weights_in_gpt2, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP) -from .modeling_xlnet import (XLNetPreTrainedModel, XLNetModel, XLNetLMHeadModel, - XLNetForSequenceClassification, XLNetForQuestionAnswering, - load_tf_weights_in_xlnet, XLNET_PRETRAINED_MODEL_ARCHIVE_MAP) -from .modeling_xlm import (XLMPreTrainedModel , XLMModel, - XLMWithLMHeadModel, XLMForSequenceClassification, - XLMForQuestionAnswering, XLM_PRETRAINED_MODEL_ARCHIVE_MAP) -from .modeling_roberta import (RobertaForMaskedLM, RobertaModel, RobertaForSequenceClassification, - ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP) -from .modeling_distilbert import (DistilBertForMaskedLM, DistilBertModel, - DistilBertForSequenceClassification, DistilBertForQuestionAnswering, - DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP) +if torch_available: + logger.info("PyTorch version {} available.".format(torch.__version__)) + + from .modeling_utils import (PreTrainedModel, prune_layer, Conv1D) + from .modeling_auto import (AutoModel, AutoModelForSequenceClassification, AutoModelForQuestionAnswering, + AutoModelWithLMHead) + + from .modeling_bert import (BertPreTrainedModel, BertModel, BertForPreTraining, + BertForMaskedLM, BertForNextSentencePrediction, + BertForSequenceClassification, BertForMultipleChoice, + BertForTokenClassification, BertForQuestionAnswering, + load_tf_weights_in_bert, BERT_PRETRAINED_MODEL_ARCHIVE_MAP) + from .modeling_openai import (OpenAIGPTPreTrainedModel, OpenAIGPTModel, + OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel, + load_tf_weights_in_openai_gpt, OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP) + from .modeling_transfo_xl import (TransfoXLPreTrainedModel, TransfoXLModel, TransfoXLLMHeadModel, + load_tf_weights_in_transfo_xl, TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP) + from .modeling_gpt2 import (GPT2PreTrainedModel, GPT2Model, + GPT2LMHeadModel, GPT2DoubleHeadsModel, + load_tf_weights_in_gpt2, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP) + from .modeling_xlnet import (XLNetPreTrainedModel, XLNetModel, XLNetLMHeadModel, + XLNetForSequenceClassification, XLNetForQuestionAnswering, + load_tf_weights_in_xlnet, XLNET_PRETRAINED_MODEL_ARCHIVE_MAP) + from .modeling_xlm import (XLMPreTrainedModel , XLMModel, + XLMWithLMHeadModel, XLMForSequenceClassification, + XLMForQuestionAnswering, XLM_PRETRAINED_MODEL_ARCHIVE_MAP) + from .modeling_roberta import (RobertaForMaskedLM, RobertaModel, RobertaForSequenceClassification, + ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP) + from .modeling_distilbert import (DistilBertForMaskedLM, DistilBertModel, + DistilBertForSequenceClassification, DistilBertForQuestionAnswering, + DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP) + + # Optimization + from .optimization import (AdamW, ConstantLRSchedule, WarmupConstantSchedule, WarmupCosineSchedule, + WarmupCosineWithHardRestartsSchedule, WarmupLinearSchedule) + + +# TensorFlow +try: + import tensorflow as tf + tf_available = True # pylint: disable=invalid-name +except ImportError: + tf_available = False # pylint: disable=invalid-name + +if tf_available: + logger.info("TensorFlow version {} available.".format(tf.__version__)) + + from .modeling_tf_utils import TFPreTrainedModel + from .modeling_tf_bert import (TFBertPreTrainedModel, TFBertModel, TFBertForPreTraining, + TFBertForMaskedLM, TFBertForNextSentencePrediction, load_pt_weights_in_bert) -# Optimization -from .optimization import (AdamW, ConstantLRSchedule, WarmupConstantSchedule, WarmupCosineSchedule, - WarmupCosineWithHardRestartsSchedule, WarmupLinearSchedule) # Files and general utilities from .file_utils import (PYTORCH_TRANSFORMERS_CACHE, PYTORCH_PRETRAINED_BERT_CACHE, diff --git a/pytorch_transformers/tests/modeling_tf_test.py b/pytorch_transformers/tests/modeling_tf_bert_test.py similarity index 94% rename from pytorch_transformers/tests/modeling_tf_test.py rename to pytorch_transformers/tests/modeling_tf_bert_test.py index ee2580bd9e..9d36bdb98e 100644 --- a/pytorch_transformers/tests/modeling_tf_test.py +++ b/pytorch_transformers/tests/modeling_tf_bert_test.py @@ -19,15 +19,19 @@ from __future__ import print_function import unittest import shutil import pytest - -import tensorflow as tf - -from pytorch_transformers import (BertConfig) -from pytorch_transformers.modeling_tf_bert import TFBertModel, TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP +import sys from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester +try: + import tensorflow as tf + + from pytorch_transformers import (BertConfig) + from pytorch_transformers.modeling_tf_bert import TFBertModel, TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP +except ImportError: + pass + class TFBertModelTest(TFCommonTestCases.TFCommonModelTester): @@ -283,39 +287,48 @@ class TFBertModelTest(TFCommonTestCases.TFCommonModelTester): def test_config(self): self.config_tester.run_common_tests() + @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") def test_bert_model(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_model(*config_and_inputs) + @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") def test_for_masked_lm(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_for_masked_lm(*config_and_inputs) + @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") def test_for_multiple_choice(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_for_multiple_choice(*config_and_inputs) + @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") def test_for_next_sequence_prediction(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_for_next_sequence_prediction(*config_and_inputs) + @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") def test_for_pretraining(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_for_pretraining(*config_and_inputs) + @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") def test_for_question_answering(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_for_question_answering(*config_and_inputs) + @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") def test_for_sequence_classification(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_for_sequence_classification(*config_and_inputs) + @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") def test_for_token_classification(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_for_token_classification(*config_and_inputs) @pytest.mark.slow + @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") def test_model_from_pretrained(self): cache_dir = "/tmp/pytorch_transformers_test/" for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: @@ -325,3 +338,4 @@ class TFBertModelTest(TFCommonTestCases.TFCommonModelTester): if __name__ == "__main__": unittest.main() + diff --git a/pytorch_transformers/tests/modeling_tf_common_test.py b/pytorch_transformers/tests/modeling_tf_common_test.py index d432ab5fdf..8e3911f766 100644 --- a/pytorch_transformers/tests/modeling_tf_common_test.py +++ b/pytorch_transformers/tests/modeling_tf_common_test.py @@ -12,24 +12,25 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import copy -import os -import shutil import json +import logging import random +import shutil +import unittest import uuid -import unittest -import logging +import pytest +import sys -import tensorflow as tf - -from pytorch_transformers import TFPreTrainedModel -# from pytorch_transformers.modeling_bert import BertModel, BertConfig, BERT_PRETRAINED_MODEL_ARCHIVE_MAP +try: + import tensorflow as tf + from pytorch_transformers import TFPreTrainedModel + # from pytorch_transformers.modeling_bert import BertModel, BertConfig, BERT_PRETRAINED_MODEL_ARCHIVE_MAP +except ImportError: + pass def _config_zero_init(config): @@ -49,6 +50,7 @@ class TFCommonTestCases: test_pruning = True test_resize_embeddings = True + @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") def test_initialization(self): pass # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -62,6 +64,7 @@ class TFCommonTestCases: # msg="Parameter {} of model {} seems not properly initialized".format(name, model_class)) + @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") def test_attention_outputs(self): pass # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -102,6 +105,7 @@ class TFCommonTestCases: # self.model_tester.key_len if hasattr(self.model_tester, 'key_len') else self.model_tester.seq_length]) + @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") def test_headmasking(self): pass # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -149,6 +153,7 @@ class TFCommonTestCases: # attentions[-1][..., -1, :, :].flatten().sum().item(), 0.0) + @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") def test_head_pruning(self): pass # if not self.test_pruning: @@ -176,6 +181,7 @@ class TFCommonTestCases: # attentions[-1].shape[-3], self.model_tester.num_attention_heads - 1) + @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") def test_hidden_states_output(self): pass # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -195,6 +201,7 @@ class TFCommonTestCases: # [self.model_tester.seq_length, self.model_tester.hidden_size]) + @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") def test_resize_tokens_embeddings(self): pass # original_config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -231,6 +238,7 @@ class TFCommonTestCases: # self.assertTrue(models_equal) + @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") def test_tie_model_weights(self): pass # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -282,6 +290,7 @@ def ids_tensor(shape, vocab_size, rng=None, name=None): class TFModelUtilsTest(unittest.TestCase): + @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") def test_model_from_pretrained(self): pass # logging.basicConfig(level=logging.INFO) From aa4c8804f243f6095a23dde538096cc9869814dc Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 5 Sep 2019 03:06:09 +0200 Subject: [PATCH 004/219] skipping tf tests if tf is not installed --- .../tests/modeling_tf_bert_test.py | 18 +++++++++--------- .../tests/modeling_tf_common_test.py | 16 ++++++++-------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/pytorch_transformers/tests/modeling_tf_bert_test.py b/pytorch_transformers/tests/modeling_tf_bert_test.py index 9d36bdb98e..c7b8b02f7b 100644 --- a/pytorch_transformers/tests/modeling_tf_bert_test.py +++ b/pytorch_transformers/tests/modeling_tf_bert_test.py @@ -287,48 +287,48 @@ class TFBertModelTest(TFCommonTestCases.TFCommonModelTester): def test_config(self): self.config_tester.run_common_tests() - @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") + @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_bert_model(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_model(*config_and_inputs) - @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") + @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_for_masked_lm(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_for_masked_lm(*config_and_inputs) - @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") + @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_for_multiple_choice(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_for_multiple_choice(*config_and_inputs) - @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") + @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_for_next_sequence_prediction(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_for_next_sequence_prediction(*config_and_inputs) - @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") + @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_for_pretraining(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_for_pretraining(*config_and_inputs) - @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") + @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_for_question_answering(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_for_question_answering(*config_and_inputs) - @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") + @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_for_sequence_classification(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_for_sequence_classification(*config_and_inputs) - @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") + @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_for_token_classification(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_for_token_classification(*config_and_inputs) @pytest.mark.slow - @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") + @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_model_from_pretrained(self): cache_dir = "/tmp/pytorch_transformers_test/" for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: diff --git a/pytorch_transformers/tests/modeling_tf_common_test.py b/pytorch_transformers/tests/modeling_tf_common_test.py index 8e3911f766..1ddfd83be1 100644 --- a/pytorch_transformers/tests/modeling_tf_common_test.py +++ b/pytorch_transformers/tests/modeling_tf_common_test.py @@ -50,7 +50,7 @@ class TFCommonTestCases: test_pruning = True test_resize_embeddings = True - @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") + @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_initialization(self): pass # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -64,7 +64,7 @@ class TFCommonTestCases: # msg="Parameter {} of model {} seems not properly initialized".format(name, model_class)) - @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") + @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_attention_outputs(self): pass # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -105,7 +105,7 @@ class TFCommonTestCases: # self.model_tester.key_len if hasattr(self.model_tester, 'key_len') else self.model_tester.seq_length]) - @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") + @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_headmasking(self): pass # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -153,7 +153,7 @@ class TFCommonTestCases: # attentions[-1][..., -1, :, :].flatten().sum().item(), 0.0) - @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") + @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_head_pruning(self): pass # if not self.test_pruning: @@ -181,7 +181,7 @@ class TFCommonTestCases: # attentions[-1].shape[-3], self.model_tester.num_attention_heads - 1) - @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") + @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_hidden_states_output(self): pass # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -201,7 +201,7 @@ class TFCommonTestCases: # [self.model_tester.seq_length, self.model_tester.hidden_size]) - @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") + @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_resize_tokens_embeddings(self): pass # original_config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -238,7 +238,7 @@ class TFCommonTestCases: # self.assertTrue(models_equal) - @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") + @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_tie_model_weights(self): pass # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -290,7 +290,7 @@ def ids_tensor(shape, vocab_size, rng=None, name=None): class TFModelUtilsTest(unittest.TestCase): - @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") + @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_model_from_pretrained(self): pass # logging.basicConfig(level=logging.INFO) From 5951d86024e2366a83019f5d676f25757540ed0a Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 5 Sep 2019 03:10:11 +0200 Subject: [PATCH 005/219] add conversion script, rename conversion scripts --- ...bert_original_tf_checkpoint_to_pytorch.py} | 0 ...bert_pytorch_checkpoint_to_original_tf.py} | 0 .../convert_bert_pytorch_checkpoint_to_tf.py | 65 +++++++++++++++++++ 3 files changed, 65 insertions(+) rename pytorch_transformers/{convert_tf_checkpoint_to_pytorch.py => convert_bert_original_tf_checkpoint_to_pytorch.py} (100%) rename pytorch_transformers/{convert_pytorch_checkpoint_to_tf.py => convert_bert_pytorch_checkpoint_to_original_tf.py} (100%) create mode 100644 pytorch_transformers/convert_bert_pytorch_checkpoint_to_tf.py diff --git a/pytorch_transformers/convert_tf_checkpoint_to_pytorch.py b/pytorch_transformers/convert_bert_original_tf_checkpoint_to_pytorch.py similarity index 100% rename from pytorch_transformers/convert_tf_checkpoint_to_pytorch.py rename to pytorch_transformers/convert_bert_original_tf_checkpoint_to_pytorch.py diff --git a/pytorch_transformers/convert_pytorch_checkpoint_to_tf.py b/pytorch_transformers/convert_bert_pytorch_checkpoint_to_original_tf.py similarity index 100% rename from pytorch_transformers/convert_pytorch_checkpoint_to_tf.py rename to pytorch_transformers/convert_bert_pytorch_checkpoint_to_original_tf.py diff --git a/pytorch_transformers/convert_bert_pytorch_checkpoint_to_tf.py b/pytorch_transformers/convert_bert_pytorch_checkpoint_to_tf.py new file mode 100644 index 0000000000..7031f3b523 --- /dev/null +++ b/pytorch_transformers/convert_bert_pytorch_checkpoint_to_tf.py @@ -0,0 +1,65 @@ +# coding=utf-8 +# Copyright 2018 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Convert BERT checkpoint.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import tensorflow as tf + +from pytorch_transformers import BertConfig, TFBertForPreTraining, load_pt_weights_in_bert + +import logging +logging.basicConfig(level=logging.INFO) + +def convert_bert_checkpoint_to_tf(pytorch_checkpoint_path, bert_config_file, tf_dump_path): + # Initialise TF model + config = BertConfig.from_json_file(bert_config_file) + print("Building TensorFlow model from configuration: {}".format(str(config))) + model = TFBertForPreTraining(config) + + # Load weights from tf checkpoint + model = load_pt_weights_in_bert(model, config, pytorch_checkpoint_path) + + # Save pytorch-model + print("Save TensorFlow model to {}".format(tf_dump_path)) + model.save_weights(tf_dump_path) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + ## Required parameters + parser.add_argument("--pytorch_checkpoint_path", + default = None, + type = str, + required = True, + help = "Path to the PyTorch checkpoint path.") + parser.add_argument("--bert_config_file", + default = None, + type = str, + required = True, + help = "The config json file corresponding to the pre-trained BERT model. \n" + "This specifies the model architecture.") + parser.add_argument("--tf_dump_path", + default = None, + type = str, + required = True, + help = "Path to the output Tensorflow dump file.") + args = parser.parse_args() + convert_bert_checkpoint_to_tf(args.pytorch_checkpoint_path, + args.bert_config_file, + args.tf_dump_path) From 33dd59e9714d44dfdb29d184eb6ad0b687a39e5c Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 5 Sep 2019 03:13:26 +0200 Subject: [PATCH 006/219] update conversion script names --- pytorch_transformers/__main__.py | 15 ++++++++------- ...ert_gpt2_original_tf_checkpoint_to_pytorch.py} | 0 ...t_openai_original_tf_checkpoint_to_pytorch.py} | 0 ...rta_original_pytorch_checkpoint_to_pytorch.py} | 0 ...ansfo_xl_original_tf_checkpoint_to_pytorch.py} | 0 ...xlm_original_pytorch_checkpoint_to_pytorch.py} | 0 ...rt_xlnet_original_tf_checkpoint_to_pytorch.py} | 0 7 files changed, 8 insertions(+), 7 deletions(-) rename pytorch_transformers/{convert_gpt2_checkpoint_to_pytorch.py => convert_gpt2_original_tf_checkpoint_to_pytorch.py} (100%) rename pytorch_transformers/{convert_openai_checkpoint_to_pytorch.py => convert_openai_original_tf_checkpoint_to_pytorch.py} (100%) rename pytorch_transformers/{convert_roberta_checkpoint_to_pytorch.py => convert_roberta_original_pytorch_checkpoint_to_pytorch.py} (100%) rename pytorch_transformers/{convert_transfo_xl_checkpoint_to_pytorch.py => convert_transfo_xl_original_tf_checkpoint_to_pytorch.py} (100%) rename pytorch_transformers/{convert_xlm_checkpoint_to_pytorch.py => convert_xlm_original_pytorch_checkpoint_to_pytorch.py} (100%) rename pytorch_transformers/{convert_xlnet_checkpoint_to_pytorch.py => convert_xlnet_original_tf_checkpoint_to_pytorch.py} (100%) diff --git a/pytorch_transformers/__main__.py b/pytorch_transformers/__main__.py index b047fa7447..ca4936fedf 100644 --- a/pytorch_transformers/__main__.py +++ b/pytorch_transformers/__main__.py @@ -3,7 +3,8 @@ def main(): import sys if (len(sys.argv) < 4 or len(sys.argv) > 6) or sys.argv[1] not in ["bert", "gpt", "transfo_xl", "gpt2", "xlnet", "xlm"]: print( - "Should be used as one of: \n" + "This command line utility let you convert original (author released) model checkpoint to pytorch.\n" + "It should be used as one of: \n" ">> pytorch_transformers bert TF_CHECKPOINT TF_CONFIG PYTORCH_DUMP_OUTPUT, \n" ">> pytorch_transformers gpt OPENAI_GPT_CHECKPOINT_FOLDER_PATH PYTORCH_DUMP_OUTPUT [OPENAI_GPT_CONFIG], \n" ">> pytorch_transformers transfo_xl TF_CHECKPOINT_OR_DATASET PYTORCH_DUMP_OUTPUT [TF_CONFIG] or \n" @@ -13,7 +14,7 @@ def main(): else: if sys.argv[1] == "bert": try: - from .convert_tf_checkpoint_to_pytorch import convert_tf_checkpoint_to_pytorch + from .convert_bert_original_tf_checkpoint_to_pytorch import convert_tf_checkpoint_to_pytorch except ImportError: print("pytorch_transformers can only be used from the commandline to convert TensorFlow models in PyTorch, " "In that case, it requires TensorFlow to be installed. Please see " @@ -29,7 +30,7 @@ def main(): TF_CHECKPOINT = sys.argv.pop() convert_tf_checkpoint_to_pytorch(TF_CHECKPOINT, TF_CONFIG, PYTORCH_DUMP_OUTPUT) elif sys.argv[1] == "gpt": - from .convert_openai_checkpoint_to_pytorch import convert_openai_checkpoint_to_pytorch + from .convert_openai_original_tf_checkpoint_to_pytorch import convert_openai_checkpoint_to_pytorch if len(sys.argv) < 4 or len(sys.argv) > 5: # pylint: disable=line-too-long print("Should be used as `pytorch_transformers gpt OPENAI_GPT_CHECKPOINT_FOLDER_PATH PYTORCH_DUMP_OUTPUT [OPENAI_GPT_CONFIG]`") @@ -45,7 +46,7 @@ def main(): PYTORCH_DUMP_OUTPUT) elif sys.argv[1] == "transfo_xl": try: - from .convert_transfo_xl_checkpoint_to_pytorch import convert_transfo_xl_checkpoint_to_pytorch + from .convert_transfo_xl_original_tf_checkpoint_to_pytorch import convert_transfo_xl_checkpoint_to_pytorch except ImportError: print("pytorch_transformers can only be used from the commandline to convert TensorFlow models in PyTorch, " "In that case, it requires TensorFlow to be installed. Please see " @@ -69,7 +70,7 @@ def main(): convert_transfo_xl_checkpoint_to_pytorch(TF_CHECKPOINT, TF_CONFIG, PYTORCH_DUMP_OUTPUT, TF_DATASET_FILE) elif sys.argv[1] == "gpt2": try: - from .convert_gpt2_checkpoint_to_pytorch import convert_gpt2_checkpoint_to_pytorch + from .convert_gpt2_original_tf_checkpoint_to_pytorch import convert_gpt2_checkpoint_to_pytorch except ImportError: print("pytorch_transformers can only be used from the commandline to convert TensorFlow models in PyTorch, " "In that case, it requires TensorFlow to be installed. Please see " @@ -89,7 +90,7 @@ def main(): convert_gpt2_checkpoint_to_pytorch(TF_CHECKPOINT, TF_CONFIG, PYTORCH_DUMP_OUTPUT) elif sys.argv[1] == "xlnet": try: - from .convert_xlnet_checkpoint_to_pytorch import convert_xlnet_checkpoint_to_pytorch + from .convert_xlnet_original_tf_checkpoint_to_pytorch import convert_xlnet_checkpoint_to_pytorch except ImportError: print("pytorch_transformers can only be used from the commandline to convert TensorFlow models in PyTorch, " "In that case, it requires TensorFlow to be installed. Please see " @@ -113,7 +114,7 @@ def main(): PYTORCH_DUMP_OUTPUT, FINETUNING_TASK) elif sys.argv[1] == "xlm": - from .convert_xlm_checkpoint_to_pytorch import convert_xlm_checkpoint_to_pytorch + from .convert_xlm_original_pytorch_checkpoint_to_pytorch import convert_xlm_checkpoint_to_pytorch if len(sys.argv) != 4: # pylint: disable=line-too-long diff --git a/pytorch_transformers/convert_gpt2_checkpoint_to_pytorch.py b/pytorch_transformers/convert_gpt2_original_tf_checkpoint_to_pytorch.py similarity index 100% rename from pytorch_transformers/convert_gpt2_checkpoint_to_pytorch.py rename to pytorch_transformers/convert_gpt2_original_tf_checkpoint_to_pytorch.py diff --git a/pytorch_transformers/convert_openai_checkpoint_to_pytorch.py b/pytorch_transformers/convert_openai_original_tf_checkpoint_to_pytorch.py similarity index 100% rename from pytorch_transformers/convert_openai_checkpoint_to_pytorch.py rename to pytorch_transformers/convert_openai_original_tf_checkpoint_to_pytorch.py diff --git a/pytorch_transformers/convert_roberta_checkpoint_to_pytorch.py b/pytorch_transformers/convert_roberta_original_pytorch_checkpoint_to_pytorch.py similarity index 100% rename from pytorch_transformers/convert_roberta_checkpoint_to_pytorch.py rename to pytorch_transformers/convert_roberta_original_pytorch_checkpoint_to_pytorch.py diff --git a/pytorch_transformers/convert_transfo_xl_checkpoint_to_pytorch.py b/pytorch_transformers/convert_transfo_xl_original_tf_checkpoint_to_pytorch.py similarity index 100% rename from pytorch_transformers/convert_transfo_xl_checkpoint_to_pytorch.py rename to pytorch_transformers/convert_transfo_xl_original_tf_checkpoint_to_pytorch.py diff --git a/pytorch_transformers/convert_xlm_checkpoint_to_pytorch.py b/pytorch_transformers/convert_xlm_original_pytorch_checkpoint_to_pytorch.py similarity index 100% rename from pytorch_transformers/convert_xlm_checkpoint_to_pytorch.py rename to pytorch_transformers/convert_xlm_original_pytorch_checkpoint_to_pytorch.py diff --git a/pytorch_transformers/convert_xlnet_checkpoint_to_pytorch.py b/pytorch_transformers/convert_xlnet_original_tf_checkpoint_to_pytorch.py similarity index 100% rename from pytorch_transformers/convert_xlnet_checkpoint_to_pytorch.py rename to pytorch_transformers/convert_xlnet_original_tf_checkpoint_to_pytorch.py From 7775a3d2edfc6d42ac17d7bf9dbc495f15efab5a Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 5 Sep 2019 10:23:04 +0200 Subject: [PATCH 007/219] update dependencies and circle-ci --- .circleci/config.yml | 40 +++++++++++++++++++++++++++++++++++----- requirements.txt | 2 -- setup.py | 3 +-- 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e54d92ab95..dfb7de5634 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,6 +1,6 @@ version: 2 jobs: - build_py3: + build_py3_torch: working_directory: ~/pytorch-transformers docker: - image: circleci/python:3.5 @@ -8,14 +8,29 @@ jobs: parallelism: 1 steps: - checkout + - run: sudo pip install torch - run: sudo pip install --progress-bar off . - run: sudo pip install pytest codecov pytest-cov - run: sudo pip install tensorboardX scikit-learn - - run: sudo pip install tensorflow==2.0.0-rc0 - run: python -m pytest -sv ./pytorch_transformers/tests/ --cov - run: python -m pytest -sv ./examples/ - run: codecov - build_py2: + build_py3_tf: + working_directory: ~/pytorch-transformers + docker: + - image: circleci/python:3.5 + resource_class: xlarge + parallelism: 1 + steps: + - checkout + - run: sudo pip install tensorflow==2.0.0-rc0 + - run: sudo pip install --progress-bar off . + - run: sudo pip install pytest codecov pytest-cov + - run: sudo pip install tensorboardX scikit-learn + - run: python -m pytest -sv ./pytorch_transformers/tests/ --cov + - run: python -m pytest -sv ./examples/ + - run: codecov + build_py2_torch: working_directory: ~/pytorch-transformers resource_class: large parallelism: 1 @@ -23,9 +38,22 @@ jobs: - image: circleci/python:2.7 steps: - checkout + - run: sudo pip install torch - run: sudo pip install --progress-bar off . - run: sudo pip install pytest codecov pytest-cov + - run: python -m pytest -sv ./pytorch_transformers/tests/ --cov + - run: codecov + build_py2_tf: + working_directory: ~/pytorch-transformers + resource_class: large + parallelism: 1 + docker: + - image: circleci/python:2.7 + steps: + - checkout - run: sudo pip install tensorflow==2.0.0-rc0 + - run: sudo pip install --progress-bar off . + - run: sudo pip install pytest codecov pytest-cov - run: python -m pytest -sv ./pytorch_transformers/tests/ --cov - run: codecov deploy_doc: @@ -49,6 +77,8 @@ workflows: version: 2 build_and_test: jobs: - - build_py3 - - build_py2 + - build_py3_torch + - build_py3_tf + - build_py2_torch + - build_py2_tf - deploy_doc: *workflow_filters \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 01dca79d23..9c43abc6d7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,3 @@ -# PyTorch -torch>=1.0.0 # progress bars in model download and training scripts tqdm # Accessing files from S3 directly. diff --git a/setup.py b/setup.py index c31bf6eaeb..903f1d8cac 100644 --- a/setup.py +++ b/setup.py @@ -49,8 +49,7 @@ setup( url="https://github.com/huggingface/pytorch-transformers", packages=find_packages(exclude=["*.tests", "*.tests.*", "tests.*", "tests"]), - install_requires=['torch>=1.0.0', - 'numpy', + install_requires=['numpy', 'boto3', 'requests', 'tqdm', From 7c0baf9521eb32cbcebd7c8a8ec4070abf1a0426 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 5 Sep 2019 11:18:55 +0200 Subject: [PATCH 008/219] test suite independent of framework --- .circleci/config.yml | 9 +- pytorch_transformers/__init__.py | 21 +- ...py => convert_pytorch_checkpoint_to_tf.py} | 37 +- pytorch_transformers/modeling_tf_bert.py | 358 ++++++++++++++++-- .../tests/modeling_auto_test.py | 19 +- .../tests/modeling_bert_test.py | 17 +- .../tests/modeling_common_test.py | 12 +- .../tests/modeling_distilbert_test.py | 12 +- .../tests/modeling_gpt2_test.py | 10 +- .../tests/modeling_openai_test.py | 10 +- .../tests/modeling_roberta_test.py | 13 +- .../tests/modeling_tf_bert_test.py | 241 ++++++------ .../tests/modeling_tf_common_test.py | 9 +- .../tests/modeling_transfo_xl_test.py | 12 +- .../tests/modeling_xlm_test.py | 16 +- .../tests/modeling_xlnet_test.py | 13 +- .../tests/optimization_test.py | 16 +- .../tests/tokenization_auto_test.py | 7 +- .../tests/tokenization_transfo_xl_test.py | 12 +- .../tokenization_transfo_xl.py | 14 +- 20 files changed, 596 insertions(+), 262 deletions(-) rename pytorch_transformers/{convert_bert_pytorch_checkpoint_to_tf.py => convert_pytorch_checkpoint_to_tf.py} (61%) diff --git a/.circleci/config.yml b/.circleci/config.yml index dfb7de5634..2bf082c850 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,7 +10,7 @@ jobs: - checkout - run: sudo pip install torch - run: sudo pip install --progress-bar off . - - run: sudo pip install pytest codecov pytest-cov + - run: sudo pip install pytest==5.0.1 codecov pytest-cov - run: sudo pip install tensorboardX scikit-learn - run: python -m pytest -sv ./pytorch_transformers/tests/ --cov - run: python -m pytest -sv ./examples/ @@ -25,10 +25,9 @@ jobs: - checkout - run: sudo pip install tensorflow==2.0.0-rc0 - run: sudo pip install --progress-bar off . - - run: sudo pip install pytest codecov pytest-cov + - run: sudo pip install pytest==5.0.1 codecov pytest-cov - run: sudo pip install tensorboardX scikit-learn - run: python -m pytest -sv ./pytorch_transformers/tests/ --cov - - run: python -m pytest -sv ./examples/ - run: codecov build_py2_torch: working_directory: ~/pytorch-transformers @@ -40,7 +39,7 @@ jobs: - checkout - run: sudo pip install torch - run: sudo pip install --progress-bar off . - - run: sudo pip install pytest codecov pytest-cov + - run: sudo pip install pytest==5.0.1 codecov pytest-cov - run: python -m pytest -sv ./pytorch_transformers/tests/ --cov - run: codecov build_py2_tf: @@ -53,7 +52,7 @@ jobs: - checkout - run: sudo pip install tensorflow==2.0.0-rc0 - run: sudo pip install --progress-bar off . - - run: sudo pip install pytest codecov pytest-cov + - run: sudo pip install pytest==5.0.1 codecov pytest-cov - run: python -m pytest -sv ./pytorch_transformers/tests/ --cov - run: codecov deploy_doc: diff --git a/pytorch_transformers/__init__.py b/pytorch_transformers/__init__.py index 43b0cb2e07..a41c838015 100644 --- a/pytorch_transformers/__init__.py +++ b/pytorch_transformers/__init__.py @@ -43,11 +43,11 @@ from .configuration_distilbert import DistilBertConfig, DISTILBERT_PRETRAINED_CO # Modeling try: import torch - torch_available = True # pylint: disable=invalid-name + _torch_available = True # pylint: disable=invalid-name except ImportError: - torch_available = False # pylint: disable=invalid-name + _torch_available = False # pylint: disable=invalid-name -if torch_available: +if _torch_available: logger.info("PyTorch version {} available.".format(torch.__version__)) from .modeling_utils import (PreTrainedModel, prune_layer, Conv1D) @@ -87,19 +87,26 @@ if torch_available: # TensorFlow try: import tensorflow as tf - tf_available = True # pylint: disable=invalid-name + assert int(tf.__version__[0]) >= 2 + _tf_available = True # pylint: disable=invalid-name except ImportError: - tf_available = False # pylint: disable=invalid-name + _tf_available = False # pylint: disable=invalid-name -if tf_available: +if _tf_available: logger.info("TensorFlow version {} available.".format(tf.__version__)) from .modeling_tf_utils import TFPreTrainedModel from .modeling_tf_bert import (TFBertPreTrainedModel, TFBertModel, TFBertForPreTraining, - TFBertForMaskedLM, TFBertForNextSentencePrediction, load_pt_weights_in_bert) + TFBertForMaskedLM, TFBertForNextSentencePrediction, load_bert_pt_weights_in_tf) # Files and general utilities from .file_utils import (PYTORCH_TRANSFORMERS_CACHE, PYTORCH_PRETRAINED_BERT_CACHE, cached_path, add_start_docstrings, add_end_docstrings, WEIGHTS_NAME, TF_WEIGHTS_NAME, CONFIG_NAME) + +def is_torch_available(): + return _torch_available + +def is_tf_available(): + return _tf_available diff --git a/pytorch_transformers/convert_bert_pytorch_checkpoint_to_tf.py b/pytorch_transformers/convert_pytorch_checkpoint_to_tf.py similarity index 61% rename from pytorch_transformers/convert_bert_pytorch_checkpoint_to_tf.py rename to pytorch_transformers/convert_pytorch_checkpoint_to_tf.py index 7031f3b523..e682f6c0d3 100644 --- a/pytorch_transformers/convert_bert_pytorch_checkpoint_to_tf.py +++ b/pytorch_transformers/convert_pytorch_checkpoint_to_tf.py @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Convert BERT checkpoint.""" +""" Convert pytorch checkpoints to TensorFlow """ from __future__ import absolute_import from __future__ import division @@ -21,19 +21,22 @@ from __future__ import print_function import argparse import tensorflow as tf -from pytorch_transformers import BertConfig, TFBertForPreTraining, load_pt_weights_in_bert +from pytorch_transformers import BertConfig, TFBertForPreTraining, load_bert_pt_weights_in_tf import logging logging.basicConfig(level=logging.INFO) -def convert_bert_checkpoint_to_tf(pytorch_checkpoint_path, bert_config_file, tf_dump_path): - # Initialise TF model - config = BertConfig.from_json_file(bert_config_file) - print("Building TensorFlow model from configuration: {}".format(str(config))) - model = TFBertForPreTraining(config) +def convert_pt_checkpoint_to_tf(model_type, pytorch_checkpoint_path, config_file, tf_dump_path): + if model_type == 'bert': + # Initialise TF model + config = BertConfig.from_json_file(config_file) + print("Building TensorFlow model from configuration: {}".format(str(config))) + model = TFBertForPreTraining(config) - # Load weights from tf checkpoint - model = load_pt_weights_in_bert(model, config, pytorch_checkpoint_path) + # Load weights from tf checkpoint + model = load_bert_pt_weights_in_tf(model, config, pytorch_checkpoint_path) + else: + raise ValueError("Unrecognized model type, should be one of ['bert'].") # Save pytorch-model print("Save TensorFlow model to {}".format(tf_dump_path)) @@ -43,16 +46,21 @@ def convert_bert_checkpoint_to_tf(pytorch_checkpoint_path, bert_config_file, tf_ if __name__ == "__main__": parser = argparse.ArgumentParser() ## Required parameters + parser.add_argument("--model_type", + default = None, + type = str, + required = True, + help = "Model type selcted in the list of.") parser.add_argument("--pytorch_checkpoint_path", default = None, type = str, required = True, help = "Path to the PyTorch checkpoint path.") - parser.add_argument("--bert_config_file", + parser.add_argument("--config_file", default = None, type = str, required = True, - help = "The config json file corresponding to the pre-trained BERT model. \n" + help = "The config json file corresponding to the pre-trained model. \n" "This specifies the model architecture.") parser.add_argument("--tf_dump_path", default = None, @@ -60,6 +68,7 @@ if __name__ == "__main__": required = True, help = "Path to the output Tensorflow dump file.") args = parser.parse_args() - convert_bert_checkpoint_to_tf(args.pytorch_checkpoint_path, - args.bert_config_file, - args.tf_dump_path) + convert_pt_checkpoint_to_tf(args.model_type.lower(), + args.pytorch_checkpoint_path, + args.config_file, + args.tf_dump_path) diff --git a/pytorch_transformers/modeling_tf_bert.py b/pytorch_transformers/modeling_tf_bert.py index dda713fe7d..bf33cb461d 100644 --- a/pytorch_transformers/modeling_tf_bert.py +++ b/pytorch_transformers/modeling_tf_bert.py @@ -51,7 +51,7 @@ TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP = { } -def load_pt_weights_in_bert(tf_model, config, pytorch_checkpoint_path): +def load_bert_pt_weights_in_tf(tf_model, config, pytorch_checkpoint_path): """ Load pytorch checkpoints in a TF 2.0 model and save it using HDF5 format We use HDF5 to easily do transfer learning (see https://github.com/tensorflow/tensorflow/blob/ee16fcac960ae660e0e4496658a366e2f745e1f0/tensorflow/python/keras/engine/network.py#L1352-L1357). @@ -150,6 +150,7 @@ class TFBertEmbeddings(tf.keras.layers.Layer): self.LayerNorm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name='LayerNorm') self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) + @tf.function def call(self, inputs, training=False): input_ids, position_ids, token_type_ids = inputs @@ -194,6 +195,7 @@ class TFBertSelfAttention(tf.keras.layers.Layer): x = tf.reshape(x, (batch_size, -1, self.num_attention_heads, self.attention_head_size)) return tf.transpose(x, perm=[0, 2, 1, 3]) + @tf.function def call(self, inputs, training=False): hidden_states, attention_mask, head_mask = inputs @@ -242,6 +244,7 @@ class TFBertSelfOutput(tf.keras.layers.Layer): self.LayerNorm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name='LayerNorm') self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) + @tf.function def call(self, inputs, training=False): hidden_states, input_tensor = inputs @@ -261,6 +264,7 @@ class TFBertAttention(tf.keras.layers.Layer): def prune_heads(self, heads): raise NotImplementedError + @tf.function def call(self, inputs, training=False): input_tensor, attention_mask, head_mask = inputs @@ -279,6 +283,7 @@ class TFBertIntermediate(tf.keras.layers.Layer): else: self.intermediate_act_fn = config.hidden_act + @tf.function def call(self, hidden_states): hidden_states = self.dense(hidden_states) hidden_states = self.intermediate_act_fn(hidden_states) @@ -292,6 +297,7 @@ class TFBertOutput(tf.keras.layers.Layer): self.LayerNorm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name='LayerNorm') self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) + @tf.function def call(self, inputs, training=False): hidden_states, input_tensor = inputs @@ -309,6 +315,7 @@ class TFBertLayer(tf.keras.layers.Layer): self.intermediate = TFBertIntermediate(config, name='intermediate') self.bert_output = TFBertOutput(config, name='output') + @tf.function def call(self, inputs, training=False): hidden_states, attention_mask, head_mask = inputs @@ -327,6 +334,7 @@ class TFBertEncoder(tf.keras.layers.Layer): self.output_hidden_states = config.output_hidden_states self.layer = [TFBertLayer(config, name='layer_{}'.format(i)) for i in range(config.num_hidden_layers)] + @tf.function def call(self, inputs, training=False): hidden_states, attention_mask, head_mask = inputs @@ -359,6 +367,7 @@ class TFBertPooler(tf.keras.layers.Layer): super(TFBertPooler, self).__init__(**kwargs) self.dense = tf.keras.layers.Dense(config.hidden_size, activation='tanh', name='dense') + @tf.function def call(self, hidden_states): # We "pool" the model by simply taking the hidden state corresponding # to the first token. @@ -377,6 +386,7 @@ class TFBertPredictionHeadTransform(tf.keras.layers.Layer): self.transform_act_fn = config.hidden_act self.LayerNorm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name='LayerNorm') + @tf.function def call(self, hidden_states): hidden_states = self.dense(hidden_states) hidden_states = self.transform_act_fn(hidden_states) @@ -400,6 +410,7 @@ class TFBertLMPredictionHead(tf.keras.layers.Layer): trainable=True, name='bias') + @tf.function def call(self, hidden_states): hidden_states = self.transform(hidden_states) hidden_states = self.decoder(hidden_states) + self.bias @@ -411,6 +422,7 @@ class TFBertMLMHead(tf.keras.layers.Layer): super(TFBertMLMHead, self).__init__(**kwargs) self.predictions = TFBertLMPredictionHead(config, name='predictions') + @tf.function def call(self, sequence_output): prediction_scores = self.predictions(sequence_output) return prediction_scores @@ -421,6 +433,7 @@ class TFBertNSPHead(tf.keras.layers.Layer): super(TFBertNSPHead, self).__init__(**kwargs) self.seq_relationship = tf.keras.layers.Dense(2, name='seq_relationship') + @tf.function def call(self, pooled_output): seq_relationship_score = self.seq_relationship(pooled_output) return seq_relationship_score @@ -447,6 +460,7 @@ class TFBertMainLayer(tf.keras.layers.Layer): """ raise NotImplementedError + @tf.function def call(self, inputs, training=False): if not isinstance(inputs, (dict, tuple, list)): input_ids = inputs @@ -459,12 +473,12 @@ class TFBertMainLayer(tf.keras.layers.Layer): head_mask = inputs[4] if len(inputs) > 4 else None assert len(inputs) <= 5, "Too many inputs." else: - input_ids = inputs.pop('input_ids') - attention_mask = inputs.pop('attention_mask', None) - token_type_ids = inputs.pop('token_type_ids', None) - position_ids = inputs.pop('position_ids', None) - head_mask = inputs.pop('head_mask', None) - assert len(inputs) == 0, "Unexpected inputs detected: {}. Check inputs dict key names.".format(list(inputs.keys())) + input_ids = inputs.get('input_ids') + attention_mask = inputs.get('attention_mask', None) + token_type_ids = inputs.get('token_type_ids', None) + position_ids = inputs.get('position_ids', None) + head_mask = inputs.get('head_mask', None) + assert len(inputs) <= 5, "Too many inputs." if attention_mask is None: attention_mask = tf.fill(tf.shape(input_ids), 1) @@ -507,23 +521,16 @@ class TFBertMainLayer(tf.keras.layers.Layer): outputs = (sequence_output, pooled_output,) + encoder_outputs[1:] # add hidden_states and attentions if they are here return outputs # sequence_output, pooled_output, (hidden_states), (attentions) + class TFBertPreTrainedModel(TFPreTrainedModel): """ An abstract class to handle weights initialization and a simple interface for dowloading and loading pretrained models. """ config_class = BertConfig pretrained_model_archive_map = TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP - load_pt_weights = load_pt_weights_in_bert + load_pt_weights = load_bert_pt_weights_in_tf base_model_prefix = "bert" - def __init__(self, *inputs, **kwargs): - super(TFBertPreTrainedModel, self).__init__(*inputs, **kwargs) - - def init_weights(self, module): - """ Initialize the weights. - """ - raise NotImplementedError - BERT_START_DOCSTRING = r""" The BERT model was proposed in `BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ @@ -635,6 +642,7 @@ class TFBertModel(TFBertPreTrainedModel): super(TFBertModel, self).__init__(config) self.bert = TFBertMainLayer(config, name='bert') + @tf.function def call(self, inputs, training=False): outputs = self.bert(inputs, training=training) return outputs @@ -687,7 +695,6 @@ class TFBertForPreTraining(TFBertPreTrainedModel): self.cls_mlm = TFBertMLMHead(config, name='cls_mlm') self.cls_nsp = TFBertNSPHead(config, name='cls_nsp') - # self.apply(self.init_weights) # TODO check added weights initialization self.tie_weights() def tie_weights(self): @@ -695,6 +702,7 @@ class TFBertForPreTraining(TFBertPreTrainedModel): """ pass # TODO add weights tying + @tf.function def call(self, inputs, training=False): outputs = self.bert(inputs, training=training) @@ -704,14 +712,6 @@ class TFBertForPreTraining(TFBertPreTrainedModel): outputs = (prediction_scores, seq_relationship_score,) + outputs[2:] # add hidden states and attention if they are here - # if masked_lm_labels is not None and next_sentence_label is not None: - # loss_fct = CrossEntropyLoss(ignore_index=-1) - # masked_lm_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), masked_lm_labels.view(-1)) - # next_sentence_loss = loss_fct(seq_relationship_score.view(-1, 2), next_sentence_label.view(-1)) - # total_loss = masked_lm_loss + next_sentence_loss - # outputs = (total_loss,) + outputs - # TODO add example with losses using model.compile and a dictionary of losses (give names to the output layers) - return outputs # prediction_scores, seq_relationship_score, (hidden_states), (attentions) @@ -753,7 +753,6 @@ class TFBertForMaskedLM(TFBertPreTrainedModel): self.bert = TFBertMainLayer(config, name='bert') self.cls_mlm = TFBertMLMHead(config, name='cls_mlm') - # self.apply(self.init_weights) self.tie_weights() def tie_weights(self): @@ -761,6 +760,7 @@ class TFBertForMaskedLM(TFBertPreTrainedModel): """ pass # TODO add weights tying + @tf.function def call(self, inputs, training=False): outputs = self.bert(inputs, training=training) @@ -768,11 +768,6 @@ class TFBertForMaskedLM(TFBertPreTrainedModel): prediction_scores = self.cls_mlm(sequence_output) outputs = (prediction_scores,) + outputs[2:] # Add hidden states and attention if they are here - # if masked_lm_labels is not None: - # loss_fct = CrossEntropyLoss(ignore_index=-1) - # masked_lm_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), masked_lm_labels.view(-1)) - # outputs = (masked_lm_loss,) + outputs - # TODO example with losses return outputs # prediction_scores, (hidden_states), (attentions) @@ -815,8 +810,7 @@ class TFBertForNextSentencePrediction(TFBertPreTrainedModel): self.bert = TFBertMainLayer(config, name='bert') self.cls_nsp = TFBertNSPHead(config, name='cls_nsp') - # self.apply(self.init_weights) - + @tf.function def call(self, inputs, training=False): outputs = self.bert(inputs, training=training) @@ -824,9 +818,299 @@ class TFBertForNextSentencePrediction(TFBertPreTrainedModel): seq_relationship_score = self.cls_nsp(pooled_output) outputs = (seq_relationship_score,) + outputs[2:] # add hidden states and attention if they are here - # if next_sentence_label is not None: - # loss_fct = CrossEntropyLoss(ignore_index=-1) - # next_sentence_loss = loss_fct(seq_relationship_score.view(-1, 2), next_sentence_label.view(-1)) - # outputs = (next_sentence_loss,) + outputs return outputs # seq_relationship_score, (hidden_states), (attentions) + + +@add_start_docstrings("""Bert Model transformer with a sequence classification/regression head on top (a linear layer on top of + the pooled output) e.g. for GLUE tasks. """, + BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) +class TFBertForSequenceClassification(TFBertPreTrainedModel): + r""" + **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for computing the sequence classification/regression loss. + Indices should be in ``[0, ..., config.num_labels - 1]``. + If ``config.num_labels == 1`` a regression loss is computed (Mean-Square loss), + If ``config.num_labels > 1`` a classification loss is computed (Cross-Entropy). + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Classification (or regression if config.num_labels==1) loss. + **logits**: ``torch.FloatTensor`` of shape ``(batch_size, config.num_labels)`` + Classification (or regression if config.num_labels==1) scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + model = TFBertForSequenceClassification.from_pretrained('bert-base-uncased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + labels = torch.tensor([1]).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, labels=labels) + loss, logits = outputs[:2] + + """ + def __init__(self, config): + super(TFBertForSequenceClassification, self).__init__(config) + self.num_labels = config.num_labels + + self.bert = TFBertMainLayer(config, name='bert') + self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) + self.classifier = tf.keras.layers.Dense(config.num_labels, name='classifier') + + @tf.function + def call(self, inputs, training=False): + outputs = self.bert(inputs, training=training) + + pooled_output = outputs[1] + + pooled_output = self.dropout(pooled_output) + logits = self.classifier(pooled_output) + + outputs = (logits,) + outputs[2:] # add hidden states and attention if they are here + + return outputs # logits, (hidden_states), (attentions) + + +@add_start_docstrings("""Bert Model with a multiple choice classification head on top (a linear layer on top of + the pooled output and a softmax) e.g. for RocStories/SWAG tasks. """, + BERT_START_DOCSTRING) +class TFBertForMultipleChoice(TFBertPreTrainedModel): + r""" + Inputs: + **input_ids**: ``torch.LongTensor`` of shape ``(batch_size, num_choices, sequence_length)``: + Indices of input sequence tokens in the vocabulary. + The second dimension of the input (`num_choices`) indicates the number of choices to score. + To match pre-training, BERT input sequence should be formatted with [CLS] and [SEP] tokens as follows: + + (a) For sequence pairs: + + ``tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]`` + + ``token_type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1`` + + (b) For single sequences: + + ``tokens: [CLS] the dog is hairy . [SEP]`` + + ``token_type_ids: 0 0 0 0 0 0 0`` + + Indices can be obtained using :class:`pytorch_transformers.BertTokenizer`. + See :func:`pytorch_transformers.PreTrainedTokenizer.encode` and + :func:`pytorch_transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + **token_type_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, num_choices, sequence_length)``: + Segment token indices to indicate first and second portions of the inputs. + The second dimension of the input (`num_choices`) indicates the number of choices to score. + Indices are selected in ``[0, 1]``: ``0`` corresponds to a `sentence A` token, ``1`` + corresponds to a `sentence B` token + (see `BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ for more details). + **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, num_choices, sequence_length)``: + Mask to avoid performing attention on padding token indices. + The second dimension of the input (`num_choices`) indicates the number of choices to score. + Mask values selected in ``[0, 1]``: + ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. + **head_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: + Mask to nullify selected heads of the self-attention modules. + Mask values selected in ``[0, 1]``: + ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. + **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for computing the multiple choice classification loss. + Indices should be in ``[0, ..., num_choices]`` where `num_choices` is the size of the second dimension + of the input tensors. (see `input_ids` above) + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Classification loss. + **classification_scores**: ``torch.FloatTensor`` of shape ``(batch_size, num_choices)`` where `num_choices` is the size of the second dimension + of the input tensors. (see `input_ids` above). + Classification scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + model = BertForMultipleChoice.from_pretrained('bert-base-uncased') + choices = ["Hello, my dog is cute", "Hello, my cat is amazing"] + input_ids = torch.tensor([tokenizer.encode(s) for s in choices]).unsqueeze(0) # Batch size 1, 2 choices + labels = torch.tensor(1).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, labels=labels) + loss, classification_scores = outputs[:2] + + """ + def __init__(self, config): + super(TFBertForMultipleChoice, self).__init__(config) + + self.bert = TFBertMainLayer(config, name='bert') + self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) + self.classifier = tf.keras.layers.Dense(1, name='classifier') + + @tf.function + def call(self, inputs, training=False): + if not isinstance(inputs, (dict, tuple, list)): + input_ids = inputs + attention_mask, head_mask, position_ids, token_type_ids = None, None, None, None + elif isinstance(inputs, (tuple, list)): + input_ids = inputs[0] + attention_mask = inputs[1] if len(inputs) > 1 else None + token_type_ids = inputs[2] if len(inputs) > 2 else None + position_ids = inputs[3] if len(inputs) > 3 else None + head_mask = inputs[4] if len(inputs) > 4 else None + assert len(inputs) <= 5, "Too many inputs." + else: + input_ids = inputs.get('input_ids') + attention_mask = inputs.get('attention_mask', None) + token_type_ids = inputs.get('token_type_ids', None) + position_ids = inputs.get('position_ids', None) + head_mask = inputs.get('head_mask', None) + assert len(inputs) <= 5, "Too many inputs." + + num_choices = tf.shape(input_ids)[1] + seq_length = tf.shape(input_ids)[2] + + flat_input_ids = tf.reshape(input_ids, (-1, seq_length)) + flat_attention_mask = tf.reshape(attention_mask, (-1, seq_length)) if attention_mask is not None else None + flat_token_type_ids = tf.reshape(token_type_ids, (-1, seq_length)) if token_type_ids is not None else None + flat_position_ids = tf.reshape(position_ids, (-1, seq_length)) if position_ids is not None else None + + flat_inputs = [flat_input_ids, flat_attention_mask, flat_token_type_ids, flat_position_ids, head_mask] + + outputs = self.bert(flat_inputs, training=training) + + pooled_output = outputs[1] + + pooled_output = self.dropout(pooled_output) + logits = self.classifier(pooled_output) + reshaped_logits = tf.reshape(logits, (-1, num_choices)) + + outputs = (reshaped_logits,) + outputs[2:] # add hidden states and attention if they are here + + return outputs # reshaped_logits, (hidden_states), (attentions) + + +@add_start_docstrings("""Bert Model with a token classification head on top (a linear layer on top of + the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks. """, + BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) +class TFBertForTokenClassification(TFBertPreTrainedModel): + r""" + **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Labels for computing the token classification loss. + Indices should be in ``[0, ..., config.num_labels - 1]``. + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Classification loss. + **scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.num_labels)`` + Classification scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + model = BertForTokenClassification.from_pretrained('bert-base-uncased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + labels = torch.tensor([1] * input_ids.size(1)).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, labels=labels) + loss, scores = outputs[:2] + + """ + def __init__(self, config): + super(TFBertForTokenClassification, self).__init__(config) + self.num_labels = config.num_labels + + self.bert = TFBertMainLayer(config, name='bert') + self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) + self.classifier = tf.keras.layers.Dense(config.num_labels, name='classifier') + + @tf.function + def call(self, inputs, training=False): + outputs = self.bert(inputs, training=training) + + sequence_output = outputs[0] + + sequence_output = self.dropout(sequence_output) + logits = self.classifier(sequence_output) + + outputs = (logits,) + outputs[2:] # add hidden states and attention if they are here + + return outputs # scores, (hidden_states), (attentions) + + +@add_start_docstrings("""Bert Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layers on top of + the hidden-states output to compute `span start logits` and `span end logits`). """, + BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) +class TFBertForQuestionAnswering(TFBertPreTrainedModel): + r""" + **start_positions**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for position (index) of the start of the labelled span for computing the token classification loss. + Positions are clamped to the length of the sequence (`sequence_length`). + Position outside of the sequence are not taken into account for computing the loss. + **end_positions**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for position (index) of the end of the labelled span for computing the token classification loss. + Positions are clamped to the length of the sequence (`sequence_length`). + Position outside of the sequence are not taken into account for computing the loss. + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Total span extraction loss is the sum of a Cross-Entropy for the start and end positions. + **start_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length,)`` + Span-start scores (before SoftMax). + **end_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length,)`` + Span-end scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + model = BertForQuestionAnswering.from_pretrained('bert-base-uncased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + start_positions = torch.tensor([1]) + end_positions = torch.tensor([3]) + outputs = model(input_ids, start_positions=start_positions, end_positions=end_positions) + loss, start_scores, end_scores = outputs[:2] + + """ + def __init__(self, config): + super(TFBertForQuestionAnswering, self).__init__(config) + self.num_labels = config.num_labels + + self.bert = TFBertMainLayer(config, name='bert') + self.qa_outputs = tf.keras.layers.Dense(config.num_labels, name='qa_outputs') + + @tf.function + def call(self, inputs, training=False): + outputs = self.bert(inputs, training=training) + + sequence_output = outputs[0] + + logits = self.qa_outputs(sequence_output) + start_logits, end_logits = tf.split(logits, 2, axis=-1) + start_logits = tf.squeeze(start_logits, axis=-1) + end_logits = tf.squeeze(end_logits, axis=-1) + + outputs = (start_logits, end_logits,) + outputs[2:] + + return outputs # start_logits, end_logits, (hidden_states), (attentions) diff --git a/pytorch_transformers/tests/modeling_auto_test.py b/pytorch_transformers/tests/modeling_auto_test.py index dfdedbbe61..169f722ed7 100644 --- a/pytorch_transformers/tests/modeling_auto_test.py +++ b/pytorch_transformers/tests/modeling_auto_test.py @@ -21,15 +21,18 @@ import shutil import pytest import logging -from pytorch_transformers import (AutoConfig, BertConfig, - AutoModel, BertModel, - AutoModelWithLMHead, BertForMaskedLM, - AutoModelForSequenceClassification, BertForSequenceClassification, - AutoModelForQuestionAnswering, BertForQuestionAnswering) -from pytorch_transformers.modeling_bert import BERT_PRETRAINED_MODEL_ARCHIVE_MAP +try: + from pytorch_transformers import (AutoConfig, BertConfig, + AutoModel, BertModel, + AutoModelWithLMHead, BertForMaskedLM, + AutoModelForSequenceClassification, BertForSequenceClassification, + AutoModelForQuestionAnswering, BertForQuestionAnswering) + from pytorch_transformers.modeling_bert import BERT_PRETRAINED_MODEL_ARCHIVE_MAP -from .modeling_common_test import (CommonTestCases, ids_tensor) -from .configuration_common_test import ConfigTester + from .modeling_common_test import (CommonTestCases, ids_tensor) + from .configuration_common_test import ConfigTester +except ImportError: + pytestmark = pytest.mark.skip("Require Torch") class AutoModelTest(unittest.TestCase): diff --git a/pytorch_transformers/tests/modeling_bert_test.py b/pytorch_transformers/tests/modeling_bert_test.py index 39d169972d..797b117762 100644 --- a/pytorch_transformers/tests/modeling_bert_test.py +++ b/pytorch_transformers/tests/modeling_bert_test.py @@ -20,21 +20,26 @@ import unittest import shutil import pytest -from pytorch_transformers import (BertConfig, BertModel, BertForMaskedLM, - BertForNextSentencePrediction, BertForPreTraining, - BertForQuestionAnswering, BertForSequenceClassification, - BertForTokenClassification, BertForMultipleChoice) -from pytorch_transformers.modeling_bert import BERT_PRETRAINED_MODEL_ARCHIVE_MAP +from pytorch_transformers import is_torch_available from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester +try: + from pytorch_transformers import (BertConfig, BertModel, BertForMaskedLM, + BertForNextSentencePrediction, BertForPreTraining, + BertForQuestionAnswering, BertForSequenceClassification, + BertForTokenClassification, BertForMultipleChoice) + from pytorch_transformers.modeling_bert import BERT_PRETRAINED_MODEL_ARCHIVE_MAP +except ImportError: + pytestmark = pytest.mark.skip("Require Torch") + class BertModelTest(CommonTestCases.CommonModelTester): all_model_classes = (BertModel, BertForMaskedLM, BertForNextSentencePrediction, BertForPreTraining, BertForQuestionAnswering, BertForSequenceClassification, - BertForTokenClassification) + BertForTokenClassification) if is_torch_available() else () class BertModelTester(object): diff --git a/pytorch_transformers/tests/modeling_common_test.py b/pytorch_transformers/tests/modeling_common_test.py index c50d6678d8..1f778d608f 100644 --- a/pytorch_transformers/tests/modeling_common_test.py +++ b/pytorch_transformers/tests/modeling_common_test.py @@ -25,12 +25,16 @@ import uuid import unittest import logging +import pytest -import torch +try: + import torch -from pytorch_transformers import (PretrainedConfig, PreTrainedModel, - BertModel, BertConfig, BERT_PRETRAINED_MODEL_ARCHIVE_MAP, - GPT2LMHeadModel, GPT2Config, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP) + from pytorch_transformers import (PretrainedConfig, PreTrainedModel, + BertModel, BertConfig, BERT_PRETRAINED_MODEL_ARCHIVE_MAP, + GPT2LMHeadModel, GPT2Config, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP) +except ImportError: + pytestmark = pytest.mark.skip("Require Torch") def _config_zero_init(config): diff --git a/pytorch_transformers/tests/modeling_distilbert_test.py b/pytorch_transformers/tests/modeling_distilbert_test.py index c1503b4355..d5f2cba312 100644 --- a/pytorch_transformers/tests/modeling_distilbert_test.py +++ b/pytorch_transformers/tests/modeling_distilbert_test.py @@ -17,9 +17,15 @@ from __future__ import division from __future__ import print_function import unittest +import pytest -from pytorch_transformers import (DistilBertConfig, DistilBertModel, DistilBertForMaskedLM, - DistilBertForQuestionAnswering, DistilBertForSequenceClassification) +from pytorch_transformers import is_torch_available + +try: + from pytorch_transformers import (DistilBertConfig, DistilBertModel, DistilBertForMaskedLM, + DistilBertForQuestionAnswering, DistilBertForSequenceClassification) +except ImportError: + pytestmark = pytest.mark.skip("Require Torch") from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester @@ -28,7 +34,7 @@ from .configuration_common_test import ConfigTester class DistilBertModelTest(CommonTestCases.CommonModelTester): all_model_classes = (DistilBertModel, DistilBertForMaskedLM, DistilBertForQuestionAnswering, - DistilBertForSequenceClassification) + DistilBertForSequenceClassification) if is_torch_available() else None test_pruning = True test_torchscript = True test_resize_embeddings = True diff --git a/pytorch_transformers/tests/modeling_gpt2_test.py b/pytorch_transformers/tests/modeling_gpt2_test.py index 2717805120..273200f24f 100644 --- a/pytorch_transformers/tests/modeling_gpt2_test.py +++ b/pytorch_transformers/tests/modeling_gpt2_test.py @@ -20,9 +20,13 @@ import unittest import pytest import shutil +from pytorch_transformers import is_torch_available -from pytorch_transformers import (GPT2Config, GPT2Model, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP, - GPT2LMHeadModel, GPT2DoubleHeadsModel) +try: + from pytorch_transformers import (GPT2Config, GPT2Model, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP, + GPT2LMHeadModel, GPT2DoubleHeadsModel) +except ImportError: + pytestmark = pytest.mark.skip("Require Torch") from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester @@ -30,7 +34,7 @@ from .configuration_common_test import ConfigTester class GPT2ModelTest(CommonTestCases.CommonModelTester): - all_model_classes = (GPT2Model, GPT2LMHeadModel, GPT2DoubleHeadsModel) + all_model_classes = (GPT2Model, GPT2LMHeadModel, GPT2DoubleHeadsModel) if is_torch_available() else () class GPT2ModelTester(object): diff --git a/pytorch_transformers/tests/modeling_openai_test.py b/pytorch_transformers/tests/modeling_openai_test.py index dbef6c52eb..b89990a181 100644 --- a/pytorch_transformers/tests/modeling_openai_test.py +++ b/pytorch_transformers/tests/modeling_openai_test.py @@ -20,9 +20,13 @@ import unittest import pytest import shutil +from pytorch_transformers import is_torch_available -from pytorch_transformers import (OpenAIGPTConfig, OpenAIGPTModel, OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP, - OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel) +try: + from pytorch_transformers import (OpenAIGPTConfig, OpenAIGPTModel, OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP, + OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel) +except ImportError: + pytestmark = pytest.mark.skip("Require Torch") from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester @@ -30,7 +34,7 @@ from .configuration_common_test import ConfigTester class OpenAIGPTModelTest(CommonTestCases.CommonModelTester): - all_model_classes = (OpenAIGPTModel, OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel) + all_model_classes = (OpenAIGPTModel, OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel) if is_torch_available() else () class OpenAIGPTModelTester(object): diff --git a/pytorch_transformers/tests/modeling_roberta_test.py b/pytorch_transformers/tests/modeling_roberta_test.py index 0471505b5e..f729110bce 100644 --- a/pytorch_transformers/tests/modeling_roberta_test.py +++ b/pytorch_transformers/tests/modeling_roberta_test.py @@ -19,10 +19,15 @@ from __future__ import print_function import unittest import shutil import pytest -import torch -from pytorch_transformers import (RobertaConfig, RobertaModel, RobertaForMaskedLM, RobertaForSequenceClassification) -from pytorch_transformers.modeling_roberta import ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP +from pytorch_transformers import is_torch_available + +try: + import torch + from pytorch_transformers import (RobertaConfig, RobertaModel, RobertaForMaskedLM, RobertaForSequenceClassification) + from pytorch_transformers.modeling_roberta import ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP +except ImportError: + pytestmark = pytest.mark.skip("Require Torch") from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester @@ -30,7 +35,7 @@ from .configuration_common_test import ConfigTester class RobertaModelTest(CommonTestCases.CommonModelTester): - all_model_classes = (RobertaForMaskedLM, RobertaModel) + all_model_classes = (RobertaForMaskedLM, RobertaModel) if is_torch_available() else () class RobertaModelTester(object): diff --git a/pytorch_transformers/tests/modeling_tf_bert_test.py b/pytorch_transformers/tests/modeling_tf_bert_test.py index c7b8b02f7b..c95e33d780 100644 --- a/pytorch_transformers/tests/modeling_tf_bert_test.py +++ b/pytorch_transformers/tests/modeling_tf_bert_test.py @@ -24,21 +24,27 @@ import sys from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester +from pytorch_transformers import BertConfig, is_tf_available + try: import tensorflow as tf - - from pytorch_transformers import (BertConfig) - from pytorch_transformers.modeling_tf_bert import TFBertModel, TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP + from pytorch_transformers.modeling_tf_bert import (TFBertModel, TFBertForMaskedLM, + TFBertForNextSentencePrediction, + TFBertForPreTraining, + TFBertForSequenceClassification, + TFBertForMultipleChoice, + TFBertForTokenClassification, + TFBertForQuestionAnswering, + TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP) except ImportError: - pass + pytestmark = pytest.mark.skip("Require TensorFlow") class TFBertModelTest(TFCommonTestCases.TFCommonModelTester): - all_model_classes = (TFBertModel,) - # BertForMaskedLM, BertForNextSentencePrediction, - # BertForPreTraining, BertForQuestionAnswering, BertForSequenceClassification, - # BertForTokenClassification) + all_model_classes = (TFBertModel, TFBertForMaskedLM, TFBertForNextSentencePrediction, + TFBertForPreTraining, TFBertForQuestionAnswering, TFBertForSequenceClassification, + TFBertForTokenClassification) if is_tf_available() else () class TFBertModelTester(object): @@ -123,14 +129,8 @@ class TFBertModelTest(TFCommonTestCases.TFCommonModelTester): return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels - def check_loss_output(self, result): - self.parent.assertListEqual( - list(result["loss"].size()), - []) - def create_and_check_bert_model(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): model = TFBertModel(config=config) - # model.eval() inputs = {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids} @@ -152,125 +152,115 @@ class TFBertModelTest(TFCommonTestCases.TFCommonModelTester): def create_and_check_bert_for_masked_lm(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): - pass - # model = BertForMaskedLM(config=config) - # model.eval() - # loss, prediction_scores = model(input_ids, token_type_ids, input_mask, token_labels) - # result = { - # "loss": loss, - # "prediction_scores": prediction_scores, - # } - # self.parent.assertListEqual( - # list(result["prediction_scores"].size()), - # [self.batch_size, self.seq_length, self.vocab_size]) - # self.check_loss_output(result) + model = TFBertForMaskedLM(config=config) + inputs = {'input_ids': input_ids, + 'attention_mask': input_mask, + 'token_type_ids': token_type_ids} + prediction_scores, = model(inputs) + result = { + "prediction_scores": prediction_scores.numpy(), + } + self.parent.assertListEqual( + list(result["prediction_scores"].shape), + [self.batch_size, self.seq_length, self.vocab_size]) def create_and_check_bert_for_next_sequence_prediction(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): - pass - # model = BertForNextSentencePrediction(config=config) - # model.eval() - # loss, seq_relationship_score = model(input_ids, token_type_ids, input_mask, sequence_labels) - # result = { - # "loss": loss, - # "seq_relationship_score": seq_relationship_score, - # } - # self.parent.assertListEqual( - # list(result["seq_relationship_score"].size()), - # [self.batch_size, 2]) - # self.check_loss_output(result) + model = TFBertForNextSentencePrediction(config=config) + inputs = {'input_ids': input_ids, + 'attention_mask': input_mask, + 'token_type_ids': token_type_ids} + seq_relationship_score, = model(inputs) + result = { + "seq_relationship_score": seq_relationship_score.numpy(), + } + self.parent.assertListEqual( + list(result["seq_relationship_score"].shape), + [self.batch_size, 2]) def create_and_check_bert_for_pretraining(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): - pass - # model = BertForPreTraining(config=config) - # model.eval() - # loss, prediction_scores, seq_relationship_score = model(input_ids, token_type_ids, input_mask, token_labels, sequence_labels) - # result = { - # "loss": loss, - # "prediction_scores": prediction_scores, - # "seq_relationship_score": seq_relationship_score, - # } - # self.parent.assertListEqual( - # list(result["prediction_scores"].size()), - # [self.batch_size, self.seq_length, self.vocab_size]) - # self.parent.assertListEqual( - # list(result["seq_relationship_score"].size()), - # [self.batch_size, 2]) - # self.check_loss_output(result) - - - def create_and_check_bert_for_question_answering(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): - pass - # model = BertForQuestionAnswering(config=config) - # model.eval() - # loss, start_logits, end_logits = model(input_ids, token_type_ids, input_mask, sequence_labels, sequence_labels) - # result = { - # "loss": loss, - # "start_logits": start_logits, - # "end_logits": end_logits, - # } - # self.parent.assertListEqual( - # list(result["start_logits"].size()), - # [self.batch_size, self.seq_length]) - # self.parent.assertListEqual( - # list(result["end_logits"].size()), - # [self.batch_size, self.seq_length]) - # self.check_loss_output(result) + model = TFBertForPreTraining(config=config) + inputs = {'input_ids': input_ids, + 'attention_mask': input_mask, + 'token_type_ids': token_type_ids} + prediction_scores, seq_relationship_score = model(inputs) + result = { + "prediction_scores": prediction_scores.numpy(), + "seq_relationship_score": seq_relationship_score.numpy(), + } + self.parent.assertListEqual( + list(result["prediction_scores"].shape), + [self.batch_size, self.seq_length, self.vocab_size]) + self.parent.assertListEqual( + list(result["seq_relationship_score"].shape), + [self.batch_size, 2]) def create_and_check_bert_for_sequence_classification(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): - pass - # config.num_labels = self.num_labels - # model = BertForSequenceClassification(config) - # model.eval() - # loss, logits = model(input_ids, token_type_ids, input_mask, sequence_labels) - # result = { - # "loss": loss, - # "logits": logits, - # } - # self.parent.assertListEqual( - # list(result["logits"].size()), - # [self.batch_size, self.num_labels]) - # self.check_loss_output(result) - - - def create_and_check_bert_for_token_classification(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): - pass - # config.num_labels = self.num_labels - # model = BertForTokenClassification(config=config) - # model.eval() - # loss, logits = model(input_ids, token_type_ids, input_mask, token_labels) - # result = { - # "loss": loss, - # "logits": logits, - # } - # self.parent.assertListEqual( - # list(result["logits"].size()), - # [self.batch_size, self.seq_length, self.num_labels]) - # self.check_loss_output(result) + config.num_labels = self.num_labels + model = TFBertForSequenceClassification(config=config) + inputs = {'input_ids': input_ids, + 'attention_mask': input_mask, + 'token_type_ids': token_type_ids} + logits, = model(inputs) + result = { + "logits": logits.numpy(), + } + self.parent.assertListEqual( + list(result["logits"].shape), + [self.batch_size, self.num_labels]) def create_and_check_bert_for_multiple_choice(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): - pass - # config.num_choices = self.num_choices - # model = BertForMultipleChoice(config=config) - # model.eval() - # multiple_choice_inputs_ids = input_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous() - # multiple_choice_token_type_ids = token_type_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous() - # multiple_choice_input_mask = input_mask.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous() - # loss, logits = model(multiple_choice_inputs_ids, - # multiple_choice_token_type_ids, - # multiple_choice_input_mask, - # choice_labels) - # result = { - # "loss": loss, - # "logits": logits, - # } - # self.parent.assertListEqual( - # list(result["logits"].size()), - # [self.batch_size, self.num_choices]) - # self.check_loss_output(result) + config.num_choices = self.num_choices + model = TFBertForMultipleChoice(config=config) + multiple_choice_inputs_ids = tf.tile(tf.expand_dims(input_ids, 1), (1, self.num_choices, 1)) + multiple_choice_input_mask = tf.tile(tf.expand_dims(input_mask, 1), (1, self.num_choices, 1)) + multiple_choice_token_type_ids = tf.tile(tf.expand_dims(token_type_ids, 1), (1, self.num_choices, 1)) + inputs = {'input_ids': multiple_choice_inputs_ids, + 'attention_mask': multiple_choice_input_mask, + 'token_type_ids': multiple_choice_token_type_ids} + logits, = model(inputs) + result = { + "logits": logits.numpy(), + } + self.parent.assertListEqual( + list(result["logits"].shape), + [self.batch_size, self.num_choices]) + + + def create_and_check_bert_for_token_classification(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + config.num_labels = self.num_labels + model = TFBertForTokenClassification(config=config) + inputs = {'input_ids': input_ids, + 'attention_mask': input_mask, + 'token_type_ids': token_type_ids} + logits, = model(inputs) + result = { + "logits": logits.numpy(), + } + self.parent.assertListEqual( + list(result["logits"].shape), + [self.batch_size, self.seq_length, self.num_labels]) + + + def create_and_check_bert_for_question_answering(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + model = TFBertForQuestionAnswering(config=config) + inputs = {'input_ids': input_ids, + 'attention_mask': input_mask, + 'token_type_ids': token_type_ids} + start_logits, end_logits = model(inputs) + result = { + "start_logits": start_logits.numpy(), + "end_logits": end_logits.numpy(), + } + self.parent.assertListEqual( + list(result["start_logits"].shape), + [self.batch_size, self.seq_length]) + self.parent.assertListEqual( + list(result["end_logits"].shape), + [self.batch_size, self.seq_length]) def prepare_config_and_inputs_for_common(self): @@ -287,48 +277,39 @@ class TFBertModelTest(TFCommonTestCases.TFCommonModelTester): def test_config(self): self.config_tester.run_common_tests() - @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_bert_model(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_model(*config_and_inputs) - @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_for_masked_lm(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_for_masked_lm(*config_and_inputs) - @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_for_multiple_choice(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_for_multiple_choice(*config_and_inputs) - @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_for_next_sequence_prediction(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_for_next_sequence_prediction(*config_and_inputs) - @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_for_pretraining(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_for_pretraining(*config_and_inputs) - @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_for_question_answering(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_for_question_answering(*config_and_inputs) - @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_for_sequence_classification(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_for_sequence_classification(*config_and_inputs) - @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_for_token_classification(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_for_token_classification(*config_and_inputs) @pytest.mark.slow - @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_model_from_pretrained(self): cache_dir = "/tmp/pytorch_transformers_test/" for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: diff --git a/pytorch_transformers/tests/modeling_tf_common_test.py b/pytorch_transformers/tests/modeling_tf_common_test.py index 1ddfd83be1..404e6ad34e 100644 --- a/pytorch_transformers/tests/modeling_tf_common_test.py +++ b/pytorch_transformers/tests/modeling_tf_common_test.py @@ -30,7 +30,7 @@ try: from pytorch_transformers import TFPreTrainedModel # from pytorch_transformers.modeling_bert import BertModel, BertConfig, BERT_PRETRAINED_MODEL_ARCHIVE_MAP except ImportError: - pass + pytestmark = pytest.mark.skip("Require TensorFlow") def _config_zero_init(config): @@ -50,7 +50,6 @@ class TFCommonTestCases: test_pruning = True test_resize_embeddings = True - @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_initialization(self): pass # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -64,7 +63,6 @@ class TFCommonTestCases: # msg="Parameter {} of model {} seems not properly initialized".format(name, model_class)) - @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_attention_outputs(self): pass # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -105,7 +103,6 @@ class TFCommonTestCases: # self.model_tester.key_len if hasattr(self.model_tester, 'key_len') else self.model_tester.seq_length]) - @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_headmasking(self): pass # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -153,7 +150,6 @@ class TFCommonTestCases: # attentions[-1][..., -1, :, :].flatten().sum().item(), 0.0) - @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_head_pruning(self): pass # if not self.test_pruning: @@ -181,7 +177,6 @@ class TFCommonTestCases: # attentions[-1].shape[-3], self.model_tester.num_attention_heads - 1) - @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_hidden_states_output(self): pass # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -201,7 +196,6 @@ class TFCommonTestCases: # [self.model_tester.seq_length, self.model_tester.hidden_size]) - @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_resize_tokens_embeddings(self): pass # original_config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -238,7 +232,6 @@ class TFCommonTestCases: # self.assertTrue(models_equal) - @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_tie_model_weights(self): pass # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() diff --git a/pytorch_transformers/tests/modeling_transfo_xl_test.py b/pytorch_transformers/tests/modeling_transfo_xl_test.py index f482c47202..9a72335157 100644 --- a/pytorch_transformers/tests/modeling_transfo_xl_test.py +++ b/pytorch_transformers/tests/modeling_transfo_xl_test.py @@ -21,17 +21,21 @@ import random import shutil import pytest -import torch +from pytorch_transformers import is_torch_available -from pytorch_transformers import (TransfoXLConfig, TransfoXLModel, TransfoXLLMHeadModel) -from pytorch_transformers.modeling_transfo_xl import TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP +try: + import torch + from pytorch_transformers import (TransfoXLConfig, TransfoXLModel, TransfoXLLMHeadModel) + from pytorch_transformers.modeling_transfo_xl import TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP +except ImportError: + pytestmark = pytest.mark.skip("Require Torch") from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester class TransfoXLModelTest(CommonTestCases.CommonModelTester): - all_model_classes = (TransfoXLModel, TransfoXLLMHeadModel) + all_model_classes = (TransfoXLModel, TransfoXLLMHeadModel) if is_torch_available() else () test_pruning = False test_torchscript = False test_resize_embeddings = False diff --git a/pytorch_transformers/tests/modeling_xlm_test.py b/pytorch_transformers/tests/modeling_xlm_test.py index dcd0963477..21cf624b9b 100644 --- a/pytorch_transformers/tests/modeling_xlm_test.py +++ b/pytorch_transformers/tests/modeling_xlm_test.py @@ -20,8 +20,14 @@ import unittest import shutil import pytest -from pytorch_transformers import (XLMConfig, XLMModel, XLMWithLMHeadModel, XLMForQuestionAnswering, XLMForSequenceClassification) -from pytorch_transformers.modeling_xlm import XLM_PRETRAINED_MODEL_ARCHIVE_MAP +from pytorch_transformers import is_torch_available + +try: + from pytorch_transformers import (XLMConfig, XLMModel, XLMWithLMHeadModel, XLMForQuestionAnswering, + XLMForSequenceClassification) + from pytorch_transformers.modeling_xlm import XLM_PRETRAINED_MODEL_ARCHIVE_MAP +except ImportError: + pytestmark = pytest.mark.skip("Require Torch") from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester @@ -29,9 +35,9 @@ from .configuration_common_test import ConfigTester class XLMModelTest(CommonTestCases.CommonModelTester): - all_model_classes = (XLMModel, XLMWithLMHeadModel, - XLMForQuestionAnswering, XLMForSequenceClassification) - # , XLMForSequenceClassification, XLMForTokenClassification), + all_model_classes = (XLMModel, XLMWithLMHeadModel, XLMForQuestionAnswering, + XLMForSequenceClassification) if is_torch_available() else () + class XLMModelTester(object): diff --git a/pytorch_transformers/tests/modeling_xlnet_test.py b/pytorch_transformers/tests/modeling_xlnet_test.py index 4445bc17ac..b280ed4592 100644 --- a/pytorch_transformers/tests/modeling_xlnet_test.py +++ b/pytorch_transformers/tests/modeling_xlnet_test.py @@ -23,10 +23,15 @@ import random import shutil import pytest -import torch +from pytorch_transformers import is_torch_available -from pytorch_transformers import (XLNetConfig, XLNetModel, XLNetLMHeadModel, XLNetForSequenceClassification, XLNetForQuestionAnswering) -from pytorch_transformers.modeling_xlnet import XLNET_PRETRAINED_MODEL_ARCHIVE_MAP +try: + import torch + + from pytorch_transformers import (XLNetConfig, XLNetModel, XLNetLMHeadModel, XLNetForSequenceClassification, XLNetForQuestionAnswering) + from pytorch_transformers.modeling_xlnet import XLNET_PRETRAINED_MODEL_ARCHIVE_MAP +except ImportError: + pytestmark = pytest.mark.skip("Require Torch") from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester @@ -34,7 +39,7 @@ from .configuration_common_test import ConfigTester class XLNetModelTest(CommonTestCases.CommonModelTester): all_model_classes=(XLNetModel, XLNetLMHeadModel, - XLNetForSequenceClassification, XLNetForQuestionAnswering) + XLNetForSequenceClassification, XLNetForQuestionAnswering) if is_torch_available() else () test_pruning = False class XLNetModelTester(object): diff --git a/pytorch_transformers/tests/optimization_test.py b/pytorch_transformers/tests/optimization_test.py index 0146541582..07dc22141d 100644 --- a/pytorch_transformers/tests/optimization_test.py +++ b/pytorch_transformers/tests/optimization_test.py @@ -18,11 +18,17 @@ from __future__ import print_function import unittest import os +import pytest -import torch +from pytorch_transformers import is_torch_available -from pytorch_transformers import (AdamW, ConstantLRSchedule, WarmupConstantSchedule, - WarmupCosineSchedule, WarmupCosineWithHardRestartsSchedule, WarmupLinearSchedule) +try: + import torch + + from pytorch_transformers import (AdamW, ConstantLRSchedule, WarmupConstantSchedule, + WarmupCosineSchedule, WarmupCosineWithHardRestartsSchedule, WarmupLinearSchedule) +except ImportError: + pytestmark = pytest.mark.skip("Require Torch") from .tokenization_tests_commons import TemporaryDirectory @@ -71,8 +77,8 @@ class OptimizationTest(unittest.TestCase): class ScheduleInitTest(unittest.TestCase): - m = torch.nn.Linear(50, 50) - optimizer = AdamW(m.parameters(), lr=10.) + m = torch.nn.Linear(50, 50) if is_torch_available() else None + optimizer = AdamW(m.parameters(), lr=10.) if is_torch_available() else None num_steps = 10 def assertListAlmostEqual(self, list1, list2, tol): diff --git a/pytorch_transformers/tests/tokenization_auto_test.py b/pytorch_transformers/tests/tokenization_auto_test.py index f4f82083f2..7cee7ebc28 100644 --- a/pytorch_transformers/tests/tokenization_auto_test.py +++ b/pytorch_transformers/tests/tokenization_auto_test.py @@ -22,20 +22,19 @@ import pytest import logging from pytorch_transformers import AutoTokenizer, BertTokenizer, AutoTokenizer, GPT2Tokenizer -from pytorch_transformers.modeling_bert import BERT_PRETRAINED_MODEL_ARCHIVE_MAP -from pytorch_transformers.modeling_gpt2 import GPT2_PRETRAINED_MODEL_ARCHIVE_MAP +from pytorch_transformers import BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP class AutoTokenizerTest(unittest.TestCase): def test_tokenizer_from_pretrained(self): logging.basicConfig(level=logging.INFO) - for model_name in list(BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: + for model_name in list(BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys())[:1]: tokenizer = AutoTokenizer.from_pretrained(model_name) self.assertIsNotNone(tokenizer) self.assertIsInstance(tokenizer, BertTokenizer) self.assertGreater(len(tokenizer), 0) - for model_name in list(GPT2_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: + for model_name in list(GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys())[:1]: tokenizer = AutoTokenizer.from_pretrained(model_name) self.assertIsNotNone(tokenizer) self.assertIsInstance(tokenizer, GPT2Tokenizer) diff --git a/pytorch_transformers/tests/tokenization_transfo_xl_test.py b/pytorch_transformers/tests/tokenization_transfo_xl_test.py index f881cf1d2b..1406bac48b 100644 --- a/pytorch_transformers/tests/tokenization_transfo_xl_test.py +++ b/pytorch_transformers/tests/tokenization_transfo_xl_test.py @@ -16,15 +16,21 @@ from __future__ import absolute_import, division, print_function, unicode_litera import os import unittest +import pytest from io import open -from pytorch_transformers.tokenization_transfo_xl import TransfoXLTokenizer, VOCAB_FILES_NAMES +from pytorch_transformers import is_torch_available -from.tokenization_tests_commons import CommonTestCases +try: + from pytorch_transformers.tokenization_transfo_xl import TransfoXLTokenizer, VOCAB_FILES_NAMES +except ImportError: + pytestmark = pytest.mark.skip("Require Torch") # TODO: untangle Transfo-XL tokenizer from torch.load and torch.save + +from .tokenization_tests_commons import CommonTestCases class TransfoXLTokenizationTest(CommonTestCases.CommonTokenizerTester): - tokenizer_class = TransfoXLTokenizer + tokenizer_class = TransfoXLTokenizer if is_torch_available() else None def setUp(self): super(TransfoXLTokenizationTest, self).setUp() diff --git a/pytorch_transformers/tokenization_transfo_xl.py b/pytorch_transformers/tokenization_transfo_xl.py index 66bc01c1bb..8d5a0ce9d4 100644 --- a/pytorch_transformers/tokenization_transfo_xl.py +++ b/pytorch_transformers/tokenization_transfo_xl.py @@ -26,16 +26,20 @@ import sys from collections import Counter, OrderedDict from io import open -import torch import numpy as np from .file_utils import cached_path from .tokenization_utils import PreTrainedTokenizer -if sys.version_info[0] == 2: - import cPickle as pickle -else: - import pickle +try: + import torch +except ImportError: + pass + +# if sys.version_info[0] == 2: +# import cPickle as pickle +# else: +# import pickle logger = logging.getLogger(__name__) From 7f6a0c0d6959ca0653c34651030f9ec9559a308c Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 5 Sep 2019 11:20:56 +0200 Subject: [PATCH 009/219] no pytest version checking --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2bf082c850..c49cf7df8a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,7 +10,7 @@ jobs: - checkout - run: sudo pip install torch - run: sudo pip install --progress-bar off . - - run: sudo pip install pytest==5.0.1 codecov pytest-cov + - run: sudo pip install pytest codecov pytest-cov - run: sudo pip install tensorboardX scikit-learn - run: python -m pytest -sv ./pytorch_transformers/tests/ --cov - run: python -m pytest -sv ./examples/ @@ -25,7 +25,7 @@ jobs: - checkout - run: sudo pip install tensorflow==2.0.0-rc0 - run: sudo pip install --progress-bar off . - - run: sudo pip install pytest==5.0.1 codecov pytest-cov + - run: sudo pip install pytest codecov pytest-cov - run: sudo pip install tensorboardX scikit-learn - run: python -m pytest -sv ./pytorch_transformers/tests/ --cov - run: codecov @@ -39,7 +39,7 @@ jobs: - checkout - run: sudo pip install torch - run: sudo pip install --progress-bar off . - - run: sudo pip install pytest==5.0.1 codecov pytest-cov + - run: sudo pip install pytest codecov pytest-cov - run: python -m pytest -sv ./pytorch_transformers/tests/ --cov - run: codecov build_py2_tf: @@ -52,7 +52,7 @@ jobs: - checkout - run: sudo pip install tensorflow==2.0.0-rc0 - run: sudo pip install --progress-bar off . - - run: sudo pip install pytest==5.0.1 codecov pytest-cov + - run: sudo pip install pytest codecov pytest-cov - run: python -m pytest -sv ./pytorch_transformers/tests/ --cov - run: codecov deploy_doc: From 04d2006f28a6746e0b0521aa27b56328a6a66293 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 5 Sep 2019 11:22:13 +0200 Subject: [PATCH 010/219] skip transfo-xl tokenizer tests with tf for now --- pytorch_transformers/tests/tokenization_transfo_xl_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pytorch_transformers/tests/tokenization_transfo_xl_test.py b/pytorch_transformers/tests/tokenization_transfo_xl_test.py index 1406bac48b..792033d82c 100644 --- a/pytorch_transformers/tests/tokenization_transfo_xl_test.py +++ b/pytorch_transformers/tests/tokenization_transfo_xl_test.py @@ -22,6 +22,7 @@ from io import open from pytorch_transformers import is_torch_available try: + import torch from pytorch_transformers.tokenization_transfo_xl import TransfoXLTokenizer, VOCAB_FILES_NAMES except ImportError: pytestmark = pytest.mark.skip("Require Torch") # TODO: untangle Transfo-XL tokenizer from torch.load and torch.save From 600a42329bba99506902df7e6d53e79b1c60f7d0 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 5 Sep 2019 12:02:14 +0200 Subject: [PATCH 011/219] add weights tying, attention and hidden states output tests --- pytorch_transformers/modeling_tf_bert.py | 81 +++++++++++----- pytorch_transformers/modeling_tf_utils.py | 25 +++-- .../tests/modeling_tf_common_test.py | 94 +++++++++---------- 3 files changed, 121 insertions(+), 79 deletions(-) diff --git a/pytorch_transformers/modeling_tf_bert.py b/pytorch_transformers/modeling_tf_bert.py index bf33cb461d..fb73515f6e 100644 --- a/pytorch_transformers/modeling_tf_bert.py +++ b/pytorch_transformers/modeling_tf_bert.py @@ -141,7 +141,9 @@ class TFBertEmbeddings(tf.keras.layers.Layer): """ def __init__(self, config, **kwargs): super(TFBertEmbeddings, self).__init__(**kwargs) - self.word_embeddings = tf.keras.layers.Embedding(config.vocab_size, config.hidden_size, name='word_embeddings') + self.vocab_size = config.vocab_size + self.hidden_size = config.hidden_size + self.position_embeddings = tf.keras.layers.Embedding(config.max_position_embeddings, config.hidden_size, name='position_embeddings') self.token_type_embeddings = tf.keras.layers.Embedding(config.type_vocab_size, config.hidden_size, name='token_type_embeddings') @@ -150,8 +152,44 @@ class TFBertEmbeddings(tf.keras.layers.Layer): self.LayerNorm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name='LayerNorm') self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) + def build(self, input_shape): + """Build shared word embedding layer """ + with tf.name_scope("word_embeddings"): + # Create and initialize weights. The random normal initializer was chosen + # arbitrarily, and works well. + self.word_embeddings = self.add_weight( + "weight", + shape=[self.vocab_size, self.hidden_size], + initializer=tf.random_normal_initializer( + mean=0., stddev=self.hidden_size**-0.5)) + super(TFBertEmbeddings, self).build(input_shape) + @tf.function - def call(self, inputs, training=False): + def call(self, inputs, mode="embedding", training=False): + """Get token embeddings of inputs. + Args: + inputs: list of three int64 tensors with shape [batch_size, length]: (input_ids, position_ids, token_type_ids) + mode: string, a valid value is one of "embedding" and "linear". + Returns: + outputs: (1) If mode == "embedding", output embedding tensor, float32 with + shape [batch_size, length, embedding_size]; (2) mode == "linear", output + linear tensor, float32 with shape [batch_size, length, vocab_size]. + Raises: + ValueError: if mode is not valid. + + Shared weights logic adapted from + https://github.com/tensorflow/models/blob/a009f4fb9d2fc4949e32192a944688925ef78659/official/transformer/v2/embedding_layer.py#L24 + """ + if mode == "embedding": + return self._embedding(inputs, training=training) + elif mode == "linear": + return self._linear(inputs) + else: + raise ValueError("mode {} is not valid.".format(mode)) + + def _embedding(self, inputs, training=False): + """Applies embedding based on inputs tensor.""" + # Create binary mask of size [batch_size, length] input_ids, position_ids, token_type_ids = inputs seq_length = tf.shape(input_ids)[1] @@ -160,7 +198,7 @@ class TFBertEmbeddings(tf.keras.layers.Layer): if token_type_ids is None: token_type_ids = tf.fill(tf.shape(input_ids), 0) - words_embeddings = self.word_embeddings(input_ids) + words_embeddings = tf.gather(self.word_embeddings, input_ids) position_embeddings = self.position_embeddings(position_ids) token_type_embeddings = self.token_type_embeddings(token_type_ids) @@ -170,6 +208,21 @@ class TFBertEmbeddings(tf.keras.layers.Layer): embeddings = self.dropout(embeddings) return embeddings + def _linear(self, inputs): + """Computes logits by running inputs through a linear layer. + Args: + inputs: A float32 tensor with shape [batch_size, length, hidden_size] + Returns: + float32 tensor with shape [batch_size, length, vocab_size]. + """ + batch_size = tf.shape(inputs)[0] + length = tf.shape(inputs)[1] + + x = tf.reshape(inputs, [-1, self.hidden_size]) + logits = tf.matmul(x, self.word_embeddings, transpose_b=True) + + return tf.reshape(logits, [batch_size, length, self.vocab_size]) + class TFBertSelfAttention(tf.keras.layers.Layer): def __init__(self, config, **kwargs): @@ -448,8 +501,6 @@ class TFBertMainLayer(tf.keras.layers.Layer): self.encoder = TFBertEncoder(config, name='encoder') self.pooler = TFBertPooler(config, name='pooler') - # self.apply(self.init_weights) # TODO check weights initialization - def _resize_token_embeddings(self, new_num_tokens): raise NotImplementedError @@ -692,22 +743,14 @@ class TFBertForPreTraining(TFBertPreTrainedModel): super(TFBertForPreTraining, self).__init__(config) self.bert = TFBertMainLayer(config, name='bert') - self.cls_mlm = TFBertMLMHead(config, name='cls_mlm') self.cls_nsp = TFBertNSPHead(config, name='cls_nsp') - self.tie_weights() - - def tie_weights(self): - """ Make sure we are sharing the input and output embeddings. - """ - pass # TODO add weights tying - @tf.function def call(self, inputs, training=False): outputs = self.bert(inputs, training=training) sequence_output, pooled_output = outputs[:2] - prediction_scores = self.cls_mlm(sequence_output) + prediction_scores = self.bert.embeddings(sequence_output, mode="linear", training=training) seq_relationship_score = self.cls_nsp(pooled_output) outputs = (prediction_scores, seq_relationship_score,) + outputs[2:] # add hidden states and attention if they are here @@ -751,21 +794,13 @@ class TFBertForMaskedLM(TFBertPreTrainedModel): super(TFBertForMaskedLM, self).__init__(config) self.bert = TFBertMainLayer(config, name='bert') - self.cls_mlm = TFBertMLMHead(config, name='cls_mlm') - - self.tie_weights() - - def tie_weights(self): - """ Make sure we are sharing the input and output embeddings. - """ - pass # TODO add weights tying @tf.function def call(self, inputs, training=False): outputs = self.bert(inputs, training=training) sequence_output = outputs[0] - prediction_scores = self.cls_mlm(sequence_output) + prediction_scores = self.bert.embeddings(sequence_output, mode="linear", training=training) outputs = (prediction_scores,) + outputs[2:] # Add hidden states and attention if they are here diff --git a/pytorch_transformers/modeling_tf_utils.py b/pytorch_transformers/modeling_tf_utils.py index 442d9dbe44..cb1588e399 100644 --- a/pytorch_transformers/modeling_tf_utils.py +++ b/pytorch_transformers/modeling_tf_utils.py @@ -64,7 +64,7 @@ class TFPreTrainedModel(tf.keras.Model): self.config = config def _get_resized_embeddings(self, old_embeddings, new_num_tokens=None): - """ Build a resized Embedding Module from a provided token Embedding Module. + """ Build a resized Embedding Variable from a provided token Embedding Module. Increasing the size will add newly initialized vectors at the end Reducing the size will remove vectors from the end @@ -77,12 +77,25 @@ class TFPreTrainedModel(tf.keras.Model): Return: ``torch.nn.Embeddings`` Pointer to the resized Embedding Module or the old Embedding Module if new_num_tokens is None """ - raise NotImplementedError + # if new_num_tokens is None: + # return old_embeddings - def _tie_or_clone_weights(self, first_module, second_module): - """ Tie or clone module weights depending of weither we are using TorchScript or not - """ - raise NotImplementedError + # old_num_tokens, old_embedding_dim = old_embeddings.weight.size() + # if old_num_tokens == new_num_tokens: + # return old_embeddings + + # # Build new embeddings + # new_embeddings = nn.Embedding(new_num_tokens, old_embedding_dim) + # new_embeddings.to(old_embeddings.weight.device) + + # # initialize all new embeddings (in particular added tokens) + # self._init_weights(new_embeddings) + + # # Copy word embeddings from the previous weights + # num_tokens_to_copy = min(old_num_tokens, new_num_tokens) + # new_embeddings.weight.data[:num_tokens_to_copy, :] = old_embeddings.weight.data[:num_tokens_to_copy, :] + + # return new_embeddings def resize_token_embeddings(self, new_num_tokens=None): """ Resize input token embeddings matrix of the model if new_num_tokens != config.vocab_size. diff --git a/pytorch_transformers/tests/modeling_tf_common_test.py b/pytorch_transformers/tests/modeling_tf_common_test.py index 404e6ad34e..f9b87eed9a 100644 --- a/pytorch_transformers/tests/modeling_tf_common_test.py +++ b/pytorch_transformers/tests/modeling_tf_common_test.py @@ -64,44 +64,40 @@ class TFCommonTestCases: def test_attention_outputs(self): - pass - # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() - # for model_class in self.all_model_classes: - # config.output_attentions = True - # config.output_hidden_states = False - # model = model_class(config) - # model.eval() - # outputs = model(**inputs_dict) - # attentions = outputs[-1] - # self.assertEqual(model.config.output_attentions, True) - # self.assertEqual(model.config.output_hidden_states, False) - # self.assertEqual(len(attentions), self.model_tester.num_hidden_layers) - # self.assertListEqual( - # list(attentions[0].shape[-3:]), - # [self.model_tester.num_attention_heads, - # self.model_tester.seq_length, - # self.model_tester.key_len if hasattr(self.model_tester, 'key_len') else self.model_tester.seq_length]) - # out_len = len(outputs) + for model_class in self.all_model_classes: + config.output_attentions = True + config.output_hidden_states = False + model = model_class(config) + outputs = model(inputs_dict) + attentions = [t.numpy() for t in outputs[-1]] + self.assertEqual(model.config.output_attentions, True) + self.assertEqual(model.config.output_hidden_states, False) + self.assertEqual(len(attentions), self.model_tester.num_hidden_layers) + self.assertListEqual( + list(attentions[0].shape[-3:]), + [self.model_tester.num_attention_heads, + self.model_tester.seq_length, + self.model_tester.key_len if hasattr(self.model_tester, 'key_len') else self.model_tester.seq_length]) + out_len = len(outputs) - # # Check attention is always last and order is fine - # config.output_attentions = True - # config.output_hidden_states = True - # model = model_class(config) - # model.eval() - # outputs = model(**inputs_dict) - # self.assertEqual(out_len+1, len(outputs)) - # self.assertEqual(model.config.output_attentions, True) - # self.assertEqual(model.config.output_hidden_states, True) - - # attentions = outputs[-1] - # self.assertEqual(len(attentions), self.model_tester.num_hidden_layers) - # self.assertListEqual( - # list(attentions[0].shape[-3:]), - # [self.model_tester.num_attention_heads, - # self.model_tester.seq_length, - # self.model_tester.key_len if hasattr(self.model_tester, 'key_len') else self.model_tester.seq_length]) + # Check attention is always last and order is fine + config.output_attentions = True + config.output_hidden_states = True + model = model_class(config) + outputs = model(inputs_dict) + self.assertEqual(out_len+1, len(outputs)) + self.assertEqual(model.config.output_attentions, True) + self.assertEqual(model.config.output_hidden_states, True) + attentions = [t.numpy() for t in outputs[-1]] + self.assertEqual(len(attentions), self.model_tester.num_hidden_layers) + self.assertListEqual( + list(attentions[0].shape[-3:]), + [self.model_tester.num_attention_heads, + self.model_tester.seq_length, + self.model_tester.key_len if hasattr(self.model_tester, 'key_len') else self.model_tester.seq_length]) def test_headmasking(self): pass @@ -178,22 +174,20 @@ class TFCommonTestCases: def test_hidden_states_output(self): - pass - # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() - # for model_class in self.all_model_classes: - # config.output_hidden_states = True - # config.output_attentions = False - # model = model_class(config) - # model.eval() - # outputs = model(**inputs_dict) - # hidden_states = outputs[-1] - # self.assertEqual(model.config.output_attentions, False) - # self.assertEqual(model.config.output_hidden_states, True) - # self.assertEqual(len(hidden_states), self.model_tester.num_hidden_layers + 1) - # self.assertListEqual( - # list(hidden_states[0].shape[-2:]), - # [self.model_tester.seq_length, self.model_tester.hidden_size]) + for model_class in self.all_model_classes: + config.output_hidden_states = True + config.output_attentions = False + model = model_class(config) + outputs = model(inputs_dict) + hidden_states = [t.numpy() for t in outputs[-1]] + self.assertEqual(model.config.output_attentions, False) + self.assertEqual(model.config.output_hidden_states, True) + self.assertEqual(len(hidden_states), self.model_tester.num_hidden_layers + 1) + self.assertListEqual( + list(hidden_states[0].shape[-2:]), + [self.model_tester.seq_length, self.model_tester.hidden_size]) def test_resize_tokens_embeddings(self): From 705237b4ecefb6e04e57f1ec2c85d64011d706d6 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 5 Sep 2019 12:21:08 +0200 Subject: [PATCH 012/219] add tf auto models + tests --- pytorch_transformers/__init__.py | 9 +- pytorch_transformers/modeling_tf_auto.py | 488 ++++++++++++++++++ pytorch_transformers/modeling_tf_utils.py | 11 - .../tests/modeling_tf_auto_test.py | 85 +++ 4 files changed, 581 insertions(+), 12 deletions(-) create mode 100644 pytorch_transformers/modeling_tf_auto.py create mode 100644 pytorch_transformers/tests/modeling_tf_auto_test.py diff --git a/pytorch_transformers/__init__.py b/pytorch_transformers/__init__.py index a41c838015..f13457f073 100644 --- a/pytorch_transformers/__init__.py +++ b/pytorch_transformers/__init__.py @@ -96,8 +96,15 @@ if _tf_available: logger.info("TensorFlow version {} available.".format(tf.__version__)) from .modeling_tf_utils import TFPreTrainedModel + from .modeling_tf_auto import (TFAutoModel, TFAutoModelForSequenceClassification, TFAutoModelForQuestionAnswering, + TFAutoModelWithLMHead) + from .modeling_tf_bert import (TFBertPreTrainedModel, TFBertModel, TFBertForPreTraining, - TFBertForMaskedLM, TFBertForNextSentencePrediction, load_bert_pt_weights_in_tf) + TFBertForMaskedLM, TFBertForNextSentencePrediction, + TFBertForSequenceClassification, TFBertForMultipleChoice, + TFBertForTokenClassification, TFBertForQuestionAnswering, + load_bert_pt_weights_in_tf, + TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP) # Files and general utilities diff --git a/pytorch_transformers/modeling_tf_auto.py b/pytorch_transformers/modeling_tf_auto.py new file mode 100644 index 0000000000..3b10d700a1 --- /dev/null +++ b/pytorch_transformers/modeling_tf_auto.py @@ -0,0 +1,488 @@ +# coding=utf-8 +# Copyright 2018 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Auto Model class. """ + +from __future__ import absolute_import, division, print_function, unicode_literals + +import logging + +from .modeling_tf_bert import TFBertModel, TFBertForMaskedLM, TFBertForSequenceClassification, TFBertForQuestionAnswering + +from .file_utils import add_start_docstrings + +logger = logging.getLogger(__name__) + + +class TFAutoModel(object): + r""" + :class:`~pytorch_transformers.TFAutoModel` is a generic model class + that will be instantiated as one of the base model classes of the library + when created with the `TFAutoModel.from_pretrained(pretrained_model_name_or_path)` + class method. + + The `from_pretrained()` method takes care of returning the correct model class instance + using pattern matching on the `pretrained_model_name_or_path` string. + + The base model class to instantiate is selected as the first pattern matching + in the `pretrained_model_name_or_path` string (in the following order): + - contains `distilbert`: DistilBertModel (DistilBERT model) + - contains `roberta`: RobertaModel (RoBERTa model) + - contains `bert`: TFBertModel (Bert model) + - contains `openai-gpt`: OpenAIGPTModel (OpenAI GPT model) + - contains `gpt2`: GPT2Model (OpenAI GPT-2 model) + - contains `transfo-xl`: TransfoXLModel (Transformer-XL model) + - contains `xlnet`: XLNetModel (XLNet model) + - contains `xlm`: XLMModel (XLM model) + + This class cannot be instantiated using `__init__()` (throws an error). + """ + def __init__(self): + raise EnvironmentError("TFAutoModel is designed to be instantiated " + "using the `TFAutoModel.from_pretrained(pretrained_model_name_or_path)` method.") + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): + r""" Instantiates one of the base model classes of the library + from a pre-trained model configuration. + + The model class to instantiate is selected as the first pattern matching + in the `pretrained_model_name_or_path` string (in the following order): + - contains `distilbert`: DistilBertModel (DistilBERT model) + - contains `roberta`: RobertaModel (RoBERTa model) + - contains `bert`: TFBertModel (Bert model) + - contains `openai-gpt`: OpenAIGPTModel (OpenAI GPT model) + - contains `gpt2`: GPT2Model (OpenAI GPT-2 model) + - contains `transfo-xl`: TransfoXLModel (Transformer-XL model) + - contains `xlnet`: XLNetModel (XLNet model) + - contains `xlm`: XLMModel (XLM model) + + The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated) + To train the model, you should first set it back in training mode with `model.train()` + + Params: + pretrained_model_name_or_path: either: + + - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. + - a path to a `directory` containing model weights saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. + - a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. + + model_args: (`optional`) Sequence of positional arguments: + All remaning positional arguments will be passed to the underlying model's ``__init__`` method + + config: (`optional`) instance of a class derived from :class:`~pytorch_transformers.PretrainedConfig`: + Configuration for the model to use instead of an automatically loaded configuation. Configuration can be automatically loaded when: + + - the model is a model provided by the library (loaded with the ``shortcut-name`` string of a pretrained model), or + - the model was saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained` and is reloaded by suppling the save directory. + - the model is loaded by suppling a local directory as ``pretrained_model_name_or_path`` and a configuration JSON file named `config.json` is found in the directory. + + state_dict: (`optional`) dict: + an optional state dictionnary for the model to use instead of a state dictionary loaded from saved weights file. + This option can be used if you want to create a model from a pretrained configuration but load your own weights. + In this case though, you should check if using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained` and :func:`~pytorch_transformers.PreTrainedModel.from_pretrained` is not a simpler option. + + cache_dir: (`optional`) string: + Path to a directory in which a downloaded pre-trained model + configuration should be cached if the standard cache should not be used. + + force_download: (`optional`) boolean, default False: + Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + + proxies: (`optional`) dict, default None: + A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. + The proxies are used on each request. + + output_loading_info: (`optional`) boolean: + Set to ``True`` to also return a dictionnary containing missing keys, unexpected keys and error messages. + + kwargs: (`optional`) Remaining dictionary of keyword arguments: + Can be used to update the configuration object (after it being loaded) and initiate the model. (e.g. ``output_attention=True``). Behave differently depending on whether a `config` is provided or automatically loaded: + + - If a configuration is provided with ``config``, ``**kwargs`` will be directly passed to the underlying model's ``__init__`` method (we assume all relevant updates to the configuration have already been done) + - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~pytorch_transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function. + + Examples:: + + model = TFAutoModel.from_pretrained('bert-base-uncased') # Download model and configuration from S3 and cache. + model = TFAutoModel.from_pretrained('./test/bert_model/') # E.g. model was saved using `save_pretrained('./test/saved_model/')` + model = TFAutoModel.from_pretrained('bert-base-uncased', output_attention=True) # Update configuration during loading + assert model.config.output_attention == True + # Loading from a TF checkpoint file instead of a PyTorch model (slower) + config = AutoConfig.from_json_file('./tf_model/bert_tf_model_config.json') + model = TFAutoModel.from_pretrained('./tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config) + + """ + if 'distilbert' in pretrained_model_name_or_path: + raise NotImplementedError + elif 'roberta' in pretrained_model_name_or_path: + raise NotImplementedError + elif 'bert' in pretrained_model_name_or_path: + return TFBertModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'openai-gpt' in pretrained_model_name_or_path: + raise NotImplementedError + elif 'gpt2' in pretrained_model_name_or_path: + raise NotImplementedError + elif 'transfo-xl' in pretrained_model_name_or_path: + raise NotImplementedError + elif 'xlnet' in pretrained_model_name_or_path: + raise NotImplementedError + elif 'xlm' in pretrained_model_name_or_path: + raise NotImplementedError + + raise ValueError("Unrecognized model identifier in {}. Should contains one of " + "'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', " + "'xlm', 'roberta'".format(pretrained_model_name_or_path)) + + +class TFAutoModelWithLMHead(object): + r""" + :class:`~pytorch_transformers.TFAutoModelWithLMHead` is a generic model class + that will be instantiated as one of the language modeling model classes of the library + when created with the `TFAutoModelWithLMHead.from_pretrained(pretrained_model_name_or_path)` + class method. + + The `from_pretrained()` method takes care of returning the correct model class instance + using pattern matching on the `pretrained_model_name_or_path` string. + + The model class to instantiate is selected as the first pattern matching + in the `pretrained_model_name_or_path` string (in the following order): + - contains `distilbert`: DistilBertForMaskedLM (DistilBERT model) + - contains `roberta`: RobertaForMaskedLM (RoBERTa model) + - contains `bert`: TFBertForMaskedLM (Bert model) + - contains `openai-gpt`: OpenAIGPTLMHeadModel (OpenAI GPT model) + - contains `gpt2`: GPT2LMHeadModel (OpenAI GPT-2 model) + - contains `transfo-xl`: TransfoXLLMHeadModel (Transformer-XL model) + - contains `xlnet`: XLNetLMHeadModel (XLNet model) + - contains `xlm`: XLMWithLMHeadModel (XLM model) + + This class cannot be instantiated using `__init__()` (throws an error). + """ + def __init__(self): + raise EnvironmentError("TFAutoModelWithLMHead is designed to be instantiated " + "using the `TFAutoModelWithLMHead.from_pretrained(pretrained_model_name_or_path)` method.") + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): + r""" Instantiates one of the language modeling model classes of the library + from a pre-trained model configuration. + + The `from_pretrained()` method takes care of returning the correct model class instance + using pattern matching on the `pretrained_model_name_or_path` string. + + The model class to instantiate is selected as the first pattern matching + in the `pretrained_model_name_or_path` string (in the following order): + - contains `distilbert`: DistilBertForMaskedLM (DistilBERT model) + - contains `roberta`: RobertaForMaskedLM (RoBERTa model) + - contains `bert`: TFBertForMaskedLM (Bert model) + - contains `openai-gpt`: OpenAIGPTLMHeadModel (OpenAI GPT model) + - contains `gpt2`: GPT2LMHeadModel (OpenAI GPT-2 model) + - contains `transfo-xl`: TransfoXLLMHeadModel (Transformer-XL model) + - contains `xlnet`: XLNetLMHeadModel (XLNet model) + - contains `xlm`: XLMWithLMHeadModel (XLM model) + + The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated) + To train the model, you should first set it back in training mode with `model.train()` + + Params: + pretrained_model_name_or_path: either: + + - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. + - a path to a `directory` containing model weights saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. + - a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. + + model_args: (`optional`) Sequence of positional arguments: + All remaning positional arguments will be passed to the underlying model's ``__init__`` method + + config: (`optional`) instance of a class derived from :class:`~pytorch_transformers.PretrainedConfig`: + Configuration for the model to use instead of an automatically loaded configuation. Configuration can be automatically loaded when: + + - the model is a model provided by the library (loaded with the ``shortcut-name`` string of a pretrained model), or + - the model was saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained` and is reloaded by suppling the save directory. + - the model is loaded by suppling a local directory as ``pretrained_model_name_or_path`` and a configuration JSON file named `config.json` is found in the directory. + + state_dict: (`optional`) dict: + an optional state dictionnary for the model to use instead of a state dictionary loaded from saved weights file. + This option can be used if you want to create a model from a pretrained configuration but load your own weights. + In this case though, you should check if using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained` and :func:`~pytorch_transformers.PreTrainedModel.from_pretrained` is not a simpler option. + + cache_dir: (`optional`) string: + Path to a directory in which a downloaded pre-trained model + configuration should be cached if the standard cache should not be used. + + force_download: (`optional`) boolean, default False: + Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + + proxies: (`optional`) dict, default None: + A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. + The proxies are used on each request. + + output_loading_info: (`optional`) boolean: + Set to ``True`` to also return a dictionnary containing missing keys, unexpected keys and error messages. + + kwargs: (`optional`) Remaining dictionary of keyword arguments: + Can be used to update the configuration object (after it being loaded) and initiate the model. (e.g. ``output_attention=True``). Behave differently depending on whether a `config` is provided or automatically loaded: + + - If a configuration is provided with ``config``, ``**kwargs`` will be directly passed to the underlying model's ``__init__`` method (we assume all relevant updates to the configuration have already been done) + - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~pytorch_transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function. + + Examples:: + + model = TFAutoModelWithLMHead.from_pretrained('bert-base-uncased') # Download model and configuration from S3 and cache. + model = TFAutoModelWithLMHead.from_pretrained('./test/bert_model/') # E.g. model was saved using `save_pretrained('./test/saved_model/')` + model = TFAutoModelWithLMHead.from_pretrained('bert-base-uncased', output_attention=True) # Update configuration during loading + assert model.config.output_attention == True + # Loading from a TF checkpoint file instead of a PyTorch model (slower) + config = AutoConfig.from_json_file('./tf_model/bert_tf_model_config.json') + model = TFAutoModelWithLMHead.from_pretrained('./tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config) + + """ + if 'distilbert' in pretrained_model_name_or_path: + raise NotImplementedError + elif 'roberta' in pretrained_model_name_or_path: + raise NotImplementedError + elif 'bert' in pretrained_model_name_or_path: + return TFBertForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'openai-gpt' in pretrained_model_name_or_path: + raise NotImplementedError + elif 'gpt2' in pretrained_model_name_or_path: + raise NotImplementedError + elif 'transfo-xl' in pretrained_model_name_or_path: + raise NotImplementedError + elif 'xlnet' in pretrained_model_name_or_path: + raise NotImplementedError + elif 'xlm' in pretrained_model_name_or_path: + raise NotImplementedError + + raise ValueError("Unrecognized model identifier in {}. Should contains one of " + "'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', " + "'xlm', 'roberta'".format(pretrained_model_name_or_path)) + + +class TFAutoModelForSequenceClassification(object): + r""" + :class:`~pytorch_transformers.TFAutoModelForSequenceClassification` is a generic model class + that will be instantiated as one of the sequence classification model classes of the library + when created with the `TFAutoModelForSequenceClassification.from_pretrained(pretrained_model_name_or_path)` + class method. + + The `from_pretrained()` method takes care of returning the correct model class instance + using pattern matching on the `pretrained_model_name_or_path` string. + + The model class to instantiate is selected as the first pattern matching + in the `pretrained_model_name_or_path` string (in the following order): + - contains `distilbert`: DistilBertForSequenceClassification (DistilBERT model) + - contains `roberta`: RobertaForSequenceClassification (RoBERTa model) + - contains `bert`: TFBertForSequenceClassification (Bert model) + - contains `xlnet`: XLNetForSequenceClassification (XLNet model) + - contains `xlm`: XLMForSequenceClassification (XLM model) + + This class cannot be instantiated using `__init__()` (throws an error). + """ + def __init__(self): + raise EnvironmentError("TFAutoModelWithLMHead is designed to be instantiated " + "using the `TFAutoModelWithLMHead.from_pretrained(pretrained_model_name_or_path)` method.") + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): + r""" Instantiates one of the sequence classification model classes of the library + from a pre-trained model configuration. + + The `from_pretrained()` method takes care of returning the correct model class instance + using pattern matching on the `pretrained_model_name_or_path` string. + + The model class to instantiate is selected as the first pattern matching + in the `pretrained_model_name_or_path` string (in the following order): + - contains `distilbert`: DistilBertForSequenceClassification (DistilBERT model) + - contains `roberta`: RobertaForSequenceClassification (RoBERTa model) + - contains `bert`: TFBertForSequenceClassification (Bert model) + - contains `xlnet`: XLNetForSequenceClassification (XLNet model) + - contains `xlm`: XLMForSequenceClassification (XLM model) + + The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated) + To train the model, you should first set it back in training mode with `model.train()` + + Params: + pretrained_model_name_or_path: either: + + - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. + - a path to a `directory` containing model weights saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. + - a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. + + model_args: (`optional`) Sequence of positional arguments: + All remaning positional arguments will be passed to the underlying model's ``__init__`` method + + config: (`optional`) instance of a class derived from :class:`~pytorch_transformers.PretrainedConfig`: + Configuration for the model to use instead of an automatically loaded configuation. Configuration can be automatically loaded when: + + - the model is a model provided by the library (loaded with the ``shortcut-name`` string of a pretrained model), or + - the model was saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained` and is reloaded by suppling the save directory. + - the model is loaded by suppling a local directory as ``pretrained_model_name_or_path`` and a configuration JSON file named `config.json` is found in the directory. + + state_dict: (`optional`) dict: + an optional state dictionnary for the model to use instead of a state dictionary loaded from saved weights file. + This option can be used if you want to create a model from a pretrained configuration but load your own weights. + In this case though, you should check if using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained` and :func:`~pytorch_transformers.PreTrainedModel.from_pretrained` is not a simpler option. + + cache_dir: (`optional`) string: + Path to a directory in which a downloaded pre-trained model + configuration should be cached if the standard cache should not be used. + + force_download: (`optional`) boolean, default False: + Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + + proxies: (`optional`) dict, default None: + A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. + The proxies are used on each request. + + output_loading_info: (`optional`) boolean: + Set to ``True`` to also return a dictionnary containing missing keys, unexpected keys and error messages. + + kwargs: (`optional`) Remaining dictionary of keyword arguments: + Can be used to update the configuration object (after it being loaded) and initiate the model. (e.g. ``output_attention=True``). Behave differently depending on whether a `config` is provided or automatically loaded: + + - If a configuration is provided with ``config``, ``**kwargs`` will be directly passed to the underlying model's ``__init__`` method (we assume all relevant updates to the configuration have already been done) + - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~pytorch_transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function. + + Examples:: + + model = TFAutoModelForSequenceClassification.from_pretrained('bert-base-uncased') # Download model and configuration from S3 and cache. + model = TFAutoModelForSequenceClassification.from_pretrained('./test/bert_model/') # E.g. model was saved using `save_pretrained('./test/saved_model/')` + model = TFAutoModelForSequenceClassification.from_pretrained('bert-base-uncased', output_attention=True) # Update configuration during loading + assert model.config.output_attention == True + # Loading from a TF checkpoint file instead of a PyTorch model (slower) + config = AutoConfig.from_json_file('./tf_model/bert_tf_model_config.json') + model = TFAutoModelForSequenceClassification.from_pretrained('./tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config) + + """ + if 'distilbert' in pretrained_model_name_or_path: + raise NotImplementedError + elif 'roberta' in pretrained_model_name_or_path: + raise NotImplementedError + elif 'bert' in pretrained_model_name_or_path: + return TFBertForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'xlnet' in pretrained_model_name_or_path: + raise NotImplementedError + elif 'xlm' in pretrained_model_name_or_path: + raise NotImplementedError + + raise ValueError("Unrecognized model identifier in {}. Should contains one of " + "'bert', 'xlnet', 'xlm', 'roberta'".format(pretrained_model_name_or_path)) + + +class TFAutoModelForQuestionAnswering(object): + r""" + :class:`~pytorch_transformers.TFAutoModelForQuestionAnswering` is a generic model class + that will be instantiated as one of the question answering model classes of the library + when created with the `TFAutoModelForQuestionAnswering.from_pretrained(pretrained_model_name_or_path)` + class method. + + The `from_pretrained()` method takes care of returning the correct model class instance + using pattern matching on the `pretrained_model_name_or_path` string. + + The model class to instantiate is selected as the first pattern matching + in the `pretrained_model_name_or_path` string (in the following order): + - contains `distilbert`: DistilBertForQuestionAnswering (DistilBERT model) + - contains `bert`: TFBertForQuestionAnswering (Bert model) + - contains `xlnet`: XLNetForQuestionAnswering (XLNet model) + - contains `xlm`: XLMForQuestionAnswering (XLM model) + + This class cannot be instantiated using `__init__()` (throws an error). + """ + def __init__(self): + raise EnvironmentError("TFAutoModelWithLMHead is designed to be instantiated " + "using the `TFAutoModelWithLMHead.from_pretrained(pretrained_model_name_or_path)` method.") + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): + r""" Instantiates one of the question answering model classes of the library + from a pre-trained model configuration. + + The `from_pretrained()` method takes care of returning the correct model class instance + using pattern matching on the `pretrained_model_name_or_path` string. + + The model class to instantiate is selected as the first pattern matching + in the `pretrained_model_name_or_path` string (in the following order): + - contains `distilbert`: DistilBertForQuestionAnswering (DistilBERT model) + - contains `bert`: TFBertForQuestionAnswering (Bert model) + - contains `xlnet`: XLNetForQuestionAnswering (XLNet model) + - contains `xlm`: XLMForQuestionAnswering (XLM model) + + The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated) + To train the model, you should first set it back in training mode with `model.train()` + + Params: + pretrained_model_name_or_path: either: + + - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. + - a path to a `directory` containing model weights saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. + - a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. + + model_args: (`optional`) Sequence of positional arguments: + All remaning positional arguments will be passed to the underlying model's ``__init__`` method + + config: (`optional`) instance of a class derived from :class:`~pytorch_transformers.PretrainedConfig`: + Configuration for the model to use instead of an automatically loaded configuation. Configuration can be automatically loaded when: + + - the model is a model provided by the library (loaded with the ``shortcut-name`` string of a pretrained model), or + - the model was saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained` and is reloaded by suppling the save directory. + - the model is loaded by suppling a local directory as ``pretrained_model_name_or_path`` and a configuration JSON file named `config.json` is found in the directory. + + state_dict: (`optional`) dict: + an optional state dictionnary for the model to use instead of a state dictionary loaded from saved weights file. + This option can be used if you want to create a model from a pretrained configuration but load your own weights. + In this case though, you should check if using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained` and :func:`~pytorch_transformers.PreTrainedModel.from_pretrained` is not a simpler option. + + cache_dir: (`optional`) string: + Path to a directory in which a downloaded pre-trained model + configuration should be cached if the standard cache should not be used. + + force_download: (`optional`) boolean, default False: + Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + + proxies: (`optional`) dict, default None: + A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. + The proxies are used on each request. + + output_loading_info: (`optional`) boolean: + Set to ``True`` to also return a dictionnary containing missing keys, unexpected keys and error messages. + + kwargs: (`optional`) Remaining dictionary of keyword arguments: + Can be used to update the configuration object (after it being loaded) and initiate the model. (e.g. ``output_attention=True``). Behave differently depending on whether a `config` is provided or automatically loaded: + + - If a configuration is provided with ``config``, ``**kwargs`` will be directly passed to the underlying model's ``__init__`` method (we assume all relevant updates to the configuration have already been done) + - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~pytorch_transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function. + + Examples:: + + model = TFAutoModelForQuestionAnswering.from_pretrained('bert-base-uncased') # Download model and configuration from S3 and cache. + model = TFAutoModelForQuestionAnswering.from_pretrained('./test/bert_model/') # E.g. model was saved using `save_pretrained('./test/saved_model/')` + model = TFAutoModelForQuestionAnswering.from_pretrained('bert-base-uncased', output_attention=True) # Update configuration during loading + assert model.config.output_attention == True + # Loading from a TF checkpoint file instead of a PyTorch model (slower) + config = AutoConfig.from_json_file('./tf_model/bert_tf_model_config.json') + model = TFAutoModelForQuestionAnswering.from_pretrained('./tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config) + + """ + if 'distilbert' in pretrained_model_name_or_path: + raise NotImplementedError + elif 'bert' in pretrained_model_name_or_path: + return TFBertForQuestionAnswering.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'xlnet' in pretrained_model_name_or_path: + raise NotImplementedError + elif 'xlm' in pretrained_model_name_or_path: + raise NotImplementedError + + raise ValueError("Unrecognized model identifier in {}. Should contains one of " + "'bert', 'xlnet', 'xlm'".format(pretrained_model_name_or_path)) diff --git a/pytorch_transformers/modeling_tf_utils.py b/pytorch_transformers/modeling_tf_utils.py index cb1588e399..ef2375c60a 100644 --- a/pytorch_transformers/modeling_tf_utils.py +++ b/pytorch_transformers/modeling_tf_utils.py @@ -170,9 +170,6 @@ class TFPreTrainedModel(tf.keras.Model): A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. - output_loading_info: (`optional`) boolean: - Set to ``True`` to also return a dictionnary containing missing keys, unexpected keys and error messages. - kwargs: (`optional`) Remaining dictionary of keyword arguments: Can be used to update the configuration object (after it being loaded) and initiate the model. (e.g. ``output_attention=True``). Behave differently depending on whether a `config` is provided or automatically loaded: @@ -195,7 +192,6 @@ class TFPreTrainedModel(tf.keras.Model): from_pt = kwargs.pop('from_pt', False) force_download = kwargs.pop('force_download', False) proxies = kwargs.pop('proxies', None) - output_loading_info = kwargs.pop('output_loading_info', False) # Load config if config is None: @@ -258,11 +254,4 @@ class TFPreTrainedModel(tf.keras.Model): ret = model(inputs, training=False) # Make sure restore ops are run - # if hasattr(model, 'tie_weights'): - # model.tie_weights() # TODO make sure word embedding weights are still tied - - if output_loading_info: - loading_info = {"missing_keys": missing_keys, "unexpected_keys": unexpected_keys, "error_msgs": error_msgs} - return model, loading_info - return model diff --git a/pytorch_transformers/tests/modeling_tf_auto_test.py b/pytorch_transformers/tests/modeling_tf_auto_test.py new file mode 100644 index 0000000000..816d6c1b1a --- /dev/null +++ b/pytorch_transformers/tests/modeling_tf_auto_test.py @@ -0,0 +1,85 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import unittest +import shutil +import pytest +import logging + +try: + from pytorch_transformers import (AutoConfig, BertConfig, + TFAutoModel, TFBertModel, + TFAutoModelWithLMHead, TFBertForMaskedLM, + TFAutoModelForSequenceClassification, TFBertForSequenceClassification, + TFAutoModelForQuestionAnswering, TFBertForQuestionAnswering) + from pytorch_transformers.modeling_tf_bert import TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP + + from .modeling_common_test import (CommonTestCases, ids_tensor) + from .configuration_common_test import ConfigTester +except ImportError: + pytestmark = pytest.mark.skip("Require TensorFlow") + + +class TFAutoModelTest(unittest.TestCase): + def test_model_from_pretrained(self): + logging.basicConfig(level=logging.INFO) + for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: + config = AutoConfig.from_pretrained(model_name) + self.assertIsNotNone(config) + self.assertIsInstance(config, BertConfig) + + model = TFAutoModel.from_pretrained(model_name) + self.assertIsNotNone(model) + self.assertIsInstance(model, TFBertModel) + + def test_lmhead_model_from_pretrained(self): + logging.basicConfig(level=logging.INFO) + for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: + config = AutoConfig.from_pretrained(model_name) + self.assertIsNotNone(config) + self.assertIsInstance(config, BertConfig) + + model = TFAutoModelWithLMHead.from_pretrained(model_name) + self.assertIsNotNone(model) + self.assertIsInstance(model, TFBertForMaskedLM) + + def test_sequence_classification_model_from_pretrained(self): + logging.basicConfig(level=logging.INFO) + for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: + config = AutoConfig.from_pretrained(model_name) + self.assertIsNotNone(config) + self.assertIsInstance(config, BertConfig) + + model = TFAutoModelForSequenceClassification.from_pretrained(model_name) + self.assertIsNotNone(model) + self.assertIsInstance(model, TFBertForSequenceClassification) + + def test_question_answering_model_from_pretrained(self): + logging.basicConfig(level=logging.INFO) + for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: + config = AutoConfig.from_pretrained(model_name) + self.assertIsNotNone(config) + self.assertIsInstance(config, BertConfig) + + model = TFAutoModelForQuestionAnswering.from_pretrained(model_name) + self.assertIsNotNone(model) + self.assertIsInstance(model, TFBertForQuestionAnswering) + + +if __name__ == "__main__": + unittest.main() From 1eb125fb9543bbc98086c83ad3612e9df001204d Mon Sep 17 00:00:00 2001 From: thomwolf Date: Wed, 4 Sep 2019 22:47:38 +0200 Subject: [PATCH 013/219] be sure we have uint8 --- pytorch_transformers/modeling_transfo_xl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pytorch_transformers/modeling_transfo_xl.py b/pytorch_transformers/modeling_transfo_xl.py index 63303dae5e..3fff0c552e 100644 --- a/pytorch_transformers/modeling_transfo_xl.py +++ b/pytorch_transformers/modeling_transfo_xl.py @@ -1135,7 +1135,7 @@ class TransfoXLModel(TransfoXLPreTrainedModel): mlen = mems[0].size(0) if mems is not None else 0 klen = mlen + qlen if self.same_length: - all_ones = word_emb.new_ones(qlen, klen) + all_ones = word_emb.new_ones((qlen, klen), dtype=torch.uint8) mask_len = klen - self.mem_len if mask_len > 0: mask_shift_len = qlen - mask_len @@ -1145,7 +1145,7 @@ class TransfoXLModel(TransfoXLPreTrainedModel): + torch.tril(all_ones, -mask_shift_len))[:, :, None] # -1 else: dec_attn_mask = torch.triu( - word_emb.new_ones(qlen, klen), diagonal=1+mlen)[:,:,None] + word_emb.new_ones((qlen, klen), dtype=torch.uint8), diagonal=1+mlen)[:,:,None] hids = [] attentions = [] From 1efb1f1660c7054037019667c6d6f72e0000c3a9 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 5 Sep 2019 00:26:57 +0200 Subject: [PATCH 014/219] split configuration and modeling files --- pytorch_transformers/__init__.py | 64 +++--- pytorch_transformers/configuration_auto.py | 135 ++++++++++++ pytorch_transformers/configuration_bert.py | 113 ++++++++++ .../configuration_distilbert.py | 89 ++++++++ pytorch_transformers/configuration_gpt2.py | 143 ++++++++++++ pytorch_transformers/configuration_openai.py | 135 ++++++++++++ pytorch_transformers/configuration_roberta.py | 35 +++ .../configuration_transfo_xl.py | 167 ++++++++++++++ pytorch_transformers/configuration_utils.py | 207 ++++++++++++++++++ pytorch_transformers/configuration_xlm.py | 184 ++++++++++++++++ pytorch_transformers/configuration_xlnet.py | 172 +++++++++++++++ pytorch_transformers/file_utils.py | 24 ++ pytorch_transformers/modeling_auto.py | 125 +---------- pytorch_transformers/modeling_bert.py | 93 +------- pytorch_transformers/modeling_distilbert.py | 67 +----- pytorch_transformers/modeling_gpt2.py | 132 +---------- pytorch_transformers/modeling_openai.py | 119 +--------- pytorch_transformers/modeling_roberta.py | 20 +- pytorch_transformers/modeling_transfo_xl.py | 151 +------------ pytorch_transformers/modeling_utils.py | 205 +---------------- pytorch_transformers/modeling_xlm.py | 166 +------------- pytorch_transformers/modeling_xlnet.py | 151 +------------ .../tests/configuration_common_test.py | 63 ++++++ .../tests/modeling_auto_test.py | 3 +- .../tests/modeling_bert_test.py | 3 +- .../tests/modeling_common_test.py | 6 +- .../tests/modeling_distilbert_test.py | 4 +- .../tests/modeling_gpt2_test.py | 3 +- .../tests/modeling_openai_test.py | 3 +- .../tests/modeling_roberta_test.py | 3 +- .../tests/modeling_transfo_xl_test.py | 3 +- .../tests/modeling_xlm_test.py | 3 +- .../tests/modeling_xlnet_test.py | 3 +- 33 files changed, 1571 insertions(+), 1223 deletions(-) create mode 100644 pytorch_transformers/configuration_auto.py create mode 100644 pytorch_transformers/configuration_bert.py create mode 100644 pytorch_transformers/configuration_distilbert.py create mode 100644 pytorch_transformers/configuration_gpt2.py create mode 100644 pytorch_transformers/configuration_openai.py create mode 100644 pytorch_transformers/configuration_roberta.py create mode 100644 pytorch_transformers/configuration_transfo_xl.py create mode 100644 pytorch_transformers/configuration_utils.py create mode 100644 pytorch_transformers/configuration_xlm.py create mode 100644 pytorch_transformers/configuration_xlnet.py create mode 100644 pytorch_transformers/tests/configuration_common_test.py diff --git a/pytorch_transformers/__init__.py b/pytorch_transformers/__init__.py index 3e8719bd8d..04a73c3abc 100644 --- a/pytorch_transformers/__init__.py +++ b/pytorch_transformers/__init__.py @@ -1,4 +1,7 @@ __version__ = "1.2.0" + +# Tokenizer +from .tokenization_utils import (PreTrainedTokenizer) from .tokenization_auto import AutoTokenizer from .tokenization_bert import BertTokenizer, BasicTokenizer, WordpieceTokenizer from .tokenization_openai import OpenAIGPTTokenizer @@ -9,46 +12,51 @@ from .tokenization_xlm import XLMTokenizer from .tokenization_roberta import RobertaTokenizer from .tokenization_distilbert import DistilBertTokenizer -from .tokenization_utils import (PreTrainedTokenizer) +# Configurations +from .configuration_utils import CONFIG_NAME, PretrainedConfig +from .configuration_auto import AutoConfig +from .configuration_bert import BertConfig, BERT_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_openai import OpenAIGPTConfig, OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_transfo_xl import TransfoXLConfig, TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_gpt2 import GPT2Config, GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_xlnet import XLNetConfig, XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_xlm import XLMConfig, XLM_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_roberta import RobertaConfig, ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_distilbert import DistilBertConfig, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP -from .modeling_auto import (AutoConfig, AutoModel, AutoModelForSequenceClassification, AutoModelForQuestionAnswering, +# Modeling +from .modeling_utils import (WEIGHTS_NAME, TF_WEIGHTS_NAME, PreTrainedModel, prune_layer, Conv1D) +from .modeling_auto import (AutoModel, AutoModelForSequenceClassification, AutoModelForQuestionAnswering, AutoModelWithLMHead) -from .modeling_bert import (BertConfig, BertPreTrainedModel, BertModel, BertForPreTraining, +from .modeling_bert import (BertPreTrainedModel, BertModel, BertForPreTraining, BertForMaskedLM, BertForNextSentencePrediction, BertForSequenceClassification, BertForMultipleChoice, BertForTokenClassification, BertForQuestionAnswering, - load_tf_weights_in_bert, BERT_PRETRAINED_MODEL_ARCHIVE_MAP, - BERT_PRETRAINED_CONFIG_ARCHIVE_MAP) -from .modeling_openai import (OpenAIGPTConfig, OpenAIGPTPreTrainedModel, OpenAIGPTModel, + load_tf_weights_in_bert, BERT_PRETRAINED_MODEL_ARCHIVE_MAP) +from .modeling_openai import (OpenAIGPTPreTrainedModel, OpenAIGPTModel, OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel, - load_tf_weights_in_openai_gpt, OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP, - OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP) -from .modeling_transfo_xl import (TransfoXLConfig, TransfoXLPreTrainedModel, TransfoXLModel, TransfoXLLMHeadModel, - load_tf_weights_in_transfo_xl, TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP, - TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP) -from .modeling_gpt2 import (GPT2Config, GPT2PreTrainedModel, GPT2Model, + load_tf_weights_in_openai_gpt, OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP) +from .modeling_transfo_xl import (TransfoXLPreTrainedModel, TransfoXLModel, TransfoXLLMHeadModel, + load_tf_weights_in_transfo_xl, TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP) +from .modeling_gpt2 import (GPT2PreTrainedModel, GPT2Model, GPT2LMHeadModel, GPT2DoubleHeadsModel, - load_tf_weights_in_gpt2, GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP, - GPT2_PRETRAINED_MODEL_ARCHIVE_MAP) -from .modeling_xlnet import (XLNetConfig, - XLNetPreTrainedModel, XLNetModel, XLNetLMHeadModel, + load_tf_weights_in_gpt2, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP) +from .modeling_xlnet import (XLNetPreTrainedModel, XLNetModel, XLNetLMHeadModel, XLNetForSequenceClassification, XLNetForQuestionAnswering, - load_tf_weights_in_xlnet, XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP, - XLNET_PRETRAINED_MODEL_ARCHIVE_MAP) -from .modeling_xlm import (XLMConfig, XLMPreTrainedModel , XLMModel, + load_tf_weights_in_xlnet, XLNET_PRETRAINED_MODEL_ARCHIVE_MAP) +from .modeling_xlm import (XLMPreTrainedModel , XLMModel, XLMWithLMHeadModel, XLMForSequenceClassification, - XLMForQuestionAnswering, XLM_PRETRAINED_CONFIG_ARCHIVE_MAP, - XLM_PRETRAINED_MODEL_ARCHIVE_MAP) -from .modeling_roberta import (RobertaConfig, RobertaForMaskedLM, RobertaModel, RobertaForSequenceClassification, - ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP) -from .modeling_distilbert import (DistilBertConfig, DistilBertForMaskedLM, DistilBertModel, + XLMForQuestionAnswering, XLM_PRETRAINED_MODEL_ARCHIVE_MAP) +from .modeling_roberta import (RobertaForMaskedLM, RobertaModel, RobertaForSequenceClassification, + ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP) +from .modeling_distilbert import (DistilBertForMaskedLM, DistilBertModel, DistilBertForSequenceClassification, DistilBertForQuestionAnswering, - DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP) -from .modeling_utils import (WEIGHTS_NAME, CONFIG_NAME, TF_WEIGHTS_NAME, - PretrainedConfig, PreTrainedModel, prune_layer, Conv1D) + DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP) +# Optimization from .optimization import (AdamW, ConstantLRSchedule, WarmupConstantSchedule, WarmupCosineSchedule, WarmupCosineWithHardRestartsSchedule, WarmupLinearSchedule) -from .file_utils import (PYTORCH_TRANSFORMERS_CACHE, PYTORCH_PRETRAINED_BERT_CACHE, cached_path) +# Files and general utilities +from .file_utils import (PYTORCH_TRANSFORMERS_CACHE, PYTORCH_PRETRAINED_BERT_CACHE, cached_path, add_start_docstrings, add_end_docstrings) diff --git a/pytorch_transformers/configuration_auto.py b/pytorch_transformers/configuration_auto.py new file mode 100644 index 0000000000..9e35f85dc7 --- /dev/null +++ b/pytorch_transformers/configuration_auto.py @@ -0,0 +1,135 @@ +# coding=utf-8 +# Copyright 2018 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Auto Model class. """ + +from __future__ import absolute_import, division, print_function, unicode_literals + +import logging + +from .configuration_bert import BertConfig +from .configuration_openai import OpenAIGPTConfig +from .configuration_gpt2 import GPT2Config +from .configuration_transfo_xl import TransfoXLConfig +from .configuration_xlnet import XLNetConfig +from .configuration_xlm import XLMConfig +from .configuration_roberta import RobertaConfig +from .configuration_distilbert import DistilBertConfig + +logger = logging.getLogger(__name__) + + +class AutoConfig(object): + r""":class:`~pytorch_transformers.AutoConfig` is a generic configuration class + that will be instantiated as one of the configuration classes of the library + when created with the `AutoConfig.from_pretrained(pretrained_model_name_or_path)` + class method. + + The `from_pretrained()` method take care of returning the correct model class instance + using pattern matching on the `pretrained_model_name_or_path` string. + + The base model class to instantiate is selected as the first pattern matching + in the `pretrained_model_name_or_path` string (in the following order): + - contains `distilbert`: DistilBertConfig (DistilBERT model) + - contains `bert`: BertConfig (Bert model) + - contains `openai-gpt`: OpenAIGPTConfig (OpenAI GPT model) + - contains `gpt2`: GPT2Config (OpenAI GPT-2 model) + - contains `transfo-xl`: TransfoXLConfig (Transformer-XL model) + - contains `xlnet`: XLNetConfig (XLNet model) + - contains `xlm`: XLMConfig (XLM model) + - contains `roberta`: RobertaConfig (RoBERTa model) + + This class cannot be instantiated using `__init__()` (throw an error). + """ + def __init__(self): + raise EnvironmentError("AutoConfig is designed to be instantiated " + "using the `AutoConfig.from_pretrained(pretrained_model_name_or_path)` method.") + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, **kwargs): + r""" Instantiate a one of the configuration classes of the library + from a pre-trained model configuration. + + The configuration class to instantiate is selected as the first pattern matching + in the `pretrained_model_name_or_path` string (in the following order): + - contains `distilbert`: DistilBertConfig (DistilBERT model) + - contains `bert`: BertConfig (Bert model) + - contains `openai-gpt`: OpenAIGPTConfig (OpenAI GPT model) + - contains `gpt2`: GPT2Config (OpenAI GPT-2 model) + - contains `transfo-xl`: TransfoXLConfig (Transformer-XL model) + - contains `xlnet`: XLNetConfig (XLNet model) + - contains `xlm`: XLMConfig (XLM model) + - contains `roberta`: RobertaConfig (RoBERTa model) + + Params: + pretrained_model_name_or_path: either: + + - a string with the `shortcut name` of a pre-trained model configuration to load from cache or download, e.g.: ``bert-base-uncased``. + - a path to a `directory` containing a configuration file saved using the :func:`~pytorch_transformers.PretrainedConfig.save_pretrained` method, e.g.: ``./my_model_directory/``. + - a path or url to a saved configuration JSON `file`, e.g.: ``./my_model_directory/configuration.json``. + + cache_dir: (`optional`) string: + Path to a directory in which a downloaded pre-trained model + configuration should be cached if the standard cache should not be used. + + kwargs: (`optional`) dict: key/value pairs with which to update the configuration object after loading. + + - The values in kwargs of any keys which are configuration attributes will be used to override the loaded values. + - Behavior concerning key/value pairs whose keys are *not* configuration attributes is controlled by the `return_unused_kwargs` keyword parameter. + + force_download: (`optional`) boolean, default False: + Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + + proxies: (`optional`) dict, default None: + A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. + The proxies are used on each request. + + return_unused_kwargs: (`optional`) bool: + + - If False, then this function returns just the final configuration object. + - If True, then this functions returns a tuple `(config, unused_kwargs)` where `unused_kwargs` is a dictionary consisting of the key/value pairs whose keys are not configuration attributes: ie the part of kwargs which has not been used to update `config` and is otherwise ignored. + + Examples:: + + config = AutoConfig.from_pretrained('bert-base-uncased') # Download configuration from S3 and cache. + config = AutoConfig.from_pretrained('./test/bert_saved_model/') # E.g. config (or model) was saved using `save_pretrained('./test/saved_model/')` + config = AutoConfig.from_pretrained('./test/bert_saved_model/my_configuration.json') + config = AutoConfig.from_pretrained('bert-base-uncased', output_attention=True, foo=False) + assert config.output_attention == True + config, unused_kwargs = AutoConfig.from_pretrained('bert-base-uncased', output_attention=True, + foo=False, return_unused_kwargs=True) + assert config.output_attention == True + assert unused_kwargs == {'foo': False} + + """ + if 'distilbert' in pretrained_model_name_or_path: + return DistilBertConfig.from_pretrained(pretrained_model_name_or_path, **kwargs) + elif 'roberta' in pretrained_model_name_or_path: + return RobertaConfig.from_pretrained(pretrained_model_name_or_path, **kwargs) + elif 'bert' in pretrained_model_name_or_path: + return BertConfig.from_pretrained(pretrained_model_name_or_path, **kwargs) + elif 'openai-gpt' in pretrained_model_name_or_path: + return OpenAIGPTConfig.from_pretrained(pretrained_model_name_or_path, **kwargs) + elif 'gpt2' in pretrained_model_name_or_path: + return GPT2Config.from_pretrained(pretrained_model_name_or_path, **kwargs) + elif 'transfo-xl' in pretrained_model_name_or_path: + return TransfoXLConfig.from_pretrained(pretrained_model_name_or_path, **kwargs) + elif 'xlnet' in pretrained_model_name_or_path: + return XLNetConfig.from_pretrained(pretrained_model_name_or_path, **kwargs) + elif 'xlm' in pretrained_model_name_or_path: + return XLMConfig.from_pretrained(pretrained_model_name_or_path, **kwargs) + + raise ValueError("Unrecognized model identifier in {}. Should contains one of " + "'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', " + "'xlm', 'roberta'".format(pretrained_model_name_or_path)) diff --git a/pytorch_transformers/configuration_bert.py b/pytorch_transformers/configuration_bert.py new file mode 100644 index 0000000000..7fff3e5d05 --- /dev/null +++ b/pytorch_transformers/configuration_bert.py @@ -0,0 +1,113 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" BERT model configuration """ + +from __future__ import absolute_import, division, print_function, unicode_literals + +import json +import logging +import sys +from io import open + +from .configuration_utils import PretrainedConfig + +logger = logging.getLogger(__name__) + +BERT_PRETRAINED_CONFIG_ARCHIVE_MAP = { + 'bert-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-config.json", + 'bert-large-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-config.json", + 'bert-base-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-config.json", + 'bert-large-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-config.json", + 'bert-base-multilingual-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-uncased-config.json", + 'bert-base-multilingual-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-cased-config.json", + 'bert-base-chinese': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-chinese-config.json", + 'bert-base-german-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-german-cased-config.json", + 'bert-large-uncased-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-whole-word-masking-config.json", + 'bert-large-cased-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-whole-word-masking-config.json", + 'bert-large-uncased-whole-word-masking-finetuned-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-whole-word-masking-finetuned-squad-config.json", + 'bert-large-cased-whole-word-masking-finetuned-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-whole-word-masking-finetuned-squad-config.json", + 'bert-base-cased-finetuned-mrpc': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-finetuned-mrpc-config.json", +} + + +class BertConfig(PretrainedConfig): + r""" + :class:`~pytorch_transformers.BertConfig` is the configuration class to store the configuration of a + `BertModel`. + + + Arguments: + vocab_size_or_config_json_file: Vocabulary size of `inputs_ids` in `BertModel`. + hidden_size: Size of the encoder layers and the pooler layer. + num_hidden_layers: Number of hidden layers in the Transformer encoder. + num_attention_heads: Number of attention heads for each attention layer in + the Transformer encoder. + intermediate_size: The size of the "intermediate" (i.e., feed-forward) + layer in the Transformer encoder. + hidden_act: The non-linear activation function (function or string) in the + encoder and pooler. If string, "gelu", "relu" and "swish" are supported. + hidden_dropout_prob: The dropout probabilitiy for all fully connected + layers in the embeddings, encoder, and pooler. + attention_probs_dropout_prob: The dropout ratio for the attention + probabilities. + max_position_embeddings: The maximum sequence length that this model might + ever be used with. Typically set this to something large just in case + (e.g., 512 or 1024 or 2048). + type_vocab_size: The vocabulary size of the `token_type_ids` passed into + `BertModel`. + initializer_range: The sttdev of the truncated_normal_initializer for + initializing all weight matrices. + layer_norm_eps: The epsilon used by LayerNorm. + """ + pretrained_config_archive_map = BERT_PRETRAINED_CONFIG_ARCHIVE_MAP + + def __init__(self, + vocab_size_or_config_json_file=30522, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=2, + initializer_range=0.02, + layer_norm_eps=1e-12, + **kwargs): + super(BertConfig, self).__init__(**kwargs) + if isinstance(vocab_size_or_config_json_file, str) or (sys.version_info[0] == 2 + and isinstance(vocab_size_or_config_json_file, unicode)): + with open(vocab_size_or_config_json_file, "r", encoding='utf-8') as reader: + json_config = json.loads(reader.read()) + for key, value in json_config.items(): + self.__dict__[key] = value + elif isinstance(vocab_size_or_config_json_file, int): + self.vocab_size = vocab_size_or_config_json_file + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.hidden_act = hidden_act + self.intermediate_size = intermediate_size + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.initializer_range = initializer_range + self.layer_norm_eps = layer_norm_eps + else: + raise ValueError("First argument must be either a vocabulary size (int)" + " or the path to a pretrained model config file (str)") diff --git a/pytorch_transformers/configuration_distilbert.py b/pytorch_transformers/configuration_distilbert.py new file mode 100644 index 0000000000..b8929eedec --- /dev/null +++ b/pytorch_transformers/configuration_distilbert.py @@ -0,0 +1,89 @@ +# coding=utf-8 +# Copyright 2019-present, the HuggingFace Inc. team, The Google AI Language Team and Facebook, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" DistilBERT model configuration """ +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +import sys +import json +import logging +from io import open + +from .configuration_utils import PretrainedConfig + +logger = logging.getLogger(__name__) + +DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = { + 'distilbert-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-config.json", + 'distilbert-base-uncased-distilled-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-config.json" +} + + +class DistilBertConfig(PretrainedConfig): + pretrained_config_archive_map = DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP + + def __init__(self, + vocab_size_or_config_json_file=30522, + max_position_embeddings=512, + sinusoidal_pos_embds=True, + n_layers=6, + n_heads=12, + dim=768, + hidden_dim=4*768, + dropout=0.1, + attention_dropout=0.1, + activation='gelu', + initializer_range=0.02, + tie_weights_=True, + qa_dropout=0.1, + seq_classif_dropout=0.2, + **kwargs): + super(DistilBertConfig, self).__init__(**kwargs) + + if isinstance(vocab_size_or_config_json_file, str) or (sys.version_info[0] == 2 + and isinstance(vocab_size_or_config_json_file, unicode)): + with open(vocab_size_or_config_json_file, "r", encoding='utf-8') as reader: + json_config = json.loads(reader.read()) + for key, value in json_config.items(): + self.__dict__[key] = value + elif isinstance(vocab_size_or_config_json_file, int): + self.vocab_size = vocab_size_or_config_json_file + self.max_position_embeddings = max_position_embeddings + self.sinusoidal_pos_embds = sinusoidal_pos_embds + self.n_layers = n_layers + self.n_heads = n_heads + self.dim = dim + self.hidden_dim = hidden_dim + self.dropout = dropout + self.attention_dropout = attention_dropout + self.activation = activation + self.initializer_range = initializer_range + self.tie_weights_ = tie_weights_ + self.qa_dropout = qa_dropout + self.seq_classif_dropout = seq_classif_dropout + else: + raise ValueError("First argument must be either a vocabulary size (int)" + " or the path to a pretrained model config file (str)") + @property + def hidden_size(self): + return self.dim + + @property + def num_attention_heads(self): + return self.n_heads + + @property + def num_hidden_layers(self): + return self.n_layers diff --git a/pytorch_transformers/configuration_gpt2.py b/pytorch_transformers/configuration_gpt2.py new file mode 100644 index 0000000000..c83d9e82ce --- /dev/null +++ b/pytorch_transformers/configuration_gpt2.py @@ -0,0 +1,143 @@ +# coding=utf-8 +# Copyright 2018 The OpenAI Team Authors and HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" OpenAI GPT-2 configuration """ + +from __future__ import absolute_import, division, print_function, unicode_literals + +import json +import logging +import sys +from io import open + +from .configuration_utils import PretrainedConfig + +logger = logging.getLogger(__name__) + +GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP = {"gpt2": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-config.json", + "gpt2-medium": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-medium-config.json", + "gpt2-large": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-large-config.json"} + +class GPT2Config(PretrainedConfig): + """Configuration class to store the configuration of a `GPT2Model`. + + Args: + vocab_size_or_config_json_file: Vocabulary size of `inputs_ids` in `GPT2Model` or a configuration json file. + n_positions: Number of positional embeddings. + n_ctx: Size of the causal mask (usually same as n_positions). + n_embd: Dimensionality of the embeddings and hidden states. + n_layer: Number of hidden layers in the Transformer encoder. + n_head: Number of attention heads for each attention layer in + the Transformer encoder. + layer_norm_epsilon: epsilon to use in the layer norm layers + resid_pdrop: The dropout probabilitiy for all fully connected + layers in the embeddings, encoder, and pooler. + attn_pdrop: The dropout ratio for the attention + probabilities. + embd_pdrop: The dropout ratio for the embeddings. + initializer_range: The sttdev of the truncated_normal_initializer for + initializing all weight matrices. + """ + pretrained_config_archive_map = GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP + + def __init__( + self, + vocab_size_or_config_json_file=50257, + n_positions=1024, + n_ctx=1024, + n_embd=768, + n_layer=12, + n_head=12, + resid_pdrop=0.1, + embd_pdrop=0.1, + attn_pdrop=0.1, + layer_norm_epsilon=1e-5, + initializer_range=0.02, + + num_labels=1, + summary_type='cls_index', + summary_use_proj=True, + summary_activation=None, + summary_proj_to_labels=True, + summary_first_dropout=0.1, + **kwargs + ): + """Constructs GPT2Config. + + Args: + vocab_size_or_config_json_file: Vocabulary size of `inputs_ids` in `GPT2Model` or a configuration json file. + n_positions: Number of positional embeddings. + n_ctx: Size of the causal mask (usually same as n_positions). + n_embd: Dimensionality of the embeddings and hidden states. + n_layer: Number of hidden layers in the Transformer encoder. + n_head: Number of attention heads for each attention layer in + the Transformer encoder. + layer_norm_epsilon: epsilon to use in the layer norm layers + resid_pdrop: The dropout probabilitiy for all fully connected + layers in the embeddings, encoder, and pooler. + attn_pdrop: The dropout ratio for the attention + probabilities. + embd_pdrop: The dropout ratio for the embeddings. + initializer_range: The sttdev of the truncated_normal_initializer for + initializing all weight matrices. + """ + super(GPT2Config, self).__init__(**kwargs) + + if isinstance(vocab_size_or_config_json_file, str) or (sys.version_info[0] == 2 + and isinstance(vocab_size_or_config_json_file, unicode)): + with open(vocab_size_or_config_json_file, "r", encoding="utf-8") as reader: + json_config = json.loads(reader.read()) + for key, value in json_config.items(): + self.__dict__[key] = value + elif isinstance(vocab_size_or_config_json_file, int): + self.vocab_size = vocab_size_or_config_json_file + self.n_ctx = n_ctx + self.n_positions = n_positions + self.n_embd = n_embd + self.n_layer = n_layer + self.n_head = n_head + self.resid_pdrop = resid_pdrop + self.embd_pdrop = embd_pdrop + self.attn_pdrop = attn_pdrop + self.layer_norm_epsilon = layer_norm_epsilon + self.initializer_range = initializer_range + + self.num_labels = num_labels + self.summary_type = summary_type + self.summary_use_proj = summary_use_proj + self.summary_activation = summary_activation + self.summary_first_dropout = summary_first_dropout + self.summary_proj_to_labels = summary_proj_to_labels + else: + raise ValueError( + "First argument must be either a vocabulary size (int)" + "or the path to a pretrained model config file (str)" + ) + + @property + def max_position_embeddings(self): + return self.n_positions + + @property + def hidden_size(self): + return self.n_embd + + @property + def num_attention_heads(self): + return self.n_head + + @property + def num_hidden_layers(self): + return self.n_layer diff --git a/pytorch_transformers/configuration_openai.py b/pytorch_transformers/configuration_openai.py new file mode 100644 index 0000000000..b27df56899 --- /dev/null +++ b/pytorch_transformers/configuration_openai.py @@ -0,0 +1,135 @@ +# coding=utf-8 +# Copyright 2018 The OpenAI Team Authors and HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" OpenAI GPT configuration """ + +from __future__ import absolute_import, division, print_function, unicode_literals + +import json +import logging +import sys +from io import open + +from .configuration_utils import PretrainedConfig + +logger = logging.getLogger(__name__) + +OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP = { + "openai-gpt": "https://s3.amazonaws.com/models.huggingface.co/bert/openai-gpt-config.json" +} + +class OpenAIGPTConfig(PretrainedConfig): + """ + Configuration class to store the configuration of a `OpenAIGPTModel`. + + Args: + vocab_size_or_config_json_file: Vocabulary size of `inputs_ids` in `OpenAIGPTModel` or a configuration json file. + n_special: The number of special tokens to learn during fine-tuning ('[SEP]', '[CLF]', ...) + n_positions: Number of positional embeddings. + n_ctx: Size of the causal mask (usually same as n_positions). + n_embd: Dimensionality of the embeddings and hidden states. + n_layer: Number of hidden layers in the Transformer encoder. + n_head: Number of attention heads for each attention layer in + the Transformer encoder. + afn: The non-linear activation function (function or string) in the + encoder and pooler. If string, "gelu", "relu" and "swish" are supported. + resid_pdrop: The dropout probabilitiy for all fully connected + layers in the embeddings, encoder, and pooler. + attn_pdrop: The dropout ratio for the attention + probabilities. + embd_pdrop: The dropout ratio for the embeddings. + layer_norm_epsilon: epsilon to use in the layer norm layers + initializer_range: The sttdev of the truncated_normal_initializer for + initializing all weight matrices. + predict_special_tokens: should we predict special tokens (when the model has a LM head) + """ + pretrained_config_archive_map = OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP + + def __init__( + self, + vocab_size_or_config_json_file=40478, + n_positions=512, + n_ctx=512, + n_embd=768, + n_layer=12, + n_head=12, + afn="gelu", + resid_pdrop=0.1, + embd_pdrop=0.1, + attn_pdrop=0.1, + layer_norm_epsilon=1e-5, + initializer_range=0.02, + predict_special_tokens=True, + + num_labels=1, + summary_type='cls_index', + summary_use_proj=True, + summary_activation=None, + summary_proj_to_labels=True, + summary_first_dropout=0.1, + **kwargs + ): + """Constructs OpenAIGPTConfig. + """ + super(OpenAIGPTConfig, self).__init__(**kwargs) + + if isinstance(vocab_size_or_config_json_file, str) or (sys.version_info[0] == 2 + and isinstance(vocab_size_or_config_json_file, unicode)): + with open(vocab_size_or_config_json_file, "r", encoding="utf-8") as reader: + json_config = json.loads(reader.read()) + for key, value in json_config.items(): + self.__dict__[key] = value + elif isinstance(vocab_size_or_config_json_file, int): + self.vocab_size = vocab_size_or_config_json_file + self.n_ctx = n_ctx + self.n_positions = n_positions + self.n_embd = n_embd + self.n_layer = n_layer + self.n_head = n_head + self.afn = afn + self.resid_pdrop = resid_pdrop + self.embd_pdrop = embd_pdrop + self.attn_pdrop = attn_pdrop + self.layer_norm_epsilon = layer_norm_epsilon + self.initializer_range = initializer_range + self.predict_special_tokens = predict_special_tokens + + self.num_labels = num_labels + self.summary_type = summary_type + self.summary_use_proj = summary_use_proj + self.summary_activation = summary_activation + self.summary_first_dropout = summary_first_dropout + self.summary_proj_to_labels = summary_proj_to_labels + else: + raise ValueError( + "First argument must be either a vocabulary size (int)" + "or the path to a pretrained model config file (str)" + ) + + @property + def max_position_embeddings(self): + return self.n_positions + + @property + def hidden_size(self): + return self.n_embd + + @property + def num_attention_heads(self): + return self.n_head + + @property + def num_hidden_layers(self): + return self.n_layer diff --git a/pytorch_transformers/configuration_roberta.py b/pytorch_transformers/configuration_roberta.py new file mode 100644 index 0000000000..b92d6a908b --- /dev/null +++ b/pytorch_transformers/configuration_roberta.py @@ -0,0 +1,35 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" RoBERTa configuration """ + +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +import logging + +from .configuration_bert import BertConfig + +logger = logging.getLogger(__name__) + +ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP = { + 'roberta-base': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-config.json", + 'roberta-large': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-config.json", + 'roberta-large-mnli': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-mnli-config.json", +} + + +class RobertaConfig(BertConfig): + pretrained_config_archive_map = ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP diff --git a/pytorch_transformers/configuration_transfo_xl.py b/pytorch_transformers/configuration_transfo_xl.py new file mode 100644 index 0000000000..2e966ee55c --- /dev/null +++ b/pytorch_transformers/configuration_transfo_xl.py @@ -0,0 +1,167 @@ +# coding=utf-8 +# Copyright 2018 Google AI, Google Brain and Carnegie Mellon University Authors and the HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Transformer XL configuration """ + +from __future__ import absolute_import, division, print_function, unicode_literals + +import json +import logging +import sys +from io import open + +from .configuration_utils import PretrainedConfig + +logger = logging.getLogger(__name__) + +TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP = { + 'transfo-xl-wt103': "https://s3.amazonaws.com/models.huggingface.co/bert/transfo-xl-wt103-config.json", +} + +class TransfoXLConfig(PretrainedConfig): + """Configuration class to store the configuration of a `TransfoXLModel`. + + Args: + vocab_size_or_config_json_file: Vocabulary size of `inputs_ids` in `TransfoXLModel` or a configuration json file. + cutoffs: cutoffs for the adaptive softmax + d_model: Dimensionality of the model's hidden states. + d_embed: Dimensionality of the embeddings + d_head: Dimensionality of the model's heads. + div_val: divident value for adapative input and softmax + pre_lnorm: apply LayerNorm to the input instead of the output + d_inner: Inner dimension in FF + n_layer: Number of hidden layers in the Transformer encoder. + n_head: Number of attention heads for each attention layer in + the Transformer encoder. + tgt_len: number of tokens to predict + ext_len: length of the extended context + mem_len: length of the retained previous heads + same_length: use the same attn length for all tokens + proj_share_all_but_first: True to share all but first projs, False not to share. + attn_type: attention type. 0 for Transformer-XL, 1 for Shaw et al, 2 for Vaswani et al, 3 for Al Rfou et al. + clamp_len: use the same pos embeddings after clamp_len + sample_softmax: number of samples in sampled softmax + adaptive: use adaptive softmax + tie_weight: tie the word embedding and softmax weights + dropout: The dropout probabilitiy for all fully connected + layers in the embeddings, encoder, and pooler. + dropatt: The dropout ratio for the attention probabilities. + untie_r: untie relative position biases + embd_pdrop: The dropout ratio for the embeddings. + init: parameter initializer to use + init_range: parameters initialized by U(-init_range, init_range). + proj_init_std: parameters initialized by N(0, init_std) + init_std: parameters initialized by N(0, init_std) + """ + pretrained_config_archive_map = TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP + + def __init__(self, + vocab_size_or_config_json_file=267735, + cutoffs=[20000, 40000, 200000], + d_model=1024, + d_embed=1024, + n_head=16, + d_head=64, + d_inner=4096, + div_val=4, + pre_lnorm=False, + n_layer=18, + tgt_len=128, + ext_len=0, + mem_len=1600, + clamp_len=1000, + same_length=True, + proj_share_all_but_first=True, + attn_type=0, + sample_softmax=-1, + adaptive=True, + tie_weight=True, + dropout=0.1, + dropatt=0.0, + untie_r=True, + init="normal", + init_range=0.01, + proj_init_std=0.01, + init_std=0.02, + **kwargs): + """Constructs TransfoXLConfig. + """ + super(TransfoXLConfig, self).__init__(**kwargs) + + if isinstance(vocab_size_or_config_json_file, str) or (sys.version_info[0] == 2 + and isinstance(vocab_size_or_config_json_file, unicode)): + with open(vocab_size_or_config_json_file, "r", encoding='utf-8') as reader: + json_config = json.loads(reader.read()) + for key, value in json_config.items(): + self.__dict__[key] = value + elif isinstance(vocab_size_or_config_json_file, int): + self.n_token = vocab_size_or_config_json_file + self.cutoffs = [] + self.cutoffs.extend(cutoffs) + self.tie_weight = tie_weight + if proj_share_all_but_first: + self.tie_projs = [False] + [True] * len(self.cutoffs) + else: + self.tie_projs = [False] + [False] * len(self.cutoffs) + self.d_model = d_model + self.d_embed = d_embed + self.d_head = d_head + self.d_inner = d_inner + self.div_val = div_val + self.pre_lnorm = pre_lnorm + self.n_layer = n_layer + self.n_head = n_head + self.tgt_len = tgt_len + self.ext_len = ext_len + self.mem_len = mem_len + self.same_length = same_length + self.attn_type = attn_type + self.clamp_len = clamp_len + self.sample_softmax = sample_softmax + self.adaptive = adaptive + self.dropout = dropout + self.dropatt = dropatt + self.untie_r = untie_r + self.init = init + self.init_range = init_range + self.proj_init_std = proj_init_std + self.init_std = init_std + else: + raise ValueError("First argument must be either a vocabulary size (int)" + " or the path to a pretrained model config file (str)") + + @property + def max_position_embeddings(self): + return self.tgt_len + self.ext_len + self.mem_len + + @property + def vocab_size(self): + return self.n_token + + @vocab_size.setter + def vocab_size(self, value): + self.n_token = value + + @property + def hidden_size(self): + return self.d_model + + @property + def num_attention_heads(self): + return self.n_head + + @property + def num_hidden_layers(self): + return self.n_layer diff --git a/pytorch_transformers/configuration_utils.py b/pytorch_transformers/configuration_utils.py new file mode 100644 index 0000000000..550b47fab8 --- /dev/null +++ b/pytorch_transformers/configuration_utils.py @@ -0,0 +1,207 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Configuration base class and utilities.""" + +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +import copy +import json +import logging +import os +from io import open + +from .file_utils import cached_path + +logger = logging.getLogger(__name__) + +CONFIG_NAME = "config.json" + +class PretrainedConfig(object): + r""" Base class for all configuration classes. + Handles a few parameters common to all models' configurations as well as methods for loading/downloading/saving configurations. + + Note: + A configuration file can be loaded and saved to disk. Loading the configuration file and using this file to initialize a model does **not** load the model weights. + It only affects the model's configuration. + + Class attributes (overridden by derived classes): + - ``pretrained_config_archive_map``: a python ``dict`` of with `short-cut-names` (string) as keys and `url` (string) of associated pretrained model configurations as values. + + Parameters: + ``finetuning_task``: string, default `None`. Name of the task used to fine-tune the model. This can be used when converting from an original (TensorFlow or PyTorch) checkpoint. + ``num_labels``: integer, default `2`. Number of classes to use when the model is a classification model (sequences/tokens) + ``output_attentions``: boolean, default `False`. Should the model returns attentions weights. + ``output_hidden_states``: string, default `False`. Should the model returns all hidden-states. + ``torchscript``: string, default `False`. Is the model used with Torchscript. + """ + pretrained_config_archive_map = {} + + def __init__(self, **kwargs): + self.finetuning_task = kwargs.pop('finetuning_task', None) + self.num_labels = kwargs.pop('num_labels', 2) + self.output_attentions = kwargs.pop('output_attentions', False) + self.output_hidden_states = kwargs.pop('output_hidden_states', False) + self.torchscript = kwargs.pop('torchscript', False) + self.pruned_heads = kwargs.pop('pruned_heads', {}) + + def save_pretrained(self, save_directory): + """ Save a configuration object to the directory `save_directory`, so that it + can be re-loaded using the :func:`~pytorch_transformers.PretrainedConfig.from_pretrained` class method. + """ + assert os.path.isdir(save_directory), "Saving path should be a directory where the model and configuration can be saved" + + # If we save using the predefined names, we can load using `from_pretrained` + output_config_file = os.path.join(save_directory, CONFIG_NAME) + + self.to_json_file(output_config_file) + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, **kwargs): + r""" Instantiate a :class:`~pytorch_transformers.PretrainedConfig` (or a derived class) from a pre-trained model configuration. + + Parameters: + pretrained_model_name_or_path: either: + + - a string with the `shortcut name` of a pre-trained model configuration to load from cache or download, e.g.: ``bert-base-uncased``. + - a path to a `directory` containing a configuration file saved using the :func:`~pytorch_transformers.PretrainedConfig.save_pretrained` method, e.g.: ``./my_model_directory/``. + - a path or url to a saved configuration JSON `file`, e.g.: ``./my_model_directory/configuration.json``. + + cache_dir: (`optional`) string: + Path to a directory in which a downloaded pre-trained model + configuration should be cached if the standard cache should not be used. + + kwargs: (`optional`) dict: key/value pairs with which to update the configuration object after loading. + + - The values in kwargs of any keys which are configuration attributes will be used to override the loaded values. + - Behavior concerning key/value pairs whose keys are *not* configuration attributes is controlled by the `return_unused_kwargs` keyword parameter. + + force_download: (`optional`) boolean, default False: + Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + + proxies: (`optional`) dict, default None: + A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. + The proxies are used on each request. + + return_unused_kwargs: (`optional`) bool: + + - If False, then this function returns just the final configuration object. + - If True, then this functions returns a tuple `(config, unused_kwargs)` where `unused_kwargs` is a dictionary consisting of the key/value pairs whose keys are not configuration attributes: ie the part of kwargs which has not been used to update `config` and is otherwise ignored. + + Examples:: + + # We can't instantiate directly the base class `PretrainedConfig` so let's show the examples on a + # derived class: BertConfig + config = BertConfig.from_pretrained('bert-base-uncased') # Download configuration from S3 and cache. + config = BertConfig.from_pretrained('./test/saved_model/') # E.g. config (or model) was saved using `save_pretrained('./test/saved_model/')` + config = BertConfig.from_pretrained('./test/saved_model/my_configuration.json') + config = BertConfig.from_pretrained('bert-base-uncased', output_attention=True, foo=False) + assert config.output_attention == True + config, unused_kwargs = BertConfig.from_pretrained('bert-base-uncased', output_attention=True, + foo=False, return_unused_kwargs=True) + assert config.output_attention == True + assert unused_kwargs == {'foo': False} + + """ + cache_dir = kwargs.pop('cache_dir', None) + force_download = kwargs.pop('force_download', False) + proxies = kwargs.pop('proxies', None) + return_unused_kwargs = kwargs.pop('return_unused_kwargs', False) + + if pretrained_model_name_or_path in cls.pretrained_config_archive_map: + config_file = cls.pretrained_config_archive_map[pretrained_model_name_or_path] + elif os.path.isdir(pretrained_model_name_or_path): + config_file = os.path.join(pretrained_model_name_or_path, CONFIG_NAME) + else: + config_file = pretrained_model_name_or_path + # redirect to the cache, if necessary + try: + resolved_config_file = cached_path(config_file, cache_dir=cache_dir, force_download=force_download, proxies=proxies) + except EnvironmentError as e: + if pretrained_model_name_or_path in cls.pretrained_config_archive_map: + logger.error( + "Couldn't reach server at '{}' to download pretrained model configuration file.".format( + config_file)) + else: + logger.error( + "Model name '{}' was not found in model name list ({}). " + "We assumed '{}' was a path or url but couldn't find any file " + "associated to this path or url.".format( + pretrained_model_name_or_path, + ', '.join(cls.pretrained_config_archive_map.keys()), + config_file)) + raise e + if resolved_config_file == config_file: + logger.info("loading configuration file {}".format(config_file)) + else: + logger.info("loading configuration file {} from cache at {}".format( + config_file, resolved_config_file)) + + # Load config + config = cls.from_json_file(resolved_config_file) + + if hasattr(config, 'pruned_heads'): + config.pruned_heads = dict((int(key), set(value)) for key, value in config.pruned_heads.items()) + + # Update config with kwargs if needed + to_remove = [] + for key, value in kwargs.items(): + if hasattr(config, key): + setattr(config, key, value) + to_remove.append(key) + for key in to_remove: + kwargs.pop(key, None) + + logger.info("Model config %s", config) + if return_unused_kwargs: + return config, kwargs + else: + return config + + @classmethod + def from_dict(cls, json_object): + """Constructs a `Config` from a Python dictionary of parameters.""" + config = cls(vocab_size_or_config_json_file=-1) + for key, value in json_object.items(): + config.__dict__[key] = value + return config + + @classmethod + def from_json_file(cls, json_file): + """Constructs a `BertConfig` from a json file of parameters.""" + with open(json_file, "r", encoding='utf-8') as reader: + text = reader.read() + return cls.from_dict(json.loads(text)) + + def __eq__(self, other): + return self.__dict__ == other.__dict__ + + def __repr__(self): + return str(self.to_json_string()) + + def to_dict(self): + """Serializes this instance to a Python dictionary.""" + output = copy.deepcopy(self.__dict__) + return output + + def to_json_string(self): + """Serializes this instance to a JSON string.""" + return json.dumps(self.to_dict(), indent=2, sort_keys=True) + "\n" + + def to_json_file(self, json_file_path): + """ Save this instance to a json file.""" + with open(json_file_path, "w", encoding='utf-8') as writer: + writer.write(self.to_json_string()) diff --git a/pytorch_transformers/configuration_xlm.py b/pytorch_transformers/configuration_xlm.py new file mode 100644 index 0000000000..ab251c8939 --- /dev/null +++ b/pytorch_transformers/configuration_xlm.py @@ -0,0 +1,184 @@ +# coding=utf-8 +# Copyright 2019-present, Facebook, Inc and the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" XLM configuration """ +from __future__ import absolute_import, division, print_function, unicode_literals + +import json +import logging +import sys +from io import open + +from .configuration_utils import PretrainedConfig + +logger = logging.getLogger(__name__) + +XLM_PRETRAINED_CONFIG_ARCHIVE_MAP = { + 'xlm-mlm-en-2048': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-mlm-en-2048-config.json", + 'xlm-mlm-ende-1024': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-mlm-ende-1024-config.json", + 'xlm-mlm-enfr-1024': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-mlm-enfr-1024-config.json", + 'xlm-mlm-enro-1024': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-mlm-enro-1024-config.json", + 'xlm-mlm-tlm-xnli15-1024': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-mlm-tlm-xnli15-1024-config.json", + 'xlm-mlm-xnli15-1024': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-mlm-xnli15-1024-config.json", + 'xlm-clm-enfr-1024': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-clm-enfr-1024-config.json", + 'xlm-clm-ende-1024': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-clm-ende-1024-config.json", + 'xlm-mlm-17-1280': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-mlm-17-1280-config.json", + 'xlm-mlm-100-1280': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-mlm-100-1280-config.json", +} + + +class XLMConfig(PretrainedConfig): + """Configuration class to store the configuration of a `XLMModel`. + + Args: + vocab_size_or_config_json_file: Vocabulary size of `inputs_ids` in `XLMModel`. + d_model: Size of the encoder layers and the pooler layer. + n_layer: Number of hidden layers in the Transformer encoder. + n_head: Number of attention heads for each attention layer in + the Transformer encoder. + d_inner: The size of the "intermediate" (i.e., feed-forward) + layer in the Transformer encoder. + ff_activation: The non-linear activation function (function or string) in the + encoder and pooler. If string, "gelu", "relu" and "swish" are supported. + untie_r: untie relative position biases + attn_type: 'bi' for XLM, 'uni' for Transformer-XL + + dropout: The dropout probabilitiy for all fully connected + layers in the embeddings, encoder, and pooler. + dropatt: The dropout ratio for the attention + probabilities. + max_position_embeddings: The maximum sequence length that this model might + ever be used with. Typically set this to something large just in case + (e.g., 512 or 1024 or 2048). + initializer_range: The sttdev of the truncated_normal_initializer for + initializing all weight matrices. + layer_norm_eps: The epsilon used by LayerNorm. + + dropout: float, dropout rate. + dropatt: float, dropout rate on attention probabilities. + init: str, the initialization scheme, either "normal" or "uniform". + init_range: float, initialize the parameters with a uniform distribution + in [-init_range, init_range]. Only effective when init="uniform". + init_std: float, initialize the parameters with a normal distribution + with mean 0 and stddev init_std. Only effective when init="normal". + mem_len: int, the number of tokens to cache. + reuse_len: int, the number of tokens in the currect batch to be cached + and reused in the future. + bi_data: bool, whether to use bidirectional input pipeline. + Usually set to True during pretraining and False during finetuning. + clamp_len: int, clamp all relative distances larger than clamp_len. + -1 means no clamping. + same_length: bool, whether to use the same attention length for each token. + """ + pretrained_config_archive_map = XLM_PRETRAINED_CONFIG_ARCHIVE_MAP + + def __init__(self, + vocab_size_or_config_json_file=30145, + emb_dim=2048, + n_layers=12, + n_heads=16, + dropout=0.1, + attention_dropout=0.1, + gelu_activation=True, + sinusoidal_embeddings=False, + causal=False, + asm=False, + n_langs=1, + use_lang_emb=True, + max_position_embeddings=512, + embed_init_std=2048 ** -0.5, + layer_norm_eps=1e-12, + init_std=0.02, + bos_index=0, + eos_index=1, + pad_index=2, + unk_index=3, + mask_index=5, + is_encoder=True, + + finetuning_task=None, + num_labels=2, + summary_type='first', + summary_use_proj=True, + summary_activation=None, + summary_proj_to_labels=True, + summary_first_dropout=0.1, + start_n_top=5, + end_n_top=5, + **kwargs): + """Constructs XLMConfig. + """ + super(XLMConfig, self).__init__(**kwargs) + + if isinstance(vocab_size_or_config_json_file, str) or (sys.version_info[0] == 2 + and isinstance(vocab_size_or_config_json_file, unicode)): + with open(vocab_size_or_config_json_file, "r", encoding='utf-8') as reader: + json_config = json.loads(reader.read()) + for key, value in json_config.items(): + self.__dict__[key] = value + elif isinstance(vocab_size_or_config_json_file, int): + self.n_words = vocab_size_or_config_json_file + self.emb_dim = emb_dim + self.n_layers = n_layers + self.n_heads = n_heads + self.dropout = dropout + self.attention_dropout = attention_dropout + self.gelu_activation = gelu_activation + self.sinusoidal_embeddings = sinusoidal_embeddings + self.causal = causal + self.asm = asm + self.n_langs = n_langs + self.use_lang_emb = use_lang_emb + self.layer_norm_eps = layer_norm_eps + self.bos_index = bos_index + self.eos_index = eos_index + self.pad_index = pad_index + self.unk_index = unk_index + self.mask_index = mask_index + self.is_encoder = is_encoder + self.max_position_embeddings = max_position_embeddings + self.embed_init_std = embed_init_std + self.init_std = init_std + self.finetuning_task = finetuning_task + self.num_labels = num_labels + self.summary_type = summary_type + self.summary_use_proj = summary_use_proj + self.summary_activation = summary_activation + self.summary_proj_to_labels = summary_proj_to_labels + self.summary_first_dropout = summary_first_dropout + self.start_n_top = start_n_top + self.end_n_top = end_n_top + else: + raise ValueError("First argument must be either a vocabulary size (int)" + " or the path to a pretrained model config file (str)") + + @property + def vocab_size(self): + return self.n_words + + @vocab_size.setter + def vocab_size(self, value): + self.n_words = value + + @property + def hidden_size(self): + return self.emb_dim + + @property + def num_attention_heads(self): + return self.n_heads + + @property + def num_hidden_layers(self): + return self.n_layers diff --git a/pytorch_transformers/configuration_xlnet.py b/pytorch_transformers/configuration_xlnet.py new file mode 100644 index 0000000000..204d44aa72 --- /dev/null +++ b/pytorch_transformers/configuration_xlnet.py @@ -0,0 +1,172 @@ +# coding=utf-8 +# Copyright 2018 Google AI, Google Brain and Carnegie Mellon University Authors and the HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" XLNet configuration """ +from __future__ import absolute_import, division, print_function, unicode_literals + +import json +import logging +import sys +from io import open + +from .configuration_utils import PretrainedConfig + +logger = logging.getLogger(__name__) + +XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP = { + 'xlnet-base-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/xlnet-base-cased-config.json", + 'xlnet-large-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/xlnet-large-cased-config.json", +} + + +class XLNetConfig(PretrainedConfig): + """Configuration class to store the configuration of a ``XLNetModel``. + + Args: + vocab_size_or_config_json_file: Vocabulary size of ``inputs_ids`` in ``XLNetModel``. + d_model: Size of the encoder layers and the pooler layer. + n_layer: Number of hidden layers in the Transformer encoder. + n_head: Number of attention heads for each attention layer in + the Transformer encoder. + d_inner: The size of the "intermediate" (i.e., feed-forward) + layer in the Transformer encoder. + ff_activation: The non-linear activation function (function or string) in the + encoder and pooler. If string, "gelu", "relu" and "swish" are supported. + untie_r: untie relative position biases + attn_type: 'bi' for XLNet, 'uni' for Transformer-XL + + dropout: The dropout probabilitiy for all fully connected + layers in the embeddings, encoder, and pooler. + dropatt: The dropout ratio for the attention + probabilities. + initializer_range: The sttdev of the truncated_normal_initializer for + initializing all weight matrices. + layer_norm_eps: The epsilon used by LayerNorm. + + dropout: float, dropout rate. + dropatt: float, dropout rate on attention probabilities. + init: str, the initialization scheme, either "normal" or "uniform". + init_range: float, initialize the parameters with a uniform distribution + in [-init_range, init_range]. Only effective when init="uniform". + init_std: float, initialize the parameters with a normal distribution + with mean 0 and stddev init_std. Only effective when init="normal". + mem_len: int, the number of tokens to cache. + reuse_len: int, the number of tokens in the currect batch to be cached + and reused in the future. + bi_data: bool, whether to use bidirectional input pipeline. + Usually set to True during pretraining and False during finetuning. + clamp_len: int, clamp all relative distances larger than clamp_len. + -1 means no clamping. + same_length: bool, whether to use the same attention length for each token. + finetuning_task: name of the glue task on which the model was fine-tuned if any + """ + pretrained_config_archive_map = XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP + + def __init__(self, + vocab_size_or_config_json_file=32000, + d_model=1024, + n_layer=24, + n_head=16, + d_inner=4096, + ff_activation="gelu", + untie_r=True, + attn_type="bi", + + initializer_range=0.02, + layer_norm_eps=1e-12, + + dropout=0.1, + mem_len=None, + reuse_len=None, + bi_data=False, + clamp_len=-1, + same_length=False, + + finetuning_task=None, + num_labels=2, + summary_type='last', + summary_use_proj=True, + summary_activation='tanh', + summary_last_dropout=0.1, + start_n_top=5, + end_n_top=5, + **kwargs): + """Constructs XLNetConfig. + """ + super(XLNetConfig, self).__init__(**kwargs) + + if isinstance(vocab_size_or_config_json_file, str) or (sys.version_info[0] == 2 + and isinstance(vocab_size_or_config_json_file, unicode)): + with open(vocab_size_or_config_json_file, "r", encoding='utf-8') as reader: + json_config = json.loads(reader.read()) + for key, value in json_config.items(): + self.__dict__[key] = value + elif isinstance(vocab_size_or_config_json_file, int): + self.n_token = vocab_size_or_config_json_file + self.d_model = d_model + self.n_layer = n_layer + self.n_head = n_head + assert d_model % n_head == 0 + self.d_head = d_model // n_head + self.ff_activation = ff_activation + self.d_inner = d_inner + self.untie_r = untie_r + self.attn_type = attn_type + + self.initializer_range = initializer_range + self.layer_norm_eps = layer_norm_eps + + self.dropout = dropout + self.mem_len = mem_len + self.reuse_len = reuse_len + self.bi_data = bi_data + self.clamp_len = clamp_len + self.same_length = same_length + + self.finetuning_task = finetuning_task + self.num_labels = num_labels + self.summary_type = summary_type + self.summary_use_proj = summary_use_proj + self.summary_activation = summary_activation + self.summary_last_dropout = summary_last_dropout + self.start_n_top = start_n_top + self.end_n_top = end_n_top + else: + raise ValueError("First argument must be either a vocabulary size (int)" + " or the path to a pretrained model config file (str)") + + @property + def max_position_embeddings(self): + return -1 + + @property + def vocab_size(self): + return self.n_token + + @vocab_size.setter + def vocab_size(self, value): + self.n_token = value + + @property + def hidden_size(self): + return self.d_model + + @property + def num_attention_heads(self): + return self.n_head + + @property + def num_hidden_layers(self): + return self.n_layer diff --git a/pytorch_transformers/file_utils.py b/pytorch_transformers/file_utils.py index f6f2151b12..37ebc57fb3 100644 --- a/pytorch_transformers/file_utils.py +++ b/pytorch_transformers/file_utils.py @@ -9,6 +9,7 @@ import sys import json import logging import os +import six import shutil import tempfile import fnmatch @@ -49,6 +50,29 @@ PYTORCH_TRANSFORMERS_CACHE = PYTORCH_PRETRAINED_BERT_CACHE # Kept for backward logger = logging.getLogger(__name__) # pylint: disable=invalid-name +if not six.PY2: + def add_start_docstrings(*docstr): + def docstring_decorator(fn): + fn.__doc__ = ''.join(docstr) + fn.__doc__ + return fn + return docstring_decorator + + def add_end_docstrings(*docstr): + def docstring_decorator(fn): + fn.__doc__ = fn.__doc__ + ''.join(docstr) + return fn + return docstring_decorator +else: + # Not possible to update class docstrings on python2 + def add_start_docstrings(*docstr): + def docstring_decorator(fn): + return fn + return docstring_decorator + + def add_end_docstrings(*docstr): + def docstring_decorator(fn): + return fn + return docstring_decorator def url_to_filename(url, etag=None): """ diff --git a/pytorch_transformers/modeling_auto.py b/pytorch_transformers/modeling_auto.py index 05ff5e5b33..31c8fafaa9 100644 --- a/pytorch_transformers/modeling_auto.py +++ b/pytorch_transformers/modeling_auto.py @@ -18,125 +18,22 @@ from __future__ import absolute_import, division, print_function, unicode_litera import logging -from .modeling_bert import BertConfig, BertModel, BertForMaskedLM, BertForSequenceClassification, BertForQuestionAnswering -from .modeling_openai import OpenAIGPTConfig, OpenAIGPTModel, OpenAIGPTLMHeadModel -from .modeling_gpt2 import GPT2Config, GPT2Model, GPT2LMHeadModel -from .modeling_transfo_xl import TransfoXLConfig, TransfoXLModel, TransfoXLLMHeadModel -from .modeling_xlnet import XLNetConfig, XLNetModel, XLNetLMHeadModel, XLNetForSequenceClassification, XLNetForQuestionAnswering -from .modeling_xlm import XLMConfig, XLMModel, XLMWithLMHeadModel, XLMForSequenceClassification, XLMForQuestionAnswering -from .modeling_roberta import RobertaConfig, RobertaModel, RobertaForMaskedLM, RobertaForSequenceClassification -from .modeling_distilbert import DistilBertConfig, DistilBertModel, DistilBertForQuestionAnswering, DistilBertForMaskedLM, DistilBertForSequenceClassification +from .modeling_bert import BertModel, BertForMaskedLM, BertForSequenceClassification, BertForQuestionAnswering +from .modeling_openai import OpenAIGPTModel, OpenAIGPTLMHeadModel +from .modeling_gpt2 import GPT2Model, GPT2LMHeadModel +from .modeling_transfo_xl import TransfoXLModel, TransfoXLLMHeadModel +from .modeling_xlnet import XLNetModel, XLNetLMHeadModel, XLNetForSequenceClassification, XLNetForQuestionAnswering +from .modeling_xlm import XLMModel, XLMWithLMHeadModel, XLMForSequenceClassification, XLMForQuestionAnswering +from .modeling_roberta import RobertaModel, RobertaForMaskedLM, RobertaForSequenceClassification +from .modeling_distilbert import DistilBertModel, DistilBertForQuestionAnswering, DistilBertForMaskedLM, DistilBertForSequenceClassification -from .modeling_utils import PreTrainedModel, SequenceSummary, add_start_docstrings +from .modeling_utils import PreTrainedModel, SequenceSummary + +from .file_utils import add_start_docstrings logger = logging.getLogger(__name__) -class AutoConfig(object): - r""":class:`~pytorch_transformers.AutoConfig` is a generic configuration class - that will be instantiated as one of the configuration classes of the library - when created with the `AutoConfig.from_pretrained(pretrained_model_name_or_path)` - class method. - - The `from_pretrained()` method take care of returning the correct model class instance - using pattern matching on the `pretrained_model_name_or_path` string. - - The base model class to instantiate is selected as the first pattern matching - in the `pretrained_model_name_or_path` string (in the following order): - - contains `distilbert`: DistilBertConfig (DistilBERT model) - - contains `bert`: BertConfig (Bert model) - - contains `openai-gpt`: OpenAIGPTConfig (OpenAI GPT model) - - contains `gpt2`: GPT2Config (OpenAI GPT-2 model) - - contains `transfo-xl`: TransfoXLConfig (Transformer-XL model) - - contains `xlnet`: XLNetConfig (XLNet model) - - contains `xlm`: XLMConfig (XLM model) - - contains `roberta`: RobertaConfig (RoBERTa model) - - This class cannot be instantiated using `__init__()` (throw an error). - """ - def __init__(self): - raise EnvironmentError("AutoConfig is designed to be instantiated " - "using the `AutoConfig.from_pretrained(pretrained_model_name_or_path)` method.") - - @classmethod - def from_pretrained(cls, pretrained_model_name_or_path, **kwargs): - r""" Instantiate a one of the configuration classes of the library - from a pre-trained model configuration. - - The configuration class to instantiate is selected as the first pattern matching - in the `pretrained_model_name_or_path` string (in the following order): - - contains `distilbert`: DistilBertConfig (DistilBERT model) - - contains `bert`: BertConfig (Bert model) - - contains `openai-gpt`: OpenAIGPTConfig (OpenAI GPT model) - - contains `gpt2`: GPT2Config (OpenAI GPT-2 model) - - contains `transfo-xl`: TransfoXLConfig (Transformer-XL model) - - contains `xlnet`: XLNetConfig (XLNet model) - - contains `xlm`: XLMConfig (XLM model) - - contains `roberta`: RobertaConfig (RoBERTa model) - - Params: - pretrained_model_name_or_path: either: - - - a string with the `shortcut name` of a pre-trained model configuration to load from cache or download, e.g.: ``bert-base-uncased``. - - a path to a `directory` containing a configuration file saved using the :func:`~pytorch_transformers.PretrainedConfig.save_pretrained` method, e.g.: ``./my_model_directory/``. - - a path or url to a saved configuration JSON `file`, e.g.: ``./my_model_directory/configuration.json``. - - cache_dir: (`optional`) string: - Path to a directory in which a downloaded pre-trained model - configuration should be cached if the standard cache should not be used. - - kwargs: (`optional`) dict: key/value pairs with which to update the configuration object after loading. - - - The values in kwargs of any keys which are configuration attributes will be used to override the loaded values. - - Behavior concerning key/value pairs whose keys are *not* configuration attributes is controlled by the `return_unused_kwargs` keyword parameter. - - force_download: (`optional`) boolean, default False: - Force to (re-)download the model weights and configuration files and override the cached versions if they exists. - - proxies: (`optional`) dict, default None: - A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. - The proxies are used on each request. - - return_unused_kwargs: (`optional`) bool: - - - If False, then this function returns just the final configuration object. - - If True, then this functions returns a tuple `(config, unused_kwargs)` where `unused_kwargs` is a dictionary consisting of the key/value pairs whose keys are not configuration attributes: ie the part of kwargs which has not been used to update `config` and is otherwise ignored. - - Examples:: - - config = AutoConfig.from_pretrained('bert-base-uncased') # Download configuration from S3 and cache. - config = AutoConfig.from_pretrained('./test/bert_saved_model/') # E.g. config (or model) was saved using `save_pretrained('./test/saved_model/')` - config = AutoConfig.from_pretrained('./test/bert_saved_model/my_configuration.json') - config = AutoConfig.from_pretrained('bert-base-uncased', output_attention=True, foo=False) - assert config.output_attention == True - config, unused_kwargs = AutoConfig.from_pretrained('bert-base-uncased', output_attention=True, - foo=False, return_unused_kwargs=True) - assert config.output_attention == True - assert unused_kwargs == {'foo': False} - - """ - if 'distilbert' in pretrained_model_name_or_path: - return DistilBertConfig.from_pretrained(pretrained_model_name_or_path, **kwargs) - elif 'roberta' in pretrained_model_name_or_path: - return RobertaConfig.from_pretrained(pretrained_model_name_or_path, **kwargs) - elif 'bert' in pretrained_model_name_or_path: - return BertConfig.from_pretrained(pretrained_model_name_or_path, **kwargs) - elif 'openai-gpt' in pretrained_model_name_or_path: - return OpenAIGPTConfig.from_pretrained(pretrained_model_name_or_path, **kwargs) - elif 'gpt2' in pretrained_model_name_or_path: - return GPT2Config.from_pretrained(pretrained_model_name_or_path, **kwargs) - elif 'transfo-xl' in pretrained_model_name_or_path: - return TransfoXLConfig.from_pretrained(pretrained_model_name_or_path, **kwargs) - elif 'xlnet' in pretrained_model_name_or_path: - return XLNetConfig.from_pretrained(pretrained_model_name_or_path, **kwargs) - elif 'xlm' in pretrained_model_name_or_path: - return XLMConfig.from_pretrained(pretrained_model_name_or_path, **kwargs) - - raise ValueError("Unrecognized model identifier in {}. Should contains one of " - "'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', " - "'xlm', 'roberta'".format(pretrained_model_name_or_path)) - - class AutoModel(object): r""" :class:`~pytorch_transformers.AutoModel` is a generic model class diff --git a/pytorch_transformers/modeling_bert.py b/pytorch_transformers/modeling_bert.py index 5c71bedba9..c541d18da5 100644 --- a/pytorch_transformers/modeling_bert.py +++ b/pytorch_transformers/modeling_bert.py @@ -28,8 +28,9 @@ import torch from torch import nn from torch.nn import CrossEntropyLoss, MSELoss -from .modeling_utils import (WEIGHTS_NAME, CONFIG_NAME, PretrainedConfig, PreTrainedModel, - prune_linear_layer, add_start_docstrings) +from .modeling_utils import PreTrainedModel, prune_linear_layer +from .configuration_bert import BertConfig +from .file_utils import add_start_docstrings logger = logging.getLogger(__name__) @@ -49,23 +50,6 @@ BERT_PRETRAINED_MODEL_ARCHIVE_MAP = { 'bert-base-cased-finetuned-mrpc': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-finetuned-mrpc-pytorch_model.bin", } -BERT_PRETRAINED_CONFIG_ARCHIVE_MAP = { - 'bert-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-config.json", - 'bert-large-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-config.json", - 'bert-base-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-config.json", - 'bert-large-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-config.json", - 'bert-base-multilingual-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-uncased-config.json", - 'bert-base-multilingual-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-cased-config.json", - 'bert-base-chinese': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-chinese-config.json", - 'bert-base-german-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-german-cased-config.json", - 'bert-large-uncased-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-whole-word-masking-config.json", - 'bert-large-cased-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-whole-word-masking-config.json", - 'bert-large-uncased-whole-word-masking-finetuned-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-whole-word-masking-finetuned-squad-config.json", - 'bert-large-cased-whole-word-masking-finetuned-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-whole-word-masking-finetuned-squad-config.json", - 'bert-base-cased-finetuned-mrpc': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-finetuned-mrpc-config.json", -} - - def load_tf_weights_in_bert(model, config, tf_checkpoint_path): """ Load tf checkpoints in a pytorch model. """ @@ -149,77 +133,6 @@ def swish(x): ACT2FN = {"gelu": gelu, "relu": torch.nn.functional.relu, "swish": swish} -class BertConfig(PretrainedConfig): - r""" - :class:`~pytorch_transformers.BertConfig` is the configuration class to store the configuration of a - `BertModel`. - - - Arguments: - vocab_size_or_config_json_file: Vocabulary size of `inputs_ids` in `BertModel`. - hidden_size: Size of the encoder layers and the pooler layer. - num_hidden_layers: Number of hidden layers in the Transformer encoder. - num_attention_heads: Number of attention heads for each attention layer in - the Transformer encoder. - intermediate_size: The size of the "intermediate" (i.e., feed-forward) - layer in the Transformer encoder. - hidden_act: The non-linear activation function (function or string) in the - encoder and pooler. If string, "gelu", "relu" and "swish" are supported. - hidden_dropout_prob: The dropout probabilitiy for all fully connected - layers in the embeddings, encoder, and pooler. - attention_probs_dropout_prob: The dropout ratio for the attention - probabilities. - max_position_embeddings: The maximum sequence length that this model might - ever be used with. Typically set this to something large just in case - (e.g., 512 or 1024 or 2048). - type_vocab_size: The vocabulary size of the `token_type_ids` passed into - `BertModel`. - initializer_range: The sttdev of the truncated_normal_initializer for - initializing all weight matrices. - layer_norm_eps: The epsilon used by LayerNorm. - """ - pretrained_config_archive_map = BERT_PRETRAINED_CONFIG_ARCHIVE_MAP - - def __init__(self, - vocab_size_or_config_json_file=30522, - hidden_size=768, - num_hidden_layers=12, - num_attention_heads=12, - intermediate_size=3072, - hidden_act="gelu", - hidden_dropout_prob=0.1, - attention_probs_dropout_prob=0.1, - max_position_embeddings=512, - type_vocab_size=2, - initializer_range=0.02, - layer_norm_eps=1e-12, - **kwargs): - super(BertConfig, self).__init__(**kwargs) - if isinstance(vocab_size_or_config_json_file, str) or (sys.version_info[0] == 2 - and isinstance(vocab_size_or_config_json_file, unicode)): - with open(vocab_size_or_config_json_file, "r", encoding='utf-8') as reader: - json_config = json.loads(reader.read()) - for key, value in json_config.items(): - self.__dict__[key] = value - elif isinstance(vocab_size_or_config_json_file, int): - self.vocab_size = vocab_size_or_config_json_file - self.hidden_size = hidden_size - self.num_hidden_layers = num_hidden_layers - self.num_attention_heads = num_attention_heads - self.hidden_act = hidden_act - self.intermediate_size = intermediate_size - self.hidden_dropout_prob = hidden_dropout_prob - self.attention_probs_dropout_prob = attention_probs_dropout_prob - self.max_position_embeddings = max_position_embeddings - self.type_vocab_size = type_vocab_size - self.initializer_range = initializer_range - self.layer_norm_eps = layer_norm_eps - else: - raise ValueError("First argument must be either a vocabulary size (int)" - " or the path to a pretrained model config file (str)") - - - try: from apex.normalization.fused_layer_norm import FusedLayerNorm as BertLayerNorm except (ImportError, AttributeError) as e: diff --git a/pytorch_transformers/modeling_distilbert.py b/pytorch_transformers/modeling_distilbert.py index 280270381c..29ecbfb846 100644 --- a/pytorch_transformers/modeling_distilbert.py +++ b/pytorch_transformers/modeling_distilbert.py @@ -31,7 +31,9 @@ import numpy as np import torch import torch.nn as nn -from pytorch_transformers.modeling_utils import PretrainedConfig, PreTrainedModel, add_start_docstrings, prune_linear_layer +from .modeling_utils import PreTrainedModel, prune_linear_layer +from .configuration_distilbert import DistilBertConfig +from .file_utils import add_start_docstrings import logging logger = logging.getLogger(__name__) @@ -42,69 +44,6 @@ DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP = { 'distilbert-base-uncased-distilled-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-pytorch_model.bin" } -DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = { - 'distilbert-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-config.json", - 'distilbert-base-uncased-distilled-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-config.json" -} - - -class DistilBertConfig(PretrainedConfig): - pretrained_config_archive_map = DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP - - def __init__(self, - vocab_size_or_config_json_file=30522, - max_position_embeddings=512, - sinusoidal_pos_embds=True, - n_layers=6, - n_heads=12, - dim=768, - hidden_dim=4*768, - dropout=0.1, - attention_dropout=0.1, - activation='gelu', - initializer_range=0.02, - tie_weights_=True, - qa_dropout=0.1, - seq_classif_dropout=0.2, - **kwargs): - super(DistilBertConfig, self).__init__(**kwargs) - - if isinstance(vocab_size_or_config_json_file, str) or (sys.version_info[0] == 2 - and isinstance(vocab_size_or_config_json_file, unicode)): - with open(vocab_size_or_config_json_file, "r", encoding='utf-8') as reader: - json_config = json.loads(reader.read()) - for key, value in json_config.items(): - self.__dict__[key] = value - elif isinstance(vocab_size_or_config_json_file, int): - self.vocab_size = vocab_size_or_config_json_file - self.max_position_embeddings = max_position_embeddings - self.sinusoidal_pos_embds = sinusoidal_pos_embds - self.n_layers = n_layers - self.n_heads = n_heads - self.dim = dim - self.hidden_dim = hidden_dim - self.dropout = dropout - self.attention_dropout = attention_dropout - self.activation = activation - self.initializer_range = initializer_range - self.tie_weights_ = tie_weights_ - self.qa_dropout = qa_dropout - self.seq_classif_dropout = seq_classif_dropout - else: - raise ValueError("First argument must be either a vocabulary size (int)" - " or the path to a pretrained model config file (str)") - @property - def hidden_size(self): - return self.dim - - @property - def num_attention_heads(self): - return self.n_heads - - @property - def num_hidden_layers(self): - return self.n_layers - ### UTILS AND BUILDING BLOCKS OF THE ARCHITECTURE ### def gelu(x): diff --git a/pytorch_transformers/modeling_gpt2.py b/pytorch_transformers/modeling_gpt2.py index d16448beaa..4268641187 100644 --- a/pytorch_transformers/modeling_gpt2.py +++ b/pytorch_transformers/modeling_gpt2.py @@ -30,19 +30,15 @@ import torch.nn as nn from torch.nn import CrossEntropyLoss from torch.nn.parameter import Parameter -from .modeling_utils import (Conv1D, CONFIG_NAME, WEIGHTS_NAME, PretrainedConfig, - PreTrainedModel, prune_conv1d_layer, SequenceSummary, - add_start_docstrings) -from .modeling_bert import BertLayerNorm as LayerNorm +from .modeling_utils import PreTrainedModel, Conv1D, prune_conv1d_layer, SequenceSummary +from .configuration_gpt2 import GPT2Config +from .file_utils import add_start_docstrings logger = logging.getLogger(__name__) GPT2_PRETRAINED_MODEL_ARCHIVE_MAP = {"gpt2": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-pytorch_model.bin", "gpt2-medium": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-medium-pytorch_model.bin", "gpt2-large": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-large-pytorch_model.bin"} -GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP = {"gpt2": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-config.json", - "gpt2-medium": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-medium-config.json", - "gpt2-large": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-large-config.json"} def load_tf_weights_in_gpt2(model, config, gpt2_checkpoint_path): """ Load tf checkpoints in a pytorch model @@ -102,120 +98,6 @@ def gelu(x): return 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) -class GPT2Config(PretrainedConfig): - """Configuration class to store the configuration of a `GPT2Model`. - - Args: - vocab_size_or_config_json_file: Vocabulary size of `inputs_ids` in `GPT2Model` or a configuration json file. - n_positions: Number of positional embeddings. - n_ctx: Size of the causal mask (usually same as n_positions). - n_embd: Dimensionality of the embeddings and hidden states. - n_layer: Number of hidden layers in the Transformer encoder. - n_head: Number of attention heads for each attention layer in - the Transformer encoder. - layer_norm_epsilon: epsilon to use in the layer norm layers - resid_pdrop: The dropout probabilitiy for all fully connected - layers in the embeddings, encoder, and pooler. - attn_pdrop: The dropout ratio for the attention - probabilities. - embd_pdrop: The dropout ratio for the embeddings. - initializer_range: The sttdev of the truncated_normal_initializer for - initializing all weight matrices. - """ - pretrained_config_archive_map = GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP - - def __init__( - self, - vocab_size_or_config_json_file=50257, - n_positions=1024, - n_ctx=1024, - n_embd=768, - n_layer=12, - n_head=12, - resid_pdrop=0.1, - embd_pdrop=0.1, - attn_pdrop=0.1, - layer_norm_epsilon=1e-5, - initializer_range=0.02, - - num_labels=1, - summary_type='cls_index', - summary_use_proj=True, - summary_activation=None, - summary_proj_to_labels=True, - summary_first_dropout=0.1, - **kwargs - ): - """Constructs GPT2Config. - - Args: - vocab_size_or_config_json_file: Vocabulary size of `inputs_ids` in `GPT2Model` or a configuration json file. - n_positions: Number of positional embeddings. - n_ctx: Size of the causal mask (usually same as n_positions). - n_embd: Dimensionality of the embeddings and hidden states. - n_layer: Number of hidden layers in the Transformer encoder. - n_head: Number of attention heads for each attention layer in - the Transformer encoder. - layer_norm_epsilon: epsilon to use in the layer norm layers - resid_pdrop: The dropout probabilitiy for all fully connected - layers in the embeddings, encoder, and pooler. - attn_pdrop: The dropout ratio for the attention - probabilities. - embd_pdrop: The dropout ratio for the embeddings. - initializer_range: The sttdev of the truncated_normal_initializer for - initializing all weight matrices. - """ - super(GPT2Config, self).__init__(**kwargs) - - if isinstance(vocab_size_or_config_json_file, str) or (sys.version_info[0] == 2 - and isinstance(vocab_size_or_config_json_file, unicode)): - with open(vocab_size_or_config_json_file, "r", encoding="utf-8") as reader: - json_config = json.loads(reader.read()) - for key, value in json_config.items(): - self.__dict__[key] = value - elif isinstance(vocab_size_or_config_json_file, int): - self.vocab_size = vocab_size_or_config_json_file - self.n_ctx = n_ctx - self.n_positions = n_positions - self.n_embd = n_embd - self.n_layer = n_layer - self.n_head = n_head - self.resid_pdrop = resid_pdrop - self.embd_pdrop = embd_pdrop - self.attn_pdrop = attn_pdrop - self.layer_norm_epsilon = layer_norm_epsilon - self.initializer_range = initializer_range - - self.num_labels = num_labels - self.summary_type = summary_type - self.summary_use_proj = summary_use_proj - self.summary_activation = summary_activation - self.summary_first_dropout = summary_first_dropout - self.summary_proj_to_labels = summary_proj_to_labels - else: - raise ValueError( - "First argument must be either a vocabulary size (int)" - "or the path to a pretrained model config file (str)" - ) - - @property - def max_position_embeddings(self): - return self.n_positions - - @property - def hidden_size(self): - return self.n_embd - - @property - def num_attention_heads(self): - return self.n_head - - @property - def num_hidden_layers(self): - return self.n_layer - - - class Attention(nn.Module): def __init__(self, nx, n_ctx, config, scale=False): super(Attention, self).__init__() @@ -336,9 +218,9 @@ class Block(nn.Module): def __init__(self, n_ctx, config, scale=False): super(Block, self).__init__() nx = config.n_embd - self.ln_1 = LayerNorm(nx, eps=config.layer_norm_epsilon) + self.ln_1 = nn.LayerNorm(nx, eps=config.layer_norm_epsilon) self.attn = Attention(nx, n_ctx, config, scale) - self.ln_2 = LayerNorm(nx, eps=config.layer_norm_epsilon) + self.ln_2 = nn.LayerNorm(nx, eps=config.layer_norm_epsilon) self.mlp = MLP(4 * nx, config) def forward(self, x, layer_past=None, attention_mask=None, head_mask=None): @@ -377,7 +259,7 @@ class GPT2PreTrainedModel(PreTrainedModel): module.weight.data.normal_(mean=0.0, std=self.config.initializer_range) if isinstance(module, (nn.Linear, Conv1D)) and module.bias is not None: module.bias.data.zero_() - elif isinstance(module, LayerNorm): + elif isinstance(module, nn.LayerNorm): module.bias.data.zero_() module.weight.data.fill_(1.0) @@ -469,7 +351,7 @@ class GPT2Model(GPT2PreTrainedModel): self.wpe = nn.Embedding(config.n_positions, config.n_embd) self.drop = nn.Dropout(config.embd_pdrop) self.h = nn.ModuleList([Block(config.n_ctx, config, scale=True) for _ in range(config.n_layer)]) - self.ln_f = LayerNorm(config.n_embd, eps=config.layer_norm_epsilon) + self.ln_f = nn.LayerNorm(config.n_embd, eps=config.layer_norm_epsilon) self.init_weights() diff --git a/pytorch_transformers/modeling_openai.py b/pytorch_transformers/modeling_openai.py index 4fbec7a768..9936e72030 100644 --- a/pytorch_transformers/modeling_openai.py +++ b/pytorch_transformers/modeling_openai.py @@ -30,15 +30,13 @@ import torch.nn as nn from torch.nn import CrossEntropyLoss from torch.nn.parameter import Parameter -from .modeling_utils import (Conv1D, CONFIG_NAME, WEIGHTS_NAME, PretrainedConfig, - PreTrainedModel, prune_conv1d_layer, SequenceSummary, - add_start_docstrings) -from .modeling_bert import BertLayerNorm as LayerNorm +from .modeling_utils import PreTrainedModel, Conv1D, prune_conv1d_layer, SequenceSummary +from .configuration_openai import OpenAIGPTConfig +from .file_utils import add_start_docstrings logger = logging.getLogger(__name__) OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP = {"openai-gpt": "https://s3.amazonaws.com/models.huggingface.co/bert/openai-gpt-pytorch_model.bin"} -OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP = {"openai-gpt": "https://s3.amazonaws.com/models.huggingface.co/bert/openai-gpt-config.json"} def load_tf_weights_in_openai_gpt(model, config, openai_checkpoint_folder_path): @@ -127,111 +125,6 @@ def swish(x): ACT_FNS = {"relu": nn.ReLU, "swish": swish, "gelu": gelu} -class OpenAIGPTConfig(PretrainedConfig): - """ - Configuration class to store the configuration of a `OpenAIGPTModel`. - - Args: - vocab_size_or_config_json_file: Vocabulary size of `inputs_ids` in `OpenAIGPTModel` or a configuration json file. - n_special: The number of special tokens to learn during fine-tuning ('[SEP]', '[CLF]', ...) - n_positions: Number of positional embeddings. - n_ctx: Size of the causal mask (usually same as n_positions). - n_embd: Dimensionality of the embeddings and hidden states. - n_layer: Number of hidden layers in the Transformer encoder. - n_head: Number of attention heads for each attention layer in - the Transformer encoder. - afn: The non-linear activation function (function or string) in the - encoder and pooler. If string, "gelu", "relu" and "swish" are supported. - resid_pdrop: The dropout probabilitiy for all fully connected - layers in the embeddings, encoder, and pooler. - attn_pdrop: The dropout ratio for the attention - probabilities. - embd_pdrop: The dropout ratio for the embeddings. - layer_norm_epsilon: epsilon to use in the layer norm layers - initializer_range: The sttdev of the truncated_normal_initializer for - initializing all weight matrices. - predict_special_tokens: should we predict special tokens (when the model has a LM head) - """ - pretrained_config_archive_map = OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP - - def __init__( - self, - vocab_size_or_config_json_file=40478, - n_positions=512, - n_ctx=512, - n_embd=768, - n_layer=12, - n_head=12, - afn="gelu", - resid_pdrop=0.1, - embd_pdrop=0.1, - attn_pdrop=0.1, - layer_norm_epsilon=1e-5, - initializer_range=0.02, - predict_special_tokens=True, - - num_labels=1, - summary_type='cls_index', - summary_use_proj=True, - summary_activation=None, - summary_proj_to_labels=True, - summary_first_dropout=0.1, - **kwargs - ): - """Constructs OpenAIGPTConfig. - """ - super(OpenAIGPTConfig, self).__init__(**kwargs) - - if isinstance(vocab_size_or_config_json_file, str) or (sys.version_info[0] == 2 - and isinstance(vocab_size_or_config_json_file, unicode)): - with open(vocab_size_or_config_json_file, "r", encoding="utf-8") as reader: - json_config = json.loads(reader.read()) - for key, value in json_config.items(): - self.__dict__[key] = value - elif isinstance(vocab_size_or_config_json_file, int): - self.vocab_size = vocab_size_or_config_json_file - self.n_ctx = n_ctx - self.n_positions = n_positions - self.n_embd = n_embd - self.n_layer = n_layer - self.n_head = n_head - self.afn = afn - self.resid_pdrop = resid_pdrop - self.embd_pdrop = embd_pdrop - self.attn_pdrop = attn_pdrop - self.layer_norm_epsilon = layer_norm_epsilon - self.initializer_range = initializer_range - self.predict_special_tokens = predict_special_tokens - - self.num_labels = num_labels - self.summary_type = summary_type - self.summary_use_proj = summary_use_proj - self.summary_activation = summary_activation - self.summary_first_dropout = summary_first_dropout - self.summary_proj_to_labels = summary_proj_to_labels - else: - raise ValueError( - "First argument must be either a vocabulary size (int)" - "or the path to a pretrained model config file (str)" - ) - - @property - def max_position_embeddings(self): - return self.n_positions - - @property - def hidden_size(self): - return self.n_embd - - @property - def num_attention_heads(self): - return self.n_head - - @property - def num_hidden_layers(self): - return self.n_layer - - class Attention(nn.Module): def __init__(self, nx, n_ctx, config, scale=False): super(Attention, self).__init__() @@ -346,9 +239,9 @@ class Block(nn.Module): super(Block, self).__init__() nx = config.n_embd self.attn = Attention(nx, n_ctx, config, scale) - self.ln_1 = LayerNorm(nx, eps=config.layer_norm_epsilon) + self.ln_1 = nn.LayerNorm(nx, eps=config.layer_norm_epsilon) self.mlp = MLP(4 * nx, config) - self.ln_2 = LayerNorm(nx, eps=config.layer_norm_epsilon) + self.ln_2 = nn.LayerNorm(nx, eps=config.layer_norm_epsilon) def forward(self, x, attention_mask=None, head_mask=None): attn_outputs = self.attn(x, attention_mask=attention_mask, head_mask=head_mask) @@ -380,7 +273,7 @@ class OpenAIGPTPreTrainedModel(PreTrainedModel): module.weight.data.normal_(mean=0.0, std=self.config.initializer_range) if isinstance(module, (nn.Linear, Conv1D)) and module.bias is not None: module.bias.data.zero_() - elif isinstance(module, LayerNorm): + elif isinstance(module, nn.LayerNorm): module.bias.data.zero_() module.weight.data.fill_(1.0) diff --git a/pytorch_transformers/modeling_roberta.py b/pytorch_transformers/modeling_roberta.py index 0694e76415..41027330a8 100644 --- a/pytorch_transformers/modeling_roberta.py +++ b/pytorch_transformers/modeling_roberta.py @@ -22,14 +22,11 @@ import logging import torch import torch.nn as nn -import torch.nn.functional as F from torch.nn import CrossEntropyLoss, MSELoss -from pytorch_transformers.modeling_bert import (BertConfig, BertEmbeddings, - BertLayerNorm, BertModel, - BertPreTrainedModel, gelu) - -from pytorch_transformers.modeling_utils import add_start_docstrings +from .modeling_bert import BertEmbeddings, BertLayerNorm, BertModel, BertPreTrainedModel, gelu +from .configuration_roberta import RobertaConfig +from .file_utils import add_start_docstrings logger = logging.getLogger(__name__) @@ -39,13 +36,6 @@ ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP = { 'roberta-large-mnli': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-mnli-pytorch_model.bin", } -ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP = { - 'roberta-base': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-config.json", - 'roberta-large': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-config.json", - 'roberta-large-mnli': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-mnli-config.json", -} - - class RobertaEmbeddings(BertEmbeddings): """ Same as BertEmbeddings with a tiny tweak for positional embeddings indexing. @@ -66,10 +56,6 @@ class RobertaEmbeddings(BertEmbeddings): position_ids=position_ids) -class RobertaConfig(BertConfig): - pretrained_config_archive_map = ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP - - ROBERTA_START_DOCSTRING = r""" The RoBERTa model was proposed in `RoBERTa: A Robustly Optimized BERT Pretraining Approach`_ by Yinhan Liu, Myle Ott, Naman Goyal, Jingfei Du, Mandar Joshi, Danqi Chen, Omer Levy, Mike Lewis, Luke Zettlemoyer, diff --git a/pytorch_transformers/modeling_transfo_xl.py b/pytorch_transformers/modeling_transfo_xl.py index 3fff0c552e..7ad9d10891 100644 --- a/pytorch_transformers/modeling_transfo_xl.py +++ b/pytorch_transformers/modeling_transfo_xl.py @@ -34,18 +34,16 @@ import torch.nn.functional as F from torch.nn import CrossEntropyLoss from torch.nn.parameter import Parameter -from .modeling_bert import BertLayerNorm as LayerNorm +from .modeling_utils import PreTrainedModel, Conv1D, prune_conv1d_layer, SequenceSummary +from .configuration_transfo_xl import TransfoXLConfig from .modeling_transfo_xl_utilities import ProjectedAdaptiveLogSoftmax, sample_logits -from .modeling_utils import (PretrainedConfig, PreTrainedModel, add_start_docstrings) +from .file_utils import add_start_docstrings logger = logging.getLogger(__name__) TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP = { 'transfo-xl-wt103': "https://s3.amazonaws.com/models.huggingface.co/bert/transfo-xl-wt103-pytorch_model.bin", } -TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP = { - 'transfo-xl-wt103': "https://s3.amazonaws.com/models.huggingface.co/bert/transfo-xl-wt103-config.json", -} def build_tf_to_pytorch_map(model, config): """ A map of modules from TF to PyTorch. @@ -175,143 +173,6 @@ def load_tf_weights_in_transfo_xl(model, config, tf_path): return model -class TransfoXLConfig(PretrainedConfig): - """Configuration class to store the configuration of a `TransfoXLModel`. - - Args: - vocab_size_or_config_json_file: Vocabulary size of `inputs_ids` in `TransfoXLModel` or a configuration json file. - cutoffs: cutoffs for the adaptive softmax - d_model: Dimensionality of the model's hidden states. - d_embed: Dimensionality of the embeddings - d_head: Dimensionality of the model's heads. - div_val: divident value for adapative input and softmax - pre_lnorm: apply LayerNorm to the input instead of the output - d_inner: Inner dimension in FF - n_layer: Number of hidden layers in the Transformer encoder. - n_head: Number of attention heads for each attention layer in - the Transformer encoder. - tgt_len: number of tokens to predict - ext_len: length of the extended context - mem_len: length of the retained previous heads - same_length: use the same attn length for all tokens - proj_share_all_but_first: True to share all but first projs, False not to share. - attn_type: attention type. 0 for Transformer-XL, 1 for Shaw et al, 2 for Vaswani et al, 3 for Al Rfou et al. - clamp_len: use the same pos embeddings after clamp_len - sample_softmax: number of samples in sampled softmax - adaptive: use adaptive softmax - tie_weight: tie the word embedding and softmax weights - dropout: The dropout probabilitiy for all fully connected - layers in the embeddings, encoder, and pooler. - dropatt: The dropout ratio for the attention probabilities. - untie_r: untie relative position biases - embd_pdrop: The dropout ratio for the embeddings. - init: parameter initializer to use - init_range: parameters initialized by U(-init_range, init_range). - proj_init_std: parameters initialized by N(0, init_std) - init_std: parameters initialized by N(0, init_std) - """ - pretrained_config_archive_map = TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP - - def __init__(self, - vocab_size_or_config_json_file=267735, - cutoffs=[20000, 40000, 200000], - d_model=1024, - d_embed=1024, - n_head=16, - d_head=64, - d_inner=4096, - div_val=4, - pre_lnorm=False, - n_layer=18, - tgt_len=128, - ext_len=0, - mem_len=1600, - clamp_len=1000, - same_length=True, - proj_share_all_but_first=True, - attn_type=0, - sample_softmax=-1, - adaptive=True, - tie_weight=True, - dropout=0.1, - dropatt=0.0, - untie_r=True, - init="normal", - init_range=0.01, - proj_init_std=0.01, - init_std=0.02, - **kwargs): - """Constructs TransfoXLConfig. - """ - super(TransfoXLConfig, self).__init__(**kwargs) - - if isinstance(vocab_size_or_config_json_file, str) or (sys.version_info[0] == 2 - and isinstance(vocab_size_or_config_json_file, unicode)): - with open(vocab_size_or_config_json_file, "r", encoding='utf-8') as reader: - json_config = json.loads(reader.read()) - for key, value in json_config.items(): - self.__dict__[key] = value - elif isinstance(vocab_size_or_config_json_file, int): - self.n_token = vocab_size_or_config_json_file - self.cutoffs = [] - self.cutoffs.extend(cutoffs) - self.tie_weight = tie_weight - if proj_share_all_but_first: - self.tie_projs = [False] + [True] * len(self.cutoffs) - else: - self.tie_projs = [False] + [False] * len(self.cutoffs) - self.d_model = d_model - self.d_embed = d_embed - self.d_head = d_head - self.d_inner = d_inner - self.div_val = div_val - self.pre_lnorm = pre_lnorm - self.n_layer = n_layer - self.n_head = n_head - self.tgt_len = tgt_len - self.ext_len = ext_len - self.mem_len = mem_len - self.same_length = same_length - self.attn_type = attn_type - self.clamp_len = clamp_len - self.sample_softmax = sample_softmax - self.adaptive = adaptive - self.dropout = dropout - self.dropatt = dropatt - self.untie_r = untie_r - self.init = init - self.init_range = init_range - self.proj_init_std = proj_init_std - self.init_std = init_std - else: - raise ValueError("First argument must be either a vocabulary size (int)" - " or the path to a pretrained model config file (str)") - - @property - def max_position_embeddings(self): - return self.tgt_len + self.ext_len + self.mem_len - - @property - def vocab_size(self): - return self.n_token - - @vocab_size.setter - def vocab_size(self, value): - self.n_token = value - - @property - def hidden_size(self): - return self.d_model - - @property - def num_attention_heads(self): - return self.n_head - - @property - def num_hidden_layers(self): - return self.n_layer - - class PositionalEmbedding(nn.Module): def __init__(self, demb): super(PositionalEmbedding, self).__init__() @@ -347,7 +208,7 @@ class PositionwiseFF(nn.Module): nn.Dropout(dropout), ) - self.layer_norm = LayerNorm(d_model) + self.layer_norm = nn.LayerNorm(d_model) self.pre_lnorm = pre_lnorm @@ -387,7 +248,7 @@ class MultiHeadAttn(nn.Module): self.dropatt = nn.Dropout(dropatt) self.o_net = nn.Linear(n_head * d_head, d_model, bias=False) - self.layer_norm = LayerNorm(d_model) + self.layer_norm = nn.LayerNorm(d_model) self.scale = 1 / (d_head ** 0.5) @@ -477,7 +338,7 @@ class RelMultiHeadAttn(nn.Module): self.dropatt = nn.Dropout(dropatt) self.o_net = nn.Linear(n_head * d_head, d_model, bias=False) - self.layer_norm = LayerNorm(d_model) + self.layer_norm = nn.LayerNorm(d_model) self.scale = 1 / (d_head ** 0.5) diff --git a/pytorch_transformers/modeling_utils.py b/pytorch_transformers/modeling_utils.py index d482addb83..48420f6d07 100644 --- a/pytorch_transformers/modeling_utils.py +++ b/pytorch_transformers/modeling_utils.py @@ -30,11 +30,11 @@ from torch import nn from torch.nn import CrossEntropyLoss from torch.nn import functional as F +from .configuration_utils import PretrainedConfig from .file_utils import cached_path logger = logging.getLogger(__name__) -CONFIG_NAME = "config.json" WEIGHTS_NAME = "pytorch_model.bin" TF_WEIGHTS_NAME = 'model.ckpt' @@ -52,209 +52,6 @@ except ImportError: def forward(self, input): return input - -if not six.PY2: - def add_start_docstrings(*docstr): - def docstring_decorator(fn): - fn.__doc__ = ''.join(docstr) + fn.__doc__ - return fn - return docstring_decorator - - def add_end_docstrings(*docstr): - def docstring_decorator(fn): - fn.__doc__ = fn.__doc__ + ''.join(docstr) - return fn - return docstring_decorator -else: - # Not possible to update class docstrings on python2 - def add_start_docstrings(*docstr): - def docstring_decorator(fn): - return fn - return docstring_decorator - - def add_end_docstrings(*docstr): - def docstring_decorator(fn): - return fn - return docstring_decorator - - -class PretrainedConfig(object): - r""" Base class for all configuration classes. - Handles a few parameters common to all models' configurations as well as methods for loading/downloading/saving configurations. - - Note: - A configuration file can be loaded and saved to disk. Loading the configuration file and using this file to initialize a model does **not** load the model weights. - It only affects the model's configuration. - - Class attributes (overridden by derived classes): - - ``pretrained_config_archive_map``: a python ``dict`` of with `short-cut-names` (string) as keys and `url` (string) of associated pretrained model configurations as values. - - Parameters: - ``finetuning_task``: string, default `None`. Name of the task used to fine-tune the model. This can be used when converting from an original (TensorFlow or PyTorch) checkpoint. - ``num_labels``: integer, default `2`. Number of classes to use when the model is a classification model (sequences/tokens) - ``output_attentions``: boolean, default `False`. Should the model returns attentions weights. - ``output_hidden_states``: string, default `False`. Should the model returns all hidden-states. - ``torchscript``: string, default `False`. Is the model used with Torchscript. - """ - pretrained_config_archive_map = {} - - def __init__(self, **kwargs): - self.finetuning_task = kwargs.pop('finetuning_task', None) - self.num_labels = kwargs.pop('num_labels', 2) - self.output_attentions = kwargs.pop('output_attentions', False) - self.output_hidden_states = kwargs.pop('output_hidden_states', False) - self.torchscript = kwargs.pop('torchscript', False) - self.pruned_heads = kwargs.pop('pruned_heads', {}) - - def save_pretrained(self, save_directory): - """ Save a configuration object to the directory `save_directory`, so that it - can be re-loaded using the :func:`~pytorch_transformers.PretrainedConfig.from_pretrained` class method. - """ - assert os.path.isdir(save_directory), "Saving path should be a directory where the model and configuration can be saved" - - # If we save using the predefined names, we can load using `from_pretrained` - output_config_file = os.path.join(save_directory, CONFIG_NAME) - - self.to_json_file(output_config_file) - - @classmethod - def from_pretrained(cls, pretrained_model_name_or_path, **kwargs): - r""" Instantiate a :class:`~pytorch_transformers.PretrainedConfig` (or a derived class) from a pre-trained model configuration. - - Parameters: - pretrained_model_name_or_path: either: - - - a string with the `shortcut name` of a pre-trained model configuration to load from cache or download, e.g.: ``bert-base-uncased``. - - a path to a `directory` containing a configuration file saved using the :func:`~pytorch_transformers.PretrainedConfig.save_pretrained` method, e.g.: ``./my_model_directory/``. - - a path or url to a saved configuration JSON `file`, e.g.: ``./my_model_directory/configuration.json``. - - cache_dir: (`optional`) string: - Path to a directory in which a downloaded pre-trained model - configuration should be cached if the standard cache should not be used. - - kwargs: (`optional`) dict: key/value pairs with which to update the configuration object after loading. - - - The values in kwargs of any keys which are configuration attributes will be used to override the loaded values. - - Behavior concerning key/value pairs whose keys are *not* configuration attributes is controlled by the `return_unused_kwargs` keyword parameter. - - force_download: (`optional`) boolean, default False: - Force to (re-)download the model weights and configuration files and override the cached versions if they exists. - - proxies: (`optional`) dict, default None: - A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. - The proxies are used on each request. - - return_unused_kwargs: (`optional`) bool: - - - If False, then this function returns just the final configuration object. - - If True, then this functions returns a tuple `(config, unused_kwargs)` where `unused_kwargs` is a dictionary consisting of the key/value pairs whose keys are not configuration attributes: ie the part of kwargs which has not been used to update `config` and is otherwise ignored. - - Examples:: - - # We can't instantiate directly the base class `PretrainedConfig` so let's show the examples on a - # derived class: BertConfig - config = BertConfig.from_pretrained('bert-base-uncased') # Download configuration from S3 and cache. - config = BertConfig.from_pretrained('./test/saved_model/') # E.g. config (or model) was saved using `save_pretrained('./test/saved_model/')` - config = BertConfig.from_pretrained('./test/saved_model/my_configuration.json') - config = BertConfig.from_pretrained('bert-base-uncased', output_attention=True, foo=False) - assert config.output_attention == True - config, unused_kwargs = BertConfig.from_pretrained('bert-base-uncased', output_attention=True, - foo=False, return_unused_kwargs=True) - assert config.output_attention == True - assert unused_kwargs == {'foo': False} - - """ - cache_dir = kwargs.pop('cache_dir', None) - force_download = kwargs.pop('force_download', False) - proxies = kwargs.pop('proxies', None) - return_unused_kwargs = kwargs.pop('return_unused_kwargs', False) - - if pretrained_model_name_or_path in cls.pretrained_config_archive_map: - config_file = cls.pretrained_config_archive_map[pretrained_model_name_or_path] - elif os.path.isdir(pretrained_model_name_or_path): - config_file = os.path.join(pretrained_model_name_or_path, CONFIG_NAME) - else: - config_file = pretrained_model_name_or_path - # redirect to the cache, if necessary - try: - resolved_config_file = cached_path(config_file, cache_dir=cache_dir, force_download=force_download, proxies=proxies) - except EnvironmentError as e: - if pretrained_model_name_or_path in cls.pretrained_config_archive_map: - logger.error( - "Couldn't reach server at '{}' to download pretrained model configuration file.".format( - config_file)) - else: - logger.error( - "Model name '{}' was not found in model name list ({}). " - "We assumed '{}' was a path or url but couldn't find any file " - "associated to this path or url.".format( - pretrained_model_name_or_path, - ', '.join(cls.pretrained_config_archive_map.keys()), - config_file)) - raise e - if resolved_config_file == config_file: - logger.info("loading configuration file {}".format(config_file)) - else: - logger.info("loading configuration file {} from cache at {}".format( - config_file, resolved_config_file)) - - # Load config - config = cls.from_json_file(resolved_config_file) - - if hasattr(config, 'pruned_heads'): - config.pruned_heads = dict((int(key), set(value)) for key, value in config.pruned_heads.items()) - - # Update config with kwargs if needed - to_remove = [] - for key, value in kwargs.items(): - if hasattr(config, key): - setattr(config, key, value) - to_remove.append(key) - for key in to_remove: - kwargs.pop(key, None) - - logger.info("Model config %s", config) - if return_unused_kwargs: - return config, kwargs - else: - return config - - @classmethod - def from_dict(cls, json_object): - """Constructs a `Config` from a Python dictionary of parameters.""" - config = cls(vocab_size_or_config_json_file=-1) - for key, value in json_object.items(): - config.__dict__[key] = value - return config - - @classmethod - def from_json_file(cls, json_file): - """Constructs a `BertConfig` from a json file of parameters.""" - with open(json_file, "r", encoding='utf-8') as reader: - text = reader.read() - return cls.from_dict(json.loads(text)) - - def __eq__(self, other): - return self.__dict__ == other.__dict__ - - def __repr__(self): - return str(self.to_json_string()) - - def to_dict(self): - """Serializes this instance to a Python dictionary.""" - output = copy.deepcopy(self.__dict__) - return output - - def to_json_string(self): - """Serializes this instance to a JSON string.""" - return json.dumps(self.to_dict(), indent=2, sort_keys=True) + "\n" - - def to_json_file(self, json_file_path): - """ Save this instance to a json file.""" - with open(json_file_path, "w", encoding='utf-8') as writer: - writer.write(self.to_json_string()) - - class PreTrainedModel(nn.Module): r""" Base class for all models. diff --git a/pytorch_transformers/modeling_xlm.py b/pytorch_transformers/modeling_xlm.py index 7e13f10495..34406f5b61 100644 --- a/pytorch_transformers/modeling_xlm.py +++ b/pytorch_transformers/modeling_xlm.py @@ -16,11 +16,8 @@ """ from __future__ import absolute_import, division, print_function, unicode_literals -import json import logging import math -import sys -from io import open import itertools import numpy as np @@ -30,8 +27,9 @@ from torch import nn from torch.nn import functional as F from torch.nn import CrossEntropyLoss, MSELoss -from .modeling_utils import (PretrainedConfig, PreTrainedModel, add_start_docstrings, - prune_linear_layer, SequenceSummary, SQuADHead) +from .modeling_utils import PreTrainedModel, prune_linear_layer, SequenceSummary, SQuADHead +from .configuration_xlm import XLMConfig +from .file_utils import add_start_docstrings logger = logging.getLogger(__name__) @@ -47,164 +45,6 @@ XLM_PRETRAINED_MODEL_ARCHIVE_MAP = { 'xlm-mlm-17-1280': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-mlm-17-1280-pytorch_model.bin", 'xlm-mlm-100-1280': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-mlm-100-1280-pytorch_model.bin", } -XLM_PRETRAINED_CONFIG_ARCHIVE_MAP = { - 'xlm-mlm-en-2048': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-mlm-en-2048-config.json", - 'xlm-mlm-ende-1024': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-mlm-ende-1024-config.json", - 'xlm-mlm-enfr-1024': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-mlm-enfr-1024-config.json", - 'xlm-mlm-enro-1024': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-mlm-enro-1024-config.json", - 'xlm-mlm-tlm-xnli15-1024': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-mlm-tlm-xnli15-1024-config.json", - 'xlm-mlm-xnli15-1024': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-mlm-xnli15-1024-config.json", - 'xlm-clm-enfr-1024': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-clm-enfr-1024-config.json", - 'xlm-clm-ende-1024': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-clm-ende-1024-config.json", - 'xlm-mlm-17-1280': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-mlm-17-1280-config.json", - 'xlm-mlm-100-1280': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-mlm-100-1280-config.json", -} - - -class XLMConfig(PretrainedConfig): - """Configuration class to store the configuration of a `XLMModel`. - - Args: - vocab_size_or_config_json_file: Vocabulary size of `inputs_ids` in `XLMModel`. - d_model: Size of the encoder layers and the pooler layer. - n_layer: Number of hidden layers in the Transformer encoder. - n_head: Number of attention heads for each attention layer in - the Transformer encoder. - d_inner: The size of the "intermediate" (i.e., feed-forward) - layer in the Transformer encoder. - ff_activation: The non-linear activation function (function or string) in the - encoder and pooler. If string, "gelu", "relu" and "swish" are supported. - untie_r: untie relative position biases - attn_type: 'bi' for XLM, 'uni' for Transformer-XL - - dropout: The dropout probabilitiy for all fully connected - layers in the embeddings, encoder, and pooler. - dropatt: The dropout ratio for the attention - probabilities. - max_position_embeddings: The maximum sequence length that this model might - ever be used with. Typically set this to something large just in case - (e.g., 512 or 1024 or 2048). - initializer_range: The sttdev of the truncated_normal_initializer for - initializing all weight matrices. - layer_norm_eps: The epsilon used by LayerNorm. - - dropout: float, dropout rate. - dropatt: float, dropout rate on attention probabilities. - init: str, the initialization scheme, either "normal" or "uniform". - init_range: float, initialize the parameters with a uniform distribution - in [-init_range, init_range]. Only effective when init="uniform". - init_std: float, initialize the parameters with a normal distribution - with mean 0 and stddev init_std. Only effective when init="normal". - mem_len: int, the number of tokens to cache. - reuse_len: int, the number of tokens in the currect batch to be cached - and reused in the future. - bi_data: bool, whether to use bidirectional input pipeline. - Usually set to True during pretraining and False during finetuning. - clamp_len: int, clamp all relative distances larger than clamp_len. - -1 means no clamping. - same_length: bool, whether to use the same attention length for each token. - """ - pretrained_config_archive_map = XLM_PRETRAINED_CONFIG_ARCHIVE_MAP - - def __init__(self, - vocab_size_or_config_json_file=30145, - emb_dim=2048, - n_layers=12, - n_heads=16, - dropout=0.1, - attention_dropout=0.1, - gelu_activation=True, - sinusoidal_embeddings=False, - causal=False, - asm=False, - n_langs=1, - use_lang_emb=True, - max_position_embeddings=512, - embed_init_std=2048 ** -0.5, - layer_norm_eps=1e-12, - init_std=0.02, - bos_index=0, - eos_index=1, - pad_index=2, - unk_index=3, - mask_index=5, - is_encoder=True, - - finetuning_task=None, - num_labels=2, - summary_type='first', - summary_use_proj=True, - summary_activation=None, - summary_proj_to_labels=True, - summary_first_dropout=0.1, - start_n_top=5, - end_n_top=5, - **kwargs): - """Constructs XLMConfig. - """ - super(XLMConfig, self).__init__(**kwargs) - - if isinstance(vocab_size_or_config_json_file, str) or (sys.version_info[0] == 2 - and isinstance(vocab_size_or_config_json_file, unicode)): - with open(vocab_size_or_config_json_file, "r", encoding='utf-8') as reader: - json_config = json.loads(reader.read()) - for key, value in json_config.items(): - self.__dict__[key] = value - elif isinstance(vocab_size_or_config_json_file, int): - self.n_words = vocab_size_or_config_json_file - self.emb_dim = emb_dim - self.n_layers = n_layers - self.n_heads = n_heads - self.dropout = dropout - self.attention_dropout = attention_dropout - self.gelu_activation = gelu_activation - self.sinusoidal_embeddings = sinusoidal_embeddings - self.causal = causal - self.asm = asm - self.n_langs = n_langs - self.use_lang_emb = use_lang_emb - self.layer_norm_eps = layer_norm_eps - self.bos_index = bos_index - self.eos_index = eos_index - self.pad_index = pad_index - self.unk_index = unk_index - self.mask_index = mask_index - self.is_encoder = is_encoder - self.max_position_embeddings = max_position_embeddings - self.embed_init_std = embed_init_std - self.init_std = init_std - self.finetuning_task = finetuning_task - self.num_labels = num_labels - self.summary_type = summary_type - self.summary_use_proj = summary_use_proj - self.summary_activation = summary_activation - self.summary_proj_to_labels = summary_proj_to_labels - self.summary_first_dropout = summary_first_dropout - self.start_n_top = start_n_top - self.end_n_top = end_n_top - else: - raise ValueError("First argument must be either a vocabulary size (int)" - " or the path to a pretrained model config file (str)") - - @property - def vocab_size(self): - return self.n_words - - @vocab_size.setter - def vocab_size(self, value): - self.n_words = value - - @property - def hidden_size(self): - return self.emb_dim - - @property - def num_attention_heads(self): - return self.n_heads - - @property - def num_hidden_layers(self): - return self.n_layers def create_sinusoidal_embeddings(n_pos, dim, out): diff --git a/pytorch_transformers/modeling_xlnet.py b/pytorch_transformers/modeling_xlnet.py index 0dfb33a27a..00c15080a1 100644 --- a/pytorch_transformers/modeling_xlnet.py +++ b/pytorch_transformers/modeling_xlnet.py @@ -29,9 +29,9 @@ from torch import nn from torch.nn import functional as F from torch.nn import CrossEntropyLoss, MSELoss -from .modeling_utils import (CONFIG_NAME, WEIGHTS_NAME, PretrainedConfig, PreTrainedModel, - SequenceSummary, PoolerAnswerClass, PoolerEndLogits, PoolerStartLogits, - add_start_docstrings) +from .modeling_utils import PreTrainedModel, prune_linear_layer, SequenceSummary, PoolerAnswerClass, PoolerEndLogits, PoolerStartLogits +from .configuration_xlnet import XLNetConfig +from .file_utils import add_start_docstrings logger = logging.getLogger(__name__) @@ -40,10 +40,6 @@ XLNET_PRETRAINED_MODEL_ARCHIVE_MAP = { 'xlnet-base-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/xlnet-base-cased-pytorch_model.bin", 'xlnet-large-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/xlnet-large-cased-pytorch_model.bin", } -XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP = { - 'xlnet-base-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/xlnet-base-cased-config.json", - 'xlnet-large-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/xlnet-large-cased-config.json", -} def build_tf_xlnet_to_pytorch_map(model, config, tf_weights=None): @@ -192,147 +188,6 @@ def swish(x): ACT2FN = {"gelu": gelu, "relu": torch.nn.functional.relu, "swish": swish} -class XLNetConfig(PretrainedConfig): - """Configuration class to store the configuration of a ``XLNetModel``. - - Args: - vocab_size_or_config_json_file: Vocabulary size of ``inputs_ids`` in ``XLNetModel``. - d_model: Size of the encoder layers and the pooler layer. - n_layer: Number of hidden layers in the Transformer encoder. - n_head: Number of attention heads for each attention layer in - the Transformer encoder. - d_inner: The size of the "intermediate" (i.e., feed-forward) - layer in the Transformer encoder. - ff_activation: The non-linear activation function (function or string) in the - encoder and pooler. If string, "gelu", "relu" and "swish" are supported. - untie_r: untie relative position biases - attn_type: 'bi' for XLNet, 'uni' for Transformer-XL - - dropout: The dropout probabilitiy for all fully connected - layers in the embeddings, encoder, and pooler. - dropatt: The dropout ratio for the attention - probabilities. - initializer_range: The sttdev of the truncated_normal_initializer for - initializing all weight matrices. - layer_norm_eps: The epsilon used by LayerNorm. - - dropout: float, dropout rate. - dropatt: float, dropout rate on attention probabilities. - init: str, the initialization scheme, either "normal" or "uniform". - init_range: float, initialize the parameters with a uniform distribution - in [-init_range, init_range]. Only effective when init="uniform". - init_std: float, initialize the parameters with a normal distribution - with mean 0 and stddev init_std. Only effective when init="normal". - mem_len: int, the number of tokens to cache. - reuse_len: int, the number of tokens in the currect batch to be cached - and reused in the future. - bi_data: bool, whether to use bidirectional input pipeline. - Usually set to True during pretraining and False during finetuning. - clamp_len: int, clamp all relative distances larger than clamp_len. - -1 means no clamping. - same_length: bool, whether to use the same attention length for each token. - finetuning_task: name of the glue task on which the model was fine-tuned if any - """ - pretrained_config_archive_map = XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP - - def __init__(self, - vocab_size_or_config_json_file=32000, - d_model=1024, - n_layer=24, - n_head=16, - d_inner=4096, - ff_activation="gelu", - untie_r=True, - attn_type="bi", - - initializer_range=0.02, - layer_norm_eps=1e-12, - - dropout=0.1, - mem_len=None, - reuse_len=None, - bi_data=False, - clamp_len=-1, - same_length=False, - - finetuning_task=None, - num_labels=2, - summary_type='last', - summary_use_proj=True, - summary_activation='tanh', - summary_last_dropout=0.1, - start_n_top=5, - end_n_top=5, - **kwargs): - """Constructs XLNetConfig. - """ - super(XLNetConfig, self).__init__(**kwargs) - - if isinstance(vocab_size_or_config_json_file, str) or (sys.version_info[0] == 2 - and isinstance(vocab_size_or_config_json_file, unicode)): - with open(vocab_size_or_config_json_file, "r", encoding='utf-8') as reader: - json_config = json.loads(reader.read()) - for key, value in json_config.items(): - self.__dict__[key] = value - elif isinstance(vocab_size_or_config_json_file, int): - self.n_token = vocab_size_or_config_json_file - self.d_model = d_model - self.n_layer = n_layer - self.n_head = n_head - assert d_model % n_head == 0 - self.d_head = d_model // n_head - self.ff_activation = ff_activation - self.d_inner = d_inner - self.untie_r = untie_r - self.attn_type = attn_type - - self.initializer_range = initializer_range - self.layer_norm_eps = layer_norm_eps - - self.dropout = dropout - self.mem_len = mem_len - self.reuse_len = reuse_len - self.bi_data = bi_data - self.clamp_len = clamp_len - self.same_length = same_length - - self.finetuning_task = finetuning_task - self.num_labels = num_labels - self.summary_type = summary_type - self.summary_use_proj = summary_use_proj - self.summary_activation = summary_activation - self.summary_last_dropout = summary_last_dropout - self.start_n_top = start_n_top - self.end_n_top = end_n_top - else: - raise ValueError("First argument must be either a vocabulary size (int)" - " or the path to a pretrained model config file (str)") - - @property - def max_position_embeddings(self): - return -1 - - @property - def vocab_size(self): - return self.n_token - - @vocab_size.setter - def vocab_size(self, value): - self.n_token = value - - @property - def hidden_size(self): - return self.d_model - - @property - def num_attention_heads(self): - return self.n_head - - @property - def num_hidden_layers(self): - return self.n_layer - - try: from apex.normalization.fused_layer_norm import FusedLayerNorm as XLNetLayerNorm except (ImportError, AttributeError) as e: diff --git a/pytorch_transformers/tests/configuration_common_test.py b/pytorch_transformers/tests/configuration_common_test.py new file mode 100644 index 0000000000..8ee751153c --- /dev/null +++ b/pytorch_transformers/tests/configuration_common_test.py @@ -0,0 +1,63 @@ +# coding=utf-8 +# Copyright 2019 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import copy +import os +import shutil +import json +import random +import uuid + +import unittest +import logging + + +class ConfigTester(object): + def __init__(self, parent, config_class=None, **kwargs): + self.parent = parent + self.config_class = config_class + self.inputs_dict = kwargs + + def create_and_test_config_common_properties(self): + config = self.config_class(**self.inputs_dict) + self.parent.assertTrue(hasattr(config, 'vocab_size')) + self.parent.assertTrue(hasattr(config, 'hidden_size')) + self.parent.assertTrue(hasattr(config, 'num_attention_heads')) + self.parent.assertTrue(hasattr(config, 'num_hidden_layers')) + + def create_and_test_config_to_json_string(self): + config = self.config_class(**self.inputs_dict) + obj = json.loads(config.to_json_string()) + for key, value in self.inputs_dict.items(): + self.parent.assertEqual(obj[key], value) + + def create_and_test_config_to_json_file(self): + config_first = self.config_class(**self.inputs_dict) + json_file_path = os.path.join(os.getcwd(), "config_" + str(uuid.uuid4()) + ".json") + config_first.to_json_file(json_file_path) + config_second = self.config_class.from_json_file(json_file_path) + os.remove(json_file_path) + self.parent.assertEqual(config_second.to_dict(), config_first.to_dict()) + + def run_common_tests(self): + self.create_and_test_config_common_properties() + self.create_and_test_config_to_json_string() + self.create_and_test_config_to_json_file() + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/pytorch_transformers/tests/modeling_auto_test.py b/pytorch_transformers/tests/modeling_auto_test.py index 09d09b28fc..dfdedbbe61 100644 --- a/pytorch_transformers/tests/modeling_auto_test.py +++ b/pytorch_transformers/tests/modeling_auto_test.py @@ -28,7 +28,8 @@ from pytorch_transformers import (AutoConfig, BertConfig, AutoModelForQuestionAnswering, BertForQuestionAnswering) from pytorch_transformers.modeling_bert import BERT_PRETRAINED_MODEL_ARCHIVE_MAP -from .modeling_common_test import (CommonTestCases, ConfigTester, ids_tensor) +from .modeling_common_test import (CommonTestCases, ids_tensor) +from .configuration_common_test import ConfigTester class AutoModelTest(unittest.TestCase): diff --git a/pytorch_transformers/tests/modeling_bert_test.py b/pytorch_transformers/tests/modeling_bert_test.py index e1cce38479..2919cc0336 100644 --- a/pytorch_transformers/tests/modeling_bert_test.py +++ b/pytorch_transformers/tests/modeling_bert_test.py @@ -26,7 +26,8 @@ from pytorch_transformers import (BertConfig, BertModel, BertForMaskedLM, BertForTokenClassification, BertForMultipleChoice) from pytorch_transformers.modeling_bert import BERT_PRETRAINED_MODEL_ARCHIVE_MAP -from .modeling_common_test import (CommonTestCases, ConfigTester, ids_tensor) +from .modeling_common_test import (CommonTestCases, ids_tensor) +from .configuration_common_test import ConfigTester class BertModelTest(CommonTestCases.CommonModelTester): diff --git a/pytorch_transformers/tests/modeling_common_test.py b/pytorch_transformers/tests/modeling_common_test.py index 8b1a70fcf3..c50d6678d8 100644 --- a/pytorch_transformers/tests/modeling_common_test.py +++ b/pytorch_transformers/tests/modeling_common_test.py @@ -28,9 +28,9 @@ import logging import torch -from pytorch_transformers import PretrainedConfig, PreTrainedModel -from pytorch_transformers.modeling_bert import BertModel, BertConfig, BERT_PRETRAINED_MODEL_ARCHIVE_MAP -from pytorch_transformers.modeling_gpt2 import GPT2LMHeadModel, GPT2Config, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP +from pytorch_transformers import (PretrainedConfig, PreTrainedModel, + BertModel, BertConfig, BERT_PRETRAINED_MODEL_ARCHIVE_MAP, + GPT2LMHeadModel, GPT2Config, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP) def _config_zero_init(config): diff --git a/pytorch_transformers/tests/modeling_distilbert_test.py b/pytorch_transformers/tests/modeling_distilbert_test.py index 7b0b6b8266..619dbf5850 100644 --- a/pytorch_transformers/tests/modeling_distilbert_test.py +++ b/pytorch_transformers/tests/modeling_distilbert_test.py @@ -18,13 +18,15 @@ from __future__ import print_function import unittest import shutil +import sys import pytest from pytorch_transformers import (DistilBertConfig, DistilBertModel, DistilBertForMaskedLM, DistilBertForQuestionAnswering, DistilBertForSequenceClassification) from pytorch_transformers.modeling_distilbert import DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP -from .modeling_common_test import (CommonTestCases, ConfigTester, ids_tensor) +from .modeling_common_test import (CommonTestCases, ids_tensor) +from .configuration_common_test import ConfigTester class DistilBertModelTest(CommonTestCases.CommonModelTester): diff --git a/pytorch_transformers/tests/modeling_gpt2_test.py b/pytorch_transformers/tests/modeling_gpt2_test.py index 1786ada54c..2717805120 100644 --- a/pytorch_transformers/tests/modeling_gpt2_test.py +++ b/pytorch_transformers/tests/modeling_gpt2_test.py @@ -24,7 +24,8 @@ import shutil from pytorch_transformers import (GPT2Config, GPT2Model, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP, GPT2LMHeadModel, GPT2DoubleHeadsModel) -from .modeling_common_test import CommonTestCases, ConfigTester, ids_tensor +from .modeling_common_test import (CommonTestCases, ids_tensor) +from .configuration_common_test import ConfigTester class GPT2ModelTest(CommonTestCases.CommonModelTester): diff --git a/pytorch_transformers/tests/modeling_openai_test.py b/pytorch_transformers/tests/modeling_openai_test.py index 0fcb4b7d64..dbef6c52eb 100644 --- a/pytorch_transformers/tests/modeling_openai_test.py +++ b/pytorch_transformers/tests/modeling_openai_test.py @@ -24,7 +24,8 @@ import shutil from pytorch_transformers import (OpenAIGPTConfig, OpenAIGPTModel, OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP, OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel) -from .modeling_common_test import CommonTestCases, ConfigTester, ids_tensor +from .modeling_common_test import (CommonTestCases, ids_tensor) +from .configuration_common_test import ConfigTester class OpenAIGPTModelTest(CommonTestCases.CommonModelTester): diff --git a/pytorch_transformers/tests/modeling_roberta_test.py b/pytorch_transformers/tests/modeling_roberta_test.py index 0279f3756b..69981af222 100644 --- a/pytorch_transformers/tests/modeling_roberta_test.py +++ b/pytorch_transformers/tests/modeling_roberta_test.py @@ -24,7 +24,8 @@ import torch from pytorch_transformers import (RobertaConfig, RobertaModel, RobertaForMaskedLM, RobertaForSequenceClassification) from pytorch_transformers.modeling_roberta import ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP -from .modeling_common_test import (CommonTestCases, ConfigTester, ids_tensor) +from .modeling_common_test import (CommonTestCases, ids_tensor) +from .configuration_common_test import ConfigTester class RobertaModelTest(CommonTestCases.CommonModelTester): diff --git a/pytorch_transformers/tests/modeling_transfo_xl_test.py b/pytorch_transformers/tests/modeling_transfo_xl_test.py index e3c0fbcdf0..a060432cc4 100644 --- a/pytorch_transformers/tests/modeling_transfo_xl_test.py +++ b/pytorch_transformers/tests/modeling_transfo_xl_test.py @@ -28,7 +28,8 @@ import torch from pytorch_transformers import (TransfoXLConfig, TransfoXLModel, TransfoXLLMHeadModel) from pytorch_transformers.modeling_transfo_xl import TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP -from .modeling_common_test import ConfigTester, CommonTestCases, ids_tensor +from .modeling_common_test import (CommonTestCases, ids_tensor) +from .configuration_common_test import ConfigTester class TransfoXLModelTest(CommonTestCases.CommonModelTester): diff --git a/pytorch_transformers/tests/modeling_xlm_test.py b/pytorch_transformers/tests/modeling_xlm_test.py index 4308c18d45..dcd0963477 100644 --- a/pytorch_transformers/tests/modeling_xlm_test.py +++ b/pytorch_transformers/tests/modeling_xlm_test.py @@ -23,7 +23,8 @@ import pytest from pytorch_transformers import (XLMConfig, XLMModel, XLMWithLMHeadModel, XLMForQuestionAnswering, XLMForSequenceClassification) from pytorch_transformers.modeling_xlm import XLM_PRETRAINED_MODEL_ARCHIVE_MAP -from .modeling_common_test import (CommonTestCases, ConfigTester, ids_tensor) +from .modeling_common_test import (CommonTestCases, ids_tensor) +from .configuration_common_test import ConfigTester class XLMModelTest(CommonTestCases.CommonModelTester): diff --git a/pytorch_transformers/tests/modeling_xlnet_test.py b/pytorch_transformers/tests/modeling_xlnet_test.py index 290c5766e2..4445bc17ac 100644 --- a/pytorch_transformers/tests/modeling_xlnet_test.py +++ b/pytorch_transformers/tests/modeling_xlnet_test.py @@ -28,7 +28,8 @@ import torch from pytorch_transformers import (XLNetConfig, XLNetModel, XLNetLMHeadModel, XLNetForSequenceClassification, XLNetForQuestionAnswering) from pytorch_transformers.modeling_xlnet import XLNET_PRETRAINED_MODEL_ARCHIVE_MAP -from .modeling_common_test import ConfigTester, CommonTestCases, ids_tensor +from .modeling_common_test import (CommonTestCases, ids_tensor) +from .configuration_common_test import ConfigTester class XLNetModelTest(CommonTestCases.CommonModelTester): From 69bff89935a74cf429bd482543cf206c9c27be2f Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 5 Sep 2019 00:41:24 +0200 Subject: [PATCH 015/219] clean ups --- pytorch_transformers/tests/modeling_distilbert_test.py | 4 ---- pytorch_transformers/tests/modeling_transfo_xl_test.py | 2 -- 2 files changed, 6 deletions(-) diff --git a/pytorch_transformers/tests/modeling_distilbert_test.py b/pytorch_transformers/tests/modeling_distilbert_test.py index 619dbf5850..0d9f231177 100644 --- a/pytorch_transformers/tests/modeling_distilbert_test.py +++ b/pytorch_transformers/tests/modeling_distilbert_test.py @@ -17,13 +17,9 @@ from __future__ import division from __future__ import print_function import unittest -import shutil -import sys -import pytest from pytorch_transformers import (DistilBertConfig, DistilBertModel, DistilBertForMaskedLM, DistilBertForQuestionAnswering, DistilBertForSequenceClassification) -from pytorch_transformers.modeling_distilbert import DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester diff --git a/pytorch_transformers/tests/modeling_transfo_xl_test.py b/pytorch_transformers/tests/modeling_transfo_xl_test.py index a060432cc4..f482c47202 100644 --- a/pytorch_transformers/tests/modeling_transfo_xl_test.py +++ b/pytorch_transformers/tests/modeling_transfo_xl_test.py @@ -16,9 +16,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import os import unittest -import json import random import shutil import pytest From 7ae642b72deef152fc61c8033065a839030fa7ef Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 5 Sep 2019 02:17:50 +0200 Subject: [PATCH 016/219] update conversion scripts --- pytorch_transformers/__init__.py | 11 +++++++++++ .../convert_gpt2_checkpoint_to_pytorch.py | 2 +- .../convert_openai_checkpoint_to_pytorch.py | 2 +- .../convert_pytorch_checkpoint_to_tf.py | 2 +- .../convert_roberta_checkpoint_to_pytorch.py | 4 ++-- .../convert_tf_checkpoint_to_pytorch.py | 2 +- .../convert_transfo_xl_checkpoint_to_pytorch.py | 2 +- .../convert_xlm_checkpoint_to_pytorch.py | 2 +- .../convert_xlnet_checkpoint_to_pytorch.py | 2 +- 9 files changed, 20 insertions(+), 9 deletions(-) diff --git a/pytorch_transformers/__init__.py b/pytorch_transformers/__init__.py index 04a73c3abc..b851c99c9d 100644 --- a/pytorch_transformers/__init__.py +++ b/pytorch_transformers/__init__.py @@ -1,4 +1,15 @@ __version__ = "1.2.0" +# Work around to update TensorFlow's absl.logging threshold which alters the +# default Python logging output behavior when present. +# see: https://github.com/abseil/abseil-py/issues/99 +# and: https://github.com/tensorflow/tensorflow/issues/26691#issuecomment-500369493 +try: + import absl.logging + absl.logging.set_verbosity('info') + absl.logging.set_stderrthreshold('info') + absl.logging._warn_preinit_stderr = False +except: + pass # Tokenizer from .tokenization_utils import (PreTrainedTokenizer) diff --git a/pytorch_transformers/convert_gpt2_checkpoint_to_pytorch.py b/pytorch_transformers/convert_gpt2_checkpoint_to_pytorch.py index e9bfa0302a..eb5b3009b4 100755 --- a/pytorch_transformers/convert_gpt2_checkpoint_to_pytorch.py +++ b/pytorch_transformers/convert_gpt2_checkpoint_to_pytorch.py @@ -21,7 +21,7 @@ from io import open import torch -from pytorch_transformers.modeling_gpt2 import (CONFIG_NAME, WEIGHTS_NAME, +from pytorch_transformers import (CONFIG_NAME, WEIGHTS_NAME, GPT2Config, GPT2Model, load_tf_weights_in_gpt2) diff --git a/pytorch_transformers/convert_openai_checkpoint_to_pytorch.py b/pytorch_transformers/convert_openai_checkpoint_to_pytorch.py index 3009f8a99e..5eecdd9648 100755 --- a/pytorch_transformers/convert_openai_checkpoint_to_pytorch.py +++ b/pytorch_transformers/convert_openai_checkpoint_to_pytorch.py @@ -21,7 +21,7 @@ from io import open import torch -from pytorch_transformers.modeling_openai import (CONFIG_NAME, WEIGHTS_NAME, +from pytorch_transformers import (CONFIG_NAME, WEIGHTS_NAME, OpenAIGPTConfig, OpenAIGPTModel, load_tf_weights_in_openai_gpt) diff --git a/pytorch_transformers/convert_pytorch_checkpoint_to_tf.py b/pytorch_transformers/convert_pytorch_checkpoint_to_tf.py index 4857ea8d80..15fd6bf5ac 100644 --- a/pytorch_transformers/convert_pytorch_checkpoint_to_tf.py +++ b/pytorch_transformers/convert_pytorch_checkpoint_to_tf.py @@ -20,7 +20,7 @@ import argparse import torch import numpy as np import tensorflow as tf -from pytorch_transformers.modeling import BertModel +from pytorch_transformers import BertModel def convert_pytorch_checkpoint_to_tf(model:BertModel, ckpt_dir:str, model_name:str): diff --git a/pytorch_transformers/convert_roberta_checkpoint_to_pytorch.py b/pytorch_transformers/convert_roberta_checkpoint_to_pytorch.py index 743013e4c4..9f74254daa 100644 --- a/pytorch_transformers/convert_roberta_checkpoint_to_pytorch.py +++ b/pytorch_transformers/convert_roberta_checkpoint_to_pytorch.py @@ -23,12 +23,12 @@ import torch from fairseq.models.roberta import RobertaModel as FairseqRobertaModel from fairseq.modules import TransformerSentenceEncoderLayer -from pytorch_transformers.modeling_bert import (BertConfig, BertEncoder, +from pytorch_transformers import (BertConfig, BertEncoder, BertIntermediate, BertLayer, BertModel, BertOutput, BertSelfAttention, BertSelfOutput) -from pytorch_transformers.modeling_roberta import (RobertaEmbeddings, +from pytorch_transformers import (RobertaEmbeddings, RobertaForMaskedLM, RobertaForSequenceClassification, RobertaModel) diff --git a/pytorch_transformers/convert_tf_checkpoint_to_pytorch.py b/pytorch_transformers/convert_tf_checkpoint_to_pytorch.py index 220204f36e..d382d3588e 100755 --- a/pytorch_transformers/convert_tf_checkpoint_to_pytorch.py +++ b/pytorch_transformers/convert_tf_checkpoint_to_pytorch.py @@ -21,7 +21,7 @@ from __future__ import print_function import argparse import torch -from pytorch_transformers.modeling_bert import BertConfig, BertForPreTraining, load_tf_weights_in_bert +from pytorch_transformers import BertConfig, BertForPreTraining, load_tf_weights_in_bert import logging logging.basicConfig(level=logging.INFO) diff --git a/pytorch_transformers/convert_transfo_xl_checkpoint_to_pytorch.py b/pytorch_transformers/convert_transfo_xl_checkpoint_to_pytorch.py index 7e79d58d7d..b310b73453 100755 --- a/pytorch_transformers/convert_transfo_xl_checkpoint_to_pytorch.py +++ b/pytorch_transformers/convert_transfo_xl_checkpoint_to_pytorch.py @@ -26,7 +26,7 @@ import torch import pytorch_transformers.tokenization_transfo_xl as data_utils from pytorch_transformers import CONFIG_NAME, WEIGHTS_NAME -from pytorch_transformers.modeling_transfo_xl import (TransfoXLConfig, TransfoXLLMHeadModel, +from pytorch_transformers import (TransfoXLConfig, TransfoXLLMHeadModel, load_tf_weights_in_transfo_xl) from pytorch_transformers.tokenization_transfo_xl import (CORPUS_NAME, VOCAB_FILES_NAMES) diff --git a/pytorch_transformers/convert_xlm_checkpoint_to_pytorch.py b/pytorch_transformers/convert_xlm_checkpoint_to_pytorch.py index bf4b99b6ea..d6a3cd89e7 100755 --- a/pytorch_transformers/convert_xlm_checkpoint_to_pytorch.py +++ b/pytorch_transformers/convert_xlm_checkpoint_to_pytorch.py @@ -23,7 +23,7 @@ from io import open import torch import numpy -from pytorch_transformers.modeling_utils import CONFIG_NAME, WEIGHTS_NAME +from pytorch_transformers import CONFIG_NAME, WEIGHTS_NAME from pytorch_transformers.tokenization_xlm import VOCAB_FILES_NAMES import logging diff --git a/pytorch_transformers/convert_xlnet_checkpoint_to_pytorch.py b/pytorch_transformers/convert_xlnet_checkpoint_to_pytorch.py index 038c706960..a36fa514b5 100755 --- a/pytorch_transformers/convert_xlnet_checkpoint_to_pytorch.py +++ b/pytorch_transformers/convert_xlnet_checkpoint_to_pytorch.py @@ -22,7 +22,7 @@ import os import argparse import torch -from pytorch_transformers.modeling_xlnet import (CONFIG_NAME, WEIGHTS_NAME, +from pytorch_transformers import (CONFIG_NAME, WEIGHTS_NAME, XLNetConfig, XLNetLMHeadModel, XLNetForQuestionAnswering, XLNetForSequenceClassification, From d68a8fe462ad836dc2127b21b456f1d8d65bec3b Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 5 Sep 2019 02:27:39 +0200 Subject: [PATCH 017/219] add tf bert files --- pytorch_transformers/file_utils.py | 6 + pytorch_transformers/modeling_tf_bert.py | 832 ++++++++++++++++++ pytorch_transformers/modeling_tf_utils.py | 255 ++++++ .../tests/modeling_tf_common_test.py | 308 +++++++ .../tests/modeling_tf_test.py | 327 +++++++ 5 files changed, 1728 insertions(+) create mode 100644 pytorch_transformers/modeling_tf_bert.py create mode 100644 pytorch_transformers/modeling_tf_utils.py create mode 100644 pytorch_transformers/tests/modeling_tf_common_test.py create mode 100644 pytorch_transformers/tests/modeling_tf_test.py diff --git a/pytorch_transformers/file_utils.py b/pytorch_transformers/file_utils.py index 37ebc57fb3..94ada93d91 100644 --- a/pytorch_transformers/file_utils.py +++ b/pytorch_transformers/file_utils.py @@ -79,6 +79,9 @@ def url_to_filename(url, etag=None): Convert `url` into a hashed filename in a repeatable way. If `etag` is specified, append its hash to the url's, delimited by a period. + If the url ends with .h5 (Keras HDF5 weights) ands '.h5' to the name + so that TF 2.0 can identify it as a HDF5 file + (see https://github.com/tensorflow/tensorflow/blob/00fad90125b18b80fe054de1055770cfb8fe4ba3/tensorflow/python/keras/engine/network.py#L1380) """ url_bytes = url.encode('utf-8') url_hash = sha256(url_bytes) @@ -89,6 +92,9 @@ def url_to_filename(url, etag=None): etag_hash = sha256(etag_bytes) filename += '.' + etag_hash.hexdigest() + if url.endswith('.h5'): + filename += '.h5' + return filename diff --git a/pytorch_transformers/modeling_tf_bert.py b/pytorch_transformers/modeling_tf_bert.py new file mode 100644 index 0000000000..dda713fe7d --- /dev/null +++ b/pytorch_transformers/modeling_tf_bert.py @@ -0,0 +1,832 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" TF 2.0 BERT model. """ + +from __future__ import absolute_import, division, print_function, unicode_literals + +import json +import logging +import math +import os +import sys +from io import open + +import numpy as np +import tensorflow as tf + +from .configuration_bert import BertConfig +from .modeling_tf_utils import TFPreTrainedModel +from .file_utils import add_start_docstrings + +logger = logging.getLogger(__name__) + + +TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP = { + 'bert-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-tf_model.h5", + 'bert-large-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-tf_model.h5", + 'bert-base-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-tf_model.h5", + 'bert-large-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-tf_model.h5", + 'bert-base-multilingual-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-uncased-tf_model.h5", + 'bert-base-multilingual-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-cased-tf_model.h5", + 'bert-base-chinese': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-chinese-tf_model.h5", + 'bert-base-german-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-german-cased-tf_model.h5", + 'bert-large-uncased-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-whole-word-masking-tf_model.h5", + 'bert-large-cased-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-whole-word-masking-tf_model.h5", + 'bert-large-uncased-whole-word-masking-finetuned-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-whole-word-masking-finetuned-squad-tf_model.h5", + 'bert-large-cased-whole-word-masking-finetuned-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-whole-word-masking-finetuned-squad-tf_model.h5", + 'bert-base-cased-finetuned-mrpc': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-finetuned-mrpc-tf_model.h5", +} + + +def load_pt_weights_in_bert(tf_model, config, pytorch_checkpoint_path): + """ Load pytorch checkpoints in a TF 2.0 model and save it using HDF5 format + We use HDF5 to easily do transfer learning + (see https://github.com/tensorflow/tensorflow/blob/ee16fcac960ae660e0e4496658a366e2f745e1f0/tensorflow/python/keras/engine/network.py#L1352-L1357). + """ + try: + import re + import torch + import numpy + from tensorflow.python.keras import backend as K + except ImportError: + logger.error("Loading a PyTorch model in TensorFlow, requires PyTorch to be installed. Please see " + "https://pytorch.org/ for installation instructions.") + raise + + pt_path = os.path.abspath(pytorch_checkpoint_path) + logger.info("Loading PyTorch weights from {}".format(pt_path)) + # Load pytorch model + state_dict = torch.load(pt_path, map_location='cpu') + + inputs_list = [[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]] + tf_inputs = tf.constant(inputs_list) + tfo = tf_model(tf_inputs, training=False) # build the network + + symbolic_weights = tf_model.trainable_weights + tf_model.non_trainable_weights + weight_value_tuples = [] + for symbolic_weight in symbolic_weights: + name = symbolic_weight.name + name = name.replace('cls_mlm', 'cls') # We had to split this layer in two in the TF model to be + name = name.replace('cls_nsp', 'cls') # able to do transfer learning (Keras only allow to remove full layers) + name = name.replace(':0', '') + name = name.replace('layer_', 'layer/') + name = name.split('/') + name = name[1:] + + transpose = bool(name[-1] == 'kernel') + if name[-1] == 'kernel' or name[-1] == 'embeddings': + name[-1] = 'weight' + + name = '.'.join(name) + assert name in state_dict + array = state_dict[name].numpy() + + if transpose: + array = numpy.transpose(array) + + try: + assert list(symbolic_weight.shape) == list(array.shape) + except AssertionError as e: + e.args += (symbolic_weight.shape, array.shape) + raise e + + logger.info("Initialize TF weight {}".format(symbolic_weight.name)) + + weight_value_tuples.append((symbolic_weight, array)) + + K.batch_set_value(weight_value_tuples) + + tfo = tf_model(tf_inputs, training=False) # Make sure restore ops are run + return tf_model + + +def gelu(x): + """Gaussian Error Linear Unit. + This is a smoother version of the RELU. + Original paper: https://arxiv.org/abs/1606.08415 + Args: + x: float Tensor to perform activation. + Returns: + `x` with the GELU activation applied. + """ + cdf = 0.5 * (1.0 + tf.tanh( + (np.sqrt(2 / np.pi) * (x + 0.044715 * tf.pow(x, 3))))) + return x * cdf + + +def swish(x): + return x * tf.sigmoid(x) + + +ACT2FN = {"gelu": tf.keras.layers.Activation(gelu), + "relu": tf.keras.activations.relu, + "swish": tf.keras.layers.Activation(swish)} + + +class TFBertEmbeddings(tf.keras.layers.Layer): + """Construct the embeddings from word, position and token_type embeddings. + """ + def __init__(self, config, **kwargs): + super(TFBertEmbeddings, self).__init__(**kwargs) + self.word_embeddings = tf.keras.layers.Embedding(config.vocab_size, config.hidden_size, name='word_embeddings') + self.position_embeddings = tf.keras.layers.Embedding(config.max_position_embeddings, config.hidden_size, name='position_embeddings') + self.token_type_embeddings = tf.keras.layers.Embedding(config.type_vocab_size, config.hidden_size, name='token_type_embeddings') + + # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load + # any TensorFlow checkpoint file + self.LayerNorm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name='LayerNorm') + self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) + + def call(self, inputs, training=False): + input_ids, position_ids, token_type_ids = inputs + + seq_length = tf.shape(input_ids)[1] + if position_ids is None: + position_ids = tf.range(seq_length, dtype=tf.int32)[tf.newaxis, :] + if token_type_ids is None: + token_type_ids = tf.fill(tf.shape(input_ids), 0) + + words_embeddings = self.word_embeddings(input_ids) + position_embeddings = self.position_embeddings(position_ids) + token_type_embeddings = self.token_type_embeddings(token_type_ids) + + embeddings = words_embeddings + position_embeddings + token_type_embeddings + embeddings = self.LayerNorm(embeddings) + if training: + embeddings = self.dropout(embeddings) + return embeddings + + +class TFBertSelfAttention(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFBertSelfAttention, self).__init__(**kwargs) + if config.hidden_size % config.num_attention_heads != 0: + raise ValueError( + "The hidden size (%d) is not a multiple of the number of attention " + "heads (%d)" % (config.hidden_size, config.num_attention_heads)) + self.output_attentions = config.output_attentions + + self.num_attention_heads = config.num_attention_heads + assert config.hidden_size % config.num_attention_heads == 0 + self.attention_head_size = int(config.hidden_size / config.num_attention_heads) + self.all_head_size = self.num_attention_heads * self.attention_head_size + + self.query = tf.keras.layers.Dense(self.all_head_size, name='query') + self.key = tf.keras.layers.Dense(self.all_head_size, name='key') + self.value = tf.keras.layers.Dense(self.all_head_size, name='value') + + self.dropout = tf.keras.layers.Dropout(config.attention_probs_dropout_prob) + + def transpose_for_scores(self, x, batch_size): + x = tf.reshape(x, (batch_size, -1, self.num_attention_heads, self.attention_head_size)) + return tf.transpose(x, perm=[0, 2, 1, 3]) + + def call(self, inputs, training=False): + hidden_states, attention_mask, head_mask = inputs + + batch_size = tf.shape(hidden_states)[0] + mixed_query_layer = self.query(hidden_states) + mixed_key_layer = self.key(hidden_states) + mixed_value_layer = self.value(hidden_states) + + query_layer = self.transpose_for_scores(mixed_query_layer, batch_size) + key_layer = self.transpose_for_scores(mixed_key_layer, batch_size) + value_layer = self.transpose_for_scores(mixed_value_layer, batch_size) + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = tf.matmul(query_layer, key_layer, transpose_b=True) # (batch size, num_heads, seq_len_q, seq_len_k) + dk = tf.cast(tf.shape(key_layer)[-1], tf.float32) # scale attention_scores + attention_scores = attention_scores / tf.math.sqrt(dk) + # Apply the attention mask is (precomputed for all layers in TFBertModel call() function) + attention_scores = attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs = tf.nn.softmax(attention_scores, axis=-1) + + if training: + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.dropout(attention_probs) + + # Mask heads if we want to + if head_mask is not None: + attention_probs = attention_probs * head_mask + + context_layer = tf.matmul(attention_probs, value_layer) + + context_layer = tf.transpose(context_layer, perm=[0, 2, 1, 3]) + context_layer = tf.reshape(context_layer, + (batch_size, -1, self.all_head_size)) # (batch_size, seq_len_q, all_head_size) + + outputs = (context_layer, attention_probs) if self.output_attentions else (context_layer,) + return outputs + + +class TFBertSelfOutput(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFBertSelfOutput, self).__init__(**kwargs) + self.dense = tf.keras.layers.Dense(config.hidden_size, name='dense') + self.LayerNorm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name='LayerNorm') + self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) + + def call(self, inputs, training=False): + hidden_states, input_tensor = inputs + + hidden_states = self.dense(hidden_states) + if training: + hidden_states = self.dropout(hidden_states) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +class TFBertAttention(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFBertAttention, self).__init__(**kwargs) + self.self_attention = TFBertSelfAttention(config, name='self') + self.dense_output = TFBertSelfOutput(config, name='output') + + def prune_heads(self, heads): + raise NotImplementedError + + def call(self, inputs, training=False): + input_tensor, attention_mask, head_mask = inputs + + self_outputs = self.self_attention([input_tensor, attention_mask, head_mask], training=training) + attention_output = self.dense_output([self_outputs[0], input_tensor], training=training) + outputs = (attention_output,) + self_outputs[1:] # add attentions if we output them + return outputs + + +class TFBertIntermediate(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFBertIntermediate, self).__init__(**kwargs) + self.dense = tf.keras.layers.Dense(config.intermediate_size, name='dense') + if isinstance(config.hidden_act, str) or (sys.version_info[0] == 2 and isinstance(config.hidden_act, unicode)): + self.intermediate_act_fn = ACT2FN[config.hidden_act] + else: + self.intermediate_act_fn = config.hidden_act + + def call(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.intermediate_act_fn(hidden_states) + return hidden_states + + +class TFBertOutput(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFBertOutput, self).__init__(**kwargs) + self.dense = tf.keras.layers.Dense(config.hidden_size, name='dense') + self.LayerNorm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name='LayerNorm') + self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) + + def call(self, inputs, training=False): + hidden_states, input_tensor = inputs + + hidden_states = self.dense(hidden_states) + if training: + hidden_states = self.dropout(hidden_states) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +class TFBertLayer(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFBertLayer, self).__init__(**kwargs) + self.attention = TFBertAttention(config, name='attention') + self.intermediate = TFBertIntermediate(config, name='intermediate') + self.bert_output = TFBertOutput(config, name='output') + + def call(self, inputs, training=False): + hidden_states, attention_mask, head_mask = inputs + + attention_outputs = self.attention([hidden_states, attention_mask, head_mask], training=training) + attention_output = attention_outputs[0] + intermediate_output = self.intermediate(attention_output) + layer_output = self.bert_output([intermediate_output, attention_output], training=training) + outputs = (layer_output,) + attention_outputs[1:] # add attentions if we output them + return outputs + + +class TFBertEncoder(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFBertEncoder, self).__init__(**kwargs) + self.output_attentions = config.output_attentions + self.output_hidden_states = config.output_hidden_states + self.layer = [TFBertLayer(config, name='layer_{}'.format(i)) for i in range(config.num_hidden_layers)] + + def call(self, inputs, training=False): + hidden_states, attention_mask, head_mask = inputs + + all_hidden_states = () + all_attentions = () + for i, layer_module in enumerate(self.layer): + if self.output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + layer_outputs = layer_module([hidden_states, attention_mask, head_mask[i]], training=training) + hidden_states = layer_outputs[0] + + if self.output_attentions: + all_attentions = all_attentions + (layer_outputs[1],) + + # Add last layer + if self.output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + outputs = (hidden_states,) + if self.output_hidden_states: + outputs = outputs + (all_hidden_states,) + if self.output_attentions: + outputs = outputs + (all_attentions,) + return outputs # outputs, (hidden states), (attentions) + + +class TFBertPooler(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFBertPooler, self).__init__(**kwargs) + self.dense = tf.keras.layers.Dense(config.hidden_size, activation='tanh', name='dense') + + def call(self, hidden_states): + # We "pool" the model by simply taking the hidden state corresponding + # to the first token. + first_token_tensor = hidden_states[:, 0] + pooled_output = self.dense(first_token_tensor) + return pooled_output + + +class TFBertPredictionHeadTransform(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFBertPredictionHeadTransform, self).__init__(**kwargs) + self.dense = tf.keras.layers.Dense(config.hidden_size, name='dense') + if isinstance(config.hidden_act, str) or (sys.version_info[0] == 2 and isinstance(config.hidden_act, unicode)): + self.transform_act_fn = ACT2FN[config.hidden_act] + else: + self.transform_act_fn = config.hidden_act + self.LayerNorm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name='LayerNorm') + + def call(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.transform_act_fn(hidden_states) + hidden_states = self.LayerNorm(hidden_states) + return hidden_states + + +class TFBertLMPredictionHead(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFBertLMPredictionHead, self).__init__(**kwargs) + self.vocab_size = config.vocab_size + self.transform = TFBertPredictionHeadTransform(config, name='transform') + + # The output weights are the same as the input embeddings, but there is + # an output-only bias for each token. + self.decoder = tf.keras.layers.Dense(config.vocab_size, use_bias=False, name='decoder') + + def build(self, input_shape): + self.bias = self.add_weight(shape=(self.vocab_size,), + initializer='zeros', + trainable=True, + name='bias') + + def call(self, hidden_states): + hidden_states = self.transform(hidden_states) + hidden_states = self.decoder(hidden_states) + self.bias + return hidden_states + + +class TFBertMLMHead(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFBertMLMHead, self).__init__(**kwargs) + self.predictions = TFBertLMPredictionHead(config, name='predictions') + + def call(self, sequence_output): + prediction_scores = self.predictions(sequence_output) + return prediction_scores + + +class TFBertNSPHead(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFBertNSPHead, self).__init__(**kwargs) + self.seq_relationship = tf.keras.layers.Dense(2, name='seq_relationship') + + def call(self, pooled_output): + seq_relationship_score = self.seq_relationship(pooled_output) + return seq_relationship_score + + +class TFBertMainLayer(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFBertMainLayer, self).__init__(**kwargs) + self.num_hidden_layers = config.num_hidden_layers + + self.embeddings = TFBertEmbeddings(config, name='embeddings') + self.encoder = TFBertEncoder(config, name='encoder') + self.pooler = TFBertPooler(config, name='pooler') + + # self.apply(self.init_weights) # TODO check weights initialization + + def _resize_token_embeddings(self, new_num_tokens): + raise NotImplementedError + + def _prune_heads(self, heads_to_prune): + """ Prunes heads of the model. + heads_to_prune: dict of {layer_num: list of heads to prune in this layer} + See base class PreTrainedModel + """ + raise NotImplementedError + + def call(self, inputs, training=False): + if not isinstance(inputs, (dict, tuple, list)): + input_ids = inputs + attention_mask, head_mask, position_ids, token_type_ids = None, None, None, None + elif isinstance(inputs, (tuple, list)): + input_ids = inputs[0] + attention_mask = inputs[1] if len(inputs) > 1 else None + token_type_ids = inputs[2] if len(inputs) > 2 else None + position_ids = inputs[3] if len(inputs) > 3 else None + head_mask = inputs[4] if len(inputs) > 4 else None + assert len(inputs) <= 5, "Too many inputs." + else: + input_ids = inputs.pop('input_ids') + attention_mask = inputs.pop('attention_mask', None) + token_type_ids = inputs.pop('token_type_ids', None) + position_ids = inputs.pop('position_ids', None) + head_mask = inputs.pop('head_mask', None) + assert len(inputs) == 0, "Unexpected inputs detected: {}. Check inputs dict key names.".format(list(inputs.keys())) + + if attention_mask is None: + attention_mask = tf.fill(tf.shape(input_ids), 1) + if token_type_ids is None: + token_type_ids = tf.fill(tf.shape(input_ids), 0) + + # We create a 3D attention mask from a 2D tensor mask. + # Sizes are [batch_size, 1, 1, to_seq_length] + # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length] + # this attention mask is more simple than the triangular masking of causal attention + # used in OpenAI GPT, we just need to prepare the broadcast dimension here. + extended_attention_mask = attention_mask[:, tf.newaxis, tf.newaxis, :] + + # Since attention_mask is 1.0 for positions we want to attend and 0.0 for + # masked positions, this operation will create a tensor which is 0.0 for + # positions we want to attend and -10000.0 for masked positions. + # Since we are adding it to the raw scores before the softmax, this is + # effectively the same as removing these entirely. + + extended_attention_mask = tf.cast(extended_attention_mask, tf.float32) + extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0 + + # Prepare head mask if needed + # 1.0 in head_mask indicate we keep the head + # attention_probs has shape bsz x n_heads x N x N + # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] + # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length] + if not head_mask is None: + raise NotImplementedError + else: + head_mask = [None] * self.num_hidden_layers + # head_mask = tf.constant([0] * self.num_hidden_layers) + + embedding_output = self.embeddings([input_ids, position_ids, token_type_ids], training=training) + encoder_outputs = self.encoder([embedding_output, extended_attention_mask, head_mask], training=training) + + sequence_output = encoder_outputs[0] + pooled_output = self.pooler(sequence_output) + + outputs = (sequence_output, pooled_output,) + encoder_outputs[1:] # add hidden_states and attentions if they are here + return outputs # sequence_output, pooled_output, (hidden_states), (attentions) + +class TFBertPreTrainedModel(TFPreTrainedModel): + """ An abstract class to handle weights initialization and + a simple interface for dowloading and loading pretrained models. + """ + config_class = BertConfig + pretrained_model_archive_map = TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP + load_pt_weights = load_pt_weights_in_bert + base_model_prefix = "bert" + + def __init__(self, *inputs, **kwargs): + super(TFBertPreTrainedModel, self).__init__(*inputs, **kwargs) + + def init_weights(self, module): + """ Initialize the weights. + """ + raise NotImplementedError + + +BERT_START_DOCSTRING = r""" The BERT model was proposed in + `BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ + by Jacob Devlin, Ming-Wei Chang, Kenton Lee and Kristina Toutanova. It's a bidirectional transformer + pre-trained using a combination of masked language modeling objective and next sentence prediction + on a large corpus comprising the Toronto Book Corpus and Wikipedia. + + This model is a tf.keras.Model `tf.keras.Model`_ sub-class. Use it as a regular TF 2.0 Keras Model and + refer to the TF 2.0 documentation for all matter related to general usage and behavior. + + .. _`BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`: + https://arxiv.org/abs/1810.04805 + + .. _`tf.keras.Model`: + https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/Model + + Important note on the model inputs: + The inputs of the TF 2.0 models are slightly different from the PyTorch ones since + TF 2.0 Keras doesn't accept named arguments with defaults values for input Tensor. + More precisely, input Tensors are gathered in the first arguments of the model call function: `model(inputs)`. + There are three possibilities to gather and feed the inputs to the model: + + - a single Tensor with input_ids only and nothing else: `model(inputs_ids) + - a list of varying length with one or several input Tensors IN THE ORDER given in the docstring: + `model([input_ids, attention_mask])` or `model([input_ids, attention_mask, token_type_ids])` + - a dictionary with one or several input Tensors associaed to the input names given in the docstring: + `model({'input_ids': input_ids, 'token_type_ids': token_type_ids})` + + Parameters: + config (:class:`~pytorch_transformers.BertConfig`): Model configuration class with all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the configuration. + Check out the :meth:`~pytorch_transformers.PreTrainedModel.from_pretrained` method to load the model weights. +""" + +BERT_INPUTS_DOCSTRING = r""" + Inputs: + **input_ids**: ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Indices of input sequence tokens in the vocabulary. + To match pre-training, BERT input sequence should be formatted with [CLS] and [SEP] tokens as follows: + + (a) For sequence pairs: + + ``tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]`` + + ``token_type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1`` + + (b) For single sequences: + + ``tokens: [CLS] the dog is hairy . [SEP]`` + + ``token_type_ids: 0 0 0 0 0 0 0`` + + Bert is a model with absolute position embeddings so it's usually advised to pad the inputs on + the right rather than the left. + + Indices can be obtained using :class:`pytorch_transformers.BertTokenizer`. + See :func:`pytorch_transformers.PreTrainedTokenizer.encode` and + :func:`pytorch_transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``: + Mask to avoid performing attention on padding token indices. + Mask values selected in ``[0, 1]``: + ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. + **token_type_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Segment token indices to indicate first and second portions of the inputs. + Indices are selected in ``[0, 1]``: ``0`` corresponds to a `sentence A` token, ``1`` + corresponds to a `sentence B` token + (see `BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ for more details). + **position_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Indices of positions of each input sequence tokens in the position embeddings. + Selected in the range ``[0, config.max_position_embeddings - 1]``. + **head_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: + Mask to nullify selected heads of the self-attention modules. + Mask values selected in ``[0, 1]``: + ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. +""" + +@add_start_docstrings("The bare Bert Model transformer outputing raw hidden-states without any specific head on top.", + BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) +class TFBertModel(TFBertPreTrainedModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)`` + Sequence of hidden-states at the output of the last layer of the model. + **pooler_output**: ``torch.FloatTensor`` of shape ``(batch_size, hidden_size)`` + Last layer hidden-state of the first token of the sequence (classification token) + further processed by a Linear layer and a Tanh activation function. The Linear + layer weights are trained from the next sentence prediction (classification) + objective during Bert pretraining. This output is usually *not* a good summary + of the semantic content of the input, you're often better with averaging or pooling + the sequence of hidden-states for the whole input sequence. + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + model = TFBertModel.from_pretrained('bert-base-uncased') + input_ids = tf.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids) + last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple + + """ + def __init__(self, config): + super(TFBertModel, self).__init__(config) + self.bert = TFBertMainLayer(config, name='bert') + + def call(self, inputs, training=False): + outputs = self.bert(inputs, training=training) + return outputs + + +@add_start_docstrings("""Bert Model with two heads on top as done during the pre-training: + a `masked language modeling` head and a `next sentence prediction (classification)` head. """, + BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) +class TFBertForPreTraining(TFBertPreTrainedModel): + r""" + **masked_lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Labels for computing the masked language modeling loss. + Indices should be in ``[-1, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring) + Tokens with indices set to ``-1`` are ignored (masked), the loss is only computed for the tokens with labels + in ``[0, ..., config.vocab_size]`` + **next_sentence_label**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair (see ``input_ids`` docstring) + Indices should be in ``[0, 1]``. + ``0`` indicates sequence B is a continuation of sequence A, + ``1`` indicates sequence B is a random sequence. + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when both ``masked_lm_labels`` and ``next_sentence_label`` are provided) ``torch.FloatTensor`` of shape ``(1,)``: + Total loss as the sum of the masked language modeling loss and the next sequence prediction (classification) loss. + **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + **seq_relationship_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, 2)`` + Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + model = TFBertForPreTraining.from_pretrained('bert-base-uncased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids) + prediction_scores, seq_relationship_scores = outputs[:2] + + """ + def __init__(self, config): + super(TFBertForPreTraining, self).__init__(config) + + self.bert = TFBertMainLayer(config, name='bert') + self.cls_mlm = TFBertMLMHead(config, name='cls_mlm') + self.cls_nsp = TFBertNSPHead(config, name='cls_nsp') + + # self.apply(self.init_weights) # TODO check added weights initialization + self.tie_weights() + + def tie_weights(self): + """ Make sure we are sharing the input and output embeddings. + """ + pass # TODO add weights tying + + def call(self, inputs, training=False): + outputs = self.bert(inputs, training=training) + + sequence_output, pooled_output = outputs[:2] + prediction_scores = self.cls_mlm(sequence_output) + seq_relationship_score = self.cls_nsp(pooled_output) + + outputs = (prediction_scores, seq_relationship_score,) + outputs[2:] # add hidden states and attention if they are here + + # if masked_lm_labels is not None and next_sentence_label is not None: + # loss_fct = CrossEntropyLoss(ignore_index=-1) + # masked_lm_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), masked_lm_labels.view(-1)) + # next_sentence_loss = loss_fct(seq_relationship_score.view(-1, 2), next_sentence_label.view(-1)) + # total_loss = masked_lm_loss + next_sentence_loss + # outputs = (total_loss,) + outputs + # TODO add example with losses using model.compile and a dictionary of losses (give names to the output layers) + + return outputs # prediction_scores, seq_relationship_score, (hidden_states), (attentions) + + +@add_start_docstrings("""Bert Model with a `language modeling` head on top. """, + BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) +class TFBertForMaskedLM(TFBertPreTrainedModel): + r""" + **masked_lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Labels for computing the masked language modeling loss. + Indices should be in ``[-1, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring) + Tokens with indices set to ``-1`` are ignored (masked), the loss is only computed for the tokens with labels + in ``[0, ..., config.vocab_size]`` + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``masked_lm_labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Masked language modeling loss. + **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + model = TFBertForMaskedLM.from_pretrained('bert-base-uncased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, masked_lm_labels=input_ids) + loss, prediction_scores = outputs[:2] + + """ + def __init__(self, config): + super(TFBertForMaskedLM, self).__init__(config) + + self.bert = TFBertMainLayer(config, name='bert') + self.cls_mlm = TFBertMLMHead(config, name='cls_mlm') + + # self.apply(self.init_weights) + self.tie_weights() + + def tie_weights(self): + """ Make sure we are sharing the input and output embeddings. + """ + pass # TODO add weights tying + + def call(self, inputs, training=False): + outputs = self.bert(inputs, training=training) + + sequence_output = outputs[0] + prediction_scores = self.cls_mlm(sequence_output) + + outputs = (prediction_scores,) + outputs[2:] # Add hidden states and attention if they are here + # if masked_lm_labels is not None: + # loss_fct = CrossEntropyLoss(ignore_index=-1) + # masked_lm_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), masked_lm_labels.view(-1)) + # outputs = (masked_lm_loss,) + outputs + # TODO example with losses + + return outputs # prediction_scores, (hidden_states), (attentions) + + +@add_start_docstrings("""Bert Model with a `next sentence prediction (classification)` head on top. """, + BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) +class TFBertForNextSentencePrediction(TFBertPreTrainedModel): + r""" + **next_sentence_label**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair (see ``input_ids`` docstring) + Indices should be in ``[0, 1]``. + ``0`` indicates sequence B is a continuation of sequence A, + ``1`` indicates sequence B is a random sequence. + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``next_sentence_label`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Next sequence prediction (classification) loss. + **seq_relationship_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, 2)`` + Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + model = TFBertForNextSentencePrediction.from_pretrained('bert-base-uncased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids) + seq_relationship_scores = outputs[0] + + """ + def __init__(self, config): + super(TFBertForNextSentencePrediction, self).__init__(config) + + self.bert = TFBertMainLayer(config, name='bert') + self.cls_nsp = TFBertNSPHead(config, name='cls_nsp') + + # self.apply(self.init_weights) + + def call(self, inputs, training=False): + outputs = self.bert(inputs, training=training) + + pooled_output = outputs[1] + seq_relationship_score = self.cls_nsp(pooled_output) + + outputs = (seq_relationship_score,) + outputs[2:] # add hidden states and attention if they are here + # if next_sentence_label is not None: + # loss_fct = CrossEntropyLoss(ignore_index=-1) + # next_sentence_loss = loss_fct(seq_relationship_score.view(-1, 2), next_sentence_label.view(-1)) + # outputs = (next_sentence_loss,) + outputs + + return outputs # seq_relationship_score, (hidden_states), (attentions) diff --git a/pytorch_transformers/modeling_tf_utils.py b/pytorch_transformers/modeling_tf_utils.py new file mode 100644 index 0000000000..442d9dbe44 --- /dev/null +++ b/pytorch_transformers/modeling_tf_utils.py @@ -0,0 +1,255 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""TF general model utils.""" + +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +import logging +import os + +import tensorflow as tf + +from .configuration_utils import PretrainedConfig +from .file_utils import cached_path, WEIGHTS_NAME, TF_WEIGHTS_NAME + +logger = logging.getLogger(__name__) + + +class TFPreTrainedModel(tf.keras.Model): + r""" Base class for all TF models. + + :class:`~pytorch_transformers.TFPreTrainedModel` takes care of storing the configuration of the models and handles methods for loading/downloading/saving models + as well as a few methods commons to all models to (i) resize the input embeddings and (ii) prune heads in the self-attention heads. + + Class attributes (overridden by derived classes): + - ``config_class``: a class derived from :class:`~pytorch_transformers.PretrainedConfig` to use as configuration class for this model architecture. + - ``pretrained_model_archive_map``: a python ``dict`` of with `short-cut-names` (string) as keys and `url` (string) of associated pretrained weights as values. + - ``load_tf_weights``: a python ``method`` for loading a TensorFlow checkpoint in a PyTorch model, taking as arguments: + + - ``model``: an instance of the relevant subclass of :class:`~pytorch_transformers.PreTrainedModel`, + - ``config``: an instance of the relevant subclass of :class:`~pytorch_transformers.PretrainedConfig`, + - ``path``: a path (string) to the TensorFlow checkpoint. + + - ``base_model_prefix``: a string indicating the attribute associated to the base model in derived classes of the same architecture adding modules on top of the base model. + """ + config_class = None + pretrained_model_archive_map = {} + load_pt_weights = lambda model, config, path: None + base_model_prefix = "" + + def __init__(self, config, *inputs, **kwargs): + super(TFPreTrainedModel, self).__init__() + if not isinstance(config, PretrainedConfig): + raise ValueError( + "Parameter config in `{}(config)` should be an instance of class `PretrainedConfig`. " + "To create a model from a pretrained model use " + "`model = {}.from_pretrained(PRETRAINED_MODEL_NAME)`".format( + self.__class__.__name__, self.__class__.__name__ + )) + # Save config in model + self.config = config + + def _get_resized_embeddings(self, old_embeddings, new_num_tokens=None): + """ Build a resized Embedding Module from a provided token Embedding Module. + Increasing the size will add newly initialized vectors at the end + Reducing the size will remove vectors from the end + + Args: + new_num_tokens: (`optional`) int + New number of tokens in the embedding matrix. + Increasing the size will add newly initialized vectors at the end + Reducing the size will remove vectors from the end + If not provided or None: return the provided token Embedding Module. + Return: ``torch.nn.Embeddings`` + Pointer to the resized Embedding Module or the old Embedding Module if new_num_tokens is None + """ + raise NotImplementedError + + def _tie_or_clone_weights(self, first_module, second_module): + """ Tie or clone module weights depending of weither we are using TorchScript or not + """ + raise NotImplementedError + + def resize_token_embeddings(self, new_num_tokens=None): + """ Resize input token embeddings matrix of the model if new_num_tokens != config.vocab_size. + Take care of tying weights embeddings afterwards if the model class has a `tie_weights()` method. + + Arguments: + + new_num_tokens: (`optional`) int: + New number of tokens in the embedding matrix. Increasing the size will add newly initialized vectors at the end. Reducing the size will remove vectors from the end. + If not provided or None: does nothing and just returns a pointer to the input tokens ``torch.nn.Embeddings`` Module of the model. + + Return: ``torch.nn.Embeddings`` + Pointer to the input tokens Embeddings Module of the model + """ + raise NotImplementedError + + def prune_heads(self, heads_to_prune): + """ Prunes heads of the base model. + + Arguments: + + heads_to_prune: dict with keys being selected layer indices (`int`) and associated values being the list of heads to prune in said layer (list of `int`). + """ + raise NotImplementedError + + def save_pretrained(self, save_directory): + """ Save a model and its configuration file to a directory, so that it + can be re-loaded using the `:func:`~pytorch_transformers.PreTrainedModel.from_pretrained`` class method. + """ + raise NotImplementedError + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): + r"""Instantiate a pretrained pytorch model from a pre-trained model configuration. + + The model is set in evaluation mode by default using ``model.eval()`` (Dropout modules are deactivated) + To train the model, you should first set it back in training mode with ``model.train()`` + + The warning ``Weights from XXX not initialized from pretrained model`` means that the weights of XXX do not come pre-trained with the rest of the model. + It is up to you to train those weights with a downstream fine-tuning task. + + The warning ``Weights from XXX not used in YYY`` means that the layer XXX is not used by YYY, therefore those weights are discarded. + + Parameters: + pretrained_model_name_or_path: either: + + - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. + - a path to a `directory` containing model weights saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. + - a path or url to a `PyTorch state_dict save file` (e.g. `./pt_model/pytorch_model.bin`). In this case, ``from_pt`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the PyTorch checkpoint in a TensorFlow model using the provided conversion scripts and loading the TensorFlow model afterwards. + + model_args: (`optional`) Sequence of positional arguments: + All remaning positional arguments will be passed to the underlying model's ``__init__`` method + + config: (`optional`) instance of a class derived from :class:`~pytorch_transformers.PretrainedConfig`: + Configuration for the model to use instead of an automatically loaded configuation. Configuration can be automatically loaded when: + + - the model is a model provided by the library (loaded with the ``shortcut-name`` string of a pretrained model), or + - the model was saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained` and is reloaded by suppling the save directory. + - the model is loaded by suppling a local directory as ``pretrained_model_name_or_path`` and a configuration JSON file named `config.json` is found in the directory. + + from_pt: (`optional`) boolean, default False: + Load the model weights from a PyTorch state_dict save file (see docstring of pretrained_model_name_or_path argument). + + cache_dir: (`optional`) string: + Path to a directory in which a downloaded pre-trained model + configuration should be cached if the standard cache should not be used. + + force_download: (`optional`) boolean, default False: + Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + + proxies: (`optional`) dict, default None: + A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. + The proxies are used on each request. + + output_loading_info: (`optional`) boolean: + Set to ``True`` to also return a dictionnary containing missing keys, unexpected keys and error messages. + + kwargs: (`optional`) Remaining dictionary of keyword arguments: + Can be used to update the configuration object (after it being loaded) and initiate the model. (e.g. ``output_attention=True``). Behave differently depending on whether a `config` is provided or automatically loaded: + + - If a configuration is provided with ``config``, ``**kwargs`` will be directly passed to the underlying model's ``__init__`` method (we assume all relevant updates to the configuration have already been done) + - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~pytorch_transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function. + + Examples:: + + model = BertModel.from_pretrained('bert-base-uncased') # Download model and configuration from S3 and cache. + model = BertModel.from_pretrained('./test/saved_model/') # E.g. model was saved using `save_pretrained('./test/saved_model/')` + model = BertModel.from_pretrained('bert-base-uncased', output_attention=True) # Update configuration during loading + assert model.config.output_attention == True + # Loading from a TF checkpoint file instead of a PyTorch model (slower) + config = BertConfig.from_json_file('./tf_model/my_tf_model_config.json') + model = BertModel.from_pretrained('./tf_model/my_tf_checkpoint.ckpt.index', from_pt=True, config=config) + + """ + config = kwargs.pop('config', None) + cache_dir = kwargs.pop('cache_dir', None) + from_pt = kwargs.pop('from_pt', False) + force_download = kwargs.pop('force_download', False) + proxies = kwargs.pop('proxies', None) + output_loading_info = kwargs.pop('output_loading_info', False) + + # Load config + if config is None: + config, model_kwargs = cls.config_class.from_pretrained( + pretrained_model_name_or_path, *model_args, + cache_dir=cache_dir, return_unused_kwargs=True, + force_download=force_download, + **kwargs + ) + else: + model_kwargs = kwargs + + # Load model + if pretrained_model_name_or_path in cls.pretrained_model_archive_map: + archive_file = cls.pretrained_model_archive_map[pretrained_model_name_or_path] + elif os.path.isdir(pretrained_model_name_or_path): + if from_pt: + # Load from a PyTorch checkpoint + archive_file = os.path.join(pretrained_model_name_or_path, WEIGHTS_NAME) + else: + archive_file = os.path.join(pretrained_model_name_or_path, TF_WEIGHTS_NAME) + else: + archive_file = pretrained_model_name_or_path + # redirect to the cache, if necessary + try: + resolved_archive_file = cached_path(archive_file, cache_dir=cache_dir, force_download=force_download, proxies=proxies) + except EnvironmentError: + if pretrained_model_name_or_path in cls.pretrained_model_archive_map: + logger.error( + "Couldn't reach server at '{}' to download pretrained weights.".format( + archive_file)) + else: + logger.error( + "Model name '{}' was not found in model name list ({}). " + "We assumed '{}' was a path or url but couldn't find any file " + "associated to this path or url.".format( + pretrained_model_name_or_path, + ', '.join(cls.pretrained_model_archive_map.keys()), + archive_file)) + return None + if resolved_archive_file == archive_file: + logger.info("loading weights file {}".format(archive_file)) + else: + logger.info("loading weights file {} from cache at {}".format( + archive_file, resolved_archive_file)) + + # Instantiate model. + model = cls(config, *model_args, **model_kwargs) + + if from_pt: + # Load from a PyTorch checkpoint + return cls.load_pt_weights(model, config, resolved_archive_file) + + inputs = tf.constant([[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]]) + ret = model(inputs, training=False) # build the network with dummy inputs + + # 'by_name' allow us to do transfer learning by skipping/adding layers + # see https://github.com/tensorflow/tensorflow/blob/00fad90125b18b80fe054de1055770cfb8fe4ba3/tensorflow/python/keras/engine/network.py#L1339-L1357 + model.load_weights(resolved_archive_file, by_name=True) + + ret = model(inputs, training=False) # Make sure restore ops are run + + # if hasattr(model, 'tie_weights'): + # model.tie_weights() # TODO make sure word embedding weights are still tied + + if output_loading_info: + loading_info = {"missing_keys": missing_keys, "unexpected_keys": unexpected_keys, "error_msgs": error_msgs} + return model, loading_info + + return model diff --git a/pytorch_transformers/tests/modeling_tf_common_test.py b/pytorch_transformers/tests/modeling_tf_common_test.py new file mode 100644 index 0000000000..d432ab5fdf --- /dev/null +++ b/pytorch_transformers/tests/modeling_tf_common_test.py @@ -0,0 +1,308 @@ +# coding=utf-8 +# Copyright 2019 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import copy +import os +import shutil +import json +import random +import uuid + +import unittest +import logging + +import tensorflow as tf + +from pytorch_transformers import TFPreTrainedModel +# from pytorch_transformers.modeling_bert import BertModel, BertConfig, BERT_PRETRAINED_MODEL_ARCHIVE_MAP + + +def _config_zero_init(config): + configs_no_init = copy.deepcopy(config) + for key in configs_no_init.__dict__.keys(): + if '_range' in key or '_std' in key: + setattr(configs_no_init, key, 0.0) + return configs_no_init + +class TFCommonTestCases: + + class TFCommonModelTester(unittest.TestCase): + + model_tester = None + all_model_classes = () + test_torchscript = True + test_pruning = True + test_resize_embeddings = True + + def test_initialization(self): + pass + # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + # configs_no_init = _config_zero_init(config) + # for model_class in self.all_model_classes: + # model = model_class(config=configs_no_init) + # for name, param in model.named_parameters(): + # if param.requires_grad: + # self.assertIn(param.data.mean().item(), [0.0, 1.0], + # msg="Parameter {} of model {} seems not properly initialized".format(name, model_class)) + + + def test_attention_outputs(self): + pass + # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + # for model_class in self.all_model_classes: + # config.output_attentions = True + # config.output_hidden_states = False + # model = model_class(config) + # model.eval() + # outputs = model(**inputs_dict) + # attentions = outputs[-1] + # self.assertEqual(model.config.output_attentions, True) + # self.assertEqual(model.config.output_hidden_states, False) + # self.assertEqual(len(attentions), self.model_tester.num_hidden_layers) + # self.assertListEqual( + # list(attentions[0].shape[-3:]), + # [self.model_tester.num_attention_heads, + # self.model_tester.seq_length, + # self.model_tester.key_len if hasattr(self.model_tester, 'key_len') else self.model_tester.seq_length]) + # out_len = len(outputs) + + # # Check attention is always last and order is fine + # config.output_attentions = True + # config.output_hidden_states = True + # model = model_class(config) + # model.eval() + # outputs = model(**inputs_dict) + # self.assertEqual(out_len+1, len(outputs)) + # self.assertEqual(model.config.output_attentions, True) + # self.assertEqual(model.config.output_hidden_states, True) + + # attentions = outputs[-1] + # self.assertEqual(len(attentions), self.model_tester.num_hidden_layers) + # self.assertListEqual( + # list(attentions[0].shape[-3:]), + # [self.model_tester.num_attention_heads, + # self.model_tester.seq_length, + # self.model_tester.key_len if hasattr(self.model_tester, 'key_len') else self.model_tester.seq_length]) + + + def test_headmasking(self): + pass + # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + # config.output_attentions = True + # config.output_hidden_states = True + # configs_no_init = _config_zero_init(config) # To be sure we have no Nan + # for model_class in self.all_model_classes: + # model = model_class(config=configs_no_init) + # model.eval() + + # # Prepare head_mask + # # Set require_grad after having prepared the tensor to avoid error (leaf variable has been moved into the graph interior) + # head_mask = torch.ones(self.model_tester.num_hidden_layers, self.model_tester.num_attention_heads) + # head_mask[0, 0] = 0 + # head_mask[-1, :-1] = 0 + # head_mask.requires_grad_(requires_grad=True) + # inputs = inputs_dict.copy() + # inputs['head_mask'] = head_mask + + # outputs = model(**inputs) + + # # Test that we can get a gradient back for importance score computation + # output = sum(t.sum() for t in outputs[0]) + # output = output.sum() + # output.backward() + # multihead_outputs = head_mask.grad + + # attentions = outputs[-1] + # hidden_states = outputs[-2] + + # # Remove Nan + + # self.assertIsNotNone(multihead_outputs) + # self.assertEqual(len(multihead_outputs), self.model_tester.num_hidden_layers) + # self.assertAlmostEqual( + # attentions[0][..., 0, :, :].flatten().sum().item(), 0.0) + # self.assertNotEqual( + # attentions[0][..., -1, :, :].flatten().sum().item(), 0.0) + # self.assertNotEqual( + # attentions[1][..., 0, :, :].flatten().sum().item(), 0.0) + # self.assertAlmostEqual( + # attentions[-1][..., -2, :, :].flatten().sum().item(), 0.0) + # self.assertNotEqual( + # attentions[-1][..., -1, :, :].flatten().sum().item(), 0.0) + + + def test_head_pruning(self): + pass + # if not self.test_pruning: + # return + + # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + # for model_class in self.all_model_classes: + # config.output_attentions = True + # config.output_hidden_states = False + # model = model_class(config=config) + # model.eval() + # heads_to_prune = {0: list(range(1, self.model_tester.num_attention_heads)), + # -1: [0]} + # model.prune_heads(heads_to_prune) + # outputs = model(**inputs_dict) + + # attentions = outputs[-1] + + # self.assertEqual( + # attentions[0].shape[-3], 1) + # self.assertEqual( + # attentions[1].shape[-3], self.model_tester.num_attention_heads) + # self.assertEqual( + # attentions[-1].shape[-3], self.model_tester.num_attention_heads - 1) + + + def test_hidden_states_output(self): + pass + # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + # for model_class in self.all_model_classes: + # config.output_hidden_states = True + # config.output_attentions = False + # model = model_class(config) + # model.eval() + # outputs = model(**inputs_dict) + # hidden_states = outputs[-1] + # self.assertEqual(model.config.output_attentions, False) + # self.assertEqual(model.config.output_hidden_states, True) + # self.assertEqual(len(hidden_states), self.model_tester.num_hidden_layers + 1) + # self.assertListEqual( + # list(hidden_states[0].shape[-2:]), + # [self.model_tester.seq_length, self.model_tester.hidden_size]) + + + def test_resize_tokens_embeddings(self): + pass + # original_config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + # if not self.test_resize_embeddings: + # return + + # for model_class in self.all_model_classes: + # config = copy.deepcopy(original_config) + # model = model_class(config) + + # model_vocab_size = config.vocab_size + # # Retrieve the embeddings and clone theme + # model_embed = model.resize_token_embeddings(model_vocab_size) + # cloned_embeddings = model_embed.weight.clone() + + # # Check that resizing the token embeddings with a larger vocab size increases the model's vocab size + # model_embed = model.resize_token_embeddings(model_vocab_size + 10) + # self.assertEqual(model.config.vocab_size, model_vocab_size + 10) + # # Check that it actually resizes the embeddings matrix + # self.assertEqual(model_embed.weight.shape[0], cloned_embeddings.shape[0] + 10) + + # # Check that resizing the token embeddings with a smaller vocab size decreases the model's vocab size + # model_embed = model.resize_token_embeddings(model_vocab_size - 15) + # self.assertEqual(model.config.vocab_size, model_vocab_size - 15) + # # Check that it actually resizes the embeddings matrix + # self.assertEqual(model_embed.weight.shape[0], cloned_embeddings.shape[0] - 15) + + # # Check that adding and removing tokens has not modified the first part of the embedding matrix. + # models_equal = True + # for p1, p2 in zip(cloned_embeddings, model_embed.weight): + # if p1.data.ne(p2.data).sum() > 0: + # models_equal = False + + # self.assertTrue(models_equal) + + + def test_tie_model_weights(self): + pass + # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + # def check_same_values(layer_1, layer_2): + # equal = True + # for p1, p2 in zip(layer_1.weight, layer_2.weight): + # if p1.data.ne(p2.data).sum() > 0: + # equal = False + # return equal + + # for model_class in self.all_model_classes: + # if not hasattr(model_class, 'tie_weights'): + # continue + + # config.torchscript = True + # model_not_tied = model_class(config) + # params_not_tied = list(model_not_tied.parameters()) + + # config_tied = copy.deepcopy(config) + # config_tied.torchscript = False + # model_tied = model_class(config_tied) + # params_tied = list(model_tied.parameters()) + + # # Check that the embedding layer and decoding layer are the same in size and in value + # self.assertGreater(len(params_not_tied), len(params_tied)) + + # # Check that after resize they remain tied. + # model_tied.resize_token_embeddings(config.vocab_size + 10) + # params_tied_2 = list(model_tied.parameters()) + # self.assertGreater(len(params_not_tied), len(params_tied)) + # self.assertEqual(len(params_tied_2), len(params_tied)) + + +def ids_tensor(shape, vocab_size, rng=None, name=None): + """Creates a random int32 tensor of the shape within the vocab size.""" + if rng is None: + rng = random.Random() + + total_dims = 1 + for dim in shape: + total_dims *= dim + + values = [] + for _ in range(total_dims): + values.append(rng.randint(0, vocab_size - 1)) + + return tf.constant(values, shape=shape) + + +class TFModelUtilsTest(unittest.TestCase): + def test_model_from_pretrained(self): + pass + # logging.basicConfig(level=logging.INFO) + # for model_name in list(BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: + # config = BertConfig.from_pretrained(model_name) + # self.assertIsNotNone(config) + # self.assertIsInstance(config, PretrainedConfig) + + # model = BertModel.from_pretrained(model_name) + # model, loading_info = BertModel.from_pretrained(model_name, output_loading_info=True) + # self.assertIsNotNone(model) + # self.assertIsInstance(model, PreTrainedModel) + # for value in loading_info.values(): + # self.assertEqual(len(value), 0) + + # config = BertConfig.from_pretrained(model_name, output_attentions=True, output_hidden_states=True) + # model = BertModel.from_pretrained(model_name, output_attentions=True, output_hidden_states=True) + # self.assertEqual(model.config.output_attentions, True) + # self.assertEqual(model.config.output_hidden_states, True) + # self.assertEqual(model.config, config) + + +if __name__ == "__main__": + unittest.main() diff --git a/pytorch_transformers/tests/modeling_tf_test.py b/pytorch_transformers/tests/modeling_tf_test.py new file mode 100644 index 0000000000..ee2580bd9e --- /dev/null +++ b/pytorch_transformers/tests/modeling_tf_test.py @@ -0,0 +1,327 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import unittest +import shutil +import pytest + +import tensorflow as tf + +from pytorch_transformers import (BertConfig) +from pytorch_transformers.modeling_tf_bert import TFBertModel, TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP + +from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) +from .configuration_common_test import ConfigTester + + +class TFBertModelTest(TFCommonTestCases.TFCommonModelTester): + + all_model_classes = (TFBertModel,) + # BertForMaskedLM, BertForNextSentencePrediction, + # BertForPreTraining, BertForQuestionAnswering, BertForSequenceClassification, + # BertForTokenClassification) + + class TFBertModelTester(object): + + def __init__(self, + parent, + batch_size=13, + seq_length=7, + is_training=True, + use_input_mask=True, + use_token_type_ids=True, + use_labels=True, + vocab_size=99, + hidden_size=32, + num_hidden_layers=5, + num_attention_heads=4, + intermediate_size=37, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + type_sequence_label_size=2, + initializer_range=0.02, + num_labels=3, + num_choices=4, + scope=None, + ): + self.parent = parent + self.batch_size = batch_size + self.seq_length = seq_length + self.is_training = is_training + self.use_input_mask = use_input_mask + self.use_token_type_ids = use_token_type_ids + self.use_labels = use_labels + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.intermediate_size = intermediate_size + self.hidden_act = hidden_act + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.type_sequence_label_size = type_sequence_label_size + self.initializer_range = initializer_range + self.num_labels = num_labels + self.num_choices = num_choices + self.scope = scope + + def prepare_config_and_inputs(self): + input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) + + input_mask = None + if self.use_input_mask: + input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2) + + token_type_ids = None + if self.use_token_type_ids: + token_type_ids = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size) + + sequence_labels = None + token_labels = None + choice_labels = None + if self.use_labels: + sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size) + token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels) + choice_labels = ids_tensor([self.batch_size], self.num_choices) + + config = BertConfig( + vocab_size_or_config_json_file=self.vocab_size, + hidden_size=self.hidden_size, + num_hidden_layers=self.num_hidden_layers, + num_attention_heads=self.num_attention_heads, + intermediate_size=self.intermediate_size, + hidden_act=self.hidden_act, + hidden_dropout_prob=self.hidden_dropout_prob, + attention_probs_dropout_prob=self.attention_probs_dropout_prob, + max_position_embeddings=self.max_position_embeddings, + type_vocab_size=self.type_vocab_size, + initializer_range=self.initializer_range) + + return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels + + def check_loss_output(self, result): + self.parent.assertListEqual( + list(result["loss"].size()), + []) + + def create_and_check_bert_model(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + model = TFBertModel(config=config) + # model.eval() + inputs = {'input_ids': input_ids, + 'attention_mask': input_mask, + 'token_type_ids': token_type_ids} + sequence_output, pooled_output = model(inputs) + + inputs = [input_ids, input_mask] + sequence_output, pooled_output = model(inputs) + + sequence_output, pooled_output = model(input_ids) + + result = { + "sequence_output": sequence_output.numpy(), + "pooled_output": pooled_output.numpy(), + } + self.parent.assertListEqual( + list(result["sequence_output"].shape), + [self.batch_size, self.seq_length, self.hidden_size]) + self.parent.assertListEqual(list(result["pooled_output"].shape), [self.batch_size, self.hidden_size]) + + + def create_and_check_bert_for_masked_lm(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + pass + # model = BertForMaskedLM(config=config) + # model.eval() + # loss, prediction_scores = model(input_ids, token_type_ids, input_mask, token_labels) + # result = { + # "loss": loss, + # "prediction_scores": prediction_scores, + # } + # self.parent.assertListEqual( + # list(result["prediction_scores"].size()), + # [self.batch_size, self.seq_length, self.vocab_size]) + # self.check_loss_output(result) + + + def create_and_check_bert_for_next_sequence_prediction(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + pass + # model = BertForNextSentencePrediction(config=config) + # model.eval() + # loss, seq_relationship_score = model(input_ids, token_type_ids, input_mask, sequence_labels) + # result = { + # "loss": loss, + # "seq_relationship_score": seq_relationship_score, + # } + # self.parent.assertListEqual( + # list(result["seq_relationship_score"].size()), + # [self.batch_size, 2]) + # self.check_loss_output(result) + + + def create_and_check_bert_for_pretraining(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + pass + # model = BertForPreTraining(config=config) + # model.eval() + # loss, prediction_scores, seq_relationship_score = model(input_ids, token_type_ids, input_mask, token_labels, sequence_labels) + # result = { + # "loss": loss, + # "prediction_scores": prediction_scores, + # "seq_relationship_score": seq_relationship_score, + # } + # self.parent.assertListEqual( + # list(result["prediction_scores"].size()), + # [self.batch_size, self.seq_length, self.vocab_size]) + # self.parent.assertListEqual( + # list(result["seq_relationship_score"].size()), + # [self.batch_size, 2]) + # self.check_loss_output(result) + + + def create_and_check_bert_for_question_answering(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + pass + # model = BertForQuestionAnswering(config=config) + # model.eval() + # loss, start_logits, end_logits = model(input_ids, token_type_ids, input_mask, sequence_labels, sequence_labels) + # result = { + # "loss": loss, + # "start_logits": start_logits, + # "end_logits": end_logits, + # } + # self.parent.assertListEqual( + # list(result["start_logits"].size()), + # [self.batch_size, self.seq_length]) + # self.parent.assertListEqual( + # list(result["end_logits"].size()), + # [self.batch_size, self.seq_length]) + # self.check_loss_output(result) + + + def create_and_check_bert_for_sequence_classification(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + pass + # config.num_labels = self.num_labels + # model = BertForSequenceClassification(config) + # model.eval() + # loss, logits = model(input_ids, token_type_ids, input_mask, sequence_labels) + # result = { + # "loss": loss, + # "logits": logits, + # } + # self.parent.assertListEqual( + # list(result["logits"].size()), + # [self.batch_size, self.num_labels]) + # self.check_loss_output(result) + + + def create_and_check_bert_for_token_classification(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + pass + # config.num_labels = self.num_labels + # model = BertForTokenClassification(config=config) + # model.eval() + # loss, logits = model(input_ids, token_type_ids, input_mask, token_labels) + # result = { + # "loss": loss, + # "logits": logits, + # } + # self.parent.assertListEqual( + # list(result["logits"].size()), + # [self.batch_size, self.seq_length, self.num_labels]) + # self.check_loss_output(result) + + + def create_and_check_bert_for_multiple_choice(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + pass + # config.num_choices = self.num_choices + # model = BertForMultipleChoice(config=config) + # model.eval() + # multiple_choice_inputs_ids = input_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous() + # multiple_choice_token_type_ids = token_type_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous() + # multiple_choice_input_mask = input_mask.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous() + # loss, logits = model(multiple_choice_inputs_ids, + # multiple_choice_token_type_ids, + # multiple_choice_input_mask, + # choice_labels) + # result = { + # "loss": loss, + # "logits": logits, + # } + # self.parent.assertListEqual( + # list(result["logits"].size()), + # [self.batch_size, self.num_choices]) + # self.check_loss_output(result) + + + def prepare_config_and_inputs_for_common(self): + config_and_inputs = self.prepare_config_and_inputs() + (config, input_ids, token_type_ids, input_mask, + sequence_labels, token_labels, choice_labels) = config_and_inputs + inputs_dict = {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask} + return config, inputs_dict + + def setUp(self): + self.model_tester = TFBertModelTest.TFBertModelTester(self) + self.config_tester = ConfigTester(self, config_class=BertConfig, hidden_size=37) + + def test_config(self): + self.config_tester.run_common_tests() + + def test_bert_model(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_bert_model(*config_and_inputs) + + def test_for_masked_lm(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_bert_for_masked_lm(*config_and_inputs) + + def test_for_multiple_choice(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_bert_for_multiple_choice(*config_and_inputs) + + def test_for_next_sequence_prediction(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_bert_for_next_sequence_prediction(*config_and_inputs) + + def test_for_pretraining(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_bert_for_pretraining(*config_and_inputs) + + def test_for_question_answering(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_bert_for_question_answering(*config_and_inputs) + + def test_for_sequence_classification(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_bert_for_sequence_classification(*config_and_inputs) + + def test_for_token_classification(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_bert_for_token_classification(*config_and_inputs) + + @pytest.mark.slow + def test_model_from_pretrained(self): + cache_dir = "/tmp/pytorch_transformers_test/" + for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: + model = TFBertModel.from_pretrained(model_name, cache_dir=cache_dir) + shutil.rmtree(cache_dir) + self.assertIsNotNone(model) + +if __name__ == "__main__": + unittest.main() From 59fe641b8bbeca2f4aa8a9d68a2e83fb7945054e Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 5 Sep 2019 02:34:09 +0200 Subject: [PATCH 018/219] also gathering file names in file_utils --- pytorch_transformers/__init__.py | 8 +++++--- pytorch_transformers/configuration_utils.py | 4 +--- pytorch_transformers/file_utils.py | 4 ++++ pytorch_transformers/modeling_utils.py | 5 +---- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/pytorch_transformers/__init__.py b/pytorch_transformers/__init__.py index b851c99c9d..24ff52e5d4 100644 --- a/pytorch_transformers/__init__.py +++ b/pytorch_transformers/__init__.py @@ -24,7 +24,7 @@ from .tokenization_roberta import RobertaTokenizer from .tokenization_distilbert import DistilBertTokenizer # Configurations -from .configuration_utils import CONFIG_NAME, PretrainedConfig +from .configuration_utils import PretrainedConfig from .configuration_auto import AutoConfig from .configuration_bert import BertConfig, BERT_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_openai import OpenAIGPTConfig, OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP @@ -36,7 +36,7 @@ from .configuration_roberta import RobertaConfig, ROBERTA_PRETRAINED_CONFIG_ARCH from .configuration_distilbert import DistilBertConfig, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP # Modeling -from .modeling_utils import (WEIGHTS_NAME, TF_WEIGHTS_NAME, PreTrainedModel, prune_layer, Conv1D) +from .modeling_utils import (PreTrainedModel, prune_layer, Conv1D) from .modeling_auto import (AutoModel, AutoModelForSequenceClassification, AutoModelForQuestionAnswering, AutoModelWithLMHead) @@ -70,4 +70,6 @@ from .optimization import (AdamW, ConstantLRSchedule, WarmupConstantSchedule, Wa WarmupCosineWithHardRestartsSchedule, WarmupLinearSchedule) # Files and general utilities -from .file_utils import (PYTORCH_TRANSFORMERS_CACHE, PYTORCH_PRETRAINED_BERT_CACHE, cached_path, add_start_docstrings, add_end_docstrings) +from .file_utils import (PYTORCH_TRANSFORMERS_CACHE, PYTORCH_PRETRAINED_BERT_CACHE, + cached_path, add_start_docstrings, add_end_docstrings, + WEIGHTS_NAME, TF_WEIGHTS_NAME, CONFIG_NAME) diff --git a/pytorch_transformers/configuration_utils.py b/pytorch_transformers/configuration_utils.py index 550b47fab8..7efc735d41 100644 --- a/pytorch_transformers/configuration_utils.py +++ b/pytorch_transformers/configuration_utils.py @@ -24,12 +24,10 @@ import logging import os from io import open -from .file_utils import cached_path +from .file_utils import cached_path, CONFIG_NAME logger = logging.getLogger(__name__) -CONFIG_NAME = "config.json" - class PretrainedConfig(object): r""" Base class for all configuration classes. Handles a few parameters common to all models' configurations as well as methods for loading/downloading/saving configurations. diff --git a/pytorch_transformers/file_utils.py b/pytorch_transformers/file_utils.py index 94ada93d91..a656e757b5 100644 --- a/pytorch_transformers/file_utils.py +++ b/pytorch_transformers/file_utils.py @@ -48,6 +48,10 @@ except (AttributeError, ImportError): PYTORCH_TRANSFORMERS_CACHE = PYTORCH_PRETRAINED_BERT_CACHE # Kept for backward compatibility +WEIGHTS_NAME = "pytorch_model.bin" +TF_WEIGHTS_NAME = 'model.ckpt' +CONFIG_NAME = "config.json" + logger = logging.getLogger(__name__) # pylint: disable=invalid-name if not six.PY2: diff --git a/pytorch_transformers/modeling_utils.py b/pytorch_transformers/modeling_utils.py index 48420f6d07..2fb4671674 100644 --- a/pytorch_transformers/modeling_utils.py +++ b/pytorch_transformers/modeling_utils.py @@ -31,13 +31,10 @@ from torch.nn import CrossEntropyLoss from torch.nn import functional as F from .configuration_utils import PretrainedConfig -from .file_utils import cached_path +from .file_utils import cached_path, WEIGHTS_NAME, TF_WEIGHTS_NAME logger = logging.getLogger(__name__) -WEIGHTS_NAME = "pytorch_model.bin" -TF_WEIGHTS_NAME = 'model.ckpt' - try: from torch.nn import Identity From ad0ab9afe9e872ea192e9cfa737f4b2aadfc4c61 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 5 Sep 2019 02:53:52 +0200 Subject: [PATCH 019/219] fix test when tf is not here --- .circleci/config.yml | 2 + pytorch_transformers/__init__.py | 90 ++++++++++++------- ...ng_tf_test.py => modeling_tf_bert_test.py} | 24 +++-- .../tests/modeling_tf_common_test.py | 31 ++++--- 4 files changed, 101 insertions(+), 46 deletions(-) rename pytorch_transformers/tests/{modeling_tf_test.py => modeling_tf_bert_test.py} (94%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 48e80beaeb..e54d92ab95 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,6 +11,7 @@ jobs: - run: sudo pip install --progress-bar off . - run: sudo pip install pytest codecov pytest-cov - run: sudo pip install tensorboardX scikit-learn + - run: sudo pip install tensorflow==2.0.0-rc0 - run: python -m pytest -sv ./pytorch_transformers/tests/ --cov - run: python -m pytest -sv ./examples/ - run: codecov @@ -24,6 +25,7 @@ jobs: - checkout - run: sudo pip install --progress-bar off . - run: sudo pip install pytest codecov pytest-cov + - run: sudo pip install tensorflow==2.0.0-rc0 - run: python -m pytest -sv ./pytorch_transformers/tests/ --cov - run: codecov deploy_doc: diff --git a/pytorch_transformers/__init__.py b/pytorch_transformers/__init__.py index 24ff52e5d4..43b0cb2e07 100644 --- a/pytorch_transformers/__init__.py +++ b/pytorch_transformers/__init__.py @@ -1,4 +1,5 @@ __version__ = "1.2.0" + # Work around to update TensorFlow's absl.logging threshold which alters the # default Python logging output behavior when present. # see: https://github.com/abseil/abseil-py/issues/99 @@ -11,6 +12,10 @@ try: except: pass +import logging + +logger = logging.getLogger(__name__) # pylint: disable=invalid-name + # Tokenizer from .tokenization_utils import (PreTrainedTokenizer) from .tokenization_auto import AutoTokenizer @@ -36,38 +41,63 @@ from .configuration_roberta import RobertaConfig, ROBERTA_PRETRAINED_CONFIG_ARCH from .configuration_distilbert import DistilBertConfig, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP # Modeling -from .modeling_utils import (PreTrainedModel, prune_layer, Conv1D) -from .modeling_auto import (AutoModel, AutoModelForSequenceClassification, AutoModelForQuestionAnswering, - AutoModelWithLMHead) +try: + import torch + torch_available = True # pylint: disable=invalid-name +except ImportError: + torch_available = False # pylint: disable=invalid-name -from .modeling_bert import (BertPreTrainedModel, BertModel, BertForPreTraining, - BertForMaskedLM, BertForNextSentencePrediction, - BertForSequenceClassification, BertForMultipleChoice, - BertForTokenClassification, BertForQuestionAnswering, - load_tf_weights_in_bert, BERT_PRETRAINED_MODEL_ARCHIVE_MAP) -from .modeling_openai import (OpenAIGPTPreTrainedModel, OpenAIGPTModel, - OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel, - load_tf_weights_in_openai_gpt, OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP) -from .modeling_transfo_xl import (TransfoXLPreTrainedModel, TransfoXLModel, TransfoXLLMHeadModel, - load_tf_weights_in_transfo_xl, TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP) -from .modeling_gpt2 import (GPT2PreTrainedModel, GPT2Model, - GPT2LMHeadModel, GPT2DoubleHeadsModel, - load_tf_weights_in_gpt2, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP) -from .modeling_xlnet import (XLNetPreTrainedModel, XLNetModel, XLNetLMHeadModel, - XLNetForSequenceClassification, XLNetForQuestionAnswering, - load_tf_weights_in_xlnet, XLNET_PRETRAINED_MODEL_ARCHIVE_MAP) -from .modeling_xlm import (XLMPreTrainedModel , XLMModel, - XLMWithLMHeadModel, XLMForSequenceClassification, - XLMForQuestionAnswering, XLM_PRETRAINED_MODEL_ARCHIVE_MAP) -from .modeling_roberta import (RobertaForMaskedLM, RobertaModel, RobertaForSequenceClassification, - ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP) -from .modeling_distilbert import (DistilBertForMaskedLM, DistilBertModel, - DistilBertForSequenceClassification, DistilBertForQuestionAnswering, - DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP) +if torch_available: + logger.info("PyTorch version {} available.".format(torch.__version__)) + + from .modeling_utils import (PreTrainedModel, prune_layer, Conv1D) + from .modeling_auto import (AutoModel, AutoModelForSequenceClassification, AutoModelForQuestionAnswering, + AutoModelWithLMHead) + + from .modeling_bert import (BertPreTrainedModel, BertModel, BertForPreTraining, + BertForMaskedLM, BertForNextSentencePrediction, + BertForSequenceClassification, BertForMultipleChoice, + BertForTokenClassification, BertForQuestionAnswering, + load_tf_weights_in_bert, BERT_PRETRAINED_MODEL_ARCHIVE_MAP) + from .modeling_openai import (OpenAIGPTPreTrainedModel, OpenAIGPTModel, + OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel, + load_tf_weights_in_openai_gpt, OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP) + from .modeling_transfo_xl import (TransfoXLPreTrainedModel, TransfoXLModel, TransfoXLLMHeadModel, + load_tf_weights_in_transfo_xl, TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP) + from .modeling_gpt2 import (GPT2PreTrainedModel, GPT2Model, + GPT2LMHeadModel, GPT2DoubleHeadsModel, + load_tf_weights_in_gpt2, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP) + from .modeling_xlnet import (XLNetPreTrainedModel, XLNetModel, XLNetLMHeadModel, + XLNetForSequenceClassification, XLNetForQuestionAnswering, + load_tf_weights_in_xlnet, XLNET_PRETRAINED_MODEL_ARCHIVE_MAP) + from .modeling_xlm import (XLMPreTrainedModel , XLMModel, + XLMWithLMHeadModel, XLMForSequenceClassification, + XLMForQuestionAnswering, XLM_PRETRAINED_MODEL_ARCHIVE_MAP) + from .modeling_roberta import (RobertaForMaskedLM, RobertaModel, RobertaForSequenceClassification, + ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP) + from .modeling_distilbert import (DistilBertForMaskedLM, DistilBertModel, + DistilBertForSequenceClassification, DistilBertForQuestionAnswering, + DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP) + + # Optimization + from .optimization import (AdamW, ConstantLRSchedule, WarmupConstantSchedule, WarmupCosineSchedule, + WarmupCosineWithHardRestartsSchedule, WarmupLinearSchedule) + + +# TensorFlow +try: + import tensorflow as tf + tf_available = True # pylint: disable=invalid-name +except ImportError: + tf_available = False # pylint: disable=invalid-name + +if tf_available: + logger.info("TensorFlow version {} available.".format(tf.__version__)) + + from .modeling_tf_utils import TFPreTrainedModel + from .modeling_tf_bert import (TFBertPreTrainedModel, TFBertModel, TFBertForPreTraining, + TFBertForMaskedLM, TFBertForNextSentencePrediction, load_pt_weights_in_bert) -# Optimization -from .optimization import (AdamW, ConstantLRSchedule, WarmupConstantSchedule, WarmupCosineSchedule, - WarmupCosineWithHardRestartsSchedule, WarmupLinearSchedule) # Files and general utilities from .file_utils import (PYTORCH_TRANSFORMERS_CACHE, PYTORCH_PRETRAINED_BERT_CACHE, diff --git a/pytorch_transformers/tests/modeling_tf_test.py b/pytorch_transformers/tests/modeling_tf_bert_test.py similarity index 94% rename from pytorch_transformers/tests/modeling_tf_test.py rename to pytorch_transformers/tests/modeling_tf_bert_test.py index ee2580bd9e..9d36bdb98e 100644 --- a/pytorch_transformers/tests/modeling_tf_test.py +++ b/pytorch_transformers/tests/modeling_tf_bert_test.py @@ -19,15 +19,19 @@ from __future__ import print_function import unittest import shutil import pytest - -import tensorflow as tf - -from pytorch_transformers import (BertConfig) -from pytorch_transformers.modeling_tf_bert import TFBertModel, TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP +import sys from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester +try: + import tensorflow as tf + + from pytorch_transformers import (BertConfig) + from pytorch_transformers.modeling_tf_bert import TFBertModel, TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP +except ImportError: + pass + class TFBertModelTest(TFCommonTestCases.TFCommonModelTester): @@ -283,39 +287,48 @@ class TFBertModelTest(TFCommonTestCases.TFCommonModelTester): def test_config(self): self.config_tester.run_common_tests() + @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") def test_bert_model(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_model(*config_and_inputs) + @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") def test_for_masked_lm(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_for_masked_lm(*config_and_inputs) + @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") def test_for_multiple_choice(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_for_multiple_choice(*config_and_inputs) + @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") def test_for_next_sequence_prediction(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_for_next_sequence_prediction(*config_and_inputs) + @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") def test_for_pretraining(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_for_pretraining(*config_and_inputs) + @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") def test_for_question_answering(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_for_question_answering(*config_and_inputs) + @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") def test_for_sequence_classification(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_for_sequence_classification(*config_and_inputs) + @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") def test_for_token_classification(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_for_token_classification(*config_and_inputs) @pytest.mark.slow + @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") def test_model_from_pretrained(self): cache_dir = "/tmp/pytorch_transformers_test/" for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: @@ -325,3 +338,4 @@ class TFBertModelTest(TFCommonTestCases.TFCommonModelTester): if __name__ == "__main__": unittest.main() + diff --git a/pytorch_transformers/tests/modeling_tf_common_test.py b/pytorch_transformers/tests/modeling_tf_common_test.py index d432ab5fdf..8e3911f766 100644 --- a/pytorch_transformers/tests/modeling_tf_common_test.py +++ b/pytorch_transformers/tests/modeling_tf_common_test.py @@ -12,24 +12,25 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import copy -import os -import shutil import json +import logging import random +import shutil +import unittest import uuid -import unittest -import logging +import pytest +import sys -import tensorflow as tf - -from pytorch_transformers import TFPreTrainedModel -# from pytorch_transformers.modeling_bert import BertModel, BertConfig, BERT_PRETRAINED_MODEL_ARCHIVE_MAP +try: + import tensorflow as tf + from pytorch_transformers import TFPreTrainedModel + # from pytorch_transformers.modeling_bert import BertModel, BertConfig, BERT_PRETRAINED_MODEL_ARCHIVE_MAP +except ImportError: + pass def _config_zero_init(config): @@ -49,6 +50,7 @@ class TFCommonTestCases: test_pruning = True test_resize_embeddings = True + @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") def test_initialization(self): pass # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -62,6 +64,7 @@ class TFCommonTestCases: # msg="Parameter {} of model {} seems not properly initialized".format(name, model_class)) + @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") def test_attention_outputs(self): pass # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -102,6 +105,7 @@ class TFCommonTestCases: # self.model_tester.key_len if hasattr(self.model_tester, 'key_len') else self.model_tester.seq_length]) + @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") def test_headmasking(self): pass # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -149,6 +153,7 @@ class TFCommonTestCases: # attentions[-1][..., -1, :, :].flatten().sum().item(), 0.0) + @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") def test_head_pruning(self): pass # if not self.test_pruning: @@ -176,6 +181,7 @@ class TFCommonTestCases: # attentions[-1].shape[-3], self.model_tester.num_attention_heads - 1) + @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") def test_hidden_states_output(self): pass # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -195,6 +201,7 @@ class TFCommonTestCases: # [self.model_tester.seq_length, self.model_tester.hidden_size]) + @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") def test_resize_tokens_embeddings(self): pass # original_config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -231,6 +238,7 @@ class TFCommonTestCases: # self.assertTrue(models_equal) + @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") def test_tie_model_weights(self): pass # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -282,6 +290,7 @@ def ids_tensor(shape, vocab_size, rng=None, name=None): class TFModelUtilsTest(unittest.TestCase): + @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") def test_model_from_pretrained(self): pass # logging.basicConfig(level=logging.INFO) From a4704b1263eba122160ef442b7d8904a8ae7e164 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 5 Sep 2019 03:06:09 +0200 Subject: [PATCH 020/219] skipping tf tests if tf is not installed --- .../tests/modeling_tf_bert_test.py | 18 +++++++++--------- .../tests/modeling_tf_common_test.py | 16 ++++++++-------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/pytorch_transformers/tests/modeling_tf_bert_test.py b/pytorch_transformers/tests/modeling_tf_bert_test.py index 9d36bdb98e..c7b8b02f7b 100644 --- a/pytorch_transformers/tests/modeling_tf_bert_test.py +++ b/pytorch_transformers/tests/modeling_tf_bert_test.py @@ -287,48 +287,48 @@ class TFBertModelTest(TFCommonTestCases.TFCommonModelTester): def test_config(self): self.config_tester.run_common_tests() - @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") + @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_bert_model(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_model(*config_and_inputs) - @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") + @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_for_masked_lm(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_for_masked_lm(*config_and_inputs) - @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") + @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_for_multiple_choice(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_for_multiple_choice(*config_and_inputs) - @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") + @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_for_next_sequence_prediction(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_for_next_sequence_prediction(*config_and_inputs) - @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") + @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_for_pretraining(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_for_pretraining(*config_and_inputs) - @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") + @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_for_question_answering(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_for_question_answering(*config_and_inputs) - @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") + @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_for_sequence_classification(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_for_sequence_classification(*config_and_inputs) - @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") + @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_for_token_classification(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_for_token_classification(*config_and_inputs) @pytest.mark.slow - @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") + @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_model_from_pretrained(self): cache_dir = "/tmp/pytorch_transformers_test/" for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: diff --git a/pytorch_transformers/tests/modeling_tf_common_test.py b/pytorch_transformers/tests/modeling_tf_common_test.py index 8e3911f766..1ddfd83be1 100644 --- a/pytorch_transformers/tests/modeling_tf_common_test.py +++ b/pytorch_transformers/tests/modeling_tf_common_test.py @@ -50,7 +50,7 @@ class TFCommonTestCases: test_pruning = True test_resize_embeddings = True - @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") + @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_initialization(self): pass # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -64,7 +64,7 @@ class TFCommonTestCases: # msg="Parameter {} of model {} seems not properly initialized".format(name, model_class)) - @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") + @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_attention_outputs(self): pass # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -105,7 +105,7 @@ class TFCommonTestCases: # self.model_tester.key_len if hasattr(self.model_tester, 'key_len') else self.model_tester.seq_length]) - @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") + @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_headmasking(self): pass # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -153,7 +153,7 @@ class TFCommonTestCases: # attentions[-1][..., -1, :, :].flatten().sum().item(), 0.0) - @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") + @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_head_pruning(self): pass # if not self.test_pruning: @@ -181,7 +181,7 @@ class TFCommonTestCases: # attentions[-1].shape[-3], self.model_tester.num_attention_heads - 1) - @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") + @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_hidden_states_output(self): pass # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -201,7 +201,7 @@ class TFCommonTestCases: # [self.model_tester.seq_length, self.model_tester.hidden_size]) - @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") + @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_resize_tokens_embeddings(self): pass # original_config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -238,7 +238,7 @@ class TFCommonTestCases: # self.assertTrue(models_equal) - @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") + @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_tie_model_weights(self): pass # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -290,7 +290,7 @@ def ids_tensor(shape, vocab_size, rng=None, name=None): class TFModelUtilsTest(unittest.TestCase): - @pytest.mark.skipif('tf' not in sys.modules, reason="requires TensorFlow") + @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_model_from_pretrained(self): pass # logging.basicConfig(level=logging.INFO) From 6f152572cd0fe5c84127a1bc25668e5fce918744 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 5 Sep 2019 03:10:11 +0200 Subject: [PATCH 021/219] add conversion script, rename conversion scripts --- ...bert_original_tf_checkpoint_to_pytorch.py} | 0 ...bert_pytorch_checkpoint_to_original_tf.py} | 0 .../convert_bert_pytorch_checkpoint_to_tf.py | 65 +++++++++++++++++++ 3 files changed, 65 insertions(+) rename pytorch_transformers/{convert_tf_checkpoint_to_pytorch.py => convert_bert_original_tf_checkpoint_to_pytorch.py} (100%) rename pytorch_transformers/{convert_pytorch_checkpoint_to_tf.py => convert_bert_pytorch_checkpoint_to_original_tf.py} (100%) create mode 100644 pytorch_transformers/convert_bert_pytorch_checkpoint_to_tf.py diff --git a/pytorch_transformers/convert_tf_checkpoint_to_pytorch.py b/pytorch_transformers/convert_bert_original_tf_checkpoint_to_pytorch.py similarity index 100% rename from pytorch_transformers/convert_tf_checkpoint_to_pytorch.py rename to pytorch_transformers/convert_bert_original_tf_checkpoint_to_pytorch.py diff --git a/pytorch_transformers/convert_pytorch_checkpoint_to_tf.py b/pytorch_transformers/convert_bert_pytorch_checkpoint_to_original_tf.py similarity index 100% rename from pytorch_transformers/convert_pytorch_checkpoint_to_tf.py rename to pytorch_transformers/convert_bert_pytorch_checkpoint_to_original_tf.py diff --git a/pytorch_transformers/convert_bert_pytorch_checkpoint_to_tf.py b/pytorch_transformers/convert_bert_pytorch_checkpoint_to_tf.py new file mode 100644 index 0000000000..7031f3b523 --- /dev/null +++ b/pytorch_transformers/convert_bert_pytorch_checkpoint_to_tf.py @@ -0,0 +1,65 @@ +# coding=utf-8 +# Copyright 2018 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Convert BERT checkpoint.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import tensorflow as tf + +from pytorch_transformers import BertConfig, TFBertForPreTraining, load_pt_weights_in_bert + +import logging +logging.basicConfig(level=logging.INFO) + +def convert_bert_checkpoint_to_tf(pytorch_checkpoint_path, bert_config_file, tf_dump_path): + # Initialise TF model + config = BertConfig.from_json_file(bert_config_file) + print("Building TensorFlow model from configuration: {}".format(str(config))) + model = TFBertForPreTraining(config) + + # Load weights from tf checkpoint + model = load_pt_weights_in_bert(model, config, pytorch_checkpoint_path) + + # Save pytorch-model + print("Save TensorFlow model to {}".format(tf_dump_path)) + model.save_weights(tf_dump_path) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + ## Required parameters + parser.add_argument("--pytorch_checkpoint_path", + default = None, + type = str, + required = True, + help = "Path to the PyTorch checkpoint path.") + parser.add_argument("--bert_config_file", + default = None, + type = str, + required = True, + help = "The config json file corresponding to the pre-trained BERT model. \n" + "This specifies the model architecture.") + parser.add_argument("--tf_dump_path", + default = None, + type = str, + required = True, + help = "Path to the output Tensorflow dump file.") + args = parser.parse_args() + convert_bert_checkpoint_to_tf(args.pytorch_checkpoint_path, + args.bert_config_file, + args.tf_dump_path) From 24a20483f5e4657495544480049ca841cbea37db Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 5 Sep 2019 03:13:26 +0200 Subject: [PATCH 022/219] update conversion script names --- pytorch_transformers/__main__.py | 15 ++++++++------- ...ert_gpt2_original_tf_checkpoint_to_pytorch.py} | 0 ...t_openai_original_tf_checkpoint_to_pytorch.py} | 0 ...rta_original_pytorch_checkpoint_to_pytorch.py} | 0 ...ansfo_xl_original_tf_checkpoint_to_pytorch.py} | 0 ...xlm_original_pytorch_checkpoint_to_pytorch.py} | 0 ...rt_xlnet_original_tf_checkpoint_to_pytorch.py} | 0 7 files changed, 8 insertions(+), 7 deletions(-) rename pytorch_transformers/{convert_gpt2_checkpoint_to_pytorch.py => convert_gpt2_original_tf_checkpoint_to_pytorch.py} (100%) rename pytorch_transformers/{convert_openai_checkpoint_to_pytorch.py => convert_openai_original_tf_checkpoint_to_pytorch.py} (100%) rename pytorch_transformers/{convert_roberta_checkpoint_to_pytorch.py => convert_roberta_original_pytorch_checkpoint_to_pytorch.py} (100%) rename pytorch_transformers/{convert_transfo_xl_checkpoint_to_pytorch.py => convert_transfo_xl_original_tf_checkpoint_to_pytorch.py} (100%) rename pytorch_transformers/{convert_xlm_checkpoint_to_pytorch.py => convert_xlm_original_pytorch_checkpoint_to_pytorch.py} (100%) rename pytorch_transformers/{convert_xlnet_checkpoint_to_pytorch.py => convert_xlnet_original_tf_checkpoint_to_pytorch.py} (100%) diff --git a/pytorch_transformers/__main__.py b/pytorch_transformers/__main__.py index b047fa7447..ca4936fedf 100644 --- a/pytorch_transformers/__main__.py +++ b/pytorch_transformers/__main__.py @@ -3,7 +3,8 @@ def main(): import sys if (len(sys.argv) < 4 or len(sys.argv) > 6) or sys.argv[1] not in ["bert", "gpt", "transfo_xl", "gpt2", "xlnet", "xlm"]: print( - "Should be used as one of: \n" + "This command line utility let you convert original (author released) model checkpoint to pytorch.\n" + "It should be used as one of: \n" ">> pytorch_transformers bert TF_CHECKPOINT TF_CONFIG PYTORCH_DUMP_OUTPUT, \n" ">> pytorch_transformers gpt OPENAI_GPT_CHECKPOINT_FOLDER_PATH PYTORCH_DUMP_OUTPUT [OPENAI_GPT_CONFIG], \n" ">> pytorch_transformers transfo_xl TF_CHECKPOINT_OR_DATASET PYTORCH_DUMP_OUTPUT [TF_CONFIG] or \n" @@ -13,7 +14,7 @@ def main(): else: if sys.argv[1] == "bert": try: - from .convert_tf_checkpoint_to_pytorch import convert_tf_checkpoint_to_pytorch + from .convert_bert_original_tf_checkpoint_to_pytorch import convert_tf_checkpoint_to_pytorch except ImportError: print("pytorch_transformers can only be used from the commandline to convert TensorFlow models in PyTorch, " "In that case, it requires TensorFlow to be installed. Please see " @@ -29,7 +30,7 @@ def main(): TF_CHECKPOINT = sys.argv.pop() convert_tf_checkpoint_to_pytorch(TF_CHECKPOINT, TF_CONFIG, PYTORCH_DUMP_OUTPUT) elif sys.argv[1] == "gpt": - from .convert_openai_checkpoint_to_pytorch import convert_openai_checkpoint_to_pytorch + from .convert_openai_original_tf_checkpoint_to_pytorch import convert_openai_checkpoint_to_pytorch if len(sys.argv) < 4 or len(sys.argv) > 5: # pylint: disable=line-too-long print("Should be used as `pytorch_transformers gpt OPENAI_GPT_CHECKPOINT_FOLDER_PATH PYTORCH_DUMP_OUTPUT [OPENAI_GPT_CONFIG]`") @@ -45,7 +46,7 @@ def main(): PYTORCH_DUMP_OUTPUT) elif sys.argv[1] == "transfo_xl": try: - from .convert_transfo_xl_checkpoint_to_pytorch import convert_transfo_xl_checkpoint_to_pytorch + from .convert_transfo_xl_original_tf_checkpoint_to_pytorch import convert_transfo_xl_checkpoint_to_pytorch except ImportError: print("pytorch_transformers can only be used from the commandline to convert TensorFlow models in PyTorch, " "In that case, it requires TensorFlow to be installed. Please see " @@ -69,7 +70,7 @@ def main(): convert_transfo_xl_checkpoint_to_pytorch(TF_CHECKPOINT, TF_CONFIG, PYTORCH_DUMP_OUTPUT, TF_DATASET_FILE) elif sys.argv[1] == "gpt2": try: - from .convert_gpt2_checkpoint_to_pytorch import convert_gpt2_checkpoint_to_pytorch + from .convert_gpt2_original_tf_checkpoint_to_pytorch import convert_gpt2_checkpoint_to_pytorch except ImportError: print("pytorch_transformers can only be used from the commandline to convert TensorFlow models in PyTorch, " "In that case, it requires TensorFlow to be installed. Please see " @@ -89,7 +90,7 @@ def main(): convert_gpt2_checkpoint_to_pytorch(TF_CHECKPOINT, TF_CONFIG, PYTORCH_DUMP_OUTPUT) elif sys.argv[1] == "xlnet": try: - from .convert_xlnet_checkpoint_to_pytorch import convert_xlnet_checkpoint_to_pytorch + from .convert_xlnet_original_tf_checkpoint_to_pytorch import convert_xlnet_checkpoint_to_pytorch except ImportError: print("pytorch_transformers can only be used from the commandline to convert TensorFlow models in PyTorch, " "In that case, it requires TensorFlow to be installed. Please see " @@ -113,7 +114,7 @@ def main(): PYTORCH_DUMP_OUTPUT, FINETUNING_TASK) elif sys.argv[1] == "xlm": - from .convert_xlm_checkpoint_to_pytorch import convert_xlm_checkpoint_to_pytorch + from .convert_xlm_original_pytorch_checkpoint_to_pytorch import convert_xlm_checkpoint_to_pytorch if len(sys.argv) != 4: # pylint: disable=line-too-long diff --git a/pytorch_transformers/convert_gpt2_checkpoint_to_pytorch.py b/pytorch_transformers/convert_gpt2_original_tf_checkpoint_to_pytorch.py similarity index 100% rename from pytorch_transformers/convert_gpt2_checkpoint_to_pytorch.py rename to pytorch_transformers/convert_gpt2_original_tf_checkpoint_to_pytorch.py diff --git a/pytorch_transformers/convert_openai_checkpoint_to_pytorch.py b/pytorch_transformers/convert_openai_original_tf_checkpoint_to_pytorch.py similarity index 100% rename from pytorch_transformers/convert_openai_checkpoint_to_pytorch.py rename to pytorch_transformers/convert_openai_original_tf_checkpoint_to_pytorch.py diff --git a/pytorch_transformers/convert_roberta_checkpoint_to_pytorch.py b/pytorch_transformers/convert_roberta_original_pytorch_checkpoint_to_pytorch.py similarity index 100% rename from pytorch_transformers/convert_roberta_checkpoint_to_pytorch.py rename to pytorch_transformers/convert_roberta_original_pytorch_checkpoint_to_pytorch.py diff --git a/pytorch_transformers/convert_transfo_xl_checkpoint_to_pytorch.py b/pytorch_transformers/convert_transfo_xl_original_tf_checkpoint_to_pytorch.py similarity index 100% rename from pytorch_transformers/convert_transfo_xl_checkpoint_to_pytorch.py rename to pytorch_transformers/convert_transfo_xl_original_tf_checkpoint_to_pytorch.py diff --git a/pytorch_transformers/convert_xlm_checkpoint_to_pytorch.py b/pytorch_transformers/convert_xlm_original_pytorch_checkpoint_to_pytorch.py similarity index 100% rename from pytorch_transformers/convert_xlm_checkpoint_to_pytorch.py rename to pytorch_transformers/convert_xlm_original_pytorch_checkpoint_to_pytorch.py diff --git a/pytorch_transformers/convert_xlnet_checkpoint_to_pytorch.py b/pytorch_transformers/convert_xlnet_original_tf_checkpoint_to_pytorch.py similarity index 100% rename from pytorch_transformers/convert_xlnet_checkpoint_to_pytorch.py rename to pytorch_transformers/convert_xlnet_original_tf_checkpoint_to_pytorch.py From 9d0a11a68c8ec41a36de3bd5f20b5f083ea4c59e Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 5 Sep 2019 10:23:04 +0200 Subject: [PATCH 023/219] update dependencies and circle-ci --- .circleci/config.yml | 40 +++++++++++++++++++++++++++++++++++----- requirements.txt | 2 -- setup.py | 3 +-- 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e54d92ab95..dfb7de5634 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,6 +1,6 @@ version: 2 jobs: - build_py3: + build_py3_torch: working_directory: ~/pytorch-transformers docker: - image: circleci/python:3.5 @@ -8,14 +8,29 @@ jobs: parallelism: 1 steps: - checkout + - run: sudo pip install torch - run: sudo pip install --progress-bar off . - run: sudo pip install pytest codecov pytest-cov - run: sudo pip install tensorboardX scikit-learn - - run: sudo pip install tensorflow==2.0.0-rc0 - run: python -m pytest -sv ./pytorch_transformers/tests/ --cov - run: python -m pytest -sv ./examples/ - run: codecov - build_py2: + build_py3_tf: + working_directory: ~/pytorch-transformers + docker: + - image: circleci/python:3.5 + resource_class: xlarge + parallelism: 1 + steps: + - checkout + - run: sudo pip install tensorflow==2.0.0-rc0 + - run: sudo pip install --progress-bar off . + - run: sudo pip install pytest codecov pytest-cov + - run: sudo pip install tensorboardX scikit-learn + - run: python -m pytest -sv ./pytorch_transformers/tests/ --cov + - run: python -m pytest -sv ./examples/ + - run: codecov + build_py2_torch: working_directory: ~/pytorch-transformers resource_class: large parallelism: 1 @@ -23,9 +38,22 @@ jobs: - image: circleci/python:2.7 steps: - checkout + - run: sudo pip install torch - run: sudo pip install --progress-bar off . - run: sudo pip install pytest codecov pytest-cov + - run: python -m pytest -sv ./pytorch_transformers/tests/ --cov + - run: codecov + build_py2_tf: + working_directory: ~/pytorch-transformers + resource_class: large + parallelism: 1 + docker: + - image: circleci/python:2.7 + steps: + - checkout - run: sudo pip install tensorflow==2.0.0-rc0 + - run: sudo pip install --progress-bar off . + - run: sudo pip install pytest codecov pytest-cov - run: python -m pytest -sv ./pytorch_transformers/tests/ --cov - run: codecov deploy_doc: @@ -49,6 +77,8 @@ workflows: version: 2 build_and_test: jobs: - - build_py3 - - build_py2 + - build_py3_torch + - build_py3_tf + - build_py2_torch + - build_py2_tf - deploy_doc: *workflow_filters \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 01dca79d23..9c43abc6d7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,3 @@ -# PyTorch -torch>=1.0.0 # progress bars in model download and training scripts tqdm # Accessing files from S3 directly. diff --git a/setup.py b/setup.py index c31bf6eaeb..903f1d8cac 100644 --- a/setup.py +++ b/setup.py @@ -49,8 +49,7 @@ setup( url="https://github.com/huggingface/pytorch-transformers", packages=find_packages(exclude=["*.tests", "*.tests.*", "tests.*", "tests"]), - install_requires=['torch>=1.0.0', - 'numpy', + install_requires=['numpy', 'boto3', 'requests', 'tqdm', From 518307dfcddd0d47b10b4cffd6ab5f1d8cb6baac Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 5 Sep 2019 11:18:55 +0200 Subject: [PATCH 024/219] test suite independent of framework --- .circleci/config.yml | 9 +- pytorch_transformers/__init__.py | 21 +- ...py => convert_pytorch_checkpoint_to_tf.py} | 37 +- pytorch_transformers/modeling_tf_bert.py | 358 ++++++++++++++++-- .../tests/modeling_auto_test.py | 19 +- .../tests/modeling_bert_test.py | 17 +- .../tests/modeling_common_test.py | 12 +- .../tests/modeling_distilbert_test.py | 12 +- .../tests/modeling_gpt2_test.py | 10 +- .../tests/modeling_openai_test.py | 10 +- .../tests/modeling_roberta_test.py | 13 +- .../tests/modeling_tf_bert_test.py | 241 ++++++------ .../tests/modeling_tf_common_test.py | 9 +- .../tests/modeling_transfo_xl_test.py | 12 +- .../tests/modeling_xlm_test.py | 16 +- .../tests/modeling_xlnet_test.py | 13 +- .../tests/optimization_test.py | 16 +- .../tests/tokenization_auto_test.py | 7 +- .../tests/tokenization_transfo_xl_test.py | 12 +- .../tokenization_transfo_xl.py | 14 +- 20 files changed, 596 insertions(+), 262 deletions(-) rename pytorch_transformers/{convert_bert_pytorch_checkpoint_to_tf.py => convert_pytorch_checkpoint_to_tf.py} (61%) diff --git a/.circleci/config.yml b/.circleci/config.yml index dfb7de5634..2bf082c850 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,7 +10,7 @@ jobs: - checkout - run: sudo pip install torch - run: sudo pip install --progress-bar off . - - run: sudo pip install pytest codecov pytest-cov + - run: sudo pip install pytest==5.0.1 codecov pytest-cov - run: sudo pip install tensorboardX scikit-learn - run: python -m pytest -sv ./pytorch_transformers/tests/ --cov - run: python -m pytest -sv ./examples/ @@ -25,10 +25,9 @@ jobs: - checkout - run: sudo pip install tensorflow==2.0.0-rc0 - run: sudo pip install --progress-bar off . - - run: sudo pip install pytest codecov pytest-cov + - run: sudo pip install pytest==5.0.1 codecov pytest-cov - run: sudo pip install tensorboardX scikit-learn - run: python -m pytest -sv ./pytorch_transformers/tests/ --cov - - run: python -m pytest -sv ./examples/ - run: codecov build_py2_torch: working_directory: ~/pytorch-transformers @@ -40,7 +39,7 @@ jobs: - checkout - run: sudo pip install torch - run: sudo pip install --progress-bar off . - - run: sudo pip install pytest codecov pytest-cov + - run: sudo pip install pytest==5.0.1 codecov pytest-cov - run: python -m pytest -sv ./pytorch_transformers/tests/ --cov - run: codecov build_py2_tf: @@ -53,7 +52,7 @@ jobs: - checkout - run: sudo pip install tensorflow==2.0.0-rc0 - run: sudo pip install --progress-bar off . - - run: sudo pip install pytest codecov pytest-cov + - run: sudo pip install pytest==5.0.1 codecov pytest-cov - run: python -m pytest -sv ./pytorch_transformers/tests/ --cov - run: codecov deploy_doc: diff --git a/pytorch_transformers/__init__.py b/pytorch_transformers/__init__.py index 43b0cb2e07..a41c838015 100644 --- a/pytorch_transformers/__init__.py +++ b/pytorch_transformers/__init__.py @@ -43,11 +43,11 @@ from .configuration_distilbert import DistilBertConfig, DISTILBERT_PRETRAINED_CO # Modeling try: import torch - torch_available = True # pylint: disable=invalid-name + _torch_available = True # pylint: disable=invalid-name except ImportError: - torch_available = False # pylint: disable=invalid-name + _torch_available = False # pylint: disable=invalid-name -if torch_available: +if _torch_available: logger.info("PyTorch version {} available.".format(torch.__version__)) from .modeling_utils import (PreTrainedModel, prune_layer, Conv1D) @@ -87,19 +87,26 @@ if torch_available: # TensorFlow try: import tensorflow as tf - tf_available = True # pylint: disable=invalid-name + assert int(tf.__version__[0]) >= 2 + _tf_available = True # pylint: disable=invalid-name except ImportError: - tf_available = False # pylint: disable=invalid-name + _tf_available = False # pylint: disable=invalid-name -if tf_available: +if _tf_available: logger.info("TensorFlow version {} available.".format(tf.__version__)) from .modeling_tf_utils import TFPreTrainedModel from .modeling_tf_bert import (TFBertPreTrainedModel, TFBertModel, TFBertForPreTraining, - TFBertForMaskedLM, TFBertForNextSentencePrediction, load_pt_weights_in_bert) + TFBertForMaskedLM, TFBertForNextSentencePrediction, load_bert_pt_weights_in_tf) # Files and general utilities from .file_utils import (PYTORCH_TRANSFORMERS_CACHE, PYTORCH_PRETRAINED_BERT_CACHE, cached_path, add_start_docstrings, add_end_docstrings, WEIGHTS_NAME, TF_WEIGHTS_NAME, CONFIG_NAME) + +def is_torch_available(): + return _torch_available + +def is_tf_available(): + return _tf_available diff --git a/pytorch_transformers/convert_bert_pytorch_checkpoint_to_tf.py b/pytorch_transformers/convert_pytorch_checkpoint_to_tf.py similarity index 61% rename from pytorch_transformers/convert_bert_pytorch_checkpoint_to_tf.py rename to pytorch_transformers/convert_pytorch_checkpoint_to_tf.py index 7031f3b523..e682f6c0d3 100644 --- a/pytorch_transformers/convert_bert_pytorch_checkpoint_to_tf.py +++ b/pytorch_transformers/convert_pytorch_checkpoint_to_tf.py @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Convert BERT checkpoint.""" +""" Convert pytorch checkpoints to TensorFlow """ from __future__ import absolute_import from __future__ import division @@ -21,19 +21,22 @@ from __future__ import print_function import argparse import tensorflow as tf -from pytorch_transformers import BertConfig, TFBertForPreTraining, load_pt_weights_in_bert +from pytorch_transformers import BertConfig, TFBertForPreTraining, load_bert_pt_weights_in_tf import logging logging.basicConfig(level=logging.INFO) -def convert_bert_checkpoint_to_tf(pytorch_checkpoint_path, bert_config_file, tf_dump_path): - # Initialise TF model - config = BertConfig.from_json_file(bert_config_file) - print("Building TensorFlow model from configuration: {}".format(str(config))) - model = TFBertForPreTraining(config) +def convert_pt_checkpoint_to_tf(model_type, pytorch_checkpoint_path, config_file, tf_dump_path): + if model_type == 'bert': + # Initialise TF model + config = BertConfig.from_json_file(config_file) + print("Building TensorFlow model from configuration: {}".format(str(config))) + model = TFBertForPreTraining(config) - # Load weights from tf checkpoint - model = load_pt_weights_in_bert(model, config, pytorch_checkpoint_path) + # Load weights from tf checkpoint + model = load_bert_pt_weights_in_tf(model, config, pytorch_checkpoint_path) + else: + raise ValueError("Unrecognized model type, should be one of ['bert'].") # Save pytorch-model print("Save TensorFlow model to {}".format(tf_dump_path)) @@ -43,16 +46,21 @@ def convert_bert_checkpoint_to_tf(pytorch_checkpoint_path, bert_config_file, tf_ if __name__ == "__main__": parser = argparse.ArgumentParser() ## Required parameters + parser.add_argument("--model_type", + default = None, + type = str, + required = True, + help = "Model type selcted in the list of.") parser.add_argument("--pytorch_checkpoint_path", default = None, type = str, required = True, help = "Path to the PyTorch checkpoint path.") - parser.add_argument("--bert_config_file", + parser.add_argument("--config_file", default = None, type = str, required = True, - help = "The config json file corresponding to the pre-trained BERT model. \n" + help = "The config json file corresponding to the pre-trained model. \n" "This specifies the model architecture.") parser.add_argument("--tf_dump_path", default = None, @@ -60,6 +68,7 @@ if __name__ == "__main__": required = True, help = "Path to the output Tensorflow dump file.") args = parser.parse_args() - convert_bert_checkpoint_to_tf(args.pytorch_checkpoint_path, - args.bert_config_file, - args.tf_dump_path) + convert_pt_checkpoint_to_tf(args.model_type.lower(), + args.pytorch_checkpoint_path, + args.config_file, + args.tf_dump_path) diff --git a/pytorch_transformers/modeling_tf_bert.py b/pytorch_transformers/modeling_tf_bert.py index dda713fe7d..bf33cb461d 100644 --- a/pytorch_transformers/modeling_tf_bert.py +++ b/pytorch_transformers/modeling_tf_bert.py @@ -51,7 +51,7 @@ TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP = { } -def load_pt_weights_in_bert(tf_model, config, pytorch_checkpoint_path): +def load_bert_pt_weights_in_tf(tf_model, config, pytorch_checkpoint_path): """ Load pytorch checkpoints in a TF 2.0 model and save it using HDF5 format We use HDF5 to easily do transfer learning (see https://github.com/tensorflow/tensorflow/blob/ee16fcac960ae660e0e4496658a366e2f745e1f0/tensorflow/python/keras/engine/network.py#L1352-L1357). @@ -150,6 +150,7 @@ class TFBertEmbeddings(tf.keras.layers.Layer): self.LayerNorm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name='LayerNorm') self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) + @tf.function def call(self, inputs, training=False): input_ids, position_ids, token_type_ids = inputs @@ -194,6 +195,7 @@ class TFBertSelfAttention(tf.keras.layers.Layer): x = tf.reshape(x, (batch_size, -1, self.num_attention_heads, self.attention_head_size)) return tf.transpose(x, perm=[0, 2, 1, 3]) + @tf.function def call(self, inputs, training=False): hidden_states, attention_mask, head_mask = inputs @@ -242,6 +244,7 @@ class TFBertSelfOutput(tf.keras.layers.Layer): self.LayerNorm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name='LayerNorm') self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) + @tf.function def call(self, inputs, training=False): hidden_states, input_tensor = inputs @@ -261,6 +264,7 @@ class TFBertAttention(tf.keras.layers.Layer): def prune_heads(self, heads): raise NotImplementedError + @tf.function def call(self, inputs, training=False): input_tensor, attention_mask, head_mask = inputs @@ -279,6 +283,7 @@ class TFBertIntermediate(tf.keras.layers.Layer): else: self.intermediate_act_fn = config.hidden_act + @tf.function def call(self, hidden_states): hidden_states = self.dense(hidden_states) hidden_states = self.intermediate_act_fn(hidden_states) @@ -292,6 +297,7 @@ class TFBertOutput(tf.keras.layers.Layer): self.LayerNorm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name='LayerNorm') self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) + @tf.function def call(self, inputs, training=False): hidden_states, input_tensor = inputs @@ -309,6 +315,7 @@ class TFBertLayer(tf.keras.layers.Layer): self.intermediate = TFBertIntermediate(config, name='intermediate') self.bert_output = TFBertOutput(config, name='output') + @tf.function def call(self, inputs, training=False): hidden_states, attention_mask, head_mask = inputs @@ -327,6 +334,7 @@ class TFBertEncoder(tf.keras.layers.Layer): self.output_hidden_states = config.output_hidden_states self.layer = [TFBertLayer(config, name='layer_{}'.format(i)) for i in range(config.num_hidden_layers)] + @tf.function def call(self, inputs, training=False): hidden_states, attention_mask, head_mask = inputs @@ -359,6 +367,7 @@ class TFBertPooler(tf.keras.layers.Layer): super(TFBertPooler, self).__init__(**kwargs) self.dense = tf.keras.layers.Dense(config.hidden_size, activation='tanh', name='dense') + @tf.function def call(self, hidden_states): # We "pool" the model by simply taking the hidden state corresponding # to the first token. @@ -377,6 +386,7 @@ class TFBertPredictionHeadTransform(tf.keras.layers.Layer): self.transform_act_fn = config.hidden_act self.LayerNorm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name='LayerNorm') + @tf.function def call(self, hidden_states): hidden_states = self.dense(hidden_states) hidden_states = self.transform_act_fn(hidden_states) @@ -400,6 +410,7 @@ class TFBertLMPredictionHead(tf.keras.layers.Layer): trainable=True, name='bias') + @tf.function def call(self, hidden_states): hidden_states = self.transform(hidden_states) hidden_states = self.decoder(hidden_states) + self.bias @@ -411,6 +422,7 @@ class TFBertMLMHead(tf.keras.layers.Layer): super(TFBertMLMHead, self).__init__(**kwargs) self.predictions = TFBertLMPredictionHead(config, name='predictions') + @tf.function def call(self, sequence_output): prediction_scores = self.predictions(sequence_output) return prediction_scores @@ -421,6 +433,7 @@ class TFBertNSPHead(tf.keras.layers.Layer): super(TFBertNSPHead, self).__init__(**kwargs) self.seq_relationship = tf.keras.layers.Dense(2, name='seq_relationship') + @tf.function def call(self, pooled_output): seq_relationship_score = self.seq_relationship(pooled_output) return seq_relationship_score @@ -447,6 +460,7 @@ class TFBertMainLayer(tf.keras.layers.Layer): """ raise NotImplementedError + @tf.function def call(self, inputs, training=False): if not isinstance(inputs, (dict, tuple, list)): input_ids = inputs @@ -459,12 +473,12 @@ class TFBertMainLayer(tf.keras.layers.Layer): head_mask = inputs[4] if len(inputs) > 4 else None assert len(inputs) <= 5, "Too many inputs." else: - input_ids = inputs.pop('input_ids') - attention_mask = inputs.pop('attention_mask', None) - token_type_ids = inputs.pop('token_type_ids', None) - position_ids = inputs.pop('position_ids', None) - head_mask = inputs.pop('head_mask', None) - assert len(inputs) == 0, "Unexpected inputs detected: {}. Check inputs dict key names.".format(list(inputs.keys())) + input_ids = inputs.get('input_ids') + attention_mask = inputs.get('attention_mask', None) + token_type_ids = inputs.get('token_type_ids', None) + position_ids = inputs.get('position_ids', None) + head_mask = inputs.get('head_mask', None) + assert len(inputs) <= 5, "Too many inputs." if attention_mask is None: attention_mask = tf.fill(tf.shape(input_ids), 1) @@ -507,23 +521,16 @@ class TFBertMainLayer(tf.keras.layers.Layer): outputs = (sequence_output, pooled_output,) + encoder_outputs[1:] # add hidden_states and attentions if they are here return outputs # sequence_output, pooled_output, (hidden_states), (attentions) + class TFBertPreTrainedModel(TFPreTrainedModel): """ An abstract class to handle weights initialization and a simple interface for dowloading and loading pretrained models. """ config_class = BertConfig pretrained_model_archive_map = TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP - load_pt_weights = load_pt_weights_in_bert + load_pt_weights = load_bert_pt_weights_in_tf base_model_prefix = "bert" - def __init__(self, *inputs, **kwargs): - super(TFBertPreTrainedModel, self).__init__(*inputs, **kwargs) - - def init_weights(self, module): - """ Initialize the weights. - """ - raise NotImplementedError - BERT_START_DOCSTRING = r""" The BERT model was proposed in `BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ @@ -635,6 +642,7 @@ class TFBertModel(TFBertPreTrainedModel): super(TFBertModel, self).__init__(config) self.bert = TFBertMainLayer(config, name='bert') + @tf.function def call(self, inputs, training=False): outputs = self.bert(inputs, training=training) return outputs @@ -687,7 +695,6 @@ class TFBertForPreTraining(TFBertPreTrainedModel): self.cls_mlm = TFBertMLMHead(config, name='cls_mlm') self.cls_nsp = TFBertNSPHead(config, name='cls_nsp') - # self.apply(self.init_weights) # TODO check added weights initialization self.tie_weights() def tie_weights(self): @@ -695,6 +702,7 @@ class TFBertForPreTraining(TFBertPreTrainedModel): """ pass # TODO add weights tying + @tf.function def call(self, inputs, training=False): outputs = self.bert(inputs, training=training) @@ -704,14 +712,6 @@ class TFBertForPreTraining(TFBertPreTrainedModel): outputs = (prediction_scores, seq_relationship_score,) + outputs[2:] # add hidden states and attention if they are here - # if masked_lm_labels is not None and next_sentence_label is not None: - # loss_fct = CrossEntropyLoss(ignore_index=-1) - # masked_lm_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), masked_lm_labels.view(-1)) - # next_sentence_loss = loss_fct(seq_relationship_score.view(-1, 2), next_sentence_label.view(-1)) - # total_loss = masked_lm_loss + next_sentence_loss - # outputs = (total_loss,) + outputs - # TODO add example with losses using model.compile and a dictionary of losses (give names to the output layers) - return outputs # prediction_scores, seq_relationship_score, (hidden_states), (attentions) @@ -753,7 +753,6 @@ class TFBertForMaskedLM(TFBertPreTrainedModel): self.bert = TFBertMainLayer(config, name='bert') self.cls_mlm = TFBertMLMHead(config, name='cls_mlm') - # self.apply(self.init_weights) self.tie_weights() def tie_weights(self): @@ -761,6 +760,7 @@ class TFBertForMaskedLM(TFBertPreTrainedModel): """ pass # TODO add weights tying + @tf.function def call(self, inputs, training=False): outputs = self.bert(inputs, training=training) @@ -768,11 +768,6 @@ class TFBertForMaskedLM(TFBertPreTrainedModel): prediction_scores = self.cls_mlm(sequence_output) outputs = (prediction_scores,) + outputs[2:] # Add hidden states and attention if they are here - # if masked_lm_labels is not None: - # loss_fct = CrossEntropyLoss(ignore_index=-1) - # masked_lm_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), masked_lm_labels.view(-1)) - # outputs = (masked_lm_loss,) + outputs - # TODO example with losses return outputs # prediction_scores, (hidden_states), (attentions) @@ -815,8 +810,7 @@ class TFBertForNextSentencePrediction(TFBertPreTrainedModel): self.bert = TFBertMainLayer(config, name='bert') self.cls_nsp = TFBertNSPHead(config, name='cls_nsp') - # self.apply(self.init_weights) - + @tf.function def call(self, inputs, training=False): outputs = self.bert(inputs, training=training) @@ -824,9 +818,299 @@ class TFBertForNextSentencePrediction(TFBertPreTrainedModel): seq_relationship_score = self.cls_nsp(pooled_output) outputs = (seq_relationship_score,) + outputs[2:] # add hidden states and attention if they are here - # if next_sentence_label is not None: - # loss_fct = CrossEntropyLoss(ignore_index=-1) - # next_sentence_loss = loss_fct(seq_relationship_score.view(-1, 2), next_sentence_label.view(-1)) - # outputs = (next_sentence_loss,) + outputs return outputs # seq_relationship_score, (hidden_states), (attentions) + + +@add_start_docstrings("""Bert Model transformer with a sequence classification/regression head on top (a linear layer on top of + the pooled output) e.g. for GLUE tasks. """, + BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) +class TFBertForSequenceClassification(TFBertPreTrainedModel): + r""" + **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for computing the sequence classification/regression loss. + Indices should be in ``[0, ..., config.num_labels - 1]``. + If ``config.num_labels == 1`` a regression loss is computed (Mean-Square loss), + If ``config.num_labels > 1`` a classification loss is computed (Cross-Entropy). + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Classification (or regression if config.num_labels==1) loss. + **logits**: ``torch.FloatTensor`` of shape ``(batch_size, config.num_labels)`` + Classification (or regression if config.num_labels==1) scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + model = TFBertForSequenceClassification.from_pretrained('bert-base-uncased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + labels = torch.tensor([1]).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, labels=labels) + loss, logits = outputs[:2] + + """ + def __init__(self, config): + super(TFBertForSequenceClassification, self).__init__(config) + self.num_labels = config.num_labels + + self.bert = TFBertMainLayer(config, name='bert') + self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) + self.classifier = tf.keras.layers.Dense(config.num_labels, name='classifier') + + @tf.function + def call(self, inputs, training=False): + outputs = self.bert(inputs, training=training) + + pooled_output = outputs[1] + + pooled_output = self.dropout(pooled_output) + logits = self.classifier(pooled_output) + + outputs = (logits,) + outputs[2:] # add hidden states and attention if they are here + + return outputs # logits, (hidden_states), (attentions) + + +@add_start_docstrings("""Bert Model with a multiple choice classification head on top (a linear layer on top of + the pooled output and a softmax) e.g. for RocStories/SWAG tasks. """, + BERT_START_DOCSTRING) +class TFBertForMultipleChoice(TFBertPreTrainedModel): + r""" + Inputs: + **input_ids**: ``torch.LongTensor`` of shape ``(batch_size, num_choices, sequence_length)``: + Indices of input sequence tokens in the vocabulary. + The second dimension of the input (`num_choices`) indicates the number of choices to score. + To match pre-training, BERT input sequence should be formatted with [CLS] and [SEP] tokens as follows: + + (a) For sequence pairs: + + ``tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]`` + + ``token_type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1`` + + (b) For single sequences: + + ``tokens: [CLS] the dog is hairy . [SEP]`` + + ``token_type_ids: 0 0 0 0 0 0 0`` + + Indices can be obtained using :class:`pytorch_transformers.BertTokenizer`. + See :func:`pytorch_transformers.PreTrainedTokenizer.encode` and + :func:`pytorch_transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + **token_type_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, num_choices, sequence_length)``: + Segment token indices to indicate first and second portions of the inputs. + The second dimension of the input (`num_choices`) indicates the number of choices to score. + Indices are selected in ``[0, 1]``: ``0`` corresponds to a `sentence A` token, ``1`` + corresponds to a `sentence B` token + (see `BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ for more details). + **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, num_choices, sequence_length)``: + Mask to avoid performing attention on padding token indices. + The second dimension of the input (`num_choices`) indicates the number of choices to score. + Mask values selected in ``[0, 1]``: + ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. + **head_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: + Mask to nullify selected heads of the self-attention modules. + Mask values selected in ``[0, 1]``: + ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. + **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for computing the multiple choice classification loss. + Indices should be in ``[0, ..., num_choices]`` where `num_choices` is the size of the second dimension + of the input tensors. (see `input_ids` above) + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Classification loss. + **classification_scores**: ``torch.FloatTensor`` of shape ``(batch_size, num_choices)`` where `num_choices` is the size of the second dimension + of the input tensors. (see `input_ids` above). + Classification scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + model = BertForMultipleChoice.from_pretrained('bert-base-uncased') + choices = ["Hello, my dog is cute", "Hello, my cat is amazing"] + input_ids = torch.tensor([tokenizer.encode(s) for s in choices]).unsqueeze(0) # Batch size 1, 2 choices + labels = torch.tensor(1).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, labels=labels) + loss, classification_scores = outputs[:2] + + """ + def __init__(self, config): + super(TFBertForMultipleChoice, self).__init__(config) + + self.bert = TFBertMainLayer(config, name='bert') + self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) + self.classifier = tf.keras.layers.Dense(1, name='classifier') + + @tf.function + def call(self, inputs, training=False): + if not isinstance(inputs, (dict, tuple, list)): + input_ids = inputs + attention_mask, head_mask, position_ids, token_type_ids = None, None, None, None + elif isinstance(inputs, (tuple, list)): + input_ids = inputs[0] + attention_mask = inputs[1] if len(inputs) > 1 else None + token_type_ids = inputs[2] if len(inputs) > 2 else None + position_ids = inputs[3] if len(inputs) > 3 else None + head_mask = inputs[4] if len(inputs) > 4 else None + assert len(inputs) <= 5, "Too many inputs." + else: + input_ids = inputs.get('input_ids') + attention_mask = inputs.get('attention_mask', None) + token_type_ids = inputs.get('token_type_ids', None) + position_ids = inputs.get('position_ids', None) + head_mask = inputs.get('head_mask', None) + assert len(inputs) <= 5, "Too many inputs." + + num_choices = tf.shape(input_ids)[1] + seq_length = tf.shape(input_ids)[2] + + flat_input_ids = tf.reshape(input_ids, (-1, seq_length)) + flat_attention_mask = tf.reshape(attention_mask, (-1, seq_length)) if attention_mask is not None else None + flat_token_type_ids = tf.reshape(token_type_ids, (-1, seq_length)) if token_type_ids is not None else None + flat_position_ids = tf.reshape(position_ids, (-1, seq_length)) if position_ids is not None else None + + flat_inputs = [flat_input_ids, flat_attention_mask, flat_token_type_ids, flat_position_ids, head_mask] + + outputs = self.bert(flat_inputs, training=training) + + pooled_output = outputs[1] + + pooled_output = self.dropout(pooled_output) + logits = self.classifier(pooled_output) + reshaped_logits = tf.reshape(logits, (-1, num_choices)) + + outputs = (reshaped_logits,) + outputs[2:] # add hidden states and attention if they are here + + return outputs # reshaped_logits, (hidden_states), (attentions) + + +@add_start_docstrings("""Bert Model with a token classification head on top (a linear layer on top of + the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks. """, + BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) +class TFBertForTokenClassification(TFBertPreTrainedModel): + r""" + **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Labels for computing the token classification loss. + Indices should be in ``[0, ..., config.num_labels - 1]``. + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Classification loss. + **scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.num_labels)`` + Classification scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + model = BertForTokenClassification.from_pretrained('bert-base-uncased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + labels = torch.tensor([1] * input_ids.size(1)).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, labels=labels) + loss, scores = outputs[:2] + + """ + def __init__(self, config): + super(TFBertForTokenClassification, self).__init__(config) + self.num_labels = config.num_labels + + self.bert = TFBertMainLayer(config, name='bert') + self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) + self.classifier = tf.keras.layers.Dense(config.num_labels, name='classifier') + + @tf.function + def call(self, inputs, training=False): + outputs = self.bert(inputs, training=training) + + sequence_output = outputs[0] + + sequence_output = self.dropout(sequence_output) + logits = self.classifier(sequence_output) + + outputs = (logits,) + outputs[2:] # add hidden states and attention if they are here + + return outputs # scores, (hidden_states), (attentions) + + +@add_start_docstrings("""Bert Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layers on top of + the hidden-states output to compute `span start logits` and `span end logits`). """, + BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) +class TFBertForQuestionAnswering(TFBertPreTrainedModel): + r""" + **start_positions**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for position (index) of the start of the labelled span for computing the token classification loss. + Positions are clamped to the length of the sequence (`sequence_length`). + Position outside of the sequence are not taken into account for computing the loss. + **end_positions**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for position (index) of the end of the labelled span for computing the token classification loss. + Positions are clamped to the length of the sequence (`sequence_length`). + Position outside of the sequence are not taken into account for computing the loss. + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Total span extraction loss is the sum of a Cross-Entropy for the start and end positions. + **start_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length,)`` + Span-start scores (before SoftMax). + **end_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length,)`` + Span-end scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + model = BertForQuestionAnswering.from_pretrained('bert-base-uncased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + start_positions = torch.tensor([1]) + end_positions = torch.tensor([3]) + outputs = model(input_ids, start_positions=start_positions, end_positions=end_positions) + loss, start_scores, end_scores = outputs[:2] + + """ + def __init__(self, config): + super(TFBertForQuestionAnswering, self).__init__(config) + self.num_labels = config.num_labels + + self.bert = TFBertMainLayer(config, name='bert') + self.qa_outputs = tf.keras.layers.Dense(config.num_labels, name='qa_outputs') + + @tf.function + def call(self, inputs, training=False): + outputs = self.bert(inputs, training=training) + + sequence_output = outputs[0] + + logits = self.qa_outputs(sequence_output) + start_logits, end_logits = tf.split(logits, 2, axis=-1) + start_logits = tf.squeeze(start_logits, axis=-1) + end_logits = tf.squeeze(end_logits, axis=-1) + + outputs = (start_logits, end_logits,) + outputs[2:] + + return outputs # start_logits, end_logits, (hidden_states), (attentions) diff --git a/pytorch_transformers/tests/modeling_auto_test.py b/pytorch_transformers/tests/modeling_auto_test.py index dfdedbbe61..169f722ed7 100644 --- a/pytorch_transformers/tests/modeling_auto_test.py +++ b/pytorch_transformers/tests/modeling_auto_test.py @@ -21,15 +21,18 @@ import shutil import pytest import logging -from pytorch_transformers import (AutoConfig, BertConfig, - AutoModel, BertModel, - AutoModelWithLMHead, BertForMaskedLM, - AutoModelForSequenceClassification, BertForSequenceClassification, - AutoModelForQuestionAnswering, BertForQuestionAnswering) -from pytorch_transformers.modeling_bert import BERT_PRETRAINED_MODEL_ARCHIVE_MAP +try: + from pytorch_transformers import (AutoConfig, BertConfig, + AutoModel, BertModel, + AutoModelWithLMHead, BertForMaskedLM, + AutoModelForSequenceClassification, BertForSequenceClassification, + AutoModelForQuestionAnswering, BertForQuestionAnswering) + from pytorch_transformers.modeling_bert import BERT_PRETRAINED_MODEL_ARCHIVE_MAP -from .modeling_common_test import (CommonTestCases, ids_tensor) -from .configuration_common_test import ConfigTester + from .modeling_common_test import (CommonTestCases, ids_tensor) + from .configuration_common_test import ConfigTester +except ImportError: + pytestmark = pytest.mark.skip("Require Torch") class AutoModelTest(unittest.TestCase): diff --git a/pytorch_transformers/tests/modeling_bert_test.py b/pytorch_transformers/tests/modeling_bert_test.py index 2919cc0336..d63d1b407c 100644 --- a/pytorch_transformers/tests/modeling_bert_test.py +++ b/pytorch_transformers/tests/modeling_bert_test.py @@ -20,21 +20,26 @@ import unittest import shutil import pytest -from pytorch_transformers import (BertConfig, BertModel, BertForMaskedLM, - BertForNextSentencePrediction, BertForPreTraining, - BertForQuestionAnswering, BertForSequenceClassification, - BertForTokenClassification, BertForMultipleChoice) -from pytorch_transformers.modeling_bert import BERT_PRETRAINED_MODEL_ARCHIVE_MAP +from pytorch_transformers import is_torch_available from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester +try: + from pytorch_transformers import (BertConfig, BertModel, BertForMaskedLM, + BertForNextSentencePrediction, BertForPreTraining, + BertForQuestionAnswering, BertForSequenceClassification, + BertForTokenClassification, BertForMultipleChoice) + from pytorch_transformers.modeling_bert import BERT_PRETRAINED_MODEL_ARCHIVE_MAP +except ImportError: + pytestmark = pytest.mark.skip("Require Torch") + class BertModelTest(CommonTestCases.CommonModelTester): all_model_classes = (BertModel, BertForMaskedLM, BertForNextSentencePrediction, BertForPreTraining, BertForQuestionAnswering, BertForSequenceClassification, - BertForTokenClassification) + BertForTokenClassification) if is_torch_available() else () class BertModelTester(object): diff --git a/pytorch_transformers/tests/modeling_common_test.py b/pytorch_transformers/tests/modeling_common_test.py index c50d6678d8..1f778d608f 100644 --- a/pytorch_transformers/tests/modeling_common_test.py +++ b/pytorch_transformers/tests/modeling_common_test.py @@ -25,12 +25,16 @@ import uuid import unittest import logging +import pytest -import torch +try: + import torch -from pytorch_transformers import (PretrainedConfig, PreTrainedModel, - BertModel, BertConfig, BERT_PRETRAINED_MODEL_ARCHIVE_MAP, - GPT2LMHeadModel, GPT2Config, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP) + from pytorch_transformers import (PretrainedConfig, PreTrainedModel, + BertModel, BertConfig, BERT_PRETRAINED_MODEL_ARCHIVE_MAP, + GPT2LMHeadModel, GPT2Config, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP) +except ImportError: + pytestmark = pytest.mark.skip("Require Torch") def _config_zero_init(config): diff --git a/pytorch_transformers/tests/modeling_distilbert_test.py b/pytorch_transformers/tests/modeling_distilbert_test.py index 0d9f231177..10bb4bb398 100644 --- a/pytorch_transformers/tests/modeling_distilbert_test.py +++ b/pytorch_transformers/tests/modeling_distilbert_test.py @@ -17,9 +17,15 @@ from __future__ import division from __future__ import print_function import unittest +import pytest -from pytorch_transformers import (DistilBertConfig, DistilBertModel, DistilBertForMaskedLM, - DistilBertForQuestionAnswering, DistilBertForSequenceClassification) +from pytorch_transformers import is_torch_available + +try: + from pytorch_transformers import (DistilBertConfig, DistilBertModel, DistilBertForMaskedLM, + DistilBertForQuestionAnswering, DistilBertForSequenceClassification) +except ImportError: + pytestmark = pytest.mark.skip("Require Torch") from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester @@ -28,7 +34,7 @@ from .configuration_common_test import ConfigTester class DistilBertModelTest(CommonTestCases.CommonModelTester): all_model_classes = (DistilBertModel, DistilBertForMaskedLM, DistilBertForQuestionAnswering, - DistilBertForSequenceClassification) + DistilBertForSequenceClassification) if is_torch_available() else None test_pruning = True test_torchscript = True test_resize_embeddings = True diff --git a/pytorch_transformers/tests/modeling_gpt2_test.py b/pytorch_transformers/tests/modeling_gpt2_test.py index 2717805120..273200f24f 100644 --- a/pytorch_transformers/tests/modeling_gpt2_test.py +++ b/pytorch_transformers/tests/modeling_gpt2_test.py @@ -20,9 +20,13 @@ import unittest import pytest import shutil +from pytorch_transformers import is_torch_available -from pytorch_transformers import (GPT2Config, GPT2Model, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP, - GPT2LMHeadModel, GPT2DoubleHeadsModel) +try: + from pytorch_transformers import (GPT2Config, GPT2Model, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP, + GPT2LMHeadModel, GPT2DoubleHeadsModel) +except ImportError: + pytestmark = pytest.mark.skip("Require Torch") from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester @@ -30,7 +34,7 @@ from .configuration_common_test import ConfigTester class GPT2ModelTest(CommonTestCases.CommonModelTester): - all_model_classes = (GPT2Model, GPT2LMHeadModel, GPT2DoubleHeadsModel) + all_model_classes = (GPT2Model, GPT2LMHeadModel, GPT2DoubleHeadsModel) if is_torch_available() else () class GPT2ModelTester(object): diff --git a/pytorch_transformers/tests/modeling_openai_test.py b/pytorch_transformers/tests/modeling_openai_test.py index dbef6c52eb..b89990a181 100644 --- a/pytorch_transformers/tests/modeling_openai_test.py +++ b/pytorch_transformers/tests/modeling_openai_test.py @@ -20,9 +20,13 @@ import unittest import pytest import shutil +from pytorch_transformers import is_torch_available -from pytorch_transformers import (OpenAIGPTConfig, OpenAIGPTModel, OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP, - OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel) +try: + from pytorch_transformers import (OpenAIGPTConfig, OpenAIGPTModel, OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP, + OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel) +except ImportError: + pytestmark = pytest.mark.skip("Require Torch") from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester @@ -30,7 +34,7 @@ from .configuration_common_test import ConfigTester class OpenAIGPTModelTest(CommonTestCases.CommonModelTester): - all_model_classes = (OpenAIGPTModel, OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel) + all_model_classes = (OpenAIGPTModel, OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel) if is_torch_available() else () class OpenAIGPTModelTester(object): diff --git a/pytorch_transformers/tests/modeling_roberta_test.py b/pytorch_transformers/tests/modeling_roberta_test.py index 69981af222..ed0f8b5cdc 100644 --- a/pytorch_transformers/tests/modeling_roberta_test.py +++ b/pytorch_transformers/tests/modeling_roberta_test.py @@ -19,10 +19,15 @@ from __future__ import print_function import unittest import shutil import pytest -import torch -from pytorch_transformers import (RobertaConfig, RobertaModel, RobertaForMaskedLM, RobertaForSequenceClassification) -from pytorch_transformers.modeling_roberta import ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP +from pytorch_transformers import is_torch_available + +try: + import torch + from pytorch_transformers import (RobertaConfig, RobertaModel, RobertaForMaskedLM, RobertaForSequenceClassification) + from pytorch_transformers.modeling_roberta import ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP +except ImportError: + pytestmark = pytest.mark.skip("Require Torch") from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester @@ -30,7 +35,7 @@ from .configuration_common_test import ConfigTester class RobertaModelTest(CommonTestCases.CommonModelTester): - all_model_classes = (RobertaForMaskedLM, RobertaModel) + all_model_classes = (RobertaForMaskedLM, RobertaModel) if is_torch_available() else () class RobertaModelTester(object): diff --git a/pytorch_transformers/tests/modeling_tf_bert_test.py b/pytorch_transformers/tests/modeling_tf_bert_test.py index c7b8b02f7b..c95e33d780 100644 --- a/pytorch_transformers/tests/modeling_tf_bert_test.py +++ b/pytorch_transformers/tests/modeling_tf_bert_test.py @@ -24,21 +24,27 @@ import sys from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester +from pytorch_transformers import BertConfig, is_tf_available + try: import tensorflow as tf - - from pytorch_transformers import (BertConfig) - from pytorch_transformers.modeling_tf_bert import TFBertModel, TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP + from pytorch_transformers.modeling_tf_bert import (TFBertModel, TFBertForMaskedLM, + TFBertForNextSentencePrediction, + TFBertForPreTraining, + TFBertForSequenceClassification, + TFBertForMultipleChoice, + TFBertForTokenClassification, + TFBertForQuestionAnswering, + TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP) except ImportError: - pass + pytestmark = pytest.mark.skip("Require TensorFlow") class TFBertModelTest(TFCommonTestCases.TFCommonModelTester): - all_model_classes = (TFBertModel,) - # BertForMaskedLM, BertForNextSentencePrediction, - # BertForPreTraining, BertForQuestionAnswering, BertForSequenceClassification, - # BertForTokenClassification) + all_model_classes = (TFBertModel, TFBertForMaskedLM, TFBertForNextSentencePrediction, + TFBertForPreTraining, TFBertForQuestionAnswering, TFBertForSequenceClassification, + TFBertForTokenClassification) if is_tf_available() else () class TFBertModelTester(object): @@ -123,14 +129,8 @@ class TFBertModelTest(TFCommonTestCases.TFCommonModelTester): return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels - def check_loss_output(self, result): - self.parent.assertListEqual( - list(result["loss"].size()), - []) - def create_and_check_bert_model(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): model = TFBertModel(config=config) - # model.eval() inputs = {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids} @@ -152,125 +152,115 @@ class TFBertModelTest(TFCommonTestCases.TFCommonModelTester): def create_and_check_bert_for_masked_lm(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): - pass - # model = BertForMaskedLM(config=config) - # model.eval() - # loss, prediction_scores = model(input_ids, token_type_ids, input_mask, token_labels) - # result = { - # "loss": loss, - # "prediction_scores": prediction_scores, - # } - # self.parent.assertListEqual( - # list(result["prediction_scores"].size()), - # [self.batch_size, self.seq_length, self.vocab_size]) - # self.check_loss_output(result) + model = TFBertForMaskedLM(config=config) + inputs = {'input_ids': input_ids, + 'attention_mask': input_mask, + 'token_type_ids': token_type_ids} + prediction_scores, = model(inputs) + result = { + "prediction_scores": prediction_scores.numpy(), + } + self.parent.assertListEqual( + list(result["prediction_scores"].shape), + [self.batch_size, self.seq_length, self.vocab_size]) def create_and_check_bert_for_next_sequence_prediction(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): - pass - # model = BertForNextSentencePrediction(config=config) - # model.eval() - # loss, seq_relationship_score = model(input_ids, token_type_ids, input_mask, sequence_labels) - # result = { - # "loss": loss, - # "seq_relationship_score": seq_relationship_score, - # } - # self.parent.assertListEqual( - # list(result["seq_relationship_score"].size()), - # [self.batch_size, 2]) - # self.check_loss_output(result) + model = TFBertForNextSentencePrediction(config=config) + inputs = {'input_ids': input_ids, + 'attention_mask': input_mask, + 'token_type_ids': token_type_ids} + seq_relationship_score, = model(inputs) + result = { + "seq_relationship_score": seq_relationship_score.numpy(), + } + self.parent.assertListEqual( + list(result["seq_relationship_score"].shape), + [self.batch_size, 2]) def create_and_check_bert_for_pretraining(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): - pass - # model = BertForPreTraining(config=config) - # model.eval() - # loss, prediction_scores, seq_relationship_score = model(input_ids, token_type_ids, input_mask, token_labels, sequence_labels) - # result = { - # "loss": loss, - # "prediction_scores": prediction_scores, - # "seq_relationship_score": seq_relationship_score, - # } - # self.parent.assertListEqual( - # list(result["prediction_scores"].size()), - # [self.batch_size, self.seq_length, self.vocab_size]) - # self.parent.assertListEqual( - # list(result["seq_relationship_score"].size()), - # [self.batch_size, 2]) - # self.check_loss_output(result) - - - def create_and_check_bert_for_question_answering(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): - pass - # model = BertForQuestionAnswering(config=config) - # model.eval() - # loss, start_logits, end_logits = model(input_ids, token_type_ids, input_mask, sequence_labels, sequence_labels) - # result = { - # "loss": loss, - # "start_logits": start_logits, - # "end_logits": end_logits, - # } - # self.parent.assertListEqual( - # list(result["start_logits"].size()), - # [self.batch_size, self.seq_length]) - # self.parent.assertListEqual( - # list(result["end_logits"].size()), - # [self.batch_size, self.seq_length]) - # self.check_loss_output(result) + model = TFBertForPreTraining(config=config) + inputs = {'input_ids': input_ids, + 'attention_mask': input_mask, + 'token_type_ids': token_type_ids} + prediction_scores, seq_relationship_score = model(inputs) + result = { + "prediction_scores": prediction_scores.numpy(), + "seq_relationship_score": seq_relationship_score.numpy(), + } + self.parent.assertListEqual( + list(result["prediction_scores"].shape), + [self.batch_size, self.seq_length, self.vocab_size]) + self.parent.assertListEqual( + list(result["seq_relationship_score"].shape), + [self.batch_size, 2]) def create_and_check_bert_for_sequence_classification(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): - pass - # config.num_labels = self.num_labels - # model = BertForSequenceClassification(config) - # model.eval() - # loss, logits = model(input_ids, token_type_ids, input_mask, sequence_labels) - # result = { - # "loss": loss, - # "logits": logits, - # } - # self.parent.assertListEqual( - # list(result["logits"].size()), - # [self.batch_size, self.num_labels]) - # self.check_loss_output(result) - - - def create_and_check_bert_for_token_classification(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): - pass - # config.num_labels = self.num_labels - # model = BertForTokenClassification(config=config) - # model.eval() - # loss, logits = model(input_ids, token_type_ids, input_mask, token_labels) - # result = { - # "loss": loss, - # "logits": logits, - # } - # self.parent.assertListEqual( - # list(result["logits"].size()), - # [self.batch_size, self.seq_length, self.num_labels]) - # self.check_loss_output(result) + config.num_labels = self.num_labels + model = TFBertForSequenceClassification(config=config) + inputs = {'input_ids': input_ids, + 'attention_mask': input_mask, + 'token_type_ids': token_type_ids} + logits, = model(inputs) + result = { + "logits": logits.numpy(), + } + self.parent.assertListEqual( + list(result["logits"].shape), + [self.batch_size, self.num_labels]) def create_and_check_bert_for_multiple_choice(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): - pass - # config.num_choices = self.num_choices - # model = BertForMultipleChoice(config=config) - # model.eval() - # multiple_choice_inputs_ids = input_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous() - # multiple_choice_token_type_ids = token_type_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous() - # multiple_choice_input_mask = input_mask.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous() - # loss, logits = model(multiple_choice_inputs_ids, - # multiple_choice_token_type_ids, - # multiple_choice_input_mask, - # choice_labels) - # result = { - # "loss": loss, - # "logits": logits, - # } - # self.parent.assertListEqual( - # list(result["logits"].size()), - # [self.batch_size, self.num_choices]) - # self.check_loss_output(result) + config.num_choices = self.num_choices + model = TFBertForMultipleChoice(config=config) + multiple_choice_inputs_ids = tf.tile(tf.expand_dims(input_ids, 1), (1, self.num_choices, 1)) + multiple_choice_input_mask = tf.tile(tf.expand_dims(input_mask, 1), (1, self.num_choices, 1)) + multiple_choice_token_type_ids = tf.tile(tf.expand_dims(token_type_ids, 1), (1, self.num_choices, 1)) + inputs = {'input_ids': multiple_choice_inputs_ids, + 'attention_mask': multiple_choice_input_mask, + 'token_type_ids': multiple_choice_token_type_ids} + logits, = model(inputs) + result = { + "logits": logits.numpy(), + } + self.parent.assertListEqual( + list(result["logits"].shape), + [self.batch_size, self.num_choices]) + + + def create_and_check_bert_for_token_classification(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + config.num_labels = self.num_labels + model = TFBertForTokenClassification(config=config) + inputs = {'input_ids': input_ids, + 'attention_mask': input_mask, + 'token_type_ids': token_type_ids} + logits, = model(inputs) + result = { + "logits": logits.numpy(), + } + self.parent.assertListEqual( + list(result["logits"].shape), + [self.batch_size, self.seq_length, self.num_labels]) + + + def create_and_check_bert_for_question_answering(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + model = TFBertForQuestionAnswering(config=config) + inputs = {'input_ids': input_ids, + 'attention_mask': input_mask, + 'token_type_ids': token_type_ids} + start_logits, end_logits = model(inputs) + result = { + "start_logits": start_logits.numpy(), + "end_logits": end_logits.numpy(), + } + self.parent.assertListEqual( + list(result["start_logits"].shape), + [self.batch_size, self.seq_length]) + self.parent.assertListEqual( + list(result["end_logits"].shape), + [self.batch_size, self.seq_length]) def prepare_config_and_inputs_for_common(self): @@ -287,48 +277,39 @@ class TFBertModelTest(TFCommonTestCases.TFCommonModelTester): def test_config(self): self.config_tester.run_common_tests() - @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_bert_model(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_model(*config_and_inputs) - @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_for_masked_lm(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_for_masked_lm(*config_and_inputs) - @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_for_multiple_choice(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_for_multiple_choice(*config_and_inputs) - @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_for_next_sequence_prediction(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_for_next_sequence_prediction(*config_and_inputs) - @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_for_pretraining(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_for_pretraining(*config_and_inputs) - @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_for_question_answering(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_for_question_answering(*config_and_inputs) - @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_for_sequence_classification(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_for_sequence_classification(*config_and_inputs) - @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_for_token_classification(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_for_token_classification(*config_and_inputs) @pytest.mark.slow - @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_model_from_pretrained(self): cache_dir = "/tmp/pytorch_transformers_test/" for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: diff --git a/pytorch_transformers/tests/modeling_tf_common_test.py b/pytorch_transformers/tests/modeling_tf_common_test.py index 1ddfd83be1..404e6ad34e 100644 --- a/pytorch_transformers/tests/modeling_tf_common_test.py +++ b/pytorch_transformers/tests/modeling_tf_common_test.py @@ -30,7 +30,7 @@ try: from pytorch_transformers import TFPreTrainedModel # from pytorch_transformers.modeling_bert import BertModel, BertConfig, BERT_PRETRAINED_MODEL_ARCHIVE_MAP except ImportError: - pass + pytestmark = pytest.mark.skip("Require TensorFlow") def _config_zero_init(config): @@ -50,7 +50,6 @@ class TFCommonTestCases: test_pruning = True test_resize_embeddings = True - @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_initialization(self): pass # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -64,7 +63,6 @@ class TFCommonTestCases: # msg="Parameter {} of model {} seems not properly initialized".format(name, model_class)) - @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_attention_outputs(self): pass # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -105,7 +103,6 @@ class TFCommonTestCases: # self.model_tester.key_len if hasattr(self.model_tester, 'key_len') else self.model_tester.seq_length]) - @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_headmasking(self): pass # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -153,7 +150,6 @@ class TFCommonTestCases: # attentions[-1][..., -1, :, :].flatten().sum().item(), 0.0) - @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_head_pruning(self): pass # if not self.test_pruning: @@ -181,7 +177,6 @@ class TFCommonTestCases: # attentions[-1].shape[-3], self.model_tester.num_attention_heads - 1) - @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_hidden_states_output(self): pass # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -201,7 +196,6 @@ class TFCommonTestCases: # [self.model_tester.seq_length, self.model_tester.hidden_size]) - @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_resize_tokens_embeddings(self): pass # original_config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -238,7 +232,6 @@ class TFCommonTestCases: # self.assertTrue(models_equal) - @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") def test_tie_model_weights(self): pass # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() diff --git a/pytorch_transformers/tests/modeling_transfo_xl_test.py b/pytorch_transformers/tests/modeling_transfo_xl_test.py index f482c47202..9a72335157 100644 --- a/pytorch_transformers/tests/modeling_transfo_xl_test.py +++ b/pytorch_transformers/tests/modeling_transfo_xl_test.py @@ -21,17 +21,21 @@ import random import shutil import pytest -import torch +from pytorch_transformers import is_torch_available -from pytorch_transformers import (TransfoXLConfig, TransfoXLModel, TransfoXLLMHeadModel) -from pytorch_transformers.modeling_transfo_xl import TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP +try: + import torch + from pytorch_transformers import (TransfoXLConfig, TransfoXLModel, TransfoXLLMHeadModel) + from pytorch_transformers.modeling_transfo_xl import TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP +except ImportError: + pytestmark = pytest.mark.skip("Require Torch") from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester class TransfoXLModelTest(CommonTestCases.CommonModelTester): - all_model_classes = (TransfoXLModel, TransfoXLLMHeadModel) + all_model_classes = (TransfoXLModel, TransfoXLLMHeadModel) if is_torch_available() else () test_pruning = False test_torchscript = False test_resize_embeddings = False diff --git a/pytorch_transformers/tests/modeling_xlm_test.py b/pytorch_transformers/tests/modeling_xlm_test.py index dcd0963477..21cf624b9b 100644 --- a/pytorch_transformers/tests/modeling_xlm_test.py +++ b/pytorch_transformers/tests/modeling_xlm_test.py @@ -20,8 +20,14 @@ import unittest import shutil import pytest -from pytorch_transformers import (XLMConfig, XLMModel, XLMWithLMHeadModel, XLMForQuestionAnswering, XLMForSequenceClassification) -from pytorch_transformers.modeling_xlm import XLM_PRETRAINED_MODEL_ARCHIVE_MAP +from pytorch_transformers import is_torch_available + +try: + from pytorch_transformers import (XLMConfig, XLMModel, XLMWithLMHeadModel, XLMForQuestionAnswering, + XLMForSequenceClassification) + from pytorch_transformers.modeling_xlm import XLM_PRETRAINED_MODEL_ARCHIVE_MAP +except ImportError: + pytestmark = pytest.mark.skip("Require Torch") from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester @@ -29,9 +35,9 @@ from .configuration_common_test import ConfigTester class XLMModelTest(CommonTestCases.CommonModelTester): - all_model_classes = (XLMModel, XLMWithLMHeadModel, - XLMForQuestionAnswering, XLMForSequenceClassification) - # , XLMForSequenceClassification, XLMForTokenClassification), + all_model_classes = (XLMModel, XLMWithLMHeadModel, XLMForQuestionAnswering, + XLMForSequenceClassification) if is_torch_available() else () + class XLMModelTester(object): diff --git a/pytorch_transformers/tests/modeling_xlnet_test.py b/pytorch_transformers/tests/modeling_xlnet_test.py index 4445bc17ac..b280ed4592 100644 --- a/pytorch_transformers/tests/modeling_xlnet_test.py +++ b/pytorch_transformers/tests/modeling_xlnet_test.py @@ -23,10 +23,15 @@ import random import shutil import pytest -import torch +from pytorch_transformers import is_torch_available -from pytorch_transformers import (XLNetConfig, XLNetModel, XLNetLMHeadModel, XLNetForSequenceClassification, XLNetForQuestionAnswering) -from pytorch_transformers.modeling_xlnet import XLNET_PRETRAINED_MODEL_ARCHIVE_MAP +try: + import torch + + from pytorch_transformers import (XLNetConfig, XLNetModel, XLNetLMHeadModel, XLNetForSequenceClassification, XLNetForQuestionAnswering) + from pytorch_transformers.modeling_xlnet import XLNET_PRETRAINED_MODEL_ARCHIVE_MAP +except ImportError: + pytestmark = pytest.mark.skip("Require Torch") from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester @@ -34,7 +39,7 @@ from .configuration_common_test import ConfigTester class XLNetModelTest(CommonTestCases.CommonModelTester): all_model_classes=(XLNetModel, XLNetLMHeadModel, - XLNetForSequenceClassification, XLNetForQuestionAnswering) + XLNetForSequenceClassification, XLNetForQuestionAnswering) if is_torch_available() else () test_pruning = False class XLNetModelTester(object): diff --git a/pytorch_transformers/tests/optimization_test.py b/pytorch_transformers/tests/optimization_test.py index 0146541582..07dc22141d 100644 --- a/pytorch_transformers/tests/optimization_test.py +++ b/pytorch_transformers/tests/optimization_test.py @@ -18,11 +18,17 @@ from __future__ import print_function import unittest import os +import pytest -import torch +from pytorch_transformers import is_torch_available -from pytorch_transformers import (AdamW, ConstantLRSchedule, WarmupConstantSchedule, - WarmupCosineSchedule, WarmupCosineWithHardRestartsSchedule, WarmupLinearSchedule) +try: + import torch + + from pytorch_transformers import (AdamW, ConstantLRSchedule, WarmupConstantSchedule, + WarmupCosineSchedule, WarmupCosineWithHardRestartsSchedule, WarmupLinearSchedule) +except ImportError: + pytestmark = pytest.mark.skip("Require Torch") from .tokenization_tests_commons import TemporaryDirectory @@ -71,8 +77,8 @@ class OptimizationTest(unittest.TestCase): class ScheduleInitTest(unittest.TestCase): - m = torch.nn.Linear(50, 50) - optimizer = AdamW(m.parameters(), lr=10.) + m = torch.nn.Linear(50, 50) if is_torch_available() else None + optimizer = AdamW(m.parameters(), lr=10.) if is_torch_available() else None num_steps = 10 def assertListAlmostEqual(self, list1, list2, tol): diff --git a/pytorch_transformers/tests/tokenization_auto_test.py b/pytorch_transformers/tests/tokenization_auto_test.py index f4f82083f2..7cee7ebc28 100644 --- a/pytorch_transformers/tests/tokenization_auto_test.py +++ b/pytorch_transformers/tests/tokenization_auto_test.py @@ -22,20 +22,19 @@ import pytest import logging from pytorch_transformers import AutoTokenizer, BertTokenizer, AutoTokenizer, GPT2Tokenizer -from pytorch_transformers.modeling_bert import BERT_PRETRAINED_MODEL_ARCHIVE_MAP -from pytorch_transformers.modeling_gpt2 import GPT2_PRETRAINED_MODEL_ARCHIVE_MAP +from pytorch_transformers import BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP class AutoTokenizerTest(unittest.TestCase): def test_tokenizer_from_pretrained(self): logging.basicConfig(level=logging.INFO) - for model_name in list(BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: + for model_name in list(BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys())[:1]: tokenizer = AutoTokenizer.from_pretrained(model_name) self.assertIsNotNone(tokenizer) self.assertIsInstance(tokenizer, BertTokenizer) self.assertGreater(len(tokenizer), 0) - for model_name in list(GPT2_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: + for model_name in list(GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys())[:1]: tokenizer = AutoTokenizer.from_pretrained(model_name) self.assertIsNotNone(tokenizer) self.assertIsInstance(tokenizer, GPT2Tokenizer) diff --git a/pytorch_transformers/tests/tokenization_transfo_xl_test.py b/pytorch_transformers/tests/tokenization_transfo_xl_test.py index f881cf1d2b..1406bac48b 100644 --- a/pytorch_transformers/tests/tokenization_transfo_xl_test.py +++ b/pytorch_transformers/tests/tokenization_transfo_xl_test.py @@ -16,15 +16,21 @@ from __future__ import absolute_import, division, print_function, unicode_litera import os import unittest +import pytest from io import open -from pytorch_transformers.tokenization_transfo_xl import TransfoXLTokenizer, VOCAB_FILES_NAMES +from pytorch_transformers import is_torch_available -from.tokenization_tests_commons import CommonTestCases +try: + from pytorch_transformers.tokenization_transfo_xl import TransfoXLTokenizer, VOCAB_FILES_NAMES +except ImportError: + pytestmark = pytest.mark.skip("Require Torch") # TODO: untangle Transfo-XL tokenizer from torch.load and torch.save + +from .tokenization_tests_commons import CommonTestCases class TransfoXLTokenizationTest(CommonTestCases.CommonTokenizerTester): - tokenizer_class = TransfoXLTokenizer + tokenizer_class = TransfoXLTokenizer if is_torch_available() else None def setUp(self): super(TransfoXLTokenizationTest, self).setUp() diff --git a/pytorch_transformers/tokenization_transfo_xl.py b/pytorch_transformers/tokenization_transfo_xl.py index 66bc01c1bb..8d5a0ce9d4 100644 --- a/pytorch_transformers/tokenization_transfo_xl.py +++ b/pytorch_transformers/tokenization_transfo_xl.py @@ -26,16 +26,20 @@ import sys from collections import Counter, OrderedDict from io import open -import torch import numpy as np from .file_utils import cached_path from .tokenization_utils import PreTrainedTokenizer -if sys.version_info[0] == 2: - import cPickle as pickle -else: - import pickle +try: + import torch +except ImportError: + pass + +# if sys.version_info[0] == 2: +# import cPickle as pickle +# else: +# import pickle logger = logging.getLogger(__name__) From e30579f7646a3755ec3b33ed9210d56e548b97fa Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 5 Sep 2019 11:20:56 +0200 Subject: [PATCH 025/219] no pytest version checking --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2bf082c850..c49cf7df8a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,7 +10,7 @@ jobs: - checkout - run: sudo pip install torch - run: sudo pip install --progress-bar off . - - run: sudo pip install pytest==5.0.1 codecov pytest-cov + - run: sudo pip install pytest codecov pytest-cov - run: sudo pip install tensorboardX scikit-learn - run: python -m pytest -sv ./pytorch_transformers/tests/ --cov - run: python -m pytest -sv ./examples/ @@ -25,7 +25,7 @@ jobs: - checkout - run: sudo pip install tensorflow==2.0.0-rc0 - run: sudo pip install --progress-bar off . - - run: sudo pip install pytest==5.0.1 codecov pytest-cov + - run: sudo pip install pytest codecov pytest-cov - run: sudo pip install tensorboardX scikit-learn - run: python -m pytest -sv ./pytorch_transformers/tests/ --cov - run: codecov @@ -39,7 +39,7 @@ jobs: - checkout - run: sudo pip install torch - run: sudo pip install --progress-bar off . - - run: sudo pip install pytest==5.0.1 codecov pytest-cov + - run: sudo pip install pytest codecov pytest-cov - run: python -m pytest -sv ./pytorch_transformers/tests/ --cov - run: codecov build_py2_tf: @@ -52,7 +52,7 @@ jobs: - checkout - run: sudo pip install tensorflow==2.0.0-rc0 - run: sudo pip install --progress-bar off . - - run: sudo pip install pytest==5.0.1 codecov pytest-cov + - run: sudo pip install pytest codecov pytest-cov - run: python -m pytest -sv ./pytorch_transformers/tests/ --cov - run: codecov deploy_doc: From 6dc4b6f34c26840b82200c1951d176698c55bb0f Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 5 Sep 2019 11:22:13 +0200 Subject: [PATCH 026/219] skip transfo-xl tokenizer tests with tf for now --- pytorch_transformers/tests/tokenization_transfo_xl_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pytorch_transformers/tests/tokenization_transfo_xl_test.py b/pytorch_transformers/tests/tokenization_transfo_xl_test.py index 1406bac48b..792033d82c 100644 --- a/pytorch_transformers/tests/tokenization_transfo_xl_test.py +++ b/pytorch_transformers/tests/tokenization_transfo_xl_test.py @@ -22,6 +22,7 @@ from io import open from pytorch_transformers import is_torch_available try: + import torch from pytorch_transformers.tokenization_transfo_xl import TransfoXLTokenizer, VOCAB_FILES_NAMES except ImportError: pytestmark = pytest.mark.skip("Require Torch") # TODO: untangle Transfo-XL tokenizer from torch.load and torch.save From f5c698b21a7e0514e1b01b86e80397d625e48616 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 5 Sep 2019 12:02:14 +0200 Subject: [PATCH 027/219] add weights tying, attention and hidden states output tests --- pytorch_transformers/modeling_tf_bert.py | 81 +++++++++++----- pytorch_transformers/modeling_tf_utils.py | 25 +++-- .../tests/modeling_tf_common_test.py | 94 +++++++++---------- 3 files changed, 121 insertions(+), 79 deletions(-) diff --git a/pytorch_transformers/modeling_tf_bert.py b/pytorch_transformers/modeling_tf_bert.py index bf33cb461d..fb73515f6e 100644 --- a/pytorch_transformers/modeling_tf_bert.py +++ b/pytorch_transformers/modeling_tf_bert.py @@ -141,7 +141,9 @@ class TFBertEmbeddings(tf.keras.layers.Layer): """ def __init__(self, config, **kwargs): super(TFBertEmbeddings, self).__init__(**kwargs) - self.word_embeddings = tf.keras.layers.Embedding(config.vocab_size, config.hidden_size, name='word_embeddings') + self.vocab_size = config.vocab_size + self.hidden_size = config.hidden_size + self.position_embeddings = tf.keras.layers.Embedding(config.max_position_embeddings, config.hidden_size, name='position_embeddings') self.token_type_embeddings = tf.keras.layers.Embedding(config.type_vocab_size, config.hidden_size, name='token_type_embeddings') @@ -150,8 +152,44 @@ class TFBertEmbeddings(tf.keras.layers.Layer): self.LayerNorm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name='LayerNorm') self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) + def build(self, input_shape): + """Build shared word embedding layer """ + with tf.name_scope("word_embeddings"): + # Create and initialize weights. The random normal initializer was chosen + # arbitrarily, and works well. + self.word_embeddings = self.add_weight( + "weight", + shape=[self.vocab_size, self.hidden_size], + initializer=tf.random_normal_initializer( + mean=0., stddev=self.hidden_size**-0.5)) + super(TFBertEmbeddings, self).build(input_shape) + @tf.function - def call(self, inputs, training=False): + def call(self, inputs, mode="embedding", training=False): + """Get token embeddings of inputs. + Args: + inputs: list of three int64 tensors with shape [batch_size, length]: (input_ids, position_ids, token_type_ids) + mode: string, a valid value is one of "embedding" and "linear". + Returns: + outputs: (1) If mode == "embedding", output embedding tensor, float32 with + shape [batch_size, length, embedding_size]; (2) mode == "linear", output + linear tensor, float32 with shape [batch_size, length, vocab_size]. + Raises: + ValueError: if mode is not valid. + + Shared weights logic adapted from + https://github.com/tensorflow/models/blob/a009f4fb9d2fc4949e32192a944688925ef78659/official/transformer/v2/embedding_layer.py#L24 + """ + if mode == "embedding": + return self._embedding(inputs, training=training) + elif mode == "linear": + return self._linear(inputs) + else: + raise ValueError("mode {} is not valid.".format(mode)) + + def _embedding(self, inputs, training=False): + """Applies embedding based on inputs tensor.""" + # Create binary mask of size [batch_size, length] input_ids, position_ids, token_type_ids = inputs seq_length = tf.shape(input_ids)[1] @@ -160,7 +198,7 @@ class TFBertEmbeddings(tf.keras.layers.Layer): if token_type_ids is None: token_type_ids = tf.fill(tf.shape(input_ids), 0) - words_embeddings = self.word_embeddings(input_ids) + words_embeddings = tf.gather(self.word_embeddings, input_ids) position_embeddings = self.position_embeddings(position_ids) token_type_embeddings = self.token_type_embeddings(token_type_ids) @@ -170,6 +208,21 @@ class TFBertEmbeddings(tf.keras.layers.Layer): embeddings = self.dropout(embeddings) return embeddings + def _linear(self, inputs): + """Computes logits by running inputs through a linear layer. + Args: + inputs: A float32 tensor with shape [batch_size, length, hidden_size] + Returns: + float32 tensor with shape [batch_size, length, vocab_size]. + """ + batch_size = tf.shape(inputs)[0] + length = tf.shape(inputs)[1] + + x = tf.reshape(inputs, [-1, self.hidden_size]) + logits = tf.matmul(x, self.word_embeddings, transpose_b=True) + + return tf.reshape(logits, [batch_size, length, self.vocab_size]) + class TFBertSelfAttention(tf.keras.layers.Layer): def __init__(self, config, **kwargs): @@ -448,8 +501,6 @@ class TFBertMainLayer(tf.keras.layers.Layer): self.encoder = TFBertEncoder(config, name='encoder') self.pooler = TFBertPooler(config, name='pooler') - # self.apply(self.init_weights) # TODO check weights initialization - def _resize_token_embeddings(self, new_num_tokens): raise NotImplementedError @@ -692,22 +743,14 @@ class TFBertForPreTraining(TFBertPreTrainedModel): super(TFBertForPreTraining, self).__init__(config) self.bert = TFBertMainLayer(config, name='bert') - self.cls_mlm = TFBertMLMHead(config, name='cls_mlm') self.cls_nsp = TFBertNSPHead(config, name='cls_nsp') - self.tie_weights() - - def tie_weights(self): - """ Make sure we are sharing the input and output embeddings. - """ - pass # TODO add weights tying - @tf.function def call(self, inputs, training=False): outputs = self.bert(inputs, training=training) sequence_output, pooled_output = outputs[:2] - prediction_scores = self.cls_mlm(sequence_output) + prediction_scores = self.bert.embeddings(sequence_output, mode="linear", training=training) seq_relationship_score = self.cls_nsp(pooled_output) outputs = (prediction_scores, seq_relationship_score,) + outputs[2:] # add hidden states and attention if they are here @@ -751,21 +794,13 @@ class TFBertForMaskedLM(TFBertPreTrainedModel): super(TFBertForMaskedLM, self).__init__(config) self.bert = TFBertMainLayer(config, name='bert') - self.cls_mlm = TFBertMLMHead(config, name='cls_mlm') - - self.tie_weights() - - def tie_weights(self): - """ Make sure we are sharing the input and output embeddings. - """ - pass # TODO add weights tying @tf.function def call(self, inputs, training=False): outputs = self.bert(inputs, training=training) sequence_output = outputs[0] - prediction_scores = self.cls_mlm(sequence_output) + prediction_scores = self.bert.embeddings(sequence_output, mode="linear", training=training) outputs = (prediction_scores,) + outputs[2:] # Add hidden states and attention if they are here diff --git a/pytorch_transformers/modeling_tf_utils.py b/pytorch_transformers/modeling_tf_utils.py index 442d9dbe44..cb1588e399 100644 --- a/pytorch_transformers/modeling_tf_utils.py +++ b/pytorch_transformers/modeling_tf_utils.py @@ -64,7 +64,7 @@ class TFPreTrainedModel(tf.keras.Model): self.config = config def _get_resized_embeddings(self, old_embeddings, new_num_tokens=None): - """ Build a resized Embedding Module from a provided token Embedding Module. + """ Build a resized Embedding Variable from a provided token Embedding Module. Increasing the size will add newly initialized vectors at the end Reducing the size will remove vectors from the end @@ -77,12 +77,25 @@ class TFPreTrainedModel(tf.keras.Model): Return: ``torch.nn.Embeddings`` Pointer to the resized Embedding Module or the old Embedding Module if new_num_tokens is None """ - raise NotImplementedError + # if new_num_tokens is None: + # return old_embeddings - def _tie_or_clone_weights(self, first_module, second_module): - """ Tie or clone module weights depending of weither we are using TorchScript or not - """ - raise NotImplementedError + # old_num_tokens, old_embedding_dim = old_embeddings.weight.size() + # if old_num_tokens == new_num_tokens: + # return old_embeddings + + # # Build new embeddings + # new_embeddings = nn.Embedding(new_num_tokens, old_embedding_dim) + # new_embeddings.to(old_embeddings.weight.device) + + # # initialize all new embeddings (in particular added tokens) + # self._init_weights(new_embeddings) + + # # Copy word embeddings from the previous weights + # num_tokens_to_copy = min(old_num_tokens, new_num_tokens) + # new_embeddings.weight.data[:num_tokens_to_copy, :] = old_embeddings.weight.data[:num_tokens_to_copy, :] + + # return new_embeddings def resize_token_embeddings(self, new_num_tokens=None): """ Resize input token embeddings matrix of the model if new_num_tokens != config.vocab_size. diff --git a/pytorch_transformers/tests/modeling_tf_common_test.py b/pytorch_transformers/tests/modeling_tf_common_test.py index 404e6ad34e..f9b87eed9a 100644 --- a/pytorch_transformers/tests/modeling_tf_common_test.py +++ b/pytorch_transformers/tests/modeling_tf_common_test.py @@ -64,44 +64,40 @@ class TFCommonTestCases: def test_attention_outputs(self): - pass - # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() - # for model_class in self.all_model_classes: - # config.output_attentions = True - # config.output_hidden_states = False - # model = model_class(config) - # model.eval() - # outputs = model(**inputs_dict) - # attentions = outputs[-1] - # self.assertEqual(model.config.output_attentions, True) - # self.assertEqual(model.config.output_hidden_states, False) - # self.assertEqual(len(attentions), self.model_tester.num_hidden_layers) - # self.assertListEqual( - # list(attentions[0].shape[-3:]), - # [self.model_tester.num_attention_heads, - # self.model_tester.seq_length, - # self.model_tester.key_len if hasattr(self.model_tester, 'key_len') else self.model_tester.seq_length]) - # out_len = len(outputs) + for model_class in self.all_model_classes: + config.output_attentions = True + config.output_hidden_states = False + model = model_class(config) + outputs = model(inputs_dict) + attentions = [t.numpy() for t in outputs[-1]] + self.assertEqual(model.config.output_attentions, True) + self.assertEqual(model.config.output_hidden_states, False) + self.assertEqual(len(attentions), self.model_tester.num_hidden_layers) + self.assertListEqual( + list(attentions[0].shape[-3:]), + [self.model_tester.num_attention_heads, + self.model_tester.seq_length, + self.model_tester.key_len if hasattr(self.model_tester, 'key_len') else self.model_tester.seq_length]) + out_len = len(outputs) - # # Check attention is always last and order is fine - # config.output_attentions = True - # config.output_hidden_states = True - # model = model_class(config) - # model.eval() - # outputs = model(**inputs_dict) - # self.assertEqual(out_len+1, len(outputs)) - # self.assertEqual(model.config.output_attentions, True) - # self.assertEqual(model.config.output_hidden_states, True) - - # attentions = outputs[-1] - # self.assertEqual(len(attentions), self.model_tester.num_hidden_layers) - # self.assertListEqual( - # list(attentions[0].shape[-3:]), - # [self.model_tester.num_attention_heads, - # self.model_tester.seq_length, - # self.model_tester.key_len if hasattr(self.model_tester, 'key_len') else self.model_tester.seq_length]) + # Check attention is always last and order is fine + config.output_attentions = True + config.output_hidden_states = True + model = model_class(config) + outputs = model(inputs_dict) + self.assertEqual(out_len+1, len(outputs)) + self.assertEqual(model.config.output_attentions, True) + self.assertEqual(model.config.output_hidden_states, True) + attentions = [t.numpy() for t in outputs[-1]] + self.assertEqual(len(attentions), self.model_tester.num_hidden_layers) + self.assertListEqual( + list(attentions[0].shape[-3:]), + [self.model_tester.num_attention_heads, + self.model_tester.seq_length, + self.model_tester.key_len if hasattr(self.model_tester, 'key_len') else self.model_tester.seq_length]) def test_headmasking(self): pass @@ -178,22 +174,20 @@ class TFCommonTestCases: def test_hidden_states_output(self): - pass - # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() - # for model_class in self.all_model_classes: - # config.output_hidden_states = True - # config.output_attentions = False - # model = model_class(config) - # model.eval() - # outputs = model(**inputs_dict) - # hidden_states = outputs[-1] - # self.assertEqual(model.config.output_attentions, False) - # self.assertEqual(model.config.output_hidden_states, True) - # self.assertEqual(len(hidden_states), self.model_tester.num_hidden_layers + 1) - # self.assertListEqual( - # list(hidden_states[0].shape[-2:]), - # [self.model_tester.seq_length, self.model_tester.hidden_size]) + for model_class in self.all_model_classes: + config.output_hidden_states = True + config.output_attentions = False + model = model_class(config) + outputs = model(inputs_dict) + hidden_states = [t.numpy() for t in outputs[-1]] + self.assertEqual(model.config.output_attentions, False) + self.assertEqual(model.config.output_hidden_states, True) + self.assertEqual(len(hidden_states), self.model_tester.num_hidden_layers + 1) + self.assertListEqual( + list(hidden_states[0].shape[-2:]), + [self.model_tester.seq_length, self.model_tester.hidden_size]) def test_resize_tokens_embeddings(self): From 01597e5b90b3bbbecb901fdf8f39f9249e0ccc84 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 5 Sep 2019 12:21:08 +0200 Subject: [PATCH 028/219] add tf auto models + tests --- pytorch_transformers/__init__.py | 9 +- pytorch_transformers/modeling_tf_auto.py | 488 ++++++++++++++++++ pytorch_transformers/modeling_tf_utils.py | 11 - .../tests/modeling_tf_auto_test.py | 85 +++ 4 files changed, 581 insertions(+), 12 deletions(-) create mode 100644 pytorch_transformers/modeling_tf_auto.py create mode 100644 pytorch_transformers/tests/modeling_tf_auto_test.py diff --git a/pytorch_transformers/__init__.py b/pytorch_transformers/__init__.py index a41c838015..f13457f073 100644 --- a/pytorch_transformers/__init__.py +++ b/pytorch_transformers/__init__.py @@ -96,8 +96,15 @@ if _tf_available: logger.info("TensorFlow version {} available.".format(tf.__version__)) from .modeling_tf_utils import TFPreTrainedModel + from .modeling_tf_auto import (TFAutoModel, TFAutoModelForSequenceClassification, TFAutoModelForQuestionAnswering, + TFAutoModelWithLMHead) + from .modeling_tf_bert import (TFBertPreTrainedModel, TFBertModel, TFBertForPreTraining, - TFBertForMaskedLM, TFBertForNextSentencePrediction, load_bert_pt_weights_in_tf) + TFBertForMaskedLM, TFBertForNextSentencePrediction, + TFBertForSequenceClassification, TFBertForMultipleChoice, + TFBertForTokenClassification, TFBertForQuestionAnswering, + load_bert_pt_weights_in_tf, + TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP) # Files and general utilities diff --git a/pytorch_transformers/modeling_tf_auto.py b/pytorch_transformers/modeling_tf_auto.py new file mode 100644 index 0000000000..3b10d700a1 --- /dev/null +++ b/pytorch_transformers/modeling_tf_auto.py @@ -0,0 +1,488 @@ +# coding=utf-8 +# Copyright 2018 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Auto Model class. """ + +from __future__ import absolute_import, division, print_function, unicode_literals + +import logging + +from .modeling_tf_bert import TFBertModel, TFBertForMaskedLM, TFBertForSequenceClassification, TFBertForQuestionAnswering + +from .file_utils import add_start_docstrings + +logger = logging.getLogger(__name__) + + +class TFAutoModel(object): + r""" + :class:`~pytorch_transformers.TFAutoModel` is a generic model class + that will be instantiated as one of the base model classes of the library + when created with the `TFAutoModel.from_pretrained(pretrained_model_name_or_path)` + class method. + + The `from_pretrained()` method takes care of returning the correct model class instance + using pattern matching on the `pretrained_model_name_or_path` string. + + The base model class to instantiate is selected as the first pattern matching + in the `pretrained_model_name_or_path` string (in the following order): + - contains `distilbert`: DistilBertModel (DistilBERT model) + - contains `roberta`: RobertaModel (RoBERTa model) + - contains `bert`: TFBertModel (Bert model) + - contains `openai-gpt`: OpenAIGPTModel (OpenAI GPT model) + - contains `gpt2`: GPT2Model (OpenAI GPT-2 model) + - contains `transfo-xl`: TransfoXLModel (Transformer-XL model) + - contains `xlnet`: XLNetModel (XLNet model) + - contains `xlm`: XLMModel (XLM model) + + This class cannot be instantiated using `__init__()` (throws an error). + """ + def __init__(self): + raise EnvironmentError("TFAutoModel is designed to be instantiated " + "using the `TFAutoModel.from_pretrained(pretrained_model_name_or_path)` method.") + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): + r""" Instantiates one of the base model classes of the library + from a pre-trained model configuration. + + The model class to instantiate is selected as the first pattern matching + in the `pretrained_model_name_or_path` string (in the following order): + - contains `distilbert`: DistilBertModel (DistilBERT model) + - contains `roberta`: RobertaModel (RoBERTa model) + - contains `bert`: TFBertModel (Bert model) + - contains `openai-gpt`: OpenAIGPTModel (OpenAI GPT model) + - contains `gpt2`: GPT2Model (OpenAI GPT-2 model) + - contains `transfo-xl`: TransfoXLModel (Transformer-XL model) + - contains `xlnet`: XLNetModel (XLNet model) + - contains `xlm`: XLMModel (XLM model) + + The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated) + To train the model, you should first set it back in training mode with `model.train()` + + Params: + pretrained_model_name_or_path: either: + + - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. + - a path to a `directory` containing model weights saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. + - a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. + + model_args: (`optional`) Sequence of positional arguments: + All remaning positional arguments will be passed to the underlying model's ``__init__`` method + + config: (`optional`) instance of a class derived from :class:`~pytorch_transformers.PretrainedConfig`: + Configuration for the model to use instead of an automatically loaded configuation. Configuration can be automatically loaded when: + + - the model is a model provided by the library (loaded with the ``shortcut-name`` string of a pretrained model), or + - the model was saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained` and is reloaded by suppling the save directory. + - the model is loaded by suppling a local directory as ``pretrained_model_name_or_path`` and a configuration JSON file named `config.json` is found in the directory. + + state_dict: (`optional`) dict: + an optional state dictionnary for the model to use instead of a state dictionary loaded from saved weights file. + This option can be used if you want to create a model from a pretrained configuration but load your own weights. + In this case though, you should check if using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained` and :func:`~pytorch_transformers.PreTrainedModel.from_pretrained` is not a simpler option. + + cache_dir: (`optional`) string: + Path to a directory in which a downloaded pre-trained model + configuration should be cached if the standard cache should not be used. + + force_download: (`optional`) boolean, default False: + Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + + proxies: (`optional`) dict, default None: + A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. + The proxies are used on each request. + + output_loading_info: (`optional`) boolean: + Set to ``True`` to also return a dictionnary containing missing keys, unexpected keys and error messages. + + kwargs: (`optional`) Remaining dictionary of keyword arguments: + Can be used to update the configuration object (after it being loaded) and initiate the model. (e.g. ``output_attention=True``). Behave differently depending on whether a `config` is provided or automatically loaded: + + - If a configuration is provided with ``config``, ``**kwargs`` will be directly passed to the underlying model's ``__init__`` method (we assume all relevant updates to the configuration have already been done) + - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~pytorch_transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function. + + Examples:: + + model = TFAutoModel.from_pretrained('bert-base-uncased') # Download model and configuration from S3 and cache. + model = TFAutoModel.from_pretrained('./test/bert_model/') # E.g. model was saved using `save_pretrained('./test/saved_model/')` + model = TFAutoModel.from_pretrained('bert-base-uncased', output_attention=True) # Update configuration during loading + assert model.config.output_attention == True + # Loading from a TF checkpoint file instead of a PyTorch model (slower) + config = AutoConfig.from_json_file('./tf_model/bert_tf_model_config.json') + model = TFAutoModel.from_pretrained('./tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config) + + """ + if 'distilbert' in pretrained_model_name_or_path: + raise NotImplementedError + elif 'roberta' in pretrained_model_name_or_path: + raise NotImplementedError + elif 'bert' in pretrained_model_name_or_path: + return TFBertModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'openai-gpt' in pretrained_model_name_or_path: + raise NotImplementedError + elif 'gpt2' in pretrained_model_name_or_path: + raise NotImplementedError + elif 'transfo-xl' in pretrained_model_name_or_path: + raise NotImplementedError + elif 'xlnet' in pretrained_model_name_or_path: + raise NotImplementedError + elif 'xlm' in pretrained_model_name_or_path: + raise NotImplementedError + + raise ValueError("Unrecognized model identifier in {}. Should contains one of " + "'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', " + "'xlm', 'roberta'".format(pretrained_model_name_or_path)) + + +class TFAutoModelWithLMHead(object): + r""" + :class:`~pytorch_transformers.TFAutoModelWithLMHead` is a generic model class + that will be instantiated as one of the language modeling model classes of the library + when created with the `TFAutoModelWithLMHead.from_pretrained(pretrained_model_name_or_path)` + class method. + + The `from_pretrained()` method takes care of returning the correct model class instance + using pattern matching on the `pretrained_model_name_or_path` string. + + The model class to instantiate is selected as the first pattern matching + in the `pretrained_model_name_or_path` string (in the following order): + - contains `distilbert`: DistilBertForMaskedLM (DistilBERT model) + - contains `roberta`: RobertaForMaskedLM (RoBERTa model) + - contains `bert`: TFBertForMaskedLM (Bert model) + - contains `openai-gpt`: OpenAIGPTLMHeadModel (OpenAI GPT model) + - contains `gpt2`: GPT2LMHeadModel (OpenAI GPT-2 model) + - contains `transfo-xl`: TransfoXLLMHeadModel (Transformer-XL model) + - contains `xlnet`: XLNetLMHeadModel (XLNet model) + - contains `xlm`: XLMWithLMHeadModel (XLM model) + + This class cannot be instantiated using `__init__()` (throws an error). + """ + def __init__(self): + raise EnvironmentError("TFAutoModelWithLMHead is designed to be instantiated " + "using the `TFAutoModelWithLMHead.from_pretrained(pretrained_model_name_or_path)` method.") + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): + r""" Instantiates one of the language modeling model classes of the library + from a pre-trained model configuration. + + The `from_pretrained()` method takes care of returning the correct model class instance + using pattern matching on the `pretrained_model_name_or_path` string. + + The model class to instantiate is selected as the first pattern matching + in the `pretrained_model_name_or_path` string (in the following order): + - contains `distilbert`: DistilBertForMaskedLM (DistilBERT model) + - contains `roberta`: RobertaForMaskedLM (RoBERTa model) + - contains `bert`: TFBertForMaskedLM (Bert model) + - contains `openai-gpt`: OpenAIGPTLMHeadModel (OpenAI GPT model) + - contains `gpt2`: GPT2LMHeadModel (OpenAI GPT-2 model) + - contains `transfo-xl`: TransfoXLLMHeadModel (Transformer-XL model) + - contains `xlnet`: XLNetLMHeadModel (XLNet model) + - contains `xlm`: XLMWithLMHeadModel (XLM model) + + The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated) + To train the model, you should first set it back in training mode with `model.train()` + + Params: + pretrained_model_name_or_path: either: + + - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. + - a path to a `directory` containing model weights saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. + - a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. + + model_args: (`optional`) Sequence of positional arguments: + All remaning positional arguments will be passed to the underlying model's ``__init__`` method + + config: (`optional`) instance of a class derived from :class:`~pytorch_transformers.PretrainedConfig`: + Configuration for the model to use instead of an automatically loaded configuation. Configuration can be automatically loaded when: + + - the model is a model provided by the library (loaded with the ``shortcut-name`` string of a pretrained model), or + - the model was saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained` and is reloaded by suppling the save directory. + - the model is loaded by suppling a local directory as ``pretrained_model_name_or_path`` and a configuration JSON file named `config.json` is found in the directory. + + state_dict: (`optional`) dict: + an optional state dictionnary for the model to use instead of a state dictionary loaded from saved weights file. + This option can be used if you want to create a model from a pretrained configuration but load your own weights. + In this case though, you should check if using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained` and :func:`~pytorch_transformers.PreTrainedModel.from_pretrained` is not a simpler option. + + cache_dir: (`optional`) string: + Path to a directory in which a downloaded pre-trained model + configuration should be cached if the standard cache should not be used. + + force_download: (`optional`) boolean, default False: + Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + + proxies: (`optional`) dict, default None: + A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. + The proxies are used on each request. + + output_loading_info: (`optional`) boolean: + Set to ``True`` to also return a dictionnary containing missing keys, unexpected keys and error messages. + + kwargs: (`optional`) Remaining dictionary of keyword arguments: + Can be used to update the configuration object (after it being loaded) and initiate the model. (e.g. ``output_attention=True``). Behave differently depending on whether a `config` is provided or automatically loaded: + + - If a configuration is provided with ``config``, ``**kwargs`` will be directly passed to the underlying model's ``__init__`` method (we assume all relevant updates to the configuration have already been done) + - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~pytorch_transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function. + + Examples:: + + model = TFAutoModelWithLMHead.from_pretrained('bert-base-uncased') # Download model and configuration from S3 and cache. + model = TFAutoModelWithLMHead.from_pretrained('./test/bert_model/') # E.g. model was saved using `save_pretrained('./test/saved_model/')` + model = TFAutoModelWithLMHead.from_pretrained('bert-base-uncased', output_attention=True) # Update configuration during loading + assert model.config.output_attention == True + # Loading from a TF checkpoint file instead of a PyTorch model (slower) + config = AutoConfig.from_json_file('./tf_model/bert_tf_model_config.json') + model = TFAutoModelWithLMHead.from_pretrained('./tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config) + + """ + if 'distilbert' in pretrained_model_name_or_path: + raise NotImplementedError + elif 'roberta' in pretrained_model_name_or_path: + raise NotImplementedError + elif 'bert' in pretrained_model_name_or_path: + return TFBertForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'openai-gpt' in pretrained_model_name_or_path: + raise NotImplementedError + elif 'gpt2' in pretrained_model_name_or_path: + raise NotImplementedError + elif 'transfo-xl' in pretrained_model_name_or_path: + raise NotImplementedError + elif 'xlnet' in pretrained_model_name_or_path: + raise NotImplementedError + elif 'xlm' in pretrained_model_name_or_path: + raise NotImplementedError + + raise ValueError("Unrecognized model identifier in {}. Should contains one of " + "'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', " + "'xlm', 'roberta'".format(pretrained_model_name_or_path)) + + +class TFAutoModelForSequenceClassification(object): + r""" + :class:`~pytorch_transformers.TFAutoModelForSequenceClassification` is a generic model class + that will be instantiated as one of the sequence classification model classes of the library + when created with the `TFAutoModelForSequenceClassification.from_pretrained(pretrained_model_name_or_path)` + class method. + + The `from_pretrained()` method takes care of returning the correct model class instance + using pattern matching on the `pretrained_model_name_or_path` string. + + The model class to instantiate is selected as the first pattern matching + in the `pretrained_model_name_or_path` string (in the following order): + - contains `distilbert`: DistilBertForSequenceClassification (DistilBERT model) + - contains `roberta`: RobertaForSequenceClassification (RoBERTa model) + - contains `bert`: TFBertForSequenceClassification (Bert model) + - contains `xlnet`: XLNetForSequenceClassification (XLNet model) + - contains `xlm`: XLMForSequenceClassification (XLM model) + + This class cannot be instantiated using `__init__()` (throws an error). + """ + def __init__(self): + raise EnvironmentError("TFAutoModelWithLMHead is designed to be instantiated " + "using the `TFAutoModelWithLMHead.from_pretrained(pretrained_model_name_or_path)` method.") + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): + r""" Instantiates one of the sequence classification model classes of the library + from a pre-trained model configuration. + + The `from_pretrained()` method takes care of returning the correct model class instance + using pattern matching on the `pretrained_model_name_or_path` string. + + The model class to instantiate is selected as the first pattern matching + in the `pretrained_model_name_or_path` string (in the following order): + - contains `distilbert`: DistilBertForSequenceClassification (DistilBERT model) + - contains `roberta`: RobertaForSequenceClassification (RoBERTa model) + - contains `bert`: TFBertForSequenceClassification (Bert model) + - contains `xlnet`: XLNetForSequenceClassification (XLNet model) + - contains `xlm`: XLMForSequenceClassification (XLM model) + + The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated) + To train the model, you should first set it back in training mode with `model.train()` + + Params: + pretrained_model_name_or_path: either: + + - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. + - a path to a `directory` containing model weights saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. + - a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. + + model_args: (`optional`) Sequence of positional arguments: + All remaning positional arguments will be passed to the underlying model's ``__init__`` method + + config: (`optional`) instance of a class derived from :class:`~pytorch_transformers.PretrainedConfig`: + Configuration for the model to use instead of an automatically loaded configuation. Configuration can be automatically loaded when: + + - the model is a model provided by the library (loaded with the ``shortcut-name`` string of a pretrained model), or + - the model was saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained` and is reloaded by suppling the save directory. + - the model is loaded by suppling a local directory as ``pretrained_model_name_or_path`` and a configuration JSON file named `config.json` is found in the directory. + + state_dict: (`optional`) dict: + an optional state dictionnary for the model to use instead of a state dictionary loaded from saved weights file. + This option can be used if you want to create a model from a pretrained configuration but load your own weights. + In this case though, you should check if using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained` and :func:`~pytorch_transformers.PreTrainedModel.from_pretrained` is not a simpler option. + + cache_dir: (`optional`) string: + Path to a directory in which a downloaded pre-trained model + configuration should be cached if the standard cache should not be used. + + force_download: (`optional`) boolean, default False: + Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + + proxies: (`optional`) dict, default None: + A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. + The proxies are used on each request. + + output_loading_info: (`optional`) boolean: + Set to ``True`` to also return a dictionnary containing missing keys, unexpected keys and error messages. + + kwargs: (`optional`) Remaining dictionary of keyword arguments: + Can be used to update the configuration object (after it being loaded) and initiate the model. (e.g. ``output_attention=True``). Behave differently depending on whether a `config` is provided or automatically loaded: + + - If a configuration is provided with ``config``, ``**kwargs`` will be directly passed to the underlying model's ``__init__`` method (we assume all relevant updates to the configuration have already been done) + - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~pytorch_transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function. + + Examples:: + + model = TFAutoModelForSequenceClassification.from_pretrained('bert-base-uncased') # Download model and configuration from S3 and cache. + model = TFAutoModelForSequenceClassification.from_pretrained('./test/bert_model/') # E.g. model was saved using `save_pretrained('./test/saved_model/')` + model = TFAutoModelForSequenceClassification.from_pretrained('bert-base-uncased', output_attention=True) # Update configuration during loading + assert model.config.output_attention == True + # Loading from a TF checkpoint file instead of a PyTorch model (slower) + config = AutoConfig.from_json_file('./tf_model/bert_tf_model_config.json') + model = TFAutoModelForSequenceClassification.from_pretrained('./tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config) + + """ + if 'distilbert' in pretrained_model_name_or_path: + raise NotImplementedError + elif 'roberta' in pretrained_model_name_or_path: + raise NotImplementedError + elif 'bert' in pretrained_model_name_or_path: + return TFBertForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'xlnet' in pretrained_model_name_or_path: + raise NotImplementedError + elif 'xlm' in pretrained_model_name_or_path: + raise NotImplementedError + + raise ValueError("Unrecognized model identifier in {}. Should contains one of " + "'bert', 'xlnet', 'xlm', 'roberta'".format(pretrained_model_name_or_path)) + + +class TFAutoModelForQuestionAnswering(object): + r""" + :class:`~pytorch_transformers.TFAutoModelForQuestionAnswering` is a generic model class + that will be instantiated as one of the question answering model classes of the library + when created with the `TFAutoModelForQuestionAnswering.from_pretrained(pretrained_model_name_or_path)` + class method. + + The `from_pretrained()` method takes care of returning the correct model class instance + using pattern matching on the `pretrained_model_name_or_path` string. + + The model class to instantiate is selected as the first pattern matching + in the `pretrained_model_name_or_path` string (in the following order): + - contains `distilbert`: DistilBertForQuestionAnswering (DistilBERT model) + - contains `bert`: TFBertForQuestionAnswering (Bert model) + - contains `xlnet`: XLNetForQuestionAnswering (XLNet model) + - contains `xlm`: XLMForQuestionAnswering (XLM model) + + This class cannot be instantiated using `__init__()` (throws an error). + """ + def __init__(self): + raise EnvironmentError("TFAutoModelWithLMHead is designed to be instantiated " + "using the `TFAutoModelWithLMHead.from_pretrained(pretrained_model_name_or_path)` method.") + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): + r""" Instantiates one of the question answering model classes of the library + from a pre-trained model configuration. + + The `from_pretrained()` method takes care of returning the correct model class instance + using pattern matching on the `pretrained_model_name_or_path` string. + + The model class to instantiate is selected as the first pattern matching + in the `pretrained_model_name_or_path` string (in the following order): + - contains `distilbert`: DistilBertForQuestionAnswering (DistilBERT model) + - contains `bert`: TFBertForQuestionAnswering (Bert model) + - contains `xlnet`: XLNetForQuestionAnswering (XLNet model) + - contains `xlm`: XLMForQuestionAnswering (XLM model) + + The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated) + To train the model, you should first set it back in training mode with `model.train()` + + Params: + pretrained_model_name_or_path: either: + + - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. + - a path to a `directory` containing model weights saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. + - a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. + + model_args: (`optional`) Sequence of positional arguments: + All remaning positional arguments will be passed to the underlying model's ``__init__`` method + + config: (`optional`) instance of a class derived from :class:`~pytorch_transformers.PretrainedConfig`: + Configuration for the model to use instead of an automatically loaded configuation. Configuration can be automatically loaded when: + + - the model is a model provided by the library (loaded with the ``shortcut-name`` string of a pretrained model), or + - the model was saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained` and is reloaded by suppling the save directory. + - the model is loaded by suppling a local directory as ``pretrained_model_name_or_path`` and a configuration JSON file named `config.json` is found in the directory. + + state_dict: (`optional`) dict: + an optional state dictionnary for the model to use instead of a state dictionary loaded from saved weights file. + This option can be used if you want to create a model from a pretrained configuration but load your own weights. + In this case though, you should check if using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained` and :func:`~pytorch_transformers.PreTrainedModel.from_pretrained` is not a simpler option. + + cache_dir: (`optional`) string: + Path to a directory in which a downloaded pre-trained model + configuration should be cached if the standard cache should not be used. + + force_download: (`optional`) boolean, default False: + Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + + proxies: (`optional`) dict, default None: + A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. + The proxies are used on each request. + + output_loading_info: (`optional`) boolean: + Set to ``True`` to also return a dictionnary containing missing keys, unexpected keys and error messages. + + kwargs: (`optional`) Remaining dictionary of keyword arguments: + Can be used to update the configuration object (after it being loaded) and initiate the model. (e.g. ``output_attention=True``). Behave differently depending on whether a `config` is provided or automatically loaded: + + - If a configuration is provided with ``config``, ``**kwargs`` will be directly passed to the underlying model's ``__init__`` method (we assume all relevant updates to the configuration have already been done) + - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~pytorch_transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function. + + Examples:: + + model = TFAutoModelForQuestionAnswering.from_pretrained('bert-base-uncased') # Download model and configuration from S3 and cache. + model = TFAutoModelForQuestionAnswering.from_pretrained('./test/bert_model/') # E.g. model was saved using `save_pretrained('./test/saved_model/')` + model = TFAutoModelForQuestionAnswering.from_pretrained('bert-base-uncased', output_attention=True) # Update configuration during loading + assert model.config.output_attention == True + # Loading from a TF checkpoint file instead of a PyTorch model (slower) + config = AutoConfig.from_json_file('./tf_model/bert_tf_model_config.json') + model = TFAutoModelForQuestionAnswering.from_pretrained('./tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config) + + """ + if 'distilbert' in pretrained_model_name_or_path: + raise NotImplementedError + elif 'bert' in pretrained_model_name_or_path: + return TFBertForQuestionAnswering.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'xlnet' in pretrained_model_name_or_path: + raise NotImplementedError + elif 'xlm' in pretrained_model_name_or_path: + raise NotImplementedError + + raise ValueError("Unrecognized model identifier in {}. Should contains one of " + "'bert', 'xlnet', 'xlm'".format(pretrained_model_name_or_path)) diff --git a/pytorch_transformers/modeling_tf_utils.py b/pytorch_transformers/modeling_tf_utils.py index cb1588e399..ef2375c60a 100644 --- a/pytorch_transformers/modeling_tf_utils.py +++ b/pytorch_transformers/modeling_tf_utils.py @@ -170,9 +170,6 @@ class TFPreTrainedModel(tf.keras.Model): A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. - output_loading_info: (`optional`) boolean: - Set to ``True`` to also return a dictionnary containing missing keys, unexpected keys and error messages. - kwargs: (`optional`) Remaining dictionary of keyword arguments: Can be used to update the configuration object (after it being loaded) and initiate the model. (e.g. ``output_attention=True``). Behave differently depending on whether a `config` is provided or automatically loaded: @@ -195,7 +192,6 @@ class TFPreTrainedModel(tf.keras.Model): from_pt = kwargs.pop('from_pt', False) force_download = kwargs.pop('force_download', False) proxies = kwargs.pop('proxies', None) - output_loading_info = kwargs.pop('output_loading_info', False) # Load config if config is None: @@ -258,11 +254,4 @@ class TFPreTrainedModel(tf.keras.Model): ret = model(inputs, training=False) # Make sure restore ops are run - # if hasattr(model, 'tie_weights'): - # model.tie_weights() # TODO make sure word embedding weights are still tied - - if output_loading_info: - loading_info = {"missing_keys": missing_keys, "unexpected_keys": unexpected_keys, "error_msgs": error_msgs} - return model, loading_info - return model diff --git a/pytorch_transformers/tests/modeling_tf_auto_test.py b/pytorch_transformers/tests/modeling_tf_auto_test.py new file mode 100644 index 0000000000..816d6c1b1a --- /dev/null +++ b/pytorch_transformers/tests/modeling_tf_auto_test.py @@ -0,0 +1,85 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import unittest +import shutil +import pytest +import logging + +try: + from pytorch_transformers import (AutoConfig, BertConfig, + TFAutoModel, TFBertModel, + TFAutoModelWithLMHead, TFBertForMaskedLM, + TFAutoModelForSequenceClassification, TFBertForSequenceClassification, + TFAutoModelForQuestionAnswering, TFBertForQuestionAnswering) + from pytorch_transformers.modeling_tf_bert import TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP + + from .modeling_common_test import (CommonTestCases, ids_tensor) + from .configuration_common_test import ConfigTester +except ImportError: + pytestmark = pytest.mark.skip("Require TensorFlow") + + +class TFAutoModelTest(unittest.TestCase): + def test_model_from_pretrained(self): + logging.basicConfig(level=logging.INFO) + for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: + config = AutoConfig.from_pretrained(model_name) + self.assertIsNotNone(config) + self.assertIsInstance(config, BertConfig) + + model = TFAutoModel.from_pretrained(model_name) + self.assertIsNotNone(model) + self.assertIsInstance(model, TFBertModel) + + def test_lmhead_model_from_pretrained(self): + logging.basicConfig(level=logging.INFO) + for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: + config = AutoConfig.from_pretrained(model_name) + self.assertIsNotNone(config) + self.assertIsInstance(config, BertConfig) + + model = TFAutoModelWithLMHead.from_pretrained(model_name) + self.assertIsNotNone(model) + self.assertIsInstance(model, TFBertForMaskedLM) + + def test_sequence_classification_model_from_pretrained(self): + logging.basicConfig(level=logging.INFO) + for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: + config = AutoConfig.from_pretrained(model_name) + self.assertIsNotNone(config) + self.assertIsInstance(config, BertConfig) + + model = TFAutoModelForSequenceClassification.from_pretrained(model_name) + self.assertIsNotNone(model) + self.assertIsInstance(model, TFBertForSequenceClassification) + + def test_question_answering_model_from_pretrained(self): + logging.basicConfig(level=logging.INFO) + for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: + config = AutoConfig.from_pretrained(model_name) + self.assertIsNotNone(config) + self.assertIsInstance(config, BertConfig) + + model = TFAutoModelForQuestionAnswering.from_pretrained(model_name) + self.assertIsNotNone(model) + self.assertIsInstance(model, TFBertForQuestionAnswering) + + +if __name__ == "__main__": + unittest.main() From 64d83c7ae0283781767fcd1e89fee22379e56acb Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 5 Sep 2019 21:21:49 +0200 Subject: [PATCH 029/219] WIP --- pytorch_transformers/modeling_tf_gpt2.py | 650 ++++++++++++++++++++++ pytorch_transformers/modeling_tf_utils.py | 18 + 2 files changed, 668 insertions(+) create mode 100644 pytorch_transformers/modeling_tf_gpt2.py diff --git a/pytorch_transformers/modeling_tf_gpt2.py b/pytorch_transformers/modeling_tf_gpt2.py new file mode 100644 index 0000000000..08900495ea --- /dev/null +++ b/pytorch_transformers/modeling_tf_gpt2.py @@ -0,0 +1,650 @@ +# coding=utf-8 +# Copyright 2018 The OpenAI Team Authors and HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" TF 2.0 OpenAI GPT-2 model. """ + +from __future__ import absolute_import, division, print_function, unicode_literals + +import collections +import json +import logging +import math +import os +import sys +from io import open + +import numpy as np +import tensorflow as tf + +from .modeling_tf_utils import TFPreTrainedModel +from .configuration_gpt2 import GPT2Config +from .file_utils import add_start_docstrings + +logger = logging.getLogger(__name__) + +GPT2_PRETRAINED_MODEL_ARCHIVE_MAP = {"gpt2": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-tf_model.h5", + "gpt2-medium": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-medium-tf_model.h5", + "gpt2-large": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-large-tf_model.h5"} + + +def load_gpt2_pt_weights_in_tf(tf_model, config, pytorch_checkpoint_path): + """ Load pytorch checkpoints in a TF 2.0 model and save it using HDF5 format + We use HDF5 to easily do transfer learning + (see https://github.com/tensorflow/tensorflow/blob/ee16fcac960ae660e0e4496658a366e2f745e1f0/tensorflow/python/keras/engine/network.py#L1352-L1357). + """ + try: + import re + import torch + import numpy + from tensorflow.python.keras import backend as K + except ImportError: + logger.error("Loading a PyTorch model in TensorFlow, requires PyTorch to be installed. Please see " + "https://pytorch.org/ for installation instructions.") + raise + + pt_path = os.path.abspath(pytorch_checkpoint_path) + logger.info("Loading PyTorch weights from {}".format(pt_path)) + # Load pytorch model + state_dict = torch.load(pt_path, map_location='cpu') + + inputs_list = [[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]] + tf_inputs = tf.constant(inputs_list) + tfo = tf_model(tf_inputs, training=False) # build the network + + symbolic_weights = tf_model.trainable_weights + tf_model.non_trainable_weights + weight_value_tuples = [] + for symbolic_weight in symbolic_weights: + name = symbolic_weight.name + name = name.replace('cls_mlm', 'cls') # We had to split this layer in two in the TF model to be + name = name.replace('cls_nsp', 'cls') # able to do transfer learning (Keras only allow to remove full layers) + name = name.replace(':0', '') + name = name.replace('layer_', 'layer/') + name = name.split('/') + name = name[1:] + + transpose = bool(name[-1] == 'kernel') + if name[-1] == 'kernel' or name[-1] == 'embeddings': + name[-1] = 'weight' + + name = '.'.join(name) + assert name in state_dict + array = state_dict[name].numpy() + + if transpose: + array = numpy.transpose(array) + + try: + assert list(symbolic_weight.shape) == list(array.shape) + except AssertionError as e: + e.args += (symbolic_weight.shape, array.shape) + raise e + + logger.info("Initialize TF weight {}".format(symbolic_weight.name)) + + weight_value_tuples.append((symbolic_weight, array)) + + K.batch_set_value(weight_value_tuples) + + tfo = tf_model(tf_inputs, training=False) # Make sure restore ops are run + return tf_model + + +def gelu(x): + """Gaussian Error Linear Unit. + This is a smoother version of the RELU. + Original paper: https://arxiv.org/abs/1606.08415 + Args: + x: float Tensor to perform activation. + Returns: + `x` with the GELU activation applied. + """ + cdf = 0.5 * (1.0 + tf.tanh( + (np.sqrt(2 / np.pi) * (x + 0.044715 * tf.pow(x, 3))))) + return x * cdf + + +class TFAttention(tf.keras.layers.Layer): + def __init__(self, nx, n_ctx, config, scale=False): + super(Attention, self).__init__() + self.output_attentions = config.output_attentions + + n_state = nx # in Attention: n_state=768 (nx=n_embd) + # [switch nx => n_state from Block to Attention to keep identical to TF implem] + assert n_state % config.n_head == 0 + self.register_buffer("bias", torch.tril(torch.ones(n_ctx, n_ctx)).view(1, 1, n_ctx, n_ctx)) + self.n_head = config.n_head + self.split_size = n_state + self.scale = scale + + self.c_attn = Conv1D(n_state * 3, nx) + self.c_proj = Conv1D(n_state, nx) + self.attn_dropout = nn.Dropout(config.attn_pdrop) + self.resid_dropout = nn.Dropout(config.resid_pdrop) + self.pruned_heads = set() + + def prune_heads(self, heads): + if len(heads) == 0: + return + mask = torch.ones(self.n_head, self.split_size // self.n_head) + heads = set(heads) - self.pruned_heads # Convert to set and emove already pruned heads + for head in heads: + # Compute how many pruned heads are before the head and move the index accordingly + head = head - sum(1 if h < head else 0 for h in self.pruned_heads) + mask[head] = 0 + mask = mask.view(-1).contiguous().eq(1) + index = torch.arange(len(mask))[mask].long() + index_attn = torch.cat([index, index + self.split_size, index + (2*self.split_size)]) + + # Prune conv1d layers + self.c_attn = prune_conv1d_layer(self.c_attn, index_attn, dim=1) + self.c_proj = prune_conv1d_layer(self.c_proj, index, dim=0) + + # Update hyper params + self.split_size = (self.split_size // self.n_head) * (self.n_head - len(heads)) + self.n_head = self.n_head - len(heads) + self.pruned_heads = self.pruned_heads.union(heads) + + def _attn(self, q, k, v, head_mask=None): + w = torch.matmul(q, k) + if self.scale: + w = w / math.sqrt(v.size(-1)) + nd, ns = w.size(-2), w.size(-1) + b = self.bias[:, :, ns-nd:ns, :ns] + w = w * b - 1e4 * (1 - b) + + w = nn.Softmax(dim=-1)(w) + w = self.attn_dropout(w) + + # Mask heads if we want to + if head_mask is not None: + w = w * head_mask + + outputs = [torch.matmul(w, v)] + if self.output_attentions: + outputs.append(w) + return outputs + + def merge_heads(self, x): + x = x.permute(0, 2, 1, 3).contiguous() + new_x_shape = x.size()[:-2] + (x.size(-2) * x.size(-1),) + return x.view(*new_x_shape) # in Tensorflow implem: fct merge_states + + def split_heads(self, x, k=False): + new_x_shape = x.size()[:-1] + (self.n_head, x.size(-1) // self.n_head) + x = x.view(*new_x_shape) # in Tensorflow implem: fct split_states + if k: + return x.permute(0, 2, 3, 1) # (batch, head, head_features, seq_length) + else: + return x.permute(0, 2, 1, 3) # (batch, head, seq_length, head_features) + + def forward(self, x, layer_past=None, head_mask=None): + x = self.c_attn(x) + query, key, value = x.split(self.split_size, dim=2) + query = self.split_heads(query) + key = self.split_heads(key, k=True) + value = self.split_heads(value) + if layer_past is not None: + past_key, past_value = layer_past[0].transpose(-2, -1), layer_past[1] # transpose back cf below + key = torch.cat((past_key, key), dim=-1) + value = torch.cat((past_value, value), dim=-2) + present = torch.stack((key.transpose(-2, -1), value)) # transpose to have same shapes for stacking + + attn_outputs = self._attn(query, key, value, head_mask) + a = attn_outputs[0] + + a = self.merge_heads(a) + a = self.c_proj(a) + a = self.resid_dropout(a) + + outputs = [a, present] + attn_outputs[1:] + return outputs # a, present, (attentions) + + +class MLP(nn.Module): + def __init__(self, n_state, config): # in MLP: n_state=3072 (4 * n_embd) + super(MLP, self).__init__() + nx = config.n_embd + self.c_fc = Conv1D(n_state, nx) + self.c_proj = Conv1D(nx, n_state) + self.act = gelu + self.dropout = nn.Dropout(config.resid_pdrop) + + def forward(self, x): + h = self.act(self.c_fc(x)) + h2 = self.c_proj(h) + return self.dropout(h2) + + +class Block(nn.Module): + def __init__(self, n_ctx, config, scale=False): + super(Block, self).__init__() + nx = config.n_embd + self.ln_1 = nn.LayerNorm(nx, eps=config.layer_norm_epsilon) + self.attn = Attention(nx, n_ctx, config, scale) + self.ln_2 = nn.LayerNorm(nx, eps=config.layer_norm_epsilon) + self.mlp = MLP(4 * nx, config) + + def forward(self, x, layer_past=None, head_mask=None): + output_attn = self.attn(self.ln_1(x), layer_past=layer_past, head_mask=head_mask) + a = output_attn[0] # output_attn: a, present, (attentions) + + x = x + a + m = self.mlp(self.ln_2(x)) + x = x + m + + outputs = [x] + output_attn[1:] + return outputs # x, present, (attentions) + + +class GPT2PreTrainedModel(PreTrainedModel): + """ An abstract class to handle weights initialization and + a simple interface for dowloading and loading pretrained models. + """ + config_class = GPT2Config + pretrained_model_archive_map = GPT2_PRETRAINED_MODEL_ARCHIVE_MAP + load_tf_weights = load_tf_weights_in_gpt2 + base_model_prefix = "transformer" + + def __init__(self, *inputs, **kwargs): + super(GPT2PreTrainedModel, self).__init__(*inputs, **kwargs) + + def _init_weights(self, module): + """ Initialize the weights. + """ + if isinstance(module, (nn.Linear, nn.Embedding, Conv1D)): + # Slightly different from the TF version which uses truncated_normal for initialization + # cf https://github.com/pytorch/pytorch/pull/5617 + module.weight.data.normal_(mean=0.0, std=self.config.initializer_range) + if isinstance(module, (nn.Linear, Conv1D)) and module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.LayerNorm): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + + +GPT2_START_DOCSTRING = r""" OpenAI GPT-2 model was proposed in + `Language Models are Unsupervised Multitask Learners`_ + by Alec Radford*, Jeffrey Wu*, Rewon Child, David Luan, Dario Amodei** and Ilya Sutskever**. + It's a causal (unidirectional) transformer pre-trained using language modeling on a very large + corpus of ~40 GB of text data. + + This model is a PyTorch `torch.nn.Module`_ sub-class. Use it as a regular PyTorch Module and + refer to the PyTorch documentation for all matter related to general usage and behavior. + + .. _`Language Models are Unsupervised Multitask Learners`: + https://openai.com/blog/better-language-models/ + + .. _`torch.nn.Module`: + https://pytorch.org/docs/stable/nn.html#module + + Parameters: + config (:class:`~pytorch_transformers.GPT2Config`): Model configuration class with all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the configuration. + Check out the :meth:`~pytorch_transformers.PreTrainedModel.from_pretrained` method to load the model weights. +""" + +GPT2_INPUTS_DOCSTRING = r""" Inputs: + **input_ids**: ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Indices of input sequence tokens in the vocabulary. + GPT-2 is a model with absolute position embeddings so it's usually advised to pad the inputs on + the right rather than the left. + Indices can be obtained using :class:`pytorch_transformers.BPT2Tokenizer`. + See :func:`pytorch_transformers.PreTrainedTokenizer.encode` and + :func:`pytorch_transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + **position_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Indices of positions of each input sequence tokens in the position embeddings. + Selected in the range ``[0, config.max_position_embeddings - 1]``. + **token_type_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + A parallel sequence of tokens (can be used to indicate various portions of the inputs). + The embeddings from these tokens will be summed with the respective token embeddings. + Indices are selected in the vocabulary (unlike BERT which has a specific vocabulary for segment indices). + **past**: + list of ``torch.FloatTensor`` (one for each layer): + that contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model + (see `past` output below). Can be used to speed up sequential decoding. + **head_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: + Mask to nullify selected heads of the self-attention modules. + Mask values selected in ``[0, 1]``: + ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. +""" + +@add_start_docstrings("The bare GPT2 Model transformer outputing raw hidden-states without any specific head on top.", + GPT2_START_DOCSTRING, GPT2_INPUTS_DOCSTRING) +class GPT2Model(GPT2PreTrainedModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)`` + Sequence of hidden-states at the last layer of the model. + **past**: + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + that contains pre-computed hidden-states (key and values in the attention blocks). + Can be used (see `past` input) to speed up sequential decoding. + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = GPT2Tokenizer.from_pretrained('gpt2') + model = GPT2Model.from_pretrained('gpt2') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids) + last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple + + """ + def __init__(self, config): + super(GPT2Model, self).__init__(config) + self.output_hidden_states = config.output_hidden_states + self.output_attentions = config.output_attentions + + self.wte = nn.Embedding(config.vocab_size, config.n_embd) + self.wpe = nn.Embedding(config.n_positions, config.n_embd) + self.drop = nn.Dropout(config.embd_pdrop) + self.h = nn.ModuleList([Block(config.n_ctx, config, scale=True) for _ in range(config.n_layer)]) + self.ln_f = nn.LayerNorm(config.n_embd, eps=config.layer_norm_epsilon) + + self.init_weights() + + def _resize_token_embeddings(self, new_num_tokens): + self.wte = self._get_resized_embeddings(self.wte, new_num_tokens) + return self.wte + + def _prune_heads(self, heads_to_prune): + """ Prunes heads of the model. + heads_to_prune: dict of {layer_num: list of heads to prune in this layer} + """ + for layer, heads in heads_to_prune.items(): + self.h[layer].attn.prune_heads(heads) + + def forward(self, input_ids, position_ids=None, token_type_ids=None, past=None, head_mask=None): + if past is None: + past_length = 0 + past = [None] * len(self.h) + else: + past_length = past[0][0].size(-2) + if position_ids is None: + position_ids = torch.arange(past_length, input_ids.size(-1) + past_length, dtype=torch.long, device=input_ids.device) + position_ids = position_ids.unsqueeze(0).expand_as(input_ids) + + # Prepare head mask if needed + # 1.0 in head_mask indicate we keep the head + # attention_probs has shape bsz x n_heads x N x N + # head_mask has shape n_layer x batch x n_heads x N x N + if head_mask is not None: + if head_mask.dim() == 1: + head_mask = head_mask.unsqueeze(0).unsqueeze(0).unsqueeze(-1).unsqueeze(-1) + head_mask = head_mask.expand(self.config.n_layer, -1, -1, -1, -1) + elif head_mask.dim() == 2: + head_mask = head_mask.unsqueeze(1).unsqueeze(-1).unsqueeze(-1) # We can specify head_mask for each layer + head_mask = head_mask.to(dtype=next(self.parameters()).dtype) # switch to fload if need + fp16 compatibility + else: + head_mask = [None] * self.config.n_layer + + input_shape = input_ids.size() + input_ids = input_ids.view(-1, input_ids.size(-1)) + position_ids = position_ids.view(-1, position_ids.size(-1)) + + inputs_embeds = self.wte(input_ids) + position_embeds = self.wpe(position_ids) + if token_type_ids is not None: + token_type_ids = token_type_ids.view(-1, token_type_ids.size(-1)) + token_type_embeds = self.wte(token_type_ids) + else: + token_type_embeds = 0 + hidden_states = inputs_embeds + position_embeds + token_type_embeds + hidden_states = self.drop(hidden_states) + + output_shape = input_shape + (hidden_states.size(-1),) + + presents = () + all_attentions = [] + all_hidden_states = () + for i, (block, layer_past) in enumerate(zip(self.h, past)): + if self.output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states.view(*output_shape),) + + outputs = block(hidden_states, layer_past, head_mask[i]) + hidden_states, present = outputs[:2] + presents = presents + (present,) + + if self.output_attentions: + all_attentions.append(outputs[2]) + + hidden_states = self.ln_f(hidden_states) + + hidden_states = hidden_states.view(*output_shape) + # Add last hidden state + if self.output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + outputs = (hidden_states, presents) + if self.output_hidden_states: + outputs = outputs + (all_hidden_states,) + if self.output_attentions: + # let the number of heads free (-1) so we can extract attention even after head pruning + attention_output_shape = input_shape[:-1] + (-1,) + all_attentions[0].shape[-2:] + all_attentions = tuple(t.view(*attention_output_shape) for t in all_attentions) + outputs = outputs + (all_attentions,) + return outputs # last hidden state, presents, (all hidden_states), (attentions) + + +@add_start_docstrings("""The GPT2 Model transformer with a language modeling head on top +(linear layer with weights tied to the input embeddings). """, GPT2_START_DOCSTRING, GPT2_INPUTS_DOCSTRING) +class GPT2LMHeadModel(GPT2PreTrainedModel): + r""" + **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Labels for language modeling. + Note that the labels **are shifted** inside the model, i.e. you can set ``lm_labels = input_ids`` + Indices are selected in ``[-1, 0, ..., config.vocab_size]`` + All labels set to ``-1`` are ignored (masked), the loss is only + computed for labels in ``[0, ..., config.vocab_size]`` + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Language modeling loss. + **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + **past**: + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + that contains pre-computed hidden-states (key and values in the attention blocks). + Can be used (see `past` input) to speed up sequential decoding. + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + import torch + from pytorch_transformers import GPT2Tokenizer, GPT2LMHeadModel + + tokenizer = GPT2Tokenizer.from_pretrained('gpt2') + model = GPT2LMHeadModel.from_pretrained('gpt2') + + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, labels=input_ids) + loss, logits = outputs[:2] + + """ + def __init__(self, config): + super(GPT2LMHeadModel, self).__init__(config) + self.transformer = GPT2Model(config) + self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False) + + self.init_weights() + self.tie_weights() + + def tie_weights(self): + """ Make sure we are sharing the input and output embeddings. + Export to TorchScript can't handle parameter sharing so we are cloning them instead. + """ + self._tie_or_clone_weights(self.lm_head, + self.transformer.wte) + + def forward(self, input_ids, position_ids=None, token_type_ids=None, labels=None, past=None, head_mask=None): + transformer_outputs = self.transformer(input_ids, position_ids=position_ids, token_type_ids=token_type_ids, + past=past, head_mask=head_mask) + hidden_states = transformer_outputs[0] + + lm_logits = self.lm_head(hidden_states) + + outputs = (lm_logits,) + transformer_outputs[1:] + if labels is not None: + # Shift so that tokens < n predict n + shift_logits = lm_logits[..., :-1, :].contiguous() + shift_labels = labels[..., 1:].contiguous() + # Flatten the tokens + loss_fct = CrossEntropyLoss(ignore_index=-1) + loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), + shift_labels.view(-1)) + outputs = (loss,) + outputs + + return outputs # (loss), lm_logits, presents, (all hidden_states), (attentions) + + +@add_start_docstrings("""The GPT2 Model transformer with a language modeling and a multiple-choice classification +head on top e.g. for RocStories/SWAG tasks. The two heads are two linear layers. +The language modeling head has its weights tied to the input embeddings, +the classification head takes as input the input of a specified classification token index in the input sequence). +""", GPT2_START_DOCSTRING) +class GPT2DoubleHeadsModel(GPT2PreTrainedModel): + r""" Inputs: + **input_ids**: ``torch.LongTensor`` of shape ``(batch_size, num_choices, sequence_length)``: + Indices of input sequence tokens in the vocabulary. + The second dimension of the input (`num_choices`) indicates the number of choices to score. + Indices can be obtained using :class:`pytorch_transformers.BPT2Tokenizer`. + See :func:`pytorch_transformers.PreTrainedTokenizer.encode` and + :func:`pytorch_transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + **mc_token_ids**: ``torch.LongTensor`` of shape ``(batch_size, num_choices)``: + Index of the classification token in each input sequence. + Selected in the range ``[0, input_ids.size(-1) - 1[``. + **position_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, num_choices, sequence_length)``: + Indices of positions of each input sequence tokens in the position embeddings. + Selected in the range ``[0, config.max_position_embeddings - 1]``. + **token_type_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, num_choices, sequence_length)``: + A parallel sequence of tokens (can be used to indicate various portions of the inputs). + The embeddings from these tokens will be summed with the respective token embeddings. + Indices are selected in the vocabulary (unlike BERT which has a specific vocabulary for segment indices). + **past**: + list of ``torch.FloatTensor`` (one for each layer): + that contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model + (see `past` output below). Can be used to speed up sequential decoding. + **head_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: + Mask to nullify selected heads of the self-attention modules. + Mask values selected in ``[0, 1]``: + ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. + **lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Labels for language modeling. + Note that the labels **are shifted** inside the model, i.e. you can set ``lm_labels = input_ids`` + Indices are selected in ``[-1, 0, ..., config.vocab_size]`` + All labels set to ``-1`` are ignored (masked), the loss is only + computed for labels in ``[0, ..., config.vocab_size]`` + **mc_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size)``: + Labels for computing the multiple choice classification loss. + Indices should be in ``[0, ..., num_choices]`` where `num_choices` is the size of the second dimension + of the input tensors. (see `input_ids` above) + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **lm_loss**: (`optional`, returned when ``lm_labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Language modeling loss. + **mc_loss**: (`optional`, returned when ``multiple_choice_labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Multiple choice classification loss. + **lm_prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, num_choices, sequence_length, config.vocab_size)`` + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + **mc_prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, num_choices)`` + Prediction scores of the multiplechoice classification head (scores for each choice before SoftMax). + **past**: + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + that contains pre-computed hidden-states (key and values in the attention blocks). + Can be used (see `past` input) to speed up sequential decoding. + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + import torch + from pytorch_transformers import GPT2Tokenizer, GPT2DoubleHeadsModel + + tokenizer = GPT2Tokenizer.from_pretrained('gpt2') + model = GPT2DoubleHeadsModel.from_pretrained('gpt2') + + # Add a [CLS] to the vocabulary (we should train it also!) + tokenizer.add_special_tokens({'cls_token': '[CLS]'}) + model.resize_token_embeddings(len(tokenizer)) # Update the model embeddings with the new vocabulary size + print(tokenizer.cls_token_id, len(tokenizer)) # The newly token the last token of the vocabulary + + choices = ["Hello, my dog is cute [CLS]", "Hello, my cat is cute [CLS]"] + encoded_choices = [tokenizer.encode(s) for s in choices] + cls_token_location = [tokens.index(tokenizer.cls_token_id) for tokens in encoded_choices] + + input_ids = torch.tensor(encoded_choices).unsqueeze(0) # Batch size: 1, number of choices: 2 + mc_token_ids = torch.tensor([cls_token_location]) # Batch size: 1 + + outputs = model(input_ids, mc_token_ids=mc_token_ids) + lm_prediction_scores, mc_prediction_scores = outputs[:2] + + """ + def __init__(self, config): + super(GPT2DoubleHeadsModel, self).__init__(config) + self.transformer = GPT2Model(config) + self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False) + self.multiple_choice_head = SequenceSummary(config) + + self.init_weights() + self.tie_weights() + + def tie_weights(self): + """ Make sure we are sharing the input and output embeddings. + Export to TorchScript can't handle parameter sharing so we are cloning them instead. + """ + self._tie_or_clone_weights(self.lm_head, + self.transformer.wte) + + def forward(self, input_ids, mc_token_ids=None, lm_labels=None, mc_labels=None, token_type_ids=None, + position_ids=None, past=None, head_mask=None): + transformer_outputs = self.transformer(input_ids, position_ids=position_ids, token_type_ids=token_type_ids, + past=past, head_mask=head_mask) + hidden_states = transformer_outputs[0] + + lm_logits = self.lm_head(hidden_states) + mc_logits = self.multiple_choice_head(hidden_states, mc_token_ids).squeeze(-1) + + outputs = (lm_logits, mc_logits) + transformer_outputs[1:] + if mc_labels is not None: + loss_fct = CrossEntropyLoss() + loss = loss_fct(mc_logits.view(-1, mc_logits.size(-1)), + mc_labels.view(-1)) + outputs = (loss,) + outputs + if lm_labels is not None: + shift_logits = lm_logits[..., :-1, :].contiguous() + shift_labels = lm_labels[..., 1:].contiguous() + loss_fct = CrossEntropyLoss(ignore_index=-1) + loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), + shift_labels.view(-1)) + outputs = (loss,) + outputs + + return outputs # (lm loss), (mc loss), lm logits, mc logits, presents, (all hidden_states), (attentions) diff --git a/pytorch_transformers/modeling_tf_utils.py b/pytorch_transformers/modeling_tf_utils.py index ef2375c60a..4f0fe17ae7 100644 --- a/pytorch_transformers/modeling_tf_utils.py +++ b/pytorch_transformers/modeling_tf_utils.py @@ -255,3 +255,21 @@ class TFPreTrainedModel(tf.keras.Model): ret = model(inputs, training=False) # Make sure restore ops are run return model + +class TFConv1D(tf.keras.layers.Layer): + def __init__(self, nf, nx): + """ TFConv1D layer as defined by Radford et al. for OpenAI GPT (and also used in GPT-2) + Basically works like a Linear layer but the weights are transposed + """ + super(TFConv1D, self).__init__() + self.nf = nf + w = torch.empty(nx, nf) + nn.init.normal_(w, std=0.02) + self.weight = nn.Parameter(w) + self.bias = nn.Parameter(torch.zeros(nf)) + + def call(self, x): + size_out = t.shape(x)[:-1] + (self.nf,) + x = tf.addmm(self.bias, x.view(-1, x.size(-1)), self.weight) + x = x.view(*size_out) + return x From ad88563bda7066d639e5f63473cf363ff14dde01 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Fri, 6 Sep 2019 23:38:51 +0300 Subject: [PATCH 030/219] WIP GPT-2 --- pytorch_transformers/modeling_tf_gpt2.py | 119 +++++++++++----------- pytorch_transformers/modeling_tf_utils.py | 26 +++-- 2 files changed, 80 insertions(+), 65 deletions(-) diff --git a/pytorch_transformers/modeling_tf_gpt2.py b/pytorch_transformers/modeling_tf_gpt2.py index 08900495ea..03fd7c6e9f 100644 --- a/pytorch_transformers/modeling_tf_gpt2.py +++ b/pytorch_transformers/modeling_tf_gpt2.py @@ -28,7 +28,7 @@ from io import open import numpy as np import tensorflow as tf -from .modeling_tf_utils import TFPreTrainedModel +from .modeling_tf_utils import TFPreTrainedModel, TFConv1D from .configuration_gpt2 import GPT2Config from .file_utils import add_start_docstrings @@ -116,97 +116,100 @@ def gelu(x): class TFAttention(tf.keras.layers.Layer): - def __init__(self, nx, n_ctx, config, scale=False): - super(Attention, self).__init__() + def __init__(self, nx, n_ctx, config, scale=False, **kwargs): + super(TFAttention, self).__init__(**kwargs) self.output_attentions = config.output_attentions n_state = nx # in Attention: n_state=768 (nx=n_embd) # [switch nx => n_state from Block to Attention to keep identical to TF implem] assert n_state % config.n_head == 0 - self.register_buffer("bias", torch.tril(torch.ones(n_ctx, n_ctx)).view(1, 1, n_ctx, n_ctx)) + self.n_ctx = n_ctx self.n_head = config.n_head self.split_size = n_state self.scale = scale - self.c_attn = Conv1D(n_state * 3, nx) - self.c_proj = Conv1D(n_state, nx) - self.attn_dropout = nn.Dropout(config.attn_pdrop) - self.resid_dropout = nn.Dropout(config.resid_pdrop) + self.c_attn = TFConv1D(n_state * 3, nx) + self.c_proj = TFConv1D(n_state, nx) + self.attn_dropout = tf.keras.layers.Dropout(config.attn_pdrop) + self.resid_dropout = tf.keras.layers.Dropout(config.resid_pdrop) self.pruned_heads = set() def prune_heads(self, heads): - if len(heads) == 0: - return - mask = torch.ones(self.n_head, self.split_size // self.n_head) - heads = set(heads) - self.pruned_heads # Convert to set and emove already pruned heads - for head in heads: - # Compute how many pruned heads are before the head and move the index accordingly - head = head - sum(1 if h < head else 0 for h in self.pruned_heads) - mask[head] = 0 - mask = mask.view(-1).contiguous().eq(1) - index = torch.arange(len(mask))[mask].long() - index_attn = torch.cat([index, index + self.split_size, index + (2*self.split_size)]) + pass - # Prune conv1d layers - self.c_attn = prune_conv1d_layer(self.c_attn, index_attn, dim=1) - self.c_proj = prune_conv1d_layer(self.c_proj, index, dim=0) + @staticmethod + @tf.function + def attention_mask(nd, ns, *, dtype): + """1's in the lower triangle, counting from the lower right corner. + Same as tf.matrix_band_part(tf.ones([nd, ns]), -1, ns-nd), but doesn't produce garbage on TPUs. + """ + i = tf.range(nd)[:,None] + j = tf.range(ns) + m = i >= j - ns + nd + return tf.cast(m, dtype) - # Update hyper params - self.split_size = (self.split_size // self.n_head) * (self.n_head - len(heads)) - self.n_head = self.n_head - len(heads) - self.pruned_heads = self.pruned_heads.union(heads) - - def _attn(self, q, k, v, head_mask=None): - w = torch.matmul(q, k) + @tf.function + def _attn(self, inputs, training=False): + q, k, v, head_mask = inputs + # q, k, v have shape [batch, heads, sequence, features] + w = tf.matmul(q, k, transpose_b=True) if self.scale: - w = w / math.sqrt(v.size(-1)) - nd, ns = w.size(-2), w.size(-1) - b = self.bias[:, :, ns-nd:ns, :ns] + n_state = shape_list(v)[-1] + w = w * tf.rsqrt(tf.cast(v.shape[-1].value, w.dtype)) + + # w has shape [batch, heads, dst_sequence, src_sequence], where information flows from src to dst. + _, _, nd, ns = shape_list(w) + b = self.attention_mask(nd, ns, dtype=w.dtype) + b = tf.reshape(b, [1, 1, nd, ns]) w = w * b - 1e4 * (1 - b) - w = nn.Softmax(dim=-1)(w) - w = self.attn_dropout(w) + w = tf.nn.softmax(w) + w = self.attn_dropout(w, training=training) # Mask heads if we want to if head_mask is not None: w = w * head_mask - outputs = [torch.matmul(w, v)] + outputs = [tf.matmul(w, v)] if self.output_attentions: outputs.append(w) return outputs + @tf.function def merge_heads(self, x): - x = x.permute(0, 2, 1, 3).contiguous() - new_x_shape = x.size()[:-2] + (x.size(-2) * x.size(-1),) - return x.view(*new_x_shape) # in Tensorflow implem: fct merge_states + x = tf.transpose(x, [0, 2, 1, 3]) + x_shape = tf.shape(x) + new_x_shape = x_shape[:-2] + (x_shape[-2] * x_shape[-1],) + return tf.reshape(x, new_x_shape) - def split_heads(self, x, k=False): - new_x_shape = x.size()[:-1] + (self.n_head, x.size(-1) // self.n_head) - x = x.view(*new_x_shape) # in Tensorflow implem: fct split_states - if k: - return x.permute(0, 2, 3, 1) # (batch, head, head_features, seq_length) - else: - return x.permute(0, 2, 1, 3) # (batch, head, seq_length, head_features) + @tf.function + def split_heads(self, x): + x_shape = tf.shape(x) + new_x_shape = x_shape[:-1] + (self.n_head, x_shape[-1] // self.n_head) + x = tf.reshape(x, new_x_shape) + return tf.transpose(x, (0, 2, 1, 3)) # (batch, head, seq_length, head_features) + + @tf.function + def call(self, inputs, training=False): + x, layer_past, head_mask = inputs - def forward(self, x, layer_past=None, head_mask=None): x = self.c_attn(x) - query, key, value = x.split(self.split_size, dim=2) + query, key, value = tf.split(x, 3, axis=2) query = self.split_heads(query) - key = self.split_heads(key, k=True) + key = self.split_heads(key) value = self.split_heads(value) if layer_past is not None: - past_key, past_value = layer_past[0].transpose(-2, -1), layer_past[1] # transpose back cf below - key = torch.cat((past_key, key), dim=-1) - value = torch.cat((past_value, value), dim=-2) - present = torch.stack((key.transpose(-2, -1), value)) # transpose to have same shapes for stacking + past_key, past_value = tf.unstack(layer_past, axis=1) + key = tf.concat([past_key, key], axis=-2) + value = tf.concat([past_value, value], axis=-2) + present = tf.stack([key, value], axis=1) attn_outputs = self._attn(query, key, value, head_mask) a = attn_outputs[0] a = self.merge_heads(a) a = self.c_proj(a) - a = self.resid_dropout(a) + a = self.resid_dropout(a, training=training) outputs = [a, present] + attn_outputs[1:] return outputs # a, present, (attentions) @@ -216,8 +219,8 @@ class MLP(nn.Module): def __init__(self, n_state, config): # in MLP: n_state=3072 (4 * n_embd) super(MLP, self).__init__() nx = config.n_embd - self.c_fc = Conv1D(n_state, nx) - self.c_proj = Conv1D(nx, n_state) + self.c_fc = TFConv1D(n_state, nx) + self.c_proj = TFConv1D(nx, n_state) self.act = gelu self.dropout = nn.Dropout(config.resid_pdrop) @@ -227,9 +230,9 @@ class MLP(nn.Module): return self.dropout(h2) -class Block(nn.Module): - def __init__(self, n_ctx, config, scale=False): - super(Block, self).__init__() +class TFBlock(tf.keras.layers.Layer): + def __init__(self, n_ctx, config, scale=False, **kwargs): + super(TFBlock, self).__init__(**kwargs) nx = config.n_embd self.ln_1 = nn.LayerNorm(nx, eps=config.layer_norm_epsilon) self.attn = Attention(nx, n_ctx, config, scale) diff --git a/pytorch_transformers/modeling_tf_utils.py b/pytorch_transformers/modeling_tf_utils.py index 4f0fe17ae7..bd5fb73f0b 100644 --- a/pytorch_transformers/modeling_tf_utils.py +++ b/pytorch_transformers/modeling_tf_utils.py @@ -263,13 +263,25 @@ class TFConv1D(tf.keras.layers.Layer): """ super(TFConv1D, self).__init__() self.nf = nf - w = torch.empty(nx, nf) - nn.init.normal_(w, std=0.02) - self.weight = nn.Parameter(w) - self.bias = nn.Parameter(torch.zeros(nf)) + self.nx = nx + def build(self, input_shape): + self.weight = self.add_weight( + "weight", + shape=[self.nx, self.nf], + initializer=tf.random_normal_initializer( + mean=0., stddev=0.02)) + self.bias = self.add_weight( + "bias", + shape=[self.nx, self.nf], + initializer=tf.zeros_initializer()) + + @tf.function def call(self, x): - size_out = t.shape(x)[:-1] + (self.nf,) - x = tf.addmm(self.bias, x.view(-1, x.size(-1)), self.weight) - x = x.view(*size_out) + size_out = tf.shape(x)[:-1] + (self.nf,) + + x = tf.reshape(x, [-1, tf.shape(x)[-1]]) + x = tf.matmul(x, self.weight) + self.bias + x = tf.reshape(x, size_out) + return x From 34f28b2a1342fd72c2e4d4e5613855bfb9f35d34 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Sun, 8 Sep 2019 15:01:12 +0300 Subject: [PATCH 031/219] WIP GPT2 --- pytorch_transformers/modeling_tf_bert.py | 34 +-- pytorch_transformers/modeling_tf_gpt2.py | 328 ++++++++++++++++++---- pytorch_transformers/modeling_tf_utils.py | 6 +- 3 files changed, 292 insertions(+), 76 deletions(-) diff --git a/pytorch_transformers/modeling_tf_bert.py b/pytorch_transformers/modeling_tf_bert.py index fb73515f6e..d1cc67923f 100644 --- a/pytorch_transformers/modeling_tf_bert.py +++ b/pytorch_transformers/modeling_tf_bert.py @@ -684,13 +684,13 @@ class TFBertModel(TFBertPreTrainedModel): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = TFBertModel.from_pretrained('bert-base-uncased') - input_ids = tf.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple """ - def __init__(self, config): - super(TFBertModel, self).__init__(config) + def __init__(self, config, *inputs, **kwargs): + super(TFBertModel, self).__init__(config, *inputs, **kwargs) self.bert = TFBertMainLayer(config, name='bert') @tf.function @@ -739,8 +739,8 @@ class TFBertForPreTraining(TFBertPreTrainedModel): prediction_scores, seq_relationship_scores = outputs[:2] """ - def __init__(self, config): - super(TFBertForPreTraining, self).__init__(config) + def __init__(self, config, *inputs, **kwargs): + super(TFBertForPreTraining, self).__init__(config, *inputs, **kwargs) self.bert = TFBertMainLayer(config, name='bert') self.cls_nsp = TFBertNSPHead(config, name='cls_nsp') @@ -790,8 +790,8 @@ class TFBertForMaskedLM(TFBertPreTrainedModel): loss, prediction_scores = outputs[:2] """ - def __init__(self, config): - super(TFBertForMaskedLM, self).__init__(config) + def __init__(self, config, *inputs, **kwargs): + super(TFBertForMaskedLM, self).__init__(config, *inputs, **kwargs) self.bert = TFBertMainLayer(config, name='bert') @@ -839,8 +839,8 @@ class TFBertForNextSentencePrediction(TFBertPreTrainedModel): seq_relationship_scores = outputs[0] """ - def __init__(self, config): - super(TFBertForNextSentencePrediction, self).__init__(config) + def __init__(self, config, *inputs, **kwargs): + super(TFBertForNextSentencePrediction, self).__init__(config, *inputs, **kwargs) self.bert = TFBertMainLayer(config, name='bert') self.cls_nsp = TFBertNSPHead(config, name='cls_nsp') @@ -891,8 +891,8 @@ class TFBertForSequenceClassification(TFBertPreTrainedModel): loss, logits = outputs[:2] """ - def __init__(self, config): - super(TFBertForSequenceClassification, self).__init__(config) + def __init__(self, config, *inputs, **kwargs): + super(TFBertForSequenceClassification, self).__init__(config, *inputs, **kwargs) self.num_labels = config.num_labels self.bert = TFBertMainLayer(config, name='bert') @@ -984,8 +984,8 @@ class TFBertForMultipleChoice(TFBertPreTrainedModel): loss, classification_scores = outputs[:2] """ - def __init__(self, config): - super(TFBertForMultipleChoice, self).__init__(config) + def __init__(self, config, *inputs, **kwargs): + super(TFBertForMultipleChoice, self).__init__(config, *inputs, **kwargs) self.bert = TFBertMainLayer(config, name='bert') self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) @@ -1066,8 +1066,8 @@ class TFBertForTokenClassification(TFBertPreTrainedModel): loss, scores = outputs[:2] """ - def __init__(self, config): - super(TFBertForTokenClassification, self).__init__(config) + def __init__(self, config, *inputs, **kwargs): + super(TFBertForTokenClassification, self).__init__(config, *inputs, **kwargs) self.num_labels = config.num_labels self.bert = TFBertMainLayer(config, name='bert') @@ -1128,8 +1128,8 @@ class TFBertForQuestionAnswering(TFBertPreTrainedModel): loss, start_scores, end_scores = outputs[:2] """ - def __init__(self, config): - super(TFBertForQuestionAnswering, self).__init__(config) + def __init__(self, config, *inputs, **kwargs): + super(TFBertForQuestionAnswering, self).__init__(config, *inputs, **kwargs) self.num_labels = config.num_labels self.bert = TFBertMainLayer(config, name='bert') diff --git a/pytorch_transformers/modeling_tf_gpt2.py b/pytorch_transformers/modeling_tf_gpt2.py index 03fd7c6e9f..dc185d6e44 100644 --- a/pytorch_transformers/modeling_tf_gpt2.py +++ b/pytorch_transformers/modeling_tf_gpt2.py @@ -128,8 +128,8 @@ class TFAttention(tf.keras.layers.Layer): self.split_size = n_state self.scale = scale - self.c_attn = TFConv1D(n_state * 3, nx) - self.c_proj = TFConv1D(n_state, nx) + self.c_attn = TFConv1D(n_state * 3, nx, name='c_attn') + self.c_proj = TFConv1D(n_state, nx, name='c_proj') self.attn_dropout = tf.keras.layers.Dropout(config.attn_pdrop) self.resid_dropout = tf.keras.layers.Dropout(config.resid_pdrop) self.pruned_heads = set() @@ -139,7 +139,7 @@ class TFAttention(tf.keras.layers.Layer): @staticmethod @tf.function - def attention_mask(nd, ns, *, dtype): + def attention_mask(nd, ns, dtype): """1's in the lower triangle, counting from the lower right corner. Same as tf.matrix_band_part(tf.ones([nd, ns]), -1, ns-nd), but doesn't produce garbage on TPUs. """ @@ -164,7 +164,8 @@ class TFAttention(tf.keras.layers.Layer): w = w * b - 1e4 * (1 - b) w = tf.nn.softmax(w) - w = self.attn_dropout(w, training=training) + if training: + w = self.attn_dropout(w) # Mask heads if we want to if head_mask is not None: @@ -204,54 +205,238 @@ class TFAttention(tf.keras.layers.Layer): value = tf.concat([past_value, value], axis=-2) present = tf.stack([key, value], axis=1) - attn_outputs = self._attn(query, key, value, head_mask) + attn_outputs = self._attn(query, key, value, head_mask, training=training) a = attn_outputs[0] a = self.merge_heads(a) a = self.c_proj(a) - a = self.resid_dropout(a, training=training) + if training: + a = self.resid_dropout(a) outputs = [a, present] + attn_outputs[1:] return outputs # a, present, (attentions) -class MLP(nn.Module): - def __init__(self, n_state, config): # in MLP: n_state=3072 (4 * n_embd) - super(MLP, self).__init__() +class TFMLP(nn.Module): + def __init__(self, n_state, config, **kwargs): + super(TFMLP, self).__init__(**kwargs) nx = config.n_embd - self.c_fc = TFConv1D(n_state, nx) - self.c_proj = TFConv1D(nx, n_state) + self.c_fc = TFConv1D(n_state, nx, name='c_fc') + self.c_proj = TFConv1D(nx, n_state, name='c_proj') self.act = gelu - self.dropout = nn.Dropout(config.resid_pdrop) + self.dropout = tf.keras.layers.Dropout(config.resid_pdrop) - def forward(self, x): + @tf.function + def call(self, x, training=False): h = self.act(self.c_fc(x)) h2 = self.c_proj(h) - return self.dropout(h2) + if training: + h2 = self.dropout(h2) + return h2 class TFBlock(tf.keras.layers.Layer): def __init__(self, n_ctx, config, scale=False, **kwargs): super(TFBlock, self).__init__(**kwargs) nx = config.n_embd - self.ln_1 = nn.LayerNorm(nx, eps=config.layer_norm_epsilon) - self.attn = Attention(nx, n_ctx, config, scale) - self.ln_2 = nn.LayerNorm(nx, eps=config.layer_norm_epsilon) - self.mlp = MLP(4 * nx, config) + self.ln_1 = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_epsilon, name='ln_1') + self.attn = TFAttention(nx, n_ctx, config, scale, name='attn') + self.ln_2 = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_epsilon, name='ln_2') + self.mlp = TFMLP(4 * nx, config, name='mlp') - def forward(self, x, layer_past=None, head_mask=None): - output_attn = self.attn(self.ln_1(x), layer_past=layer_past, head_mask=head_mask) + @tf.function + def call(self, x, layer_past=None, head_mask=None, training=False): + output_attn = self.attn(self.ln_1(x), + layer_past=layer_past, + head_mask=head_mask, + training=training) a = output_attn[0] # output_attn: a, present, (attentions) x = x + a - m = self.mlp(self.ln_2(x)) + m = self.mlp(self.ln_2(x), training=training) x = x + m outputs = [x] + output_attn[1:] return outputs # x, present, (attentions) +class TFGPT2Embeddings(tf.keras.layers.Layer): + """Construct the embeddings from word, position and token_type embeddings. + """ + def __init__(self, config, **kwargs): + super(TFGPT2Embeddings, self).__init__(**kwargs) + self.vocab_size = config.vocab_size + self.hidden_size = config.hidden_size -class GPT2PreTrainedModel(PreTrainedModel): + def build(self, input_shape): + """Build shared word embedding layer + Shared weights logic adapted from + https://github.com/tensorflow/models/blob/a009f4fb9d2fc4949e32192a944688925ef78659/official/transformer/v2/embedding_layer.py#L24 + """ + self.weight = self.add_weight( + "weight", + shape=[self.vocab_size, self.n_embed], + initializer=tf.random_normal_initializer( + mean=0., stddev=self.n_embed**-0.5)) + super(TFBertEmbeddings, self).build(input_shape) + + @tf.function + def call(self, inputs, mode="embedding", training=False): + """Get token embeddings of inputs. + Args: + inputs: list of three int64 tensors with shape [batch_size, length]: (input_ids, position_ids, token_type_ids) + mode: string, a valid value is one of "embedding" and "linear". + Returns: + outputs: (1) If mode == "embedding", output embedding tensor, float32 with + shape [batch_size, length, embedding_size]; (2) mode == "linear", output + linear tensor, float32 with shape [batch_size, length, vocab_size]. + Raises: + ValueError: if mode is not valid. + + Shared weights logic adapted from + https://github.com/tensorflow/models/blob/a009f4fb9d2fc4949e32192a944688925ef78659/official/transformer/v2/embedding_layer.py#L24 + """ + if mode == "embedding": + return self._embedding(inputs, training=training) + elif mode == "linear": + return self._linear(inputs) + else: + raise ValueError("mode {} is not valid.".format(mode)) + + def _embedding(self, input_ids): + """Applies embedding based on inputs tensor.""" + return tf.gather(self.weight, input_ids) + + def _linear(self, inputs): + """Computes logits by running inputs through a linear layer. + Args: + inputs: A float32 tensor with shape [batch_size, length, hidden_size] + Returns: + float32 tensor with shape [batch_size, length, vocab_size]. + """ + batch_size = tf.shape(inputs)[0] + length = tf.shape(inputs)[1] + + x = tf.reshape(inputs, [-1, self.n_embed]) + logits = tf.matmul(x, self.weight, transpose_b=True) + + return tf.reshape(logits, [batch_size, length, self.vocab_size]) + +class TFGPT2MainLayer(tf.keras.layers.Layer): + def __init__(self, config, *inputs, **kwargs): + super(TFGPT2MainLayer, self).__init__(config, *inputs, **kwargs) + self.output_hidden_states = config.output_hidden_states + self.output_attentions = config.output_attentions + self.vocab_size = config.vocab_size + self.n_embd = config.n_embd + + self.wte = TFGPT2Embeddings(config, name='wte') + self.wpe = tf.keras.layers.Embedding(config.n_positions, config.n_embd, name='wpe') + self.drop = tf.keras.layers.Dropout(config.embd_pdrop) + self.h = [TFBlock(config.n_ctx, config, scale=Truename='h_{}'.format(i)) for i in range(config.n_layer)] + self.ln_f = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_epsilon, name='ln_f') + + def _resize_token_embeddings(self, new_num_tokens): + raise NotImplementedError + + def _prune_heads(self, heads_to_prune): + """ Prunes heads of the model. + heads_to_prune: dict of {layer_num: list of heads to prune in this layer} + """ + raise NotImplementedError + + @tf.function + def call(self, inputs, training=False): + input_ids, position_ids=None, token_type_ids=None, past=None, head_mask=None): + + if not isinstance(inputs, (dict, tuple, list)): + input_ids = inputs + attention_mask, head_mask, position_ids, token_type_ids = None, None, None, None + elif isinstance(inputs, (tuple, list)): + input_ids = inputs[0] + attention_mask = inputs[1] if len(inputs) > 1 else None + token_type_ids = inputs[2] if len(inputs) > 2 else None + position_ids = inputs[3] if len(inputs) > 3 else None + head_mask = inputs[4] if len(inputs) > 4 else None + assert len(inputs) <= 5, "Too many inputs." + else: + input_ids = inputs.get('input_ids') + attention_mask = inputs.get('attention_mask', None) + token_type_ids = inputs.get('token_type_ids', None) + position_ids = inputs.get('position_ids', None) + head_mask = inputs.get('head_mask', None) + assert len(inputs) <= 5, "Too many inputs." + + if past is None: + past_length = 0 + past = [None] * len(self.h) + else: + past_length = past[0][0].size(-2) + if position_ids is None: + position_ids = torch.arange(past_length, input_ids.size(-1) + past_length, dtype=torch.long, device=input_ids.device) + position_ids = position_ids.unsqueeze(0).expand_as(input_ids) + + # Prepare head mask if needed + # 1.0 in head_mask indicate we keep the head + # attention_probs has shape bsz x n_heads x N x N + # head_mask has shape n_layer x batch x n_heads x N x N + if head_mask is not None: + if head_mask.dim() == 1: + head_mask = head_mask.unsqueeze(0).unsqueeze(0).unsqueeze(-1).unsqueeze(-1) + head_mask = head_mask.expand(self.config.n_layer, -1, -1, -1, -1) + elif head_mask.dim() == 2: + head_mask = head_mask.unsqueeze(1).unsqueeze(-1).unsqueeze(-1) # We can specify head_mask for each layer + head_mask = head_mask.to(dtype=next(self.parameters()).dtype) # switch to fload if need + fp16 compatibility + else: + head_mask = [None] * self.config.n_layer + + input_shape = input_ids.size() + input_ids = input_ids.view(-1, input_ids.size(-1)) + position_ids = position_ids.view(-1, position_ids.size(-1)) + + inputs_embeds = self.wte(input_ids) + position_embeds = self.wpe(position_ids) + if token_type_ids is not None: + token_type_ids = token_type_ids.view(-1, token_type_ids.size(-1)) + token_type_embeds = self.wte(token_type_ids) + else: + token_type_embeds = 0 + hidden_states = inputs_embeds + position_embeds + token_type_embeds + hidden_states = self.drop(hidden_states) + + output_shape = input_shape + (hidden_states.size(-1),) + + presents = () + all_attentions = [] + all_hidden_states = () + for i, (block, layer_past) in enumerate(zip(self.h, past)): + if self.output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states.view(*output_shape),) + + outputs = block(hidden_states, layer_past, head_mask[i]) + hidden_states, present = outputs[:2] + presents = presents + (present,) + + if self.output_attentions: + all_attentions.append(outputs[2]) + + hidden_states = self.ln_f(hidden_states) + + hidden_states = hidden_states.view(*output_shape) + # Add last hidden state + if self.output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + outputs = (hidden_states, presents) + if self.output_hidden_states: + outputs = outputs + (all_hidden_states,) + if self.output_attentions: + # let the number of heads free (-1) so we can extract attention even after head pruning + attention_output_shape = input_shape[:-1] + (-1,) + all_attentions[0].shape[-2:] + all_attentions = tuple(t.view(*attention_output_shape) for t in all_attentions) + outputs = outputs + (all_attentions,) + return outputs # last hidden state, presents, (all hidden_states), (attentions) + +class TFGPT2PreTrainedModel(TFPreTrainedModel): """ An abstract class to handle weights initialization and a simple interface for dowloading and loading pretrained models. """ @@ -260,22 +445,6 @@ class GPT2PreTrainedModel(PreTrainedModel): load_tf_weights = load_tf_weights_in_gpt2 base_model_prefix = "transformer" - def __init__(self, *inputs, **kwargs): - super(GPT2PreTrainedModel, self).__init__(*inputs, **kwargs) - - def _init_weights(self, module): - """ Initialize the weights. - """ - if isinstance(module, (nn.Linear, nn.Embedding, Conv1D)): - # Slightly different from the TF version which uses truncated_normal for initialization - # cf https://github.com/pytorch/pytorch/pull/5617 - module.weight.data.normal_(mean=0.0, std=self.config.initializer_range) - if isinstance(module, (nn.Linear, Conv1D)) and module.bias is not None: - module.bias.data.zero_() - elif isinstance(module, nn.LayerNorm): - module.bias.data.zero_() - module.weight.data.fill_(1.0) - GPT2_START_DOCSTRING = r""" OpenAI GPT-2 model was proposed in `Language Models are Unsupervised Multitask Learners`_ @@ -283,14 +452,26 @@ GPT2_START_DOCSTRING = r""" OpenAI GPT-2 model was proposed in It's a causal (unidirectional) transformer pre-trained using language modeling on a very large corpus of ~40 GB of text data. - This model is a PyTorch `torch.nn.Module`_ sub-class. Use it as a regular PyTorch Module and - refer to the PyTorch documentation for all matter related to general usage and behavior. + This model is a tf.keras.Model `tf.keras.Model`_ sub-class. Use it as a regular TF 2.0 Keras Model and + refer to the TF 2.0 documentation for all matter related to general usage and behavior. .. _`Language Models are Unsupervised Multitask Learners`: https://openai.com/blog/better-language-models/ - .. _`torch.nn.Module`: - https://pytorch.org/docs/stable/nn.html#module + .. _`tf.keras.Model`: + https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/Model + + Important note on the model inputs: + The inputs of the TF 2.0 models are slightly different from the PyTorch ones since + TF 2.0 Keras doesn't accept named arguments with defaults values for input Tensor. + More precisely, input Tensors are gathered in the first arguments of the model call function: `model(inputs)`. + There are three possibilities to gather and feed the inputs to the model: + + - a single Tensor with input_ids only and nothing else: `model(inputs_ids) + - a list of varying length with one or several input Tensors IN THE ORDER given in the docstring: + `model([input_ids, attention_mask])` or `model([input_ids, attention_mask, token_type_ids])` + - a dictionary with one or several input Tensors associaed to the input names given in the docstring: + `model({'input_ids': input_ids, 'token_type_ids': token_type_ids})` Parameters: config (:class:`~pytorch_transformers.GPT2Config`): Model configuration class with all the parameters of the model. @@ -325,7 +506,7 @@ GPT2_INPUTS_DOCSTRING = r""" Inputs: @add_start_docstrings("The bare GPT2 Model transformer outputing raw hidden-states without any specific head on top.", GPT2_START_DOCSTRING, GPT2_INPUTS_DOCSTRING) -class GPT2Model(GPT2PreTrainedModel): +class TFGPT2Model(TFGPT2PreTrainedModel): r""" Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)`` @@ -345,37 +526,72 @@ class GPT2Model(GPT2PreTrainedModel): Examples:: tokenizer = GPT2Tokenizer.from_pretrained('gpt2') - model = GPT2Model.from_pretrained('gpt2') + model = TFGPT2Model.from_pretrained('gpt2') input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple """ - def __init__(self, config): - super(GPT2Model, self).__init__(config) + def __init__(self, config, *inputs, **kwargs): + super(TFGPT2Model, self).__init__(config, *inputs, **kwargs) self.output_hidden_states = config.output_hidden_states self.output_attentions = config.output_attentions + self.vocab_size = config.vocab_size + self.n_embd = config.n_embd - self.wte = nn.Embedding(config.vocab_size, config.n_embd) - self.wpe = nn.Embedding(config.n_positions, config.n_embd) - self.drop = nn.Dropout(config.embd_pdrop) - self.h = nn.ModuleList([Block(config.n_ctx, config, scale=True) for _ in range(config.n_layer)]) - self.ln_f = nn.LayerNorm(config.n_embd, eps=config.layer_norm_epsilon) + self.wpe = tf.keras.layers.Embedding(config.n_positions, config.n_embd, name='wpe') + self.drop = tf.keras.layers.Dropout(config.embd_pdrop) + self.h = [TFBlock(config.n_ctx, config, scale=Truename='h_{}'.format(i)) for i in range(config.n_layer)] + self.ln_f = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_epsilon, name='ln_f') self.init_weights() + def build(self, input_shape): + """Build shared word embedding layer + Shared weights logic adapted from + https://github.com/tensorflow/models/blob/a009f4fb9d2fc4949e32192a944688925ef78659/official/transformer/v2/embedding_layer.py#L24 + """ + with tf.name_scope("wte"): + # Create and initialize weights. The random normal initializer was chosen + # arbitrarily, and works well. + self.wte = self.add_weight( + "weight", + shape=[self.vocab_size, self.n_embed], + initializer=tf.random_normal_initializer( + mean=0., stddev=self.n_embed**-0.5)) + super(TFGPT2Model, self).build(input_shape) + def _resize_token_embeddings(self, new_num_tokens): - self.wte = self._get_resized_embeddings(self.wte, new_num_tokens) - return self.wte + raise NotImplementedError def _prune_heads(self, heads_to_prune): """ Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} """ - for layer, heads in heads_to_prune.items(): - self.h[layer].attn.prune_heads(heads) + raise NotImplementedError + + @tf.function + def call(self, inputs, training=False): + input_ids, position_ids=None, token_type_ids=None, past=None, head_mask=None): + + if not isinstance(inputs, (dict, tuple, list)): + input_ids = inputs + attention_mask, head_mask, position_ids, token_type_ids = None, None, None, None + elif isinstance(inputs, (tuple, list)): + input_ids = inputs[0] + attention_mask = inputs[1] if len(inputs) > 1 else None + token_type_ids = inputs[2] if len(inputs) > 2 else None + position_ids = inputs[3] if len(inputs) > 3 else None + head_mask = inputs[4] if len(inputs) > 4 else None + assert len(inputs) <= 5, "Too many inputs." + else: + input_ids = inputs.get('input_ids') + attention_mask = inputs.get('attention_mask', None) + token_type_ids = inputs.get('token_type_ids', None) + position_ids = inputs.get('position_ids', None) + head_mask = inputs.get('head_mask', None) + assert len(inputs) <= 5, "Too many inputs." - def forward(self, input_ids, position_ids=None, token_type_ids=None, past=None, head_mask=None): if past is None: past_length = 0 past = [None] * len(self.h) diff --git a/pytorch_transformers/modeling_tf_utils.py b/pytorch_transformers/modeling_tf_utils.py index bd5fb73f0b..bea4efaa0e 100644 --- a/pytorch_transformers/modeling_tf_utils.py +++ b/pytorch_transformers/modeling_tf_utils.py @@ -52,7 +52,7 @@ class TFPreTrainedModel(tf.keras.Model): base_model_prefix = "" def __init__(self, config, *inputs, **kwargs): - super(TFPreTrainedModel, self).__init__() + super(TFPreTrainedModel, self).__init__(*inputs, **kwargs) if not isinstance(config, PretrainedConfig): raise ValueError( "Parameter config in `{}(config)` should be an instance of class `PretrainedConfig`. " @@ -257,11 +257,11 @@ class TFPreTrainedModel(tf.keras.Model): return model class TFConv1D(tf.keras.layers.Layer): - def __init__(self, nf, nx): + def __init__(self, nf, nx, *inputs, **kwargs): """ TFConv1D layer as defined by Radford et al. for OpenAI GPT (and also used in GPT-2) Basically works like a Linear layer but the weights are transposed """ - super(TFConv1D, self).__init__() + super(TFConv1D, self).__init__(*inputs, **kwargs) self.nf = nf self.nx = nx From 728637356c7ff9ab969f107d3c67790b4400bac9 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Mon, 9 Sep 2019 10:18:55 +0300 Subject: [PATCH 032/219] WIP GPT2 --- pytorch_transformers/modeling_tf_bert.py | 119 +---- pytorch_transformers/modeling_tf_gpt2.py | 468 ++++++------------ pytorch_transformers/modeling_tf_utils.py | 110 +++- .../tests/modeling_gpt2_test.py | 16 +- .../tests/modeling_tf_gpt2_test.py | 216 ++++++++ 5 files changed, 505 insertions(+), 424 deletions(-) create mode 100644 pytorch_transformers/tests/modeling_tf_gpt2_test.py diff --git a/pytorch_transformers/modeling_tf_bert.py b/pytorch_transformers/modeling_tf_bert.py index d1cc67923f..a48c2d8660 100644 --- a/pytorch_transformers/modeling_tf_bert.py +++ b/pytorch_transformers/modeling_tf_bert.py @@ -704,20 +704,7 @@ class TFBertModel(TFBertPreTrainedModel): BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) class TFBertForPreTraining(TFBertPreTrainedModel): r""" - **masked_lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: - Labels for computing the masked language modeling loss. - Indices should be in ``[-1, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring) - Tokens with indices set to ``-1`` are ignored (masked), the loss is only computed for the tokens with labels - in ``[0, ..., config.vocab_size]`` - **next_sentence_label**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: - Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair (see ``input_ids`` docstring) - Indices should be in ``[0, 1]``. - ``0`` indicates sequence B is a continuation of sequence A, - ``1`` indicates sequence B is a random sequence. - Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: - **loss**: (`optional`, returned when both ``masked_lm_labels`` and ``next_sentence_label`` are provided) ``torch.FloatTensor`` of shape ``(1,)``: - Total loss as the sum of the masked language modeling loss and the next sequence prediction (classification) loss. **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). **seq_relationship_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, 2)`` @@ -762,15 +749,7 @@ class TFBertForPreTraining(TFBertPreTrainedModel): BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) class TFBertForMaskedLM(TFBertPreTrainedModel): r""" - **masked_lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: - Labels for computing the masked language modeling loss. - Indices should be in ``[-1, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring) - Tokens with indices set to ``-1`` are ignored (masked), the loss is only computed for the tokens with labels - in ``[0, ..., config.vocab_size]`` - Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: - **loss**: (`optional`, returned when ``masked_lm_labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: - Masked language modeling loss. **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) @@ -786,8 +765,8 @@ class TFBertForMaskedLM(TFBertPreTrainedModel): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = TFBertForMaskedLM.from_pretrained('bert-base-uncased') input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 - outputs = model(input_ids, masked_lm_labels=input_ids) - loss, prediction_scores = outputs[:2] + outputs = model(input_ids) + prediction_scores = outputs[:2] """ def __init__(self, config, *inputs, **kwargs): @@ -811,12 +790,6 @@ class TFBertForMaskedLM(TFBertPreTrainedModel): BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) class TFBertForNextSentencePrediction(TFBertPreTrainedModel): r""" - **next_sentence_label**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: - Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair (see ``input_ids`` docstring) - Indices should be in ``[0, 1]``. - ``0`` indicates sequence B is a continuation of sequence A, - ``1`` indicates sequence B is a random sequence. - Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: **loss**: (`optional`, returned when ``next_sentence_label`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: Next sequence prediction (classification) loss. @@ -862,15 +835,7 @@ class TFBertForNextSentencePrediction(TFBertPreTrainedModel): BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) class TFBertForSequenceClassification(TFBertPreTrainedModel): r""" - **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: - Labels for computing the sequence classification/regression loss. - Indices should be in ``[0, ..., config.num_labels - 1]``. - If ``config.num_labels == 1`` a regression loss is computed (Mean-Square loss), - If ``config.num_labels > 1`` a classification loss is computed (Cross-Entropy). - Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: - **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: - Classification (or regression if config.num_labels==1) loss. **logits**: ``torch.FloatTensor`` of shape ``(batch_size, config.num_labels)`` Classification (or regression if config.num_labels==1) scores (before SoftMax). **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) @@ -886,8 +851,7 @@ class TFBertForSequenceClassification(TFBertPreTrainedModel): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = TFBertForSequenceClassification.from_pretrained('bert-base-uncased') input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 - labels = torch.tensor([1]).unsqueeze(0) # Batch size 1 - outputs = model(input_ids, labels=labels) + outputs = model(input_ids) loss, logits = outputs[:2] """ @@ -905,7 +869,8 @@ class TFBertForSequenceClassification(TFBertPreTrainedModel): pooled_output = outputs[1] - pooled_output = self.dropout(pooled_output) + if training: + pooled_output = self.dropout(pooled_output) logits = self.classifier(pooled_output) outputs = (logits,) + outputs[2:] # add hidden states and attention if they are here @@ -915,53 +880,10 @@ class TFBertForSequenceClassification(TFBertPreTrainedModel): @add_start_docstrings("""Bert Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a softmax) e.g. for RocStories/SWAG tasks. """, - BERT_START_DOCSTRING) + BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) class TFBertForMultipleChoice(TFBertPreTrainedModel): r""" - Inputs: - **input_ids**: ``torch.LongTensor`` of shape ``(batch_size, num_choices, sequence_length)``: - Indices of input sequence tokens in the vocabulary. - The second dimension of the input (`num_choices`) indicates the number of choices to score. - To match pre-training, BERT input sequence should be formatted with [CLS] and [SEP] tokens as follows: - - (a) For sequence pairs: - - ``tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]`` - - ``token_type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1`` - - (b) For single sequences: - - ``tokens: [CLS] the dog is hairy . [SEP]`` - - ``token_type_ids: 0 0 0 0 0 0 0`` - - Indices can be obtained using :class:`pytorch_transformers.BertTokenizer`. - See :func:`pytorch_transformers.PreTrainedTokenizer.encode` and - :func:`pytorch_transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. - **token_type_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, num_choices, sequence_length)``: - Segment token indices to indicate first and second portions of the inputs. - The second dimension of the input (`num_choices`) indicates the number of choices to score. - Indices are selected in ``[0, 1]``: ``0`` corresponds to a `sentence A` token, ``1`` - corresponds to a `sentence B` token - (see `BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ for more details). - **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, num_choices, sequence_length)``: - Mask to avoid performing attention on padding token indices. - The second dimension of the input (`num_choices`) indicates the number of choices to score. - Mask values selected in ``[0, 1]``: - ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. - **head_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: - Mask to nullify selected heads of the self-attention modules. - Mask values selected in ``[0, 1]``: - ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. - **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: - Labels for computing the multiple choice classification loss. - Indices should be in ``[0, ..., num_choices]`` where `num_choices` is the size of the second dimension - of the input tensors. (see `input_ids` above) - Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: - **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: - Classification loss. **classification_scores**: ``torch.FloatTensor`` of shape ``(batch_size, num_choices)`` where `num_choices` is the size of the second dimension of the input tensors. (see `input_ids` above). Classification scores (before SoftMax). @@ -979,8 +901,7 @@ class TFBertForMultipleChoice(TFBertPreTrainedModel): model = BertForMultipleChoice.from_pretrained('bert-base-uncased') choices = ["Hello, my dog is cute", "Hello, my cat is amazing"] input_ids = torch.tensor([tokenizer.encode(s) for s in choices]).unsqueeze(0) # Batch size 1, 2 choices - labels = torch.tensor(1).unsqueeze(0) # Batch size 1 - outputs = model(input_ids, labels=labels) + outputs = model(input_ids) loss, classification_scores = outputs[:2] """ @@ -1025,7 +946,8 @@ class TFBertForMultipleChoice(TFBertPreTrainedModel): pooled_output = outputs[1] - pooled_output = self.dropout(pooled_output) + if training: + pooled_output = self.dropout(pooled_output) logits = self.classifier(pooled_output) reshaped_logits = tf.reshape(logits, (-1, num_choices)) @@ -1039,13 +961,7 @@ class TFBertForMultipleChoice(TFBertPreTrainedModel): BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) class TFBertForTokenClassification(TFBertPreTrainedModel): r""" - **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: - Labels for computing the token classification loss. - Indices should be in ``[0, ..., config.num_labels - 1]``. - Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: - **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: - Classification loss. **scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.num_labels)`` Classification scores (before SoftMax). **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) @@ -1061,8 +977,7 @@ class TFBertForTokenClassification(TFBertPreTrainedModel): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = BertForTokenClassification.from_pretrained('bert-base-uncased') input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 - labels = torch.tensor([1] * input_ids.size(1)).unsqueeze(0) # Batch size 1 - outputs = model(input_ids, labels=labels) + outputs = model(input_ids) loss, scores = outputs[:2] """ @@ -1080,7 +995,8 @@ class TFBertForTokenClassification(TFBertPreTrainedModel): sequence_output = outputs[0] - sequence_output = self.dropout(sequence_output) + if training: + sequence_output = self.dropout(sequence_output) logits = self.classifier(sequence_output) outputs = (logits,) + outputs[2:] # add hidden states and attention if they are here @@ -1093,18 +1009,7 @@ class TFBertForTokenClassification(TFBertPreTrainedModel): BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) class TFBertForQuestionAnswering(TFBertPreTrainedModel): r""" - **start_positions**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). - Position outside of the sequence are not taken into account for computing the loss. - **end_positions**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). - Position outside of the sequence are not taken into account for computing the loss. - Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: - **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: - Total span extraction loss is the sum of a Cross-Entropy for the start and end positions. **start_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length,)`` Span-start scores (before SoftMax). **end_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length,)`` diff --git a/pytorch_transformers/modeling_tf_gpt2.py b/pytorch_transformers/modeling_tf_gpt2.py index dc185d6e44..27e51fb752 100644 --- a/pytorch_transformers/modeling_tf_gpt2.py +++ b/pytorch_transformers/modeling_tf_gpt2.py @@ -28,13 +28,13 @@ from io import open import numpy as np import tensorflow as tf -from .modeling_tf_utils import TFPreTrainedModel, TFConv1D +from .modeling_tf_utils import TFPreTrainedModel, TFConv1D, TFSequenceSummary, shape_list from .configuration_gpt2 import GPT2Config from .file_utils import add_start_docstrings logger = logging.getLogger(__name__) -GPT2_PRETRAINED_MODEL_ARCHIVE_MAP = {"gpt2": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-tf_model.h5", +TF_GPT2_PRETRAINED_MODEL_ARCHIVE_MAP = {"gpt2": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-tf_model.h5", "gpt2-medium": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-medium-tf_model.h5", "gpt2-large": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-large-tf_model.h5"} @@ -139,7 +139,7 @@ class TFAttention(tf.keras.layers.Layer): @staticmethod @tf.function - def attention_mask(nd, ns, dtype): + def causal_attention_mask(nd, ns, dtype): """1's in the lower triangle, counting from the lower right corner. Same as tf.matrix_band_part(tf.ones([nd, ns]), -1, ns-nd), but doesn't produce garbage on TPUs. """ @@ -150,20 +150,24 @@ class TFAttention(tf.keras.layers.Layer): @tf.function def _attn(self, inputs, training=False): - q, k, v, head_mask = inputs + q, k, v, attention_mask, head_mask = inputs # q, k, v have shape [batch, heads, sequence, features] w = tf.matmul(q, k, transpose_b=True) if self.scale: - n_state = shape_list(v)[-1] - w = w * tf.rsqrt(tf.cast(v.shape[-1].value, w.dtype)) + dk = tf.cast(tf.shape(k)[-1], tf.float32) # scale attention_scores + w = w / tf.math.sqrt(dk) # w has shape [batch, heads, dst_sequence, src_sequence], where information flows from src to dst. _, _, nd, ns = shape_list(w) - b = self.attention_mask(nd, ns, dtype=w.dtype) + b = self.causal_attention_mask(nd, ns, dtype=w.dtype) b = tf.reshape(b, [1, 1, nd, ns]) w = w * b - 1e4 * (1 - b) - w = tf.nn.softmax(w) + if attention_mask is not None: + # Apply the attention mask + w = w + attention_mask + + w = tf.nn.softmax(w, axis=-1) if training: w = self.attn_dropout(w) @@ -179,20 +183,20 @@ class TFAttention(tf.keras.layers.Layer): @tf.function def merge_heads(self, x): x = tf.transpose(x, [0, 2, 1, 3]) - x_shape = tf.shape(x) - new_x_shape = x_shape[:-2] + (x_shape[-2] * x_shape[-1],) + x_shape = shape_list(x) + new_x_shape = x_shape[:-2] + [x_shape[-2] * x_shape[-1]] return tf.reshape(x, new_x_shape) @tf.function def split_heads(self, x): - x_shape = tf.shape(x) - new_x_shape = x_shape[:-1] + (self.n_head, x_shape[-1] // self.n_head) + x_shape = shape_list(x) + new_x_shape = x_shape[:-1] + [self.n_head, x_shape[-1] // self.n_head] x = tf.reshape(x, new_x_shape) return tf.transpose(x, (0, 2, 1, 3)) # (batch, head, seq_length, head_features) @tf.function def call(self, inputs, training=False): - x, layer_past, head_mask = inputs + x, layer_past, attention_mask, head_mask = inputs x = self.c_attn(x) query, key, value = tf.split(x, 3, axis=2) @@ -205,7 +209,7 @@ class TFAttention(tf.keras.layers.Layer): value = tf.concat([past_value, value], axis=-2) present = tf.stack([key, value], axis=1) - attn_outputs = self._attn(query, key, value, head_mask, training=training) + attn_outputs = self._attn([query, key, value, attention_mask, head_mask], training=training) a = attn_outputs[0] a = self.merge_heads(a) @@ -217,7 +221,7 @@ class TFAttention(tf.keras.layers.Layer): return outputs # a, present, (attentions) -class TFMLP(nn.Module): +class TFMLP(tf.keras.layers.Layer): def __init__(self, n_state, config, **kwargs): super(TFMLP, self).__init__(**kwargs) nx = config.n_embd @@ -245,15 +249,16 @@ class TFBlock(tf.keras.layers.Layer): self.mlp = TFMLP(4 * nx, config, name='mlp') @tf.function - def call(self, x, layer_past=None, head_mask=None, training=False): - output_attn = self.attn(self.ln_1(x), - layer_past=layer_past, - head_mask=head_mask, - training=training) - a = output_attn[0] # output_attn: a, present, (attentions) + def call(self, inputs, training=False): + x, layer_past, attention_mask, head_mask = inputs + a = self.ln_1(x) + output_attn = self.attn([a, layer_past, attention_mask, head_mask], training=training) + a = output_attn[0] # output_attn: a, present, (attentions) x = x + a - m = self.mlp(self.ln_2(x), training=training) + + m = self.ln_2(x) + m = self.mlp(m, training=training) x = x + m outputs = [x] + output_attn[1:] @@ -274,13 +279,13 @@ class TFGPT2Embeddings(tf.keras.layers.Layer): """ self.weight = self.add_weight( "weight", - shape=[self.vocab_size, self.n_embed], + shape=[self.vocab_size, self.hidden_size], initializer=tf.random_normal_initializer( - mean=0., stddev=self.n_embed**-0.5)) - super(TFBertEmbeddings, self).build(input_shape) + mean=0., stddev=self.hidden_size**-0.5)) + super(TFGPT2Embeddings, self).build(input_shape) @tf.function - def call(self, inputs, mode="embedding", training=False): + def call(self, inputs, mode="embedding"): """Get token embeddings of inputs. Args: inputs: list of three int64 tensors with shape [batch_size, length]: (input_ids, position_ids, token_type_ids) @@ -296,7 +301,7 @@ class TFGPT2Embeddings(tf.keras.layers.Layer): https://github.com/tensorflow/models/blob/a009f4fb9d2fc4949e32192a944688925ef78659/official/transformer/v2/embedding_layer.py#L24 """ if mode == "embedding": - return self._embedding(inputs, training=training) + return self._embedding(inputs) elif mode == "linear": return self._linear(inputs) else: @@ -313,10 +318,10 @@ class TFGPT2Embeddings(tf.keras.layers.Layer): Returns: float32 tensor with shape [batch_size, length, vocab_size]. """ - batch_size = tf.shape(inputs)[0] - length = tf.shape(inputs)[1] + batch_size = shape_list(inputs)[0] + length = shape_list(inputs)[1] - x = tf.reshape(inputs, [-1, self.n_embed]) + x = tf.reshape(inputs, [-1, self.hidden_size]) logits = tf.matmul(x, self.weight, transpose_b=True) return tf.reshape(logits, [batch_size, length, self.vocab_size]) @@ -326,13 +331,14 @@ class TFGPT2MainLayer(tf.keras.layers.Layer): super(TFGPT2MainLayer, self).__init__(config, *inputs, **kwargs) self.output_hidden_states = config.output_hidden_states self.output_attentions = config.output_attentions + self.num_hidden_layers = config.n_layer self.vocab_size = config.vocab_size self.n_embd = config.n_embd self.wte = TFGPT2Embeddings(config, name='wte') self.wpe = tf.keras.layers.Embedding(config.n_positions, config.n_embd, name='wpe') self.drop = tf.keras.layers.Dropout(config.embd_pdrop) - self.h = [TFBlock(config.n_ctx, config, scale=Truename='h_{}'.format(i)) for i in range(config.n_layer)] + self.h = [TFBlock(config.n_ctx, config, scale=True, name='h_{}'.format(i)) for i in range(config.n_layer)] self.ln_f = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_epsilon, name='ln_f') def _resize_token_embeddings(self, new_num_tokens): @@ -346,20 +352,20 @@ class TFGPT2MainLayer(tf.keras.layers.Layer): @tf.function def call(self, inputs, training=False): - input_ids, position_ids=None, token_type_ids=None, past=None, head_mask=None): - if not isinstance(inputs, (dict, tuple, list)): input_ids = inputs - attention_mask, head_mask, position_ids, token_type_ids = None, None, None, None + past, attention_mask, token_type_ids, position_ids, head_mask = None, None, None, None, None elif isinstance(inputs, (tuple, list)): input_ids = inputs[0] - attention_mask = inputs[1] if len(inputs) > 1 else None - token_type_ids = inputs[2] if len(inputs) > 2 else None - position_ids = inputs[3] if len(inputs) > 3 else None - head_mask = inputs[4] if len(inputs) > 4 else None - assert len(inputs) <= 5, "Too many inputs." + past = inputs[1] if len(inputs) > 1 else None + attention_mask = inputs[2] if len(inputs) > 2 else None + token_type_ids = inputs[3] if len(inputs) > 3 else None + position_ids = inputs[4] if len(inputs) > 4 else None + head_mask = inputs[5] if len(inputs) > 5 else None + assert len(inputs) <= 6, "Too many inputs." else: input_ids = inputs.get('input_ids') + past = inputs.get('past', None) attention_mask = inputs.get('attention_mask', None) token_type_ids = inputs.get('token_type_ids', None) position_ids = inputs.get('position_ids', None) @@ -370,49 +376,66 @@ class TFGPT2MainLayer(tf.keras.layers.Layer): past_length = 0 past = [None] * len(self.h) else: - past_length = past[0][0].size(-2) + past_length = shape_list(past[0][0])[-2] if position_ids is None: - position_ids = torch.arange(past_length, input_ids.size(-1) + past_length, dtype=torch.long, device=input_ids.device) - position_ids = position_ids.unsqueeze(0).expand_as(input_ids) + position_ids = tf.range(past_length, shape_list(input_ids)[-1] + past_length, dtype=tf.int32)[tf.newaxis, :] + + if attention_mask is not None: + # We create a 3D attention mask from a 2D tensor mask. + # Sizes are [batch_size, 1, 1, to_seq_length] + # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length] + # this attention mask is more simple than the triangular masking of causal attention + # used in OpenAI GPT, we just need to prepare the broadcast dimension here. + attention_mask = attention_mask[:, tf.newaxis, tf.newaxis, :] + + # Since attention_mask is 1.0 for positions we want to attend and 0.0 for + # masked positions, this operation will create a tensor which is 0.0 for + # positions we want to attend and -10000.0 for masked positions. + # Since we are adding it to the raw scores before the softmax, this is + # effectively the same as removing these entirely. + + attention_mask = tf.cast(attention_mask, tf.float32) + attention_mask = (1.0 - attention_mask) * -10000.0 + else: + attention_mask = None # Prepare head mask if needed # 1.0 in head_mask indicate we keep the head # attention_probs has shape bsz x n_heads x N x N - # head_mask has shape n_layer x batch x n_heads x N x N - if head_mask is not None: - if head_mask.dim() == 1: - head_mask = head_mask.unsqueeze(0).unsqueeze(0).unsqueeze(-1).unsqueeze(-1) - head_mask = head_mask.expand(self.config.n_layer, -1, -1, -1, -1) - elif head_mask.dim() == 2: - head_mask = head_mask.unsqueeze(1).unsqueeze(-1).unsqueeze(-1) # We can specify head_mask for each layer - head_mask = head_mask.to(dtype=next(self.parameters()).dtype) # switch to fload if need + fp16 compatibility + # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] + # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length] + if not head_mask is None: + raise NotImplementedError else: - head_mask = [None] * self.config.n_layer + head_mask = [None] * self.num_hidden_layers + # head_mask = tf.constant([0] * self.num_hidden_layers) - input_shape = input_ids.size() - input_ids = input_ids.view(-1, input_ids.size(-1)) - position_ids = position_ids.view(-1, position_ids.size(-1)) + input_shape = shape_list(input_ids) + input_ids = tf.reshape(input_ids, [-1, input_shape[-1]]) + position_ids = tf.reshape(position_ids, [-1, shape_list(position_ids)[-1]]) - inputs_embeds = self.wte(input_ids) + inputs_embeds = self.wte(input_ids, mode='embedding') position_embeds = self.wpe(position_ids) if token_type_ids is not None: - token_type_ids = token_type_ids.view(-1, token_type_ids.size(-1)) - token_type_embeds = self.wte(token_type_ids) + token_type_ids = tf.reshape(token_type_ids, [-1, shape_list(token_type_ids)[-1]]) + token_type_embeds = self.wte(token_type_ids, mode='embedding') else: token_type_embeds = 0 hidden_states = inputs_embeds + position_embeds + token_type_embeds - hidden_states = self.drop(hidden_states) + if training: + hidden_states = self.drop(hidden_states) - output_shape = input_shape + (hidden_states.size(-1),) + output_shape = input_shape + [shape_list(hidden_states)[-1]] presents = () all_attentions = [] all_hidden_states = () for i, (block, layer_past) in enumerate(zip(self.h, past)): if self.output_hidden_states: - all_hidden_states = all_hidden_states + (hidden_states.view(*output_shape),) + all_hidden_states = all_hidden_states + (tf.reshape(hidden_states, output_shape),) + + outputs = block([hidden_states, layer_past, attention_mask, head_mask[i]], training=training) - outputs = block(hidden_states, layer_past, head_mask[i]) hidden_states, present = outputs[:2] presents = presents + (present,) @@ -421,7 +444,7 @@ class TFGPT2MainLayer(tf.keras.layers.Layer): hidden_states = self.ln_f(hidden_states) - hidden_states = hidden_states.view(*output_shape) + hidden_states = tf.reshape(hidden_states, output_shape) # Add last hidden state if self.output_hidden_states: all_hidden_states = all_hidden_states + (hidden_states,) @@ -431,18 +454,19 @@ class TFGPT2MainLayer(tf.keras.layers.Layer): outputs = outputs + (all_hidden_states,) if self.output_attentions: # let the number of heads free (-1) so we can extract attention even after head pruning - attention_output_shape = input_shape[:-1] + (-1,) + all_attentions[0].shape[-2:] - all_attentions = tuple(t.view(*attention_output_shape) for t in all_attentions) + attention_output_shape = input_shape[:-1] + [-1] + shape_list(all_attentions[0])[-2:] + all_attentions = tuple(tf.reshape(t, attention_output_shape) for t in all_attentions) outputs = outputs + (all_attentions,) return outputs # last hidden state, presents, (all hidden_states), (attentions) + class TFGPT2PreTrainedModel(TFPreTrainedModel): """ An abstract class to handle weights initialization and a simple interface for dowloading and loading pretrained models. """ config_class = GPT2Config - pretrained_model_archive_map = GPT2_PRETRAINED_MODEL_ARCHIVE_MAP - load_tf_weights = load_tf_weights_in_gpt2 + pretrained_model_archive_map = TF_GPT2_PRETRAINED_MODEL_ARCHIVE_MAP + load_pt_weights = load_gpt2_pt_weights_in_tf base_model_prefix = "transformer" @@ -487,17 +511,21 @@ GPT2_INPUTS_DOCSTRING = r""" Inputs: Indices can be obtained using :class:`pytorch_transformers.BPT2Tokenizer`. See :func:`pytorch_transformers.PreTrainedTokenizer.encode` and :func:`pytorch_transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. - **position_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: - Indices of positions of each input sequence tokens in the position embeddings. - Selected in the range ``[0, config.max_position_embeddings - 1]``. - **token_type_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: - A parallel sequence of tokens (can be used to indicate various portions of the inputs). - The embeddings from these tokens will be summed with the respective token embeddings. - Indices are selected in the vocabulary (unlike BERT which has a specific vocabulary for segment indices). **past**: list of ``torch.FloatTensor`` (one for each layer): that contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model (see `past` output below). Can be used to speed up sequential decoding. + **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``: + Mask to avoid performing attention on padding token indices. + Mask values selected in ``[0, 1]``: + ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. + **token_type_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + A parallel sequence of tokens (can be used to indicate various portions of the inputs). + The embeddings from these tokens will be summed with the respective token embeddings. + Indices are selected in the vocabulary (unlike BERT which has a specific vocabulary for segment indices). + **position_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Indices of positions of each input sequence tokens in the position embeddings. + Selected in the range ``[0, config.max_position_embeddings - 1]``. **head_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: @@ -526,7 +554,7 @@ class TFGPT2Model(TFGPT2PreTrainedModel): Examples:: tokenizer = GPT2Tokenizer.from_pretrained('gpt2') - model = TFGPT2Model.from_pretrained('gpt2') + model = GPT2Model.from_pretrained('gpt2') input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -534,149 +562,19 @@ class TFGPT2Model(TFGPT2PreTrainedModel): """ def __init__(self, config, *inputs, **kwargs): super(TFGPT2Model, self).__init__(config, *inputs, **kwargs) - self.output_hidden_states = config.output_hidden_states - self.output_attentions = config.output_attentions - self.vocab_size = config.vocab_size - self.n_embd = config.n_embd - - self.wpe = tf.keras.layers.Embedding(config.n_positions, config.n_embd, name='wpe') - self.drop = tf.keras.layers.Dropout(config.embd_pdrop) - self.h = [TFBlock(config.n_ctx, config, scale=Truename='h_{}'.format(i)) for i in range(config.n_layer)] - self.ln_f = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_epsilon, name='ln_f') - - self.init_weights() - - def build(self, input_shape): - """Build shared word embedding layer - Shared weights logic adapted from - https://github.com/tensorflow/models/blob/a009f4fb9d2fc4949e32192a944688925ef78659/official/transformer/v2/embedding_layer.py#L24 - """ - with tf.name_scope("wte"): - # Create and initialize weights. The random normal initializer was chosen - # arbitrarily, and works well. - self.wte = self.add_weight( - "weight", - shape=[self.vocab_size, self.n_embed], - initializer=tf.random_normal_initializer( - mean=0., stddev=self.n_embed**-0.5)) - super(TFGPT2Model, self).build(input_shape) - - def _resize_token_embeddings(self, new_num_tokens): - raise NotImplementedError - - def _prune_heads(self, heads_to_prune): - """ Prunes heads of the model. - heads_to_prune: dict of {layer_num: list of heads to prune in this layer} - """ - raise NotImplementedError + self.transformer = TFGPT2MainLayer(config, name='transformer') @tf.function def call(self, inputs, training=False): - input_ids, position_ids=None, token_type_ids=None, past=None, head_mask=None): - - if not isinstance(inputs, (dict, tuple, list)): - input_ids = inputs - attention_mask, head_mask, position_ids, token_type_ids = None, None, None, None - elif isinstance(inputs, (tuple, list)): - input_ids = inputs[0] - attention_mask = inputs[1] if len(inputs) > 1 else None - token_type_ids = inputs[2] if len(inputs) > 2 else None - position_ids = inputs[3] if len(inputs) > 3 else None - head_mask = inputs[4] if len(inputs) > 4 else None - assert len(inputs) <= 5, "Too many inputs." - else: - input_ids = inputs.get('input_ids') - attention_mask = inputs.get('attention_mask', None) - token_type_ids = inputs.get('token_type_ids', None) - position_ids = inputs.get('position_ids', None) - head_mask = inputs.get('head_mask', None) - assert len(inputs) <= 5, "Too many inputs." - - if past is None: - past_length = 0 - past = [None] * len(self.h) - else: - past_length = past[0][0].size(-2) - if position_ids is None: - position_ids = torch.arange(past_length, input_ids.size(-1) + past_length, dtype=torch.long, device=input_ids.device) - position_ids = position_ids.unsqueeze(0).expand_as(input_ids) - - # Prepare head mask if needed - # 1.0 in head_mask indicate we keep the head - # attention_probs has shape bsz x n_heads x N x N - # head_mask has shape n_layer x batch x n_heads x N x N - if head_mask is not None: - if head_mask.dim() == 1: - head_mask = head_mask.unsqueeze(0).unsqueeze(0).unsqueeze(-1).unsqueeze(-1) - head_mask = head_mask.expand(self.config.n_layer, -1, -1, -1, -1) - elif head_mask.dim() == 2: - head_mask = head_mask.unsqueeze(1).unsqueeze(-1).unsqueeze(-1) # We can specify head_mask for each layer - head_mask = head_mask.to(dtype=next(self.parameters()).dtype) # switch to fload if need + fp16 compatibility - else: - head_mask = [None] * self.config.n_layer - - input_shape = input_ids.size() - input_ids = input_ids.view(-1, input_ids.size(-1)) - position_ids = position_ids.view(-1, position_ids.size(-1)) - - inputs_embeds = self.wte(input_ids) - position_embeds = self.wpe(position_ids) - if token_type_ids is not None: - token_type_ids = token_type_ids.view(-1, token_type_ids.size(-1)) - token_type_embeds = self.wte(token_type_ids) - else: - token_type_embeds = 0 - hidden_states = inputs_embeds + position_embeds + token_type_embeds - hidden_states = self.drop(hidden_states) - - output_shape = input_shape + (hidden_states.size(-1),) - - presents = () - all_attentions = [] - all_hidden_states = () - for i, (block, layer_past) in enumerate(zip(self.h, past)): - if self.output_hidden_states: - all_hidden_states = all_hidden_states + (hidden_states.view(*output_shape),) - - outputs = block(hidden_states, layer_past, head_mask[i]) - hidden_states, present = outputs[:2] - presents = presents + (present,) - - if self.output_attentions: - all_attentions.append(outputs[2]) - - hidden_states = self.ln_f(hidden_states) - - hidden_states = hidden_states.view(*output_shape) - # Add last hidden state - if self.output_hidden_states: - all_hidden_states = all_hidden_states + (hidden_states,) - - outputs = (hidden_states, presents) - if self.output_hidden_states: - outputs = outputs + (all_hidden_states,) - if self.output_attentions: - # let the number of heads free (-1) so we can extract attention even after head pruning - attention_output_shape = input_shape[:-1] + (-1,) + all_attentions[0].shape[-2:] - all_attentions = tuple(t.view(*attention_output_shape) for t in all_attentions) - outputs = outputs + (all_attentions,) - return outputs # last hidden state, presents, (all hidden_states), (attentions) + outputs = self.transformer(inputs, training=training) + return outputs @add_start_docstrings("""The GPT2 Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings). """, GPT2_START_DOCSTRING, GPT2_INPUTS_DOCSTRING) -class GPT2LMHeadModel(GPT2PreTrainedModel): +class TFGPT2LMHeadModel(TFGPT2PreTrainedModel): r""" - **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: - Labels for language modeling. - Note that the labels **are shifted** inside the model, i.e. you can set ``lm_labels = input_ids`` - Indices are selected in ``[-1, 0, ..., config.vocab_size]`` - All labels set to ``-1`` are ignored (masked), the loss is only - computed for labels in ``[0, ..., config.vocab_size]`` - Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: - **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: - Language modeling loss. **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). **past**: @@ -700,93 +598,38 @@ class GPT2LMHeadModel(GPT2PreTrainedModel): model = GPT2LMHeadModel.from_pretrained('gpt2') input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 - outputs = model(input_ids, labels=input_ids) - loss, logits = outputs[:2] + outputs = model(input_ids) + logits = outputs[:2] """ - def __init__(self, config): - super(GPT2LMHeadModel, self).__init__(config) - self.transformer = GPT2Model(config) - self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False) + def __init__(self, config, *inputs, **kwargs): + super(TFGPT2LMHeadModel, self).__init__(config, *inputs, **kwargs) + self.transformer = TFGPT2MainLayer(config, name='transformer') - self.init_weights() - self.tie_weights() - - def tie_weights(self): - """ Make sure we are sharing the input and output embeddings. - Export to TorchScript can't handle parameter sharing so we are cloning them instead. - """ - self._tie_or_clone_weights(self.lm_head, - self.transformer.wte) - - def forward(self, input_ids, position_ids=None, token_type_ids=None, labels=None, past=None, head_mask=None): - transformer_outputs = self.transformer(input_ids, position_ids=position_ids, token_type_ids=token_type_ids, - past=past, head_mask=head_mask) + @tf.function + def call(self, inputs, training=False): + transformer_outputs = self.transformer(inputs, training=training) hidden_states = transformer_outputs[0] - lm_logits = self.lm_head(hidden_states) + lm_logits = self.transformer.wte(hidden_states, mode="linear") outputs = (lm_logits,) + transformer_outputs[1:] - if labels is not None: - # Shift so that tokens < n predict n - shift_logits = lm_logits[..., :-1, :].contiguous() - shift_labels = labels[..., 1:].contiguous() - # Flatten the tokens - loss_fct = CrossEntropyLoss(ignore_index=-1) - loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), - shift_labels.view(-1)) - outputs = (loss,) + outputs - return outputs # (loss), lm_logits, presents, (all hidden_states), (attentions) + return outputs # lm_logits, presents, (all hidden_states), (attentions) @add_start_docstrings("""The GPT2 Model transformer with a language modeling and a multiple-choice classification head on top e.g. for RocStories/SWAG tasks. The two heads are two linear layers. The language modeling head has its weights tied to the input embeddings, the classification head takes as input the input of a specified classification token index in the input sequence). -""", GPT2_START_DOCSTRING) -class GPT2DoubleHeadsModel(GPT2PreTrainedModel): - r""" Inputs: - **input_ids**: ``torch.LongTensor`` of shape ``(batch_size, num_choices, sequence_length)``: - Indices of input sequence tokens in the vocabulary. - The second dimension of the input (`num_choices`) indicates the number of choices to score. - Indices can be obtained using :class:`pytorch_transformers.BPT2Tokenizer`. - See :func:`pytorch_transformers.PreTrainedTokenizer.encode` and - :func:`pytorch_transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. - **mc_token_ids**: ``torch.LongTensor`` of shape ``(batch_size, num_choices)``: +""", GPT2_START_DOCSTRING, GPT2_INPUTS_DOCSTRING) +class TFGPT2DoubleHeadsModel(TFGPT2PreTrainedModel): + r""" + **mc_token_ids**: (`optional`, default to index of the last token of the input) ``torch.LongTensor`` of shape ``(batch_size, num_choices)``: Index of the classification token in each input sequence. Selected in the range ``[0, input_ids.size(-1) - 1[``. - **position_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, num_choices, sequence_length)``: - Indices of positions of each input sequence tokens in the position embeddings. - Selected in the range ``[0, config.max_position_embeddings - 1]``. - **token_type_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, num_choices, sequence_length)``: - A parallel sequence of tokens (can be used to indicate various portions of the inputs). - The embeddings from these tokens will be summed with the respective token embeddings. - Indices are selected in the vocabulary (unlike BERT which has a specific vocabulary for segment indices). - **past**: - list of ``torch.FloatTensor`` (one for each layer): - that contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model - (see `past` output below). Can be used to speed up sequential decoding. - **head_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: - Mask to nullify selected heads of the self-attention modules. - Mask values selected in ``[0, 1]``: - ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. - **lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: - Labels for language modeling. - Note that the labels **are shifted** inside the model, i.e. you can set ``lm_labels = input_ids`` - Indices are selected in ``[-1, 0, ..., config.vocab_size]`` - All labels set to ``-1`` are ignored (masked), the loss is only - computed for labels in ``[0, ..., config.vocab_size]`` - **mc_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size)``: - Labels for computing the multiple choice classification loss. - Indices should be in ``[0, ..., num_choices]`` where `num_choices` is the size of the second dimension - of the input tensors. (see `input_ids` above) Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: - **lm_loss**: (`optional`, returned when ``lm_labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: - Language modeling loss. - **mc_loss**: (`optional`, returned when ``multiple_choice_labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: - Multiple choice classification loss. **lm_prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, num_choices, sequence_length, config.vocab_size)`` Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). **mc_prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, num_choices)`` @@ -827,43 +670,52 @@ class GPT2DoubleHeadsModel(GPT2PreTrainedModel): lm_prediction_scores, mc_prediction_scores = outputs[:2] """ - def __init__(self, config): - super(GPT2DoubleHeadsModel, self).__init__(config) - self.transformer = GPT2Model(config) - self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False) - self.multiple_choice_head = SequenceSummary(config) + def __init__(self, config, *inputs, **kwargs): + super(TFGPT2DoubleHeadsModel, self).__init__(config, *inputs, **kwargs) + self.transformer = TFGPT2MainLayer(config, name='transformer') + self.multiple_choice_head = TFSequenceSummary(config, name='multiple_choice_head') - self.init_weights() - self.tie_weights() - def tie_weights(self): - """ Make sure we are sharing the input and output embeddings. - Export to TorchScript can't handle parameter sharing so we are cloning them instead. - """ - self._tie_or_clone_weights(self.lm_head, - self.transformer.wte) + @tf.function + def call(self, inputs, training=False): + if not isinstance(inputs, (dict, tuple, list)): + raise ValueError("Inputs should be a list or a dict with at least two elements: 'inputs_ids' and 'mc_token_ids'") + elif isinstance(inputs, (tuple, list)): + input_ids = inputs[0] + mc_token_ids = inputs[1] + past = inputs[2] if len(inputs) > 2 else None + attention_mask = inputs[3] if len(inputs) > 3 else None + token_type_ids = inputs[4] if len(inputs) > 4 else None + position_ids = inputs[5] if len(inputs) > 5 else None + head_mask = inputs[6] if len(inputs) > 6 else None + assert len(inputs) <= 7, "Too many inputs." + else: + input_ids = inputs.get('input_ids') + mc_token_ids = inputs.get('mc_token_ids') + past = inputs.get('past', None) + attention_mask = inputs.get('attention_mask', None) + token_type_ids = inputs.get('token_type_ids', None) + position_ids = inputs.get('position_ids', None) + head_mask = inputs.get('head_mask', None) + assert len(inputs) <= 5, "Too many inputs." + + num_choices = shape_list(input_ids)[1] + seq_length = shape_list(input_ids)[2] + + flat_input_ids = tf.reshape(input_ids, (-1, seq_length)) + flat_attention_mask = tf.reshape(attention_mask, (-1, seq_length)) if attention_mask is not None else None + flat_token_type_ids = tf.reshape(token_type_ids, (-1, seq_length)) if token_type_ids is not None else None + flat_position_ids = tf.reshape(position_ids, (-1, seq_length)) if position_ids is not None else None + + flat_inputs = [flat_input_ids, past, flat_attention_mask, flat_token_type_ids, flat_position_ids, head_mask] + + outputs = self.transformer(flat_inputs, training=training) - def forward(self, input_ids, mc_token_ids=None, lm_labels=None, mc_labels=None, token_type_ids=None, - position_ids=None, past=None, head_mask=None): - transformer_outputs = self.transformer(input_ids, position_ids=position_ids, token_type_ids=token_type_ids, - past=past, head_mask=head_mask) hidden_states = transformer_outputs[0] - lm_logits = self.lm_head(hidden_states) - mc_logits = self.multiple_choice_head(hidden_states, mc_token_ids).squeeze(-1) + lm_logits = self.transformer.wte(hidden_states, mode="linear") + mc_logits = self.multiple_choice_head([hidden_states, mc_token_ids], training=training) outputs = (lm_logits, mc_logits) + transformer_outputs[1:] - if mc_labels is not None: - loss_fct = CrossEntropyLoss() - loss = loss_fct(mc_logits.view(-1, mc_logits.size(-1)), - mc_labels.view(-1)) - outputs = (loss,) + outputs - if lm_labels is not None: - shift_logits = lm_logits[..., :-1, :].contiguous() - shift_labels = lm_labels[..., 1:].contiguous() - loss_fct = CrossEntropyLoss(ignore_index=-1) - loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), - shift_labels.view(-1)) - outputs = (loss,) + outputs return outputs # (lm loss), (mc loss), lm logits, mc logits, presents, (all hidden_states), (attentions) diff --git a/pytorch_transformers/modeling_tf_utils.py b/pytorch_transformers/modeling_tf_utils.py index bea4efaa0e..9dfd42c36b 100644 --- a/pytorch_transformers/modeling_tf_utils.py +++ b/pytorch_transformers/modeling_tf_utils.py @@ -273,15 +273,117 @@ class TFConv1D(tf.keras.layers.Layer): mean=0., stddev=0.02)) self.bias = self.add_weight( "bias", - shape=[self.nx, self.nf], + shape=[1, self.nf], initializer=tf.zeros_initializer()) @tf.function def call(self, x): - size_out = tf.shape(x)[:-1] + (self.nf,) + bz, sl = shape_list(x)[:2] - x = tf.reshape(x, [-1, tf.shape(x)[-1]]) + x = tf.reshape(x, [-1, self.nx]) x = tf.matmul(x, self.weight) + self.bias - x = tf.reshape(x, size_out) + + x = tf.reshape(x, [bz, sl, self.nf]) return x + + +class TFSequenceSummary(tf.keras.layers.Layer): + r""" Compute a single vector summary of a sequence hidden states according to various possibilities: + Args of the config class: + summary_type: + - 'last' => [default] take the last token hidden state (like XLNet) + - 'first' => take the first token hidden state (like Bert) + - 'mean' => take the mean of all tokens hidden states + - 'cls_index' => supply a Tensor of classification token position (GPT/GPT-2) + - 'attn' => Not implemented now, use multi-head attention + summary_use_proj: Add a projection after the vector extraction + summary_proj_to_labels: If True, the projection outputs to config.num_labels classes (otherwise to hidden_size). Default: False. + summary_activation: 'tanh' => add a tanh activation to the output, Other => no activation. Default + summary_first_dropout: Add a dropout before the projection and activation + summary_last_dropout: Add a dropout after the projection and activation + """ + def __init__(self, config, **kwargs): + super(TFSequenceSummary, self).__init__(**kwargs) + + self.summary_type = config.summary_type if hasattr(config, 'summary_use_proj') else 'last' + if self.summary_type == 'attn': + # We should use a standard multi-head attention module with absolute positional embedding for that. + # Cf. https://github.com/zihangdai/xlnet/blob/master/modeling.py#L253-L276 + # We can probably just use the multi-head attention module of PyTorch >=1.1.0 + raise NotImplementedError + + self.summary = tf.keras.layers.Identity(name='summary') + if hasattr(config, 'summary_use_proj') and config.summary_use_proj: + if hasattr(config, 'summary_proj_to_labels') and config.summary_proj_to_labels and config.num_labels > 0: + num_classes = config.num_labels + else: + num_classes = config.hidden_size + self.summary = tf.keras.layers.Dense(num_classes, name='summary') + + self.activation = None + if hasattr(config, 'summary_activation') and config.summary_activation == 'tanh': + self.activation = tf.keras.layers.Tanh() + + self.first_dropout = None + if hasattr(config, 'summary_first_dropout') and config.summary_first_dropout > 0: + self.first_dropout = tf.keras.layers.Dropout(config.summary_first_dropout) + + self.last_dropout = None + if hasattr(config, 'summary_last_dropout') and config.summary_last_dropout > 0: + self.last_dropout = tf.keras.layers.Dropout(config.summary_last_dropout) + + @tf.function + def call(self, inputs, training=False): + """ hidden_states: float Tensor in shape [bsz, seq_len, hidden_size], the hidden-states of the last layer. + cls_index: [optional] position of the classification token if summary_type == 'cls_index', + shape (bsz,) or more generally (bsz, ...) where ... are optional leading dimensions of hidden_states. + if summary_type == 'cls_index' and cls_index is None: + we take the last token of the sequence as classification token + """ + if not isinstance(inputs, (dict, tuple, list)): + hidden_states = inputs + cls_index = None + elif isinstance(inputs, (tuple, list)): + hidden_states = inputs[0] + cls_index = inputs[1] if len(inputs) > 1 else None + assert len(inputs) <= 2, "Too many inputs." + else: + input_ids = inputs.get('input_ids') + cls_index = inputs.get('cls_index', None) + + if self.summary_type == 'last': + output = hidden_states[:, -1] + elif self.summary_type == 'first': + output = hidden_states[:, 0] + elif self.summary_type == 'mean': + output = tf.mean(hidden_states, axis=1) + elif self.summary_type == 'cls_index': + if cls_index is None: + cls_index = tf.fill(tf.shape(hidden_states[..., :1, :]), hidden_states.shape[-2]-1, dtype=tf.int32) + else: + cls_index = cls_index[..., tf.newaxis, tf.newaxis] + cls_index = cls_index.expand((-1,) * (cls_index.dim()-1) + (hidden_states.size(-1),)) + # shape of cls_index: (bsz, XX, 1, hidden_size) where XX are optional leading dim of hidden_states + output = hidden_states.gather(-2, cls_index).squeeze(-2) # shape (bsz, XX, hidden_size) + elif self.summary_type == 'attn': + raise NotImplementedError + + if training and self.first_dropout is not None: + output = self.first_dropout(output) + + output = self.summary(output) + + if self.activation is not None: + output = self.activation(output) + + if training and self.last_dropout is not None: + output = self.last_dropout(output) + + return output + +def shape_list(x): + """Deal with dynamic shape in tensorflow cleanly.""" + static = x.shape.as_list() + dynamic = tf.shape(x) + return [dynamic[i] if s is None else s for i, s in enumerate(static)] diff --git a/pytorch_transformers/tests/modeling_gpt2_test.py b/pytorch_transformers/tests/modeling_gpt2_test.py index 273200f24f..e8f2ff20d2 100644 --- a/pytorch_transformers/tests/modeling_gpt2_test.py +++ b/pytorch_transformers/tests/modeling_gpt2_test.py @@ -44,6 +44,7 @@ class GPT2ModelTest(CommonTestCases.CommonModelTester): seq_length=7, is_training=True, use_token_type_ids=True, + use_input_mask=True, use_labels=True, vocab_size=99, hidden_size=32, @@ -66,6 +67,7 @@ class GPT2ModelTest(CommonTestCases.CommonModelTester): self.seq_length = seq_length self.is_training = is_training self.use_token_type_ids = use_token_type_ids + self.use_input_mask = use_input_mask self.use_labels = use_labels self.vocab_size = vocab_size self.hidden_size = hidden_size @@ -86,6 +88,10 @@ class GPT2ModelTest(CommonTestCases.CommonModelTester): def prepare_config_and_inputs(self): input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) + input_mask = None + if self.use_input_mask: + input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2) + token_type_ids = None if self.use_token_type_ids: token_type_ids = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size) @@ -115,14 +121,14 @@ class GPT2ModelTest(CommonTestCases.CommonModelTester): head_mask = ids_tensor([self.num_hidden_layers, self.num_attention_heads], 2) - return config, input_ids, head_mask, token_type_ids, sequence_labels, token_labels, choice_labels + return config, input_ids, input_mask, head_mask, token_type_ids, sequence_labels, token_labels, choice_labels def check_loss_output(self, result): self.parent.assertListEqual( list(result["loss"].size()), []) - def create_and_check_gpt2_model(self, config, input_ids, head_mask, token_type_ids, *args): + def create_and_check_gpt2_model(self, config, input_ids, input_mask, head_mask, token_type_ids, *args): model = GPT2Model(config=config) model.eval() @@ -139,7 +145,7 @@ class GPT2ModelTest(CommonTestCases.CommonModelTester): [self.batch_size, self.seq_length, self.hidden_size]) self.parent.assertEqual(len(result["presents"]), config.n_layer) - def create_and_check_lm_head_model(self, config, input_ids, head_mask, token_type_ids, *args): + def create_and_check_lm_head_model(self, config, input_ids, input_mask, head_mask, token_type_ids, *args): model = GPT2LMHeadModel(config) model.eval() @@ -157,7 +163,7 @@ class GPT2ModelTest(CommonTestCases.CommonModelTester): list(result["lm_logits"].size()), [self.batch_size, self.seq_length, self.vocab_size]) - def create_and_check_double_lm_head_model(self, config, input_ids, head_mask, token_type_ids, *args): + def create_and_check_double_lm_head_model(self, config, input_ids, input_mask, head_mask, token_type_ids, *args): model = GPT2DoubleHeadsModel(config) model.eval() @@ -177,7 +183,7 @@ class GPT2ModelTest(CommonTestCases.CommonModelTester): def prepare_config_and_inputs_for_common(self): config_and_inputs = self.prepare_config_and_inputs() - (config, input_ids, head_mask, token_type_ids, sequence_labels, token_labels, choice_labels) = config_and_inputs + (config, input_ids, input_mask, head_mask, token_type_ids, sequence_labels, token_labels, choice_labels) = config_and_inputs inputs_dict = { 'input_ids': input_ids, 'token_type_ids': token_type_ids, diff --git a/pytorch_transformers/tests/modeling_tf_gpt2_test.py b/pytorch_transformers/tests/modeling_tf_gpt2_test.py new file mode 100644 index 0000000000..2710488169 --- /dev/null +++ b/pytorch_transformers/tests/modeling_tf_gpt2_test.py @@ -0,0 +1,216 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import unittest +import shutil +import pytest +import sys + +from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) +from .configuration_common_test import ConfigTester + +from pytorch_transformers import GPT2Config, is_tf_available + +try: + import tensorflow as tf + from pytorch_transformers.modeling_tf_gpt2 import (TFGPT2Model, TFGPT2LMHeadModel, + TFGPT2DoubleHeadsModel, + TF_GPT2_PRETRAINED_MODEL_ARCHIVE_MAP) +except ImportError: + pytestmark = pytest.mark.skip("Require TensorFlow") + + +class TFGPT2ModelTest(TFCommonTestCases.TFCommonModelTester): + + all_model_classes = (TFGPT2Model, TFGPT2LMHeadModel, + TFGPT2DoubleHeadsModel) if is_tf_available() else () + + class TFGPT2ModelTester(object): + + def __init__(self, + parent, + batch_size=13, + seq_length=7, + is_training=True, + use_token_type_ids=True, + use_input_mask=True, + use_labels=True, + vocab_size=99, + hidden_size=32, + num_hidden_layers=5, + num_attention_heads=4, + intermediate_size=37, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + type_sequence_label_size=2, + initializer_range=0.02, + num_labels=3, + num_choices=4, + scope=None, + ): + self.parent = parent + self.batch_size = batch_size + self.seq_length = seq_length + self.is_training = is_training + self.use_token_type_ids = use_token_type_ids + self.use_input_mask = use_input_mask + self.use_labels = use_labels + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.intermediate_size = intermediate_size + self.hidden_act = hidden_act + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.type_sequence_label_size = type_sequence_label_size + self.initializer_range = initializer_range + self.num_labels = num_labels + self.num_choices = num_choices + self.scope = scope + + def prepare_config_and_inputs(self): + input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) + + input_mask = None + if self.use_input_mask: + input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2) + + token_type_ids = None + if self.use_token_type_ids: + token_type_ids = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size) + + sequence_labels = None + token_labels = None + choice_labels = None + if self.use_labels: + sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size) + token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels) + choice_labels = ids_tensor([self.batch_size], self.num_choices) + + config = GPT2Config( + vocab_size_or_config_json_file=self.vocab_size, + n_embd=self.hidden_size, + n_layer=self.num_hidden_layers, + n_head=self.num_attention_heads, + # intermediate_size=self.intermediate_size, + # hidden_act=self.hidden_act, + # hidden_dropout_prob=self.hidden_dropout_prob, + # attention_probs_dropout_prob=self.attention_probs_dropout_prob, + n_positions=self.max_position_embeddings, + n_ctx=self.max_position_embeddings + # type_vocab_size=self.type_vocab_size, + # initializer_range=self.initializer_range + ) + + head_mask = ids_tensor([self.num_hidden_layers, self.num_attention_heads], 2) + + return config, input_ids, input_mask, head_mask, token_type_ids, sequence_labels, token_labels, choice_labels + + def create_and_check_gpt2_model(self, config, input_ids, input_mask, head_mask, token_type_ids, *args): + model = TFGPT2Model(config=config) + inputs = {'input_ids': input_ids, + 'attention_mask': input_mask, + 'token_type_ids': token_type_ids} + sequence_output = model(inputs)[0] + + inputs = [input_ids, None, input_mask] # None is the input for 'past' + sequence_output = model(inputs)[0] + + sequence_output = model(input_ids)[0] + + result = { + "sequence_output": sequence_output.numpy(), + } + self.parent.assertListEqual( + list(result["sequence_output"].shape), + [self.batch_size, self.seq_length, self.hidden_size]) + + + def create_and_check_gpt2_lm_head(self, config, input_ids, input_mask, head_mask, token_type_ids, *args): + model = TFGPT2LMHeadModel(config=config) + inputs = {'input_ids': input_ids, + 'attention_mask': input_mask, + 'token_type_ids': token_type_ids} + prediction_scores = model(inputs)[0] + result = { + "prediction_scores": prediction_scores.numpy(), + } + self.parent.assertListEqual( + list(result["prediction_scores"].shape), + [self.batch_size, self.seq_length, self.vocab_size]) + + + def create_and_check_gpt2_double_head(self, config, input_ids, input_mask, head_mask, token_type_ids, *args): + pass + # model = TFGPT2DoubleHeadsModel(config=config) + # inputs = {'input_ids': input_ids, + # 'attention_mask': input_mask, + # 'token_type_ids': token_type_ids} + # seq_relationship_score, = model(inputs)[0] + # result = { + # "seq_relationship_score": seq_relationship_score.numpy(), + # } + # self.parent.assertListEqual( + # list(result["seq_relationship_score"].shape), + # [self.batch_size, 2]) + + def prepare_config_and_inputs_for_common(self): + config_and_inputs = self.prepare_config_and_inputs() + + (config, input_ids, input_mask, head_mask, token_type_ids, + sequence_labels, token_labels, choice_labels) = config_and_inputs + + inputs_dict = {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask} + return config, inputs_dict + + def setUp(self): + self.model_tester = TFGPT2ModelTest.TFGPT2ModelTester(self) + self.config_tester = ConfigTester(self, config_class=GPT2Config, hidden_size=37) + + def test_config(self): + self.config_tester.run_common_tests() + + def test_gpt2_model(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_gpt2_model(*config_and_inputs) + + def test_gpt2_lm_head(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_gpt2_lm_head(*config_and_inputs) + + def test_gpt2_double_head(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_gpt2_double_head(*config_and_inputs) + + @pytest.mark.slow + def test_model_from_pretrained(self): + cache_dir = "/tmp/pytorch_transformers_test/" + for model_name in list(TF_gpt2_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: + model = TFGPT2Model.from_pretrained(model_name, cache_dir=cache_dir) + shutil.rmtree(cache_dir) + self.assertIsNotNone(model) + +if __name__ == "__main__": + unittest.main() + From b7175a27018425d46a01b9e5a0595e6e9b1ab6a1 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Mon, 9 Sep 2019 11:04:03 +0200 Subject: [PATCH 033/219] fixed imports in tests and gpt2 config test --- pytorch_transformers/__init__.py | 2 +- pytorch_transformers/modeling_tf_gpt2.py | 1 + pytorch_transformers/modeling_tf_utils.py | 5 +++-- pytorch_transformers/modeling_xlnet.py | 6 ++++-- pytorch_transformers/tests/modeling_auto_test.py | 6 ++++-- pytorch_transformers/tests/modeling_bert_test.py | 4 ++-- pytorch_transformers/tests/modeling_common_test.py | 6 ++++-- .../tests/modeling_distilbert_test.py | 4 ++-- pytorch_transformers/tests/modeling_gpt2_test.py | 4 ++-- pytorch_transformers/tests/modeling_openai_test.py | 4 ++-- pytorch_transformers/tests/modeling_roberta_test.py | 4 ++-- pytorch_transformers/tests/modeling_tf_auto_test.py | 6 ++++-- pytorch_transformers/tests/modeling_tf_bert_test.py | 4 ++-- pytorch_transformers/tests/modeling_tf_common_test.py | 6 ++++-- pytorch_transformers/tests/modeling_tf_gpt2_test.py | 11 ++++++----- .../tests/modeling_transfo_xl_test.py | 4 ++-- pytorch_transformers/tests/modeling_xlm_test.py | 4 ++-- pytorch_transformers/tests/modeling_xlnet_test.py | 4 ++-- pytorch_transformers/tests/optimization_test.py | 4 ++-- .../tests/tokenization_transfo_xl_test.py | 4 ++-- 20 files changed, 53 insertions(+), 40 deletions(-) diff --git a/pytorch_transformers/__init__.py b/pytorch_transformers/__init__.py index f13457f073..7bc7ddf7e1 100644 --- a/pytorch_transformers/__init__.py +++ b/pytorch_transformers/__init__.py @@ -89,7 +89,7 @@ try: import tensorflow as tf assert int(tf.__version__[0]) >= 2 _tf_available = True # pylint: disable=invalid-name -except ImportError: +except (ImportError, AssertionError): _tf_available = False # pylint: disable=invalid-name if _tf_available: diff --git a/pytorch_transformers/modeling_tf_gpt2.py b/pytorch_transformers/modeling_tf_gpt2.py index 27e51fb752..bcb9f5309a 100644 --- a/pytorch_transformers/modeling_tf_gpt2.py +++ b/pytorch_transformers/modeling_tf_gpt2.py @@ -699,6 +699,7 @@ class TFGPT2DoubleHeadsModel(TFGPT2PreTrainedModel): head_mask = inputs.get('head_mask', None) assert len(inputs) <= 5, "Too many inputs." + assert len(shape_list(input_ids)) == 3, "Inputs should have 3 dimensions: batch, choices, sequence length" num_choices = shape_list(input_ids)[1] seq_length = shape_list(input_ids)[2] diff --git a/pytorch_transformers/modeling_tf_utils.py b/pytorch_transformers/modeling_tf_utils.py index 9dfd42c36b..af67a8442e 100644 --- a/pytorch_transformers/modeling_tf_utils.py +++ b/pytorch_transformers/modeling_tf_utils.py @@ -313,7 +313,7 @@ class TFSequenceSummary(tf.keras.layers.Layer): # We can probably just use the multi-head attention module of PyTorch >=1.1.0 raise NotImplementedError - self.summary = tf.keras.layers.Identity(name='summary') + self.summary = None if hasattr(config, 'summary_use_proj') and config.summary_use_proj: if hasattr(config, 'summary_proj_to_labels') and config.summary_proj_to_labels and config.num_labels > 0: num_classes = config.num_labels @@ -372,7 +372,8 @@ class TFSequenceSummary(tf.keras.layers.Layer): if training and self.first_dropout is not None: output = self.first_dropout(output) - output = self.summary(output) + if self.summary is not None: + output = self.summary(output) if self.activation is not None: output = self.activation(output) diff --git a/pytorch_transformers/modeling_xlnet.py b/pytorch_transformers/modeling_xlnet.py index 00c15080a1..97feaad371 100644 --- a/pytorch_transformers/modeling_xlnet.py +++ b/pytorch_transformers/modeling_xlnet.py @@ -525,8 +525,10 @@ XLNET_INPUTS_DOCSTRING = r""" Only used during pretraining for partial prediction or for sequential decoding (generation). **token_type_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: A parallel sequence of tokens (can be used to indicate various portions of the inputs). - The embeddings from these tokens will be summed with the respective token embeddings. - Indices are selected in the vocabulary (unlike BERT which has a specific vocabulary for segment indices). + The type indices in XLNet are NOT selected in the vocabulary, they can be arbitrary numbers and + the important thing is that they should be different for tokens which belong to different segments. + The model will compute relative segment differences from the given type indices: + 0 if the segment id of two tokens are the same, 1 if not. **input_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``: Mask to avoid performing attention on padding token indices. Negative of `attention_mask`, i.e. with 0 for real tokens and 1 for padding. diff --git a/pytorch_transformers/tests/modeling_auto_test.py b/pytorch_transformers/tests/modeling_auto_test.py index 169f722ed7..4b00891c38 100644 --- a/pytorch_transformers/tests/modeling_auto_test.py +++ b/pytorch_transformers/tests/modeling_auto_test.py @@ -21,7 +21,9 @@ import shutil import pytest import logging -try: +from pytorch_transformers import is_torch_available + +if is_torch_available(): from pytorch_transformers import (AutoConfig, BertConfig, AutoModel, BertModel, AutoModelWithLMHead, BertForMaskedLM, @@ -31,7 +33,7 @@ try: from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester -except ImportError: +else: pytestmark = pytest.mark.skip("Require Torch") diff --git a/pytorch_transformers/tests/modeling_bert_test.py b/pytorch_transformers/tests/modeling_bert_test.py index d63d1b407c..cac1f996e9 100644 --- a/pytorch_transformers/tests/modeling_bert_test.py +++ b/pytorch_transformers/tests/modeling_bert_test.py @@ -25,13 +25,13 @@ from pytorch_transformers import is_torch_available from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester -try: +if is_torch_available(): from pytorch_transformers import (BertConfig, BertModel, BertForMaskedLM, BertForNextSentencePrediction, BertForPreTraining, BertForQuestionAnswering, BertForSequenceClassification, BertForTokenClassification, BertForMultipleChoice) from pytorch_transformers.modeling_bert import BERT_PRETRAINED_MODEL_ARCHIVE_MAP -except ImportError: +else: pytestmark = pytest.mark.skip("Require Torch") diff --git a/pytorch_transformers/tests/modeling_common_test.py b/pytorch_transformers/tests/modeling_common_test.py index 1f778d608f..a7d56041a3 100644 --- a/pytorch_transformers/tests/modeling_common_test.py +++ b/pytorch_transformers/tests/modeling_common_test.py @@ -27,13 +27,15 @@ import unittest import logging import pytest -try: +from pytorch_transformers import is_torch_available + +if is_torch_available(): import torch from pytorch_transformers import (PretrainedConfig, PreTrainedModel, BertModel, BertConfig, BERT_PRETRAINED_MODEL_ARCHIVE_MAP, GPT2LMHeadModel, GPT2Config, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP) -except ImportError: +else: pytestmark = pytest.mark.skip("Require Torch") diff --git a/pytorch_transformers/tests/modeling_distilbert_test.py b/pytorch_transformers/tests/modeling_distilbert_test.py index 10bb4bb398..8fef9d5833 100644 --- a/pytorch_transformers/tests/modeling_distilbert_test.py +++ b/pytorch_transformers/tests/modeling_distilbert_test.py @@ -21,10 +21,10 @@ import pytest from pytorch_transformers import is_torch_available -try: +if is_torch_available(): from pytorch_transformers import (DistilBertConfig, DistilBertModel, DistilBertForMaskedLM, DistilBertForQuestionAnswering, DistilBertForSequenceClassification) -except ImportError: +else: pytestmark = pytest.mark.skip("Require Torch") from .modeling_common_test import (CommonTestCases, ids_tensor) diff --git a/pytorch_transformers/tests/modeling_gpt2_test.py b/pytorch_transformers/tests/modeling_gpt2_test.py index e8f2ff20d2..dc7c0d1816 100644 --- a/pytorch_transformers/tests/modeling_gpt2_test.py +++ b/pytorch_transformers/tests/modeling_gpt2_test.py @@ -22,10 +22,10 @@ import shutil from pytorch_transformers import is_torch_available -try: +if is_torch_available(): from pytorch_transformers import (GPT2Config, GPT2Model, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP, GPT2LMHeadModel, GPT2DoubleHeadsModel) -except ImportError: +else: pytestmark = pytest.mark.skip("Require Torch") from .modeling_common_test import (CommonTestCases, ids_tensor) diff --git a/pytorch_transformers/tests/modeling_openai_test.py b/pytorch_transformers/tests/modeling_openai_test.py index b89990a181..6df4406d03 100644 --- a/pytorch_transformers/tests/modeling_openai_test.py +++ b/pytorch_transformers/tests/modeling_openai_test.py @@ -22,10 +22,10 @@ import shutil from pytorch_transformers import is_torch_available -try: +if is_torch_available(): from pytorch_transformers import (OpenAIGPTConfig, OpenAIGPTModel, OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP, OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel) -except ImportError: +else: pytestmark = pytest.mark.skip("Require Torch") from .modeling_common_test import (CommonTestCases, ids_tensor) diff --git a/pytorch_transformers/tests/modeling_roberta_test.py b/pytorch_transformers/tests/modeling_roberta_test.py index ed0f8b5cdc..11f2893671 100644 --- a/pytorch_transformers/tests/modeling_roberta_test.py +++ b/pytorch_transformers/tests/modeling_roberta_test.py @@ -22,11 +22,11 @@ import pytest from pytorch_transformers import is_torch_available -try: +if is_torch_available(): import torch from pytorch_transformers import (RobertaConfig, RobertaModel, RobertaForMaskedLM, RobertaForSequenceClassification) from pytorch_transformers.modeling_roberta import ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP -except ImportError: +else: pytestmark = pytest.mark.skip("Require Torch") from .modeling_common_test import (CommonTestCases, ids_tensor) diff --git a/pytorch_transformers/tests/modeling_tf_auto_test.py b/pytorch_transformers/tests/modeling_tf_auto_test.py index 816d6c1b1a..4617ab817b 100644 --- a/pytorch_transformers/tests/modeling_tf_auto_test.py +++ b/pytorch_transformers/tests/modeling_tf_auto_test.py @@ -21,7 +21,9 @@ import shutil import pytest import logging -try: +from pytorch_transformers import is_tf_available + +if is_tf_available(): from pytorch_transformers import (AutoConfig, BertConfig, TFAutoModel, TFBertModel, TFAutoModelWithLMHead, TFBertForMaskedLM, @@ -31,7 +33,7 @@ try: from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester -except ImportError: +else: pytestmark = pytest.mark.skip("Require TensorFlow") diff --git a/pytorch_transformers/tests/modeling_tf_bert_test.py b/pytorch_transformers/tests/modeling_tf_bert_test.py index c95e33d780..55bbe36feb 100644 --- a/pytorch_transformers/tests/modeling_tf_bert_test.py +++ b/pytorch_transformers/tests/modeling_tf_bert_test.py @@ -26,7 +26,7 @@ from .configuration_common_test import ConfigTester from pytorch_transformers import BertConfig, is_tf_available -try: +if is_tf_available(): import tensorflow as tf from pytorch_transformers.modeling_tf_bert import (TFBertModel, TFBertForMaskedLM, TFBertForNextSentencePrediction, @@ -36,7 +36,7 @@ try: TFBertForTokenClassification, TFBertForQuestionAnswering, TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP) -except ImportError: +else: pytestmark = pytest.mark.skip("Require TensorFlow") diff --git a/pytorch_transformers/tests/modeling_tf_common_test.py b/pytorch_transformers/tests/modeling_tf_common_test.py index f9b87eed9a..da3263ffde 100644 --- a/pytorch_transformers/tests/modeling_tf_common_test.py +++ b/pytorch_transformers/tests/modeling_tf_common_test.py @@ -25,11 +25,13 @@ import uuid import pytest import sys -try: +from pytorch_transformers import is_tf_available + +if is_tf_available(): import tensorflow as tf from pytorch_transformers import TFPreTrainedModel # from pytorch_transformers.modeling_bert import BertModel, BertConfig, BERT_PRETRAINED_MODEL_ARCHIVE_MAP -except ImportError: +else: pytestmark = pytest.mark.skip("Require TensorFlow") diff --git a/pytorch_transformers/tests/modeling_tf_gpt2_test.py b/pytorch_transformers/tests/modeling_tf_gpt2_test.py index 2710488169..5fef1f6453 100644 --- a/pytorch_transformers/tests/modeling_tf_gpt2_test.py +++ b/pytorch_transformers/tests/modeling_tf_gpt2_test.py @@ -26,19 +26,20 @@ from .configuration_common_test import ConfigTester from pytorch_transformers import GPT2Config, is_tf_available -try: +if is_tf_available(): import tensorflow as tf from pytorch_transformers.modeling_tf_gpt2 import (TFGPT2Model, TFGPT2LMHeadModel, TFGPT2DoubleHeadsModel, TF_GPT2_PRETRAINED_MODEL_ARCHIVE_MAP) -except ImportError: +else: pytestmark = pytest.mark.skip("Require TensorFlow") class TFGPT2ModelTest(TFCommonTestCases.TFCommonModelTester): - all_model_classes = (TFGPT2Model, TFGPT2LMHeadModel, - TFGPT2DoubleHeadsModel) if is_tf_available() else () + # all_model_classes = (TFGPT2Model, TFGPT2LMHeadModel, + # TFGPT2DoubleHeadsModel) if is_tf_available() else () + all_model_classes = (TFGPT2Model, TFGPT2LMHeadModel) if is_tf_available() else () class TFGPT2ModelTester(object): @@ -186,7 +187,7 @@ class TFGPT2ModelTest(TFCommonTestCases.TFCommonModelTester): def setUp(self): self.model_tester = TFGPT2ModelTest.TFGPT2ModelTester(self) - self.config_tester = ConfigTester(self, config_class=GPT2Config, hidden_size=37) + self.config_tester = ConfigTester(self, config_class=GPT2Config, n_embd=37) def test_config(self): self.config_tester.run_common_tests() diff --git a/pytorch_transformers/tests/modeling_transfo_xl_test.py b/pytorch_transformers/tests/modeling_transfo_xl_test.py index 9a72335157..035f20991f 100644 --- a/pytorch_transformers/tests/modeling_transfo_xl_test.py +++ b/pytorch_transformers/tests/modeling_transfo_xl_test.py @@ -23,11 +23,11 @@ import pytest from pytorch_transformers import is_torch_available -try: +if is_torch_available(): import torch from pytorch_transformers import (TransfoXLConfig, TransfoXLModel, TransfoXLLMHeadModel) from pytorch_transformers.modeling_transfo_xl import TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP -except ImportError: +else: pytestmark = pytest.mark.skip("Require Torch") from .modeling_common_test import (CommonTestCases, ids_tensor) diff --git a/pytorch_transformers/tests/modeling_xlm_test.py b/pytorch_transformers/tests/modeling_xlm_test.py index 21cf624b9b..acc8a68066 100644 --- a/pytorch_transformers/tests/modeling_xlm_test.py +++ b/pytorch_transformers/tests/modeling_xlm_test.py @@ -22,11 +22,11 @@ import pytest from pytorch_transformers import is_torch_available -try: +if is_torch_available(): from pytorch_transformers import (XLMConfig, XLMModel, XLMWithLMHeadModel, XLMForQuestionAnswering, XLMForSequenceClassification) from pytorch_transformers.modeling_xlm import XLM_PRETRAINED_MODEL_ARCHIVE_MAP -except ImportError: +else: pytestmark = pytest.mark.skip("Require Torch") from .modeling_common_test import (CommonTestCases, ids_tensor) diff --git a/pytorch_transformers/tests/modeling_xlnet_test.py b/pytorch_transformers/tests/modeling_xlnet_test.py index b280ed4592..8c8b3f964f 100644 --- a/pytorch_transformers/tests/modeling_xlnet_test.py +++ b/pytorch_transformers/tests/modeling_xlnet_test.py @@ -25,12 +25,12 @@ import pytest from pytorch_transformers import is_torch_available -try: +if is_torch_available(): import torch from pytorch_transformers import (XLNetConfig, XLNetModel, XLNetLMHeadModel, XLNetForSequenceClassification, XLNetForQuestionAnswering) from pytorch_transformers.modeling_xlnet import XLNET_PRETRAINED_MODEL_ARCHIVE_MAP -except ImportError: +else: pytestmark = pytest.mark.skip("Require Torch") from .modeling_common_test import (CommonTestCases, ids_tensor) diff --git a/pytorch_transformers/tests/optimization_test.py b/pytorch_transformers/tests/optimization_test.py index 07dc22141d..c1c6270f32 100644 --- a/pytorch_transformers/tests/optimization_test.py +++ b/pytorch_transformers/tests/optimization_test.py @@ -22,12 +22,12 @@ import pytest from pytorch_transformers import is_torch_available -try: +if is_torch_available(): import torch from pytorch_transformers import (AdamW, ConstantLRSchedule, WarmupConstantSchedule, WarmupCosineSchedule, WarmupCosineWithHardRestartsSchedule, WarmupLinearSchedule) -except ImportError: +else: pytestmark = pytest.mark.skip("Require Torch") from .tokenization_tests_commons import TemporaryDirectory diff --git a/pytorch_transformers/tests/tokenization_transfo_xl_test.py b/pytorch_transformers/tests/tokenization_transfo_xl_test.py index 792033d82c..563871f690 100644 --- a/pytorch_transformers/tests/tokenization_transfo_xl_test.py +++ b/pytorch_transformers/tests/tokenization_transfo_xl_test.py @@ -21,10 +21,10 @@ from io import open from pytorch_transformers import is_torch_available -try: +if is_torch_available(): import torch from pytorch_transformers.tokenization_transfo_xl import TransfoXLTokenizer, VOCAB_FILES_NAMES -except ImportError: +else: pytestmark = pytest.mark.skip("Require Torch") # TODO: untangle Transfo-XL tokenizer from torch.load and torch.save from .tokenization_tests_commons import CommonTestCases From 6b3438df21782fe5736b267c05415f64a26a65d9 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Mon, 9 Sep 2019 12:48:36 +0200 Subject: [PATCH 034/219] fixing GPT2 double head model and updating the torch version tests --- pytorch_transformers/modeling_gpt2.py | 13 +++-- pytorch_transformers/modeling_tf_gpt2.py | 29 ++++++----- pytorch_transformers/modeling_tf_utils.py | 15 ++++-- pytorch_transformers/modeling_utils.py | 2 +- .../tests/modeling_gpt2_test.py | 36 ++++++++++--- .../tests/modeling_tf_gpt2_test.py | 51 ++++++++++++------- 6 files changed, 98 insertions(+), 48 deletions(-) diff --git a/pytorch_transformers/modeling_gpt2.py b/pytorch_transformers/modeling_gpt2.py index 4268641187..324d020fc3 100644 --- a/pytorch_transformers/modeling_gpt2.py +++ b/pytorch_transformers/modeling_gpt2.py @@ -367,6 +367,13 @@ class GPT2Model(GPT2PreTrainedModel): self.h[layer].attn.prune_heads(heads) def forward(self, input_ids, past=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None): + input_shape = input_ids.size() + input_ids = input_ids.view(-1, input_shape[-1]) + if token_type_ids is not None: + token_type_ids = token_type_ids.view(-1, input_shape[-1]) + if position_ids is not None: + position_ids = position_ids.view(-1, input_shape[-1]) + if past is None: past_length = 0 past = [None] * len(self.h) @@ -378,6 +385,7 @@ class GPT2Model(GPT2PreTrainedModel): # Attention mask. if attention_mask is not None: + attention_mask = attention_mask.view(-1, input_shape[-1]) # We create a 3D attention mask from a 2D tensor mask. # Sizes are [batch_size, 1, 1, to_seq_length] # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length] @@ -407,14 +415,9 @@ class GPT2Model(GPT2PreTrainedModel): else: head_mask = [None] * self.config.n_layer - input_shape = input_ids.size() - input_ids = input_ids.view(-1, input_ids.size(-1)) - position_ids = position_ids.view(-1, position_ids.size(-1)) - inputs_embeds = self.wte(input_ids) position_embeds = self.wpe(position_ids) if token_type_ids is not None: - token_type_ids = token_type_ids.view(-1, token_type_ids.size(-1)) token_type_embeds = self.wte(token_type_ids) else: token_type_embeds = 0 diff --git a/pytorch_transformers/modeling_tf_gpt2.py b/pytorch_transformers/modeling_tf_gpt2.py index bcb9f5309a..85873c9d1b 100644 --- a/pytorch_transformers/modeling_tf_gpt2.py +++ b/pytorch_transformers/modeling_tf_gpt2.py @@ -314,17 +314,16 @@ class TFGPT2Embeddings(tf.keras.layers.Layer): def _linear(self, inputs): """Computes logits by running inputs through a linear layer. Args: - inputs: A float32 tensor with shape [batch_size, length, hidden_size] + inputs: A float32 tensor with shape [..., hidden_size] Returns: - float32 tensor with shape [batch_size, length, vocab_size]. + float32 tensor with shape [..., vocab_size]. """ - batch_size = shape_list(inputs)[0] - length = shape_list(inputs)[1] + first_dims = shape_list(inputs)[:-1] x = tf.reshape(inputs, [-1, self.hidden_size]) logits = tf.matmul(x, self.weight, transpose_b=True) - return tf.reshape(logits, [batch_size, length, self.vocab_size]) + return tf.reshape(logits, first_dims + [self.vocab_size]) class TFGPT2MainLayer(tf.keras.layers.Layer): def __init__(self, config, *inputs, **kwargs): @@ -679,10 +678,11 @@ class TFGPT2DoubleHeadsModel(TFGPT2PreTrainedModel): @tf.function def call(self, inputs, training=False): if not isinstance(inputs, (dict, tuple, list)): - raise ValueError("Inputs should be a list or a dict with at least two elements: 'inputs_ids' and 'mc_token_ids'") + input_ids = inputs + mc_token_ids, past, attention_mask, token_type_ids, position_ids, head_mask = None, None, None, None, None elif isinstance(inputs, (tuple, list)): input_ids = inputs[0] - mc_token_ids = inputs[1] + mc_token_ids = inputs[1] if len(inputs) > 1 else None past = inputs[2] if len(inputs) > 2 else None attention_mask = inputs[3] if len(inputs) > 3 else None token_type_ids = inputs[4] if len(inputs) > 4 else None @@ -691,7 +691,7 @@ class TFGPT2DoubleHeadsModel(TFGPT2PreTrainedModel): assert len(inputs) <= 7, "Too many inputs." else: input_ids = inputs.get('input_ids') - mc_token_ids = inputs.get('mc_token_ids') + mc_token_ids = inputs.get('mc_token_ids', None) past = inputs.get('past', None) attention_mask = inputs.get('attention_mask', None) token_type_ids = inputs.get('token_type_ids', None) @@ -699,9 +699,9 @@ class TFGPT2DoubleHeadsModel(TFGPT2PreTrainedModel): head_mask = inputs.get('head_mask', None) assert len(inputs) <= 5, "Too many inputs." - assert len(shape_list(input_ids)) == 3, "Inputs should have 3 dimensions: batch, choices, sequence length" - num_choices = shape_list(input_ids)[1] - seq_length = shape_list(input_ids)[2] + input_shapes = shape_list(input_ids) + + seq_length = input_shapes[-1] flat_input_ids = tf.reshape(input_ids, (-1, seq_length)) flat_attention_mask = tf.reshape(attention_mask, (-1, seq_length)) if attention_mask is not None else None @@ -710,13 +710,16 @@ class TFGPT2DoubleHeadsModel(TFGPT2PreTrainedModel): flat_inputs = [flat_input_ids, past, flat_attention_mask, flat_token_type_ids, flat_position_ids, head_mask] - outputs = self.transformer(flat_inputs, training=training) - + transformer_outputs = self.transformer(flat_inputs, training=training) hidden_states = transformer_outputs[0] + hidden_states = tf.reshape(hidden_states, input_shapes + shape_list(hidden_states)[-1:]) + lm_logits = self.transformer.wte(hidden_states, mode="linear") mc_logits = self.multiple_choice_head([hidden_states, mc_token_ids], training=training) + mc_logits = tf.squeeze(mc_logits, axis=-1) + outputs = (lm_logits, mc_logits) + transformer_outputs[1:] return outputs # (lm loss), (mc loss), lm logits, mc logits, presents, (all hidden_states), (attentions) diff --git a/pytorch_transformers/modeling_tf_utils.py b/pytorch_transformers/modeling_tf_utils.py index af67a8442e..9630e27478 100644 --- a/pytorch_transformers/modeling_tf_utils.py +++ b/pytorch_transformers/modeling_tf_utils.py @@ -359,13 +359,18 @@ class TFSequenceSummary(tf.keras.layers.Layer): elif self.summary_type == 'mean': output = tf.mean(hidden_states, axis=1) elif self.summary_type == 'cls_index': + hidden_shape = shape_list(hidden_states) # e.g. [batch, num choices, seq length, hidden dims] if cls_index is None: - cls_index = tf.fill(tf.shape(hidden_states[..., :1, :]), hidden_states.shape[-2]-1, dtype=tf.int32) - else: - cls_index = cls_index[..., tf.newaxis, tf.newaxis] - cls_index = cls_index.expand((-1,) * (cls_index.dim()-1) + (hidden_states.size(-1),)) + cls_index = tf.fill(hidden_shape[:-2], hidden_shape[-2] - 1) # A tensor full of shape [batch] or [batch, num choices] full of sequence length + cls_shape = shape_list(cls_index) + if len(cls_shape) <= len(hidden_shape) - 2: + cls_index = cls_index[..., tf.newaxis] + # else: + # cls_index = cls_index[..., tf.newaxis] + # cls_index = cls_index.expand((-1,) * (cls_index.dim()-1) + (hidden_states.size(-1),)) # shape of cls_index: (bsz, XX, 1, hidden_size) where XX are optional leading dim of hidden_states - output = hidden_states.gather(-2, cls_index).squeeze(-2) # shape (bsz, XX, hidden_size) + output = tf.gather(hidden_states, cls_index, batch_dims=len(hidden_shape) - 2) + output = tf.squeeze(output, axis=len(hidden_shape) - 2) # shape of output: (batch, num choices, hidden_size) elif self.summary_type == 'attn': raise NotImplementedError diff --git a/pytorch_transformers/modeling_utils.py b/pytorch_transformers/modeling_utils.py index 2fb4671674..9fd7a2c0c2 100644 --- a/pytorch_transformers/modeling_utils.py +++ b/pytorch_transformers/modeling_utils.py @@ -679,7 +679,7 @@ class SequenceSummary(nn.Module): self.last_dropout = nn.Dropout(config.summary_last_dropout) def forward(self, hidden_states, cls_index=None): - """ hidden_states: float Tensor in shape [bsz, seq_len, hidden_size], the hidden-states of the last layer. + """ hidden_states: float Tensor in shape [bsz, ..., seq_len, hidden_size], the hidden-states of the last layer. cls_index: [optional] position of the classification token if summary_type == 'cls_index', shape (bsz,) or more generally (bsz, ...) where ... are optional leading dimensions of hidden_states. if summary_type == 'cls_index' and cls_index is None: diff --git a/pytorch_transformers/tests/modeling_gpt2_test.py b/pytorch_transformers/tests/modeling_gpt2_test.py index dc7c0d1816..e5accfa8cf 100644 --- a/pytorch_transformers/tests/modeling_gpt2_test.py +++ b/pytorch_transformers/tests/modeling_gpt2_test.py @@ -46,6 +46,7 @@ class GPT2ModelTest(CommonTestCases.CommonModelTester): use_token_type_ids=True, use_input_mask=True, use_labels=True, + use_mc_token_ids=True, vocab_size=99, hidden_size=32, num_hidden_layers=5, @@ -69,6 +70,7 @@ class GPT2ModelTest(CommonTestCases.CommonModelTester): self.use_token_type_ids = use_token_type_ids self.use_input_mask = use_input_mask self.use_labels = use_labels + self.use_mc_token_ids = use_mc_token_ids self.vocab_size = vocab_size self.hidden_size = hidden_size self.num_hidden_layers = num_hidden_layers @@ -96,6 +98,10 @@ class GPT2ModelTest(CommonTestCases.CommonModelTester): if self.use_token_type_ids: token_type_ids = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size) + mc_token_ids = None + if self.use_mc_token_ids: + mc_token_ids = ids_tensor([self.batch_size, self.num_choices], self.seq_length) + sequence_labels = None token_labels = None choice_labels = None @@ -121,7 +127,7 @@ class GPT2ModelTest(CommonTestCases.CommonModelTester): head_mask = ids_tensor([self.num_hidden_layers, self.num_attention_heads], 2) - return config, input_ids, input_mask, head_mask, token_type_ids, sequence_labels, token_labels, choice_labels + return config, input_ids, input_mask, head_mask, token_type_ids, mc_token_ids, sequence_labels, token_labels, choice_labels def check_loss_output(self, result): self.parent.assertListEqual( @@ -163,15 +169,27 @@ class GPT2ModelTest(CommonTestCases.CommonModelTester): list(result["lm_logits"].size()), [self.batch_size, self.seq_length, self.vocab_size]) - def create_and_check_double_lm_head_model(self, config, input_ids, input_mask, head_mask, token_type_ids, *args): + def create_and_check_double_lm_head_model(self, config, input_ids, input_mask, head_mask, token_type_ids, mc_token_ids, *args): model = GPT2DoubleHeadsModel(config) model.eval() - loss, lm_logits, mc_logits, _ = model(input_ids, token_type_ids=token_type_ids, lm_labels=input_ids) + + multiple_choice_inputs_ids = input_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous() + multiple_choice_input_mask = input_mask.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous() + multiple_choice_token_type_ids = token_type_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous() + + inputs = {'input_ids': multiple_choice_inputs_ids, + 'mc_token_ids': mc_token_ids, + 'attention_mask': multiple_choice_input_mask, + 'token_type_ids': multiple_choice_token_type_ids, + 'lm_labels': multiple_choice_inputs_ids} + + loss, lm_logits, mc_logits, _ = model(**inputs) result = { "loss": loss, - "lm_logits": lm_logits + "lm_logits": lm_logits, + "mc_logits": mc_logits } self.parent.assertListEqual( @@ -179,11 +197,17 @@ class GPT2ModelTest(CommonTestCases.CommonModelTester): []) self.parent.assertListEqual( list(result["lm_logits"].size()), - [self.batch_size, self.seq_length, self.vocab_size]) + [self.batch_size, self.num_choices, self.seq_length, self.vocab_size]) + self.parent.assertListEqual( + list(result["mc_logits"].size()), + [self.batch_size, self.num_choices]) def prepare_config_and_inputs_for_common(self): config_and_inputs = self.prepare_config_and_inputs() - (config, input_ids, input_mask, head_mask, token_type_ids, sequence_labels, token_labels, choice_labels) = config_and_inputs + + (config, input_ids, input_mask, head_mask, token_type_ids, + mc_token_ids, sequence_labels, token_labels, choice_labels) = config_and_inputs + inputs_dict = { 'input_ids': input_ids, 'token_type_ids': token_type_ids, diff --git a/pytorch_transformers/tests/modeling_tf_gpt2_test.py b/pytorch_transformers/tests/modeling_tf_gpt2_test.py index 5fef1f6453..490d5c4e32 100644 --- a/pytorch_transformers/tests/modeling_tf_gpt2_test.py +++ b/pytorch_transformers/tests/modeling_tf_gpt2_test.py @@ -37,9 +37,9 @@ else: class TFGPT2ModelTest(TFCommonTestCases.TFCommonModelTester): - # all_model_classes = (TFGPT2Model, TFGPT2LMHeadModel, - # TFGPT2DoubleHeadsModel) if is_tf_available() else () - all_model_classes = (TFGPT2Model, TFGPT2LMHeadModel) if is_tf_available() else () + all_model_classes = (TFGPT2Model, TFGPT2LMHeadModel, + TFGPT2DoubleHeadsModel) if is_tf_available() else () + # all_model_classes = (TFGPT2Model, TFGPT2LMHeadModel) if is_tf_available() else () class TFGPT2ModelTester(object): @@ -51,6 +51,7 @@ class TFGPT2ModelTest(TFCommonTestCases.TFCommonModelTester): use_token_type_ids=True, use_input_mask=True, use_labels=True, + use_mc_token_ids=True, vocab_size=99, hidden_size=32, num_hidden_layers=5, @@ -74,6 +75,7 @@ class TFGPT2ModelTest(TFCommonTestCases.TFCommonModelTester): self.use_token_type_ids = use_token_type_ids self.use_input_mask = use_input_mask self.use_labels = use_labels + self.use_mc_token_ids = use_mc_token_ids self.vocab_size = vocab_size self.hidden_size = hidden_size self.num_hidden_layers = num_hidden_layers @@ -101,6 +103,10 @@ class TFGPT2ModelTest(TFCommonTestCases.TFCommonModelTester): if self.use_token_type_ids: token_type_ids = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size) + mc_token_ids = None + if self.use_mc_token_ids: + mc_token_ids = ids_tensor([self.batch_size, self.num_choices], self.seq_length) + sequence_labels = None token_labels = None choice_labels = None @@ -126,7 +132,7 @@ class TFGPT2ModelTest(TFCommonTestCases.TFCommonModelTester): head_mask = ids_tensor([self.num_hidden_layers, self.num_attention_heads], 2) - return config, input_ids, input_mask, head_mask, token_type_ids, sequence_labels, token_labels, choice_labels + return config, input_ids, input_mask, head_mask, token_type_ids, mc_token_ids, sequence_labels, token_labels, choice_labels def create_and_check_gpt2_model(self, config, input_ids, input_mask, head_mask, token_type_ids, *args): model = TFGPT2Model(config=config) @@ -162,25 +168,34 @@ class TFGPT2ModelTest(TFCommonTestCases.TFCommonModelTester): [self.batch_size, self.seq_length, self.vocab_size]) - def create_and_check_gpt2_double_head(self, config, input_ids, input_mask, head_mask, token_type_ids, *args): - pass - # model = TFGPT2DoubleHeadsModel(config=config) - # inputs = {'input_ids': input_ids, - # 'attention_mask': input_mask, - # 'token_type_ids': token_type_ids} - # seq_relationship_score, = model(inputs)[0] - # result = { - # "seq_relationship_score": seq_relationship_score.numpy(), - # } - # self.parent.assertListEqual( - # list(result["seq_relationship_score"].shape), - # [self.batch_size, 2]) + def create_and_check_gpt2_double_head(self, config, input_ids, input_mask, head_mask, token_type_ids, mc_token_ids, *args): + model = TFGPT2DoubleHeadsModel(config=config) + + multiple_choice_inputs_ids = tf.tile(tf.expand_dims(input_ids, 1), (1, self.num_choices, 1)) + multiple_choice_input_mask = tf.tile(tf.expand_dims(input_mask, 1), (1, self.num_choices, 1)) + multiple_choice_token_type_ids = tf.tile(tf.expand_dims(token_type_ids, 1), (1, self.num_choices, 1)) + + inputs = {'input_ids': multiple_choice_inputs_ids, + 'mc_token_ids': mc_token_ids, + 'attention_mask': multiple_choice_input_mask, + 'token_type_ids': multiple_choice_token_type_ids} + lm_logits, mc_logits = model(inputs)[:2] + result = { + "lm_logits": lm_logits.numpy(), + "mc_logits": mc_logits.numpy() + } + self.parent.assertListEqual( + list(result["lm_logits"].shape), + [self.batch_size, self.num_choices, self.seq_length, self.vocab_size]) + self.parent.assertListEqual( + list(result["mc_logits"].shape), + [self.batch_size, self.num_choices]) def prepare_config_and_inputs_for_common(self): config_and_inputs = self.prepare_config_and_inputs() (config, input_ids, input_mask, head_mask, token_type_ids, - sequence_labels, token_labels, choice_labels) = config_and_inputs + mc_token_ids, sequence_labels, token_labels, choice_labels) = config_and_inputs inputs_dict = {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask} return config, inputs_dict From 78b2a53f106fa051169c0c6cdab1f22d462dcfd9 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Mon, 9 Sep 2019 13:38:10 +0200 Subject: [PATCH 035/219] debug file download in tests error --- ..._checkpoint_to_tf.py => convert_pytorch_checkpoint_to_tf2.py} | 0 pytorch_transformers/modeling_tf_utils.py | 1 + 2 files changed, 1 insertion(+) rename pytorch_transformers/{convert_pytorch_checkpoint_to_tf.py => convert_pytorch_checkpoint_to_tf2.py} (100%) diff --git a/pytorch_transformers/convert_pytorch_checkpoint_to_tf.py b/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py similarity index 100% rename from pytorch_transformers/convert_pytorch_checkpoint_to_tf.py rename to pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py diff --git a/pytorch_transformers/modeling_tf_utils.py b/pytorch_transformers/modeling_tf_utils.py index 9630e27478..27be3bf9e9 100644 --- a/pytorch_transformers/modeling_tf_utils.py +++ b/pytorch_transformers/modeling_tf_utils.py @@ -248,6 +248,7 @@ class TFPreTrainedModel(tf.keras.Model): inputs = tf.constant([[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]]) ret = model(inputs, training=False) # build the network with dummy inputs + assert os.path.isfile(resolved_archive_file), "Error retrieving file {}".format(resolved_archive_file) # 'by_name' allow us to do transfer learning by skipping/adding layers # see https://github.com/tensorflow/tensorflow/blob/00fad90125b18b80fe054de1055770cfb8fe4ba3/tensorflow/python/keras/engine/network.py#L1339-L1357 model.load_weights(resolved_archive_file, by_name=True) From 33cb00f41a9017ee8a8cb5f315352149e9b6a038 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Mon, 9 Sep 2019 14:29:24 +0200 Subject: [PATCH 036/219] add GPT2 to init - fix weights loading - remove tf.function --- pytorch_transformers/__init__.py | 10 +++- .../convert_pytorch_checkpoint_to_tf2.py | 34 ++++++++----- pytorch_transformers/modeling_tf_bert.py | 48 +++++++++---------- pytorch_transformers/modeling_tf_gpt2.py | 45 +++++++++-------- 4 files changed, 79 insertions(+), 58 deletions(-) diff --git a/pytorch_transformers/__init__.py b/pytorch_transformers/__init__.py index 7bc7ddf7e1..9546492b3c 100644 --- a/pytorch_transformers/__init__.py +++ b/pytorch_transformers/__init__.py @@ -99,13 +99,19 @@ if _tf_available: from .modeling_tf_auto import (TFAutoModel, TFAutoModelForSequenceClassification, TFAutoModelForQuestionAnswering, TFAutoModelWithLMHead) - from .modeling_tf_bert import (TFBertPreTrainedModel, TFBertModel, TFBertForPreTraining, + from .modeling_tf_bert import (TFBertPreTrainedModel, TFBertMainLayer, TFBertEmbeddings, + TFBertModel, TFBertForPreTraining, TFBertForMaskedLM, TFBertForNextSentencePrediction, TFBertForSequenceClassification, TFBertForMultipleChoice, TFBertForTokenClassification, TFBertForQuestionAnswering, - load_bert_pt_weights_in_tf, + load_bert_pt_weights_in_tf2, TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP) + from .modeling_tf_gpt2 import (TFGPT2PreTrainedModel, TFGPT2MainLayer, TFGPT2Embeddings, + TFGPT2Model, TFGPT2LMHeadModel, TFGPT2DoubleHeadsModel, + load_gpt2_pt_weights_in_tf2, + TF_GPT2_PRETRAINED_MODEL_ARCHIVE_MAP) + # Files and general utilities from .file_utils import (PYTORCH_TRANSFORMERS_CACHE, PYTORCH_PRETRAINED_BERT_CACHE, diff --git a/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py b/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py index e682f6c0d3..ab9b6dd06a 100644 --- a/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py +++ b/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py @@ -21,22 +21,32 @@ from __future__ import print_function import argparse import tensorflow as tf -from pytorch_transformers import BertConfig, TFBertForPreTraining, load_bert_pt_weights_in_tf +import pytorch_transformers + +from pytorch_transformers import (BertConfig, TFBertForPreTraining, load_bert_pt_weights_in_tf2, + GPT2Config, TFGPT2LMHeadModel, load_gpt2_pt_weights_in_tf2) import logging logging.basicConfig(level=logging.INFO) -def convert_pt_checkpoint_to_tf(model_type, pytorch_checkpoint_path, config_file, tf_dump_path): - if model_type == 'bert': - # Initialise TF model - config = BertConfig.from_json_file(config_file) - print("Building TensorFlow model from configuration: {}".format(str(config))) - model = TFBertForPreTraining(config) +MODEL_CLASSES = { + 'bert': (BertConfig, TFBertForPreTraining, load_bert_pt_weights_in_tf2), + 'gpt2': (GPT2Config, TFGPT2LMHeadModel, load_gpt2_pt_weights_in_tf2), +} - # Load weights from tf checkpoint - model = load_bert_pt_weights_in_tf(model, config, pytorch_checkpoint_path) - else: - raise ValueError("Unrecognized model type, should be one of ['bert'].") +def convert_pt_checkpoint_to_tf(model_type, pytorch_checkpoint_path, config_file, tf_dump_path): + if model_type not in MODEL_CLASSES: + raise ValueError("Unrecognized model type, should be one of {}.".format(list(MODEL_CLASSES.keys()))) + + config_class, model_class, loading_fct = MODEL_CLASSES[model_type] + + # Initialise TF model + config = config_class.from_json_file(config_file) + print("Building TensorFlow model from configuration: {}".format(str(config))) + model = model_class(config) + + # Load weights from tf checkpoint + model = loading_fct(model, config, pytorch_checkpoint_path) # Save pytorch-model print("Save TensorFlow model to {}".format(tf_dump_path)) @@ -50,7 +60,7 @@ if __name__ == "__main__": default = None, type = str, required = True, - help = "Model type selcted in the list of.") + help = "Model type selcted in the list of {}.".format(list(MODEL_CLASSES.keys()))) parser.add_argument("--pytorch_checkpoint_path", default = None, type = str, diff --git a/pytorch_transformers/modeling_tf_bert.py b/pytorch_transformers/modeling_tf_bert.py index a48c2d8660..b6b72bed14 100644 --- a/pytorch_transformers/modeling_tf_bert.py +++ b/pytorch_transformers/modeling_tf_bert.py @@ -51,7 +51,7 @@ TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP = { } -def load_bert_pt_weights_in_tf(tf_model, config, pytorch_checkpoint_path): +def load_bert_pt_weights_in_tf2(tf_model, config, pytorch_checkpoint_path): """ Load pytorch checkpoints in a TF 2.0 model and save it using HDF5 format We use HDF5 to easily do transfer learning (see https://github.com/tensorflow/tensorflow/blob/ee16fcac960ae660e0e4496658a366e2f745e1f0/tensorflow/python/keras/engine/network.py#L1352-L1357). @@ -164,7 +164,7 @@ class TFBertEmbeddings(tf.keras.layers.Layer): mean=0., stddev=self.hidden_size**-0.5)) super(TFBertEmbeddings, self).build(input_shape) - @tf.function + # @tf.function def call(self, inputs, mode="embedding", training=False): """Get token embeddings of inputs. Args: @@ -248,7 +248,7 @@ class TFBertSelfAttention(tf.keras.layers.Layer): x = tf.reshape(x, (batch_size, -1, self.num_attention_heads, self.attention_head_size)) return tf.transpose(x, perm=[0, 2, 1, 3]) - @tf.function + # @tf.function def call(self, inputs, training=False): hidden_states, attention_mask, head_mask = inputs @@ -297,7 +297,7 @@ class TFBertSelfOutput(tf.keras.layers.Layer): self.LayerNorm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name='LayerNorm') self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) - @tf.function + # @tf.function def call(self, inputs, training=False): hidden_states, input_tensor = inputs @@ -317,7 +317,7 @@ class TFBertAttention(tf.keras.layers.Layer): def prune_heads(self, heads): raise NotImplementedError - @tf.function + # @tf.function def call(self, inputs, training=False): input_tensor, attention_mask, head_mask = inputs @@ -336,7 +336,7 @@ class TFBertIntermediate(tf.keras.layers.Layer): else: self.intermediate_act_fn = config.hidden_act - @tf.function + # @tf.function def call(self, hidden_states): hidden_states = self.dense(hidden_states) hidden_states = self.intermediate_act_fn(hidden_states) @@ -350,7 +350,7 @@ class TFBertOutput(tf.keras.layers.Layer): self.LayerNorm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name='LayerNorm') self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) - @tf.function + # @tf.function def call(self, inputs, training=False): hidden_states, input_tensor = inputs @@ -368,7 +368,7 @@ class TFBertLayer(tf.keras.layers.Layer): self.intermediate = TFBertIntermediate(config, name='intermediate') self.bert_output = TFBertOutput(config, name='output') - @tf.function + # @tf.function def call(self, inputs, training=False): hidden_states, attention_mask, head_mask = inputs @@ -387,7 +387,7 @@ class TFBertEncoder(tf.keras.layers.Layer): self.output_hidden_states = config.output_hidden_states self.layer = [TFBertLayer(config, name='layer_{}'.format(i)) for i in range(config.num_hidden_layers)] - @tf.function + # @tf.function def call(self, inputs, training=False): hidden_states, attention_mask, head_mask = inputs @@ -420,7 +420,7 @@ class TFBertPooler(tf.keras.layers.Layer): super(TFBertPooler, self).__init__(**kwargs) self.dense = tf.keras.layers.Dense(config.hidden_size, activation='tanh', name='dense') - @tf.function + # @tf.function def call(self, hidden_states): # We "pool" the model by simply taking the hidden state corresponding # to the first token. @@ -439,7 +439,7 @@ class TFBertPredictionHeadTransform(tf.keras.layers.Layer): self.transform_act_fn = config.hidden_act self.LayerNorm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name='LayerNorm') - @tf.function + # @tf.function def call(self, hidden_states): hidden_states = self.dense(hidden_states) hidden_states = self.transform_act_fn(hidden_states) @@ -463,7 +463,7 @@ class TFBertLMPredictionHead(tf.keras.layers.Layer): trainable=True, name='bias') - @tf.function + # @tf.function def call(self, hidden_states): hidden_states = self.transform(hidden_states) hidden_states = self.decoder(hidden_states) + self.bias @@ -475,7 +475,7 @@ class TFBertMLMHead(tf.keras.layers.Layer): super(TFBertMLMHead, self).__init__(**kwargs) self.predictions = TFBertLMPredictionHead(config, name='predictions') - @tf.function + # @tf.function def call(self, sequence_output): prediction_scores = self.predictions(sequence_output) return prediction_scores @@ -486,7 +486,7 @@ class TFBertNSPHead(tf.keras.layers.Layer): super(TFBertNSPHead, self).__init__(**kwargs) self.seq_relationship = tf.keras.layers.Dense(2, name='seq_relationship') - @tf.function + # @tf.function def call(self, pooled_output): seq_relationship_score = self.seq_relationship(pooled_output) return seq_relationship_score @@ -511,7 +511,7 @@ class TFBertMainLayer(tf.keras.layers.Layer): """ raise NotImplementedError - @tf.function + # @tf.function def call(self, inputs, training=False): if not isinstance(inputs, (dict, tuple, list)): input_ids = inputs @@ -579,7 +579,7 @@ class TFBertPreTrainedModel(TFPreTrainedModel): """ config_class = BertConfig pretrained_model_archive_map = TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP - load_pt_weights = load_bert_pt_weights_in_tf + load_pt_weights = load_bert_pt_weights_in_tf2 base_model_prefix = "bert" @@ -693,7 +693,7 @@ class TFBertModel(TFBertPreTrainedModel): super(TFBertModel, self).__init__(config, *inputs, **kwargs) self.bert = TFBertMainLayer(config, name='bert') - @tf.function + # @tf.function def call(self, inputs, training=False): outputs = self.bert(inputs, training=training) return outputs @@ -732,7 +732,7 @@ class TFBertForPreTraining(TFBertPreTrainedModel): self.bert = TFBertMainLayer(config, name='bert') self.cls_nsp = TFBertNSPHead(config, name='cls_nsp') - @tf.function + # @tf.function def call(self, inputs, training=False): outputs = self.bert(inputs, training=training) @@ -774,7 +774,7 @@ class TFBertForMaskedLM(TFBertPreTrainedModel): self.bert = TFBertMainLayer(config, name='bert') - @tf.function + # @tf.function def call(self, inputs, training=False): outputs = self.bert(inputs, training=training) @@ -818,7 +818,7 @@ class TFBertForNextSentencePrediction(TFBertPreTrainedModel): self.bert = TFBertMainLayer(config, name='bert') self.cls_nsp = TFBertNSPHead(config, name='cls_nsp') - @tf.function + # @tf.function def call(self, inputs, training=False): outputs = self.bert(inputs, training=training) @@ -863,7 +863,7 @@ class TFBertForSequenceClassification(TFBertPreTrainedModel): self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) self.classifier = tf.keras.layers.Dense(config.num_labels, name='classifier') - @tf.function + # @tf.function def call(self, inputs, training=False): outputs = self.bert(inputs, training=training) @@ -912,7 +912,7 @@ class TFBertForMultipleChoice(TFBertPreTrainedModel): self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) self.classifier = tf.keras.layers.Dense(1, name='classifier') - @tf.function + # @tf.function def call(self, inputs, training=False): if not isinstance(inputs, (dict, tuple, list)): input_ids = inputs @@ -989,7 +989,7 @@ class TFBertForTokenClassification(TFBertPreTrainedModel): self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) self.classifier = tf.keras.layers.Dense(config.num_labels, name='classifier') - @tf.function + # @tf.function def call(self, inputs, training=False): outputs = self.bert(inputs, training=training) @@ -1040,7 +1040,7 @@ class TFBertForQuestionAnswering(TFBertPreTrainedModel): self.bert = TFBertMainLayer(config, name='bert') self.qa_outputs = tf.keras.layers.Dense(config.num_labels, name='qa_outputs') - @tf.function + # @tf.function def call(self, inputs, training=False): outputs = self.bert(inputs, training=training) diff --git a/pytorch_transformers/modeling_tf_gpt2.py b/pytorch_transformers/modeling_tf_gpt2.py index 85873c9d1b..b0adc8a5d6 100644 --- a/pytorch_transformers/modeling_tf_gpt2.py +++ b/pytorch_transformers/modeling_tf_gpt2.py @@ -39,7 +39,7 @@ TF_GPT2_PRETRAINED_MODEL_ARCHIVE_MAP = {"gpt2": "https://s3.amazonaws.com/models "gpt2-large": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-large-tf_model.h5"} -def load_gpt2_pt_weights_in_tf(tf_model, config, pytorch_checkpoint_path): +def load_gpt2_pt_weights_in_tf2(tf_model, config, pytorch_checkpoint_path): """ Load pytorch checkpoints in a TF 2.0 model and save it using HDF5 format We use HDF5 to easily do transfer learning (see https://github.com/tensorflow/tensorflow/blob/ee16fcac960ae660e0e4496658a366e2f745e1f0/tensorflow/python/keras/engine/network.py#L1352-L1357). @@ -67,24 +67,29 @@ def load_gpt2_pt_weights_in_tf(tf_model, config, pytorch_checkpoint_path): weight_value_tuples = [] for symbolic_weight in symbolic_weights: name = symbolic_weight.name - name = name.replace('cls_mlm', 'cls') # We had to split this layer in two in the TF model to be - name = name.replace('cls_nsp', 'cls') # able to do transfer learning (Keras only allow to remove full layers) name = name.replace(':0', '') - name = name.replace('layer_', 'layer/') + name = name.replace('h_', 'h/') name = name.split('/') - name = name[1:] + name = name[2:] transpose = bool(name[-1] == 'kernel') - if name[-1] == 'kernel' or name[-1] == 'embeddings': + if name[-1] == 'kernel' or name[-1] == 'embeddings' or name[-1] == 'gamma': name[-1] = 'weight' + if name[-1] == 'beta': + name[-1] = 'bias' name = '.'.join(name) - assert name in state_dict + assert name in state_dict, "Weight {} not in PyTorch model".format(name) array = state_dict[name].numpy() if transpose: array = numpy.transpose(array) + if len(symbolic_weight.shape) > len(array.shape): + array = array[None, ...] + if len(symbolic_weight.shape) < len(array.shape): + array = np.squeeze(array) + try: assert list(symbolic_weight.shape) == list(array.shape) except AssertionError as e: @@ -138,7 +143,7 @@ class TFAttention(tf.keras.layers.Layer): pass @staticmethod - @tf.function + # @tf.function def causal_attention_mask(nd, ns, dtype): """1's in the lower triangle, counting from the lower right corner. Same as tf.matrix_band_part(tf.ones([nd, ns]), -1, ns-nd), but doesn't produce garbage on TPUs. @@ -148,7 +153,7 @@ class TFAttention(tf.keras.layers.Layer): m = i >= j - ns + nd return tf.cast(m, dtype) - @tf.function + # @tf.function def _attn(self, inputs, training=False): q, k, v, attention_mask, head_mask = inputs # q, k, v have shape [batch, heads, sequence, features] @@ -180,21 +185,21 @@ class TFAttention(tf.keras.layers.Layer): outputs.append(w) return outputs - @tf.function + # @tf.function def merge_heads(self, x): x = tf.transpose(x, [0, 2, 1, 3]) x_shape = shape_list(x) new_x_shape = x_shape[:-2] + [x_shape[-2] * x_shape[-1]] return tf.reshape(x, new_x_shape) - @tf.function + # @tf.function def split_heads(self, x): x_shape = shape_list(x) new_x_shape = x_shape[:-1] + [self.n_head, x_shape[-1] // self.n_head] x = tf.reshape(x, new_x_shape) return tf.transpose(x, (0, 2, 1, 3)) # (batch, head, seq_length, head_features) - @tf.function + # @tf.function def call(self, inputs, training=False): x, layer_past, attention_mask, head_mask = inputs @@ -230,7 +235,7 @@ class TFMLP(tf.keras.layers.Layer): self.act = gelu self.dropout = tf.keras.layers.Dropout(config.resid_pdrop) - @tf.function + # @tf.function def call(self, x, training=False): h = self.act(self.c_fc(x)) h2 = self.c_proj(h) @@ -248,7 +253,7 @@ class TFBlock(tf.keras.layers.Layer): self.ln_2 = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_epsilon, name='ln_2') self.mlp = TFMLP(4 * nx, config, name='mlp') - @tf.function + # @tf.function def call(self, inputs, training=False): x, layer_past, attention_mask, head_mask = inputs @@ -284,7 +289,7 @@ class TFGPT2Embeddings(tf.keras.layers.Layer): mean=0., stddev=self.hidden_size**-0.5)) super(TFGPT2Embeddings, self).build(input_shape) - @tf.function + # @tf.function def call(self, inputs, mode="embedding"): """Get token embeddings of inputs. Args: @@ -349,7 +354,7 @@ class TFGPT2MainLayer(tf.keras.layers.Layer): """ raise NotImplementedError - @tf.function + # @tf.function def call(self, inputs, training=False): if not isinstance(inputs, (dict, tuple, list)): input_ids = inputs @@ -465,7 +470,7 @@ class TFGPT2PreTrainedModel(TFPreTrainedModel): """ config_class = GPT2Config pretrained_model_archive_map = TF_GPT2_PRETRAINED_MODEL_ARCHIVE_MAP - load_pt_weights = load_gpt2_pt_weights_in_tf + load_pt_weights = load_gpt2_pt_weights_in_tf2 base_model_prefix = "transformer" @@ -563,7 +568,7 @@ class TFGPT2Model(TFGPT2PreTrainedModel): super(TFGPT2Model, self).__init__(config, *inputs, **kwargs) self.transformer = TFGPT2MainLayer(config, name='transformer') - @tf.function + # @tf.function def call(self, inputs, training=False): outputs = self.transformer(inputs, training=training) return outputs @@ -605,7 +610,7 @@ class TFGPT2LMHeadModel(TFGPT2PreTrainedModel): super(TFGPT2LMHeadModel, self).__init__(config, *inputs, **kwargs) self.transformer = TFGPT2MainLayer(config, name='transformer') - @tf.function + # @tf.function def call(self, inputs, training=False): transformer_outputs = self.transformer(inputs, training=training) hidden_states = transformer_outputs[0] @@ -675,7 +680,7 @@ class TFGPT2DoubleHeadsModel(TFGPT2PreTrainedModel): self.multiple_choice_head = TFSequenceSummary(config, name='multiple_choice_head') - @tf.function + # @tf.function def call(self, inputs, training=False): if not isinstance(inputs, (dict, tuple, list)): input_ids = inputs From 0537139b2bad2acabc16244fd7fa4cfdb7deb761 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Mon, 9 Sep 2019 14:47:31 +0200 Subject: [PATCH 037/219] removing tf.function --- pytorch_transformers/modeling_tf_bert.py | 22 ---------------------- pytorch_transformers/modeling_tf_gpt2.py | 12 ------------ pytorch_transformers/modeling_tf_utils.py | 2 -- 3 files changed, 36 deletions(-) diff --git a/pytorch_transformers/modeling_tf_bert.py b/pytorch_transformers/modeling_tf_bert.py index b6b72bed14..9137c0af9a 100644 --- a/pytorch_transformers/modeling_tf_bert.py +++ b/pytorch_transformers/modeling_tf_bert.py @@ -164,7 +164,6 @@ class TFBertEmbeddings(tf.keras.layers.Layer): mean=0., stddev=self.hidden_size**-0.5)) super(TFBertEmbeddings, self).build(input_shape) - # @tf.function def call(self, inputs, mode="embedding", training=False): """Get token embeddings of inputs. Args: @@ -248,7 +247,6 @@ class TFBertSelfAttention(tf.keras.layers.Layer): x = tf.reshape(x, (batch_size, -1, self.num_attention_heads, self.attention_head_size)) return tf.transpose(x, perm=[0, 2, 1, 3]) - # @tf.function def call(self, inputs, training=False): hidden_states, attention_mask, head_mask = inputs @@ -297,7 +295,6 @@ class TFBertSelfOutput(tf.keras.layers.Layer): self.LayerNorm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name='LayerNorm') self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) - # @tf.function def call(self, inputs, training=False): hidden_states, input_tensor = inputs @@ -317,7 +314,6 @@ class TFBertAttention(tf.keras.layers.Layer): def prune_heads(self, heads): raise NotImplementedError - # @tf.function def call(self, inputs, training=False): input_tensor, attention_mask, head_mask = inputs @@ -336,7 +332,6 @@ class TFBertIntermediate(tf.keras.layers.Layer): else: self.intermediate_act_fn = config.hidden_act - # @tf.function def call(self, hidden_states): hidden_states = self.dense(hidden_states) hidden_states = self.intermediate_act_fn(hidden_states) @@ -350,7 +345,6 @@ class TFBertOutput(tf.keras.layers.Layer): self.LayerNorm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name='LayerNorm') self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) - # @tf.function def call(self, inputs, training=False): hidden_states, input_tensor = inputs @@ -368,7 +362,6 @@ class TFBertLayer(tf.keras.layers.Layer): self.intermediate = TFBertIntermediate(config, name='intermediate') self.bert_output = TFBertOutput(config, name='output') - # @tf.function def call(self, inputs, training=False): hidden_states, attention_mask, head_mask = inputs @@ -387,7 +380,6 @@ class TFBertEncoder(tf.keras.layers.Layer): self.output_hidden_states = config.output_hidden_states self.layer = [TFBertLayer(config, name='layer_{}'.format(i)) for i in range(config.num_hidden_layers)] - # @tf.function def call(self, inputs, training=False): hidden_states, attention_mask, head_mask = inputs @@ -420,7 +412,6 @@ class TFBertPooler(tf.keras.layers.Layer): super(TFBertPooler, self).__init__(**kwargs) self.dense = tf.keras.layers.Dense(config.hidden_size, activation='tanh', name='dense') - # @tf.function def call(self, hidden_states): # We "pool" the model by simply taking the hidden state corresponding # to the first token. @@ -439,7 +430,6 @@ class TFBertPredictionHeadTransform(tf.keras.layers.Layer): self.transform_act_fn = config.hidden_act self.LayerNorm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name='LayerNorm') - # @tf.function def call(self, hidden_states): hidden_states = self.dense(hidden_states) hidden_states = self.transform_act_fn(hidden_states) @@ -463,7 +453,6 @@ class TFBertLMPredictionHead(tf.keras.layers.Layer): trainable=True, name='bias') - # @tf.function def call(self, hidden_states): hidden_states = self.transform(hidden_states) hidden_states = self.decoder(hidden_states) + self.bias @@ -475,7 +464,6 @@ class TFBertMLMHead(tf.keras.layers.Layer): super(TFBertMLMHead, self).__init__(**kwargs) self.predictions = TFBertLMPredictionHead(config, name='predictions') - # @tf.function def call(self, sequence_output): prediction_scores = self.predictions(sequence_output) return prediction_scores @@ -486,7 +474,6 @@ class TFBertNSPHead(tf.keras.layers.Layer): super(TFBertNSPHead, self).__init__(**kwargs) self.seq_relationship = tf.keras.layers.Dense(2, name='seq_relationship') - # @tf.function def call(self, pooled_output): seq_relationship_score = self.seq_relationship(pooled_output) return seq_relationship_score @@ -511,7 +498,6 @@ class TFBertMainLayer(tf.keras.layers.Layer): """ raise NotImplementedError - # @tf.function def call(self, inputs, training=False): if not isinstance(inputs, (dict, tuple, list)): input_ids = inputs @@ -693,7 +679,6 @@ class TFBertModel(TFBertPreTrainedModel): super(TFBertModel, self).__init__(config, *inputs, **kwargs) self.bert = TFBertMainLayer(config, name='bert') - # @tf.function def call(self, inputs, training=False): outputs = self.bert(inputs, training=training) return outputs @@ -732,7 +717,6 @@ class TFBertForPreTraining(TFBertPreTrainedModel): self.bert = TFBertMainLayer(config, name='bert') self.cls_nsp = TFBertNSPHead(config, name='cls_nsp') - # @tf.function def call(self, inputs, training=False): outputs = self.bert(inputs, training=training) @@ -774,7 +758,6 @@ class TFBertForMaskedLM(TFBertPreTrainedModel): self.bert = TFBertMainLayer(config, name='bert') - # @tf.function def call(self, inputs, training=False): outputs = self.bert(inputs, training=training) @@ -818,7 +801,6 @@ class TFBertForNextSentencePrediction(TFBertPreTrainedModel): self.bert = TFBertMainLayer(config, name='bert') self.cls_nsp = TFBertNSPHead(config, name='cls_nsp') - # @tf.function def call(self, inputs, training=False): outputs = self.bert(inputs, training=training) @@ -863,7 +845,6 @@ class TFBertForSequenceClassification(TFBertPreTrainedModel): self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) self.classifier = tf.keras.layers.Dense(config.num_labels, name='classifier') - # @tf.function def call(self, inputs, training=False): outputs = self.bert(inputs, training=training) @@ -912,7 +893,6 @@ class TFBertForMultipleChoice(TFBertPreTrainedModel): self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) self.classifier = tf.keras.layers.Dense(1, name='classifier') - # @tf.function def call(self, inputs, training=False): if not isinstance(inputs, (dict, tuple, list)): input_ids = inputs @@ -989,7 +969,6 @@ class TFBertForTokenClassification(TFBertPreTrainedModel): self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) self.classifier = tf.keras.layers.Dense(config.num_labels, name='classifier') - # @tf.function def call(self, inputs, training=False): outputs = self.bert(inputs, training=training) @@ -1040,7 +1019,6 @@ class TFBertForQuestionAnswering(TFBertPreTrainedModel): self.bert = TFBertMainLayer(config, name='bert') self.qa_outputs = tf.keras.layers.Dense(config.num_labels, name='qa_outputs') - # @tf.function def call(self, inputs, training=False): outputs = self.bert(inputs, training=training) diff --git a/pytorch_transformers/modeling_tf_gpt2.py b/pytorch_transformers/modeling_tf_gpt2.py index b0adc8a5d6..05165ce084 100644 --- a/pytorch_transformers/modeling_tf_gpt2.py +++ b/pytorch_transformers/modeling_tf_gpt2.py @@ -143,7 +143,6 @@ class TFAttention(tf.keras.layers.Layer): pass @staticmethod - # @tf.function def causal_attention_mask(nd, ns, dtype): """1's in the lower triangle, counting from the lower right corner. Same as tf.matrix_band_part(tf.ones([nd, ns]), -1, ns-nd), but doesn't produce garbage on TPUs. @@ -153,7 +152,6 @@ class TFAttention(tf.keras.layers.Layer): m = i >= j - ns + nd return tf.cast(m, dtype) - # @tf.function def _attn(self, inputs, training=False): q, k, v, attention_mask, head_mask = inputs # q, k, v have shape [batch, heads, sequence, features] @@ -185,21 +183,18 @@ class TFAttention(tf.keras.layers.Layer): outputs.append(w) return outputs - # @tf.function def merge_heads(self, x): x = tf.transpose(x, [0, 2, 1, 3]) x_shape = shape_list(x) new_x_shape = x_shape[:-2] + [x_shape[-2] * x_shape[-1]] return tf.reshape(x, new_x_shape) - # @tf.function def split_heads(self, x): x_shape = shape_list(x) new_x_shape = x_shape[:-1] + [self.n_head, x_shape[-1] // self.n_head] x = tf.reshape(x, new_x_shape) return tf.transpose(x, (0, 2, 1, 3)) # (batch, head, seq_length, head_features) - # @tf.function def call(self, inputs, training=False): x, layer_past, attention_mask, head_mask = inputs @@ -235,7 +230,6 @@ class TFMLP(tf.keras.layers.Layer): self.act = gelu self.dropout = tf.keras.layers.Dropout(config.resid_pdrop) - # @tf.function def call(self, x, training=False): h = self.act(self.c_fc(x)) h2 = self.c_proj(h) @@ -253,7 +247,6 @@ class TFBlock(tf.keras.layers.Layer): self.ln_2 = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_epsilon, name='ln_2') self.mlp = TFMLP(4 * nx, config, name='mlp') - # @tf.function def call(self, inputs, training=False): x, layer_past, attention_mask, head_mask = inputs @@ -289,7 +282,6 @@ class TFGPT2Embeddings(tf.keras.layers.Layer): mean=0., stddev=self.hidden_size**-0.5)) super(TFGPT2Embeddings, self).build(input_shape) - # @tf.function def call(self, inputs, mode="embedding"): """Get token embeddings of inputs. Args: @@ -354,7 +346,6 @@ class TFGPT2MainLayer(tf.keras.layers.Layer): """ raise NotImplementedError - # @tf.function def call(self, inputs, training=False): if not isinstance(inputs, (dict, tuple, list)): input_ids = inputs @@ -568,7 +559,6 @@ class TFGPT2Model(TFGPT2PreTrainedModel): super(TFGPT2Model, self).__init__(config, *inputs, **kwargs) self.transformer = TFGPT2MainLayer(config, name='transformer') - # @tf.function def call(self, inputs, training=False): outputs = self.transformer(inputs, training=training) return outputs @@ -610,7 +600,6 @@ class TFGPT2LMHeadModel(TFGPT2PreTrainedModel): super(TFGPT2LMHeadModel, self).__init__(config, *inputs, **kwargs) self.transformer = TFGPT2MainLayer(config, name='transformer') - # @tf.function def call(self, inputs, training=False): transformer_outputs = self.transformer(inputs, training=training) hidden_states = transformer_outputs[0] @@ -680,7 +669,6 @@ class TFGPT2DoubleHeadsModel(TFGPT2PreTrainedModel): self.multiple_choice_head = TFSequenceSummary(config, name='multiple_choice_head') - # @tf.function def call(self, inputs, training=False): if not isinstance(inputs, (dict, tuple, list)): input_ids = inputs diff --git a/pytorch_transformers/modeling_tf_utils.py b/pytorch_transformers/modeling_tf_utils.py index 27be3bf9e9..160f97d157 100644 --- a/pytorch_transformers/modeling_tf_utils.py +++ b/pytorch_transformers/modeling_tf_utils.py @@ -277,7 +277,6 @@ class TFConv1D(tf.keras.layers.Layer): shape=[1, self.nf], initializer=tf.zeros_initializer()) - @tf.function def call(self, x): bz, sl = shape_list(x)[:2] @@ -334,7 +333,6 @@ class TFSequenceSummary(tf.keras.layers.Layer): if hasattr(config, 'summary_last_dropout') and config.summary_last_dropout > 0: self.last_dropout = tf.keras.layers.Dropout(config.summary_last_dropout) - @tf.function def call(self, inputs, training=False): """ hidden_states: float Tensor in shape [bsz, seq_len, hidden_size], the hidden-states of the last layer. cls_index: [optional] position of the classification token if summary_type == 'cls_index', From 50c6bc4195a3446ede1a94a92c9be50ecf45bc6c Mon Sep 17 00:00:00 2001 From: thomwolf Date: Mon, 9 Sep 2019 17:46:01 +0200 Subject: [PATCH 038/219] fix tf bert model --- pytorch_transformers/configuration_bert.py | 2 +- .../convert_pytorch_checkpoint_to_tf2.py | 48 ++++++++++--- pytorch_transformers/modeling_bert.py | 22 +++--- pytorch_transformers/modeling_tf_bert.py | 45 ++++++++---- pytorch_transformers/modeling_tf_gpt2.py | 5 ++ pytorch_transformers/modeling_utils.py | 70 ++++++++++--------- 6 files changed, 129 insertions(+), 63 deletions(-) diff --git a/pytorch_transformers/configuration_bert.py b/pytorch_transformers/configuration_bert.py index 7fff3e5d05..00a22770ac 100644 --- a/pytorch_transformers/configuration_bert.py +++ b/pytorch_transformers/configuration_bert.py @@ -58,7 +58,7 @@ class BertConfig(PretrainedConfig): intermediate_size: The size of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder. hidden_act: The non-linear activation function (function or string) in the - encoder and pooler. If string, "gelu", "relu" and "swish" are supported. + encoder and pooler. If string, "gelu", "relu", "swish" and "gelu_new" are supported. hidden_dropout_prob: The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler. attention_probs_dropout_prob: The dropout ratio for the attention diff --git a/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py b/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py index ab9b6dd06a..03b14d4517 100644 --- a/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py +++ b/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py @@ -21,36 +21,62 @@ from __future__ import print_function import argparse import tensorflow as tf -import pytorch_transformers +from pytorch_transformers import is_torch_available from pytorch_transformers import (BertConfig, TFBertForPreTraining, load_bert_pt_weights_in_tf2, GPT2Config, TFGPT2LMHeadModel, load_gpt2_pt_weights_in_tf2) +if is_torch_available(): + import torch + import numpy as np + from pytorch_transformers import BertForPreTraining, GPT2LMHeadModel +else: + BertForPreTraining, GPT2LMHeadModel = None, None + + import logging logging.basicConfig(level=logging.INFO) MODEL_CLASSES = { - 'bert': (BertConfig, TFBertForPreTraining, load_bert_pt_weights_in_tf2), - 'gpt2': (GPT2Config, TFGPT2LMHeadModel, load_gpt2_pt_weights_in_tf2), + 'bert': (BertConfig, TFBertForPreTraining, load_bert_pt_weights_in_tf2, BertForPreTraining), + 'gpt2': (GPT2Config, TFGPT2LMHeadModel, load_gpt2_pt_weights_in_tf2, GPT2LMHeadModel), } -def convert_pt_checkpoint_to_tf(model_type, pytorch_checkpoint_path, config_file, tf_dump_path): +def convert_pt_checkpoint_to_tf(model_type, pytorch_checkpoint_path, config_file, tf_dump_path, compare_with_pt_model=False): if model_type not in MODEL_CLASSES: raise ValueError("Unrecognized model type, should be one of {}.".format(list(MODEL_CLASSES.keys()))) - config_class, model_class, loading_fct = MODEL_CLASSES[model_type] + config_class, model_class, loading_fct, pt_model_class = MODEL_CLASSES[model_type] # Initialise TF model config = config_class.from_json_file(config_file) print("Building TensorFlow model from configuration: {}".format(str(config))) - model = model_class(config) + tf_model = model_class(config) # Load weights from tf checkpoint - model = loading_fct(model, config, pytorch_checkpoint_path) + tf_model = loading_fct(tf_model, config, pytorch_checkpoint_path) + + if compare_with_pt_model: + inputs_list = [[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]] + tf_inputs = tf.constant(inputs_list) + tfo = tf_model(tf_inputs, training=False) # build the network + + pt_model = pt_model_class.from_pretrained(None, + config=config, + state_dict=torch.load(pytorch_checkpoint_path, + map_location='cpu')) + pt_inputs = torch.tensor(inputs_list) + with torch.no_grad(): + pto = pt_model(pt_inputs) + + np_pt = pto[0].detach().numpy() + np_tf = tfo[0].numpy() + diff = np.amax(np.abs(np_pt - np_tf)) + print("Max absolute difference between models outputs {}".format(diff)) # Save pytorch-model print("Save TensorFlow model to {}".format(tf_dump_path)) - model.save_weights(tf_dump_path) + tf_model.save_weights(tf_dump_path) if __name__ == "__main__": @@ -77,8 +103,12 @@ if __name__ == "__main__": type = str, required = True, help = "Path to the output Tensorflow dump file.") + parser.add_argument("--compare_with_pt_model", + action='store_true', + help = "Compare Tensorflow and PyTorch model predictions.") args = parser.parse_args() convert_pt_checkpoint_to_tf(args.model_type.lower(), args.pytorch_checkpoint_path, args.config_file, - args.tf_dump_path) + args.tf_dump_path, + compare_with_pt_model=args.compare_with_pt_model) diff --git a/pytorch_transformers/modeling_bert.py b/pytorch_transformers/modeling_bert.py index c541d18da5..64ea5f947c 100644 --- a/pytorch_transformers/modeling_bert.py +++ b/pytorch_transformers/modeling_bert.py @@ -118,19 +118,24 @@ def load_tf_weights_in_bert(model, config, tf_checkpoint_path): def gelu(x): - """Implementation of the gelu activation function. + """ Original Implementation of the gelu activation function in Google Bert repo when initialy created. For information: OpenAI GPT's gelu is slightly different (and gives slightly different results): 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) Also see https://arxiv.org/abs/1606.08415 """ return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0))) +def gelu_new(x): + """ Implementation of the gelu activation function currently in Google Bert repo (identical to OpenAI GPT). + Also see https://arxiv.org/abs/1606.08415 + """ + return 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) def swish(x): return x * torch.sigmoid(x) -ACT2FN = {"gelu": gelu, "relu": torch.nn.functional.relu, "swish": swish} +ACT2FN = {"gelu": gelu, "relu": torch.nn.functional.relu, "swish": swish, "gelu_new": gelu_new} try: @@ -195,7 +200,7 @@ class BertSelfAttention(nn.Module): x = x.view(*new_x_shape) return x.permute(0, 2, 1, 3) - def forward(self, hidden_states, attention_mask, head_mask=None): + def forward(self, hidden_states, attention_mask=None, head_mask=None): mixed_query_layer = self.query(hidden_states) mixed_key_layer = self.key(hidden_states) mixed_value_layer = self.value(hidden_states) @@ -207,8 +212,9 @@ class BertSelfAttention(nn.Module): # Take the dot product between "query" and "key" to get the raw attention scores. attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2)) attention_scores = attention_scores / math.sqrt(self.attention_head_size) - # Apply the attention mask is (precomputed for all layers in BertModel forward() function) - attention_scores = attention_scores + attention_mask + if attention_mask is not None: + # Apply the attention mask is (precomputed for all layers in BertModel forward() function) + attention_scores = attention_scores + attention_mask # Normalize the attention scores to probabilities. attention_probs = nn.Softmax(dim=-1)(attention_scores) @@ -275,7 +281,7 @@ class BertAttention(nn.Module): self.self.all_head_size = self.self.attention_head_size * self.self.num_attention_heads self.pruned_heads = self.pruned_heads.union(heads) - def forward(self, input_tensor, attention_mask, head_mask=None): + def forward(self, input_tensor, attention_mask=None, head_mask=None): self_outputs = self.self(input_tensor, attention_mask, head_mask) attention_output = self.output(self_outputs[0], input_tensor) outputs = (attention_output,) + self_outputs[1:] # add attentions if we output them @@ -318,7 +324,7 @@ class BertLayer(nn.Module): self.intermediate = BertIntermediate(config) self.output = BertOutput(config) - def forward(self, hidden_states, attention_mask, head_mask=None): + def forward(self, hidden_states, attention_mask=None, head_mask=None): attention_outputs = self.attention(hidden_states, attention_mask, head_mask) attention_output = attention_outputs[0] intermediate_output = self.intermediate(attention_output) @@ -334,7 +340,7 @@ class BertEncoder(nn.Module): self.output_hidden_states = config.output_hidden_states self.layer = nn.ModuleList([BertLayer(config) for _ in range(config.num_hidden_layers)]) - def forward(self, hidden_states, attention_mask, head_mask=None): + def forward(self, hidden_states, attention_mask=None, head_mask=None): all_hidden_states = () all_attentions = () for i, layer_module in enumerate(self.layer): diff --git a/pytorch_transformers/modeling_tf_bert.py b/pytorch_transformers/modeling_tf_bert.py index 9137c0af9a..ed82f4e8f3 100644 --- a/pytorch_transformers/modeling_tf_bert.py +++ b/pytorch_transformers/modeling_tf_bert.py @@ -77,6 +77,7 @@ def load_bert_pt_weights_in_tf2(tf_model, config, pytorch_checkpoint_path): symbolic_weights = tf_model.trainable_weights + tf_model.non_trainable_weights weight_value_tuples = [] + all_pytorch_weights = set(list(state_dict.keys())) for symbolic_weight in symbolic_weights: name = symbolic_weight.name name = name.replace('cls_mlm', 'cls') # We had to split this layer in two in the TF model to be @@ -91,7 +92,7 @@ def load_bert_pt_weights_in_tf2(tf_model, config, pytorch_checkpoint_path): name[-1] = 'weight' name = '.'.join(name) - assert name in state_dict + assert name in state_dict, "{} not found in PyTorch model".format(name) array = state_dict[name].numpy() if transpose: @@ -106,14 +107,28 @@ def load_bert_pt_weights_in_tf2(tf_model, config, pytorch_checkpoint_path): logger.info("Initialize TF weight {}".format(symbolic_weight.name)) weight_value_tuples.append((symbolic_weight, array)) + all_pytorch_weights.discard(name) K.batch_set_value(weight_value_tuples) tfo = tf_model(tf_inputs, training=False) # Make sure restore ops are run + + logger.info("Weights not loaded: {}".format(all_pytorch_weights)) + return tf_model def gelu(x): + """ Gaussian Error Linear Unit. + Original Implementation of the gelu activation function in Google Bert repo when initialy created. + For information: OpenAI GPT's gelu is slightly different (and gives slightly different results): + 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) + Also see https://arxiv.org/abs/1606.08415 + """ + cdf = 0.5 * (1.0 + tf.math.erf(x / tf.math.sqrt(2.0))) + return x * cdf + +def gelu_new(x): """Gaussian Error Linear Unit. This is a smoother version of the RELU. Original paper: https://arxiv.org/abs/1606.08415 @@ -126,14 +141,14 @@ def gelu(x): (np.sqrt(2 / np.pi) * (x + 0.044715 * tf.pow(x, 3))))) return x * cdf - def swish(x): return x * tf.sigmoid(x) ACT2FN = {"gelu": tf.keras.layers.Activation(gelu), "relu": tf.keras.activations.relu, - "swish": tf.keras.layers.Activation(swish)} + "swish": tf.keras.layers.Activation(swish), + "gelu_new": tf.keras.layers.Activation(gelu_new)} class TFBertEmbeddings(tf.keras.layers.Layer): @@ -263,8 +278,10 @@ class TFBertSelfAttention(tf.keras.layers.Layer): attention_scores = tf.matmul(query_layer, key_layer, transpose_b=True) # (batch size, num_heads, seq_len_q, seq_len_k) dk = tf.cast(tf.shape(key_layer)[-1], tf.float32) # scale attention_scores attention_scores = attention_scores / tf.math.sqrt(dk) - # Apply the attention mask is (precomputed for all layers in TFBertModel call() function) - attention_scores = attention_scores + attention_mask + + if attention_mask is not None: + # Apply the attention mask is (precomputed for all layers in TFBertModel call() function) + attention_scores = attention_scores + attention_mask # Normalize the attention scores to probabilities. attention_probs = tf.nn.softmax(attention_scores, axis=-1) @@ -438,31 +455,33 @@ class TFBertPredictionHeadTransform(tf.keras.layers.Layer): class TFBertLMPredictionHead(tf.keras.layers.Layer): - def __init__(self, config, **kwargs): + def __init__(self, config, input_embeddings, **kwargs): super(TFBertLMPredictionHead, self).__init__(**kwargs) self.vocab_size = config.vocab_size self.transform = TFBertPredictionHeadTransform(config, name='transform') # The output weights are the same as the input embeddings, but there is # an output-only bias for each token. - self.decoder = tf.keras.layers.Dense(config.vocab_size, use_bias=False, name='decoder') + self.input_embeddings = input_embeddings def build(self, input_shape): self.bias = self.add_weight(shape=(self.vocab_size,), initializer='zeros', trainable=True, name='bias') + super(TFBertLMPredictionHead, self).build(input_shape) def call(self, hidden_states): hidden_states = self.transform(hidden_states) - hidden_states = self.decoder(hidden_states) + self.bias + hidden_states = self.input_embeddings(hidden_states, mode="linear") + hidden_states = hidden_states + self.bias return hidden_states class TFBertMLMHead(tf.keras.layers.Layer): - def __init__(self, config, **kwargs): + def __init__(self, config, input_embeddings, **kwargs): super(TFBertMLMHead, self).__init__(**kwargs) - self.predictions = TFBertLMPredictionHead(config, name='predictions') + self.predictions = TFBertLMPredictionHead(config, input_embeddings, name='predictions') def call(self, sequence_output): prediction_scores = self.predictions(sequence_output) @@ -716,12 +735,13 @@ class TFBertForPreTraining(TFBertPreTrainedModel): self.bert = TFBertMainLayer(config, name='bert') self.cls_nsp = TFBertNSPHead(config, name='cls_nsp') + self.cls_mlm = TFBertMLMHead(config, self.bert.embeddings, name='cls_mlm') def call(self, inputs, training=False): outputs = self.bert(inputs, training=training) sequence_output, pooled_output = outputs[:2] - prediction_scores = self.bert.embeddings(sequence_output, mode="linear", training=training) + prediction_scores = self.cls_mlm(sequence_output, training=training) seq_relationship_score = self.cls_nsp(pooled_output) outputs = (prediction_scores, seq_relationship_score,) + outputs[2:] # add hidden states and attention if they are here @@ -757,12 +777,13 @@ class TFBertForMaskedLM(TFBertPreTrainedModel): super(TFBertForMaskedLM, self).__init__(config, *inputs, **kwargs) self.bert = TFBertMainLayer(config, name='bert') + self.cls_mlm = TFBertMLMHead(config, self.bert.embeddings, name='cls_mlm') def call(self, inputs, training=False): outputs = self.bert(inputs, training=training) sequence_output = outputs[0] - prediction_scores = self.bert.embeddings(sequence_output, mode="linear", training=training) + prediction_scores = self.cls_mlm(sequence_output, training=training) outputs = (prediction_scores,) + outputs[2:] # Add hidden states and attention if they are here diff --git a/pytorch_transformers/modeling_tf_gpt2.py b/pytorch_transformers/modeling_tf_gpt2.py index 05165ce084..a896ee5a5f 100644 --- a/pytorch_transformers/modeling_tf_gpt2.py +++ b/pytorch_transformers/modeling_tf_gpt2.py @@ -100,9 +100,14 @@ def load_gpt2_pt_weights_in_tf2(tf_model, config, pytorch_checkpoint_path): weight_value_tuples.append((symbolic_weight, array)) + state_dict.pop(name) + K.batch_set_value(weight_value_tuples) tfo = tf_model(tf_inputs, training=False) # Make sure restore ops are run + + assert not state_dict, "Weights not loaded: {}".format(list(state_dict.keys())) + return tf_model diff --git a/pytorch_transformers/modeling_utils.py b/pytorch_transformers/modeling_utils.py index 9fd7a2c0c2..c316b66bc9 100644 --- a/pytorch_transformers/modeling_utils.py +++ b/pytorch_transformers/modeling_utils.py @@ -222,6 +222,7 @@ class PreTrainedModel(nn.Module): - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. - a path to a `directory` containing model weights saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. - a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. + - None if you are both providing the configuration and state dictionary (resp. with keyword arguments ``config`` and ``state_dict``) model_args: (`optional`) Sequence of positional arguments: All remaning positional arguments will be passed to the underlying model's ``__init__`` method @@ -289,42 +290,45 @@ class PreTrainedModel(nn.Module): model_kwargs = kwargs # Load model - if pretrained_model_name_or_path in cls.pretrained_model_archive_map: - archive_file = cls.pretrained_model_archive_map[pretrained_model_name_or_path] - elif os.path.isdir(pretrained_model_name_or_path): - if from_tf: - # Directly load from a TensorFlow checkpoint - archive_file = os.path.join(pretrained_model_name_or_path, TF_WEIGHTS_NAME + ".index") - else: - archive_file = os.path.join(pretrained_model_name_or_path, WEIGHTS_NAME) - else: - if from_tf: - # Directly load from a TensorFlow checkpoint - archive_file = pretrained_model_name_or_path + ".index" - else: - archive_file = pretrained_model_name_or_path - # redirect to the cache, if necessary - try: - resolved_archive_file = cached_path(archive_file, cache_dir=cache_dir, force_download=force_download, proxies=proxies) - except EnvironmentError as e: + if pretrained_model_name_or_path is not None: if pretrained_model_name_or_path in cls.pretrained_model_archive_map: - logger.error( - "Couldn't reach server at '{}' to download pretrained weights.".format( - archive_file)) + archive_file = cls.pretrained_model_archive_map[pretrained_model_name_or_path] + elif os.path.isdir(pretrained_model_name_or_path): + if from_tf: + # Directly load from a TensorFlow checkpoint + archive_file = os.path.join(pretrained_model_name_or_path, TF_WEIGHTS_NAME + ".index") + else: + archive_file = os.path.join(pretrained_model_name_or_path, WEIGHTS_NAME) else: - logger.error( - "Model name '{}' was not found in model name list ({}). " - "We assumed '{}' was a path or url but couldn't find any file " - "associated to this path or url.".format( - pretrained_model_name_or_path, - ', '.join(cls.pretrained_model_archive_map.keys()), - archive_file)) - raise e - if resolved_archive_file == archive_file: - logger.info("loading weights file {}".format(archive_file)) + if from_tf: + # Directly load from a TensorFlow checkpoint + archive_file = pretrained_model_name_or_path + ".index" + else: + archive_file = pretrained_model_name_or_path + # redirect to the cache, if necessary + try: + resolved_archive_file = cached_path(archive_file, cache_dir=cache_dir, force_download=force_download, proxies=proxies) + except EnvironmentError as e: + if pretrained_model_name_or_path in cls.pretrained_model_archive_map: + logger.error( + "Couldn't reach server at '{}' to download pretrained weights.".format( + archive_file)) + else: + logger.error( + "Model name '{}' was not found in model name list ({}). " + "We assumed '{}' was a path or url but couldn't find any file " + "associated to this path or url.".format( + pretrained_model_name_or_path, + ', '.join(cls.pretrained_model_archive_map.keys()), + archive_file)) + raise e + if resolved_archive_file == archive_file: + logger.info("loading weights file {}".format(archive_file)) + else: + logger.info("loading weights file {} from cache at {}".format( + archive_file, resolved_archive_file)) else: - logger.info("loading weights file {} from cache at {}".format( - archive_file, resolved_archive_file)) + resolved_archive_file = None # Instantiate model. model = cls(config, *model_args, **model_kwargs) From f851fb55ca0745eb8a2e39324bafcd3da5a21175 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 10 Sep 2019 09:24:08 +0200 Subject: [PATCH 039/219] fixing error message --- pytorch_transformers/modeling_tf_bert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_transformers/modeling_tf_bert.py b/pytorch_transformers/modeling_tf_bert.py index ed82f4e8f3..4a45707b98 100644 --- a/pytorch_transformers/modeling_tf_bert.py +++ b/pytorch_transformers/modeling_tf_bert.py @@ -113,7 +113,7 @@ def load_bert_pt_weights_in_tf2(tf_model, config, pytorch_checkpoint_path): tfo = tf_model(tf_inputs, training=False) # Make sure restore ops are run - logger.info("Weights not loaded: {}".format(all_pytorch_weights)) + logger.info("Weights or buffers not loaded from PyTorch model: {}".format(all_pytorch_weights)) return tf_model From 32aabe8c33b65dd9877a7df474c9aa84ea1fe354 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 10 Sep 2019 12:17:18 +0200 Subject: [PATCH 040/219] WIP XLNet --- pytorch_transformers/__init__.py | 4 +- pytorch_transformers/configuration_utils.py | 1 + pytorch_transformers/modeling_tf_gpt2.py | 74 +- pytorch_transformers/modeling_tf_utils.py | 63 + pytorch_transformers/modeling_tf_xlnet.py | 1121 +++++++++++++++++ .../tests/modeling_tf_common_test.py | 4 +- .../tests/modeling_tf_xlnet_test.py | 341 +++++ 7 files changed, 1540 insertions(+), 68 deletions(-) create mode 100644 pytorch_transformers/modeling_tf_xlnet.py create mode 100644 pytorch_transformers/tests/modeling_tf_xlnet_test.py diff --git a/pytorch_transformers/__init__.py b/pytorch_transformers/__init__.py index 9546492b3c..dbe979c564 100644 --- a/pytorch_transformers/__init__.py +++ b/pytorch_transformers/__init__.py @@ -95,7 +95,7 @@ except (ImportError, AssertionError): if _tf_available: logger.info("TensorFlow version {} available.".format(tf.__version__)) - from .modeling_tf_utils import TFPreTrainedModel + from .modeling_tf_utils import TFPreTrainedModel, TFSharedEmbeddings, TFSequenceSummary from .modeling_tf_auto import (TFAutoModel, TFAutoModelForSequenceClassification, TFAutoModelForQuestionAnswering, TFAutoModelWithLMHead) @@ -107,7 +107,7 @@ if _tf_available: load_bert_pt_weights_in_tf2, TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP) - from .modeling_tf_gpt2 import (TFGPT2PreTrainedModel, TFGPT2MainLayer, TFGPT2Embeddings, + from .modeling_tf_gpt2 import (TFGPT2PreTrainedModel, TFGPT2MainLayer, TFGPT2Model, TFGPT2LMHeadModel, TFGPT2DoubleHeadsModel, load_gpt2_pt_weights_in_tf2, TF_GPT2_PRETRAINED_MODEL_ARCHIVE_MAP) diff --git a/pytorch_transformers/configuration_utils.py b/pytorch_transformers/configuration_utils.py index 7efc735d41..42346d6b6c 100644 --- a/pytorch_transformers/configuration_utils.py +++ b/pytorch_transformers/configuration_utils.py @@ -54,6 +54,7 @@ class PretrainedConfig(object): self.output_attentions = kwargs.pop('output_attentions', False) self.output_hidden_states = kwargs.pop('output_hidden_states', False) self.torchscript = kwargs.pop('torchscript', False) + self.use_bfloat16 = kwargs.pop('use_bfloat16', False) self.pruned_heads = kwargs.pop('pruned_heads', {}) def save_pretrained(self, save_directory): diff --git a/pytorch_transformers/modeling_tf_gpt2.py b/pytorch_transformers/modeling_tf_gpt2.py index a896ee5a5f..900acb94a4 100644 --- a/pytorch_transformers/modeling_tf_gpt2.py +++ b/pytorch_transformers/modeling_tf_gpt2.py @@ -28,7 +28,8 @@ from io import open import numpy as np import tensorflow as tf -from .modeling_tf_utils import TFPreTrainedModel, TFConv1D, TFSequenceSummary, shape_list +from .modeling_tf_utils import (TFPreTrainedModel, TFConv1D, TFSharedEmbeddings, + TFSequenceSummary, shape_list) from .configuration_gpt2 import GPT2Config from .file_utils import add_start_docstrings @@ -65,6 +66,7 @@ def load_gpt2_pt_weights_in_tf2(tf_model, config, pytorch_checkpoint_path): symbolic_weights = tf_model.trainable_weights + tf_model.non_trainable_weights weight_value_tuples = [] + all_pytorch_weights = set(list(state_dict.keys())) for symbolic_weight in symbolic_weights: name = symbolic_weight.name name = name.replace(':0', '') @@ -100,13 +102,13 @@ def load_gpt2_pt_weights_in_tf2(tf_model, config, pytorch_checkpoint_path): weight_value_tuples.append((symbolic_weight, array)) - state_dict.pop(name) + all_pytorch_weights.discard(name) K.batch_set_value(weight_value_tuples) tfo = tf_model(tf_inputs, training=False) # Make sure restore ops are run - assert not state_dict, "Weights not loaded: {}".format(list(state_dict.keys())) + logger.info("Weights or buffers not loaded from PyTorch model: {}".format(all_pytorch_weights)) return tf_model @@ -267,65 +269,6 @@ class TFBlock(tf.keras.layers.Layer): outputs = [x] + output_attn[1:] return outputs # x, present, (attentions) -class TFGPT2Embeddings(tf.keras.layers.Layer): - """Construct the embeddings from word, position and token_type embeddings. - """ - def __init__(self, config, **kwargs): - super(TFGPT2Embeddings, self).__init__(**kwargs) - self.vocab_size = config.vocab_size - self.hidden_size = config.hidden_size - - def build(self, input_shape): - """Build shared word embedding layer - Shared weights logic adapted from - https://github.com/tensorflow/models/blob/a009f4fb9d2fc4949e32192a944688925ef78659/official/transformer/v2/embedding_layer.py#L24 - """ - self.weight = self.add_weight( - "weight", - shape=[self.vocab_size, self.hidden_size], - initializer=tf.random_normal_initializer( - mean=0., stddev=self.hidden_size**-0.5)) - super(TFGPT2Embeddings, self).build(input_shape) - - def call(self, inputs, mode="embedding"): - """Get token embeddings of inputs. - Args: - inputs: list of three int64 tensors with shape [batch_size, length]: (input_ids, position_ids, token_type_ids) - mode: string, a valid value is one of "embedding" and "linear". - Returns: - outputs: (1) If mode == "embedding", output embedding tensor, float32 with - shape [batch_size, length, embedding_size]; (2) mode == "linear", output - linear tensor, float32 with shape [batch_size, length, vocab_size]. - Raises: - ValueError: if mode is not valid. - - Shared weights logic adapted from - https://github.com/tensorflow/models/blob/a009f4fb9d2fc4949e32192a944688925ef78659/official/transformer/v2/embedding_layer.py#L24 - """ - if mode == "embedding": - return self._embedding(inputs) - elif mode == "linear": - return self._linear(inputs) - else: - raise ValueError("mode {} is not valid.".format(mode)) - - def _embedding(self, input_ids): - """Applies embedding based on inputs tensor.""" - return tf.gather(self.weight, input_ids) - - def _linear(self, inputs): - """Computes logits by running inputs through a linear layer. - Args: - inputs: A float32 tensor with shape [..., hidden_size] - Returns: - float32 tensor with shape [..., vocab_size]. - """ - first_dims = shape_list(inputs)[:-1] - - x = tf.reshape(inputs, [-1, self.hidden_size]) - logits = tf.matmul(x, self.weight, transpose_b=True) - - return tf.reshape(logits, first_dims + [self.vocab_size]) class TFGPT2MainLayer(tf.keras.layers.Layer): def __init__(self, config, *inputs, **kwargs): @@ -336,10 +279,13 @@ class TFGPT2MainLayer(tf.keras.layers.Layer): self.vocab_size = config.vocab_size self.n_embd = config.n_embd - self.wte = TFGPT2Embeddings(config, name='wte') + self.wte = TFSharedEmbeddings(config.vocab_size, config.hidden_size, name='wte') self.wpe = tf.keras.layers.Embedding(config.n_positions, config.n_embd, name='wpe') self.drop = tf.keras.layers.Dropout(config.embd_pdrop) - self.h = [TFBlock(config.n_ctx, config, scale=True, name='h_{}'.format(i)) for i in range(config.n_layer)] + self.h = [TFBlock(config.n_ctx, + config, + scale=True, + name='h_{}'.format(i)) for i in range(config.n_layer)] self.ln_f = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_epsilon, name='ln_f') def _resize_token_embeddings(self, new_num_tokens): diff --git a/pytorch_transformers/modeling_tf_utils.py b/pytorch_transformers/modeling_tf_utils.py index 160f97d157..1860ab4f8b 100644 --- a/pytorch_transformers/modeling_tf_utils.py +++ b/pytorch_transformers/modeling_tf_utils.py @@ -288,6 +288,69 @@ class TFConv1D(tf.keras.layers.Layer): return x +class TFSharedEmbeddings(tf.keras.layers.Layer): + """Construct shared token embeddings. + """ + def __init__(self, vocab_size, hidden_size, initializer_range=None, **kwargs): + super(TFSharedEmbeddings, self).__init__(**kwargs) + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.initializer_range = initializer_range + + def build(self, input_shape): + """Build shared word embedding layer + Shared weights logic adapted from + https://github.com/tensorflow/models/blob/a009f4fb9d2fc4949e32192a944688925ef78659/official/transformer/v2/embedding_layer.py#L24 + """ + initializer_range = self.hidden_size**-0.5 if self.initializer_range is None else self.initializer_range + self.weight = self.add_weight( + "weight", + shape=[self.vocab_size, self.hidden_size], + initializer=tf.random_normal_initializer( + mean=0., stddev=initializer_range)) + super(TFSharedEmbeddings, self).build(input_shape) + + def call(self, inputs, mode="embedding"): + """Get token embeddings of inputs. + Args: + inputs: list of three int64 tensors with shape [batch_size, length]: (input_ids, position_ids, token_type_ids) + mode: string, a valid value is one of "embedding" and "linear". + Returns: + outputs: (1) If mode == "embedding", output embedding tensor, float32 with + shape [batch_size, length, embedding_size]; (2) mode == "linear", output + linear tensor, float32 with shape [batch_size, length, vocab_size]. + Raises: + ValueError: if mode is not valid. + + Shared weights logic adapted from + https://github.com/tensorflow/models/blob/a009f4fb9d2fc4949e32192a944688925ef78659/official/transformer/v2/embedding_layer.py#L24 + """ + if mode == "embedding": + return self._embedding(inputs) + elif mode == "linear": + return self._linear(inputs) + else: + raise ValueError("mode {} is not valid.".format(mode)) + + def _embedding(self, input_ids): + """Applies embedding based on inputs tensor.""" + return tf.gather(self.weight, input_ids) + + def _linear(self, inputs): + """Computes logits by running inputs through a linear layer. + Args: + inputs: A float32 tensor with shape [..., hidden_size] + Returns: + float32 tensor with shape [..., vocab_size]. + """ + first_dims = shape_list(inputs)[:-1] + + x = tf.reshape(inputs, [-1, self.hidden_size]) + logits = tf.matmul(x, self.weight, transpose_b=True) + + return tf.reshape(logits, first_dims + [self.vocab_size]) + + class TFSequenceSummary(tf.keras.layers.Layer): r""" Compute a single vector summary of a sequence hidden states according to various possibilities: Args of the config class: diff --git a/pytorch_transformers/modeling_tf_xlnet.py b/pytorch_transformers/modeling_tf_xlnet.py new file mode 100644 index 0000000000..fa24296d76 --- /dev/null +++ b/pytorch_transformers/modeling_tf_xlnet.py @@ -0,0 +1,1121 @@ +# coding=utf-8 +# Copyright 2018 Google AI, Google Brain and Carnegie Mellon University Authors and the HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" TF 2.0 XLNet model. +""" +from __future__ import absolute_import, division, print_function, unicode_literals + +import json +import logging +import math +import os +import sys +from io import open + +import numpy as np +import tensorflow as tf + +from .configuration_xlnet import XLNetConfig +from .modeling_tf_utils import TFPreTrainedModel, TFSharedEmbeddings, shape_list +from .file_utils import add_start_docstrings + + +logger = logging.getLogger(__name__) + +TF_XLNET_PRETRAINED_MODEL_ARCHIVE_MAP = { + 'xlnet-base-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/xlnet-base-cased-tf_model.h5", + 'xlnet-large-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/xlnet-large-cased-tf_model.h5", +} + + +def load_xlnet_pt_weights_in_tf2(tf_model, config, pytorch_checkpoint_path): + """ Load pytorch checkpoints in a TF 2.0 model and save it using HDF5 format + We use HDF5 to easily do transfer learning + (see https://github.com/tensorflow/tensorflow/blob/ee16fcac960ae660e0e4496658a366e2f745e1f0/tensorflow/python/keras/engine/network.py#L1352-L1357). + """ + try: + import re + import torch + import numpy + from tensorflow.python.keras import backend as K + except ImportError: + logger.error("Loading a PyTorch model in TensorFlow, requires PyTorch to be installed. Please see " + "https://pytorch.org/ for installation instructions.") + raise + + pt_path = os.path.abspath(pytorch_checkpoint_path) + logger.info("Loading PyTorch weights from {}".format(pt_path)) + # Load pytorch model + state_dict = torch.load(pt_path, map_location='cpu') + + inputs_list = [[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]] + tf_inputs = tf.constant(inputs_list) + tfo = tf_model(tf_inputs, training=False) # build the network + + symbolic_weights = tf_model.trainable_weights + tf_model.non_trainable_weights + weight_value_tuples = [] + all_pytorch_weights = set(list(state_dict.keys())) + for symbolic_weight in symbolic_weights: + name = symbolic_weight.name + name = name.replace('cls_mlm', 'cls') # We had to split this layer in two in the TF model to be + name = name.replace('cls_nsp', 'cls') # able to do transfer learning (Keras only allow to remove full layers) + name = name.replace(':0', '') + name = name.replace('layer_', 'layer/') + name = name.split('/') + name = name[1:] + + transpose = bool(name[-1] == 'kernel') + if name[-1] == 'kernel' or name[-1] == 'embeddings': + name[-1] = 'weight' + + name = '.'.join(name) + assert name in state_dict, "{} not found in PyTorch model".format(name) + array = state_dict[name].numpy() + + if transpose: + array = numpy.transpose(array) + + try: + assert list(symbolic_weight.shape) == list(array.shape) + except AssertionError as e: + e.args += (symbolic_weight.shape, array.shape) + raise e + + logger.info("Initialize TF weight {}".format(symbolic_weight.name)) + + weight_value_tuples.append((symbolic_weight, array)) + all_pytorch_weights.discard(name) + + K.batch_set_value(weight_value_tuples) + + tfo = tf_model(tf_inputs, training=False) # Make sure restore ops are run + + logger.info("Weights or buffers not loaded from PyTorch model: {}".format(all_pytorch_weights)) + + return tf_model + + +def gelu(x): + """ Implementation of the gelu activation function. + XLNet is using OpenAI GPT's gelu + Also see https://arxiv.org/abs/1606.08415 + """ + cdf = 0.5 * (1.0 + tf.tanh( + (np.sqrt(2 / np.pi) * (x + 0.044715 * tf.pow(x, 3))))) + return x * cdf + + +def swish(x): + return x * tf.sigmoid(x) + + +ACT2FN = {"gelu": tf.keras.layers.Activation(gelu), + "relu": tf.keras.activations.relu, + "swish": tf.keras.layers.Activation(swish)} + + +class TFXLNetRelativeAttention(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFXLNetRelativeAttention, self).__init__(**kwargs) + self.output_attentions = config.output_attentions + + if config.d_model % config.n_head != 0: + raise ValueError( + "The hidden size (%d) is not a multiple of the number of attention " + "heads (%d)" % (config.d_model, config.n_head)) + + self.n_head = config.n_head + self.d_head = config.d_head + self.d_model = config.d_model + self.scale = 1 / (config.d_head ** 0.5) + self.initializer_range = config.initializer_range + + self.layer_norm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name='layer_norm') + self.dropout = tf.keras.layers.Dropout(config.dropout) + + def build(input_shape): + initializer = tf.random_normal_initializer(mean=0., stddev=self.initializer_range) + self.q = self.add_weight(shape=(self.d_model, self.n_head, self.d_head), + initializer=initializer, + trainable=True, name='q') + self.k = self.add_weight(shape=(self.d_model, self.n_head, self.d_head), + initializer=initializer, + trainable=True, name='k') + self.v = self.add_weight(shape=(self.d_model, self.n_head, self.d_head), + initializer=initializer, + trainable=True, name='v') + self.o = self.add_weight(shape=(self.d_model, self.n_head, self.d_head), + initializer=initializer, + trainable=True, name='o') + self.r = self.add_weight(shape=(self.d_model, self.n_head, self.d_head), + initializer=initializer, + trainable=True, name='r') + self.r_r_bias = self.add_weight(shape=(self.n_head, self.d_head), + initializer=initializer, + trainable=True, name='r_r_bias') + self.r_s_bias = self.add_weight(shape=(self.n_head, self.d_head), + initializer=initializer, + trainable=True, name='r_s_bias') + self.r_w_bias = self.add_weight(shape=(self.n_head, self.d_head), + initializer=initializer, + trainable=True, name='r_w_bias') + self.seg_embed = self.add_weight(shape=(2, self.n_head, self.d_head), + initializer=initializer, + trainable=True, name='seg_embed') + super(TFXLNetRelativeAttention, self).build(input_shape) + + def prune_heads(self, heads): + raise NotImplementedError + + @staticmethod + def rel_shift(x, klen=-1): + """perform relative shift to form the relative attention score.""" + x_size = shape_list(x) + + x = tf.reshape(x, (x_size[1], x_size[0], x_size[2], x_size[3])) + x = x[1:, ...] + x = tf.reshape(x, (x_size[0], x_size[1] - 1, x_size[2], x_size[3])) + x = x[:, 0:klen, :, :] + # x = torch.index_select(x, 1, torch.arange(klen, device=x.device, dtype=torch.long)) + + return x + + def rel_attn_core(self, inputs, training=False): + """Core relative positional attention operations.""" + + q_head, k_head_h, v_head_h, k_head_r, seg_mat, attn_mask, head_mask = inputs + + # content based attention score + ac = tf.einsum('ibnd,jbnd->ijbn', q_head + self.r_w_bias, k_head_h) + + # position based attention score + bd = tf.einsum('ibnd,jbnd->ijbn', q_head + self.r_r_bias, k_head_r) + bd = self.rel_shift(bd, klen=ac.shape[1]) + + # segment based attention score + if seg_mat is None: + ef = 0 + else: + ef = tf.einsum('ibnd,snd->ibns', q_head + self.r_s_bias, self.seg_embed) + ef = tf.einsum('ijbs,ibns->ijbn', seg_mat, ef) + + # merge attention scores and perform masking + attn_score = (ac + bd + ef) * self.scale + if attn_mask is not None: + # attn_score = attn_score * (1 - attn_mask) - 1e30 * attn_mask + if attn_mask.dtype == tf.float16: + attn_score = attn_score - 65500 * attn_mask + else: + attn_score = attn_score - 1e30 * attn_mask + + # attention probability + attn_prob = tf.softmax(attn_score, axis=1) + + if training: + attn_prob = self.dropout(attn_prob) + + # Mask heads if we want to + if head_mask is not None: + attn_prob = attn_prob * head_mask + + # attention output + attn_vec = tf.einsum('ijbn,jbnd->ibnd', attn_prob, v_head_h) + + if self.output_attentions: + return attn_vec, attn_prob + + return attn_vec + + def post_attention(self, inputs, training=False): + """Post-attention processing.""" + # post-attention projection (back to `d_model`) + h, attn_vec, residual = inputs + + attn_out = tf.einsum('ibnd,hnd->ibh', attn_vec, self.o) + + if training: + attn_out = self.dropout(attn_out) + + if residual: + attn_out = attn_out + h + output = self.layer_norm(attn_out) + + return output + + def call(self, inputs, training=False): + (h, g, attn_mask_h, attn_mask_g, + r, seg_mat, mems, target_mapping, head_mask) = inputs + + if g is not None: + ###### Two-stream attention with relative positional encoding. + # content based attention score + if mems is not None and mems.dim() > 1: + cat = torch.cat([mems, h], dim=0) + else: + cat = h + + # content-based key head + k_head_h = torch.einsum('ibh,hnd->ibnd', cat, self.k) + + # content-based value head + v_head_h = torch.einsum('ibh,hnd->ibnd', cat, self.v) + + # position-based key head + k_head_r = torch.einsum('ibh,hnd->ibnd', r, self.r) + + ##### h-stream + # content-stream query head + q_head_h = torch.einsum('ibh,hnd->ibnd', h, self.q) + + # core attention ops + attn_vec_h = self.rel_attn_core( + [q_head_h, k_head_h, v_head_h, k_head_r, seg_mat, attn_mask_h, head_mask], + training=training) + + if self.output_attentions: + attn_vec_h, attn_prob_h = attn_vec_h + + # post processing + output_h = self.post_attention([h, attn_vec_h], training=training) + + ##### g-stream + # query-stream query head + q_head_g = torch.einsum('ibh,hnd->ibnd', g, self.q) + + # core attention ops + if target_mapping is not None: + q_head_g = torch.einsum('mbnd,mlb->lbnd', q_head_g, target_mapping) + attn_vec_g = self.rel_attn_core( + [q_head_g, k_head_h, v_head_h, k_head_r, seg_mat, attn_mask_g, head_mask], + training=training) + + if self.output_attentions: + attn_vec_g, attn_prob_g = attn_vec_g + + attn_vec_g = torch.einsum('lbnd,mlb->mbnd', attn_vec_g, target_mapping) + else: + attn_vec_g = self.rel_attn_core( + [q_head_g, k_head_h, v_head_h, k_head_r, seg_mat, attn_mask_g, head_mask], + training=training) + + if self.output_attentions: + attn_vec_g, attn_prob_g = attn_vec_g + + # post processing + output_g = self.post_attention([g, attn_vec_g], training=training) + + if self.output_attentions: + attn_prob = attn_prob_h, attn_prob_g + + else: + ###### Multi-head attention with relative positional encoding + if mems is not None and mems.dim() > 1: + cat = tf.concat([mems, h], dim=0) + else: + cat = h + + # content heads + q_head_h = tf.einsum('ibh,hnd->ibnd', h, self.q) + k_head_h = tf.einsum('ibh,hnd->ibnd', cat, self.k) + v_head_h = tf.einsum('ibh,hnd->ibnd', cat, self.v) + + # positional heads + k_head_r = tf.einsum('ibh,hnd->ibnd', r, self.r) + + # core attention ops + attn_vec = self.rel_attn_core( + [q_head_h, k_head_h, v_head_h, k_head_r, seg_mat, attn_mask_h, head_mask], + training=training) + + if self.output_attentions: + attn_vec, attn_prob = attn_vec + + # post processing + output_h = self.post_attention([h, attn_vec], training=training) + output_g = None + + outputs = (output_h, output_g) + if self.output_attentions: + outputs = outputs + (attn_prob,) + return outputs + +class TFXLNetFeedForward(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFXLNetFeedForward, self).__init__(**kwargs) + self.layer_norm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name='layer_norm') + self.layer_1 = tf.keras.layers.Dense(config.d_inner, name='layer_1') + self.layer_2 = tf.keras.layers.Dense(config.d_model, name='layer_2') + self.dropout = tf.keras.layers.Dropout(config.dropout) + if isinstance(config.ff_activation, str) or \ + (sys.version_info[0] == 2 and isinstance(config.ff_activation, unicode)): + self.activation_function = ACT2FN[config.ff_activation] + else: + self.activation_function = config.ff_activation + + def call(self, inp, training=False): + output = inp + output = self.layer_1(output) + output = self.activation_function(output) + if training: + output = self.dropout(output) + output = self.layer_2(output) + if training: + output = self.dropout(output) + output = self.layer_norm(output + inp) + return output + +class TFXLNetLayer(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFXLNetLayer, self).__init__(**kwargs) + self.rel_attn = TFXLNetRelativeAttention(config, name='rel_attn') + self.ff = TFXLNetFeedForward(config, name='ff') + self.dropout = tf.keras.layers.Dropout(config.dropout) + + def call(self, inputs, training=False): + outputs = self.rel_attn(inputs, training=training) + output_h, output_g = outputs[:2] + + if output_g is not None: + output_g = self.ff(output_g, training=training) + output_h = self.ff(output_h, training=training) + + outputs = (output_h, output_g) + outputs[2:] # Add again attentions if there are there + return outputs + + +class TFXLNetMainLayer(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFXLNetMainLayer, self).__init__(**kwargs) + self.output_attentions = config.output_attentions + self.output_hidden_states = config.output_hidden_states + + self.mem_len = config.mem_len + self.reuse_len = config.reuse_len + self.d_model = config.d_model + self.same_length = config.same_length + self.attn_type = config.attn_type + self.bi_data = config.bi_data + self.clamp_len = config.clamp_len + self.n_layer = config.n_layer + self.use_bfloat16 = config.use_bfloat16 + self.initializer_range = config.initializer_range + + self.word_embedding = TFSharedEmbeddings(config.n_token, config.d_model, initializer_range=config.initializer_range, name='word_embedding') + self.layer = [XLNetLayer(config, name='layer_{}'.format(i)) for i in range(config.n_layer)] + self.dropout = tf.keras.layers.Dropout(config.dropout) + + def build(input_shape): + initializer = tf.random_normal_initializer(mean=0., stddev=self.initializer_range) + self.mask_emb = self.add_weight(shape=(1, 1, config.d_model), + initializer=initializer, + trainable=True, name='mask_emb') + + def _resize_token_embeddings(self, new_num_tokens): + raise NotImplementedError + + def _prune_heads(self, heads_to_prune): + raise NotImplementedError + + def create_mask(self, qlen, mlen, dtype=tf.float32): + """ + Creates causal attention mask. Float mask where 1.0 indicates masked, 0.0 indicates not-masked. + + Args: + qlen: TODO Lysandre didn't fill + mlen: TODO Lysandre didn't fill + + :: + + same_length=False: same_length=True: + < qlen > < qlen > + ^ [0 0 0 0 0 1 1 1 1] [0 0 0 0 0 1 1 1 1] + [0 0 0 0 0 0 1 1 1] [1 0 0 0 0 0 1 1 1] + qlen [0 0 0 0 0 0 0 1 1] [1 1 0 0 0 0 0 1 1] + [0 0 0 0 0 0 0 0 1] [1 1 1 0 0 0 0 0 1] + v [0 0 0 0 0 0 0 0 0] [1 1 1 1 0 0 0 0 0] + + """ + attn_mask = tf.ones([qlen, qlen], dtype=dtype) + mask_u = tf.matrix_band_part(attn_mask, 0, -1) + mask_dia = tf.matrix_band_part(attn_mask, 0, 0) + attn_mask_pad = tf.zeros([qlen, mlen], dtype=dtype) + ret = tf.concat([attn_mask_pad, mask_u - mask_dia], 1) + if self.same_length: + mask_l = tf.matrix_band_part(attn_mask, -1, 0) + ret = tf.concat([ret[:, :qlen] + mask_l - mask_dia, ret[:, qlen:]], 1) + return ret + + def cache_mem(self, curr_out, prev_mem): + """cache hidden states into memory.""" + if self.mem_len is None or self.mem_len == 0: + return None + else: + if self.reuse_len is not None and self.reuse_len > 0: + curr_out = curr_out[:self.reuse_len] + + if prev_mem is None: + new_mem = curr_out[-self.mem_len:] + else: + new_mem = tf.concat([prev_mem, curr_out], 0)[-mem_len:] + + return tf.stop_gradient(new_mem) + + @staticmethod + def positional_embedding(pos_seq, inv_freq, bsz=None): + sinusoid_inp = tf.einsum('i,d->id', pos_seq, inv_freq) + pos_emb = tf.concat([tf.sin(sinusoid_inp), tf.cos(sinusoid_inp)], axis=-1) + pos_emb = pos_emb[:, None, :] + + if bsz is not None: + pos_emb = tf.tile(pos_emb, [1, bsz, 1]) + + return pos_emb + + def relative_positional_encoding(self, qlen, klen, bsz=None, dtype=None): + """create relative positional encoding.""" + freq_seq = tf.range(0, self.d_model, 2.0) + if dtype is not None and dtype != tf.float32: + freq_seq = tf.cast(freq_seq, dtype=dtype) + inv_freq = 1 / (10000 ** (freq_seq / self.d_model)) + + if self.attn_type == 'bi': + # beg, end = klen - 1, -qlen + beg, end = klen, -qlen + elif self.attn_type == 'uni': + # beg, end = klen - 1, -1 + beg, end = klen, -1 + else: + raise ValueError('Unknown `attn_type` {}.'.format(self.attn_type)) + + if self.bi_data: + fwd_pos_seq = tf.range(beg, end, -1.0) + bwd_pos_seq = tf.range(-beg, -end, 1.0) + + if dtype is not None and dtype != tf.float32: + fwd_pos_seq = tf.cast(fwd_pos_seq, dtype=dtype) + bwd_pos_seq = tf.cast(bwd_pos_seq, dtype=dtype) + + if self.clamp_len > 0: + fwd_pos_seq = tf.clip_by_value(fwd_pos_seq, -self.clamp_len, self.clamp_len) + bwd_pos_seq = tf.clip_by_value(bwd_pos_seq, -self.clamp_len, self.clamp_len) + + if bsz is not None: + # With bi_data, the batch size should be divisible by 2. + assert bsz%2 == 0 + fwd_pos_emb = self.positional_embedding(fwd_pos_seq, inv_freq, bsz//2) + bwd_pos_emb = self.positional_embedding(bwd_pos_seq, inv_freq, bsz//2) + else: + fwd_pos_emb = self.positional_embedding(fwd_pos_seq, inv_freq) + bwd_pos_emb = self.positional_embedding(bwd_pos_seq, inv_freq) + + pos_emb = tf.concat([fwd_pos_emb, bwd_pos_emb], axis=1) + else: + fwd_pos_seq = tf.range(beg, end, -1.0) + if dtype is not None and dtype != tf.float32: + fwd_pos_seq = tf.cast(fwd_pos_seq, dtype=dtype) + if self.clamp_len > 0: + fwd_pos_seq = tf.clip_by_value(fwd_pos_seq, -clamp_len, clamp_len) + pos_emb = self.positional_embedding(fwd_pos_seq, inv_freq, bsz) + + return pos_emb + + def call(self, inputs, training=False): + (input_ids, attention_mask, mems, perm_mask, target_mapping, + token_type_ids, input_mask, head_mask) = inputs + # the original code for XLNet uses shapes [len, bsz] with the batch dimension at the end + # but we want a unified interface in the library with the batch size on the first dimension + # so we move here the first dimension (batch) to the end + + input_ids = tf.transpose(input_ids, perm=(0, 1)) + token_type_ids = tf.transpose(token_type_ids, perm=(0, 1)) if token_type_ids is not None else None + input_mask = tf.transpose(input_mask, perm=(0, 1)) if input_mask is not None else None + attention_mask = tf.transpose(attention_mask, perm=(0, 1)) if attention_mask is not None else None + perm_mask = tf.transpose(perm_mask, perm=(1, 2, 0)) if perm_mask is not None else None + target_mapping = tf.transpose(target_mapping, perm=(1, 2, 0)) if target_mapping is not None else None + + qlen, bsz = shape_list(input_ids)[:2] + mlen = shape_list(mems[0])[0] if mems is not None and mems[0] is not None else 0 + klen = mlen + qlen + + dtype_float = tf.bfloat16 if self.use_bfloat16 else tf.float32 + + ##### Attention mask + # causal attention mask + if self.attn_type == 'uni': + attn_mask = self.create_mask(qlen, mlen) + attn_mask = attn_mask[:, :, None, None] + elif self.attn_type == 'bi': + attn_mask = None + else: + raise ValueError('Unsupported attention type: {}'.format(self.attn_type)) + + # data mask: input mask & perm mask + assert input_mask is None or attention_mask is None, "You can only use one of input_mask (uses 1 for padding) " + "or attention_mask (uses 0 for padding, added for compatbility with BERT). Please choose one." + if input_mask is None and attention_mask is not None: + input_mask = 1.0 - attention_mask + if input_mask is not None and perm_mask is not None: + data_mask = input_mask[None] + perm_mask + elif input_mask is not None and perm_mask is None: + data_mask = input_mask[None] + elif input_mask is None and perm_mask is not None: + data_mask = perm_mask + else: + data_mask = None + + if data_mask is not None: + # all mems can be attended to + mems_mask = tf.zeros([tf.shape(data_mask)[0], mlen, bsz], + dtype=dtype_float) + data_mask = tf.concat([mems_mask, data_mask], axis=1) + if attn_mask is None: + attn_mask = data_mask[:, :, :, None] + else: + attn_mask += data_mask[:, :, :, None] + + if attn_mask is not None: + attn_mask = tf.cast(attn_mask > 0, dtype=dtype_float) + + if attn_mask is not None: + non_tgt_mask = -tf.eye(qlen, dtype=dtype_float) + non_tgt_mask = tf.concat([tf.zeros([qlen, mlen], dtype=dtype_float), non_tgt_mask], axis=-1) + non_tgt_mask = tf.cast((attn_mask + non_tgt_mask[:, :, None, None]) > 0, dtype=dtype_float) + else: + non_tgt_mask = None + + ##### Word embeddings and prepare h & g hidden states + word_emb_k = self.word_embedding(input_ids) + if training: + output_h = self.dropout(word_emb_k) + if target_mapping is not None: + word_emb_q = tf.tile(mask_emb, [tf.shape(target_mapping)[0], bsz, 1]) + # else: # We removed the inp_q input which was same as target mapping + # inp_q_ext = inp_q[:, :, None] + # word_emb_q = inp_q_ext * self.mask_emb + (1 - inp_q_ext) * word_emb_k + if training: + output_g = self.dropout(word_emb_q) + else: + output_g = None + + ##### Segment embedding + if token_type_ids is not None: + # Convert `token_type_ids` to one-hot `seg_mat` + mem_pad = tf.zeros([mlen, bsz], dtype=tf.int32) + cat_ids = tf.concat([mem_pad, token_type_ids], 0) + + # `1` indicates not in the same segment [qlen x klen x bsz] + seg_mat = tf.cast( + tf.logical_not(tf.equal(token_type_ids[:, None], cat_ids[None, :])), + tf.int32) + seg_mat = tf.one_hot(seg_mat, 2, dtype=dtype_float) + else: + seg_mat = None + + ##### Positional encoding + pos_emb = self.relative_positional_encoding(qlen, klen, bsz=bsz, dtype=dtype_float) + if training: + pos_emb = self.dropout(pos_emb) + + # Prepare head mask if needed + # 1.0 in head_mask indicate we keep the head + # attention_probs has shape bsz x n_heads x N x N + # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] (a head_mask for each layer) + # and head_mask is converted to shape [num_hidden_layers x qlen x klen x bsz x n_head] + if head_mask is not None: + if head_mask.dim() == 1: + head_mask = head_mask.unsqueeze(0).unsqueeze(0).unsqueeze(0).unsqueeze(0) + head_mask = head_mask.expand(self.n_layer, -1, -1, -1, -1) + elif head_mask.dim() == 2: + head_mask = head_mask.unsqueeze(1).unsqueeze(1).unsqueeze(1) + head_mask = head_mask.to(dtype=next(self.parameters()).dtype) # switch to fload if need + fp16 compatibility + else: + head_mask = [None] * self.n_layer + + new_mems = () + if mems is None: + mems = [None] * len(self.layer) + + attentions = [] + hidden_states = [] + for i, layer_module in enumerate(self.layer): + # cache new mems + new_mems = new_mems + (self.cache_mem(output_h, mems[i]),) + if self.output_hidden_states: + hidden_states.append((output_h, output_g) if output_g is not None else output_h) + + outputs = layer_module([output_h, output_g, non_tgt_mask, attn_mask, + pos_emb, seg_mat, mems[i], target_mapping, + head_mask[i]], training=training) + output_h, output_g = outputs[:2] + if self.output_attentions: + attentions.append(outputs[2]) + + # Add last hidden state + if self.output_hidden_states: + hidden_states.append((output_h, output_g) if output_g is not None else output_h) + + if training: + output = self.dropout(output_g if output_g is not None else output_h) + + # Prepare outputs, we transpose back here to shape [bsz, len, hidden_dim] (cf. beginning of forward() method) + outputs = (tf.transpose(output, perm=(1, 0, 2)), new_mems) + if self.output_hidden_states: + if output_g is not None: + hidden_states = tuple(tf.transpose(h, perm=(1, 0, 2)) for hs in hidden_states for h in hs) + else: + hidden_states = tuple(tf.transpose(hs, perm=(1, 0, 2)) for hs in hidden_states) + outputs = outputs + (hidden_states,) + if self.output_attentions: + attentions = tuple(tf.transpose(t, perm=(2, 3, 0, 1)) for t in attentions) + outputs = outputs + (attentions,) + + return outputs # outputs, new_mems, (hidden_states), (attentions) + + +class TFXLNetPreTrainedModel(TFPreTrainedModel): + """ An abstract class to handle weights initialization and + a simple interface for dowloading and loading pretrained models. + """ + config_class = XLNetConfig + pretrained_model_archive_map = TF_XLNET_PRETRAINED_MODEL_ARCHIVE_MAP + load_pt_weights = load_xlnet_pt_weights_in_tf2 + base_model_prefix = "transformer" + + +XLNET_START_DOCSTRING = r""" The XLNet model was proposed in + `XLNet: Generalized Autoregressive Pretraining for Language Understanding`_ + by Zhilin Yang*, Zihang Dai*, Yiming Yang, Jaime Carbonell, Ruslan Salakhutdinov, Quoc V. Le. + XLnet is an extension of the Transformer-XL model pre-trained using an autoregressive method + to learn bidirectional contexts by maximizing the expected likelihood over all permutations + of the input sequence factorization order. + + The specific attention pattern can be controlled at training and test time using the `perm_mask` input. + + Do to the difficulty of training a fully auto-regressive model over various factorization order, + XLNet is pretrained using only a sub-set of the output tokens as target which are selected + with the `target_mapping` input. + + To use XLNet for sequential decoding (i.e. not in fully bi-directional setting), use the `perm_mask` and + `target_mapping` inputs to control the attention span and outputs (see examples in `examples/run_generation.py`) + + This model is a PyTorch `torch.tf.keras.layers.Layer`_ sub-class. Use it as a regular PyTorch Module and + refer to the PyTorch documentation for all matter related to general usage and behavior. + + .. _`XLNet: Generalized Autoregressive Pretraining for Language Understanding`: + http://arxiv.org/abs/1906.08237 + + .. _`torch.tf.keras.layers.Layer`: + https://pytorch.org/docs/stable/nn.html#module + + Parameters: + config (:class:`~pytorch_transformers.XLNetConfig`): Model configuration class with all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the configuration. + Check out the :meth:`~pytorch_transformers.PreTrainedModel.from_pretrained` method to load the model weights. +""" + +XLNET_INPUTS_DOCSTRING = r""" + Inputs: + **input_ids**: ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Indices of input sequence tokens in the vocabulary. + XLNet is a model with relative position embeddings so you can either pad the inputs on + the right or on the left. + Indices can be obtained using :class:`pytorch_transformers.XLNetTokenizer`. + See :func:`pytorch_transformers.PreTrainedTokenizer.encode` and + :func:`pytorch_transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``: + Mask to avoid performing attention on padding token indices. + Mask values selected in ``[0, 1]``: + ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. + **mems**: (`optional`) + list of ``torch.FloatTensor`` (one for each layer): + that contains pre-computed hidden-states (key and values in the attention blocks) as output by the model + (see `mems` output below). Can be used to speed up sequential decoding and attend to longer context. + To activate mems you need to set up config.mem_len to a positive value which will be the max number of tokens in + the memory output by the model. E.g. `model = XLNetModel.from_pretrained('xlnet-base-case, mem_len=1024)` will + instantiate a model which can use up to 1024 tokens of memory (in addition to the input it self). + **perm_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, sequence_length)``: + Mask to indicate the attention pattern for each input token with values selected in ``[0, 1]``: + If ``perm_mask[k, i, j] = 0``, i attend to j in batch k; + if ``perm_mask[k, i, j] = 1``, i does not attend to j in batch k. + If None, each token attends to all the others (full bidirectional attention). + Only used during pretraining (to define factorization order) or for sequential decoding (generation). + **target_mapping**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, num_predict, sequence_length)``: + Mask to indicate the output tokens to use. + If ``target_mapping[k, i, j] = 1``, the i-th predict in batch k is on the j-th token. + Only used during pretraining for partial prediction or for sequential decoding (generation). + **token_type_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + A parallel sequence of tokens (can be used to indicate various portions of the inputs). + The type indices in XLNet are NOT selected in the vocabulary, they can be arbitrary numbers and + the important thing is that they should be different for tokens which belong to different segments. + The model will compute relative segment differences from the given type indices: + 0 if the segment id of two tokens are the same, 1 if not. + **input_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``: + Mask to avoid performing attention on padding token indices. + Negative of `attention_mask`, i.e. with 0 for real tokens and 1 for padding. + Kept for compatibility with the original code base. + You can only uses one of `input_mask` and `attention_mask` + Mask values selected in ``[0, 1]``: + ``1`` for tokens that are MASKED, ``0`` for tokens that are NOT MASKED. + **head_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: + Mask to nullify selected heads of the self-attention modules. + Mask values selected in ``[0, 1]``: + ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. +""" + +@add_start_docstrings("The bare XLNet Model transformer outputing raw hidden-states without any specific head on top.", + XLNET_START_DOCSTRING, XLNET_INPUTS_DOCSTRING) +class TFXLNetModel(TFXLNetPreTrainedModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)`` + Sequence of hidden-states at the last layer of the model. + **mems**: + list of ``torch.FloatTensor`` (one for each layer): + that contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model + if config.mem_len > 0 else tuple of None. Can be used to speed up sequential decoding and attend to longer context. + See details in the docstring of the `mems` input above. + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = XLNetTokenizer.from_pretrained('xlnet-large-cased') + model = XLNetModel.from_pretrained('xlnet-large-cased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids) + last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple + + """ + def __init__(self, config, *inputs, **kwargs): + super(TFXLNetModel, self).__init__(config, *inputs, **kwargs) + self.transformer = TFBertMainLayer(config, name='transformer') + + def call(self, inputs, training=False): + outputs = self.transformer(inputs, training=training) + return outputs + + +# @add_start_docstrings("""XLNet Model with a language modeling head on top +# (linear layer with weights tied to the input embeddings). """, +# XLNET_START_DOCSTRING, XLNET_INPUTS_DOCSTRING) +# class XLNetLMHeadModel(XLNetPreTrainedModel): +# r""" +# **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: +# Labels for language modeling. +# Note that the labels **are shifted** inside the model, i.e. you can set ``lm_labels = input_ids`` +# Indices are selected in ``[-1, 0, ..., config.vocab_size]`` +# All labels set to ``-1`` are ignored (masked), the loss is only +# computed for labels in ``[0, ..., config.vocab_size]`` + +# Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: +# **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: +# Language modeling loss. +# **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` +# Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). +# **mems**: +# list of ``torch.FloatTensor`` (one for each layer): +# that contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model +# if config.mem_len > 0 else tuple of None. Can be used to speed up sequential decoding and attend to longer context. +# See details in the docstring of the `mems` input above. +# **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) +# list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) +# of shape ``(batch_size, sequence_length, hidden_size)``: +# Hidden-states of the model at the output of each layer plus the initial embedding outputs. +# **attentions**: (`optional`, returned when ``config.output_attentions=True``) +# list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: +# Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + +# Examples:: + +# tokenizer = XLNetTokenizer.from_pretrained('xlnet-large-cased') +# model = XLNetLMHeadModel.from_pretrained('xlnet-large-cased') +# # We show how to setup inputs to predict a next token using a bi-directional context. +# input_ids = torch.tensor(tokenizer.encode("Hello, my dog is very ")).unsqueeze(0) # We will predict the masked token +# perm_mask = torch.zeros((1, input_ids.shape[1], input_ids.shape[1]), dtype=torch.float) +# perm_mask[:, :, -1] = 1.0 # Previous tokens don't see last token +# target_mapping = torch.zeros((1, 1, input_ids.shape[1]), dtype=torch.float) # Shape [1, 1, seq_length] => let's predict one token +# target_mapping[0, 0, -1] = 1.0 # Our first (and only) prediction will be the last token of the sequence (the masked token) +# outputs = model(input_ids, perm_mask=perm_mask, target_mapping=target_mapping) +# next_token_logits = outputs[0] # Output has shape [target_mapping.size(0), target_mapping.size(1), config.vocab_size] + +# """ +# def __init__(self, config, **kwargs): +# super(XLNetLMHeadModel, self).__init__(config) +# self.attn_type = config.attn_type +# self.same_length = config.same_length + +# self.transformer = XLNetModel(config) +# self.lm_loss = nn.Linear(config.d_model, config.n_token, bias=True) + +# self.init_weights() +# self.tie_weights() + +# def tie_weights(self): +# """ Make sure we are sharing the embeddings +# """ +# self._tie_or_clone_weights(self.lm_loss, self.transformer.word_embedding) + +# def forward(self, input_ids, attention_mask=None, mems=None, perm_mask=None, target_mapping=None, +# token_type_ids=None, input_mask=None, head_mask=None, labels=None): +# transformer_outputs = self.transformer(input_ids, +# attention_mask=attention_mask, +# mems=mems, +# perm_mask=perm_mask, +# target_mapping=target_mapping, +# token_type_ids=token_type_ids, +# input_mask=input_mask, +# head_mask=head_mask) + +# logits = self.lm_loss(transformer_outputs[0]) + +# outputs = (logits,) + transformer_outputs[1:] # Keep mems, hidden states, attentions if there are in it + +# if labels is not None: +# # Flatten the tokens +# loss_fct = CrossEntropyLoss(ignore_index=-1) +# loss = loss_fct(logits.view(-1, logits.size(-1)), +# labels.view(-1)) +# outputs = (loss,) + outputs + +# return outputs # return (loss), logits, mems, (hidden states), (attentions) + + +# @add_start_docstrings("""XLNet Model with a sequence classification/regression head on top (a linear layer on top of +# the pooled output) e.g. for GLUE tasks. """, +# XLNET_START_DOCSTRING, XLNET_INPUTS_DOCSTRING) +# class XLNetForSequenceClassification(XLNetPreTrainedModel): +# r""" +# **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: +# Labels for computing the sequence classification/regression loss. +# Indices should be in ``[0, ..., config.num_labels - 1]``. +# If ``config.num_labels == 1`` a regression loss is computed (Mean-Square loss), +# If ``config.num_labels > 1`` a classification loss is computed (Cross-Entropy). + +# Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: +# **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: +# Classification (or regression if config.num_labels==1) loss. +# **logits**: ``torch.FloatTensor`` of shape ``(batch_size, config.num_labels)`` +# Classification (or regression if config.num_labels==1) scores (before SoftMax). +# **mems**: +# list of ``torch.FloatTensor`` (one for each layer): +# that contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model +# if config.mem_len > 0 else tuple of None. Can be used to speed up sequential decoding and attend to longer context. +# See details in the docstring of the `mems` input above. +# **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) +# list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) +# of shape ``(batch_size, sequence_length, hidden_size)``: +# Hidden-states of the model at the output of each layer plus the initial embedding outputs. +# **attentions**: (`optional`, returned when ``config.output_attentions=True``) +# list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: +# Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + +# Examples:: + +# tokenizer = XLNetTokenizer.from_pretrained('xlnet-large-cased') +# model = XLNetForSequenceClassification.from_pretrained('xlnet-large-cased') +# input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 +# labels = torch.tensor([1]).unsqueeze(0) # Batch size 1 +# outputs = model(input_ids, labels=labels) +# loss, logits = outputs[:2] + +# """ +# def __init__(self, config, **kwargs): +# super(XLNetForSequenceClassification, self).__init__(config) +# self.num_labels = config.num_labels + +# self.transformer = XLNetModel(config) +# self.sequence_summary = SequenceSummary(config) +# self.logits_proj = nn.Linear(config.d_model, config.num_labels) + +# self.init_weights() + +# def forward(self, input_ids, attention_mask=None, mems=None, perm_mask=None, target_mapping=None, +# token_type_ids=None, input_mask=None, head_mask=None, labels=None): +# transformer_outputs = self.transformer(input_ids, +# attention_mask=attention_mask, +# mems=mems, +# perm_mask=perm_mask, +# target_mapping=target_mapping, +# token_type_ids=token_type_ids, +# input_mask=input_mask, +# head_mask=head_mask) +# output = transformer_outputs[0] + +# output = self.sequence_summary(output) +# logits = self.logits_proj(output) + +# outputs = (logits,) + transformer_outputs[1:] # Keep mems, hidden states, attentions if there are in it + +# if labels is not None: +# if self.num_labels == 1: +# # We are doing regression +# loss_fct = MSELoss() +# loss = loss_fct(logits.view(-1), labels.view(-1)) +# else: +# loss_fct = CrossEntropyLoss() +# loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1)) +# outputs = (loss,) + outputs + +# return outputs # return (loss), logits, mems, (hidden states), (attentions) + + +# @add_start_docstrings("""XLNet Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layers on top of +# the hidden-states output to compute `span start logits` and `span end logits`). """, +# XLNET_START_DOCSTRING, XLNET_INPUTS_DOCSTRING) +# class XLNetForQuestionAnswering(XLNetPreTrainedModel): +# r""" +# **start_positions**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: +# Labels for position (index) of the start of the labelled span for computing the token classification loss. +# Positions are clamped to the length of the sequence (`sequence_length`). +# Position outside of the sequence are not taken into account for computing the loss. +# **end_positions**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: +# Labels for position (index) of the end of the labelled span for computing the token classification loss. +# Positions are clamped to the length of the sequence (`sequence_length`). +# Position outside of the sequence are not taken into account for computing the loss. +# **is_impossible**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: +# Labels whether a question has an answer or no answer (SQuAD 2.0) +# **cls_index**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: +# Labels for position (index) of the classification token to use as input for computing plausibility of the answer. +# **p_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``: +# Optional mask of tokens which can't be in answers (e.g. [CLS], [PAD], ...). +# 1.0 means token should be masked. 0.0 mean token is not masked. + +# Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: +# **loss**: (`optional`, returned if both ``start_positions`` and ``end_positions`` are provided) ``torch.FloatTensor`` of shape ``(1,)``: +# Classification loss as the sum of start token, end token (and is_impossible if provided) classification losses. +# **start_top_log_probs**: (`optional`, returned if ``start_positions`` or ``end_positions`` is not provided) +# ``torch.FloatTensor`` of shape ``(batch_size, config.start_n_top)`` +# Log probabilities for the top config.start_n_top start token possibilities (beam-search). +# **start_top_index**: (`optional`, returned if ``start_positions`` or ``end_positions`` is not provided) +# ``torch.LongTensor`` of shape ``(batch_size, config.start_n_top)`` +# Indices for the top config.start_n_top start token possibilities (beam-search). +# **end_top_log_probs**: (`optional`, returned if ``start_positions`` or ``end_positions`` is not provided) +# ``torch.FloatTensor`` of shape ``(batch_size, config.start_n_top * config.end_n_top)`` +# Log probabilities for the top ``config.start_n_top * config.end_n_top`` end token possibilities (beam-search). +# **end_top_index**: (`optional`, returned if ``start_positions`` or ``end_positions`` is not provided) +# ``torch.LongTensor`` of shape ``(batch_size, config.start_n_top * config.end_n_top)`` +# Indices for the top ``config.start_n_top * config.end_n_top`` end token possibilities (beam-search). +# **cls_logits**: (`optional`, returned if ``start_positions`` or ``end_positions`` is not provided) +# ``torch.FloatTensor`` of shape ``(batch_size,)`` +# Log probabilities for the ``is_impossible`` label of the answers. +# **mems**: +# list of ``torch.FloatTensor`` (one for each layer): +# that contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model +# if config.mem_len > 0 else tuple of None. Can be used to speed up sequential decoding and attend to longer context. +# See details in the docstring of the `mems` input above. +# **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) +# list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) +# of shape ``(batch_size, sequence_length, hidden_size)``: +# Hidden-states of the model at the output of each layer plus the initial embedding outputs. +# **attentions**: (`optional`, returned when ``config.output_attentions=True``) +# list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: +# Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + +# Examples:: + +# tokenizer = XLMTokenizer.from_pretrained('xlm-mlm-en-2048') +# model = XLMForQuestionAnswering.from_pretrained('xlnet-large-cased') +# input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 +# start_positions = torch.tensor([1]) +# end_positions = torch.tensor([3]) +# outputs = model(input_ids, start_positions=start_positions, end_positions=end_positions) +# loss, start_scores, end_scores = outputs[:2] + +# """ +# def __init__(self, config, **kwargs): +# super(XLNetForQuestionAnswering, self).__init__(config) +# self.start_n_top = config.start_n_top +# self.end_n_top = config.end_n_top + +# self.transformer = XLNetModel(config) +# self.start_logits = PoolerStartLogits(config) +# self.end_logits = PoolerEndLogits(config) +# self.answer_class = PoolerAnswerClass(config) + +# self.init_weights() + +# def forward(self, input_ids, attention_mask=None, mems=None, perm_mask=None, target_mapping=None, +# token_type_ids=None, input_mask=None, head_mask=None, +# start_positions=None, end_positions=None, is_impossible=None, cls_index=None, p_mask=None,): +# transformer_outputs = self.transformer(input_ids, +# attention_mask=attention_mask, +# mems=mems, +# perm_mask=perm_mask, +# target_mapping=target_mapping, +# token_type_ids=token_type_ids, +# input_mask=input_mask, +# head_mask=head_mask) +# hidden_states = transformer_outputs[0] +# start_logits = self.start_logits(hidden_states, p_mask=p_mask) + +# outputs = transformer_outputs[1:] # Keep mems, hidden states, attentions if there are in it + +# if start_positions is not None and end_positions is not None: +# # If we are on multi-GPU, let's remove the dimension added by batch splitting +# for x in (start_positions, end_positions, cls_index, is_impossible): +# if x is not None and x.dim() > 1: +# x.squeeze_(-1) + +# # during training, compute the end logits based on the ground truth of the start position +# end_logits = self.end_logits(hidden_states, start_positions=start_positions, p_mask=p_mask) + +# loss_fct = CrossEntropyLoss() +# start_loss = loss_fct(start_logits, start_positions) +# end_loss = loss_fct(end_logits, end_positions) +# total_loss = (start_loss + end_loss) / 2 + +# if cls_index is not None and is_impossible is not None: +# # Predict answerability from the representation of CLS and START +# cls_logits = self.answer_class(hidden_states, start_positions=start_positions, cls_index=cls_index) +# loss_fct_cls = nn.BCEWithLogitsLoss() +# cls_loss = loss_fct_cls(cls_logits, is_impossible) + +# # note(zhiliny): by default multiply the loss by 0.5 so that the scale is comparable to start_loss and end_loss +# total_loss += cls_loss * 0.5 + +# outputs = (total_loss,) + outputs + +# else: +# # during inference, compute the end logits based on beam search +# bsz, slen, hsz = hidden_states.size() +# start_log_probs = F.softmax(start_logits, dim=-1) # shape (bsz, slen) + +# start_top_log_probs, start_top_index = torch.topk(start_log_probs, self.start_n_top, dim=-1) # shape (bsz, start_n_top) +# start_top_index_exp = start_top_index.unsqueeze(-1).expand(-1, -1, hsz) # shape (bsz, start_n_top, hsz) +# start_states = torch.gather(hidden_states, -2, start_top_index_exp) # shape (bsz, start_n_top, hsz) +# start_states = start_states.unsqueeze(1).expand(-1, slen, -1, -1) # shape (bsz, slen, start_n_top, hsz) + +# hidden_states_expanded = hidden_states.unsqueeze(2).expand_as(start_states) # shape (bsz, slen, start_n_top, hsz) +# p_mask = p_mask.unsqueeze(-1) if p_mask is not None else None +# end_logits = self.end_logits(hidden_states_expanded, start_states=start_states, p_mask=p_mask) +# end_log_probs = F.softmax(end_logits, dim=1) # shape (bsz, slen, start_n_top) + +# end_top_log_probs, end_top_index = torch.topk(end_log_probs, self.end_n_top, dim=1) # shape (bsz, end_n_top, start_n_top) +# end_top_log_probs = end_top_log_probs.view(-1, self.start_n_top * self.end_n_top) +# end_top_index = end_top_index.view(-1, self.start_n_top * self.end_n_top) + +# start_states = torch.einsum("blh,bl->bh", hidden_states, start_log_probs) # get the representation of START as weighted sum of hidden states +# cls_logits = self.answer_class(hidden_states, start_states=start_states, cls_index=cls_index) # Shape (batch size,): one single `cls_logits` for each sample + +# outputs = (start_top_log_probs, start_top_index, end_top_log_probs, end_top_index, cls_logits) + outputs + +# # return start_top_log_probs, start_top_index, end_top_log_probs, end_top_index, cls_logits +# # or (if labels are provided) (total_loss,) +# return outputs diff --git a/pytorch_transformers/tests/modeling_tf_common_test.py b/pytorch_transformers/tests/modeling_tf_common_test.py index da3263ffde..3dae24b283 100644 --- a/pytorch_transformers/tests/modeling_tf_common_test.py +++ b/pytorch_transformers/tests/modeling_tf_common_test.py @@ -262,7 +262,7 @@ class TFCommonTestCases: # self.assertEqual(len(params_tied_2), len(params_tied)) -def ids_tensor(shape, vocab_size, rng=None, name=None): +def ids_tensor(shape, vocab_size, rng=None, name=None, dtype=tf.int32): """Creates a random int32 tensor of the shape within the vocab size.""" if rng is None: rng = random.Random() @@ -275,7 +275,7 @@ def ids_tensor(shape, vocab_size, rng=None, name=None): for _ in range(total_dims): values.append(rng.randint(0, vocab_size - 1)) - return tf.constant(values, shape=shape) + return tf.constant(values, shape=shape, dtype=dtype) class TFModelUtilsTest(unittest.TestCase): diff --git a/pytorch_transformers/tests/modeling_tf_xlnet_test.py b/pytorch_transformers/tests/modeling_tf_xlnet_test.py new file mode 100644 index 0000000000..58e53e1311 --- /dev/null +++ b/pytorch_transformers/tests/modeling_tf_xlnet_test.py @@ -0,0 +1,341 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import unittest +import json +import random +import shutil +import pytest + +from pytorch_transformers import XLNetConfig, is_tf_available + +if is_tf_available(): + import tensorflow as tf + + from pytorch_transformers.modeling_tf_xlnet import (TFXLNetModel, TF_XLNET_PRETRAINED_MODEL_ARCHIVE_MAP) + # XLNetLMHeadModel, + # XLNetForSequenceClassification, XLNetForQuestionAnswering) +else: + pytestmark = pytest.mark.skip("Require TensorFlow") + +from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) +from .configuration_common_test import ConfigTester + +class TFXLNetModelTest(TFCommonTestCases.TFCommonModelTester): + + all_model_classes=(TFXLNetModel, ) if is_tf_available() else () + # all_model_classes=(TFXLNetModel, TFXLNetLMHeadModel, + # TFXLNetForSequenceClassification, TFXLNetForQuestionAnswering) if is_tf_available() else () + test_pruning = False + + class TFXLNetModelTester(object): + + def __init__(self, + parent, + batch_size=13, + seq_length=7, + mem_len=10, + clamp_len=-1, + reuse_len=15, + is_training=True, + use_labels=True, + vocab_size=99, + cutoffs=[10, 50, 80], + hidden_size=32, + num_attention_heads=4, + d_inner=128, + num_hidden_layers=5, + max_position_embeddings=10, + type_sequence_label_size=2, + untie_r=True, + bi_data=False, + same_length=False, + initializer_range=0.05, + seed=1, + type_vocab_size=2, + ): + self.parent = parent + self.batch_size = batch_size + self.seq_length = seq_length + self.mem_len = mem_len + # self.key_len = seq_length + mem_len + self.clamp_len = clamp_len + self.reuse_len = reuse_len + self.is_training = is_training + self.use_labels = use_labels + self.vocab_size = vocab_size + self.cutoffs = cutoffs + self.hidden_size = hidden_size + self.num_attention_heads = num_attention_heads + self.d_inner = d_inner + self.num_hidden_layers = num_hidden_layers + self.max_position_embeddings = max_position_embeddings + self.bi_data = bi_data + self.untie_r = untie_r + self.same_length = same_length + self.initializer_range = initializer_range + self.seed = seed + self.type_vocab_size = type_vocab_size + self.type_sequence_label_size = type_sequence_label_size + + def prepare_config_and_inputs(self): + input_ids_1 = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) + input_ids_2 = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) + segment_ids = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size) + input_mask = ids_tensor([self.batch_size, self.seq_length], 2, dtype=tf.float32) + + input_ids_q = ids_tensor([self.batch_size, self.seq_length + 1], self.vocab_size) + perm_mask = tf.zeros((self.batch_size, self.seq_length + 1, self.seq_length), dtype=tf.float32) + perm_mask_last = tf.ones((self.batch_size, self.seq_length + 1, 1), dtype=tf.float32) + perm_mask = tf.concat([perm_mask, perm_mask_last], axis=-1) + # perm_mask[:, :, -1] = 1.0 # Previous tokens don't see last token + target_mapping = tf.zeros((self.batch_size, 1, self.seq_length), dtype=torch.float32) + target_mapping_last = tf.ones((self.batch_size, 1, 1), dtype=torch.float32) + target_mapping = tf.concat([target_mapping, target_mapping_last], axis=-1) + # target_mapping[:, 0, -1] = 1.0 # predict last token + + sequence_labels = None + lm_labels = None + is_impossible_labels = None + if self.use_labels: + lm_labels = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) + sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size) + is_impossible_labels = ids_tensor([self.batch_size], 2, dtype=tf.float32) + + config = XLNetConfig( + vocab_size_or_config_json_file=self.vocab_size, + d_model=self.hidden_size, + n_head=self.num_attention_heads, + d_inner=self.d_inner, + n_layer=self.num_hidden_layers, + untie_r=self.untie_r, + max_position_embeddings=self.max_position_embeddings, + mem_len=self.mem_len, + clamp_len=self.clamp_len, + same_length=self.same_length, + reuse_len=self.reuse_len, + bi_data=self.bi_data, + initializer_range=self.initializer_range, + num_labels=self.type_sequence_label_size) + + return (config, input_ids_1, input_ids_2, input_ids_q, perm_mask, input_mask, + target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels) + + def set_seed(self): + random.seed(self.seed) + tf.random.set_seed(self.seed) + + def create_and_check_xlnet_base_model(self, config, input_ids_1, input_ids_2, input_ids_q, perm_mask, input_mask, + target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels): + model = TFXLNetModel(config) + + inputs = {'input_ids': input_ids, + 'input_mask': input_mask, + 'token_type_ids': token_type_ids} + + _, _ = model(inputs) + + inputs = [input_ids, input_mask] + + outputs, mems_1 = model(inputs) + + result = { + "mems_1": [mem.numpy() for m in mems_1], + "outputs": outputs.numpy(), + } + + self.parent.assertListEqual( + list(result["outputs"].shape), + [self.batch_size, self.seq_length, self.hidden_size]) + self.parent.assertListEqual( + list(list(mem.shape) for mem in result["mems_1"]), + [[self.seq_length, self.batch_size, self.hidden_size]] * self.num_hidden_layers) + + def create_and_check_xlnet_lm_head(self, config, input_ids_1, input_ids_2, input_ids_q, perm_mask, input_mask, + target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels): + pass + # model = XLNetLMHeadModel(config) + # model.eval() + + # loss_1, all_logits_1, mems_1 = model(input_ids_1, token_type_ids=segment_ids, labels=lm_labels) + + # loss_2, all_logits_2, mems_2 = model(input_ids_2, token_type_ids=segment_ids, labels=lm_labels, mems=mems_1) + + # logits, _ = model(input_ids_q, perm_mask=perm_mask, target_mapping=target_mapping) + + # result = { + # "loss_1": loss_1, + # "mems_1": mems_1, + # "all_logits_1": all_logits_1, + # "loss_2": loss_2, + # "mems_2": mems_2, + # "all_logits_2": all_logits_2, + # } + + # self.parent.assertListEqual( + # list(result["loss_1"].size()), + # []) + # self.parent.assertListEqual( + # list(result["all_logits_1"].size()), + # [self.batch_size, self.seq_length, self.vocab_size]) + # self.parent.assertListEqual( + # list(list(mem.size()) for mem in result["mems_1"]), + # [[self.seq_length, self.batch_size, self.hidden_size]] * self.num_hidden_layers) + + # self.parent.assertListEqual( + # list(result["loss_2"].size()), + # []) + # self.parent.assertListEqual( + # list(result["all_logits_2"].size()), + # [self.batch_size, self.seq_length, self.vocab_size]) + # self.parent.assertListEqual( + # list(list(mem.size()) for mem in result["mems_2"]), + # [[self.mem_len, self.batch_size, self.hidden_size]] * self.num_hidden_layers) + + def create_and_check_xlnet_qa(self, config, input_ids_1, input_ids_2, input_ids_q, perm_mask, input_mask, + target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels): + pass + # model = XLNetForQuestionAnswering(config) + # model.eval() + + # outputs = model(input_ids_1) + # start_top_log_probs, start_top_index, end_top_log_probs, end_top_index, cls_logits, mems = outputs + + # outputs = model(input_ids_1, start_positions=sequence_labels, + # end_positions=sequence_labels, + # cls_index=sequence_labels, + # is_impossible=is_impossible_labels, + # p_mask=input_mask) + + # outputs = model(input_ids_1, start_positions=sequence_labels, + # end_positions=sequence_labels, + # cls_index=sequence_labels, + # is_impossible=is_impossible_labels) + + # total_loss, mems = outputs + + # outputs = model(input_ids_1, start_positions=sequence_labels, + # end_positions=sequence_labels) + + # total_loss, mems = outputs + + # result = { + # "loss": total_loss, + # "start_top_log_probs": start_top_log_probs, + # "start_top_index": start_top_index, + # "end_top_log_probs": end_top_log_probs, + # "end_top_index": end_top_index, + # "cls_logits": cls_logits, + # "mems": mems, + # } + + # self.parent.assertListEqual( + # list(result["loss"].size()), + # []) + # self.parent.assertListEqual( + # list(result["start_top_log_probs"].size()), + # [self.batch_size, model.config.start_n_top]) + # self.parent.assertListEqual( + # list(result["start_top_index"].size()), + # [self.batch_size, model.config.start_n_top]) + # self.parent.assertListEqual( + # list(result["end_top_log_probs"].size()), + # [self.batch_size, model.config.start_n_top * model.config.end_n_top]) + # self.parent.assertListEqual( + # list(result["end_top_index"].size()), + # [self.batch_size, model.config.start_n_top * model.config.end_n_top]) + # self.parent.assertListEqual( + # list(result["cls_logits"].size()), + # [self.batch_size]) + # self.parent.assertListEqual( + # list(list(mem.size()) for mem in result["mems"]), + # [[self.seq_length, self.batch_size, self.hidden_size]] * self.num_hidden_layers) + + def create_and_check_xlnet_sequence_classif(self, config, input_ids_1, input_ids_2, input_ids_q, perm_mask, input_mask, + target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels): + pass + # model = XLNetForSequenceClassification(config) + # model.eval() + + # logits, mems_1 = model(input_ids_1) + # loss, logits, mems_1 = model(input_ids_1, labels=sequence_labels) + + # result = { + # "loss": loss, + # "mems_1": mems_1, + # "logits": logits, + # } + + # self.parent.assertListEqual( + # list(result["loss"].size()), + # []) + # self.parent.assertListEqual( + # list(result["logits"].size()), + # [self.batch_size, self.type_sequence_label_size]) + # self.parent.assertListEqual( + # list(list(mem.size()) for mem in result["mems_1"]), + # [[self.seq_length, self.batch_size, self.hidden_size]] * self.num_hidden_layers) + + def prepare_config_and_inputs_for_common(self): + config_and_inputs = self.prepare_config_and_inputs() + (config, input_ids_1, input_ids_2, input_ids_q, perm_mask, input_mask, + target_mapping, segment_ids, lm_labels, + sequence_labels, is_impossible_labels) = config_and_inputs + inputs_dict = {'input_ids': input_ids_1} + return config, inputs_dict + + + def setUp(self): + self.model_tester = TFXLNetModelTest.TFXLNetModelTester(self) + self.config_tester = ConfigTester(self, config_class=XLNetConfig, d_inner=37) + + def test_config(self): + self.config_tester.run_common_tests() + + def test_xlnet_base_model(self): + self.model_tester.set_seed() + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_xlnet_base_model(*config_and_inputs) + + def test_xlnet_lm_head(self): + self.model_tester.set_seed() + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_xlnet_lm_head(*config_and_inputs) + + def test_xlnet_sequence_classif(self): + self.model_tester.set_seed() + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_xlnet_sequence_classif(*config_and_inputs) + + def test_xlnet_qa(self): + self.model_tester.set_seed() + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_xlnet_qa(*config_and_inputs) + + @pytest.mark.slow + def test_model_from_pretrained(self): + cache_dir = "/tmp/pytorch_transformers_test/" + for model_name in list(TF_XLNET_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: + model = TFXLNetModel.from_pretrained(model_name, cache_dir=cache_dir) + shutil.rmtree(cache_dir) + self.assertIsNotNone(model) + + +if __name__ == "__main__": + unittest.main() From 16b6361792389416127787019799f20aadb7e18b Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 10 Sep 2019 12:39:27 +0200 Subject: [PATCH 041/219] xlnet paassing first test --- pytorch_transformers/modeling_tf_bert.py | 25 +++--- pytorch_transformers/modeling_tf_gpt2.py | 12 +-- pytorch_transformers/modeling_tf_xlnet.py | 79 +++++++++++-------- .../tests/modeling_tf_xlnet_test.py | 12 +-- 4 files changed, 66 insertions(+), 62 deletions(-) diff --git a/pytorch_transformers/modeling_tf_bert.py b/pytorch_transformers/modeling_tf_bert.py index 4a45707b98..74951fed34 100644 --- a/pytorch_transformers/modeling_tf_bert.py +++ b/pytorch_transformers/modeling_tf_bert.py @@ -218,8 +218,7 @@ class TFBertEmbeddings(tf.keras.layers.Layer): embeddings = words_embeddings + position_embeddings + token_type_embeddings embeddings = self.LayerNorm(embeddings) - if training: - embeddings = self.dropout(embeddings) + embeddings = self.dropout(embeddings, training=training) return embeddings def _linear(self, inputs): @@ -286,10 +285,9 @@ class TFBertSelfAttention(tf.keras.layers.Layer): # Normalize the attention scores to probabilities. attention_probs = tf.nn.softmax(attention_scores, axis=-1) - if training: - # This is actually dropping out entire tokens to attend to, which might - # seem a bit unusual, but is taken from the original Transformer paper. - attention_probs = self.dropout(attention_probs) + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.dropout(attention_probs, training=training) # Mask heads if we want to if head_mask is not None: @@ -316,8 +314,7 @@ class TFBertSelfOutput(tf.keras.layers.Layer): hidden_states, input_tensor = inputs hidden_states = self.dense(hidden_states) - if training: - hidden_states = self.dropout(hidden_states) + hidden_states = self.dropout(hidden_states, training=training) hidden_states = self.LayerNorm(hidden_states + input_tensor) return hidden_states @@ -366,8 +363,7 @@ class TFBertOutput(tf.keras.layers.Layer): hidden_states, input_tensor = inputs hidden_states = self.dense(hidden_states) - if training: - hidden_states = self.dropout(hidden_states) + hidden_states = self.dropout(hidden_states, training=training) hidden_states = self.LayerNorm(hidden_states + input_tensor) return hidden_states @@ -871,8 +867,7 @@ class TFBertForSequenceClassification(TFBertPreTrainedModel): pooled_output = outputs[1] - if training: - pooled_output = self.dropout(pooled_output) + pooled_output = self.dropout(pooled_output, training=training) logits = self.classifier(pooled_output) outputs = (logits,) + outputs[2:] # add hidden states and attention if they are here @@ -947,8 +942,7 @@ class TFBertForMultipleChoice(TFBertPreTrainedModel): pooled_output = outputs[1] - if training: - pooled_output = self.dropout(pooled_output) + pooled_output = self.dropout(pooled_output, training=training) logits = self.classifier(pooled_output) reshaped_logits = tf.reshape(logits, (-1, num_choices)) @@ -995,8 +989,7 @@ class TFBertForTokenClassification(TFBertPreTrainedModel): sequence_output = outputs[0] - if training: - sequence_output = self.dropout(sequence_output) + sequence_output = self.dropout(sequence_output, training=training) logits = self.classifier(sequence_output) outputs = (logits,) + outputs[2:] # add hidden states and attention if they are here diff --git a/pytorch_transformers/modeling_tf_gpt2.py b/pytorch_transformers/modeling_tf_gpt2.py index 900acb94a4..72421370bb 100644 --- a/pytorch_transformers/modeling_tf_gpt2.py +++ b/pytorch_transformers/modeling_tf_gpt2.py @@ -178,8 +178,7 @@ class TFAttention(tf.keras.layers.Layer): w = w + attention_mask w = tf.nn.softmax(w, axis=-1) - if training: - w = self.attn_dropout(w) + w = self.attn_dropout(w, training=training) # Mask heads if we want to if head_mask is not None: @@ -221,8 +220,7 @@ class TFAttention(tf.keras.layers.Layer): a = self.merge_heads(a) a = self.c_proj(a) - if training: - a = self.resid_dropout(a) + a = self.resid_dropout(a, training=training) outputs = [a, present] + attn_outputs[1:] return outputs # a, present, (attentions) @@ -240,8 +238,7 @@ class TFMLP(tf.keras.layers.Layer): def call(self, x, training=False): h = self.act(self.c_fc(x)) h2 = self.c_proj(h) - if training: - h2 = self.dropout(h2) + h2 = self.dropout(h2, training=training) return h2 @@ -368,8 +365,7 @@ class TFGPT2MainLayer(tf.keras.layers.Layer): else: token_type_embeds = 0 hidden_states = inputs_embeds + position_embeds + token_type_embeds - if training: - hidden_states = self.drop(hidden_states) + hidden_states = self.drop(hidden_states, training=training) output_shape = input_shape + [shape_list(hidden_states)[-1]] diff --git a/pytorch_transformers/modeling_tf_xlnet.py b/pytorch_transformers/modeling_tf_xlnet.py index fa24296d76..7708e3abfd 100644 --- a/pytorch_transformers/modeling_tf_xlnet.py +++ b/pytorch_transformers/modeling_tf_xlnet.py @@ -145,7 +145,7 @@ class TFXLNetRelativeAttention(tf.keras.layers.Layer): self.layer_norm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name='layer_norm') self.dropout = tf.keras.layers.Dropout(config.dropout) - def build(input_shape): + def build(self, input_shape): initializer = tf.random_normal_initializer(mean=0., stddev=self.initializer_range) self.q = self.add_weight(shape=(self.d_model, self.n_head, self.d_head), initializer=initializer, @@ -221,10 +221,9 @@ class TFXLNetRelativeAttention(tf.keras.layers.Layer): attn_score = attn_score - 1e30 * attn_mask # attention probability - attn_prob = tf.softmax(attn_score, axis=1) + attn_prob = tf.nn.softmax(attn_score, axis=1) - if training: - attn_prob = self.dropout(attn_prob) + attn_prob = self.dropout(attn_prob, training=training) # Mask heads if we want to if head_mask is not None: @@ -245,10 +244,9 @@ class TFXLNetRelativeAttention(tf.keras.layers.Layer): attn_out = tf.einsum('ibnd,hnd->ibh', attn_vec, self.o) - if training: - attn_out = self.dropout(attn_out) + attn_out = self.dropout(attn_out, training=training) - if residual: + if residual is not None: attn_out = attn_out + h output = self.layer_norm(attn_out) @@ -288,7 +286,7 @@ class TFXLNetRelativeAttention(tf.keras.layers.Layer): attn_vec_h, attn_prob_h = attn_vec_h # post processing - output_h = self.post_attention([h, attn_vec_h], training=training) + output_h = self.post_attention([h, attn_vec_h, None], training=training) ##### g-stream # query-stream query head @@ -314,7 +312,7 @@ class TFXLNetRelativeAttention(tf.keras.layers.Layer): attn_vec_g, attn_prob_g = attn_vec_g # post processing - output_g = self.post_attention([g, attn_vec_g], training=training) + output_g = self.post_attention([g, attn_vec_g, None], training=training) if self.output_attentions: attn_prob = attn_prob_h, attn_prob_g @@ -343,7 +341,7 @@ class TFXLNetRelativeAttention(tf.keras.layers.Layer): attn_vec, attn_prob = attn_vec # post processing - output_h = self.post_attention([h, attn_vec], training=training) + output_h = self.post_attention([h, attn_vec, None], training=training) output_g = None outputs = (output_h, output_g) @@ -368,11 +366,9 @@ class TFXLNetFeedForward(tf.keras.layers.Layer): output = inp output = self.layer_1(output) output = self.activation_function(output) - if training: - output = self.dropout(output) + output = self.dropout(output, training=training) output = self.layer_2(output) - if training: - output = self.dropout(output) + output = self.dropout(output, training=training) output = self.layer_norm(output + inp) return output @@ -413,12 +409,12 @@ class TFXLNetMainLayer(tf.keras.layers.Layer): self.initializer_range = config.initializer_range self.word_embedding = TFSharedEmbeddings(config.n_token, config.d_model, initializer_range=config.initializer_range, name='word_embedding') - self.layer = [XLNetLayer(config, name='layer_{}'.format(i)) for i in range(config.n_layer)] + self.layer = [TFXLNetLayer(config, name='layer_{}'.format(i)) for i in range(config.n_layer)] self.dropout = tf.keras.layers.Dropout(config.dropout) - def build(input_shape): + def build(self, input_shape): initializer = tf.random_normal_initializer(mean=0., stddev=self.initializer_range) - self.mask_emb = self.add_weight(shape=(1, 1, config.d_model), + self.mask_emb = self.add_weight(shape=(1, 1, self.d_model), initializer=initializer, trainable=True, name='mask_emb') @@ -532,16 +528,39 @@ class TFXLNetMainLayer(tf.keras.layers.Layer): return pos_emb def call(self, inputs, training=False): - (input_ids, attention_mask, mems, perm_mask, target_mapping, - token_type_ids, input_mask, head_mask) = inputs + if not isinstance(inputs, (dict, tuple, list)): + input_ids = inputs + (attention_mask, mems, perm_mask, target_mapping, + token_type_ids, input_mask, head_mask) = None, None, None, None, None, None, None + elif isinstance(inputs, (tuple, list)): + input_ids = inputs[0] + attention_mask = inputs[1] if len(inputs) > 1 else None + mems = inputs[2] if len(inputs) > 2 else None + perm_mask = inputs[3] if len(inputs) > 3 else None + target_mapping = inputs[4] if len(inputs) > 4 else None + token_type_ids = inputs[5] if len(inputs) > 5 else None + input_mask = inputs[6] if len(inputs) > 6 else None + head_mask = inputs[7] if len(inputs) > 7 else None + assert len(inputs) <= 8, "Too many inputs." + else: + input_ids = inputs.get('input_ids') + attention_mask = inputs.get('attention_mask', None) + mems = inputs.get('mems', None) + perm_mask = inputs.get('perm_mask', None) + target_mapping = inputs.get('target_mapping', None) + token_type_ids = inputs.get('token_type_ids', None) + input_mask = inputs.get('input_mask', None) + head_mask = inputs.get('head_mask', None) + assert len(inputs) <= 8, "Too many inputs." + # the original code for XLNet uses shapes [len, bsz] with the batch dimension at the end # but we want a unified interface in the library with the batch size on the first dimension # so we move here the first dimension (batch) to the end - input_ids = tf.transpose(input_ids, perm=(0, 1)) - token_type_ids = tf.transpose(token_type_ids, perm=(0, 1)) if token_type_ids is not None else None - input_mask = tf.transpose(input_mask, perm=(0, 1)) if input_mask is not None else None - attention_mask = tf.transpose(attention_mask, perm=(0, 1)) if attention_mask is not None else None + input_ids = tf.transpose(input_ids, perm=(1, 0)) + token_type_ids = tf.transpose(token_type_ids, perm=(1, 0)) if token_type_ids is not None else None + input_mask = tf.transpose(input_mask, perm=(1, 0)) if input_mask is not None else None + attention_mask = tf.transpose(attention_mask, perm=(1, 0)) if attention_mask is not None else None perm_mask = tf.transpose(perm_mask, perm=(1, 2, 0)) if perm_mask is not None else None target_mapping = tf.transpose(target_mapping, perm=(1, 2, 0)) if target_mapping is not None else None @@ -597,15 +616,13 @@ class TFXLNetMainLayer(tf.keras.layers.Layer): ##### Word embeddings and prepare h & g hidden states word_emb_k = self.word_embedding(input_ids) - if training: - output_h = self.dropout(word_emb_k) + output_h = self.dropout(word_emb_k, training=training) if target_mapping is not None: word_emb_q = tf.tile(mask_emb, [tf.shape(target_mapping)[0], bsz, 1]) # else: # We removed the inp_q input which was same as target mapping # inp_q_ext = inp_q[:, :, None] # word_emb_q = inp_q_ext * self.mask_emb + (1 - inp_q_ext) * word_emb_k - if training: - output_g = self.dropout(word_emb_q) + output_g = self.dropout(word_emb_q, training=training) else: output_g = None @@ -625,8 +642,7 @@ class TFXLNetMainLayer(tf.keras.layers.Layer): ##### Positional encoding pos_emb = self.relative_positional_encoding(qlen, klen, bsz=bsz, dtype=dtype_float) - if training: - pos_emb = self.dropout(pos_emb) + pos_emb = self.dropout(pos_emb, training=training) # Prepare head mask if needed # 1.0 in head_mask indicate we keep the head @@ -666,8 +682,7 @@ class TFXLNetMainLayer(tf.keras.layers.Layer): if self.output_hidden_states: hidden_states.append((output_h, output_g) if output_g is not None else output_h) - if training: - output = self.dropout(output_g if output_g is not None else output_h) + output = self.dropout(output_g if output_g is not None else output_h, training=training) # Prepare outputs, we transpose back here to shape [bsz, len, hidden_dim] (cf. beginning of forward() method) outputs = (tf.transpose(output, perm=(1, 0, 2)), new_mems) @@ -805,7 +820,7 @@ class TFXLNetModel(TFXLNetPreTrainedModel): """ def __init__(self, config, *inputs, **kwargs): super(TFXLNetModel, self).__init__(config, *inputs, **kwargs) - self.transformer = TFBertMainLayer(config, name='transformer') + self.transformer = TFXLNetMainLayer(config, name='transformer') def call(self, inputs, training=False): outputs = self.transformer(inputs, training=training) diff --git a/pytorch_transformers/tests/modeling_tf_xlnet_test.py b/pytorch_transformers/tests/modeling_tf_xlnet_test.py index 58e53e1311..8d61d6b5dc 100644 --- a/pytorch_transformers/tests/modeling_tf_xlnet_test.py +++ b/pytorch_transformers/tests/modeling_tf_xlnet_test.py @@ -105,8 +105,8 @@ class TFXLNetModelTest(TFCommonTestCases.TFCommonModelTester): perm_mask_last = tf.ones((self.batch_size, self.seq_length + 1, 1), dtype=tf.float32) perm_mask = tf.concat([perm_mask, perm_mask_last], axis=-1) # perm_mask[:, :, -1] = 1.0 # Previous tokens don't see last token - target_mapping = tf.zeros((self.batch_size, 1, self.seq_length), dtype=torch.float32) - target_mapping_last = tf.ones((self.batch_size, 1, 1), dtype=torch.float32) + target_mapping = tf.zeros((self.batch_size, 1, self.seq_length), dtype=tf.float32) + target_mapping_last = tf.ones((self.batch_size, 1, 1), dtype=tf.float32) target_mapping = tf.concat([target_mapping, target_mapping_last], axis=-1) # target_mapping[:, 0, -1] = 1.0 # predict last token @@ -145,18 +145,18 @@ class TFXLNetModelTest(TFCommonTestCases.TFCommonModelTester): target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels): model = TFXLNetModel(config) - inputs = {'input_ids': input_ids, + inputs = {'input_ids': input_ids_1, 'input_mask': input_mask, - 'token_type_ids': token_type_ids} + 'token_type_ids': segment_ids} _, _ = model(inputs) - inputs = [input_ids, input_mask] + inputs = [input_ids_1, input_mask] outputs, mems_1 = model(inputs) result = { - "mems_1": [mem.numpy() for m in mems_1], + "mems_1": [mem.numpy() for mem in mems_1], "outputs": outputs.numpy(), } From 465870c33fe4ade66863ca0edfe13616f9d24da5 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 10 Sep 2019 16:44:41 +0200 Subject: [PATCH 042/219] Xlnet working - also added simple question answering model for XLNet --- pytorch_transformers/__init__.py | 9 +- pytorch_transformers/configuration_utils.py | 2 +- pytorch_transformers/configuration_xlnet.py | 2 +- .../convert_pytorch_checkpoint_to_tf2.py | 8 +- pytorch_transformers/modeling_tf_bert.py | 4 +- pytorch_transformers/modeling_tf_gpt2.py | 4 +- pytorch_transformers/modeling_tf_utils.py | 2 +- pytorch_transformers/modeling_tf_xlnet.py | 396 +++++++++--------- pytorch_transformers/modeling_xlnet.py | 95 +++++ .../tests/modeling_tf_xlnet_test.py | 181 ++++---- 10 files changed, 384 insertions(+), 319 deletions(-) diff --git a/pytorch_transformers/__init__.py b/pytorch_transformers/__init__.py index dbe979c564..0dc86563aa 100644 --- a/pytorch_transformers/__init__.py +++ b/pytorch_transformers/__init__.py @@ -68,7 +68,8 @@ if _torch_available: GPT2LMHeadModel, GPT2DoubleHeadsModel, load_tf_weights_in_gpt2, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_xlnet import (XLNetPreTrainedModel, XLNetModel, XLNetLMHeadModel, - XLNetForSequenceClassification, XLNetForQuestionAnswering, + XLNetForSequenceClassification, XLNetForQuestionAnsweringSimple, + XLNetForQuestionAnswering, load_tf_weights_in_xlnet, XLNET_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_xlm import (XLMPreTrainedModel , XLMModel, XLMWithLMHeadModel, XLMForSequenceClassification, @@ -112,6 +113,12 @@ if _tf_available: load_gpt2_pt_weights_in_tf2, TF_GPT2_PRETRAINED_MODEL_ARCHIVE_MAP) + from .modeling_tf_xlnet import (TFXLNetPreTrainedModel, TFXLNetMainLayer, + TFXLNetModel, TFXLNetLMHeadModel, + TFXLNetForSequenceClassification, + TFXLNetForQuestionAnsweringSimple, + load_xlnet_pt_weights_in_tf2, + TF_XLNET_PRETRAINED_MODEL_ARCHIVE_MAP) # Files and general utilities from .file_utils import (PYTORCH_TRANSFORMERS_CACHE, PYTORCH_PRETRAINED_BERT_CACHE, diff --git a/pytorch_transformers/configuration_utils.py b/pytorch_transformers/configuration_utils.py index 42346d6b6c..fb1fe82f43 100644 --- a/pytorch_transformers/configuration_utils.py +++ b/pytorch_transformers/configuration_utils.py @@ -175,7 +175,7 @@ class PretrainedConfig(object): """Constructs a `Config` from a Python dictionary of parameters.""" config = cls(vocab_size_or_config_json_file=-1) for key, value in json_object.items(): - config.__dict__[key] = value + setattr(config, key, value) return config @classmethod diff --git a/pytorch_transformers/configuration_xlnet.py b/pytorch_transformers/configuration_xlnet.py index 204d44aa72..cb325dfe17 100644 --- a/pytorch_transformers/configuration_xlnet.py +++ b/pytorch_transformers/configuration_xlnet.py @@ -112,7 +112,7 @@ class XLNetConfig(PretrainedConfig): with open(vocab_size_or_config_json_file, "r", encoding='utf-8') as reader: json_config = json.loads(reader.read()) for key, value in json_config.items(): - self.__dict__[key] = value + setattr(config, key, value) elif isinstance(vocab_size_or_config_json_file, int): self.n_token = vocab_size_or_config_json_file self.d_model = d_model diff --git a/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py b/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py index 03b14d4517..0acaf92788 100644 --- a/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py +++ b/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py @@ -24,12 +24,13 @@ import tensorflow as tf from pytorch_transformers import is_torch_available from pytorch_transformers import (BertConfig, TFBertForPreTraining, load_bert_pt_weights_in_tf2, - GPT2Config, TFGPT2LMHeadModel, load_gpt2_pt_weights_in_tf2) + GPT2Config, TFGPT2LMHeadModel, load_gpt2_pt_weights_in_tf2, + XLNetConfig, TFXLNetLMHeadModel, load_xlnet_pt_weights_in_tf2) if is_torch_available(): import torch import numpy as np - from pytorch_transformers import BertForPreTraining, GPT2LMHeadModel + from pytorch_transformers import BertForPreTraining, GPT2LMHeadModel, XLNetLMHeadModel else: BertForPreTraining, GPT2LMHeadModel = None, None @@ -40,6 +41,7 @@ logging.basicConfig(level=logging.INFO) MODEL_CLASSES = { 'bert': (BertConfig, TFBertForPreTraining, load_bert_pt_weights_in_tf2, BertForPreTraining), 'gpt2': (GPT2Config, TFGPT2LMHeadModel, load_gpt2_pt_weights_in_tf2, GPT2LMHeadModel), + 'xlnet': (XLNetConfig, TFXLNetLMHeadModel, load_xlnet_pt_weights_in_tf2, XLNetLMHeadModel), } def convert_pt_checkpoint_to_tf(model_type, pytorch_checkpoint_path, config_file, tf_dump_path, compare_with_pt_model=False): @@ -50,6 +52,8 @@ def convert_pt_checkpoint_to_tf(model_type, pytorch_checkpoint_path, config_file # Initialise TF model config = config_class.from_json_file(config_file) + config.output_hidden_states = True + config.output_attentions = True print("Building TensorFlow model from configuration: {}".format(str(config))) tf_model = model_class(config) diff --git a/pytorch_transformers/modeling_tf_bert.py b/pytorch_transformers/modeling_tf_bert.py index 74951fed34..f1b5ec7109 100644 --- a/pytorch_transformers/modeling_tf_bert.py +++ b/pytorch_transformers/modeling_tf_bert.py @@ -83,7 +83,7 @@ def load_bert_pt_weights_in_tf2(tf_model, config, pytorch_checkpoint_path): name = name.replace('cls_mlm', 'cls') # We had to split this layer in two in the TF model to be name = name.replace('cls_nsp', 'cls') # able to do transfer learning (Keras only allow to remove full layers) name = name.replace(':0', '') - name = name.replace('layer_', 'layer/') + name = name.replace('__', '/') name = name.split('/') name = name[1:] @@ -391,7 +391,7 @@ class TFBertEncoder(tf.keras.layers.Layer): super(TFBertEncoder, self).__init__(**kwargs) self.output_attentions = config.output_attentions self.output_hidden_states = config.output_hidden_states - self.layer = [TFBertLayer(config, name='layer_{}'.format(i)) for i in range(config.num_hidden_layers)] + self.layer = [TFBertLayer(config, name='layer__{}'.format(i)) for i in range(config.num_hidden_layers)] def call(self, inputs, training=False): hidden_states, attention_mask, head_mask = inputs diff --git a/pytorch_transformers/modeling_tf_gpt2.py b/pytorch_transformers/modeling_tf_gpt2.py index 72421370bb..491b592b13 100644 --- a/pytorch_transformers/modeling_tf_gpt2.py +++ b/pytorch_transformers/modeling_tf_gpt2.py @@ -70,7 +70,7 @@ def load_gpt2_pt_weights_in_tf2(tf_model, config, pytorch_checkpoint_path): for symbolic_weight in symbolic_weights: name = symbolic_weight.name name = name.replace(':0', '') - name = name.replace('h_', 'h/') + name = name.replace('__', '/') name = name.split('/') name = name[2:] @@ -282,7 +282,7 @@ class TFGPT2MainLayer(tf.keras.layers.Layer): self.h = [TFBlock(config.n_ctx, config, scale=True, - name='h_{}'.format(i)) for i in range(config.n_layer)] + name='h__{}'.format(i)) for i in range(config.n_layer)] self.ln_f = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_epsilon, name='ln_f') def _resize_token_embeddings(self, new_num_tokens): diff --git a/pytorch_transformers/modeling_tf_utils.py b/pytorch_transformers/modeling_tf_utils.py index 1860ab4f8b..704de3b62f 100644 --- a/pytorch_transformers/modeling_tf_utils.py +++ b/pytorch_transformers/modeling_tf_utils.py @@ -386,7 +386,7 @@ class TFSequenceSummary(tf.keras.layers.Layer): self.activation = None if hasattr(config, 'summary_activation') and config.summary_activation == 'tanh': - self.activation = tf.keras.layers.Tanh() + self.activation = tf.keras.activations.tanh self.first_dropout = None if hasattr(config, 'summary_first_dropout') and config.summary_first_dropout > 0: diff --git a/pytorch_transformers/modeling_tf_xlnet.py b/pytorch_transformers/modeling_tf_xlnet.py index 7708e3abfd..12f0640531 100644 --- a/pytorch_transformers/modeling_tf_xlnet.py +++ b/pytorch_transformers/modeling_tf_xlnet.py @@ -28,7 +28,7 @@ import numpy as np import tensorflow as tf from .configuration_xlnet import XLNetConfig -from .modeling_tf_utils import TFPreTrainedModel, TFSharedEmbeddings, shape_list +from .modeling_tf_utils import TFPreTrainedModel, TFSharedEmbeddings, TFSequenceSummary, shape_list from .file_utils import add_start_docstrings @@ -72,13 +72,15 @@ def load_xlnet_pt_weights_in_tf2(tf_model, config, pytorch_checkpoint_path): name = name.replace('cls_mlm', 'cls') # We had to split this layer in two in the TF model to be name = name.replace('cls_nsp', 'cls') # able to do transfer learning (Keras only allow to remove full layers) name = name.replace(':0', '') - name = name.replace('layer_', 'layer/') + name = name.replace('layer__', 'layer/') name = name.split('/') name = name[1:] transpose = bool(name[-1] == 'kernel') - if name[-1] == 'kernel' or name[-1] == 'embeddings': + if name[-1] == 'kernel' or name[-1] == 'embeddings' or name[-1] == 'gamma': name[-1] = 'weight' + if name[-1] == 'beta': + name[-1] = 'bias' name = '.'.join(name) assert name in state_dict, "{} not found in PyTorch model".format(name) @@ -237,16 +239,16 @@ class TFXLNetRelativeAttention(tf.keras.layers.Layer): return attn_vec - def post_attention(self, inputs, training=False): + def post_attention(self, inputs, residual=True, training=False): """Post-attention processing.""" # post-attention projection (back to `d_model`) - h, attn_vec, residual = inputs + h, attn_vec = inputs attn_out = tf.einsum('ibnd,hnd->ibh', attn_vec, self.o) attn_out = self.dropout(attn_out, training=training) - if residual is not None: + if residual: attn_out = attn_out + h output = self.layer_norm(attn_out) @@ -259,23 +261,23 @@ class TFXLNetRelativeAttention(tf.keras.layers.Layer): if g is not None: ###### Two-stream attention with relative positional encoding. # content based attention score - if mems is not None and mems.dim() > 1: - cat = torch.cat([mems, h], dim=0) + if mems is not None and mems.shape.ndims > 1: + cat = tf.concat([mems, h], axis=0) else: cat = h # content-based key head - k_head_h = torch.einsum('ibh,hnd->ibnd', cat, self.k) + k_head_h = tf.einsum('ibh,hnd->ibnd', cat, self.k) # content-based value head - v_head_h = torch.einsum('ibh,hnd->ibnd', cat, self.v) + v_head_h = tf.einsum('ibh,hnd->ibnd', cat, self.v) # position-based key head - k_head_r = torch.einsum('ibh,hnd->ibnd', r, self.r) + k_head_r = tf.einsum('ibh,hnd->ibnd', r, self.r) ##### h-stream # content-stream query head - q_head_h = torch.einsum('ibh,hnd->ibnd', h, self.q) + q_head_h = tf.einsum('ibh,hnd->ibnd', h, self.q) # core attention ops attn_vec_h = self.rel_attn_core( @@ -286,15 +288,15 @@ class TFXLNetRelativeAttention(tf.keras.layers.Layer): attn_vec_h, attn_prob_h = attn_vec_h # post processing - output_h = self.post_attention([h, attn_vec_h, None], training=training) + output_h = self.post_attention([h, attn_vec_h], training=training) ##### g-stream # query-stream query head - q_head_g = torch.einsum('ibh,hnd->ibnd', g, self.q) + q_head_g = tf.einsum('ibh,hnd->ibnd', g, self.q) # core attention ops if target_mapping is not None: - q_head_g = torch.einsum('mbnd,mlb->lbnd', q_head_g, target_mapping) + q_head_g = tf.einsum('mbnd,mlb->lbnd', q_head_g, target_mapping) attn_vec_g = self.rel_attn_core( [q_head_g, k_head_h, v_head_h, k_head_r, seg_mat, attn_mask_g, head_mask], training=training) @@ -302,7 +304,7 @@ class TFXLNetRelativeAttention(tf.keras.layers.Layer): if self.output_attentions: attn_vec_g, attn_prob_g = attn_vec_g - attn_vec_g = torch.einsum('lbnd,mlb->mbnd', attn_vec_g, target_mapping) + attn_vec_g = tf.einsum('lbnd,mlb->mbnd', attn_vec_g, target_mapping) else: attn_vec_g = self.rel_attn_core( [q_head_g, k_head_h, v_head_h, k_head_r, seg_mat, attn_mask_g, head_mask], @@ -312,15 +314,15 @@ class TFXLNetRelativeAttention(tf.keras.layers.Layer): attn_vec_g, attn_prob_g = attn_vec_g # post processing - output_g = self.post_attention([g, attn_vec_g, None], training=training) + output_g = self.post_attention([g, attn_vec_g], training=training) if self.output_attentions: attn_prob = attn_prob_h, attn_prob_g else: ###### Multi-head attention with relative positional encoding - if mems is not None and mems.dim() > 1: - cat = tf.concat([mems, h], dim=0) + if mems is not None and mems.shape.ndims > 1: + cat = tf.concat([mems, h], axis=0) else: cat = h @@ -341,7 +343,7 @@ class TFXLNetRelativeAttention(tf.keras.layers.Layer): attn_vec, attn_prob = attn_vec # post processing - output_h = self.post_attention([h, attn_vec, None], training=training) + output_h = self.post_attention([h, attn_vec], training=training) output_g = None outputs = (output_h, output_g) @@ -391,6 +393,27 @@ class TFXLNetLayer(tf.keras.layers.Layer): return outputs +class TFXLNetLMHead(tf.keras.layers.Layer): + def __init__(self, config, input_embeddings, **kwargs): + super(TFXLNetLMHead, self).__init__(**kwargs) + self.vocab_size = config.vocab_size + # The output weights are the same as the input embeddings, but there is + # an output-only bias for each token. + self.input_embeddings = input_embeddings + + def build(self, input_shape): + self.bias = self.add_weight(shape=(self.vocab_size,), + initializer='zeros', + trainable=True, + name='bias') + super(TFXLNetLMHead, self).build(input_shape) + + def call(self, hidden_states): + hidden_states = self.input_embeddings(hidden_states, mode="linear") + hidden_states = hidden_states + self.bias + return hidden_states + + class TFXLNetMainLayer(tf.keras.layers.Layer): def __init__(self, config, **kwargs): super(TFXLNetMainLayer, self).__init__(**kwargs) @@ -409,7 +432,7 @@ class TFXLNetMainLayer(tf.keras.layers.Layer): self.initializer_range = config.initializer_range self.word_embedding = TFSharedEmbeddings(config.n_token, config.d_model, initializer_range=config.initializer_range, name='word_embedding') - self.layer = [TFXLNetLayer(config, name='layer_{}'.format(i)) for i in range(config.n_layer)] + self.layer = [TFXLNetLayer(config, name='layer__{}'.format(i)) for i in range(config.n_layer)] self.dropout = tf.keras.layers.Dropout(config.dropout) def build(self, input_shape): @@ -464,7 +487,7 @@ class TFXLNetMainLayer(tf.keras.layers.Layer): if prev_mem is None: new_mem = curr_out[-self.mem_len:] else: - new_mem = tf.concat([prev_mem, curr_out], 0)[-mem_len:] + new_mem = tf.concat([prev_mem, curr_out], 0)[-self.mem_len:] return tf.stop_gradient(new_mem) @@ -618,7 +641,7 @@ class TFXLNetMainLayer(tf.keras.layers.Layer): word_emb_k = self.word_embedding(input_ids) output_h = self.dropout(word_emb_k, training=training) if target_mapping is not None: - word_emb_q = tf.tile(mask_emb, [tf.shape(target_mapping)[0], bsz, 1]) + word_emb_q = tf.tile(self.mask_emb, [tf.shape(target_mapping)[0], bsz, 1]) # else: # We removed the inp_q input which was same as target mapping # inp_q_ext = inp_q[:, :, None] # word_emb_q = inp_q_ext * self.mask_emb + (1 - inp_q_ext) * word_emb_k @@ -827,187 +850,173 @@ class TFXLNetModel(TFXLNetPreTrainedModel): return outputs -# @add_start_docstrings("""XLNet Model with a language modeling head on top -# (linear layer with weights tied to the input embeddings). """, -# XLNET_START_DOCSTRING, XLNET_INPUTS_DOCSTRING) -# class XLNetLMHeadModel(XLNetPreTrainedModel): -# r""" -# **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: -# Labels for language modeling. -# Note that the labels **are shifted** inside the model, i.e. you can set ``lm_labels = input_ids`` -# Indices are selected in ``[-1, 0, ..., config.vocab_size]`` -# All labels set to ``-1`` are ignored (masked), the loss is only -# computed for labels in ``[0, ..., config.vocab_size]`` +@add_start_docstrings("""XLNet Model with a language modeling head on top + (linear layer with weights tied to the input embeddings). """, + XLNET_START_DOCSTRING, XLNET_INPUTS_DOCSTRING) +class TFXLNetLMHeadModel(TFXLNetPreTrainedModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + **mems**: + list of ``torch.FloatTensor`` (one for each layer): + that contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model + if config.mem_len > 0 else tuple of None. Can be used to speed up sequential decoding and attend to longer context. + See details in the docstring of the `mems` input above. + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. -# Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: -# **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: -# Language modeling loss. -# **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` -# Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). -# **mems**: -# list of ``torch.FloatTensor`` (one for each layer): -# that contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model -# if config.mem_len > 0 else tuple of None. Can be used to speed up sequential decoding and attend to longer context. -# See details in the docstring of the `mems` input above. -# **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) -# list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) -# of shape ``(batch_size, sequence_length, hidden_size)``: -# Hidden-states of the model at the output of each layer plus the initial embedding outputs. -# **attentions**: (`optional`, returned when ``config.output_attentions=True``) -# list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: -# Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + Examples:: -# Examples:: + tokenizer = XLNetTokenizer.from_pretrained('xlnet-large-cased') + model = TFXLNetLMHeadModel.from_pretrained('xlnet-large-cased') + # We show how to setup inputs to predict a next token using a bi-directional context. + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is very ")).unsqueeze(0) # We will predict the masked token + perm_mask = torch.zeros((1, input_ids.shape[1], input_ids.shape[1]), dtype=torch.float) + perm_mask[:, :, -1] = 1.0 # Previous tokens don't see last token + target_mapping = torch.zeros((1, 1, input_ids.shape[1]), dtype=torch.float) # Shape [1, 1, seq_length] => let's predict one token + target_mapping[0, 0, -1] = 1.0 # Our first (and only) prediction will be the last token of the sequence (the masked token) + outputs = model(input_ids, perm_mask=perm_mask, target_mapping=target_mapping) + next_token_logits = outputs[0] # Output has shape [target_mapping.size(0), target_mapping.size(1), config.vocab_size] -# tokenizer = XLNetTokenizer.from_pretrained('xlnet-large-cased') -# model = XLNetLMHeadModel.from_pretrained('xlnet-large-cased') -# # We show how to setup inputs to predict a next token using a bi-directional context. -# input_ids = torch.tensor(tokenizer.encode("Hello, my dog is very ")).unsqueeze(0) # We will predict the masked token -# perm_mask = torch.zeros((1, input_ids.shape[1], input_ids.shape[1]), dtype=torch.float) -# perm_mask[:, :, -1] = 1.0 # Previous tokens don't see last token -# target_mapping = torch.zeros((1, 1, input_ids.shape[1]), dtype=torch.float) # Shape [1, 1, seq_length] => let's predict one token -# target_mapping[0, 0, -1] = 1.0 # Our first (and only) prediction will be the last token of the sequence (the masked token) -# outputs = model(input_ids, perm_mask=perm_mask, target_mapping=target_mapping) -# next_token_logits = outputs[0] # Output has shape [target_mapping.size(0), target_mapping.size(1), config.vocab_size] + """ + def __init__(self, config, *inputs, **kwargs): + super(TFXLNetLMHeadModel, self).__init__(config, *inputs, **kwargs) + self.n_token = config.n_token -# """ -# def __init__(self, config, **kwargs): -# super(XLNetLMHeadModel, self).__init__(config) -# self.attn_type = config.attn_type -# self.same_length = config.same_length + self.transformer = TFXLNetMainLayer(config, name='transformer') + self.lm_loss = TFXLNetLMHead(config, self.transformer.word_embedding, name='lm_loss') -# self.transformer = XLNetModel(config) -# self.lm_loss = nn.Linear(config.d_model, config.n_token, bias=True) + def call(self, inputs, training=False): + transformer_outputs = self.transformer(inputs, training=training) + hidden_state = transformer_outputs[0] + logits = self.lm_loss(hidden_state) -# self.init_weights() -# self.tie_weights() + outputs = (logits,) + transformer_outputs[1:] # Keep mems, hidden states, attentions if there are in it -# def tie_weights(self): -# """ Make sure we are sharing the embeddings -# """ -# self._tie_or_clone_weights(self.lm_loss, self.transformer.word_embedding) - -# def forward(self, input_ids, attention_mask=None, mems=None, perm_mask=None, target_mapping=None, -# token_type_ids=None, input_mask=None, head_mask=None, labels=None): -# transformer_outputs = self.transformer(input_ids, -# attention_mask=attention_mask, -# mems=mems, -# perm_mask=perm_mask, -# target_mapping=target_mapping, -# token_type_ids=token_type_ids, -# input_mask=input_mask, -# head_mask=head_mask) - -# logits = self.lm_loss(transformer_outputs[0]) - -# outputs = (logits,) + transformer_outputs[1:] # Keep mems, hidden states, attentions if there are in it - -# if labels is not None: -# # Flatten the tokens -# loss_fct = CrossEntropyLoss(ignore_index=-1) -# loss = loss_fct(logits.view(-1, logits.size(-1)), -# labels.view(-1)) -# outputs = (loss,) + outputs - -# return outputs # return (loss), logits, mems, (hidden states), (attentions) + return outputs # return logits, mems, (hidden states), (attentions) -# @add_start_docstrings("""XLNet Model with a sequence classification/regression head on top (a linear layer on top of -# the pooled output) e.g. for GLUE tasks. """, -# XLNET_START_DOCSTRING, XLNET_INPUTS_DOCSTRING) -# class XLNetForSequenceClassification(XLNetPreTrainedModel): -# r""" -# **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: -# Labels for computing the sequence classification/regression loss. -# Indices should be in ``[0, ..., config.num_labels - 1]``. -# If ``config.num_labels == 1`` a regression loss is computed (Mean-Square loss), -# If ``config.num_labels > 1`` a classification loss is computed (Cross-Entropy). +@add_start_docstrings("""XLNet Model with a sequence classification/regression head on top (a linear layer on top of + the pooled output) e.g. for GLUE tasks. """, + XLNET_START_DOCSTRING, XLNET_INPUTS_DOCSTRING) +class TFXLNetForSequenceClassification(TFXLNetPreTrainedModel): + r""" + **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for computing the sequence classification/regression loss. + Indices should be in ``[0, ..., config.num_labels - 1]``. + If ``config.num_labels == 1`` a regression loss is computed (Mean-Square loss), + If ``config.num_labels > 1`` a classification loss is computed (Cross-Entropy). -# Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: -# **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: -# Classification (or regression if config.num_labels==1) loss. -# **logits**: ``torch.FloatTensor`` of shape ``(batch_size, config.num_labels)`` -# Classification (or regression if config.num_labels==1) scores (before SoftMax). -# **mems**: -# list of ``torch.FloatTensor`` (one for each layer): -# that contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model -# if config.mem_len > 0 else tuple of None. Can be used to speed up sequential decoding and attend to longer context. -# See details in the docstring of the `mems` input above. -# **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) -# list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) -# of shape ``(batch_size, sequence_length, hidden_size)``: -# Hidden-states of the model at the output of each layer plus the initial embedding outputs. -# **attentions**: (`optional`, returned when ``config.output_attentions=True``) -# list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: -# Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Classification (or regression if config.num_labels==1) loss. + **logits**: ``torch.FloatTensor`` of shape ``(batch_size, config.num_labels)`` + Classification (or regression if config.num_labels==1) scores (before SoftMax). + **mems**: + list of ``torch.FloatTensor`` (one for each layer): + that contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model + if config.mem_len > 0 else tuple of None. Can be used to speed up sequential decoding and attend to longer context. + See details in the docstring of the `mems` input above. + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. -# Examples:: + Examples:: -# tokenizer = XLNetTokenizer.from_pretrained('xlnet-large-cased') -# model = XLNetForSequenceClassification.from_pretrained('xlnet-large-cased') -# input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 -# labels = torch.tensor([1]).unsqueeze(0) # Batch size 1 -# outputs = model(input_ids, labels=labels) -# loss, logits = outputs[:2] + tokenizer = XLNetTokenizer.from_pretrained('xlnet-large-cased') + model = XLNetForSequenceClassification.from_pretrained('xlnet-large-cased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + labels = torch.tensor([1]).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, labels=labels) + loss, logits = outputs[:2] -# """ -# def __init__(self, config, **kwargs): -# super(XLNetForSequenceClassification, self).__init__(config) -# self.num_labels = config.num_labels + """ + def __init__(self, config, *inputs, **kwargs): + super(TFXLNetForSequenceClassification, self).__init__(config, *inputs, **kwargs) + self.num_labels = config.num_labels -# self.transformer = XLNetModel(config) -# self.sequence_summary = SequenceSummary(config) -# self.logits_proj = nn.Linear(config.d_model, config.num_labels) + self.transformer = TFXLNetMainLayer(config, name='transformer') + self.sequence_summary = TFSequenceSummary(config, name='sequence_summary') + self.logits_proj = tf.keras.layers.Dense(config.num_labels, name='logits_proj') -# self.init_weights() + def call(self, inputs, training=False): + transformer_outputs = self.transformer(inputs, training=training) + output = transformer_outputs[0] -# def forward(self, input_ids, attention_mask=None, mems=None, perm_mask=None, target_mapping=None, -# token_type_ids=None, input_mask=None, head_mask=None, labels=None): -# transformer_outputs = self.transformer(input_ids, -# attention_mask=attention_mask, -# mems=mems, -# perm_mask=perm_mask, -# target_mapping=target_mapping, -# token_type_ids=token_type_ids, -# input_mask=input_mask, -# head_mask=head_mask) -# output = transformer_outputs[0] + output = self.sequence_summary(output) + logits = self.logits_proj(output) -# output = self.sequence_summary(output) -# logits = self.logits_proj(output) + outputs = (logits,) + transformer_outputs[1:] # Keep mems, hidden states, attentions if there are in it -# outputs = (logits,) + transformer_outputs[1:] # Keep mems, hidden states, attentions if there are in it - -# if labels is not None: -# if self.num_labels == 1: -# # We are doing regression -# loss_fct = MSELoss() -# loss = loss_fct(logits.view(-1), labels.view(-1)) -# else: -# loss_fct = CrossEntropyLoss() -# loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1)) -# outputs = (loss,) + outputs - -# return outputs # return (loss), logits, mems, (hidden states), (attentions) + return outputs # return logits, mems, (hidden states), (attentions) # @add_start_docstrings("""XLNet Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layers on top of # the hidden-states output to compute `span start logits` and `span end logits`). """, # XLNET_START_DOCSTRING, XLNET_INPUTS_DOCSTRING) -# class XLNetForQuestionAnswering(XLNetPreTrainedModel): +# class TFXLNetForQuestionAnswering(TFXLNetPreTrainedModel): +class TFXLNetForQuestionAnsweringSimple(TFXLNetPreTrainedModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **start_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length,)`` + Span-start scores (before SoftMax). + **end_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length,)`` + Span-end scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = XLNetTokenizer.from_pretrained('xlnet-base-cased') + model = TFXLNetForQuestionAnsweringSimple.from_pretrained('xlnet-base-cased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + start_positions = torch.tensor([1]) + end_positions = torch.tensor([3]) + outputs = model(input_ids, start_positions=start_positions, end_positions=end_positions) + loss, start_scores, end_scores = outputs[:2] + + """ + def __init__(self, config, *inputs, **kwargs): + super(TFXLNetForQuestionAnsweringSimple, self).__init__(config, *inputs, **kwargs) + self.num_labels = config.num_labels + + self.transformer = TFXLNetMainLayer(config, name='transformer') + self.qa_outputs = tf.keras.layers.Dense(config.num_labels, name='qa_outputs') + + def call(self, inputs, training=False): + transformer_outputs = self.transformer(inputs, training=training) + + sequence_output = transformer_outputs[0] + + logits = self.qa_outputs(sequence_output) + start_logits, end_logits = tf.split(logits, 2, axis=-1) + start_logits = tf.squeeze(start_logits, axis=-1) + end_logits = tf.squeeze(end_logits, axis=-1) + + outputs = (start_logits, end_logits,) + transformer_outputs[1:] # Keep mems, hidden states, attentions if there are in it + + return outputs # start_logits, end_logits, (hidden_states), (attentions) + +# @add_start_docstrings("""XLNet Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layers on top of +# the hidden-states output to compute `span start logits` and `span end logits`). """, +# XLNET_START_DOCSTRING, XLNET_INPUTS_DOCSTRING) +# class TFXLNetForQuestionAnswering(TFXLNetPreTrainedModel): # r""" -# **start_positions**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: -# Labels for position (index) of the start of the labelled span for computing the token classification loss. -# Positions are clamped to the length of the sequence (`sequence_length`). -# Position outside of the sequence are not taken into account for computing the loss. -# **end_positions**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: -# Labels for position (index) of the end of the labelled span for computing the token classification loss. -# Positions are clamped to the length of the sequence (`sequence_length`). -# Position outside of the sequence are not taken into account for computing the loss. -# **is_impossible**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: -# Labels whether a question has an answer or no answer (SQuAD 2.0) -# **cls_index**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: -# Labels for position (index) of the classification token to use as input for computing plausibility of the answer. # **p_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``: # Optional mask of tokens which can't be in answers (e.g. [CLS], [PAD], ...). # 1.0 means token should be masked. 0.0 mean token is not masked. @@ -1054,29 +1063,18 @@ class TFXLNetModel(TFXLNetPreTrainedModel): # loss, start_scores, end_scores = outputs[:2] # """ -# def __init__(self, config, **kwargs): -# super(XLNetForQuestionAnswering, self).__init__(config) +# def __init__(self, config, *inputs, **kwargs): +# super(TFXLNetForQuestionAnswering, self).__init__(config, *inputs, **kwargs) # self.start_n_top = config.start_n_top # self.end_n_top = config.end_n_top -# self.transformer = XLNetModel(config) -# self.start_logits = PoolerStartLogits(config) -# self.end_logits = PoolerEndLogits(config) -# self.answer_class = PoolerAnswerClass(config) +# self.transformer = TFXLNetMainLayer(config, name='transformer') +# self.start_logits = TFPoolerStartLogits(config, name='start_logits') +# self.end_logits = TFPoolerEndLogits(config, name='end_logits') +# self.answer_class = TFPoolerAnswerClass(config, name='answer_class') -# self.init_weights() - -# def forward(self, input_ids, attention_mask=None, mems=None, perm_mask=None, target_mapping=None, -# token_type_ids=None, input_mask=None, head_mask=None, -# start_positions=None, end_positions=None, is_impossible=None, cls_index=None, p_mask=None,): -# transformer_outputs = self.transformer(input_ids, -# attention_mask=attention_mask, -# mems=mems, -# perm_mask=perm_mask, -# target_mapping=target_mapping, -# token_type_ids=token_type_ids, -# input_mask=input_mask, -# head_mask=head_mask) +# def call(self, inputs, training=False): +# transformer_outputs = self.transformer(inputs, training=training) # hidden_states = transformer_outputs[0] # start_logits = self.start_logits(hidden_states, p_mask=p_mask) diff --git a/pytorch_transformers/modeling_xlnet.py b/pytorch_transformers/modeling_xlnet.py index 97feaad371..c8e55d2107 100644 --- a/pytorch_transformers/modeling_xlnet.py +++ b/pytorch_transformers/modeling_xlnet.py @@ -1003,6 +1003,101 @@ class XLNetForSequenceClassification(XLNetPreTrainedModel): return outputs # return (loss), logits, mems, (hidden states), (attentions) +@add_start_docstrings("""XLNet Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layers on top of + the hidden-states output to compute `span start logits` and `span end logits`). """, + XLNET_START_DOCSTRING, XLNET_INPUTS_DOCSTRING) +class XLNetForQuestionAnsweringSimple(XLNetPreTrainedModel): + r""" + **start_positions**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for position (index) of the start of the labelled span for computing the token classification loss. + Positions are clamped to the length of the sequence (`sequence_length`). + Position outside of the sequence are not taken into account for computing the loss. + **end_positions**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for position (index) of the end of the labelled span for computing the token classification loss. + Positions are clamped to the length of the sequence (`sequence_length`). + Position outside of the sequence are not taken into account for computing the loss. + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned if both ``start_positions`` and ``end_positions`` are provided) ``torch.FloatTensor`` of shape ``(1,)``: + Classification loss as the sum of start token, end token (and is_impossible if provided) classification losses. + **start_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length,)`` + Span-start scores (before SoftMax). + **end_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length,)`` + Span-end scores (before SoftMax). + **mems**: + list of ``torch.FloatTensor`` (one for each layer): + that contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model + if config.mem_len > 0 else tuple of None. Can be used to speed up sequential decoding and attend to longer context. + See details in the docstring of the `mems` input above. + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = XLMTokenizer.from_pretrained('xlm-mlm-en-2048') + model = XLMForQuestionAnswering.from_pretrained('xlnet-large-cased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + start_positions = torch.tensor([1]) + end_positions = torch.tensor([3]) + outputs = model(input_ids, start_positions=start_positions, end_positions=end_positions) + loss, start_scores, end_scores = outputs[:2] + + """ + def __init__(self, config): + super(XLNetForQuestionAnsweringSimple, self).__init__(config) + self.num_labels = config.num_labels + + self.transformer = XLNetModel(config) + self.qa_outputs = nn.Linear(config.hidden_size, config.num_labels) + + self.init_weights() + + def forward(self, input_ids, attention_mask=None, mems=None, perm_mask=None, target_mapping=None, + token_type_ids=None, input_mask=None, head_mask=None, + start_positions=None, end_positions=None): + + outputs = self.transformer(input_ids, + attention_mask=attention_mask, + mems=mems, + perm_mask=perm_mask, + target_mapping=target_mapping, + token_type_ids=token_type_ids, + input_mask=input_mask, + head_mask=head_mask) + + sequence_output = outputs[0] + + logits = self.qa_outputs(sequence_output) + start_logits, end_logits = logits.split(1, dim=-1) + start_logits = start_logits.squeeze(-1) + end_logits = end_logits.squeeze(-1) + + outputs = (start_logits, end_logits,) + outputs[2:] + if start_positions is not None and end_positions is not None: + # If we are on multi-GPU, split add a dimension + if len(start_positions.size()) > 1: + start_positions = start_positions.squeeze(-1) + if len(end_positions.size()) > 1: + end_positions = end_positions.squeeze(-1) + # sometimes the start/end positions are outside our model inputs, we ignore these terms + ignored_index = start_logits.size(1) + start_positions.clamp_(0, ignored_index) + end_positions.clamp_(0, ignored_index) + + loss_fct = CrossEntropyLoss(ignore_index=ignored_index) + start_loss = loss_fct(start_logits, start_positions) + end_loss = loss_fct(end_logits, end_positions) + total_loss = (start_loss + end_loss) / 2 + outputs = (total_loss,) + outputs + + return outputs # (loss), start_logits, end_logits, (hidden_states), (attentions) + + @add_start_docstrings("""XLNet Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`). """, XLNET_START_DOCSTRING, XLNET_INPUTS_DOCSTRING) diff --git a/pytorch_transformers/tests/modeling_tf_xlnet_test.py b/pytorch_transformers/tests/modeling_tf_xlnet_test.py index 8d61d6b5dc..01c4494664 100644 --- a/pytorch_transformers/tests/modeling_tf_xlnet_test.py +++ b/pytorch_transformers/tests/modeling_tf_xlnet_test.py @@ -28,9 +28,10 @@ from pytorch_transformers import XLNetConfig, is_tf_available if is_tf_available(): import tensorflow as tf - from pytorch_transformers.modeling_tf_xlnet import (TFXLNetModel, TF_XLNET_PRETRAINED_MODEL_ARCHIVE_MAP) - # XLNetLMHeadModel, - # XLNetForSequenceClassification, XLNetForQuestionAnswering) + from pytorch_transformers.modeling_tf_xlnet import (TFXLNetModel, TFXLNetLMHeadModel, + TFXLNetForSequenceClassification, + TFXLNetForQuestionAnsweringSimple, + TF_XLNET_PRETRAINED_MODEL_ARCHIVE_MAP) else: pytestmark = pytest.mark.skip("Require TensorFlow") @@ -39,9 +40,9 @@ from .configuration_common_test import ConfigTester class TFXLNetModelTest(TFCommonTestCases.TFCommonModelTester): - all_model_classes=(TFXLNetModel, ) if is_tf_available() else () - # all_model_classes=(TFXLNetModel, TFXLNetLMHeadModel, - # TFXLNetForSequenceClassification, TFXLNetForQuestionAnswering) if is_tf_available() else () + all_model_classes=(TFXLNetModel, TFXLNetLMHeadModel, + TFXLNetForSequenceClassification, + TFXLNetForQuestionAnsweringSimple) if is_tf_available() else () test_pruning = False class TFXLNetModelTester(object): @@ -169,128 +170,88 @@ class TFXLNetModelTest(TFCommonTestCases.TFCommonModelTester): def create_and_check_xlnet_lm_head(self, config, input_ids_1, input_ids_2, input_ids_q, perm_mask, input_mask, target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels): - pass - # model = XLNetLMHeadModel(config) - # model.eval() + model = TFXLNetLMHeadModel(config) - # loss_1, all_logits_1, mems_1 = model(input_ids_1, token_type_ids=segment_ids, labels=lm_labels) + inputs_1 = {'input_ids': input_ids_1, + 'token_type_ids': segment_ids} - # loss_2, all_logits_2, mems_2 = model(input_ids_2, token_type_ids=segment_ids, labels=lm_labels, mems=mems_1) + all_logits_1, mems_1 = model(inputs_1) - # logits, _ = model(input_ids_q, perm_mask=perm_mask, target_mapping=target_mapping) + inputs_2 = {'input_ids': input_ids_2, + 'mems': mems_1, + 'token_type_ids': segment_ids} - # result = { - # "loss_1": loss_1, - # "mems_1": mems_1, - # "all_logits_1": all_logits_1, - # "loss_2": loss_2, - # "mems_2": mems_2, - # "all_logits_2": all_logits_2, - # } + all_logits_2, mems_2 = model(inputs_2) - # self.parent.assertListEqual( - # list(result["loss_1"].size()), - # []) - # self.parent.assertListEqual( - # list(result["all_logits_1"].size()), - # [self.batch_size, self.seq_length, self.vocab_size]) - # self.parent.assertListEqual( - # list(list(mem.size()) for mem in result["mems_1"]), - # [[self.seq_length, self.batch_size, self.hidden_size]] * self.num_hidden_layers) + inputs_3 = {'input_ids': input_ids_q, + 'perm_mask': perm_mask, + 'target_mapping': target_mapping} - # self.parent.assertListEqual( - # list(result["loss_2"].size()), - # []) - # self.parent.assertListEqual( - # list(result["all_logits_2"].size()), - # [self.batch_size, self.seq_length, self.vocab_size]) - # self.parent.assertListEqual( - # list(list(mem.size()) for mem in result["mems_2"]), - # [[self.mem_len, self.batch_size, self.hidden_size]] * self.num_hidden_layers) + logits, _ = model(inputs_3) + + result = { + "mems_1": [mem.numpy() for mem in mems_1], + "all_logits_1": all_logits_1.numpy(), + "mems_2": [mem.numpy() for mem in mems_2], + "all_logits_2": all_logits_2.numpy(), + } + + self.parent.assertListEqual( + list(result["all_logits_1"].shape), + [self.batch_size, self.seq_length, self.vocab_size]) + self.parent.assertListEqual( + list(list(mem.shape) for mem in result["mems_1"]), + [[self.seq_length, self.batch_size, self.hidden_size]] * self.num_hidden_layers) + + self.parent.assertListEqual( + list(result["all_logits_2"].shape), + [self.batch_size, self.seq_length, self.vocab_size]) + self.parent.assertListEqual( + list(list(mem.shape) for mem in result["mems_2"]), + [[self.mem_len, self.batch_size, self.hidden_size]] * self.num_hidden_layers) def create_and_check_xlnet_qa(self, config, input_ids_1, input_ids_2, input_ids_q, perm_mask, input_mask, target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels): - pass - # model = XLNetForQuestionAnswering(config) - # model.eval() + model = TFXLNetForQuestionAnsweringSimple(config) - # outputs = model(input_ids_1) - # start_top_log_probs, start_top_index, end_top_log_probs, end_top_index, cls_logits, mems = outputs + inputs = {'input_ids': input_ids_1, + 'attention_mask': input_mask, + 'token_type_ids': segment_ids} + start_logits, end_logits, mems = model(inputs) - # outputs = model(input_ids_1, start_positions=sequence_labels, - # end_positions=sequence_labels, - # cls_index=sequence_labels, - # is_impossible=is_impossible_labels, - # p_mask=input_mask) + result = { + "start_logits": start_logits.numpy(), + "end_logits": end_logits.numpy(), + "mems": [m.numpy() for m in mems], + } - # outputs = model(input_ids_1, start_positions=sequence_labels, - # end_positions=sequence_labels, - # cls_index=sequence_labels, - # is_impossible=is_impossible_labels) - - # total_loss, mems = outputs - - # outputs = model(input_ids_1, start_positions=sequence_labels, - # end_positions=sequence_labels) - - # total_loss, mems = outputs - - # result = { - # "loss": total_loss, - # "start_top_log_probs": start_top_log_probs, - # "start_top_index": start_top_index, - # "end_top_log_probs": end_top_log_probs, - # "end_top_index": end_top_index, - # "cls_logits": cls_logits, - # "mems": mems, - # } - - # self.parent.assertListEqual( - # list(result["loss"].size()), - # []) - # self.parent.assertListEqual( - # list(result["start_top_log_probs"].size()), - # [self.batch_size, model.config.start_n_top]) - # self.parent.assertListEqual( - # list(result["start_top_index"].size()), - # [self.batch_size, model.config.start_n_top]) - # self.parent.assertListEqual( - # list(result["end_top_log_probs"].size()), - # [self.batch_size, model.config.start_n_top * model.config.end_n_top]) - # self.parent.assertListEqual( - # list(result["end_top_index"].size()), - # [self.batch_size, model.config.start_n_top * model.config.end_n_top]) - # self.parent.assertListEqual( - # list(result["cls_logits"].size()), - # [self.batch_size]) - # self.parent.assertListEqual( - # list(list(mem.size()) for mem in result["mems"]), - # [[self.seq_length, self.batch_size, self.hidden_size]] * self.num_hidden_layers) + self.parent.assertListEqual( + list(result["start_logits"].shape), + [self.batch_size, self.seq_length]) + self.parent.assertListEqual( + list(result["end_logits"].shape), + [self.batch_size, self.seq_length]) + self.parent.assertListEqual( + list(list(mem.shape) for mem in result["mems"]), + [[self.seq_length, self.batch_size, self.hidden_size]] * self.num_hidden_layers) def create_and_check_xlnet_sequence_classif(self, config, input_ids_1, input_ids_2, input_ids_q, perm_mask, input_mask, target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels): - pass - # model = XLNetForSequenceClassification(config) - # model.eval() + model = TFXLNetForSequenceClassification(config) - # logits, mems_1 = model(input_ids_1) - # loss, logits, mems_1 = model(input_ids_1, labels=sequence_labels) + logits, mems_1 = model(input_ids_1) - # result = { - # "loss": loss, - # "mems_1": mems_1, - # "logits": logits, - # } + result = { + "mems_1": [mem.numpy() for mem in mems_1], + "logits": logits.numpy(), + } - # self.parent.assertListEqual( - # list(result["loss"].size()), - # []) - # self.parent.assertListEqual( - # list(result["logits"].size()), - # [self.batch_size, self.type_sequence_label_size]) - # self.parent.assertListEqual( - # list(list(mem.size()) for mem in result["mems_1"]), - # [[self.seq_length, self.batch_size, self.hidden_size]] * self.num_hidden_layers) + self.parent.assertListEqual( + list(result["logits"].shape), + [self.batch_size, self.type_sequence_label_size]) + self.parent.assertListEqual( + list(list(mem.shape) for mem in result["mems_1"]), + [[self.seq_length, self.batch_size, self.hidden_size]] * self.num_hidden_layers) def prepare_config_and_inputs_for_common(self): config_and_inputs = self.prepare_config_and_inputs() From 4356f791a2ba1bb1283ea073438b67e9b37d1c76 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Wed, 11 Sep 2019 11:49:54 +0200 Subject: [PATCH 043/219] XLM passing tests --- pytorch_transformers/__init__.py | 7 + pytorch_transformers/modeling_tf_xlm.py | 798 ++++++++++++++++++ pytorch_transformers/modeling_tf_xlnet.py | 8 +- pytorch_transformers/modeling_xlm.py | 5 - .../tests/modeling_tf_xlm_test.py | 264 ++++++ .../tests/modeling_xlm_test.py | 21 +- 6 files changed, 1079 insertions(+), 24 deletions(-) create mode 100644 pytorch_transformers/modeling_tf_xlm.py create mode 100644 pytorch_transformers/tests/modeling_tf_xlm_test.py diff --git a/pytorch_transformers/__init__.py b/pytorch_transformers/__init__.py index 0dc86563aa..2f7bdb3de4 100644 --- a/pytorch_transformers/__init__.py +++ b/pytorch_transformers/__init__.py @@ -120,6 +120,13 @@ if _tf_available: load_xlnet_pt_weights_in_tf2, TF_XLNET_PRETRAINED_MODEL_ARCHIVE_MAP) + from .modeling_tf_xlm import (TFXLMPreTrainedModel, TFXLMMainLayer, + TFXLMModel, TFXLMWithLMHeadModel, + TFXLMForSequenceClassification, + TFXLMForQuestionAnsweringSimple, + load_xlm_pt_weights_in_tf2, + TF_XLM_PRETRAINED_MODEL_ARCHIVE_MAP) + # Files and general utilities from .file_utils import (PYTORCH_TRANSFORMERS_CACHE, PYTORCH_PRETRAINED_BERT_CACHE, cached_path, add_start_docstrings, add_end_docstrings, diff --git a/pytorch_transformers/modeling_tf_xlm.py b/pytorch_transformers/modeling_tf_xlm.py new file mode 100644 index 0000000000..d0db638013 --- /dev/null +++ b/pytorch_transformers/modeling_tf_xlm.py @@ -0,0 +1,798 @@ +# coding=utf-8 +# Copyright 2019-present, Facebook, Inc and the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" TF 2.0 XLM model. +""" +from __future__ import absolute_import, division, print_function, unicode_literals + +import logging +import math + +import itertools +import numpy as np +import tensorflow as tf + +from .configuration_xlm import XLMConfig +from .modeling_tf_utils import TFPreTrainedModel, TFSharedEmbeddings, TFSequenceSummary, shape_list +from .file_utils import add_start_docstrings + +logger = logging.getLogger(__name__) + +TF_XLM_PRETRAINED_MODEL_ARCHIVE_MAP = { + 'xlm-mlm-en-2048': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-mlm-en-2048-tf_model.h5", + 'xlm-mlm-ende-1024': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-mlm-ende-1024-tf_model.h5", + 'xlm-mlm-enfr-1024': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-mlm-enfr-1024-tf_model.h5", + 'xlm-mlm-enro-1024': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-mlm-enro-1024-tf_model.h5", + 'xlm-mlm-tlm-xnli15-1024': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-mlm-tlm-xnli15-1024-tf_model.h5", + 'xlm-mlm-xnli15-1024': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-mlm-xnli15-1024-tf_model.h5", + 'xlm-clm-enfr-1024': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-clm-enfr-1024-tf_model.h5", + 'xlm-clm-ende-1024': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-clm-ende-1024-tf_model.h5", + 'xlm-mlm-17-1280': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-mlm-17-1280-tf_model.h5", + 'xlm-mlm-100-1280': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-mlm-100-1280-tf_model.h5", +} + + +def load_xlm_pt_weights_in_tf2(tf_model, config, pytorch_checkpoint_path): + """ Load pytorch checkpoints in a TF 2.0 model and save it using HDF5 format + We use HDF5 to easily do transfer learning + (see https://github.com/tensorflow/tensorflow/blob/ee16fcac960ae660e0e4496658a366e2f745e1f0/tensorflow/python/keras/engine/network.py#L1352-L1357). + """ + try: + import re + import torch + import numpy + from tensorflow.python.keras import backend as K + except ImportError: + logger.error("Loading a PyTorch model in TensorFlow, requires PyTorch to be installed. Please see " + "https://pytorch.org/ for installation instructions.") + raise + + pt_path = os.path.abspath(pytorch_checkpoint_path) + logger.info("Loading PyTorch weights from {}".format(pt_path)) + # Load pytorch model + state_dict = torch.load(pt_path, map_location='cpu') + + inputs_list = [[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]] + tf_inputs = tf.constant(inputs_list) + tfo = tf_model(tf_inputs, training=False) # build the network + + symbolic_weights = tf_model.trainable_weights + tf_model.non_trainable_weights + weight_value_tuples = [] + all_pytorch_weights = set(list(state_dict.keys())) + for symbolic_weight in symbolic_weights: + name = symbolic_weight.name + name = name.replace(':0', '') + name = name.replace('__', '/') + name = name.split('/') + name = name[1:] + + transpose = bool(name[-1] == 'kernel') + if name[-1] == 'kernel' or name[-1] == 'embeddings' or name[-1] == 'gamma': + name[-1] = 'weight' + if name[-1] == 'beta': + name[-1] = 'bias' + + name = '.'.join(name) + assert name in state_dict, "{} not found in PyTorch model".format(name) + array = state_dict[name].numpy() + + if transpose: + array = numpy.transpose(array) + + try: + assert list(symbolic_weight.shape) == list(array.shape) + except AssertionError as e: + e.args += (symbolic_weight.shape, array.shape) + raise e + + logger.info("Initialize TF weight {}".format(symbolic_weight.name)) + + weight_value_tuples.append((symbolic_weight, array)) + all_pytorch_weights.discard(name) + + K.batch_set_value(weight_value_tuples) + + tfo = tf_model(tf_inputs, training=False) # Make sure restore ops are run + + logger.info("Weights or buffers not loaded from PyTorch model: {}".format(all_pytorch_weights)) + + return tf_model + + +def create_sinusoidal_embeddings(n_pos, dim, out): + position_enc = np.array([ + [pos / np.power(10000, 2 * (j // 2) / dim) for j in range(dim)] + for pos in range(n_pos) + ]) + out[:, 0::2] = tf.constant(np.sin(position_enc[:, 0::2])) + out[:, 1::2] = tf.constant(np.cos(position_enc[:, 1::2])) + + +def gelu(x): + """ Gaussian Error Linear Unit. + Original Implementation of the gelu activation function in Google Bert repo when initialy created. + For information: OpenAI GPT's gelu is slightly different (and gives slightly different results): + 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) + Also see https://arxiv.org/abs/1606.08415 + """ + cdf = 0.5 * (1.0 + tf.math.erf(x / tf.math.sqrt(2.0))) + return x * cdf + + +def get_masks(slen, lengths, causal, padding_mask=None, dtype=tf.float32): + """ + Generate hidden states mask, and optionally an attention mask. + """ + bs = shape_list(lengths)[0] + if padding_mask is not None: + mask = padding_mask + else: + # assert lengths.max().item() <= slen + alen = tf.range(slen) + mask = tf.math.less(alen, lengths[:, tf.newaxis]) + + # attention mask is the same as mask, or triangular inferior attention (causal) + if causal: + attn_mask = tf.less_equal(tf.tile(alen[tf.newaxis, tf.newaxis, :], (bs, slen, 1)), + alen[tf.newaxis, :, tf.newaxis]) + else: + attn_mask = mask + + # sanity check + assert shape_list(mask) == [bs, slen] + assert causal is False or shape_list(attn_mask) == [bs, slen, slen] + + mask = tf.cast(mask, dtype=dtype) + attn_mask = tf.cast(attn_mask, dtype=dtype) + + return mask, attn_mask + + +class TFMultiHeadAttention(tf.keras.layers.Layer): + + NEW_ID = itertools.count() + + def __init__(self, n_heads, dim, config, **kwargs): + super(TFMultiHeadAttention, self).__init__(**kwargs) + self.layer_id = next(TFMultiHeadAttention.NEW_ID) + self.output_attentions = config.output_attentions + self.dim = dim + self.n_heads = n_heads + assert self.dim % self.n_heads == 0 + + self.q_lin = tf.keras.layers.Dense(dim, name='q_lin') + self.k_lin = tf.keras.layers.Dense(dim, name='k_lin') + self.v_lin = tf.keras.layers.Dense(dim, name='v_lin') + self.out_lin = tf.keras.layers.Dense(dim, name='out_lin') + self.dropout = tf.keras.layers.Dropout(config.attention_dropout) + self.pruned_heads = set() + + def prune_heads(self, heads): + raise NotImplementedError + + def call(self, inputs, training=False): + """ + Self-attention (if kv is None) or attention over source sentence (provided by kv). + """ + input, mask, kv, cache, head_mask = inputs + # Input is (bs, qlen, dim) + # Mask is (bs, klen) (non-causal) or (bs, klen, klen) + bs, qlen, dim = shape_list(input) + if kv is None: + klen = qlen if cache is None else cache['slen'] + qlen + else: + klen = shape_list(kv)[1] + # assert dim == self.dim, 'Dimensions do not match: %s input vs %s configured' % (dim, self.dim) + n_heads = self.n_heads + dim_per_head = self.dim // n_heads + mask_reshape = (bs, 1, qlen, klen) if len(shape_list(mask)) == 3 else (bs, 1, 1, klen) + + def shape(x): + """ projection """ + return tf.transpose(tf.reshape(x, (bs, -1, self.n_heads, dim_per_head)), perm=(0, 2, 1, 3)) + + def unshape(x): + """ compute context """ + return tf.reshape(tf.transpose(x, perm=(0, 2, 1, 3)), (bs, -1, self.n_heads * dim_per_head)) + + q = shape(self.q_lin(input)) # (bs, n_heads, qlen, dim_per_head) + if kv is None: + k = shape(self.k_lin(input)) # (bs, n_heads, qlen, dim_per_head) + v = shape(self.v_lin(input)) # (bs, n_heads, qlen, dim_per_head) + elif cache is None or self.layer_id not in cache: + k = v = kv + k = shape(self.k_lin(k)) # (bs, n_heads, qlen, dim_per_head) + v = shape(self.v_lin(v)) # (bs, n_heads, qlen, dim_per_head) + + if cache is not None: + if self.layer_id in cache: + if kv is None: + k_, v_ = cache[self.layer_id] + k = tf.concat([k_, k], axis=2) # (bs, n_heads, klen, dim_per_head) + v = tf.concat([v_, v], axis=2) # (bs, n_heads, klen, dim_per_head) + else: + k, v = cache[self.layer_id] + cache[self.layer_id] = (k, v) + + q = q / math.sqrt(dim_per_head) # (bs, n_heads, qlen, dim_per_head) + scores = tf.matmul(q, k, transpose_b=True) # (bs, n_heads, qlen, klen) + mask = tf.reshape(mask, mask_reshape) # (bs, n_heads, qlen, klen) + # scores.masked_fill_(mask, -float('inf')) # (bs, n_heads, qlen, klen) + scores = scores - 1e30 * (1.0 - mask) + + weights = tf.nn.softmax(scores, axis=-1) # (bs, n_heads, qlen, klen) + weights = self.dropout(weights, training=training) # (bs, n_heads, qlen, klen) + + # Mask heads if we want to + if head_mask is not None: + weights = weights * head_mask + + context = tf.matmul(weights, v) # (bs, n_heads, qlen, dim_per_head) + context = unshape(context) # (bs, qlen, dim) + + outputs = (self.out_lin(context),) + if self.output_attentions: + outputs = outputs + (weights,) + return outputs + + +class TFTransformerFFN(tf.keras.layers.Layer): + + def __init__(self, in_dim, dim_hidden, out_dim, config, **kwargs): + super(TFTransformerFFN, self).__init__(**kwargs) + self.lin1 = tf.keras.layers.Dense(dim_hidden, name='lin1') + self.lin2 = tf.keras.layers.Dense(out_dim, name='lin2') + self.act = tf.keras.layers.Activation(gelu) if config.gelu_activation else tf.keras.activations.relu + self.dropout = tf.keras.layers.Dropout(config.dropout) + + def call(self, input, training=False): + x = self.lin1(input) + x = self.act(x) + x = self.lin2(x) + x = self.dropout(x, training=training) + return x + + +class TFXLMMainLayer(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFXLMMainLayer, self).__init__(**kwargs) + self.output_attentions = config.output_attentions + self.output_hidden_states = config.output_hidden_states + + # encoder / decoder, output layer + self.is_encoder = config.is_encoder + self.is_decoder = not config.is_encoder + if self.is_decoder: + raise NotImplementedError("Currently XLM can only be used as an encoder") + # self.with_output = with_output + self.causal = config.causal + + # dictionary / languages + self.n_langs = config.n_langs + self.use_lang_emb = config.use_lang_emb + self.n_words = config.n_words + self.eos_index = config.eos_index + self.pad_index = config.pad_index + # self.dico = dico + # self.id2lang = config.id2lang + # self.lang2id = config.lang2id + # assert len(self.dico) == self.n_words + # assert len(self.id2lang) == len(self.lang2id) == self.n_langs + + # model parameters + self.dim = config.emb_dim # 512 by default + self.hidden_dim = self.dim * 4 # 2048 by default + self.n_heads = config.n_heads # 8 by default + self.n_layers = config.n_layers + assert self.dim % self.n_heads == 0, 'transformer dim must be a multiple of n_heads' + + # embeddings + self.dropout = tf.keras.layers.Dropout(config.dropout) + self.attention_dropout = tf.keras.layers.Dropout(config.attention_dropout) + + self.position_embeddings = tf.keras.layers.Embedding(config.max_position_embeddings, self.dim, name='position_embeddings') + if config.sinusoidal_embeddings: + raise NotImplementedError + # create_sinusoidal_embeddings(config.max_position_embeddings, self.dim, out=self.position_embeddings.weight) + if config.n_langs > 1 and config.use_lang_emb: + self.lang_embeddings = tf.keras.layers.Embedding(self.n_langs, self.dim, name='lang_embeddings') + self.embeddings = TFSharedEmbeddings(self.n_words, self.dim, name='embeddings') # padding_idx=self.pad_index) + self.layer_norm_emb = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name='layer_norm_emb') + + # transformer layers + self.attentions = [] + self.layer_norm1 = [] + self.ffns = [] + self.layer_norm2 = [] + # if self.is_decoder: + # self.layer_norm15 = tf.keras.layers.LayerList() + # self.encoder_attn = tf.keras.layers.LayerList() + + for i in range(self.n_layers): + self.attentions.append(TFMultiHeadAttention(self.n_heads, self.dim, config=config, name='attentions__{}'.format(i))) + self.layer_norm1.append(tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name='layer_norm1__{}'.format(i))) + # if self.is_decoder: + # self.layer_norm15.append(nn.LayerNorm(self.dim, eps=config.layer_norm_eps)) + # self.encoder_attn.append(MultiHeadAttention(self.n_heads, self.dim, dropout=self.attention_dropout)) + self.ffns.append(TFTransformerFFN(self.dim, self.hidden_dim, self.dim, config=config, name='ffns__{}'.format(i))) + self.layer_norm2.append(tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name='layer_norm2__{}'.format(i))) + + if hasattr(config, "pruned_heads"): + pruned_heads = config.pruned_heads.copy().items() + config.pruned_heads = {} + for layer, heads in pruned_heads: + if self.attentions[int(layer)].n_heads == config.n_heads: + self.prune_heads({int(layer): list(map(int, heads))}) + + + def _resize_token_embeddings(self, new_num_tokens): + raise NotImplementedError + + def _prune_heads(self, heads_to_prune): + """ Prunes heads of the model. + heads_to_prune: dict of {layer_num: list of heads to prune in this layer} + See base class PreTrainedModel + """ + raise NotImplementedError + + def call(self, inputs, training=False): # removed: src_enc=None, src_len=None + if not isinstance(inputs, (dict, tuple, list)): + input_ids = inputs + (attention_mask, langs, token_type_ids, position_ids, + lengths, cache, head_mask) = None, None, None, None, None, None, None + elif isinstance(inputs, (tuple, list)): + input_ids = inputs[0] + attention_mask = inputs[1] if len(inputs) > 1 else None + langs = inputs[2] if len(inputs) > 2 else None + token_type_ids = inputs[3] if len(inputs) > 3 else None + position_ids = inputs[4] if len(inputs) > 4 else None + lengths = inputs[5] if len(inputs) > 5 else None + cache = inputs[6] if len(inputs) > 6 else None + head_mask = inputs[7] if len(inputs) > 7 else None + assert len(inputs) <= 8, "Too many inputs." + else: + input_ids = inputs.get('input_ids') + attention_mask = inputs.get('attention_mask', None) + langs = inputs.get('langs', None) + token_type_ids = inputs.get('token_type_ids', None) + position_ids = inputs.get('position_ids', None) + lengths = inputs.get('lengths', None) + cache = inputs.get('cache', None) + head_mask = inputs.get('head_mask', None) + assert len(inputs) <= 8, "Too many inputs." + + if lengths is None: + lengths = tf.reduce_sum(tf.cast(tf.not_equal(input_ids, self.pad_index), dtype=tf.int32), axis=1) + # mask = input_ids != self.pad_index + + # check inputs + bs, slen = shape_list(input_ids) + assert shape_list(lengths)[0] == bs + # assert lengths.max().item() <= slen + # input_ids = input_ids.transpose(0, 1) # batch size as dimension 0 + # assert (src_enc is None) == (src_len is None) + # if src_enc is not None: + # assert self.is_decoder + # assert src_enc.size(0) == bs + + # generate masks + mask, attn_mask = get_masks(slen, lengths, self.causal, padding_mask=attention_mask) + # if self.is_decoder and src_enc is not None: + # src_mask = torch.arange(src_len.max(), dtype=torch.long, device=lengths.device) < src_len[:, None] + + # position_ids + if position_ids is None: + position_ids = tf.expand_dims(tf.range(slen), axis=0) + else: + assert shape_list(position_ids) == [bs, slen] # (slen, bs) + # position_ids = position_ids.transpose(0, 1) + + # langs + if langs is not None: + assert shape_list(langs) == [bs, slen] # (slen, bs) + # langs = langs.transpose(0, 1) + + # Prepare head mask if needed + # 1.0 in head_mask indicate we keep the head + # attention_probs has shape bsz x n_heads x N x N + # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] + # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x qlen x klen] + if head_mask is not None: + raise NotImplementedError + else: + head_mask = [None] * self.n_layers + + # do not recompute cached elements + if cache is not None: + _slen = slen - cache['slen'] + input_ids = input_ids[:, -_slen:] + position_ids = position_ids[:, -_slen:] + if langs is not None: + langs = langs[:, -_slen:] + mask = mask[:, -_slen:] + attn_mask = attn_mask[:, -_slen:] + + # embeddings + tensor = self.embeddings(input_ids) + tensor = tensor + self.position_embeddings(position_ids) + if langs is not None and self.use_lang_emb: + tensor = tensor + self.lang_embeddings(langs) + if token_type_ids is not None: + tensor = tensor + self.embeddings(token_type_ids) + tensor = self.layer_norm_emb(tensor) + tensor = self.dropout(tensor, training=training) + tensor = tensor * mask[..., tf.newaxis] + + # transformer layers + hidden_states = () + attentions = () + for i in range(self.n_layers): + if self.output_hidden_states: + hidden_states = hidden_states + (tensor,) + + # self attention + attn_outputs = self.attentions[i]([tensor, attn_mask, None, cache, head_mask[i]], training=training) + attn = attn_outputs[0] + if self.output_attentions: + attentions = attentions + (attn_outputs[1],) + attn = self.dropout(attn, training=training) + tensor = tensor + attn + tensor = self.layer_norm1[i](tensor) + + # encoder attention (for decoder only) + # if self.is_decoder and src_enc is not None: + # attn = self.encoder_attn[i](tensor, src_mask, kv=src_enc, cache=cache) + # attn = F.dropout(attn, p=self.dropout, training=self.training) + # tensor = tensor + attn + # tensor = self.layer_norm15[i](tensor) + + # FFN + tensor = tensor + self.ffns[i](tensor) + tensor = self.layer_norm2[i](tensor) + tensor = tensor * mask[..., tf.newaxis] + + # Add last hidden state + if self.output_hidden_states: + hidden_states = hidden_states + (tensor,) + + # update cache length + if cache is not None: + cache['slen'] += tensor.size(1) + + # move back sequence length to dimension 0 + # tensor = tensor.transpose(0, 1) + + outputs = (tensor,) + if self.output_hidden_states: + outputs = outputs + (hidden_states,) + if self.output_attentions: + outputs = outputs + (attentions,) + return outputs # outputs, (hidden_states), (attentions) + + +class TFXLMPreTrainedModel(TFPreTrainedModel): + """ An abstract class to handle weights initialization and + a simple interface for dowloading and loading pretrained models. + """ + config_class = XLMConfig + pretrained_model_archive_map = TF_XLM_PRETRAINED_MODEL_ARCHIVE_MAP + load_pt_weights = load_xlm_pt_weights_in_tf2 + base_model_prefix = "transformer" + + +XLM_START_DOCSTRING = r""" The XLM model was proposed in + `Cross-lingual Language Model Pretraining`_ + by Guillaume Lample*, Alexis Conneau*. It's a transformer pre-trained using one of the following objectives: + + - a causal language modeling (CLM) objective (next token prediction), + - a masked language modeling (MLM) objective (Bert-like), or + - a Translation Language Modeling (TLM) object (extension of Bert's MLM to multiple language inputs) + + Original code can be found `here`_. + + This model is a PyTorch `torch.tf.keras.layers.Layer`_ sub-class. Use it as a regular PyTorch Module and + refer to the PyTorch documentation for all matter related to general usage and behavior. + + .. _`Cross-lingual Language Model Pretraining`: + https://arxiv.org/abs/1901.07291 + + .. _`torch.tf.keras.layers.Layer`: + https://pytorch.org/docs/stable/nn.html#module + + .. _`here`: + https://github.com/facebookresearch/XLM + + Parameters: + config (:class:`~pytorch_transformers.XLMConfig`): Model configuration class with all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the configuration. + Check out the :meth:`~pytorch_transformers.PreTrainedModel.from_pretrained` method to load the model weights. +""" + +XLM_INPUTS_DOCSTRING = r""" + Inputs: + **input_ids**: ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Indices of input sequence tokens in the vocabulary. + + XLM is a model with absolute position embeddings so it's usually advised to pad the inputs on + the right rather than the left. + + Indices can be obtained using :class:`pytorch_transformers.XLMTokenizer`. + See :func:`pytorch_transformers.PreTrainedTokenizer.encode` and + :func:`pytorch_transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``: + Mask to avoid performing attention on padding token indices. + Mask values selected in ``[0, 1]``: + ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. + **langs**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + A parallel sequence of tokens to be used to indicate the language of each token in the input. + Indices are languages ids which can be obtained from the language names by using two conversion mappings + provided in the configuration of the model (only provided for multilingual models). + More precisely, the `language name -> language id` mapping is in `model.config.lang2id` (dict str -> int) and + the `language id -> language name` mapping is `model.config.id2lang` (dict int -> str). + **token_type_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + A parallel sequence of tokens (can be used to indicate various portions of the inputs). + The embeddings from these tokens will be summed with the respective token embeddings. + Indices are selected in the vocabulary (unlike BERT which has a specific vocabulary for segment indices). + **position_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Indices of positions of each input sequence tokens in the position embeddings. + Selected in the range ``[0, config.max_position_embeddings - 1]``. + **lengths**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Length of each sentence that can be used to avoid performing attention on padding token indices. + You can also use `attention_mask` for the same result (see above), kept here for compatbility. + Indices selected in ``[0, ..., input_ids.size(-1)]``: + **cache**: + dictionary with ``torch.FloatTensor`` that contains pre-computed + hidden-states (key and values in the attention blocks) as computed by the model + (see `cache` output below). Can be used to speed up sequential decoding. + The dictionary object will be modified in-place during the forward pass to add newly computed hidden-states. + **head_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: + Mask to nullify selected heads of the self-attention modules. + Mask values selected in ``[0, 1]``: + ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. +""" + +@add_start_docstrings("The bare XLM Model transformer outputing raw hidden-states without any specific head on top.", + XLM_START_DOCSTRING, XLM_INPUTS_DOCSTRING) +class TFXLMModel(TFXLMPreTrainedModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)`` + Sequence of hidden-states at the last layer of the model. + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = XLMTokenizer.from_pretrained('xlm-mlm-en-2048') + model = XLMModel.from_pretrained('xlm-mlm-en-2048') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids) + last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple + + """ + def __init__(self, config, *inputs, **kwargs): + super(TFXLMModel, self).__init__(config, *inputs, **kwargs) + self.transformer = TFXLMMainLayer(config, name='transformer') + + def call(self, inputs, training=False): + outputs = self.transformer(inputs, training=training) + return outputs + + + +class TFXLMPredLayer(tf.keras.layers.Layer): + """ + Prediction layer (cross_entropy or adaptive_softmax). + """ + def __init__(self, config, input_embeddings, **kwargs): + super(TFXLMPredLayer, self).__init__(**kwargs) + self.asm = config.asm + self.n_words = config.n_words + self.pad_index = config.pad_index + if config.asm is False: + self.input_embeddings = input_embeddings + else: + raise NotImplementedError + # self.proj = nn.AdaptiveLogSoftmaxWithLoss( + # in_features=dim, + # n_classes=config.n_words, + # cutoffs=config.asm_cutoffs, + # div_value=config.asm_div_value, + # head_bias=True, # default is False + # ) + + def build(self, input_shape): + # The output weights are the same as the input embeddings, but there is an output-only bias for each token. + self.bias = self.add_weight(shape=(self.n_words,), + initializer='zeros', + trainable=True, + name='bias') + super(TFXLMPredLayer, self).build(input_shape) + + def call(self, hidden_states): + hidden_states = self.input_embeddings(hidden_states, mode="linear") + hidden_states = hidden_states + self.bias + return hidden_states + + +@add_start_docstrings("""The XLM Model transformer with a language modeling head on top + (linear layer with weights tied to the input embeddings). """, + XLM_START_DOCSTRING, XLM_INPUTS_DOCSTRING) +class TFXLMWithLMHeadModel(TFXLMPreTrainedModel): + r""" + **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Labels for language modeling. + Note that the labels **are shifted** inside the model, i.e. you can set ``lm_labels = input_ids`` + Indices are selected in ``[-1, 0, ..., config.vocab_size]`` + All labels set to ``-1`` are ignored (masked), the loss is only + computed for labels in ``[0, ..., config.vocab_size]`` + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Language modeling loss. + **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = XLMTokenizer.from_pretrained('xlm-mlm-en-2048') + model = XLMWithLMHeadModel.from_pretrained('xlm-mlm-en-2048') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids) + last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple + + """ + def __init__(self, config, *inputs, **kwargs): + super(TFXLMWithLMHeadModel, self).__init__(config, *inputs, **kwargs) + self.transformer = TFXLMMainLayer(config, name='transformer') + self.pred_layer = TFXLMPredLayer(config, self.transformer.embeddings, name='pred_layer') + + + def call(self, inputs, training=False): + transformer_outputs = self.transformer(inputs, training=training) + + output = transformer_outputs[0] + outputs = self.pred_layer(output) + outputs = (outputs,) + transformer_outputs[1:] # Keep new_mems and attention/hidden states if they are here + + return outputs + + +@add_start_docstrings("""XLM Model with a sequence classification/regression head on top (a linear layer on top of + the pooled output) e.g. for GLUE tasks. """, + XLM_START_DOCSTRING, XLM_INPUTS_DOCSTRING) +class TFXLMForSequenceClassification(TFXLMPreTrainedModel): + r""" + **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for computing the sequence classification/regression loss. + Indices should be in ``[0, ..., config.num_labels - 1]``. + If ``config.num_labels == 1`` a regression loss is computed (Mean-Square loss), + If ``config.num_labels > 1`` a classification loss is computed (Cross-Entropy). + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Classification (or regression if config.num_labels==1) loss. + **logits**: ``torch.FloatTensor`` of shape ``(batch_size, config.num_labels)`` + Classification (or regression if config.num_labels==1) scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = XLMTokenizer.from_pretrained('xlm-mlm-en-2048') + model = XLMForSequenceClassification.from_pretrained('xlm-mlm-en-2048') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + labels = torch.tensor([1]).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, labels=labels) + loss, logits = outputs[:2] + + """ + def __init__(self, config, *inputs, **kwargs): + super(TFXLMForSequenceClassification, self).__init__(config, *inputs, **kwargs) + self.num_labels = config.num_labels + + self.transformer = TFXLMMainLayer(config, name='transformer') + self.sequence_summary = TFSequenceSummary(config, name='sequence_summary') + + def call(self, inputs, training=False): + transformer_outputs = self.transformer(inputs, training=training) + output = transformer_outputs[0] + + logits = self.sequence_summary(output) + + outputs = (logits,) + transformer_outputs[1:] # Keep new_mems and attention/hidden states if they are here + return outputs + + +@add_start_docstrings("""XLM Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layers on top of + the hidden-states output to compute `span start logits` and `span end logits`). """, + XLM_START_DOCSTRING, XLM_INPUTS_DOCSTRING) +class TFXLMForQuestionAnsweringSimple(TFXLMPreTrainedModel): + r""" + **start_positions**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for position (index) of the start of the labelled span for computing the token classification loss. + Positions are clamped to the length of the sequence (`sequence_length`). + Position outside of the sequence are not taken into account for computing the loss. + **end_positions**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for position (index) of the end of the labelled span for computing the token classification loss. + Positions are clamped to the length of the sequence (`sequence_length`). + Position outside of the sequence are not taken into account for computing the loss. + **is_impossible**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels whether a question has an answer or no answer (SQuAD 2.0) + **cls_index**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for position (index) of the classification token to use as input for computing plausibility of the answer. + **p_mask**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Optional mask of tokens which can't be in answers (e.g. [CLS], [PAD], ...) + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Total span extraction loss is the sum of a Cross-Entropy for the start and end positions. + **start_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length,)`` + Span-start scores (before SoftMax). + **end_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length,)`` + Span-end scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = XLMTokenizer.from_pretrained('xlm-mlm-en-2048') + model = XLMForQuestionAnswering.from_pretrained('xlm-mlm-en-2048') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + start_positions = torch.tensor([1]) + end_positions = torch.tensor([3]) + outputs = model(input_ids, start_positions=start_positions, end_positions=end_positions) + loss, start_scores, end_scores = outputs[:2] + + """ + def __init__(self, config, *inputs, **kwargs): + super(TFXLMForQuestionAnsweringSimple, self).__init__(config, *inputs, **kwargs) + self.transformer = TFXLMMainLayer(config, name='transformer') + self.qa_outputs = tf.keras.layers.Dense(config.num_labels, name='qa_outputs') + + def call(self, inputs, training=False): + transformer_outputs = self.transformer(inputs, training=training) + + sequence_output = transformer_outputs[0] + + logits = self.qa_outputs(sequence_output) + start_logits, end_logits = tf.split(logits, 2, axis=-1) + start_logits = tf.squeeze(start_logits, axis=-1) + end_logits = tf.squeeze(end_logits, axis=-1) + + outputs = (start_logits, end_logits,) + transformer_outputs[1:] # Keep mems, hidden states, attentions if there are in it + + return outputs # start_logits, end_logits, (hidden_states), (attentions) diff --git a/pytorch_transformers/modeling_tf_xlnet.py b/pytorch_transformers/modeling_tf_xlnet.py index 12f0640531..a65213cb33 100644 --- a/pytorch_transformers/modeling_tf_xlnet.py +++ b/pytorch_transformers/modeling_tf_xlnet.py @@ -69,10 +69,8 @@ def load_xlnet_pt_weights_in_tf2(tf_model, config, pytorch_checkpoint_path): all_pytorch_weights = set(list(state_dict.keys())) for symbolic_weight in symbolic_weights: name = symbolic_weight.name - name = name.replace('cls_mlm', 'cls') # We had to split this layer in two in the TF model to be - name = name.replace('cls_nsp', 'cls') # able to do transfer learning (Keras only allow to remove full layers) name = name.replace(':0', '') - name = name.replace('layer__', 'layer/') + name = name.replace('__', '/') name = name.split('/') name = name[1:] @@ -887,8 +885,6 @@ class TFXLNetLMHeadModel(TFXLNetPreTrainedModel): """ def __init__(self, config, *inputs, **kwargs): super(TFXLNetLMHeadModel, self).__init__(config, *inputs, **kwargs) - self.n_token = config.n_token - self.transformer = TFXLNetMainLayer(config, name='transformer') self.lm_loss = TFXLNetLMHead(config, self.transformer.word_embedding, name='lm_loss') @@ -993,8 +989,6 @@ class TFXLNetForQuestionAnsweringSimple(TFXLNetPreTrainedModel): """ def __init__(self, config, *inputs, **kwargs): super(TFXLNetForQuestionAnsweringSimple, self).__init__(config, *inputs, **kwargs) - self.num_labels = config.num_labels - self.transformer = TFXLNetMainLayer(config, name='transformer') self.qa_outputs = tf.keras.layers.Dense(config.num_labels, name='qa_outputs') diff --git a/pytorch_transformers/modeling_xlm.py b/pytorch_transformers/modeling_xlm.py index 34406f5b61..aeb1fcf689 100644 --- a/pytorch_transformers/modeling_xlm.py +++ b/pytorch_transformers/modeling_xlm.py @@ -337,11 +337,6 @@ class XLMModel(XLMPreTrainedModel): last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple """ - ATTRIBUTES = ['encoder', 'eos_index', 'pad_index', # 'with_output', - 'n_langs', 'use_lang_emb', 'n_words', 'dim', 'n_layers', 'n_heads', - 'hidden_dim', 'dropout', 'attention_dropout', 'asm', - 'asm_cutoffs', 'asm_div_value'] - def __init__(self, config): #, dico, is_encoder, with_output): super(XLMModel, self).__init__(config) self.output_attentions = config.output_attentions diff --git a/pytorch_transformers/tests/modeling_tf_xlm_test.py b/pytorch_transformers/tests/modeling_tf_xlm_test.py new file mode 100644 index 0000000000..5aed818ddd --- /dev/null +++ b/pytorch_transformers/tests/modeling_tf_xlm_test.py @@ -0,0 +1,264 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import unittest +import shutil +import pytest + +from pytorch_transformers import is_tf_available + +if is_tf_available(): + import tensorflow as tf + from pytorch_transformers import (XLMConfig, TFXLMModel, + TFXLMWithLMHeadModel, + TFXLMForSequenceClassification, + TFXLMForQuestionAnsweringSimple, + TF_XLM_PRETRAINED_MODEL_ARCHIVE_MAP) +else: + pytestmark = pytest.mark.skip("Require TensorFlow") + +from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) +from .configuration_common_test import ConfigTester + + +class TFXLMModelTest(TFCommonTestCases.TFCommonModelTester): + + all_model_classes = (TFXLMModel, TFXLMWithLMHeadModel, + TFXLMForSequenceClassification, + TFXLMForQuestionAnsweringSimple) if is_tf_available() else () + + + class TFXLMModelTester(object): + + def __init__(self, + parent, + batch_size=13, + seq_length=7, + is_training=True, + use_input_lengths=True, + use_token_type_ids=True, + use_labels=True, + gelu_activation=True, + sinusoidal_embeddings=False, + causal=False, + asm=False, + n_langs=2, + vocab_size=99, + n_special=0, + hidden_size=32, + num_hidden_layers=5, + num_attention_heads=4, + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + type_sequence_label_size=2, + initializer_range=0.02, + num_labels=3, + num_choices=4, + summary_type="last", + use_proj=True, + scope=None, + ): + self.parent = parent + self.batch_size = batch_size + self.seq_length = seq_length + self.is_training = is_training + self.use_input_lengths = use_input_lengths + self.use_token_type_ids = use_token_type_ids + self.use_labels = use_labels + self.gelu_activation = gelu_activation + self.sinusoidal_embeddings = sinusoidal_embeddings + self.asm = asm + self.n_langs = n_langs + self.vocab_size = vocab_size + self.n_special = n_special + self.summary_type = summary_type + self.causal = causal + self.use_proj = use_proj + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.n_langs = n_langs + self.type_sequence_label_size = type_sequence_label_size + self.initializer_range = initializer_range + self.summary_type = summary_type + self.num_labels = num_labels + self.num_choices = num_choices + self.scope = scope + + def prepare_config_and_inputs(self): + input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) + input_mask = ids_tensor([self.batch_size, self.seq_length], 2, dtype=tf.float32) + + input_lengths = None + if self.use_input_lengths: + input_lengths = ids_tensor([self.batch_size], vocab_size=2) + self.seq_length - 2 # small variation of seq_length + + token_type_ids = None + if self.use_token_type_ids: + token_type_ids = ids_tensor([self.batch_size, self.seq_length], self.n_langs) + + sequence_labels = None + token_labels = None + is_impossible_labels = None + if self.use_labels: + sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size) + token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels) + is_impossible_labels = ids_tensor([self.batch_size], 2, dtype=tf.float32) + + config = XLMConfig( + vocab_size_or_config_json_file=self.vocab_size, + n_special=self.n_special, + emb_dim=self.hidden_size, + n_layers=self.num_hidden_layers, + n_heads=self.num_attention_heads, + dropout=self.hidden_dropout_prob, + attention_dropout=self.attention_probs_dropout_prob, + gelu_activation=self.gelu_activation, + sinusoidal_embeddings=self.sinusoidal_embeddings, + asm=self.asm, + causal=self.causal, + n_langs=self.n_langs, + max_position_embeddings=self.max_position_embeddings, + initializer_range=self.initializer_range, + summary_type=self.summary_type, + use_proj=self.use_proj) + + return config, input_ids, token_type_ids, input_lengths, sequence_labels, token_labels, is_impossible_labels, input_mask + + def create_and_check_xlm_model(self, config, input_ids, token_type_ids, input_lengths, sequence_labels, token_labels, is_impossible_labels, input_mask): + model = TFXLMModel(config=config) + inputs = {'input_ids': input_ids, + 'lengths': input_lengths, + 'langs': token_type_ids} + outputs = model(inputs) + + inputs = [input_ids, input_mask] + outputs = model(inputs) + sequence_output = outputs[0] + result = { + "sequence_output": sequence_output.numpy(), + } + self.parent.assertListEqual( + list(result["sequence_output"].shape), + [self.batch_size, self.seq_length, self.hidden_size]) + + + def create_and_check_xlm_lm_head(self, config, input_ids, token_type_ids, input_lengths, sequence_labels, token_labels, is_impossible_labels, input_mask): + model = TFXLMWithLMHeadModel(config) + + inputs = {'input_ids': input_ids, + 'lengths': input_lengths, + 'langs': token_type_ids} + outputs = model(inputs) + + logits = outputs[0] + + result = { + "logits": logits.numpy(), + } + + self.parent.assertListEqual( + list(result["logits"].shape), + [self.batch_size, self.seq_length, self.vocab_size]) + + + def create_and_check_xlm_qa(self, config, input_ids, token_type_ids, input_lengths, sequence_labels, token_labels, is_impossible_labels, input_mask): + model = TFXLMForQuestionAnsweringSimple(config) + + inputs = {'input_ids': input_ids, + 'lengths': input_lengths} + + outputs = model(inputs) + start_logits, end_logits = model(inputs) + + result = { + "start_logits": start_logits.numpy(), + "end_logits": end_logits.numpy(), + } + + self.parent.assertListEqual( + list(result["start_logits"].shape), + [self.batch_size, self.seq_length]) + self.parent.assertListEqual( + list(result["end_logits"].shape), + [self.batch_size, self.seq_length]) + + + def create_and_check_xlm_sequence_classif(self, config, input_ids, token_type_ids, input_lengths, sequence_labels, token_labels, is_impossible_labels, input_mask): + model = TFXLMForSequenceClassification(config) + + inputs = {'input_ids': input_ids, + 'lengths': input_lengths} + + (logits,) = model(inputs) + + result = { + "logits": logits.numpy(), + } + + self.parent.assertListEqual( + list(result["logits"].shape), + [self.batch_size, self.type_sequence_label_size]) + + + def prepare_config_and_inputs_for_common(self): + config_and_inputs = self.prepare_config_and_inputs() + (config, input_ids, token_type_ids, input_lengths, + sequence_labels, token_labels, is_impossible_labels, input_mask) = config_and_inputs + inputs_dict = {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'lengths': input_lengths} + return config, inputs_dict + + def setUp(self): + self.model_tester = TFXLMModelTest.TFXLMModelTester(self) + self.config_tester = ConfigTester(self, config_class=XLMConfig, emb_dim=37) + + def test_config(self): + self.config_tester.run_common_tests() + + def test_xlm_model(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_xlm_model(*config_and_inputs) + + def test_xlm_lm_head(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_xlm_lm_head(*config_and_inputs) + + def test_xlm_qa(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_xlm_qa(*config_and_inputs) + + def test_xlm_sequence_classif(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_xlm_sequence_classif(*config_and_inputs) + + @pytest.mark.slow + def test_model_from_pretrained(self): + cache_dir = "/tmp/pytorch_transformers_test/" + for model_name in list(TF_XLM_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: + model = XLMModel.from_pretrained(model_name, cache_dir=cache_dir) + shutil.rmtree(cache_dir) + self.assertIsNotNone(model) + + +if __name__ == "__main__": + unittest.main() diff --git a/pytorch_transformers/tests/modeling_xlm_test.py b/pytorch_transformers/tests/modeling_xlm_test.py index acc8a68066..841bea94b7 100644 --- a/pytorch_transformers/tests/modeling_xlm_test.py +++ b/pytorch_transformers/tests/modeling_xlm_test.py @@ -272,20 +272,17 @@ class XLMModelTest(CommonTestCases.CommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_xlm_model(*config_and_inputs) - # config_and_inputs = tester.prepare_config_and_inputs() - # tester.create_and_check_xlm_for_masked_lm(*config_and_inputs) + def test_xlm_lm_head(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_xlm_lm_head(*config_and_inputs) - # config_and_inputs = tester.prepare_config_and_inputs() - # tester.create_and_check_xlm_for_multiple_choice(*config_and_inputs) + def test_xlm_qa(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_xlm_qa(*config_and_inputs) - # config_and_inputs = tester.prepare_config_and_inputs() - # tester.create_and_check_xlm_for_question_answering(*config_and_inputs) - - # config_and_inputs = tester.prepare_config_and_inputs() - # tester.create_and_check_xlm_for_sequence_classification(*config_and_inputs) - - # config_and_inputs = tester.prepare_config_and_inputs() - # tester.create_and_check_xlm_for_token_classification(*config_and_inputs) + def test_xlm_sequence_classif(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_xlm_sequence_classif(*config_and_inputs) @pytest.mark.slow def test_model_from_pretrained(self): From 646711e1e279b9af21bc225e5113d04b0c243029 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Wed, 11 Sep 2019 15:34:17 +0200 Subject: [PATCH 044/219] standardize scopes names - add conversion methods --- .../convert_pytorch_checkpoint_to_tf2.py | 8 +- ..._original_pytorch_checkpoint_to_pytorch.py | 12 +- pytorch_transformers/modeling_tf_bert.py | 86 ++---------- pytorch_transformers/modeling_tf_gpt2.py | 76 +---------- .../modeling_tf_pytorch_utils.py | 122 ++++++++++++++++++ pytorch_transformers/modeling_tf_xlm.py | 85 +++--------- pytorch_transformers/modeling_tf_xlnet.py | 67 +--------- 7 files changed, 177 insertions(+), 279 deletions(-) create mode 100644 pytorch_transformers/modeling_tf_pytorch_utils.py diff --git a/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py b/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py index 0acaf92788..b3389dcbf3 100644 --- a/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py +++ b/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py @@ -25,12 +25,13 @@ from pytorch_transformers import is_torch_available from pytorch_transformers import (BertConfig, TFBertForPreTraining, load_bert_pt_weights_in_tf2, GPT2Config, TFGPT2LMHeadModel, load_gpt2_pt_weights_in_tf2, - XLNetConfig, TFXLNetLMHeadModel, load_xlnet_pt_weights_in_tf2) + XLNetConfig, TFXLNetLMHeadModel, load_xlnet_pt_weights_in_tf2, + XLMConfig, TFXLMWithLMHeadModel, load_xlm_pt_weights_in_tf2,) if is_torch_available(): import torch import numpy as np - from pytorch_transformers import BertForPreTraining, GPT2LMHeadModel, XLNetLMHeadModel + from pytorch_transformers import BertForPreTraining, GPT2LMHeadModel, XLNetLMHeadModel, XLMWithLMHeadModel else: BertForPreTraining, GPT2LMHeadModel = None, None @@ -42,6 +43,7 @@ MODEL_CLASSES = { 'bert': (BertConfig, TFBertForPreTraining, load_bert_pt_weights_in_tf2, BertForPreTraining), 'gpt2': (GPT2Config, TFGPT2LMHeadModel, load_gpt2_pt_weights_in_tf2, GPT2LMHeadModel), 'xlnet': (XLNetConfig, TFXLNetLMHeadModel, load_xlnet_pt_weights_in_tf2, XLNetLMHeadModel), + 'xlm': (XLMConfig, TFXLMWithLMHeadModel, load_xlm_pt_weights_in_tf2, XLMWithLMHeadModel), } def convert_pt_checkpoint_to_tf(model_type, pytorch_checkpoint_path, config_file, tf_dump_path, compare_with_pt_model=False): @@ -58,7 +60,7 @@ def convert_pt_checkpoint_to_tf(model_type, pytorch_checkpoint_path, config_file tf_model = model_class(config) # Load weights from tf checkpoint - tf_model = loading_fct(tf_model, config, pytorch_checkpoint_path) + tf_model = loading_fct(tf_model, pytorch_checkpoint_path) if compare_with_pt_model: inputs_list = [[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]] diff --git a/pytorch_transformers/convert_xlm_original_pytorch_checkpoint_to_pytorch.py b/pytorch_transformers/convert_xlm_original_pytorch_checkpoint_to_pytorch.py index d6a3cd89e7..aecd936920 100755 --- a/pytorch_transformers/convert_xlm_original_pytorch_checkpoint_to_pytorch.py +++ b/pytorch_transformers/convert_xlm_original_pytorch_checkpoint_to_pytorch.py @@ -33,7 +33,15 @@ def convert_xlm_checkpoint_to_pytorch(xlm_checkpoint_path, pytorch_dump_folder_p # Load checkpoint chkpt = torch.load(xlm_checkpoint_path, map_location='cpu') - model = chkpt['model'] + state_dict = chkpt['model'] + + # We have the base model one level deeper than the original XLM repository + two_levels_state_dict = {} + for k, v in state_dict.items(): + if 'pred_layer' in k: + two_levels_state_dict[k] = v + else: + two_levels_state_dict['transformer.' + k] = v config = chkpt['params'] config = dict((n, v) for n, v in config.items() if not isinstance(v, (torch.FloatTensor, numpy.ndarray))) @@ -47,7 +55,7 @@ def convert_xlm_checkpoint_to_pytorch(xlm_checkpoint_path, pytorch_dump_folder_p pytorch_vocab_dump_path = pytorch_dump_folder_path + '/' + VOCAB_FILES_NAMES['vocab_file'] print("Save PyTorch model to {}".format(pytorch_weights_dump_path)) - torch.save(model, pytorch_weights_dump_path) + torch.save(two_levels_state_dict, pytorch_weights_dump_path) print("Save configuration file to {}".format(pytorch_config_dump_path)) with open(pytorch_config_dump_path, "w", encoding="utf-8") as f: diff --git a/pytorch_transformers/modeling_tf_bert.py b/pytorch_transformers/modeling_tf_bert.py index f1b5ec7109..acbdec86fd 100644 --- a/pytorch_transformers/modeling_tf_bert.py +++ b/pytorch_transformers/modeling_tf_bert.py @@ -30,6 +30,7 @@ import tensorflow as tf from .configuration_bert import BertConfig from .modeling_tf_utils import TFPreTrainedModel from .file_utils import add_start_docstrings +from .modeling_tf_pytorch_utils import load_pytorch_checkpoint_in_tf2_model logger = logging.getLogger(__name__) @@ -51,71 +52,12 @@ TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP = { } -def load_bert_pt_weights_in_tf2(tf_model, config, pytorch_checkpoint_path): - """ Load pytorch checkpoints in a TF 2.0 model and save it using HDF5 format - We use HDF5 to easily do transfer learning - (see https://github.com/tensorflow/tensorflow/blob/ee16fcac960ae660e0e4496658a366e2f745e1f0/tensorflow/python/keras/engine/network.py#L1352-L1357). - """ - try: - import re - import torch - import numpy - from tensorflow.python.keras import backend as K - except ImportError: - logger.error("Loading a PyTorch model in TensorFlow, requires PyTorch to be installed. Please see " - "https://pytorch.org/ for installation instructions.") - raise - - pt_path = os.path.abspath(pytorch_checkpoint_path) - logger.info("Loading PyTorch weights from {}".format(pt_path)) - # Load pytorch model - state_dict = torch.load(pt_path, map_location='cpu') - +def load_bert_pt_weights_in_tf2(tf_model, pytorch_checkpoint_path): + # build the network inputs_list = [[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]] tf_inputs = tf.constant(inputs_list) - tfo = tf_model(tf_inputs, training=False) # build the network - - symbolic_weights = tf_model.trainable_weights + tf_model.non_trainable_weights - weight_value_tuples = [] - all_pytorch_weights = set(list(state_dict.keys())) - for symbolic_weight in symbolic_weights: - name = symbolic_weight.name - name = name.replace('cls_mlm', 'cls') # We had to split this layer in two in the TF model to be - name = name.replace('cls_nsp', 'cls') # able to do transfer learning (Keras only allow to remove full layers) - name = name.replace(':0', '') - name = name.replace('__', '/') - name = name.split('/') - name = name[1:] - - transpose = bool(name[-1] == 'kernel') - if name[-1] == 'kernel' or name[-1] == 'embeddings': - name[-1] = 'weight' - - name = '.'.join(name) - assert name in state_dict, "{} not found in PyTorch model".format(name) - array = state_dict[name].numpy() - - if transpose: - array = numpy.transpose(array) - - try: - assert list(symbolic_weight.shape) == list(array.shape) - except AssertionError as e: - e.args += (symbolic_weight.shape, array.shape) - raise e - - logger.info("Initialize TF weight {}".format(symbolic_weight.name)) - - weight_value_tuples.append((symbolic_weight, array)) - all_pytorch_weights.discard(name) - - K.batch_set_value(weight_value_tuples) - - tfo = tf_model(tf_inputs, training=False) # Make sure restore ops are run - - logger.info("Weights or buffers not loaded from PyTorch model: {}".format(all_pytorch_weights)) - - return tf_model + tfo = tf_model(tf_inputs, training=False) + return load_pytorch_checkpoint_in_tf2_model(tf_model, pytorch_checkpoint_path) def gelu(x): @@ -391,7 +333,7 @@ class TFBertEncoder(tf.keras.layers.Layer): super(TFBertEncoder, self).__init__(**kwargs) self.output_attentions = config.output_attentions self.output_hidden_states = config.output_hidden_states - self.layer = [TFBertLayer(config, name='layer__{}'.format(i)) for i in range(config.num_hidden_layers)] + self.layer = [TFBertLayer(config, name='layer_._{}'.format(i)) for i in range(config.num_hidden_layers)] def call(self, inputs, training=False): hidden_states, attention_mask, head_mask = inputs @@ -730,15 +672,15 @@ class TFBertForPreTraining(TFBertPreTrainedModel): super(TFBertForPreTraining, self).__init__(config, *inputs, **kwargs) self.bert = TFBertMainLayer(config, name='bert') - self.cls_nsp = TFBertNSPHead(config, name='cls_nsp') - self.cls_mlm = TFBertMLMHead(config, self.bert.embeddings, name='cls_mlm') + self.nsp = TFBertNSPHead(config, name='nsp___cls') + self.mlm = TFBertMLMHead(config, self.bert.embeddings, name='mlm___cls') def call(self, inputs, training=False): outputs = self.bert(inputs, training=training) sequence_output, pooled_output = outputs[:2] - prediction_scores = self.cls_mlm(sequence_output, training=training) - seq_relationship_score = self.cls_nsp(pooled_output) + prediction_scores = self.mlm(sequence_output, training=training) + seq_relationship_score = self.nsp(pooled_output) outputs = (prediction_scores, seq_relationship_score,) + outputs[2:] # add hidden states and attention if they are here @@ -773,13 +715,13 @@ class TFBertForMaskedLM(TFBertPreTrainedModel): super(TFBertForMaskedLM, self).__init__(config, *inputs, **kwargs) self.bert = TFBertMainLayer(config, name='bert') - self.cls_mlm = TFBertMLMHead(config, self.bert.embeddings, name='cls_mlm') + self.mlm = TFBertMLMHead(config, self.bert.embeddings, name='mlm___cls') def call(self, inputs, training=False): outputs = self.bert(inputs, training=training) sequence_output = outputs[0] - prediction_scores = self.cls_mlm(sequence_output, training=training) + prediction_scores = self.mlm(sequence_output, training=training) outputs = (prediction_scores,) + outputs[2:] # Add hidden states and attention if they are here @@ -816,13 +758,13 @@ class TFBertForNextSentencePrediction(TFBertPreTrainedModel): super(TFBertForNextSentencePrediction, self).__init__(config, *inputs, **kwargs) self.bert = TFBertMainLayer(config, name='bert') - self.cls_nsp = TFBertNSPHead(config, name='cls_nsp') + self.nsp = TFBertNSPHead(config, name='nsp___cls') def call(self, inputs, training=False): outputs = self.bert(inputs, training=training) pooled_output = outputs[1] - seq_relationship_score = self.cls_nsp(pooled_output) + seq_relationship_score = self.nsp(pooled_output) outputs = (seq_relationship_score,) + outputs[2:] # add hidden states and attention if they are here diff --git a/pytorch_transformers/modeling_tf_gpt2.py b/pytorch_transformers/modeling_tf_gpt2.py index 491b592b13..6be0190880 100644 --- a/pytorch_transformers/modeling_tf_gpt2.py +++ b/pytorch_transformers/modeling_tf_gpt2.py @@ -32,6 +32,7 @@ from .modeling_tf_utils import (TFPreTrainedModel, TFConv1D, TFSharedEmbeddings, TFSequenceSummary, shape_list) from .configuration_gpt2 import GPT2Config from .file_utils import add_start_docstrings +from .modeling_tf_pytorch_utils import load_pytorch_checkpoint_in_tf2_model logger = logging.getLogger(__name__) @@ -40,77 +41,12 @@ TF_GPT2_PRETRAINED_MODEL_ARCHIVE_MAP = {"gpt2": "https://s3.amazonaws.com/models "gpt2-large": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-large-tf_model.h5"} -def load_gpt2_pt_weights_in_tf2(tf_model, config, pytorch_checkpoint_path): - """ Load pytorch checkpoints in a TF 2.0 model and save it using HDF5 format - We use HDF5 to easily do transfer learning - (see https://github.com/tensorflow/tensorflow/blob/ee16fcac960ae660e0e4496658a366e2f745e1f0/tensorflow/python/keras/engine/network.py#L1352-L1357). - """ - try: - import re - import torch - import numpy - from tensorflow.python.keras import backend as K - except ImportError: - logger.error("Loading a PyTorch model in TensorFlow, requires PyTorch to be installed. Please see " - "https://pytorch.org/ for installation instructions.") - raise - - pt_path = os.path.abspath(pytorch_checkpoint_path) - logger.info("Loading PyTorch weights from {}".format(pt_path)) - # Load pytorch model - state_dict = torch.load(pt_path, map_location='cpu') - +def load_gpt2_pt_weights_in_tf2(tf_model, pytorch_checkpoint_path): + # build the network inputs_list = [[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]] tf_inputs = tf.constant(inputs_list) - tfo = tf_model(tf_inputs, training=False) # build the network - - symbolic_weights = tf_model.trainable_weights + tf_model.non_trainable_weights - weight_value_tuples = [] - all_pytorch_weights = set(list(state_dict.keys())) - for symbolic_weight in symbolic_weights: - name = symbolic_weight.name - name = name.replace(':0', '') - name = name.replace('__', '/') - name = name.split('/') - name = name[2:] - - transpose = bool(name[-1] == 'kernel') - if name[-1] == 'kernel' or name[-1] == 'embeddings' or name[-1] == 'gamma': - name[-1] = 'weight' - if name[-1] == 'beta': - name[-1] = 'bias' - - name = '.'.join(name) - assert name in state_dict, "Weight {} not in PyTorch model".format(name) - array = state_dict[name].numpy() - - if transpose: - array = numpy.transpose(array) - - if len(symbolic_weight.shape) > len(array.shape): - array = array[None, ...] - if len(symbolic_weight.shape) < len(array.shape): - array = np.squeeze(array) - - try: - assert list(symbolic_weight.shape) == list(array.shape) - except AssertionError as e: - e.args += (symbolic_weight.shape, array.shape) - raise e - - logger.info("Initialize TF weight {}".format(symbolic_weight.name)) - - weight_value_tuples.append((symbolic_weight, array)) - - all_pytorch_weights.discard(name) - - K.batch_set_value(weight_value_tuples) - - tfo = tf_model(tf_inputs, training=False) # Make sure restore ops are run - - logger.info("Weights or buffers not loaded from PyTorch model: {}".format(all_pytorch_weights)) - - return tf_model + tfo = tf_model(tf_inputs, training=False) + return load_pytorch_checkpoint_in_tf2_model(tf_model, pytorch_checkpoint_path) def gelu(x): @@ -282,7 +218,7 @@ class TFGPT2MainLayer(tf.keras.layers.Layer): self.h = [TFBlock(config.n_ctx, config, scale=True, - name='h__{}'.format(i)) for i in range(config.n_layer)] + name='h_._{}'.format(i)) for i in range(config.n_layer)] self.ln_f = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_epsilon, name='ln_f') def _resize_token_embeddings(self, new_num_tokens): diff --git a/pytorch_transformers/modeling_tf_pytorch_utils.py b/pytorch_transformers/modeling_tf_pytorch_utils.py new file mode 100644 index 0000000000..990319c7e4 --- /dev/null +++ b/pytorch_transformers/modeling_tf_pytorch_utils.py @@ -0,0 +1,122 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" PyTorch - TF 2.0 general utilities.""" + +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +import logging + +from pytorch_transformers import is_tf_available, is_torch_available + +logger = logging.getLogger(__name__) + + +def load_pytorch_checkpoint_in_tf2_model(tf_model, pytorch_checkpoint_path): + """ Load pytorch checkpoints in a TF 2.0 model + Conventions for TF2.0 scopes -> PyTorch attribute names conversions: + - '$1___$2' is replaced by $2 (can be used to duplicate or remove layers in TF2.0 vs PyTorch) + - '_._' is replaced by a new level separation (can be used to convert TF2.0 lists in PyTorch nn.ModulesList) + """ + if not is_tf_available() or not is_torch_available(): + logger.error("Loading a PyTorch model in TensorFlow, requires both PyTorch and TensorFlow to be installed. Please see " + "https://pytorch.org/ and https://www.tensorflow.org/install/ for installation instructions.") + raise ImportError + + import torch + + pt_path = os.path.abspath(pytorch_checkpoint_path) + logger.info("Loading PyTorch weights from {}".format(pt_path)) + + pt_state_dict = torch.load(pt_path, map_location='cpu') + + return load_pytorch_state_dict_in_tf2_model(tf_model, pt_state_dict) + + +def load_pytorch_state_dict_in_tf2_model(tf_model, pt_state_dict): + """ Load pytorch state_dict in a TF 2.0 model. + Conventions for TF2.0 scopes -> PyTorch attribute names conversions: + - '$1___$2' is replaced by $2 (can be used to duplicate or remove layers in TF2.0 vs PyTorch) + - '_._' is replaced by a level separation (can be used to convert TF2.0 lists in PyTorch nn.ModulesList) + """ + try: + import re + import torch + import numpy + from tensorflow.python.keras import backend as K + except ImportError as e: + logger.error("Loading a PyTorch model in TensorFlow, requires both PyTorch and TensorFlow to be installed. Please see " + "https://pytorch.org/ and https://www.tensorflow.org/install/ for installation instructions.") + raise e + + symbolic_weights = tf_model.trainable_weights + tf_model.non_trainable_weights + + weight_value_tuples = [] + all_pytorch_weights = set(list(pt_state_dict.keys())) + for symbolic_weight in symbolic_weights: + name = symbolic_weight.name + name = name.replace(':0', '') # device ids + name = re.sub(r'/[^/]*___([^/]*)/', r'/\1/', name) # '$1___$2' is replaced by $2 (can be used to duplicate or remove layers in TF2.0 vs PyTorch) + name = name.replace('_._', '/') # '_._' is replaced by a level separation (can be used to convert TF2.0 lists in PyTorch nn.ModulesList) + name = re.sub(r'//+', '/', name) # Remove empty levels at the end + name = name.split('/') # Convert from TF2.0 '/' separators to PyTorch '.' separators + name = name[1:] # Remove level zero + + # Convert standard TF2.0 names in PyTorch names + transpose = bool(name[-1] == 'kernel') + if name[-1] == 'kernel' or name[-1] == 'embeddings' or name[-1] == 'gamma': + name[-1] = 'weight' + if name[-1] == 'beta': + name[-1] = 'bias' + + name = '.'.join(name) + assert name in pt_state_dict, "{} not found in PyTorch model".format(name) + array = pt_state_dict[name].numpy() + + if transpose: + array = numpy.transpose(array) + + try: + assert list(symbolic_weight.shape) == list(array.shape) + except AssertionError as e: + e.args += (symbolic_weight.shape, array.shape) + raise e + + logger.info("Initialize TF weight {}".format(symbolic_weight.name)) + + weight_value_tuples.append((symbolic_weight, array)) + all_pytorch_weights.discard(name) + + K.batch_set_value(weight_value_tuples) + + tfo = tf_model(tf_inputs, training=False) # Make sure restore ops are run + + logger.info("Weights or buffers not loaded from PyTorch model: {}".format(all_pytorch_weights)) + + return tf_model + + +def load_tf2_checkpoint_in_pytorch_model(pt_model, tf_checkpoint_path): + """ Load TF 2.0 HDF5 checkpoint in a PyTorch model + We use HDF5 to easily do transfer learning + (see https://github.com/tensorflow/tensorflow/blob/ee16fcac960ae660e0e4496658a366e2f745e1f0/tensorflow/python/keras/engine/network.py#L1352-L1357). + """ + raise NotImplementedError + +def load_tf2_weights_in_pytorch_model(pt_model, tf_model): + """ Load TF2.0 symbolic weights in a PyTorch model + """ + raise NotImplementedError diff --git a/pytorch_transformers/modeling_tf_xlm.py b/pytorch_transformers/modeling_tf_xlm.py index d0db638013..1b4d99c14a 100644 --- a/pytorch_transformers/modeling_tf_xlm.py +++ b/pytorch_transformers/modeling_tf_xlm.py @@ -18,6 +18,7 @@ from __future__ import absolute_import, division, print_function, unicode_litera import logging import math +import os import itertools import numpy as np @@ -26,6 +27,7 @@ import tensorflow as tf from .configuration_xlm import XLMConfig from .modeling_tf_utils import TFPreTrainedModel, TFSharedEmbeddings, TFSequenceSummary, shape_list from .file_utils import add_start_docstrings +from .modeling_tf_pytorch_utils import load_pytorch_checkpoint_in_tf2_model logger = logging.getLogger(__name__) @@ -43,71 +45,16 @@ TF_XLM_PRETRAINED_MODEL_ARCHIVE_MAP = { } -def load_xlm_pt_weights_in_tf2(tf_model, config, pytorch_checkpoint_path): - """ Load pytorch checkpoints in a TF 2.0 model and save it using HDF5 format - We use HDF5 to easily do transfer learning - (see https://github.com/tensorflow/tensorflow/blob/ee16fcac960ae660e0e4496658a366e2f745e1f0/tensorflow/python/keras/engine/network.py#L1352-L1357). - """ - try: - import re - import torch - import numpy - from tensorflow.python.keras import backend as K - except ImportError: - logger.error("Loading a PyTorch model in TensorFlow, requires PyTorch to be installed. Please see " - "https://pytorch.org/ for installation instructions.") - raise - - pt_path = os.path.abspath(pytorch_checkpoint_path) - logger.info("Loading PyTorch weights from {}".format(pt_path)) - # Load pytorch model - state_dict = torch.load(pt_path, map_location='cpu') - +def load_xlm_pt_weights_in_tf2(tf_model, pytorch_checkpoint_path): + # build the network inputs_list = [[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]] + attns_list = [[1, 1, 0, 0, 1], [1, 1, 1, 0, 0], [1, 0, 0, 1, 1]] + langs_list = [[1, 1, 0, 0, 1], [1, 1, 1, 0, 0], [1, 0, 0, 1, 1]] tf_inputs = tf.constant(inputs_list) - tfo = tf_model(tf_inputs, training=False) # build the network - - symbolic_weights = tf_model.trainable_weights + tf_model.non_trainable_weights - weight_value_tuples = [] - all_pytorch_weights = set(list(state_dict.keys())) - for symbolic_weight in symbolic_weights: - name = symbolic_weight.name - name = name.replace(':0', '') - name = name.replace('__', '/') - name = name.split('/') - name = name[1:] - - transpose = bool(name[-1] == 'kernel') - if name[-1] == 'kernel' or name[-1] == 'embeddings' or name[-1] == 'gamma': - name[-1] = 'weight' - if name[-1] == 'beta': - name[-1] = 'bias' - - name = '.'.join(name) - assert name in state_dict, "{} not found in PyTorch model".format(name) - array = state_dict[name].numpy() - - if transpose: - array = numpy.transpose(array) - - try: - assert list(symbolic_weight.shape) == list(array.shape) - except AssertionError as e: - e.args += (symbolic_weight.shape, array.shape) - raise e - - logger.info("Initialize TF weight {}".format(symbolic_weight.name)) - - weight_value_tuples.append((symbolic_weight, array)) - all_pytorch_weights.discard(name) - - K.batch_set_value(weight_value_tuples) - - tfo = tf_model(tf_inputs, training=False) # Make sure restore ops are run - - logger.info("Weights or buffers not loaded from PyTorch model: {}".format(all_pytorch_weights)) - - return tf_model + tf_attns = tf.constant(attns_list) + tf_langs = tf.constant(langs_list) + tfo = tf_model([tf_inputs, tf_attns, tf_langs], training=False) + return load_pytorch_checkpoint_in_tf2_model(tf_model, pytorch_checkpoint_path) def create_sinusoidal_embeddings(n_pos, dim, out): @@ -320,13 +267,13 @@ class TFXLMMainLayer(tf.keras.layers.Layer): # self.encoder_attn = tf.keras.layers.LayerList() for i in range(self.n_layers): - self.attentions.append(TFMultiHeadAttention(self.n_heads, self.dim, config=config, name='attentions__{}'.format(i))) - self.layer_norm1.append(tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name='layer_norm1__{}'.format(i))) + self.attentions.append(TFMultiHeadAttention(self.n_heads, self.dim, config=config, name='attentions_._{}'.format(i))) + self.layer_norm1.append(tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name='layer_norm1_._{}'.format(i))) # if self.is_decoder: # self.layer_norm15.append(nn.LayerNorm(self.dim, eps=config.layer_norm_eps)) # self.encoder_attn.append(MultiHeadAttention(self.n_heads, self.dim, dropout=self.attention_dropout)) - self.ffns.append(TFTransformerFFN(self.dim, self.hidden_dim, self.dim, config=config, name='ffns__{}'.format(i))) - self.layer_norm2.append(tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name='layer_norm2__{}'.format(i))) + self.ffns.append(TFTransformerFFN(self.dim, self.hidden_dim, self.dim, config=config, name='ffns_._{}'.format(i))) + self.layer_norm2.append(tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name='layer_norm2_._{}'.format(i))) if hasattr(config, "pruned_heads"): pruned_heads = config.pruned_heads.copy().items() @@ -667,8 +614,8 @@ class TFXLMWithLMHeadModel(TFXLMPreTrainedModel): """ def __init__(self, config, *inputs, **kwargs): super(TFXLMWithLMHeadModel, self).__init__(config, *inputs, **kwargs) - self.transformer = TFXLMMainLayer(config, name='transformer') - self.pred_layer = TFXLMPredLayer(config, self.transformer.embeddings, name='pred_layer') + self.transformer = TFXLMMainLayer(config, name='transformer___') + self.pred_layer = TFXLMPredLayer(config, self.transformer.embeddings, name='pred_layer_._proj') def call(self, inputs, training=False): diff --git a/pytorch_transformers/modeling_tf_xlnet.py b/pytorch_transformers/modeling_tf_xlnet.py index a65213cb33..c0754e8302 100644 --- a/pytorch_transformers/modeling_tf_xlnet.py +++ b/pytorch_transformers/modeling_tf_xlnet.py @@ -30,6 +30,7 @@ import tensorflow as tf from .configuration_xlnet import XLNetConfig from .modeling_tf_utils import TFPreTrainedModel, TFSharedEmbeddings, TFSequenceSummary, shape_list from .file_utils import add_start_docstrings +from .modeling_tf_pytorch_utils import load_pytorch_checkpoint_in_tf2_model logger = logging.getLogger(__name__) @@ -40,71 +41,11 @@ TF_XLNET_PRETRAINED_MODEL_ARCHIVE_MAP = { } -def load_xlnet_pt_weights_in_tf2(tf_model, config, pytorch_checkpoint_path): - """ Load pytorch checkpoints in a TF 2.0 model and save it using HDF5 format - We use HDF5 to easily do transfer learning - (see https://github.com/tensorflow/tensorflow/blob/ee16fcac960ae660e0e4496658a366e2f745e1f0/tensorflow/python/keras/engine/network.py#L1352-L1357). - """ - try: - import re - import torch - import numpy - from tensorflow.python.keras import backend as K - except ImportError: - logger.error("Loading a PyTorch model in TensorFlow, requires PyTorch to be installed. Please see " - "https://pytorch.org/ for installation instructions.") - raise - - pt_path = os.path.abspath(pytorch_checkpoint_path) - logger.info("Loading PyTorch weights from {}".format(pt_path)) - # Load pytorch model - state_dict = torch.load(pt_path, map_location='cpu') - +def load_xlnet_pt_weights_in_tf2(tf_model, pytorch_checkpoint_path): inputs_list = [[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]] tf_inputs = tf.constant(inputs_list) tfo = tf_model(tf_inputs, training=False) # build the network - - symbolic_weights = tf_model.trainable_weights + tf_model.non_trainable_weights - weight_value_tuples = [] - all_pytorch_weights = set(list(state_dict.keys())) - for symbolic_weight in symbolic_weights: - name = symbolic_weight.name - name = name.replace(':0', '') - name = name.replace('__', '/') - name = name.split('/') - name = name[1:] - - transpose = bool(name[-1] == 'kernel') - if name[-1] == 'kernel' or name[-1] == 'embeddings' or name[-1] == 'gamma': - name[-1] = 'weight' - if name[-1] == 'beta': - name[-1] = 'bias' - - name = '.'.join(name) - assert name in state_dict, "{} not found in PyTorch model".format(name) - array = state_dict[name].numpy() - - if transpose: - array = numpy.transpose(array) - - try: - assert list(symbolic_weight.shape) == list(array.shape) - except AssertionError as e: - e.args += (symbolic_weight.shape, array.shape) - raise e - - logger.info("Initialize TF weight {}".format(symbolic_weight.name)) - - weight_value_tuples.append((symbolic_weight, array)) - all_pytorch_weights.discard(name) - - K.batch_set_value(weight_value_tuples) - - tfo = tf_model(tf_inputs, training=False) # Make sure restore ops are run - - logger.info("Weights or buffers not loaded from PyTorch model: {}".format(all_pytorch_weights)) - - return tf_model + return load_pytorch_checkpoint_in_tf2_model(tf_model, pytorch_checkpoint_path) def gelu(x): @@ -430,7 +371,7 @@ class TFXLNetMainLayer(tf.keras.layers.Layer): self.initializer_range = config.initializer_range self.word_embedding = TFSharedEmbeddings(config.n_token, config.d_model, initializer_range=config.initializer_range, name='word_embedding') - self.layer = [TFXLNetLayer(config, name='layer__{}'.format(i)) for i in range(config.n_layer)] + self.layer = [TFXLNetLayer(config, name='layer_._{}'.format(i)) for i in range(config.n_layer)] self.dropout = tf.keras.layers.Dropout(config.dropout) def build(self, input_shape): From 969d3ae95e26dd7ca10c41f3fef26b033d6c94c5 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Wed, 11 Sep 2019 15:47:33 +0200 Subject: [PATCH 045/219] XLMWithLMHead fixed - standardize conversion --- pytorch_transformers/modeling_tf_bert.py | 2 +- pytorch_transformers/modeling_tf_gpt2.py | 2 +- .../modeling_tf_pytorch_utils.py | 21 ++++++++++--------- pytorch_transformers/modeling_tf_xlm.py | 10 ++++----- pytorch_transformers/modeling_tf_xlnet.py | 2 +- pytorch_transformers/modeling_xlm.py | 4 ++-- 6 files changed, 20 insertions(+), 21 deletions(-) diff --git a/pytorch_transformers/modeling_tf_bert.py b/pytorch_transformers/modeling_tf_bert.py index acbdec86fd..6500d3a3e8 100644 --- a/pytorch_transformers/modeling_tf_bert.py +++ b/pytorch_transformers/modeling_tf_bert.py @@ -57,7 +57,7 @@ def load_bert_pt_weights_in_tf2(tf_model, pytorch_checkpoint_path): inputs_list = [[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]] tf_inputs = tf.constant(inputs_list) tfo = tf_model(tf_inputs, training=False) - return load_pytorch_checkpoint_in_tf2_model(tf_model, pytorch_checkpoint_path) + return load_pytorch_checkpoint_in_tf2_model(tf_model, pytorch_checkpoint_path, tf_inputs=tf_inputs) def gelu(x): diff --git a/pytorch_transformers/modeling_tf_gpt2.py b/pytorch_transformers/modeling_tf_gpt2.py index 6be0190880..8bb5eb52f9 100644 --- a/pytorch_transformers/modeling_tf_gpt2.py +++ b/pytorch_transformers/modeling_tf_gpt2.py @@ -46,7 +46,7 @@ def load_gpt2_pt_weights_in_tf2(tf_model, pytorch_checkpoint_path): inputs_list = [[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]] tf_inputs = tf.constant(inputs_list) tfo = tf_model(tf_inputs, training=False) - return load_pytorch_checkpoint_in_tf2_model(tf_model, pytorch_checkpoint_path) + return load_pytorch_checkpoint_in_tf2_model(tf_model, pytorch_checkpoint_path, tf_inputs=tf_inputs) def gelu(x): diff --git a/pytorch_transformers/modeling_tf_pytorch_utils.py b/pytorch_transformers/modeling_tf_pytorch_utils.py index 990319c7e4..6784596944 100644 --- a/pytorch_transformers/modeling_tf_pytorch_utils.py +++ b/pytorch_transformers/modeling_tf_pytorch_utils.py @@ -19,34 +19,34 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) import logging - -from pytorch_transformers import is_tf_available, is_torch_available +import os logger = logging.getLogger(__name__) -def load_pytorch_checkpoint_in_tf2_model(tf_model, pytorch_checkpoint_path): +def load_pytorch_checkpoint_in_tf2_model(tf_model, pytorch_checkpoint_path, tf_inputs=None): """ Load pytorch checkpoints in a TF 2.0 model Conventions for TF2.0 scopes -> PyTorch attribute names conversions: - '$1___$2' is replaced by $2 (can be used to duplicate or remove layers in TF2.0 vs PyTorch) - '_._' is replaced by a new level separation (can be used to convert TF2.0 lists in PyTorch nn.ModulesList) """ - if not is_tf_available() or not is_torch_available(): + try: + import tensorflow as tf + import torch + except ImportError as e: logger.error("Loading a PyTorch model in TensorFlow, requires both PyTorch and TensorFlow to be installed. Please see " "https://pytorch.org/ and https://www.tensorflow.org/install/ for installation instructions.") - raise ImportError - - import torch + raise e pt_path = os.path.abspath(pytorch_checkpoint_path) logger.info("Loading PyTorch weights from {}".format(pt_path)) pt_state_dict = torch.load(pt_path, map_location='cpu') - return load_pytorch_state_dict_in_tf2_model(tf_model, pt_state_dict) + return load_pytorch_state_dict_in_tf2_model(tf_model, pt_state_dict, tf_inputs=tf_inputs) -def load_pytorch_state_dict_in_tf2_model(tf_model, pt_state_dict): +def load_pytorch_state_dict_in_tf2_model(tf_model, pt_state_dict, tf_inputs=None): """ Load pytorch state_dict in a TF 2.0 model. Conventions for TF2.0 scopes -> PyTorch attribute names conversions: - '$1___$2' is replaced by $2 (can be used to duplicate or remove layers in TF2.0 vs PyTorch) @@ -102,7 +102,8 @@ def load_pytorch_state_dict_in_tf2_model(tf_model, pt_state_dict): K.batch_set_value(weight_value_tuples) - tfo = tf_model(tf_inputs, training=False) # Make sure restore ops are run + if tf_inputs is not None: + tfo = tf_model(tf_inputs, training=False) # Make sure restore ops are run logger.info("Weights or buffers not loaded from PyTorch model: {}".format(all_pytorch_weights)) diff --git a/pytorch_transformers/modeling_tf_xlm.py b/pytorch_transformers/modeling_tf_xlm.py index 1b4d99c14a..b0c1b594cf 100644 --- a/pytorch_transformers/modeling_tf_xlm.py +++ b/pytorch_transformers/modeling_tf_xlm.py @@ -50,11 +50,9 @@ def load_xlm_pt_weights_in_tf2(tf_model, pytorch_checkpoint_path): inputs_list = [[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]] attns_list = [[1, 1, 0, 0, 1], [1, 1, 1, 0, 0], [1, 0, 0, 1, 1]] langs_list = [[1, 1, 0, 0, 1], [1, 1, 1, 0, 0], [1, 0, 0, 1, 1]] - tf_inputs = tf.constant(inputs_list) - tf_attns = tf.constant(attns_list) - tf_langs = tf.constant(langs_list) - tfo = tf_model([tf_inputs, tf_attns, tf_langs], training=False) - return load_pytorch_checkpoint_in_tf2_model(tf_model, pytorch_checkpoint_path) + tf_inputs = [tf.constant(inputs_list), tf.constant(attns_list), tf.constant(langs_list)] + tfo = tf_model(tf_inputs, training=False) + return load_pytorch_checkpoint_in_tf2_model(tf_model, pytorch_checkpoint_path, tf_inputs=tf_inputs) def create_sinusoidal_embeddings(n_pos, dim, out): @@ -614,7 +612,7 @@ class TFXLMWithLMHeadModel(TFXLMPreTrainedModel): """ def __init__(self, config, *inputs, **kwargs): super(TFXLMWithLMHeadModel, self).__init__(config, *inputs, **kwargs) - self.transformer = TFXLMMainLayer(config, name='transformer___') + self.transformer = TFXLMMainLayer(config, name='transformer') self.pred_layer = TFXLMPredLayer(config, self.transformer.embeddings, name='pred_layer_._proj') diff --git a/pytorch_transformers/modeling_tf_xlnet.py b/pytorch_transformers/modeling_tf_xlnet.py index c0754e8302..1605564802 100644 --- a/pytorch_transformers/modeling_tf_xlnet.py +++ b/pytorch_transformers/modeling_tf_xlnet.py @@ -45,7 +45,7 @@ def load_xlnet_pt_weights_in_tf2(tf_model, pytorch_checkpoint_path): inputs_list = [[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]] tf_inputs = tf.constant(inputs_list) tfo = tf_model(tf_inputs, training=False) # build the network - return load_pytorch_checkpoint_in_tf2_model(tf_model, pytorch_checkpoint_path) + return load_pytorch_checkpoint_in_tf2_model(tf_model, pytorch_checkpoint_path, tf_inputs=tf_inputs) def gelu(x): diff --git a/pytorch_transformers/modeling_xlm.py b/pytorch_transformers/modeling_xlm.py index aeb1fcf689..6516bd2aa1 100644 --- a/pytorch_transformers/modeling_xlm.py +++ b/pytorch_transformers/modeling_xlm.py @@ -563,10 +563,10 @@ class XLMPredLayer(nn.Module): """ outputs = () if self.asm is False: - scores = self.proj(x).view(-1, self.n_words) + scores = self.proj(x) outputs = (scores,) + outputs if y is not None: - loss = F.cross_entropy(scores, y, reduction='elementwise_mean') + loss = F.cross_entropy(scores.view(-1, self.n_words), y, reduction='elementwise_mean') outputs = (loss,) + outputs else: scores = self.proj.log_prob(x) From a84adddd1b8c3db75855d86a86a709c0e021fdb3 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 12 Sep 2019 13:14:07 +0200 Subject: [PATCH 046/219] convert all models --- .../convert_pytorch_checkpoint_to_tf2.py | 113 +- .../modeling_tf_transfo_xl.py | 1108 +++++++++++++++++ 2 files changed, 1189 insertions(+), 32 deletions(-) create mode 100644 pytorch_transformers/modeling_tf_transfo_xl.py diff --git a/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py b/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py index b3389dcbf3..0a4119af78 100644 --- a/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py +++ b/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py @@ -18,10 +18,11 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import os import argparse import tensorflow as tf -from pytorch_transformers import is_torch_available +from pytorch_transformers import is_torch_available, cached_path from pytorch_transformers import (BertConfig, TFBertForPreTraining, load_bert_pt_weights_in_tf2, GPT2Config, TFGPT2LMHeadModel, load_gpt2_pt_weights_in_tf2, @@ -31,26 +32,36 @@ from pytorch_transformers import (BertConfig, TFBertForPreTraining, load_bert_pt if is_torch_available(): import torch import numpy as np - from pytorch_transformers import BertForPreTraining, GPT2LMHeadModel, XLNetLMHeadModel, XLMWithLMHeadModel + from pytorch_transformers import (BertForPreTraining, BERT_PRETRAINED_MODEL_ARCHIVE_MAP, BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, + GPT2LMHeadModel, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP, GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP, + XLNetLMHeadModel, XLNET_PRETRAINED_MODEL_ARCHIVE_MAP, XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP, + XLMWithLMHeadModel, XLM_PRETRAINED_MODEL_ARCHIVE_MAP, XLM_PRETRAINED_CONFIG_ARCHIVE_MAP,) else: - BertForPreTraining, GPT2LMHeadModel = None, None + (BertForPreTraining, BERT_PRETRAINED_MODEL_ARCHIVE_MAP, BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, + GPT2LMHeadModel, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP, GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP, + XLNetLMHeadModel, XLNET_PRETRAINED_MODEL_ARCHIVE_MAP, XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP, + XLMWithLMHeadModel, XLM_PRETRAINED_MODEL_ARCHIVE_MAP, XLM_PRETRAINED_CONFIG_ARCHIVE_MAP,) = ( + None, None, None, + None, None, None, + None, None, None, + None, None, None,) import logging logging.basicConfig(level=logging.INFO) MODEL_CLASSES = { - 'bert': (BertConfig, TFBertForPreTraining, load_bert_pt_weights_in_tf2, BertForPreTraining), - 'gpt2': (GPT2Config, TFGPT2LMHeadModel, load_gpt2_pt_weights_in_tf2, GPT2LMHeadModel), - 'xlnet': (XLNetConfig, TFXLNetLMHeadModel, load_xlnet_pt_weights_in_tf2, XLNetLMHeadModel), - 'xlm': (XLMConfig, TFXLMWithLMHeadModel, load_xlm_pt_weights_in_tf2, XLMWithLMHeadModel), + 'bert': (BertConfig, TFBertForPreTraining, load_bert_pt_weights_in_tf2, BertForPreTraining, BERT_PRETRAINED_MODEL_ARCHIVE_MAP, BERT_PRETRAINED_CONFIG_ARCHIVE_MAP), + 'gpt2': (GPT2Config, TFGPT2LMHeadModel, load_gpt2_pt_weights_in_tf2, GPT2LMHeadModel, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP, GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP), + 'xlnet': (XLNetConfig, TFXLNetLMHeadModel, load_xlnet_pt_weights_in_tf2, XLNetLMHeadModel, XLNET_PRETRAINED_MODEL_ARCHIVE_MAP, XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP), + 'xlm': (XLMConfig, TFXLMWithLMHeadModel, load_xlm_pt_weights_in_tf2, XLMWithLMHeadModel, XLM_PRETRAINED_MODEL_ARCHIVE_MAP, XLM_PRETRAINED_CONFIG_ARCHIVE_MAP), } def convert_pt_checkpoint_to_tf(model_type, pytorch_checkpoint_path, config_file, tf_dump_path, compare_with_pt_model=False): if model_type not in MODEL_CLASSES: raise ValueError("Unrecognized model type, should be one of {}.".format(list(MODEL_CLASSES.keys()))) - config_class, model_class, loading_fct, pt_model_class = MODEL_CLASSES[model_type] + config_class, model_class, loading_fct, pt_model_class, aws_model_maps, aws_config_map = MODEL_CLASSES[model_type] # Initialise TF model config = config_class.from_json_file(config_file) @@ -68,8 +79,8 @@ def convert_pt_checkpoint_to_tf(model_type, pytorch_checkpoint_path, config_file tfo = tf_model(tf_inputs, training=False) # build the network pt_model = pt_model_class.from_pretrained(None, - config=config, - state_dict=torch.load(pytorch_checkpoint_path, + config=config, + state_dict=torch.load(pytorch_checkpoint_path, map_location='cpu')) pt_inputs = torch.tensor(inputs_list) with torch.no_grad(): @@ -79,42 +90,80 @@ def convert_pt_checkpoint_to_tf(model_type, pytorch_checkpoint_path, config_file np_tf = tfo[0].numpy() diff = np.amax(np.abs(np_pt - np_tf)) print("Max absolute difference between models outputs {}".format(diff)) + assert diff <= 1e-3, "Error, model absolute difference is >1e-3" # Save pytorch-model print("Save TensorFlow model to {}".format(tf_dump_path)) - tf_model.save_weights(tf_dump_path) + tf_model.save_weights(tf_dump_path, save_format='h5') + + +def convert_all_pt_checkpoints_to_tf(args_model_type, tf_dump_path, compare_with_pt_model=False): + assert os.path.isdir(args.tf_dump_path), "--tf_dump_path should be a directory" + + if args_model_type is None: + model_types = list(MODEL_CLASSES.keys()) + else: + model_types = [args_model_type] + + for j, model_type in enumerate(model_types, start=1): + print("=" * 100) + print(" Converting model type {}/{}: {}".format(j, len(model_types), model_type)) + print("=" * 100) + if model_type not in MODEL_CLASSES: + raise ValueError("Unrecognized model type {}, should be one of {}.".format(model_type, list(MODEL_CLASSES.keys()))) + + config_class, model_class, loading_fct, pt_model_class, aws_model_maps, aws_config_map = MODEL_CLASSES[model_type] + + for i, shortcut_name in enumerate(aws_config_map.keys(), start=1): + print("-" * 100) + print(" Converting checkpoint {}/{}: {}".format(i, len(aws_config_map), shortcut_name)) + print("-" * 100) + config_file = cached_path(aws_config_map[shortcut_name], force_download=True) + model_file = cached_path(aws_model_maps[shortcut_name], force_download=True) + + convert_pt_checkpoint_to_tf(model_type, + model_file, + config_file, + os.path.join(tf_dump_path, shortcut_name + '-tf_model.h5'), + compare_with_pt_model=compare_with_pt_model) if __name__ == "__main__": parser = argparse.ArgumentParser() ## Required parameters - parser.add_argument("--model_type", - default = None, - type = str, - required = True, - help = "Model type selcted in the list of {}.".format(list(MODEL_CLASSES.keys()))) - parser.add_argument("--pytorch_checkpoint_path", - default = None, - type = str, - required = True, - help = "Path to the PyTorch checkpoint path.") - parser.add_argument("--config_file", - default = None, - type = str, - required = True, - help = "The config json file corresponding to the pre-trained model. \n" - "This specifies the model architecture.") parser.add_argument("--tf_dump_path", default = None, type = str, required = True, help = "Path to the output Tensorflow dump file.") + parser.add_argument("--model_type", + default = None, + type = str, + help = "Model type selected in the list of {}. If not given, will download and convert all the models from AWS.".format(list(MODEL_CLASSES.keys()))) + parser.add_argument("--pytorch_checkpoint_path", + default = None, + type = str, + help = "Path to the PyTorch checkpoint path or shortcut name to download from AWS. " + "If not given, will download and convert all the checkpoints from AWS.") + parser.add_argument("--config_file", + default = None, + type = str, + help = "The config json file corresponding to the pre-trained model. \n" + "This specifies the model architecture. If not given and " + "--pytorch_checkpoint_path is not given or is a shortcut name" + "use the configuration associated to teh shortcut name on the AWS") parser.add_argument("--compare_with_pt_model", action='store_true', help = "Compare Tensorflow and PyTorch model predictions.") args = parser.parse_args() - convert_pt_checkpoint_to_tf(args.model_type.lower(), - args.pytorch_checkpoint_path, - args.config_file, - args.tf_dump_path, - compare_with_pt_model=args.compare_with_pt_model) + + if args.pytorch_checkpoint_path is not None: + convert_pt_checkpoint_to_tf(args.model_type.lower(), + args.pytorch_checkpoint_path, + args.config_file, + args.tf_dump_path, + compare_with_pt_model=args.compare_with_pt_model) + else: + convert_all_pt_checkpoints_to_tf(args.model_type.lower() if args.model_type is not None else None, + args.tf_dump_path, + compare_with_pt_model=args.compare_with_pt_model) diff --git a/pytorch_transformers/modeling_tf_transfo_xl.py b/pytorch_transformers/modeling_tf_transfo_xl.py new file mode 100644 index 0000000000..6aa413811e --- /dev/null +++ b/pytorch_transformers/modeling_tf_transfo_xl.py @@ -0,0 +1,1108 @@ +# coding=utf-8 +# Copyright 2018 Google AI, Google Brain and Carnegie Mellon University Authors and the HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" TF 2.0 Transformer XL model. +""" + +from __future__ import absolute_import, division, print_function, unicode_literals + +import os +import json +import math +import logging +import collections +import sys +from io import open + +import numpy as np +import tensorflow as tf + +from .configuration_transfo_xl import TransfoXLConfig +from .modeling_tf_utils import TFPreTrainedModel, TFConv1D, TFSequenceSummary +from .modeling_transfo_xl_utilities import ProjectedAdaptiveLogSoftmax, sample_logits +from .file_utils import add_start_docstrings +from .modeling_tf_pytorch_utils import load_pytorch_checkpoint_in_tf2_model + +logger = logging.getLogger(__name__) + +TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP = { + 'transfo-xl-wt103': "https://s3.amazonaws.com/models.huggingface.co/bert/transfo-xl-wt103-tf_model.h5", +} + +def load_transfo_xl_pt_weights_in_tf2(tf_model, pytorch_checkpoint_path): + # build the network + inputs_list = [[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]] + tf_inputs = tf.constant(inputs_list) + tfo = tf_model(tf_inputs, training=False) + return load_pytorch_checkpoint_in_tf2_model(tf_model, pytorch_checkpoint_path, tf_inputs=tf_inputs) + + +class PositionalEmbedding(nn.Module): + def __init__(self, demb): + super(PositionalEmbedding, self).__init__() + + self.demb = demb + + inv_freq = 1 / (10000 ** (torch.arange(0.0, demb, 2.0) / demb)) + self.register_buffer('inv_freq', inv_freq) + + def forward(self, pos_seq, bsz=None): + sinusoid_inp = torch.ger(pos_seq, self.inv_freq) + pos_emb = torch.cat([sinusoid_inp.sin(), sinusoid_inp.cos()], dim=-1) + + if bsz is not None: + return pos_emb[:,None,:].expand(-1, bsz, -1) + else: + return pos_emb[:,None,:] + + + +class PositionwiseFF(nn.Module): + def __init__(self, d_model, d_inner, dropout, pre_lnorm=False): + super(PositionwiseFF, self).__init__() + + self.d_model = d_model + self.d_inner = d_inner + self.dropout = dropout + + self.CoreNet = nn.Sequential( + nn.Linear(d_model, d_inner), nn.ReLU(inplace=True), + nn.Dropout(dropout), + nn.Linear(d_inner, d_model), + nn.Dropout(dropout), + ) + + self.layer_norm = nn.LayerNorm(d_model) + + self.pre_lnorm = pre_lnorm + + def forward(self, inp): + if self.pre_lnorm: + ##### layer normalization + positionwise feed-forward + core_out = self.CoreNet(self.layer_norm(inp)) + + ##### residual connection + output = core_out + inp + else: + ##### positionwise feed-forward + core_out = self.CoreNet(inp) + + ##### residual connection + layer normalization + output = self.layer_norm(inp + core_out) + + return output + + + +class MultiHeadAttn(nn.Module): + def __init__(self, n_head, d_model, d_head, dropout, dropatt=0, + pre_lnorm=False, r_r_bias=None, r_w_bias=None, output_attentions=False): + super(MultiHeadAttn, self).__init__() + + self.output_attentions = output_attentions + self.n_head = n_head + self.d_model = d_model + self.d_head = d_head + self.dropout = dropout + + self.q_net = nn.Linear(d_model, n_head * d_head, bias=False) + self.kv_net = nn.Linear(d_model, 2 * n_head * d_head, bias=False) + + self.drop = nn.Dropout(dropout) + self.dropatt = nn.Dropout(dropatt) + self.o_net = nn.Linear(n_head * d_head, d_model, bias=False) + + self.layer_norm = nn.LayerNorm(d_model) + + self.scale = 1 / (d_head ** 0.5) + + self.pre_lnorm = pre_lnorm + + if r_r_bias is None or r_w_bias is None: # Biases are not shared + self.r_r_bias = nn.Parameter(torch.FloatTensor(self.n_head, self.d_head)) + self.r_w_bias = nn.Parameter(torch.FloatTensor(self.n_head, self.d_head)) + else: + self.r_r_bias = r_r_bias + self.r_w_bias = r_w_bias + + def forward(self, h, attn_mask=None, mems=None, head_mask=None): + ##### multihead attention + # [hlen x bsz x n_head x d_head] + + if mems is not None: + c = torch.cat([mems, h], 0) + else: + c = h + + if self.pre_lnorm: + ##### layer normalization + c = self.layer_norm(c) + + head_q = self.q_net(h) + head_k, head_v = torch.chunk(self.kv_net(c), 2, -1) + + head_q = head_q.view(h.size(0), h.size(1), self.n_head, self.d_head) + head_k = head_k.view(c.size(0), c.size(1), self.n_head, self.d_head) + head_v = head_v.view(c.size(0), c.size(1), self.n_head, self.d_head) + + # [qlen x klen x bsz x n_head] + attn_score = torch.einsum('ibnd,jbnd->ijbn', (head_q, head_k)) + attn_score.mul_(self.scale) + if attn_mask is not None and torch.sum(attn_mask).item(): + attn_mask = (attn_mask == 1) # Switch to bool + if attn_mask.dim() == 2: + attn_score.masked_fill_(attn_mask[None,:,:,None], -float('inf')) + elif attn_mask.dim() == 3: + attn_score.masked_fill_(attn_mask[:,:,:,None], -float('inf')) + + # [qlen x klen x bsz x n_head] + attn_prob = F.softmax(attn_score, dim=1) + attn_prob = self.dropatt(attn_prob) + + # Mask heads if we want to + if head_mask is not None: + attn_prob = attn_prob * head_mask + + # [qlen x klen x bsz x n_head] + [klen x bsz x n_head x d_head] -> [qlen x bsz x n_head x d_head] + attn_vec = torch.einsum('ijbn,jbnd->ibnd', (attn_prob, head_v)) + attn_vec = attn_vec.contiguous().view( + attn_vec.size(0), attn_vec.size(1), self.n_head * self.d_head) + + ##### linear projection + attn_out = self.o_net(attn_vec) + attn_out = self.drop(attn_out) + + if self.pre_lnorm: + ##### residual connection + outputs = [h + attn_out] + else: + ##### residual connection + layer normalization + outputs = [self.layer_norm(h + attn_out)] + + if self.output_attentions: + outputs.append(attn_prob) + + return outputs + +class RelMultiHeadAttn(nn.Module): + def __init__(self, n_head, d_model, d_head, dropout, dropatt=0, + tgt_len=None, ext_len=None, mem_len=None, pre_lnorm=False, + r_r_bias=None, r_w_bias=None, output_attentions=False): + super(RelMultiHeadAttn, self).__init__() + + self.output_attentions = output_attentions + self.n_head = n_head + self.d_model = d_model + self.d_head = d_head + self.dropout = dropout + + self.qkv_net = nn.Linear(d_model, 3 * n_head * d_head, bias=False) + + self.drop = nn.Dropout(dropout) + self.dropatt = nn.Dropout(dropatt) + self.o_net = nn.Linear(n_head * d_head, d_model, bias=False) + + self.layer_norm = nn.LayerNorm(d_model) + + self.scale = 1 / (d_head ** 0.5) + + self.pre_lnorm = pre_lnorm + + if r_r_bias is None or r_w_bias is None: # Biases are not shared + self.r_r_bias = nn.Parameter(torch.FloatTensor(self.n_head, self.d_head)) + self.r_w_bias = nn.Parameter(torch.FloatTensor(self.n_head, self.d_head)) + else: + self.r_r_bias = r_r_bias + self.r_w_bias = r_w_bias + + def _parallelogram_mask(self, h, w, left=False): + mask = torch.ones((h, w)).byte() + m = min(h, w) + mask[:m,:m] = torch.triu(mask[:m,:m]) + mask[-m:,-m:] = torch.tril(mask[-m:,-m:]) + + if left: + return mask + else: + return mask.flip(0) + + def _shift(self, x, qlen, klen, mask, left=False): + if qlen > 1: + zero_pad = torch.zeros((x.size(0), qlen-1, x.size(2), x.size(3)), + device=x.device, dtype=x.dtype) + else: + zero_pad = torch.zeros(0, device=x.device, dtype=x.dtype) + + if left: + mask = mask.flip(1) + x_padded = torch.cat([zero_pad, x], dim=1).expand(qlen, -1, -1, -1) + else: + x_padded = torch.cat([x, zero_pad], dim=1).expand(qlen, -1, -1, -1) + + x = x_padded.masked_select(mask[:,:,None,None]) \ + .view(qlen, klen, x.size(2), x.size(3)) + + return x + + def _rel_shift(self, x, zero_triu=False): + zero_pad_shape = (x.size(0), 1) + x.size()[2:] + zero_pad = torch.zeros(zero_pad_shape, device=x.device, dtype=x.dtype) + x_padded = torch.cat([zero_pad, x], dim=1) + + x_padded_shape = (x.size(1) + 1, x.size(0)) + x.size()[2:] + x_padded = x_padded.view(*x_padded_shape) + + x = x_padded[1:].view_as(x) + + if zero_triu: + ones = torch.ones((x.size(0), x.size(1))) + x = x * torch.tril(ones, x.size(1) - x.size(0))[:,:,None,None] + + return x + + def forward(self, w, r, attn_mask=None, mems=None): + raise NotImplementedError + +class RelPartialLearnableMultiHeadAttn(RelMultiHeadAttn): + def __init__(self, *args, **kwargs): + super(RelPartialLearnableMultiHeadAttn, self).__init__(*args, **kwargs) + + self.r_net = nn.Linear(self.d_model, self.n_head * self.d_head, bias=False) + + def forward(self, w, r, attn_mask=None, mems=None, head_mask=None): + qlen, rlen, bsz = w.size(0), r.size(0), w.size(1) + + if mems is not None: + cat = torch.cat([mems, w], 0) + if self.pre_lnorm: + w_heads = self.qkv_net(self.layer_norm(cat)) + else: + w_heads = self.qkv_net(cat) + r_head_k = self.r_net(r) + + w_head_q, w_head_k, w_head_v = torch.chunk(w_heads, 3, dim=-1) + w_head_q = w_head_q[-qlen:] + else: + if self.pre_lnorm: + w_heads = self.qkv_net(self.layer_norm(w)) + else: + w_heads = self.qkv_net(w) + r_head_k = self.r_net(r) + + w_head_q, w_head_k, w_head_v = torch.chunk(w_heads, 3, dim=-1) + + klen = w_head_k.size(0) + + w_head_q = w_head_q.view(qlen, bsz, self.n_head, self.d_head) # qlen x bsz x n_head x d_head + w_head_k = w_head_k.view(klen, bsz, self.n_head, self.d_head) # qlen x bsz x n_head x d_head + w_head_v = w_head_v.view(klen, bsz, self.n_head, self.d_head) # qlen x bsz x n_head x d_head + + r_head_k = r_head_k.view(rlen, self.n_head, self.d_head) # qlen x n_head x d_head + + #### compute attention score + rw_head_q = w_head_q + self.r_w_bias # qlen x bsz x n_head x d_head + AC = torch.einsum('ibnd,jbnd->ijbn', (rw_head_q, w_head_k)) # qlen x klen x bsz x n_head + + rr_head_q = w_head_q + self.r_r_bias + BD = torch.einsum('ibnd,jnd->ijbn', (rr_head_q, r_head_k)) # qlen x klen x bsz x n_head + BD = self._rel_shift(BD) + + # [qlen x klen x bsz x n_head] + attn_score = AC + BD + attn_score.mul_(self.scale) + + #### compute attention probability + if attn_mask is not None and torch.sum(attn_mask).item(): + attn_mask = (attn_mask == 1) # Switch to bool + if attn_mask.dim() == 2: + attn_score = attn_score.float().masked_fill( + attn_mask[None,:,:,None], -1e30).type_as(attn_score) + elif attn_mask.dim() == 3: + attn_score = attn_score.float().masked_fill( + attn_mask[:,:,:,None], -1e30).type_as(attn_score) + + # [qlen x klen x bsz x n_head] + attn_prob = F.softmax(attn_score, dim=1) + attn_prob = self.dropatt(attn_prob) + + # Mask heads if we want to + if head_mask is not None: + attn_prob = attn_prob * head_mask + + #### compute attention vector + attn_vec = torch.einsum('ijbn,jbnd->ibnd', (attn_prob, w_head_v)) + + # [qlen x bsz x n_head x d_head] + attn_vec = attn_vec.contiguous().view( + attn_vec.size(0), attn_vec.size(1), self.n_head * self.d_head) + + ##### linear projection + attn_out = self.o_net(attn_vec) + attn_out = self.drop(attn_out) + + if self.pre_lnorm: + ##### residual connection + outputs = [w + attn_out] + else: + ##### residual connection + layer normalization + outputs = [self.layer_norm(w + attn_out)] + + if self.output_attentions: + outputs.append(attn_prob) + + return outputs + +class RelLearnableMultiHeadAttn(RelMultiHeadAttn): + def __init__(self, *args, **kwargs): + super(RelLearnableMultiHeadAttn, self).__init__(*args, **kwargs) + + def forward(self, w, r_emb, r_w_bias, r_bias, attn_mask=None, mems=None, head_mask=None): + # r_emb: [klen, n_head, d_head], used for term B + # r_w_bias: [n_head, d_head], used for term C + # r_bias: [klen, n_head], used for term D + + qlen, bsz = w.size(0), w.size(1) + + if mems is not None: + cat = torch.cat([mems, w], 0) + if self.pre_lnorm: + w_heads = self.qkv_net(self.layer_norm(cat)) + else: + w_heads = self.qkv_net(cat) + w_head_q, w_head_k, w_head_v = torch.chunk(w_heads, 3, dim=-1) + + w_head_q = w_head_q[-qlen:] + else: + if self.pre_lnorm: + w_heads = self.qkv_net(self.layer_norm(w)) + else: + w_heads = self.qkv_net(w) + w_head_q, w_head_k, w_head_v = torch.chunk(w_heads, 3, dim=-1) + + klen = w_head_k.size(0) + + w_head_q = w_head_q.view(qlen, bsz, self.n_head, self.d_head) + w_head_k = w_head_k.view(klen, bsz, self.n_head, self.d_head) + w_head_v = w_head_v.view(klen, bsz, self.n_head, self.d_head) + + if klen > r_emb.size(0): + r_emb_pad = r_emb[0:1].expand(klen-r_emb.size(0), -1, -1) + r_emb = torch.cat([r_emb_pad, r_emb], 0) + r_bias_pad = r_bias[0:1].expand(klen-r_bias.size(0), -1) + r_bias = torch.cat([r_bias_pad, r_bias], 0) + else: + r_emb = r_emb[-klen:] + r_bias = r_bias[-klen:] + + #### compute attention score + rw_head_q = w_head_q + r_w_bias[None] # qlen x bsz x n_head x d_head + + AC = torch.einsum('ibnd,jbnd->ijbn', (rw_head_q, w_head_k)) # qlen x klen x bsz x n_head + B_ = torch.einsum('ibnd,jnd->ijbn', (w_head_q, r_emb)) # qlen x klen x bsz x n_head + D_ = r_bias[None, :, None] # 1 x klen x 1 x n_head + BD = self._rel_shift(B_ + D_) + + # [qlen x klen x bsz x n_head] + attn_score = AC + BD + attn_score.mul_(self.scale) + + #### compute attention probability + if attn_mask is not None and torch.sum(attn_mask).item(): + attn_mask = (attn_mask == 1) # Switch to bool + if attn_mask.dim() == 2: + attn_score.masked_fill_(attn_mask[None,:,:,None], -float('inf')) + elif attn_mask.dim() == 3: + attn_score.masked_fill_(attn_mask[:,:,:,None], -float('inf')) + + # [qlen x klen x bsz x n_head] + attn_prob = F.softmax(attn_score, dim=1) + attn_prob = self.dropatt(attn_prob) + + if head_mask is not None: + attn_prob = attn_prob * head_mask + + #### compute attention vector + attn_vec = torch.einsum('ijbn,jbnd->ibnd', (attn_prob, w_head_v)) + + # [qlen x bsz x n_head x d_head] + attn_vec = attn_vec.contiguous().view( + attn_vec.size(0), attn_vec.size(1), self.n_head * self.d_head) + + ##### linear projection + attn_out = self.o_net(attn_vec) + attn_out = self.drop(attn_out) + + if self.pre_lnorm: + ##### residual connection + outputs = [w + attn_out] + else: + ##### residual connection + layer normalization + outputs = [self.layer_norm(w + attn_out)] + + if self.output_attentions: + outputs.append(attn_prob) + + return outputs + + + +class DecoderLayer(nn.Module): + def __init__(self, n_head, d_model, d_head, d_inner, dropout, **kwargs): + super(DecoderLayer, self).__init__() + + self.dec_attn = MultiHeadAttn(n_head, d_model, d_head, dropout, **kwargs) + self.pos_ff = PositionwiseFF(d_model, d_inner, dropout, + pre_lnorm=kwargs.get('pre_lnorm')) + + def forward(self, dec_inp, dec_attn_mask=None, mems=None, head_mask=None): + + attn_outputs = self.dec_attn(dec_inp, attn_mask=dec_attn_mask, + mems=mems, head_mask=head_mask) + ff_output = self.pos_ff(attn_outputs[0]) + + outputs = [ff_output] + attn_outputs[1:] + + return outputs + +class RelLearnableDecoderLayer(nn.Module): + def __init__(self, n_head, d_model, d_head, d_inner, dropout, + **kwargs): + super(RelLearnableDecoderLayer, self).__init__() + + self.dec_attn = RelLearnableMultiHeadAttn(n_head, d_model, d_head, dropout, + **kwargs) + self.pos_ff = PositionwiseFF(d_model, d_inner, dropout, + pre_lnorm=kwargs.get('pre_lnorm')) + + def forward(self, dec_inp, r_emb, r_w_bias, r_bias, dec_attn_mask=None, mems=None, head_mask=None): + + attn_outputs = self.dec_attn(dec_inp, r_emb, r_w_bias, r_bias, + attn_mask=dec_attn_mask, + mems=mems, head_mask=head_mask) + ff_output = self.pos_ff(attn_outputs[0]) + + outputs = [ff_output] + attn_outputs[1:] + + return outputs + +class RelPartialLearnableDecoderLayer(nn.Module): + def __init__(self, n_head, d_model, d_head, d_inner, dropout, + **kwargs): + super(RelPartialLearnableDecoderLayer, self).__init__() + + self.dec_attn = RelPartialLearnableMultiHeadAttn(n_head, d_model, + d_head, dropout, **kwargs) + self.pos_ff = PositionwiseFF(d_model, d_inner, dropout, + pre_lnorm=kwargs.get('pre_lnorm')) + + def forward(self, dec_inp, r, dec_attn_mask=None, mems=None, head_mask=None): + + attn_outputs = self.dec_attn(dec_inp, r, + attn_mask=dec_attn_mask, + mems=mems, head_mask=head_mask) + ff_output = self.pos_ff(attn_outputs[0]) + + outputs = [ff_output] + attn_outputs[1:] + + return outputs + + + +class AdaptiveEmbedding(nn.Module): + def __init__(self, n_token, d_embed, d_proj, cutoffs, div_val=1, + sample_softmax=False): + super(AdaptiveEmbedding, self).__init__() + + self.n_token = n_token + self.d_embed = d_embed + + self.cutoffs = cutoffs + [n_token] + self.div_val = div_val + self.d_proj = d_proj + + self.emb_scale = d_proj ** 0.5 + + self.cutoff_ends = [0] + self.cutoffs + + self.emb_layers = nn.ModuleList() + self.emb_projs = nn.ParameterList() + if div_val == 1: + self.emb_layers.append( + nn.Embedding(n_token, d_embed, sparse=sample_softmax>0) + ) + if d_proj != d_embed: + self.emb_projs.append(nn.Parameter(torch.FloatTensor(d_proj, d_embed))) + else: + for i in range(len(self.cutoffs)): + l_idx, r_idx = self.cutoff_ends[i], self.cutoff_ends[i+1] + d_emb_i = d_embed // (div_val ** i) + self.emb_layers.append(nn.Embedding(r_idx-l_idx, d_emb_i)) + self.emb_projs.append(nn.Parameter(torch.FloatTensor(d_proj, d_emb_i))) + + def forward(self, inp): + if self.div_val == 1: + embed = self.emb_layers[0](inp) + if self.d_proj != self.d_embed: + embed = F.linear(embed, self.emb_projs[0]) + else: + param = next(self.parameters()) + inp_flat = inp.view(-1) + emb_flat = torch.zeros([inp_flat.size(0), self.d_proj], + dtype=param.dtype, device=param.device) + for i in range(len(self.cutoffs)): + l_idx, r_idx = self.cutoff_ends[i], self.cutoff_ends[i + 1] + + mask_i = (inp_flat >= l_idx) & (inp_flat < r_idx) + indices_i = mask_i.nonzero().squeeze() + + if indices_i.numel() == 0: + continue + + inp_i = inp_flat.index_select(0, indices_i) - l_idx + emb_i = self.emb_layers[i](inp_i) + emb_i = F.linear(emb_i, self.emb_projs[i]) + + emb_flat.index_copy_(0, indices_i, emb_i) + + embed_shape = inp.size() + (self.d_proj,) + embed = emb_flat.view(embed_shape) + + embed.mul_(self.emb_scale) + + return embed + + +class TransfoXLPreTrainedModel(PreTrainedModel): + """ An abstract class to handle weights initialization and + a simple interface for dowloading and loading pretrained models. + """ + config_class = TransfoXLConfig + pretrained_model_archive_map = TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP + load_tf_weights = load_tf_weights_in_transfo_xl + base_model_prefix = "transformer" + + def _init_weight(self, weight): + if self.config.init == 'uniform': + nn.init.uniform_(weight, -self.config.init_range, self.config.init_range) + elif self.config.init == 'normal': + nn.init.normal_(weight, 0.0, self.config.init_std) + + def _init_bias(self, bias): + nn.init.constant_(bias, 0.0) + + def _init_weights(self, m): + """ Initialize the weights. + """ + classname = m.__class__.__name__ + if classname.find('Linear') != -1: + if hasattr(m, 'weight') and m.weight is not None: + self._init_weight(m.weight) + if hasattr(m, 'bias') and m.bias is not None: + self._init_bias(m.bias) + elif classname.find('AdaptiveEmbedding') != -1: + if hasattr(m, 'emb_projs'): + for i in range(len(m.emb_projs)): + if m.emb_projs[i] is not None: + nn.init.normal_(m.emb_projs[i], 0.0, self.config.proj_init_std) + elif classname.find('Embedding') != -1: + if hasattr(m, 'weight'): + self._init_weight(m.weight) + elif classname.find('ProjectedAdaptiveLogSoftmax') != -1: + if hasattr(m, 'cluster_weight') and m.cluster_weight is not None: + self._init_weight(m.cluster_weight) + if hasattr(m, 'cluster_bias') and m.cluster_bias is not None: + self._init_bias(m.cluster_bias) + if hasattr(m, 'out_projs'): + for i in range(len(m.out_projs)): + if m.out_projs[i] is not None: + nn.init.normal_(m.out_projs[i], 0.0, self.config.proj_init_std) + elif classname.find('LayerNorm') != -1: + if hasattr(m, 'weight'): + nn.init.normal_(m.weight, 1.0, self.config.init_std) + if hasattr(m, 'bias') and m.bias is not None: + self._init_bias(m.bias) + else: + if hasattr(m, 'r_emb'): + self._init_weight(m.r_emb) + if hasattr(m, 'r_w_bias'): + self._init_weight(m.r_w_bias) + if hasattr(m, 'r_r_bias'): + self._init_weight(m.r_r_bias) + if hasattr(m, 'r_bias'): + self._init_bias(m.r_bias) + + def set_num_special_tokens(self, num_special_tokens): + pass + + +TRANSFO_XL_START_DOCSTRING = r""" The Transformer-XL model was proposed in + `Transformer-XL: Attentive Language Models Beyond a Fixed-Length Context`_ + by Zihang Dai*, Zhilin Yang*, Yiming Yang, Jaime Carbonell, Quoc V. Le, Ruslan Salakhutdinov. + It's a causal (uni-directional) transformer with relative positioning (sinusoïdal) embeddings which can reuse + previously computed hidden-states to attend to longer context (memory). + This model also uses adaptive softmax inputs and outputs (tied). + + This model is a PyTorch `torch.nn.Module`_ sub-class. Use it as a regular PyTorch Module and + refer to the PyTorch documentation for all matter related to general usage and behavior. + + .. _`Transformer-XL: Attentive Language Models Beyond a Fixed-Length Context`: + https://arxiv.org/abs/1901.02860 + + .. _`torch.nn.Module`: + https://pytorch.org/docs/stable/nn.html#module + + Parameters: + config (:class:`~pytorch_transformers.TransfoXLConfig`): Model configuration class with all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the configuration. + Check out the :meth:`~pytorch_transformers.PreTrainedModel.from_pretrained` method to load the model weights. +""" + +TRANSFO_XL_INPUTS_DOCSTRING = r""" + Inputs: + **input_ids**: ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Indices of input sequence tokens in the vocabulary. + Transformer-XL is a model with relative position embeddings so you can either pad the inputs on + the right or on the left. + Indices can be obtained using :class:`pytorch_transformers.TransfoXLTokenizer`. + See :func:`pytorch_transformers.PreTrainedTokenizer.encode` and + :func:`pytorch_transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + **mems**: (`optional`) + list of ``torch.FloatTensor`` (one for each layer): + that contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model + (see `mems` output below). Can be used to speed up sequential decoding and attend to longer context. + **head_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: + Mask to nullify selected heads of the self-attention modules. + Mask values selected in ``[0, 1]``: + ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. +""" + +@add_start_docstrings("The bare Bert Model transformer outputing raw hidden-states without any specific head on top.", + TRANSFO_XL_START_DOCSTRING, TRANSFO_XL_INPUTS_DOCSTRING) +class TransfoXLModel(TransfoXLPreTrainedModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)`` + Sequence of hidden-states at the last layer of the model. + **mems**: + list of ``torch.FloatTensor`` (one for each layer): + that contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model + (see `mems` input above). Can be used to speed up sequential decoding and attend to longer context. + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = TransfoXLTokenizer.from_pretrained('transfo-xl-wt103') + model = TransfoXLModel.from_pretrained('transfo-xl-wt103') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids) + last_hidden_states, mems = outputs[:2] + + """ + def __init__(self, config): + super(TransfoXLModel, self).__init__(config) + self.output_attentions = config.output_attentions + self.output_hidden_states = config.output_hidden_states + + self.n_token = config.n_token + + self.d_embed = config.d_embed + self.d_model = config.d_model + self.n_head = config.n_head + self.d_head = config.d_head + + self.word_emb = AdaptiveEmbedding(config.n_token, config.d_embed, config.d_model, config.cutoffs, + div_val=config.div_val) + + self.drop = nn.Dropout(config.dropout) + + self.n_layer = config.n_layer + + self.tgt_len = config.tgt_len + self.mem_len = config.mem_len + self.ext_len = config.ext_len + self.max_klen = config.tgt_len + config.ext_len + config.mem_len + + self.attn_type = config.attn_type + + if not config.untie_r: + self.r_w_bias = nn.Parameter(torch.FloatTensor(self.n_head, self.d_head)) + self.r_r_bias = nn.Parameter(torch.FloatTensor(self.n_head, self.d_head)) + + self.layers = nn.ModuleList() + if config.attn_type == 0: # the default attention + for i in range(config.n_layer): + self.layers.append( + RelPartialLearnableDecoderLayer( + config.n_head, config.d_model, config.d_head, config.d_inner, config.dropout, + tgt_len=config.tgt_len, ext_len=config.ext_len, mem_len=config.mem_len, + dropatt=config.dropatt, pre_lnorm=config.pre_lnorm, + r_w_bias=None if config.untie_r else self.r_w_bias, + r_r_bias=None if config.untie_r else self.r_r_bias, + output_attentions=self.output_attentions) + ) + elif config.attn_type == 1: # learnable embeddings + for i in range(config.n_layer): + self.layers.append( + RelLearnableDecoderLayer( + config.n_head, config.d_model, config.d_head, config.d_inner, config.dropout, + tgt_len=config.tgt_len, ext_len=config.ext_len, mem_len=config.mem_len, + dropatt=config.dropatt, pre_lnorm=config.pre_lnorm, + r_w_bias=None if config.untie_r else self.r_w_bias, + r_r_bias=None if config.untie_r else self.r_r_bias, + output_attentions=self.output_attentions) + ) + elif config.attn_type in [2, 3]: # absolute embeddings + for i in range(config.n_layer): + self.layers.append( + DecoderLayer( + config.n_head, config.d_model, config.d_head, config.d_inner, config.dropout, + dropatt=config.dropatt, pre_lnorm=config.pre_lnorm, + r_w_bias=None if config.untie_r else self.r_w_bias, + r_r_bias=None if config.untie_r else self.r_r_bias, + output_attentions=self.output_attentions) + ) + + self.same_length = config.same_length + self.clamp_len = config.clamp_len + + if self.attn_type == 0: # default attention + self.pos_emb = PositionalEmbedding(self.d_model) + elif self.attn_type == 1: # learnable + self.r_emb = nn.Parameter(torch.FloatTensor( + self.n_layer, self.max_klen, self.n_head, self.d_head)) + self.r_bias = nn.Parameter(torch.FloatTensor( + self.n_layer, self.max_klen, self.n_head)) + elif self.attn_type == 2: # absolute standard + self.pos_emb = PositionalEmbedding(self.d_model) + elif self.attn_type == 3: # absolute deeper SA + self.r_emb = nn.Parameter(torch.FloatTensor( + self.n_layer, self.max_klen, self.n_head, self.d_head)) + + self.init_weights() + + def _resize_token_embeddings(self, new_num_tokens): + return self.word_emb + + def backward_compatible(self): + self.sample_softmax = -1 + + def reset_length(self, tgt_len, ext_len, mem_len): + self.tgt_len = tgt_len + self.mem_len = mem_len + self.ext_len = ext_len + + def _prune_heads(self, heads): + logger.info("Head pruning is not implemented for Transformer-XL model") + pass + + def init_mems(self, data): + if self.mem_len > 0: + mems = [] + param = next(self.parameters()) + for i in range(self.n_layer): + empty = torch.zeros(self.mem_len, data.size(1), self.config.d_model, + dtype=param.dtype, device=param.device) + mems.append(empty) + + return mems + else: + return None + + def _update_mems(self, hids, mems, qlen, mlen): + # does not deal with None + if mems is None: return None + + # mems is not None + assert len(hids) == len(mems), 'len(hids) != len(mems)' + + # There are `mlen + qlen` steps that can be cached into mems + # For the next step, the last `ext_len` of the `qlen` tokens + # will be used as the extended context. Hence, we only cache + # the tokens from `mlen + qlen - self.ext_len - self.mem_len` + # to `mlen + qlen - self.ext_len`. + with torch.no_grad(): + new_mems = [] + end_idx = mlen + max(0, qlen - 0 - self.ext_len) + beg_idx = max(0, end_idx - self.mem_len) + for i in range(len(hids)): + + cat = torch.cat([mems[i], hids[i]], dim=0) + new_mems.append(cat[beg_idx:end_idx].detach()) + + return new_mems + + def _forward(self, dec_inp, mems=None, head_mask=None): + qlen, bsz = dec_inp.size() + + # Prepare head mask if needed + # 1.0 in head_mask indicate we keep the head + # attention_probs has shape bsz x n_heads x N x N + # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] (a head_mask for each layer) + # and head_mask is converted to shape [num_hidden_layers x qlen x klen x bsz x n_head] + if head_mask is not None: + if head_mask.dim() == 1: + head_mask = head_mask.unsqueeze(0).unsqueeze(0).unsqueeze(0).unsqueeze(0) + head_mask = head_mask.expand(self.n_layer, -1, -1, -1, -1) + elif head_mask.dim() == 2: + head_mask = head_mask.unsqueeze(1).unsqueeze(1).unsqueeze(1) + head_mask = head_mask.to(dtype=next(self.parameters()).dtype) # switch to fload if need + fp16 compatibility + else: + head_mask = [None] * self.n_layer + + word_emb = self.word_emb(dec_inp) + + mlen = mems[0].size(0) if mems is not None else 0 + klen = mlen + qlen + if self.same_length: + all_ones = word_emb.new_ones((qlen, klen), dtype=torch.uint8) + mask_len = klen - self.mem_len + if mask_len > 0: + mask_shift_len = qlen - mask_len + else: + mask_shift_len = qlen + dec_attn_mask = (torch.triu(all_ones, 1+mlen) + + torch.tril(all_ones, -mask_shift_len))[:, :, None] # -1 + else: + dec_attn_mask = torch.triu( + word_emb.new_ones((qlen, klen), dtype=torch.uint8), diagonal=1+mlen)[:,:,None] + + hids = [] + attentions = [] + if self.attn_type == 0: # default + pos_seq = torch.arange(klen-1, -1, -1.0, device=word_emb.device, + dtype=word_emb.dtype) + if self.clamp_len > 0: + pos_seq.clamp_(max=self.clamp_len) + pos_emb = self.pos_emb(pos_seq) + + core_out = self.drop(word_emb) + pos_emb = self.drop(pos_emb) + + for i, layer in enumerate(self.layers): + hids.append(core_out) + mems_i = None if mems is None else mems[i] + layer_outputs = layer(core_out, pos_emb, dec_attn_mask=dec_attn_mask, + mems=mems_i, head_mask=head_mask[i]) + core_out = layer_outputs[0] + if self.output_attentions: + attentions.append(layer_outputs[1]) + elif self.attn_type == 1: # learnable + core_out = self.drop(word_emb) + for i, layer in enumerate(self.layers): + hids.append(core_out) + if self.clamp_len > 0: + r_emb = self.r_emb[i][-self.clamp_len :] + r_bias = self.r_bias[i][-self.clamp_len :] + else: + r_emb, r_bias = self.r_emb[i], self.r_bias[i] + + mems_i = None if mems is None else mems[i] + layer_outputs = layer(core_out, r_emb, self.r_w_bias[i], + r_bias, dec_attn_mask=dec_attn_mask, + mems=mems_i, head_mask=head_mask[i]) + core_out = layer_outputs[0] + if self.output_attentions: + attentions.append(layer_outputs[1]) + elif self.attn_type == 2: # absolute + pos_seq = torch.arange(klen - 1, -1, -1.0, device=word_emb.device, + dtype=word_emb.dtype) + if self.clamp_len > 0: + pos_seq.clamp_(max=self.clamp_len) + pos_emb = self.pos_emb(pos_seq) + + core_out = self.drop(word_emb + pos_emb[-qlen:]) + + for i, layer in enumerate(self.layers): + hids.append(core_out) + mems_i = None if mems is None else mems[i] + if mems_i is not None and i == 0: + mems_i += pos_emb[:mlen] + layer_outputs = layer(core_out, dec_attn_mask=dec_attn_mask, + mems=mems_i, head_mask=head_mask[i]) + core_out = layer_outputs[0] + if self.output_attentions: + attentions.append(layer_outputs[1]) + elif self.attn_type == 3: + core_out = self.drop(word_emb) + + for i, layer in enumerate(self.layers): + hids.append(core_out) + mems_i = None if mems is None else mems[i] + if mems_i is not None and mlen > 0: + cur_emb = self.r_emb[i][:-qlen] + cur_size = cur_emb.size(0) + if cur_size < mlen: + cur_emb_pad = cur_emb[0:1].expand(mlen-cur_size, -1, -1) + cur_emb = torch.cat([cur_emb_pad, cur_emb], 0) + else: + cur_emb = cur_emb[-mlen:] + mems_i += cur_emb.view(mlen, 1, -1) + core_out += self.r_emb[i][-qlen:].view(qlen, 1, -1) + + layer_outputs = layer(core_out, dec_attn_mask=dec_attn_mask, + mems=mems_i, head_mask=head_mask[i]) + core_out = layer_outputs[0] + if self.output_attentions: + attentions.append(layer_outputs[1]) + + core_out = self.drop(core_out) + + new_mems = self._update_mems(hids, mems, mlen, qlen) + + # We transpose back here to shape [bsz, len, hidden_dim] + outputs = [core_out.transpose(0, 1).contiguous(), new_mems] + if self.output_hidden_states: + # Add last layer and transpose to library standard shape [bsz, len, hidden_dim] + hids.append(core_out) + hids = list(t.transpose(0, 1).contiguous() for t in hids) + outputs.append(hids) + if self.output_attentions: + # Transpose to library standard shape [bsz, n_heads, query_seq_len, key_seq_len] + attentions = list(t.permute(2, 3, 0, 1).contiguous() for t in attentions) + outputs.append(attentions) + return outputs # last hidden state, new_mems, (all hidden states), (all attentions) + + def forward(self, input_ids, mems=None, head_mask=None): + # the original code for Transformer-XL used shapes [len, bsz] but we want a unified interface in the library + # so we transpose here from shape [bsz, len] to shape [len, bsz] + input_ids = input_ids.transpose(0, 1).contiguous() + + if mems is None: + mems = self.init_mems(input_ids) + outputs = self._forward(input_ids, mems=mems, head_mask=head_mask) + + return outputs # last hidden state, new_mems, (all hidden states), (all attentions) + + +@add_start_docstrings("""The Transformer-XL Model with a language modeling head on top + (adaptive softmax with weights tied to the adaptive input embeddings)""", + TRANSFO_XL_START_DOCSTRING, TRANSFO_XL_INPUTS_DOCSTRING) +class TransfoXLLMHeadModel(TransfoXLPreTrainedModel): + r""" + **lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Labels for language modeling. + Note that the labels **are shifted** inside the model, i.e. you can set ``lm_labels = input_ids`` + Indices are selected in ``[-1, 0, ..., config.vocab_size]`` + All labels set to ``-1`` are ignored (masked), the loss is only + computed for labels in ``[0, ..., config.vocab_size]`` + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``lm_labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Language modeling loss. + **prediction_scores**: ``None`` if ``lm_labels`` is provided else ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + We don't output them when the loss is computed to speedup adaptive softmax decoding. + **mems**: + list of ``torch.FloatTensor`` (one for each layer): + that contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model + (see `mems` input above). Can be used to speed up sequential decoding and attend to longer context. + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = TransfoXLTokenizer.from_pretrained('transfo-xl-wt103') + model = TransfoXLLMHeadModel.from_pretrained('transfo-xl-wt103') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids) + prediction_scores, mems = outputs[:2] + + """ + def __init__(self, config): + super(TransfoXLLMHeadModel, self).__init__(config) + self.transformer = TransfoXLModel(config) + self.sample_softmax = config.sample_softmax + # use sampled softmax + if config.sample_softmax > 0: + self.out_layer = nn.Linear(config.d_model, config.n_token) + self.sampler = LogUniformSampler(config.n_token, config.sample_softmax) + # use adaptive softmax (including standard softmax) + else: + self.crit = ProjectedAdaptiveLogSoftmax(config.n_token, config.d_embed, config.d_model, + config.cutoffs, div_val=config.div_val) + self.init_weights() + self.tie_weights() + + def tie_weights(self): + """ + Run this to be sure output and input (adaptive) softmax weights are tied + """ + # sampled softmax + if self.sample_softmax > 0: + if self.config.tie_weight: + self.out_layer.weight = self.transformer.word_emb.weight + # adaptive softmax (including standard softmax) + else: + if self.config.tie_weight: + for i in range(len(self.crit.out_layers)): + self._tie_or_clone_weights(self.crit.out_layers[i], + self.transformer.word_emb.emb_layers[i]) + if self.config.tie_projs: + for i, tie_proj in enumerate(self.config.tie_projs): + if tie_proj and self.config.div_val == 1 and self.config.d_model != self.config.d_embed: + if self.config.torchscript: + self.crit.out_projs[i] = nn.Parameter(self.transformer.word_emb.emb_projs[0].clone()) + else: + self.crit.out_projs[i] = self.transformer.word_emb.emb_projs[0] + elif tie_proj and self.config.div_val != 1: + if self.config.torchscript: + self.crit.out_projs[i] = nn.Parameter(self.transformer.word_emb.emb_projs[i].clone()) + else: + self.crit.out_projs[i] = self.transformer.word_emb.emb_projs[i] + + def reset_length(self, tgt_len, ext_len, mem_len): + self.transformer.reset_length(tgt_len, ext_len, mem_len) + + def init_mems(self, data): + return self.transformer.init_mems(data) + + def forward(self, input_ids, mems=None, head_mask=None, labels=None): + bsz = input_ids.size(0) + tgt_len = input_ids.size(1) + + transformer_outputs = self.transformer(input_ids, mems=mems, head_mask=head_mask) + + last_hidden = transformer_outputs[0] + pred_hid = last_hidden[:, -tgt_len:] + outputs = transformer_outputs[1:] + if self.sample_softmax > 0 and self.training: + assert self.config.tie_weight + logit = sample_logits(self.transformer.word_emb, self.out_layer.bias, labels, pred_hid, self.sampler) + softmax_output = -F.log_softmax(logit, -1)[:, :, 0] + outputs = [softmax_output] + outputs + if labels is not None: + # TODO: This is not implemented + raise NotImplementedError + else: + softmax_output = self.crit(pred_hid.view(-1, pred_hid.size(-1)), labels) + if labels is None: + softmax_output = softmax_output.view(bsz, tgt_len, -1) + outputs = [softmax_output] + outputs + else: + softmax_output = softmax_output.view(bsz, tgt_len) + outputs = [softmax_output, None] + outputs + + return outputs # (loss), logits or None if labels is not None (speed up adaptive softmax), new_mems, (all hidden states), (all attentions) From d3a3a0353c2d7e7ce0999974cdf3aa53932e6a1e Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 12 Sep 2019 16:42:52 +0200 Subject: [PATCH 047/219] clean up cache after conversion --- pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py b/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py index 0a4119af78..d586fecbc5 100644 --- a/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py +++ b/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py @@ -126,6 +126,8 @@ def convert_all_pt_checkpoints_to_tf(args_model_type, tf_dump_path, compare_with config_file, os.path.join(tf_dump_path, shortcut_name + '-tf_model.h5'), compare_with_pt_model=compare_with_pt_model) + os.remove(config_file) + os.remove(model_file) if __name__ == "__main__": From dcddf498c8b34cfeb99fb563e771877a1bc7ded5 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 12 Sep 2019 16:46:32 +0200 Subject: [PATCH 048/219] fix bert layernorm --- pytorch_transformers/modeling_tf_pytorch_utils.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pytorch_transformers/modeling_tf_pytorch_utils.py b/pytorch_transformers/modeling_tf_pytorch_utils.py index 6784596944..d979c0e1a4 100644 --- a/pytorch_transformers/modeling_tf_pytorch_utils.py +++ b/pytorch_transformers/modeling_tf_pytorch_utils.py @@ -62,6 +62,19 @@ def load_pytorch_state_dict_in_tf2_model(tf_model, pt_state_dict, tf_inputs=None "https://pytorch.org/ and https://www.tensorflow.org/install/ for installation instructions.") raise e + # Adapt state dict - TODO remove this and update the AWS weights files instead + for key in pt_state_dict.keys(): + new_key = None + if 'gamma' in key: + new_key = key.replace('gamma', 'weight') + if 'beta' in key: + new_key = key.replace('beta', 'bias') + if new_key: + old_keys.append(key) + new_keys.append(new_key) + for old_key, new_key in zip(old_keys, new_keys): + pt_state_dict[new_key] = pt_state_dict.pop(old_key) + symbolic_weights = tf_model.trainable_weights + tf_model.non_trainable_weights weight_value_tuples = [] From 39c38b2ea0f1cdfe4425c377c7b296bad6056843 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 12 Sep 2019 16:47:11 +0200 Subject: [PATCH 049/219] fix --- pytorch_transformers/modeling_tf_pytorch_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pytorch_transformers/modeling_tf_pytorch_utils.py b/pytorch_transformers/modeling_tf_pytorch_utils.py index d979c0e1a4..13cd6dd14d 100644 --- a/pytorch_transformers/modeling_tf_pytorch_utils.py +++ b/pytorch_transformers/modeling_tf_pytorch_utils.py @@ -63,6 +63,9 @@ def load_pytorch_state_dict_in_tf2_model(tf_model, pt_state_dict, tf_inputs=None raise e # Adapt state dict - TODO remove this and update the AWS weights files instead + # Convert old format to new format if needed from a PyTorch state_dict + old_keys = [] + new_keys = [] for key in pt_state_dict.keys(): new_key = None if 'gamma' in key: From 65c49bb27e180ed0b3fa2d08a914d6cea64dccce Mon Sep 17 00:00:00 2001 From: thomwolf Date: Fri, 13 Sep 2019 15:50:51 +0200 Subject: [PATCH 050/219] adding TF 2.0 adaptive softmax with logits + loss outputs --- pytorch_transformers/modeling_tf_bert.py | 2 + .../modeling_tf_transfo_xl.py | 1153 ++++++----------- .../modeling_tf_transfo_xl_utilities.py | 279 ++++ pytorch_transformers/modeling_tf_xlm.py | 4 +- pytorch_transformers/modeling_transfo_xl.py | 393 +----- .../tests/modeling_tf_bert_test.py | 6 +- .../tests/modeling_tf_transfo_xl_test.py | 217 ++++ 7 files changed, 909 insertions(+), 1145 deletions(-) create mode 100644 pytorch_transformers/modeling_tf_transfo_xl_utilities.py create mode 100644 pytorch_transformers/tests/modeling_tf_transfo_xl_test.py diff --git a/pytorch_transformers/modeling_tf_bert.py b/pytorch_transformers/modeling_tf_bert.py index 6500d3a3e8..9aa9e21c99 100644 --- a/pytorch_transformers/modeling_tf_bert.py +++ b/pytorch_transformers/modeling_tf_bert.py @@ -455,6 +455,8 @@ class TFBertMainLayer(tf.keras.layers.Layer): """ raise NotImplementedError + # def call(self, input_ids, attention_mask=None, token_type_ids=None, + # position_ids=None, head_mask=None, training=False): def call(self, inputs, training=False): if not isinstance(inputs, (dict, tuple, list)): input_ids = inputs diff --git a/pytorch_transformers/modeling_tf_transfo_xl.py b/pytorch_transformers/modeling_tf_transfo_xl.py index 6aa413811e..a4a333d969 100644 --- a/pytorch_transformers/modeling_tf_transfo_xl.py +++ b/pytorch_transformers/modeling_tf_transfo_xl.py @@ -30,8 +30,8 @@ import numpy as np import tensorflow as tf from .configuration_transfo_xl import TransfoXLConfig -from .modeling_tf_utils import TFPreTrainedModel, TFConv1D, TFSequenceSummary -from .modeling_transfo_xl_utilities import ProjectedAdaptiveLogSoftmax, sample_logits +from .modeling_tf_utils import TFPreTrainedModel, TFConv1D, TFSequenceSummary, shape_list +from .modeling_tf_transfo_xl_utilities import TFAdaptiveSoftmaxMask from .file_utils import add_start_docstrings from .modeling_tf_pytorch_utils import load_pytorch_checkpoint_in_tf2_model @@ -49,55 +49,56 @@ def load_transfo_xl_pt_weights_in_tf2(tf_model, pytorch_checkpoint_path): return load_pytorch_checkpoint_in_tf2_model(tf_model, pytorch_checkpoint_path, tf_inputs=tf_inputs) -class PositionalEmbedding(nn.Module): - def __init__(self, demb): - super(PositionalEmbedding, self).__init__() +class TFPositionalEmbedding(tf.keras.layers.Layer): + def __init__(self, demb, **kwargs): + super(TFPositionalEmbedding, self).__init__(**kwargs) - self.demb = demb + self.inv_freq = 1 / (10000 ** (tf.range(0, demb, 2.0) / demb)) - inv_freq = 1 / (10000 ** (torch.arange(0.0, demb, 2.0) / demb)) - self.register_buffer('inv_freq', inv_freq) - - def forward(self, pos_seq, bsz=None): - sinusoid_inp = torch.ger(pos_seq, self.inv_freq) - pos_emb = torch.cat([sinusoid_inp.sin(), sinusoid_inp.cos()], dim=-1) + def call(self, pos_seq, bsz=None): + sinusoid_inp = tf.einsum('i,j->ij', pos_seq, self.inv_freq) + pos_emb = tf.concat([tf.sin(sinusoid_inp), tf.cos(sinusoid_inp)], -1) if bsz is not None: - return pos_emb[:,None,:].expand(-1, bsz, -1) + return tf.tile(pos_emb[:, None, :], [1, bsz, 1]) else: - return pos_emb[:,None,:] + return pos_emb[:, None, :] - -class PositionwiseFF(nn.Module): - def __init__(self, d_model, d_inner, dropout, pre_lnorm=False): - super(PositionwiseFF, self).__init__() +class TFPositionwiseFF(tf.keras.layers.Layer): + def __init__(self, d_model, d_inner, dropout, pre_lnorm=False, **kwargs): + super(TFPositionwiseFF, self).__init__(**kwargs) self.d_model = d_model self.d_inner = d_inner self.dropout = dropout - self.CoreNet = nn.Sequential( - nn.Linear(d_model, d_inner), nn.ReLU(inplace=True), - nn.Dropout(dropout), - nn.Linear(d_inner, d_model), - nn.Dropout(dropout), - ) + self.layer_1 = tf.keras.layers.Dense(d_inner, activation=tf.nn.relu, name='CoreNet_._0') + self.drop_1 = tf.keras.layers.Dropout(dropout) + self.layer_2 = tf.keras.layers.Dense(d_model, name='CoreNet_._2') + self.drop_2 = tf.keras.layers.Dropout(dropout) - self.layer_norm = nn.LayerNorm(d_model) + self.layer_norm = tf.keras.layers.LayerNormalization(epsilon=1e-12, name='layer_norm') self.pre_lnorm = pre_lnorm - def forward(self, inp): + def call(self, inp, training=False): if self.pre_lnorm: ##### layer normalization + positionwise feed-forward - core_out = self.CoreNet(self.layer_norm(inp)) + core_out = self.layer_norm(inp) + core_out = self.layer_1(core_out) + core_out = self.drop_1(core_out, training=training) + core_out = self.layer_2(core_out) + core_out = self.drop_2(core_out, training=training) ##### residual connection output = core_out + inp else: ##### positionwise feed-forward - core_out = self.CoreNet(inp) + core_out = self.layer_1(inp) + core_out = self.drop_1(core_out, training=training) + core_out = self.layer_2(core_out) + core_out = self.drop_2(core_out, training=training) ##### residual connection + layer normalization output = self.layer_norm(inp + core_out) @@ -105,102 +106,11 @@ class PositionwiseFF(nn.Module): return output - -class MultiHeadAttn(nn.Module): - def __init__(self, n_head, d_model, d_head, dropout, dropatt=0, - pre_lnorm=False, r_r_bias=None, r_w_bias=None, output_attentions=False): - super(MultiHeadAttn, self).__init__() - - self.output_attentions = output_attentions - self.n_head = n_head - self.d_model = d_model - self.d_head = d_head - self.dropout = dropout - - self.q_net = nn.Linear(d_model, n_head * d_head, bias=False) - self.kv_net = nn.Linear(d_model, 2 * n_head * d_head, bias=False) - - self.drop = nn.Dropout(dropout) - self.dropatt = nn.Dropout(dropatt) - self.o_net = nn.Linear(n_head * d_head, d_model, bias=False) - - self.layer_norm = nn.LayerNorm(d_model) - - self.scale = 1 / (d_head ** 0.5) - - self.pre_lnorm = pre_lnorm - - if r_r_bias is None or r_w_bias is None: # Biases are not shared - self.r_r_bias = nn.Parameter(torch.FloatTensor(self.n_head, self.d_head)) - self.r_w_bias = nn.Parameter(torch.FloatTensor(self.n_head, self.d_head)) - else: - self.r_r_bias = r_r_bias - self.r_w_bias = r_w_bias - - def forward(self, h, attn_mask=None, mems=None, head_mask=None): - ##### multihead attention - # [hlen x bsz x n_head x d_head] - - if mems is not None: - c = torch.cat([mems, h], 0) - else: - c = h - - if self.pre_lnorm: - ##### layer normalization - c = self.layer_norm(c) - - head_q = self.q_net(h) - head_k, head_v = torch.chunk(self.kv_net(c), 2, -1) - - head_q = head_q.view(h.size(0), h.size(1), self.n_head, self.d_head) - head_k = head_k.view(c.size(0), c.size(1), self.n_head, self.d_head) - head_v = head_v.view(c.size(0), c.size(1), self.n_head, self.d_head) - - # [qlen x klen x bsz x n_head] - attn_score = torch.einsum('ibnd,jbnd->ijbn', (head_q, head_k)) - attn_score.mul_(self.scale) - if attn_mask is not None and torch.sum(attn_mask).item(): - attn_mask = (attn_mask == 1) # Switch to bool - if attn_mask.dim() == 2: - attn_score.masked_fill_(attn_mask[None,:,:,None], -float('inf')) - elif attn_mask.dim() == 3: - attn_score.masked_fill_(attn_mask[:,:,:,None], -float('inf')) - - # [qlen x klen x bsz x n_head] - attn_prob = F.softmax(attn_score, dim=1) - attn_prob = self.dropatt(attn_prob) - - # Mask heads if we want to - if head_mask is not None: - attn_prob = attn_prob * head_mask - - # [qlen x klen x bsz x n_head] + [klen x bsz x n_head x d_head] -> [qlen x bsz x n_head x d_head] - attn_vec = torch.einsum('ijbn,jbnd->ibnd', (attn_prob, head_v)) - attn_vec = attn_vec.contiguous().view( - attn_vec.size(0), attn_vec.size(1), self.n_head * self.d_head) - - ##### linear projection - attn_out = self.o_net(attn_vec) - attn_out = self.drop(attn_out) - - if self.pre_lnorm: - ##### residual connection - outputs = [h + attn_out] - else: - ##### residual connection + layer normalization - outputs = [self.layer_norm(h + attn_out)] - - if self.output_attentions: - outputs.append(attn_prob) - - return outputs - -class RelMultiHeadAttn(nn.Module): +class TFRelPartialLearnableMultiHeadAttn(tf.keras.layers.Layer): def __init__(self, n_head, d_model, d_head, dropout, dropatt=0, tgt_len=None, ext_len=None, mem_len=None, pre_lnorm=False, - r_r_bias=None, r_w_bias=None, output_attentions=False): - super(RelMultiHeadAttn, self).__init__() + r_r_bias=None, r_w_bias=None, output_attentions=False, **kwargs): + super(TFRelPartialLearnableMultiHeadAttn, self).__init__(**kwargs) self.output_attentions = output_attentions self.n_head = n_head @@ -208,91 +118,60 @@ class RelMultiHeadAttn(nn.Module): self.d_head = d_head self.dropout = dropout - self.qkv_net = nn.Linear(d_model, 3 * n_head * d_head, bias=False) + self.qkv_net = tf.keras.layers.Dense(3 * n_head * d_head, use_bias=False, name='qkv_net') - self.drop = nn.Dropout(dropout) - self.dropatt = nn.Dropout(dropatt) - self.o_net = nn.Linear(n_head * d_head, d_model, bias=False) + self.drop = tf.keras.layers.Dropout(dropout) + self.dropatt = tf.keras.layers.Dropout(dropatt) + self.o_net = tf.keras.layers.Dense(d_model, use_bias=False, name='o_net') - self.layer_norm = nn.LayerNorm(d_model) + self.layer_norm = tf.keras.layers.LayerNormalization(epsilon=1e-12, name='layer_norm') self.scale = 1 / (d_head ** 0.5) self.pre_lnorm = pre_lnorm - if r_r_bias is None or r_w_bias is None: # Biases are not shared - self.r_r_bias = nn.Parameter(torch.FloatTensor(self.n_head, self.d_head)) - self.r_w_bias = nn.Parameter(torch.FloatTensor(self.n_head, self.d_head)) - else: + if r_r_bias is not None and r_w_bias is not None: # Biases are shared self.r_r_bias = r_r_bias self.r_w_bias = r_w_bias - - def _parallelogram_mask(self, h, w, left=False): - mask = torch.ones((h, w)).byte() - m = min(h, w) - mask[:m,:m] = torch.triu(mask[:m,:m]) - mask[-m:,-m:] = torch.tril(mask[-m:,-m:]) - - if left: - return mask else: - return mask.flip(0) + self.r_r_bias = None + self.r_w_bias = None - def _shift(self, x, qlen, klen, mask, left=False): - if qlen > 1: - zero_pad = torch.zeros((x.size(0), qlen-1, x.size(2), x.size(3)), - device=x.device, dtype=x.dtype) - else: - zero_pad = torch.zeros(0, device=x.device, dtype=x.dtype) + self.r_net = tf.keras.layers.Dense(self.n_head * self.d_head, use_bias=False, name='r_net') - if left: - mask = mask.flip(1) - x_padded = torch.cat([zero_pad, x], dim=1).expand(qlen, -1, -1, -1) - else: - x_padded = torch.cat([x, zero_pad], dim=1).expand(qlen, -1, -1, -1) + def build(self, input_shape): + if self.r_r_bias is None or self.r_w_bias is None: # Biases are not shared + self.r_r_bias = self.add_weight(shape=(self.n_head, self.d_head), + trainable=True, + name='r_r_bias') + self.r_w_bias = self.add_weight(shape=(self.n_head, self.d_head), + trainable=True, + name='r_w_bias') + super(TFRelPartialLearnableMultiHeadAttn, self).build(input_shape) - x = x_padded.masked_select(mask[:,:,None,None]) \ - .view(qlen, klen, x.size(2), x.size(3)) + def _rel_shift(self, x): + x_size = shape_list(x) + + x = tf.pad(x, [[0, 0], [1, 0], [0, 0], [0, 0]]) + x = tf.reshape(x, [x_size[1] + 1, x_size[0], x_size[2], x_size[3]]) + x = tf.slice(x, [1, 0, 0, 0], [-1, -1, -1, -1]) + x = tf.reshape(x, x_size) return x - def _rel_shift(self, x, zero_triu=False): - zero_pad_shape = (x.size(0), 1) + x.size()[2:] - zero_pad = torch.zeros(zero_pad_shape, device=x.device, dtype=x.dtype) - x_padded = torch.cat([zero_pad, x], dim=1) - - x_padded_shape = (x.size(1) + 1, x.size(0)) + x.size()[2:] - x_padded = x_padded.view(*x_padded_shape) - - x = x_padded[1:].view_as(x) - - if zero_triu: - ones = torch.ones((x.size(0), x.size(1))) - x = x * torch.tril(ones, x.size(1) - x.size(0))[:,:,None,None] - - return x - - def forward(self, w, r, attn_mask=None, mems=None): - raise NotImplementedError - -class RelPartialLearnableMultiHeadAttn(RelMultiHeadAttn): - def __init__(self, *args, **kwargs): - super(RelPartialLearnableMultiHeadAttn, self).__init__(*args, **kwargs) - - self.r_net = nn.Linear(self.d_model, self.n_head * self.d_head, bias=False) - - def forward(self, w, r, attn_mask=None, mems=None, head_mask=None): - qlen, rlen, bsz = w.size(0), r.size(0), w.size(1) + def call(self, inputs, training=False): + w, r, attn_mask, mems, head_mask = inputs + qlen, rlen, bsz = shape_list(w)[0], shape_list(r)[0], shape_list(w)[1] if mems is not None: - cat = torch.cat([mems, w], 0) + cat = tf.concat([mems, w], 0) if self.pre_lnorm: w_heads = self.qkv_net(self.layer_norm(cat)) else: w_heads = self.qkv_net(cat) r_head_k = self.r_net(r) - w_head_q, w_head_k, w_head_v = torch.chunk(w_heads, 3, dim=-1) + w_head_q, w_head_k, w_head_v = tf.split(w_heads, 3, axis=-1) w_head_q = w_head_q[-qlen:] else: if self.pre_lnorm: @@ -301,148 +180,52 @@ class RelPartialLearnableMultiHeadAttn(RelMultiHeadAttn): w_heads = self.qkv_net(w) r_head_k = self.r_net(r) - w_head_q, w_head_k, w_head_v = torch.chunk(w_heads, 3, dim=-1) + w_head_q, w_head_k, w_head_v = tf.split(w_heads, 3, axis=-1) - klen = w_head_k.size(0) + klen = shape_list(w_head_k)[0] - w_head_q = w_head_q.view(qlen, bsz, self.n_head, self.d_head) # qlen x bsz x n_head x d_head - w_head_k = w_head_k.view(klen, bsz, self.n_head, self.d_head) # qlen x bsz x n_head x d_head - w_head_v = w_head_v.view(klen, bsz, self.n_head, self.d_head) # qlen x bsz x n_head x d_head + w_head_q = tf.reshape(w_head_q, (qlen, bsz, self.n_head, self.d_head)) # qlen x bsz x n_head x d_head + w_head_k = tf.reshape(w_head_k, (klen, bsz, self.n_head, self.d_head)) # qlen x bsz x n_head x d_head + w_head_v = tf.reshape(w_head_v, (klen, bsz, self.n_head, self.d_head)) # qlen x bsz x n_head x d_head - r_head_k = r_head_k.view(rlen, self.n_head, self.d_head) # qlen x n_head x d_head + r_head_k = tf.reshape(r_head_k, (rlen, self.n_head, self.d_head)) # qlen x n_head x d_head #### compute attention score rw_head_q = w_head_q + self.r_w_bias # qlen x bsz x n_head x d_head - AC = torch.einsum('ibnd,jbnd->ijbn', (rw_head_q, w_head_k)) # qlen x klen x bsz x n_head + AC = tf.einsum('ibnd,jbnd->ijbn', rw_head_q, w_head_k) # qlen x klen x bsz x n_head rr_head_q = w_head_q + self.r_r_bias - BD = torch.einsum('ibnd,jnd->ijbn', (rr_head_q, r_head_k)) # qlen x klen x bsz x n_head + BD = tf.einsum('ibnd,jnd->ijbn', rr_head_q, r_head_k) # qlen x klen x bsz x n_head BD = self._rel_shift(BD) # [qlen x klen x bsz x n_head] attn_score = AC + BD - attn_score.mul_(self.scale) + attn_score = attn_score * self.scale #### compute attention probability - if attn_mask is not None and torch.sum(attn_mask).item(): - attn_mask = (attn_mask == 1) # Switch to bool - if attn_mask.dim() == 2: - attn_score = attn_score.float().masked_fill( - attn_mask[None,:,:,None], -1e30).type_as(attn_score) - elif attn_mask.dim() == 3: - attn_score = attn_score.float().masked_fill( - attn_mask[:,:,:,None], -1e30).type_as(attn_score) + if attn_mask is not None: + attn_mask_t = attn_mask[:, :, None, None] + attn_score = attn_score * (1 - attn_mask_t) - 1e30 * attn_mask_t # [qlen x klen x bsz x n_head] - attn_prob = F.softmax(attn_score, dim=1) - attn_prob = self.dropatt(attn_prob) + attn_prob = tf.nn.softmax(attn_score, axis=1) + attn_prob = self.dropatt(attn_prob, training=training) # Mask heads if we want to if head_mask is not None: attn_prob = attn_prob * head_mask #### compute attention vector - attn_vec = torch.einsum('ijbn,jbnd->ibnd', (attn_prob, w_head_v)) + attn_vec = tf.einsum('ijbn,jbnd->ibnd', attn_prob, w_head_v) # [qlen x bsz x n_head x d_head] - attn_vec = attn_vec.contiguous().view( - attn_vec.size(0), attn_vec.size(1), self.n_head * self.d_head) + attn_vec_sizes = shape_list(attn_vec) + attn_vec = tf.reshape(attn_vec, + (attn_vec_sizes[0], attn_vec_sizes[1], self.n_head * self.d_head)) ##### linear projection attn_out = self.o_net(attn_vec) - attn_out = self.drop(attn_out) - - if self.pre_lnorm: - ##### residual connection - outputs = [w + attn_out] - else: - ##### residual connection + layer normalization - outputs = [self.layer_norm(w + attn_out)] - - if self.output_attentions: - outputs.append(attn_prob) - - return outputs - -class RelLearnableMultiHeadAttn(RelMultiHeadAttn): - def __init__(self, *args, **kwargs): - super(RelLearnableMultiHeadAttn, self).__init__(*args, **kwargs) - - def forward(self, w, r_emb, r_w_bias, r_bias, attn_mask=None, mems=None, head_mask=None): - # r_emb: [klen, n_head, d_head], used for term B - # r_w_bias: [n_head, d_head], used for term C - # r_bias: [klen, n_head], used for term D - - qlen, bsz = w.size(0), w.size(1) - - if mems is not None: - cat = torch.cat([mems, w], 0) - if self.pre_lnorm: - w_heads = self.qkv_net(self.layer_norm(cat)) - else: - w_heads = self.qkv_net(cat) - w_head_q, w_head_k, w_head_v = torch.chunk(w_heads, 3, dim=-1) - - w_head_q = w_head_q[-qlen:] - else: - if self.pre_lnorm: - w_heads = self.qkv_net(self.layer_norm(w)) - else: - w_heads = self.qkv_net(w) - w_head_q, w_head_k, w_head_v = torch.chunk(w_heads, 3, dim=-1) - - klen = w_head_k.size(0) - - w_head_q = w_head_q.view(qlen, bsz, self.n_head, self.d_head) - w_head_k = w_head_k.view(klen, bsz, self.n_head, self.d_head) - w_head_v = w_head_v.view(klen, bsz, self.n_head, self.d_head) - - if klen > r_emb.size(0): - r_emb_pad = r_emb[0:1].expand(klen-r_emb.size(0), -1, -1) - r_emb = torch.cat([r_emb_pad, r_emb], 0) - r_bias_pad = r_bias[0:1].expand(klen-r_bias.size(0), -1) - r_bias = torch.cat([r_bias_pad, r_bias], 0) - else: - r_emb = r_emb[-klen:] - r_bias = r_bias[-klen:] - - #### compute attention score - rw_head_q = w_head_q + r_w_bias[None] # qlen x bsz x n_head x d_head - - AC = torch.einsum('ibnd,jbnd->ijbn', (rw_head_q, w_head_k)) # qlen x klen x bsz x n_head - B_ = torch.einsum('ibnd,jnd->ijbn', (w_head_q, r_emb)) # qlen x klen x bsz x n_head - D_ = r_bias[None, :, None] # 1 x klen x 1 x n_head - BD = self._rel_shift(B_ + D_) - - # [qlen x klen x bsz x n_head] - attn_score = AC + BD - attn_score.mul_(self.scale) - - #### compute attention probability - if attn_mask is not None and torch.sum(attn_mask).item(): - attn_mask = (attn_mask == 1) # Switch to bool - if attn_mask.dim() == 2: - attn_score.masked_fill_(attn_mask[None,:,:,None], -float('inf')) - elif attn_mask.dim() == 3: - attn_score.masked_fill_(attn_mask[:,:,:,None], -float('inf')) - - # [qlen x klen x bsz x n_head] - attn_prob = F.softmax(attn_score, dim=1) - attn_prob = self.dropatt(attn_prob) - - if head_mask is not None: - attn_prob = attn_prob * head_mask - - #### compute attention vector - attn_vec = torch.einsum('ijbn,jbnd->ibnd', (attn_prob, w_head_v)) - - # [qlen x bsz x n_head x d_head] - attn_vec = attn_vec.contiguous().view( - attn_vec.size(0), attn_vec.size(1), self.n_head * self.d_head) - - ##### linear projection - attn_out = self.o_net(attn_vec) - attn_out = self.drop(attn_out) + attn_out = self.drop(attn_out, training=training) if self.pre_lnorm: ##### residual connection @@ -457,73 +240,39 @@ class RelLearnableMultiHeadAttn(RelMultiHeadAttn): return outputs - -class DecoderLayer(nn.Module): - def __init__(self, n_head, d_model, d_head, d_inner, dropout, **kwargs): - super(DecoderLayer, self).__init__() - - self.dec_attn = MultiHeadAttn(n_head, d_model, d_head, dropout, **kwargs) - self.pos_ff = PositionwiseFF(d_model, d_inner, dropout, - pre_lnorm=kwargs.get('pre_lnorm')) - - def forward(self, dec_inp, dec_attn_mask=None, mems=None, head_mask=None): - - attn_outputs = self.dec_attn(dec_inp, attn_mask=dec_attn_mask, - mems=mems, head_mask=head_mask) - ff_output = self.pos_ff(attn_outputs[0]) - - outputs = [ff_output] + attn_outputs[1:] - - return outputs - -class RelLearnableDecoderLayer(nn.Module): +class TFRelPartialLearnableDecoderLayer(tf.keras.layers.Layer): def __init__(self, n_head, d_model, d_head, d_inner, dropout, + tgt_len=None, ext_len=None, mem_len=None, + dropatt=0., pre_lnorm=False, + r_w_bias=None, + r_r_bias=None, + output_attentions=False, **kwargs): - super(RelLearnableDecoderLayer, self).__init__() + super(TFRelPartialLearnableDecoderLayer, self).__init__(**kwargs) - self.dec_attn = RelLearnableMultiHeadAttn(n_head, d_model, d_head, dropout, - **kwargs) - self.pos_ff = PositionwiseFF(d_model, d_inner, dropout, - pre_lnorm=kwargs.get('pre_lnorm')) + self.dec_attn = TFRelPartialLearnableMultiHeadAttn(n_head, d_model, + d_head, dropout, tgt_len=tgt_len, ext_len=ext_len, + mem_len=mem_len, dropatt=dropatt, pre_lnorm=pre_lnorm, + r_w_bias=r_w_bias, r_r_bias=r_r_bias, + output_attentions=output_attentions, name='dec_attn') + self.pos_ff = TFPositionwiseFF(d_model, d_inner, dropout, + pre_lnorm=pre_lnorm, name='pos_ff') - def forward(self, dec_inp, r_emb, r_w_bias, r_bias, dec_attn_mask=None, mems=None, head_mask=None): - - attn_outputs = self.dec_attn(dec_inp, r_emb, r_w_bias, r_bias, - attn_mask=dec_attn_mask, - mems=mems, head_mask=head_mask) - ff_output = self.pos_ff(attn_outputs[0]) - - outputs = [ff_output] + attn_outputs[1:] - - return outputs - -class RelPartialLearnableDecoderLayer(nn.Module): - def __init__(self, n_head, d_model, d_head, d_inner, dropout, - **kwargs): - super(RelPartialLearnableDecoderLayer, self).__init__() - - self.dec_attn = RelPartialLearnableMultiHeadAttn(n_head, d_model, - d_head, dropout, **kwargs) - self.pos_ff = PositionwiseFF(d_model, d_inner, dropout, - pre_lnorm=kwargs.get('pre_lnorm')) - - def forward(self, dec_inp, r, dec_attn_mask=None, mems=None, head_mask=None): - - attn_outputs = self.dec_attn(dec_inp, r, - attn_mask=dec_attn_mask, - mems=mems, head_mask=head_mask) - ff_output = self.pos_ff(attn_outputs[0]) + def call(self, inputs, training=False): + dec_inp, r, dec_attn_mask, mems, head_mask = inputs + attn_outputs = self.dec_attn([dec_inp, r, dec_attn_mask, + mems, head_mask], training=training) + ff_output = self.pos_ff(attn_outputs[0], training=training) outputs = [ff_output] + attn_outputs[1:] return outputs - -class AdaptiveEmbedding(nn.Module): +class TFAdaptiveEmbedding(tf.keras.layers.Layer): def __init__(self, n_token, d_embed, d_proj, cutoffs, div_val=1, - sample_softmax=False): - super(AdaptiveEmbedding, self).__init__() + sample_softmax=False, **kwargs): + super(TFAdaptiveEmbedding, self).__init__(**kwargs) self.n_token = n_token self.d_embed = d_embed @@ -536,116 +285,273 @@ class AdaptiveEmbedding(nn.Module): self.cutoff_ends = [0] + self.cutoffs - self.emb_layers = nn.ModuleList() - self.emb_projs = nn.ParameterList() + self.emb_layers = [] + self.emb_projs = [] if div_val == 1: - self.emb_layers.append( - nn.Embedding(n_token, d_embed, sparse=sample_softmax>0) - ) - if d_proj != d_embed: - self.emb_projs.append(nn.Parameter(torch.FloatTensor(d_proj, d_embed))) + raise NotImplementedError # Removed these to avoid maintaining dead code - They are not used in our pretrained checkpoint else: for i in range(len(self.cutoffs)): l_idx, r_idx = self.cutoff_ends[i], self.cutoff_ends[i+1] d_emb_i = d_embed // (div_val ** i) - self.emb_layers.append(nn.Embedding(r_idx-l_idx, d_emb_i)) - self.emb_projs.append(nn.Parameter(torch.FloatTensor(d_proj, d_emb_i))) + self.emb_layers.append(tf.keras.layers.Embedding(r_idx-l_idx, d_emb_i, name='emb_layers_._{}'.format(i))) - def forward(self, inp): + def build(self, input_shape): + for i in range(len(self.cutoffs)): + d_emb_i = self.d_embed // (self.div_val ** i) + self.emb_projs.append(self.add_weight(shape=(d_emb_i, self.d_proj), + trainable=True, + name='emb_projs._{}'.format(i))) + super(TFAdaptiveEmbedding, self).build(input_shape) + + def call(self, inp): if self.div_val == 1: - embed = self.emb_layers[0](inp) - if self.d_proj != self.d_embed: - embed = F.linear(embed, self.emb_projs[0]) + raise NotImplementedError # Removed these to avoid maintaining dead code - They are not used in our pretrained checkpoint else: - param = next(self.parameters()) - inp_flat = inp.view(-1) - emb_flat = torch.zeros([inp_flat.size(0), self.d_proj], - dtype=param.dtype, device=param.device) + inp_flat = tf.reshape(inp, (-1,)) + emb_flat = tf.zeros([shape_list(inp_flat)[0], self.d_proj]) for i in range(len(self.cutoffs)): l_idx, r_idx = self.cutoff_ends[i], self.cutoff_ends[i + 1] mask_i = (inp_flat >= l_idx) & (inp_flat < r_idx) - indices_i = mask_i.nonzero().squeeze() - if indices_i.numel() == 0: - continue - - inp_i = inp_flat.index_select(0, indices_i) - l_idx + inp_i = tf.boolean_mask(inp_flat, mask_i) - l_idx emb_i = self.emb_layers[i](inp_i) - emb_i = F.linear(emb_i, self.emb_projs[i]) + emb_i = tf.einsum('id,de->ie', emb_i, self.emb_projs[i]) - emb_flat.index_copy_(0, indices_i, emb_i) + mask_idx = tf.cast(tf.where(mask_i), dtype=tf.int64) + emb_flat += tf.scatter_nd(mask_idx, emb_i, tf.cast(tf.shape(emb_flat), dtype=tf.int64)) - embed_shape = inp.size() + (self.d_proj,) - embed = emb_flat.view(embed_shape) + embed_shape = shape_list(inp) + [self.d_proj] + embed = tf.reshape(emb_flat, embed_shape) - embed.mul_(self.emb_scale) + embed *= self.emb_scale return embed -class TransfoXLPreTrainedModel(PreTrainedModel): +class TFTransfoXLMainLayer(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFTransfoXLMainLayer, self).__init__(**kwargs) + self.output_attentions = config.output_attentions + self.output_hidden_states = config.output_hidden_states + + self.n_token = config.n_token + + self.d_embed = config.d_embed + self.d_model = config.d_model + self.n_head = config.n_head + self.d_head = config.d_head + self.untie_r = config.untie_r + + self.word_emb = TFAdaptiveEmbedding(config.n_token, config.d_embed, config.d_model, config.cutoffs, + div_val=config.div_val, name='word_emb') + + self.drop = tf.keras.layers.Dropout(config.dropout) + + self.n_layer = config.n_layer + + self.tgt_len = config.tgt_len + self.mem_len = config.mem_len + self.ext_len = config.ext_len + self.max_klen = config.tgt_len + config.ext_len + config.mem_len + + self.attn_type = config.attn_type + + self.layers = [] + if config.attn_type == 0: # the default attention + for i in range(config.n_layer): + self.layers.append( + TFRelPartialLearnableDecoderLayer( + config.n_head, config.d_model, config.d_head, config.d_inner, config.dropout, + tgt_len=config.tgt_len, ext_len=config.ext_len, mem_len=config.mem_len, + dropatt=config.dropatt, pre_lnorm=config.pre_lnorm, + r_w_bias=None if self.untie_r else self.r_w_bias, + r_r_bias=None if self.untie_r else self.r_r_bias, + output_attentions=self.output_attentions, + name='layers_._{}'.format(i)) + ) + else: # learnable embeddings and absolute embeddings + raise NotImplementedError # Removed these to avoid maintaining dead code - They are not used in our pretrained checkpoint + + self.same_length = config.same_length + self.clamp_len = config.clamp_len + + if self.attn_type == 0: # default attention + self.pos_emb = TFPositionalEmbedding(self.d_model, name='pos_emb') + else: # learnable embeddings and absolute embeddings + raise NotImplementedError # Removed these to avoid maintaining dead code - They are not used in our pretrained checkpoint + + def build(self, input_shape): + if not self.untie_r: + self.r_w_bias = self.add_weight(shape=(self.n_head, self.d_head), + initializer='zeros', + trainable=True, + name='r_w_bias') + self.r_r_bias = self.add_weight(shape=(self.n_head, self.d_head), + initializer='zeros', + trainable=True, + name='r_r_bias') + super(TFTransfoXLMainLayer, self).build(input_shape) + + def _resize_token_embeddings(self, new_num_tokens): + return self.word_emb + + def backward_compatible(self): + self.sample_softmax = -1 + + def reset_length(self, tgt_len, ext_len, mem_len): + self.tgt_len = tgt_len + self.mem_len = mem_len + self.ext_len = ext_len + + def _prune_heads(self, heads): + raise NotImplementedError + + def init_mems(self, data): + if self.mem_len > 0: + mems = [] + for i in range(self.n_layer): + empty = tf.zeros([self.mem_len, shape_list(data)[1], self.d_model]) + mems.append(empty) + + return mems + else: + return None + + def _update_mems(self, hids, mems, qlen, mlen): + # does not deal with None + if mems is None: return None + + # mems is not None + assert len(hids) == len(mems), 'len(hids) != len(mems)' + + # There are `mlen + qlen` steps that can be cached into mems + # For the next step, the last `ext_len` of the `qlen` tokens + # will be used as the extended context. Hence, we only cache + # the tokens from `mlen + qlen - self.ext_len - self.mem_len` + # to `mlen + qlen - self.ext_len`. + new_mems = [] + end_idx = mlen + max(0, qlen - 0 - self.ext_len) + beg_idx = max(0, end_idx - self.mem_len) + for i in range(len(hids)): + + cat = tf.concat([mems[i], hids[i]], axis=0) + tf.stop_gradient(cat) + new_mems.append(cat[beg_idx:end_idx]) + + return new_mems + + def call(self, inputs, training=False): + if not isinstance(inputs, (dict, tuple, list)): + input_ids = inputs + mems, head_mask = None, None + elif isinstance(inputs, (tuple, list)): + input_ids = inputs[0] + mems = inputs[1] if len(inputs) > 1 else None + head_mask = inputs[2] if len(inputs) > 2 else None + assert len(inputs) <= 3, "Too many inputs." + else: + input_ids = inputs.get('input_ids') + mems = inputs.get('mems', None) + head_mask = inputs.get('head_mask', None) + assert len(inputs) <= 3, "Too many inputs." + + # the original code for Transformer-XL used shapes [len, bsz] but we want a unified interface in the library + # so we transpose here from shape [bsz, len] to shape [len, bsz] + input_ids = tf.transpose(input_ids, perm=(1, 0)) + + if mems is None: + mems = self.init_mems(input_ids) + + qlen, bsz = shape_list(input_ids) + + # Prepare head mask if needed + # 1.0 in head_mask indicate we keep the head + # attention_probs has shape bsz x n_heads x N x N + # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] (a head_mask for each layer) + # and head_mask is converted to shape [num_hidden_layers x qlen x klen x bsz x n_head] + if not head_mask is None: + raise NotImplementedError + else: + head_mask = [None] * self.n_layer + + word_emb = self.word_emb(input_ids) + + mlen = shape_list(mems[0])[0] if mems is not None else 0 + klen = mlen + qlen + + attn_mask = tf.ones([qlen, qlen]) + mask_u = tf.linalg.band_part(attn_mask, 0, -1) + mask_dia = tf.linalg.band_part(attn_mask, 0, 0) + attn_mask_pad = tf.zeros([qlen, mlen]) + dec_attn_mask = tf.concat([attn_mask_pad, mask_u - mask_dia], 1) + if self.same_length: + mask_l = tf.linalg.band_part(attn_mask, -1, 0) + dec_attn_mask = tf.concat([dec_attn_mask[:, :qlen] + mask_l - mask_dia, + dec_attn_mask[:, qlen:]], 1) + # ::: PyTorch masking code for reference ::: + # if self.same_length: + # all_ones = word_emb.new_ones((qlen, klen), dtype=torch.uint8) + # mask_len = klen - self.mem_len + # if mask_len > 0: + # mask_shift_len = qlen - mask_len + # else: + # mask_shift_len = qlen + # dec_attn_mask = (torch.triu(all_ones, 1+mlen) + # + torch.tril(all_ones, -mask_shift_len))[:, :, None] # -1 + # else: + # dec_attn_mask = torch.triu( + # word_emb.new_ones((qlen, klen), dtype=torch.uint8), diagonal=1+mlen)[:,:,None] + + hids = [] + attentions = [] + if self.attn_type == 0: # default + pos_seq = tf.range(klen-1, -1, -1.0) + if self.clamp_len > 0: + pos_seq = tf.minimum(pos_seq, self.clamp_len) + pos_emb = self.pos_emb(pos_seq) + + core_out = self.drop(word_emb, training=training) + pos_emb = self.drop(pos_emb, training=training) + + for i, layer in enumerate(self.layers): + hids.append(core_out) + mems_i = None if mems is None else mems[i] + layer_outputs = layer([core_out, pos_emb, dec_attn_mask, + mems_i, head_mask[i]], training=training) + core_out = layer_outputs[0] + if self.output_attentions: + attentions.append(layer_outputs[1]) + else: # learnable embeddings and absolute embeddings + raise NotImplementedError # Removed these to avoid maintaining dead code - They are not used in our pretrained checkpoint + + core_out = self.drop(core_out, training=training) + + new_mems = self._update_mems(hids, mems, mlen, qlen) + + # We transpose back here to shape [bsz, len, hidden_dim] + outputs = [tf.transpose(core_out, perm=(1, 0, 2)), new_mems] + if self.output_hidden_states: + # Add last layer and transpose to library standard shape [bsz, len, hidden_dim] + hids.append(core_out) + hids = list(tf.transpose(t, perm=(1, 0, 2)) for t in hids) + outputs.append(hids) + if self.output_attentions: + # Transpose to library standard shape [bsz, n_heads, query_seq_len, key_seq_len] + attentions = list(tf.transpose(t, perm=(2, 3, 0, 1)) for t in attentions) + outputs.append(attentions) + return outputs # last hidden state, new_mems, (all hidden states), (all attentions) + + +class TFTransfoXLPreTrainedModel(TFPreTrainedModel): """ An abstract class to handle weights initialization and a simple interface for dowloading and loading pretrained models. """ config_class = TransfoXLConfig - pretrained_model_archive_map = TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP - load_tf_weights = load_tf_weights_in_transfo_xl + pretrained_model_archive_map = TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP + load_pt_weights = load_transfo_xl_pt_weights_in_tf2 base_model_prefix = "transformer" - def _init_weight(self, weight): - if self.config.init == 'uniform': - nn.init.uniform_(weight, -self.config.init_range, self.config.init_range) - elif self.config.init == 'normal': - nn.init.normal_(weight, 0.0, self.config.init_std) - - def _init_bias(self, bias): - nn.init.constant_(bias, 0.0) - - def _init_weights(self, m): - """ Initialize the weights. - """ - classname = m.__class__.__name__ - if classname.find('Linear') != -1: - if hasattr(m, 'weight') and m.weight is not None: - self._init_weight(m.weight) - if hasattr(m, 'bias') and m.bias is not None: - self._init_bias(m.bias) - elif classname.find('AdaptiveEmbedding') != -1: - if hasattr(m, 'emb_projs'): - for i in range(len(m.emb_projs)): - if m.emb_projs[i] is not None: - nn.init.normal_(m.emb_projs[i], 0.0, self.config.proj_init_std) - elif classname.find('Embedding') != -1: - if hasattr(m, 'weight'): - self._init_weight(m.weight) - elif classname.find('ProjectedAdaptiveLogSoftmax') != -1: - if hasattr(m, 'cluster_weight') and m.cluster_weight is not None: - self._init_weight(m.cluster_weight) - if hasattr(m, 'cluster_bias') and m.cluster_bias is not None: - self._init_bias(m.cluster_bias) - if hasattr(m, 'out_projs'): - for i in range(len(m.out_projs)): - if m.out_projs[i] is not None: - nn.init.normal_(m.out_projs[i], 0.0, self.config.proj_init_std) - elif classname.find('LayerNorm') != -1: - if hasattr(m, 'weight'): - nn.init.normal_(m.weight, 1.0, self.config.init_std) - if hasattr(m, 'bias') and m.bias is not None: - self._init_bias(m.bias) - else: - if hasattr(m, 'r_emb'): - self._init_weight(m.r_emb) - if hasattr(m, 'r_w_bias'): - self._init_weight(m.r_w_bias) - if hasattr(m, 'r_r_bias'): - self._init_weight(m.r_r_bias) - if hasattr(m, 'r_bias'): - self._init_bias(m.r_bias) - - def set_num_special_tokens(self, num_special_tokens): - pass - TRANSFO_XL_START_DOCSTRING = r""" The Transformer-XL model was proposed in `Transformer-XL: Attentive Language Models Beyond a Fixed-Length Context`_ @@ -654,13 +560,13 @@ TRANSFO_XL_START_DOCSTRING = r""" The Transformer-XL model was proposed in previously computed hidden-states to attend to longer context (memory). This model also uses adaptive softmax inputs and outputs (tied). - This model is a PyTorch `torch.nn.Module`_ sub-class. Use it as a regular PyTorch Module and + This model is a PyTorch `torch.tf.keras.layers.Layer`_ sub-class. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and behavior. .. _`Transformer-XL: Attentive Language Models Beyond a Fixed-Length Context`: https://arxiv.org/abs/1901.02860 - .. _`torch.nn.Module`: + .. _`torch.tf.keras.layers.Layer`: https://pytorch.org/docs/stable/nn.html#module Parameters: @@ -690,7 +596,7 @@ TRANSFO_XL_INPUTS_DOCSTRING = r""" @add_start_docstrings("The bare Bert Model transformer outputing raw hidden-states without any specific head on top.", TRANSFO_XL_START_DOCSTRING, TRANSFO_XL_INPUTS_DOCSTRING) -class TransfoXLModel(TransfoXLPreTrainedModel): +class TFTransfoXLModel(TFTransfoXLPreTrainedModel): r""" Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)`` @@ -716,286 +622,19 @@ class TransfoXLModel(TransfoXLPreTrainedModel): last_hidden_states, mems = outputs[:2] """ - def __init__(self, config): - super(TransfoXLModel, self).__init__(config) - self.output_attentions = config.output_attentions - self.output_hidden_states = config.output_hidden_states + def __init__(self, config, *inputs, **kwargs): + super(TFTransfoXLModel, self).__init__(config, *inputs, **kwargs) + self.transformer = TFTransfoXLMainLayer(config, name='transformer') - self.n_token = config.n_token - - self.d_embed = config.d_embed - self.d_model = config.d_model - self.n_head = config.n_head - self.d_head = config.d_head - - self.word_emb = AdaptiveEmbedding(config.n_token, config.d_embed, config.d_model, config.cutoffs, - div_val=config.div_val) - - self.drop = nn.Dropout(config.dropout) - - self.n_layer = config.n_layer - - self.tgt_len = config.tgt_len - self.mem_len = config.mem_len - self.ext_len = config.ext_len - self.max_klen = config.tgt_len + config.ext_len + config.mem_len - - self.attn_type = config.attn_type - - if not config.untie_r: - self.r_w_bias = nn.Parameter(torch.FloatTensor(self.n_head, self.d_head)) - self.r_r_bias = nn.Parameter(torch.FloatTensor(self.n_head, self.d_head)) - - self.layers = nn.ModuleList() - if config.attn_type == 0: # the default attention - for i in range(config.n_layer): - self.layers.append( - RelPartialLearnableDecoderLayer( - config.n_head, config.d_model, config.d_head, config.d_inner, config.dropout, - tgt_len=config.tgt_len, ext_len=config.ext_len, mem_len=config.mem_len, - dropatt=config.dropatt, pre_lnorm=config.pre_lnorm, - r_w_bias=None if config.untie_r else self.r_w_bias, - r_r_bias=None if config.untie_r else self.r_r_bias, - output_attentions=self.output_attentions) - ) - elif config.attn_type == 1: # learnable embeddings - for i in range(config.n_layer): - self.layers.append( - RelLearnableDecoderLayer( - config.n_head, config.d_model, config.d_head, config.d_inner, config.dropout, - tgt_len=config.tgt_len, ext_len=config.ext_len, mem_len=config.mem_len, - dropatt=config.dropatt, pre_lnorm=config.pre_lnorm, - r_w_bias=None if config.untie_r else self.r_w_bias, - r_r_bias=None if config.untie_r else self.r_r_bias, - output_attentions=self.output_attentions) - ) - elif config.attn_type in [2, 3]: # absolute embeddings - for i in range(config.n_layer): - self.layers.append( - DecoderLayer( - config.n_head, config.d_model, config.d_head, config.d_inner, config.dropout, - dropatt=config.dropatt, pre_lnorm=config.pre_lnorm, - r_w_bias=None if config.untie_r else self.r_w_bias, - r_r_bias=None if config.untie_r else self.r_r_bias, - output_attentions=self.output_attentions) - ) - - self.same_length = config.same_length - self.clamp_len = config.clamp_len - - if self.attn_type == 0: # default attention - self.pos_emb = PositionalEmbedding(self.d_model) - elif self.attn_type == 1: # learnable - self.r_emb = nn.Parameter(torch.FloatTensor( - self.n_layer, self.max_klen, self.n_head, self.d_head)) - self.r_bias = nn.Parameter(torch.FloatTensor( - self.n_layer, self.max_klen, self.n_head)) - elif self.attn_type == 2: # absolute standard - self.pos_emb = PositionalEmbedding(self.d_model) - elif self.attn_type == 3: # absolute deeper SA - self.r_emb = nn.Parameter(torch.FloatTensor( - self.n_layer, self.max_klen, self.n_head, self.d_head)) - - self.init_weights() - - def _resize_token_embeddings(self, new_num_tokens): - return self.word_emb - - def backward_compatible(self): - self.sample_softmax = -1 - - def reset_length(self, tgt_len, ext_len, mem_len): - self.tgt_len = tgt_len - self.mem_len = mem_len - self.ext_len = ext_len - - def _prune_heads(self, heads): - logger.info("Head pruning is not implemented for Transformer-XL model") - pass - - def init_mems(self, data): - if self.mem_len > 0: - mems = [] - param = next(self.parameters()) - for i in range(self.n_layer): - empty = torch.zeros(self.mem_len, data.size(1), self.config.d_model, - dtype=param.dtype, device=param.device) - mems.append(empty) - - return mems - else: - return None - - def _update_mems(self, hids, mems, qlen, mlen): - # does not deal with None - if mems is None: return None - - # mems is not None - assert len(hids) == len(mems), 'len(hids) != len(mems)' - - # There are `mlen + qlen` steps that can be cached into mems - # For the next step, the last `ext_len` of the `qlen` tokens - # will be used as the extended context. Hence, we only cache - # the tokens from `mlen + qlen - self.ext_len - self.mem_len` - # to `mlen + qlen - self.ext_len`. - with torch.no_grad(): - new_mems = [] - end_idx = mlen + max(0, qlen - 0 - self.ext_len) - beg_idx = max(0, end_idx - self.mem_len) - for i in range(len(hids)): - - cat = torch.cat([mems[i], hids[i]], dim=0) - new_mems.append(cat[beg_idx:end_idx].detach()) - - return new_mems - - def _forward(self, dec_inp, mems=None, head_mask=None): - qlen, bsz = dec_inp.size() - - # Prepare head mask if needed - # 1.0 in head_mask indicate we keep the head - # attention_probs has shape bsz x n_heads x N x N - # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] (a head_mask for each layer) - # and head_mask is converted to shape [num_hidden_layers x qlen x klen x bsz x n_head] - if head_mask is not None: - if head_mask.dim() == 1: - head_mask = head_mask.unsqueeze(0).unsqueeze(0).unsqueeze(0).unsqueeze(0) - head_mask = head_mask.expand(self.n_layer, -1, -1, -1, -1) - elif head_mask.dim() == 2: - head_mask = head_mask.unsqueeze(1).unsqueeze(1).unsqueeze(1) - head_mask = head_mask.to(dtype=next(self.parameters()).dtype) # switch to fload if need + fp16 compatibility - else: - head_mask = [None] * self.n_layer - - word_emb = self.word_emb(dec_inp) - - mlen = mems[0].size(0) if mems is not None else 0 - klen = mlen + qlen - if self.same_length: - all_ones = word_emb.new_ones((qlen, klen), dtype=torch.uint8) - mask_len = klen - self.mem_len - if mask_len > 0: - mask_shift_len = qlen - mask_len - else: - mask_shift_len = qlen - dec_attn_mask = (torch.triu(all_ones, 1+mlen) - + torch.tril(all_ones, -mask_shift_len))[:, :, None] # -1 - else: - dec_attn_mask = torch.triu( - word_emb.new_ones((qlen, klen), dtype=torch.uint8), diagonal=1+mlen)[:,:,None] - - hids = [] - attentions = [] - if self.attn_type == 0: # default - pos_seq = torch.arange(klen-1, -1, -1.0, device=word_emb.device, - dtype=word_emb.dtype) - if self.clamp_len > 0: - pos_seq.clamp_(max=self.clamp_len) - pos_emb = self.pos_emb(pos_seq) - - core_out = self.drop(word_emb) - pos_emb = self.drop(pos_emb) - - for i, layer in enumerate(self.layers): - hids.append(core_out) - mems_i = None if mems is None else mems[i] - layer_outputs = layer(core_out, pos_emb, dec_attn_mask=dec_attn_mask, - mems=mems_i, head_mask=head_mask[i]) - core_out = layer_outputs[0] - if self.output_attentions: - attentions.append(layer_outputs[1]) - elif self.attn_type == 1: # learnable - core_out = self.drop(word_emb) - for i, layer in enumerate(self.layers): - hids.append(core_out) - if self.clamp_len > 0: - r_emb = self.r_emb[i][-self.clamp_len :] - r_bias = self.r_bias[i][-self.clamp_len :] - else: - r_emb, r_bias = self.r_emb[i], self.r_bias[i] - - mems_i = None if mems is None else mems[i] - layer_outputs = layer(core_out, r_emb, self.r_w_bias[i], - r_bias, dec_attn_mask=dec_attn_mask, - mems=mems_i, head_mask=head_mask[i]) - core_out = layer_outputs[0] - if self.output_attentions: - attentions.append(layer_outputs[1]) - elif self.attn_type == 2: # absolute - pos_seq = torch.arange(klen - 1, -1, -1.0, device=word_emb.device, - dtype=word_emb.dtype) - if self.clamp_len > 0: - pos_seq.clamp_(max=self.clamp_len) - pos_emb = self.pos_emb(pos_seq) - - core_out = self.drop(word_emb + pos_emb[-qlen:]) - - for i, layer in enumerate(self.layers): - hids.append(core_out) - mems_i = None if mems is None else mems[i] - if mems_i is not None and i == 0: - mems_i += pos_emb[:mlen] - layer_outputs = layer(core_out, dec_attn_mask=dec_attn_mask, - mems=mems_i, head_mask=head_mask[i]) - core_out = layer_outputs[0] - if self.output_attentions: - attentions.append(layer_outputs[1]) - elif self.attn_type == 3: - core_out = self.drop(word_emb) - - for i, layer in enumerate(self.layers): - hids.append(core_out) - mems_i = None if mems is None else mems[i] - if mems_i is not None and mlen > 0: - cur_emb = self.r_emb[i][:-qlen] - cur_size = cur_emb.size(0) - if cur_size < mlen: - cur_emb_pad = cur_emb[0:1].expand(mlen-cur_size, -1, -1) - cur_emb = torch.cat([cur_emb_pad, cur_emb], 0) - else: - cur_emb = cur_emb[-mlen:] - mems_i += cur_emb.view(mlen, 1, -1) - core_out += self.r_emb[i][-qlen:].view(qlen, 1, -1) - - layer_outputs = layer(core_out, dec_attn_mask=dec_attn_mask, - mems=mems_i, head_mask=head_mask[i]) - core_out = layer_outputs[0] - if self.output_attentions: - attentions.append(layer_outputs[1]) - - core_out = self.drop(core_out) - - new_mems = self._update_mems(hids, mems, mlen, qlen) - - # We transpose back here to shape [bsz, len, hidden_dim] - outputs = [core_out.transpose(0, 1).contiguous(), new_mems] - if self.output_hidden_states: - # Add last layer and transpose to library standard shape [bsz, len, hidden_dim] - hids.append(core_out) - hids = list(t.transpose(0, 1).contiguous() for t in hids) - outputs.append(hids) - if self.output_attentions: - # Transpose to library standard shape [bsz, n_heads, query_seq_len, key_seq_len] - attentions = list(t.permute(2, 3, 0, 1).contiguous() for t in attentions) - outputs.append(attentions) - return outputs # last hidden state, new_mems, (all hidden states), (all attentions) - - def forward(self, input_ids, mems=None, head_mask=None): - # the original code for Transformer-XL used shapes [len, bsz] but we want a unified interface in the library - # so we transpose here from shape [bsz, len] to shape [len, bsz] - input_ids = input_ids.transpose(0, 1).contiguous() - - if mems is None: - mems = self.init_mems(input_ids) - outputs = self._forward(input_ids, mems=mems, head_mask=head_mask) - - return outputs # last hidden state, new_mems, (all hidden states), (all attentions) + def call(self, inputs, training=False, **kwargs): + outputs = self.transformer(inputs, training=training, **kwargs) + return outputs @add_start_docstrings("""The Transformer-XL Model with a language modeling head on top (adaptive softmax with weights tied to the adaptive input embeddings)""", TRANSFO_XL_START_DOCSTRING, TRANSFO_XL_INPUTS_DOCSTRING) -class TransfoXLLMHeadModel(TransfoXLPreTrainedModel): +class TFTransfoXLLMHeadModel(TFTransfoXLPreTrainedModel): r""" **lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: Labels for language modeling. @@ -1032,46 +671,16 @@ class TransfoXLLMHeadModel(TransfoXLPreTrainedModel): """ def __init__(self, config): - super(TransfoXLLMHeadModel, self).__init__(config) - self.transformer = TransfoXLModel(config) + super(TFTransfoXLLMHeadModel, self).__init__(config) + self.transformer = TFTransfoXLMainLayer(config, name='transformer') self.sample_softmax = config.sample_softmax # use sampled softmax if config.sample_softmax > 0: - self.out_layer = nn.Linear(config.d_model, config.n_token) - self.sampler = LogUniformSampler(config.n_token, config.sample_softmax) + raise NotImplementedError # use adaptive softmax (including standard softmax) else: - self.crit = ProjectedAdaptiveLogSoftmax(config.n_token, config.d_embed, config.d_model, - config.cutoffs, div_val=config.div_val) - self.init_weights() - self.tie_weights() - - def tie_weights(self): - """ - Run this to be sure output and input (adaptive) softmax weights are tied - """ - # sampled softmax - if self.sample_softmax > 0: - if self.config.tie_weight: - self.out_layer.weight = self.transformer.word_emb.weight - # adaptive softmax (including standard softmax) - else: - if self.config.tie_weight: - for i in range(len(self.crit.out_layers)): - self._tie_or_clone_weights(self.crit.out_layers[i], - self.transformer.word_emb.emb_layers[i]) - if self.config.tie_projs: - for i, tie_proj in enumerate(self.config.tie_projs): - if tie_proj and self.config.div_val == 1 and self.config.d_model != self.config.d_embed: - if self.config.torchscript: - self.crit.out_projs[i] = nn.Parameter(self.transformer.word_emb.emb_projs[0].clone()) - else: - self.crit.out_projs[i] = self.transformer.word_emb.emb_projs[0] - elif tie_proj and self.config.div_val != 1: - if self.config.torchscript: - self.crit.out_projs[i] = nn.Parameter(self.transformer.word_emb.emb_projs[i].clone()) - else: - self.crit.out_projs[i] = self.transformer.word_emb.emb_projs[i] + self.crit = TFAdaptiveSoftmaxMask(config.n_token, config.d_embed, config.d_model, + config.cutoffs, div_val=config.div_val, name='crit') def reset_length(self, tgt_len, ext_len, mem_len): self.transformer.reset_length(tgt_len, ext_len, mem_len) @@ -1079,30 +688,36 @@ class TransfoXLLMHeadModel(TransfoXLPreTrainedModel): def init_mems(self, data): return self.transformer.init_mems(data) - def forward(self, input_ids, mems=None, head_mask=None, labels=None): - bsz = input_ids.size(0) - tgt_len = input_ids.size(1) + def call(self, inputs, training=False): + if not isinstance(inputs, (dict, tuple, list)): + input_ids = inputs + mems, head_mask, labels = None, None, None + elif isinstance(inputs, (tuple, list)): + input_ids = inputs[0] + mems = inputs[1] if len(inputs) > 1 else None + head_mask = inputs[2] if len(inputs) > 2 else None + labels = inputs[3] if len(inputs) > 3 else None + assert len(inputs) <= 4, "Too many inputs." + else: + input_ids = inputs.get('input_ids') + mems = inputs.get('mems', None) + head_mask = inputs.get('head_mask', None) + labels = inputs.get('labels', None) + assert len(inputs) <= 4, "Too many inputs." - transformer_outputs = self.transformer(input_ids, mems=mems, head_mask=head_mask) + bsz, tgt_len = shape_list(input_ids)[:2] + + transformer_outputs = self.transformer([input_ids, mems, head_mask], training=training) last_hidden = transformer_outputs[0] pred_hid = last_hidden[:, -tgt_len:] outputs = transformer_outputs[1:] - if self.sample_softmax > 0 and self.training: - assert self.config.tie_weight - logit = sample_logits(self.transformer.word_emb, self.out_layer.bias, labels, pred_hid, self.sampler) - softmax_output = -F.log_softmax(logit, -1)[:, :, 0] - outputs = [softmax_output] + outputs - if labels is not None: - # TODO: This is not implemented - raise NotImplementedError + if self.sample_softmax > 0 and training: + raise NotImplementedError else: - softmax_output = self.crit(pred_hid.view(-1, pred_hid.size(-1)), labels) - if labels is None: - softmax_output = softmax_output.view(bsz, tgt_len, -1) - outputs = [softmax_output] + outputs - else: - softmax_output = softmax_output.view(bsz, tgt_len) - outputs = [softmax_output, None] + outputs + # pred_hid = tf.reshape(pred_hid, (-1, shape_list(pred_hid)[-1])) + softmax_output = self.crit([pred_hid, labels], training=training) + # softmax_output = tf.reshape(softmax_output, (bsz, tgt_len, -1)) + outputs = [softmax_output] + outputs - return outputs # (loss), logits or None if labels is not None (speed up adaptive softmax), new_mems, (all hidden states), (all attentions) + return outputs # logits, new_mems, (all hidden states), (all attentions) diff --git a/pytorch_transformers/modeling_tf_transfo_xl_utilities.py b/pytorch_transformers/modeling_tf_transfo_xl_utilities.py new file mode 100644 index 0000000000..d313ba177e --- /dev/null +++ b/pytorch_transformers/modeling_tf_transfo_xl_utilities.py @@ -0,0 +1,279 @@ +# coding=utf-8 +# Copyright 2018 Google AI, Google Brain and Carnegie Mellon University Authors and the HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Utilities for PyTorch Transformer XL model. + Directly adapted from https://github.com/kimiyoung/transformer-xl. +""" + +from collections import defaultdict + +import numpy as np + +import tensorflow as tf + +from .modeling_tf_utils import shape_list + +class TFAdaptiveSoftmaxMask(tf.keras.layers.Layer): + def __init__(self, n_token, d_embed, d_proj, cutoffs, div_val=1, + keep_order=False, **kwargs): + super(TFAdaptiveSoftmaxMask, self).__init__(**kwargs) + + self.n_token = n_token + self.d_embed = d_embed + self.d_proj = d_proj + + self.cutoffs = cutoffs + [n_token] + self.cutoff_ends = [0] + self.cutoffs + self.div_val = div_val + + self.shortlist_size = self.cutoffs[0] + self.n_clusters = len(self.cutoffs) - 1 + self.head_size = self.shortlist_size + self.n_clusters + self.keep_order = keep_order + + self.out_layers = [] + self.out_projs = [] + + def build(self, input_shape): + if self.n_clusters > 0: + self.cluster_weight = self.add_weight(shape=(self.n_clusters, self.d_embed), + initializer='zeros', + trainable=True, + name='cluster_weight') + self.cluster_bias = self.add_weight(shape=(self.n_clusters,), + initializer='zeros', + trainable=True, + name='cluster_bias') + + if self.div_val == 1: + for i in range(len(self.cutoffs)): + if self.d_proj != self.d_embed: + weight = self.add_weight(shape=(self.d_embed, self.d_proj), + initializer='zeros', + trainable=True, + name='out_projs_._{}'.format(i)) + self.out_projs.append(weight) + else: + self.out_projs.append(None) + weight = self.add_weight(shape=(self.n_token, self.d_embed,), + initializer='zeros', + trainable=True, + name='out_layers_._{}_._weight'.format(i)) + bias = self.add_weight(shape=(self.n_token,), + initializer='zeros', + trainable=True, + name='out_layers_._{}_._bias'.format(i)) + self.out_layers.append((weight, bias)) + else: + for i in range(len(self.cutoffs)): + l_idx, r_idx = self.cutoff_ends[i], self.cutoff_ends[i+1] + d_emb_i = self.d_embed // (self.div_val ** i) + + weight = self.add_weight(shape=(d_emb_i, self.d_proj), + initializer='zeros', + trainable=True, + name='out_projs_._{}'.format(i)) + self.out_projs.append(weight) + weight = self.add_weight(shape=(r_idx-l_idx, d_emb_i,), + initializer='zeros', + trainable=True, + name='out_layers_._{}_._weight'.format(i)) + bias = self.add_weight(shape=(r_idx-l_idx,), + initializer='zeros', + trainable=True, + name='out_layers_._{}_._bias'.format(i)) + self.out_layers.append((weight, bias)) + super(TFAdaptiveSoftmaxMask, self).build(input_shape) + + @staticmethod + def _logit(x, W, b, proj=None): + y = x + if proj is not None: + y = tf.einsum('ibd,ed->ibe', y, proj) + return tf.einsum('ibd,nd->ibn', y, W) + b + + @staticmethod + def _gather_logprob(logprob, target): + lp_size = tf.shape(logprob) + r = tf.range(lp_size[0]) + idx = tf.stack([r, target], 1) + return tf.gather_nd(logprob, idx) + + def call(self, inputs, return_mean=True, training=False): + hidden, target = inputs + head_logprob = 0 + if self.n_clusters == 0: + softmax_b = tf.get_variable('bias', [n_token], initializer=tf.zeros_initializer()) + output = self._logit(hidden, self.out_layers[0][0], self.out_layers[0][1], self.out_projs[0]) + if target is not None: + loss = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=target, logits=output) + out = tf.nn.log_softmax(output, axis=-1) + else: + hidden_sizes = shape_list(hidden) + out = [] + loss = tf.zeros(hidden_sizes[:2], dtype=tf.float32) + for i in range(len(self.cutoffs)): + l_idx, r_idx = self.cutoff_ends[i], self.cutoff_ends[i + 1] + if target is not None: + mask = (target >= l_idx) & (target < r_idx) + mask_idx = tf.where(mask) + cur_target = tf.boolean_mask(target, mask) - l_idx + + if self.div_val == 1: + cur_W = self.out_layers[0][0][l_idx:r_idx] + cur_b = self.out_layers[0][1][l_idx:r_idx] + else: + cur_W = self.out_layers[i][0] + cur_b = self.out_layers[i][1] + + if i == 0: + cur_W = tf.concat([cur_W, self.cluster_weight], 0) + cur_b = tf.concat([cur_b, self.cluster_bias], 0) + + head_logit = self._logit(hidden, cur_W, cur_b, self.out_projs[0]) + head_logprob = tf.nn.log_softmax(head_logit) + out.append(head_logprob[..., :self.cutoffs[0]]) + if target is not None: + cur_head_logprob = tf.boolean_mask(head_logprob, mask) + cur_logprob = self._gather_logprob(cur_head_logprob, cur_target) + else: + tail_logit = self._logit(hidden, cur_W, cur_b, self.out_projs[i]) + tail_logprob = tf.nn.log_softmax(tail_logit) + cluster_prob_idx = self.cutoffs[0] + i - 1 # No probability for the head cluster + logprob_i = head_logprob[..., cluster_prob_idx, None] + tail_logprob + out.append(logprob_i) + if target is not None: + cur_head_logprob = tf.boolean_mask(head_logprob, mask) + cur_tail_logprob = tf.boolean_mask(tail_logprob, mask) + cur_logprob = self._gather_logprob(cur_tail_logprob, cur_target) + cur_logprob += cur_head_logprob[:, self.cutoff_ends[1] + i - 1] + if target is not None: + loss += tf.scatter_nd(mask_idx, -cur_logprob, tf.cast(tf.shape(loss), dtype=tf.int64)) + out = tf.concat(out, axis=-1) + + if target is not None: + if return_mean: + loss = tf.reduce_mean(loss) + # Add the training-time loss value to the layer using `self.add_loss()`. + self.add_loss(loss) + + # Log the loss as a metric (we could log arbitrary metrics, + # including different metrics for training and inference. + self.add_metric(loss, name=self.name, aggregation='mean' if return_mean else '') + + return out + + +def mul_adaptive_logsoftmax(hidden, target, n_token, d_embed, d_proj, cutoffs, + params, tie_projs, + initializer=None, proj_initializer=None, + div_val=1, perms=None, proj_same_dim=True, + scope='adaptive_softmax', + **kwargs): + def _logit(x, W, b, proj): + y = x + if x.shape.ndims == 3: + if proj is not None: + y = tf.einsum('ibd,ed->ibe', y, proj) + return tf.einsum('ibd,nd->ibn', y, W) + b + else: + if proj is not None: + y = tf.einsum('id,ed->ie', y, proj) + return tf.einsum('id,nd->in', y, W) + b + + params_W, params_projs = params[0], params[1] + + with tf.variable_scope(scope): + if len(cutoffs) == 0: + softmax_b = tf.get_variable('bias', [n_token], + initializer=tf.zeros_initializer()) + output = _logit(hidden, params_W, softmax_b, params_projs) + nll = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=target, + logits=output) + nll = tf.reduce_mean(nll) + else: + total_loss, total_cnt = 0, 0 + cutoff_ends = [0] + cutoffs + [n_token] + for i in range(len(cutoff_ends) - 1): + with tf.variable_scope('cutoff_{}'.format(i)): + l_idx, r_idx = cutoff_ends[i], cutoff_ends[i + 1] + + cur_d_embed = d_embed // (div_val ** i) + + if div_val == 1: + cur_W = params_W[l_idx: r_idx] + else: + cur_W = params_W[i] + cur_b = tf.get_variable('b', [r_idx - l_idx], + initializer=tf.zeros_initializer()) + if tie_projs[i]: + if div_val == 1: + cur_proj = params_projs + else: + cur_proj = params_projs[i] + else: + if (div_val == 1 or not proj_same_dim) and d_proj == cur_d_embed: + cur_proj = None + else: + cur_proj = tf.get_variable('proj', [cur_d_embed, d_proj], + initializer=proj_initializer) + + if i == 0: + cluster_W = tf.get_variable('cluster_W', [len(cutoffs), d_embed], + initializer=tf.zeros_initializer()) + cluster_b = tf.get_variable('cluster_b', [len(cutoffs)], + initializer=tf.zeros_initializer()) + cur_W = tf.concat([cur_W, cluster_W], 0) + cur_b = tf.concat([cur_b, cluster_b], 0) + + head_logit = _logit(hidden, cur_W, cur_b, cur_proj) + + head_target = kwargs.get("head_target") + head_nll = tf.nn.sparse_softmax_cross_entropy_with_logits( + labels=head_target, + logits=head_logit) + + masked_loss = head_nll * perms[i] + total_loss += tf.reduce_sum(masked_loss) + total_cnt += tf.reduce_sum(perms[i]) + + # head_logprob = tf.nn.log_softmax(head_logit) + + # final_logprob = head_logprob * perms[i][:, :, None] + # final_target = tf.one_hot(target, tf.shape(head_logprob)[2]) + # total_loss -= tf.einsum('ibn,ibn->', final_logprob, final_target) + # total_cnt += tf.reduce_sum(perms[i]) + else: + cur_head_nll = tf.einsum('ib,ibk->k', head_nll, perms[i]) + + cur_hidden = tf.einsum('ibd,ibk->kd', hidden, perms[i]) + tail_logit = _logit(cur_hidden, cur_W, cur_b, cur_proj) + + tail_target = tf.einsum('ib,ibk->k', tf.to_float(target - l_idx), + perms[i]) + tail_nll = tf.nn.sparse_softmax_cross_entropy_with_logits( + labels=tf.to_int32(tail_target), + logits=tail_logit) + + sum_nll = cur_head_nll + tail_nll + mask = tf.reduce_sum(perms[i], [0, 1]) + + masked_loss = sum_nll * mask + total_loss += tf.reduce_sum(masked_loss) + total_cnt += tf.reduce_sum(mask) + + nll = total_loss / total_cnt + + return nll \ No newline at end of file diff --git a/pytorch_transformers/modeling_tf_xlm.py b/pytorch_transformers/modeling_tf_xlm.py index b0c1b594cf..7d04443745 100644 --- a/pytorch_transformers/modeling_tf_xlm.py +++ b/pytorch_transformers/modeling_tf_xlm.py @@ -261,8 +261,8 @@ class TFXLMMainLayer(tf.keras.layers.Layer): self.ffns = [] self.layer_norm2 = [] # if self.is_decoder: - # self.layer_norm15 = tf.keras.layers.LayerList() - # self.encoder_attn = tf.keras.layers.LayerList() + # self.layer_norm15 = [] + # self.encoder_attn = [] for i in range(self.n_layers): self.attentions.append(TFMultiHeadAttention(self.n_heads, self.dim, config=config, name='attentions_._{}'.format(i))) diff --git a/pytorch_transformers/modeling_transfo_xl.py b/pytorch_transformers/modeling_transfo_xl.py index 7ad9d10891..557694e84f 100644 --- a/pytorch_transformers/modeling_transfo_xl.py +++ b/pytorch_transformers/modeling_transfo_xl.py @@ -229,102 +229,11 @@ class PositionwiseFF(nn.Module): return output - -class MultiHeadAttn(nn.Module): - def __init__(self, n_head, d_model, d_head, dropout, dropatt=0, - pre_lnorm=False, r_r_bias=None, r_w_bias=None, output_attentions=False): - super(MultiHeadAttn, self).__init__() - - self.output_attentions = output_attentions - self.n_head = n_head - self.d_model = d_model - self.d_head = d_head - self.dropout = dropout - - self.q_net = nn.Linear(d_model, n_head * d_head, bias=False) - self.kv_net = nn.Linear(d_model, 2 * n_head * d_head, bias=False) - - self.drop = nn.Dropout(dropout) - self.dropatt = nn.Dropout(dropatt) - self.o_net = nn.Linear(n_head * d_head, d_model, bias=False) - - self.layer_norm = nn.LayerNorm(d_model) - - self.scale = 1 / (d_head ** 0.5) - - self.pre_lnorm = pre_lnorm - - if r_r_bias is None or r_w_bias is None: # Biases are not shared - self.r_r_bias = nn.Parameter(torch.FloatTensor(self.n_head, self.d_head)) - self.r_w_bias = nn.Parameter(torch.FloatTensor(self.n_head, self.d_head)) - else: - self.r_r_bias = r_r_bias - self.r_w_bias = r_w_bias - - def forward(self, h, attn_mask=None, mems=None, head_mask=None): - ##### multihead attention - # [hlen x bsz x n_head x d_head] - - if mems is not None: - c = torch.cat([mems, h], 0) - else: - c = h - - if self.pre_lnorm: - ##### layer normalization - c = self.layer_norm(c) - - head_q = self.q_net(h) - head_k, head_v = torch.chunk(self.kv_net(c), 2, -1) - - head_q = head_q.view(h.size(0), h.size(1), self.n_head, self.d_head) - head_k = head_k.view(c.size(0), c.size(1), self.n_head, self.d_head) - head_v = head_v.view(c.size(0), c.size(1), self.n_head, self.d_head) - - # [qlen x klen x bsz x n_head] - attn_score = torch.einsum('ibnd,jbnd->ijbn', (head_q, head_k)) - attn_score.mul_(self.scale) - if attn_mask is not None and torch.sum(attn_mask).item(): - attn_mask = (attn_mask == 1) # Switch to bool - if attn_mask.dim() == 2: - attn_score.masked_fill_(attn_mask[None,:,:,None], -float('inf')) - elif attn_mask.dim() == 3: - attn_score.masked_fill_(attn_mask[:,:,:,None], -float('inf')) - - # [qlen x klen x bsz x n_head] - attn_prob = F.softmax(attn_score, dim=1) - attn_prob = self.dropatt(attn_prob) - - # Mask heads if we want to - if head_mask is not None: - attn_prob = attn_prob * head_mask - - # [qlen x klen x bsz x n_head] + [klen x bsz x n_head x d_head] -> [qlen x bsz x n_head x d_head] - attn_vec = torch.einsum('ijbn,jbnd->ibnd', (attn_prob, head_v)) - attn_vec = attn_vec.contiguous().view( - attn_vec.size(0), attn_vec.size(1), self.n_head * self.d_head) - - ##### linear projection - attn_out = self.o_net(attn_vec) - attn_out = self.drop(attn_out) - - if self.pre_lnorm: - ##### residual connection - outputs = [h + attn_out] - else: - ##### residual connection + layer normalization - outputs = [self.layer_norm(h + attn_out)] - - if self.output_attentions: - outputs.append(attn_prob) - - return outputs - -class RelMultiHeadAttn(nn.Module): +class RelPartialLearnableMultiHeadAttn(nn.Module): def __init__(self, n_head, d_model, d_head, dropout, dropatt=0, tgt_len=None, ext_len=None, mem_len=None, pre_lnorm=False, r_r_bias=None, r_w_bias=None, output_attentions=False): - super(RelMultiHeadAttn, self).__init__() + super(RelPartialLearnableMultiHeadAttn, self).__init__() self.output_attentions = output_attentions self.n_head = n_head @@ -351,36 +260,9 @@ class RelMultiHeadAttn(nn.Module): self.r_r_bias = r_r_bias self.r_w_bias = r_w_bias - def _parallelogram_mask(self, h, w, left=False): - mask = torch.ones((h, w)).byte() - m = min(h, w) - mask[:m,:m] = torch.triu(mask[:m,:m]) - mask[-m:,-m:] = torch.tril(mask[-m:,-m:]) + self.r_net = nn.Linear(self.d_model, self.n_head * self.d_head, bias=False) - if left: - return mask - else: - return mask.flip(0) - - def _shift(self, x, qlen, klen, mask, left=False): - if qlen > 1: - zero_pad = torch.zeros((x.size(0), qlen-1, x.size(2), x.size(3)), - device=x.device, dtype=x.dtype) - else: - zero_pad = torch.zeros(0, device=x.device, dtype=x.dtype) - - if left: - mask = mask.flip(1) - x_padded = torch.cat([zero_pad, x], dim=1).expand(qlen, -1, -1, -1) - else: - x_padded = torch.cat([x, zero_pad], dim=1).expand(qlen, -1, -1, -1) - - x = x_padded.masked_select(mask[:,:,None,None]) \ - .view(qlen, klen, x.size(2), x.size(3)) - - return x - - def _rel_shift(self, x, zero_triu=False): + def _rel_shift(self, x): zero_pad_shape = (x.size(0), 1) + x.size()[2:] zero_pad = torch.zeros(zero_pad_shape, device=x.device, dtype=x.dtype) x_padded = torch.cat([zero_pad, x], dim=1) @@ -390,21 +272,8 @@ class RelMultiHeadAttn(nn.Module): x = x_padded[1:].view_as(x) - if zero_triu: - ones = torch.ones((x.size(0), x.size(1))) - x = x * torch.tril(ones, x.size(1) - x.size(0))[:,:,None,None] - return x - def forward(self, w, r, attn_mask=None, mems=None): - raise NotImplementedError - -class RelPartialLearnableMultiHeadAttn(RelMultiHeadAttn): - def __init__(self, *args, **kwargs): - super(RelPartialLearnableMultiHeadAttn, self).__init__(*args, **kwargs) - - self.r_net = nn.Linear(self.d_model, self.n_head * self.d_head, bias=False) - def forward(self, w, r, attn_mask=None, mems=None, head_mask=None): qlen, rlen, bsz = w.size(0), r.size(0), w.size(1) @@ -488,138 +357,6 @@ class RelPartialLearnableMultiHeadAttn(RelMultiHeadAttn): return outputs -class RelLearnableMultiHeadAttn(RelMultiHeadAttn): - def __init__(self, *args, **kwargs): - super(RelLearnableMultiHeadAttn, self).__init__(*args, **kwargs) - - def forward(self, w, r_emb, r_w_bias, r_bias, attn_mask=None, mems=None, head_mask=None): - # r_emb: [klen, n_head, d_head], used for term B - # r_w_bias: [n_head, d_head], used for term C - # r_bias: [klen, n_head], used for term D - - qlen, bsz = w.size(0), w.size(1) - - if mems is not None: - cat = torch.cat([mems, w], 0) - if self.pre_lnorm: - w_heads = self.qkv_net(self.layer_norm(cat)) - else: - w_heads = self.qkv_net(cat) - w_head_q, w_head_k, w_head_v = torch.chunk(w_heads, 3, dim=-1) - - w_head_q = w_head_q[-qlen:] - else: - if self.pre_lnorm: - w_heads = self.qkv_net(self.layer_norm(w)) - else: - w_heads = self.qkv_net(w) - w_head_q, w_head_k, w_head_v = torch.chunk(w_heads, 3, dim=-1) - - klen = w_head_k.size(0) - - w_head_q = w_head_q.view(qlen, bsz, self.n_head, self.d_head) - w_head_k = w_head_k.view(klen, bsz, self.n_head, self.d_head) - w_head_v = w_head_v.view(klen, bsz, self.n_head, self.d_head) - - if klen > r_emb.size(0): - r_emb_pad = r_emb[0:1].expand(klen-r_emb.size(0), -1, -1) - r_emb = torch.cat([r_emb_pad, r_emb], 0) - r_bias_pad = r_bias[0:1].expand(klen-r_bias.size(0), -1) - r_bias = torch.cat([r_bias_pad, r_bias], 0) - else: - r_emb = r_emb[-klen:] - r_bias = r_bias[-klen:] - - #### compute attention score - rw_head_q = w_head_q + r_w_bias[None] # qlen x bsz x n_head x d_head - - AC = torch.einsum('ibnd,jbnd->ijbn', (rw_head_q, w_head_k)) # qlen x klen x bsz x n_head - B_ = torch.einsum('ibnd,jnd->ijbn', (w_head_q, r_emb)) # qlen x klen x bsz x n_head - D_ = r_bias[None, :, None] # 1 x klen x 1 x n_head - BD = self._rel_shift(B_ + D_) - - # [qlen x klen x bsz x n_head] - attn_score = AC + BD - attn_score.mul_(self.scale) - - #### compute attention probability - if attn_mask is not None and torch.sum(attn_mask).item(): - attn_mask = (attn_mask == 1) # Switch to bool - if attn_mask.dim() == 2: - attn_score.masked_fill_(attn_mask[None,:,:,None], -float('inf')) - elif attn_mask.dim() == 3: - attn_score.masked_fill_(attn_mask[:,:,:,None], -float('inf')) - - # [qlen x klen x bsz x n_head] - attn_prob = F.softmax(attn_score, dim=1) - attn_prob = self.dropatt(attn_prob) - - if head_mask is not None: - attn_prob = attn_prob * head_mask - - #### compute attention vector - attn_vec = torch.einsum('ijbn,jbnd->ibnd', (attn_prob, w_head_v)) - - # [qlen x bsz x n_head x d_head] - attn_vec = attn_vec.contiguous().view( - attn_vec.size(0), attn_vec.size(1), self.n_head * self.d_head) - - ##### linear projection - attn_out = self.o_net(attn_vec) - attn_out = self.drop(attn_out) - - if self.pre_lnorm: - ##### residual connection - outputs = [w + attn_out] - else: - ##### residual connection + layer normalization - outputs = [self.layer_norm(w + attn_out)] - - if self.output_attentions: - outputs.append(attn_prob) - - return outputs - - - -class DecoderLayer(nn.Module): - def __init__(self, n_head, d_model, d_head, d_inner, dropout, **kwargs): - super(DecoderLayer, self).__init__() - - self.dec_attn = MultiHeadAttn(n_head, d_model, d_head, dropout, **kwargs) - self.pos_ff = PositionwiseFF(d_model, d_inner, dropout, - pre_lnorm=kwargs.get('pre_lnorm')) - - def forward(self, dec_inp, dec_attn_mask=None, mems=None, head_mask=None): - - attn_outputs = self.dec_attn(dec_inp, attn_mask=dec_attn_mask, - mems=mems, head_mask=head_mask) - ff_output = self.pos_ff(attn_outputs[0]) - - outputs = [ff_output] + attn_outputs[1:] - - return outputs - -class RelLearnableDecoderLayer(nn.Module): - def __init__(self, n_head, d_model, d_head, d_inner, dropout, - **kwargs): - super(RelLearnableDecoderLayer, self).__init__() - - self.dec_attn = RelLearnableMultiHeadAttn(n_head, d_model, d_head, dropout, - **kwargs) - self.pos_ff = PositionwiseFF(d_model, d_inner, dropout, - pre_lnorm=kwargs.get('pre_lnorm')) - - def forward(self, dec_inp, r_emb, r_w_bias, r_bias, dec_attn_mask=None, mems=None, head_mask=None): - - attn_outputs = self.dec_attn(dec_inp, r_emb, r_w_bias, r_bias, - attn_mask=dec_attn_mask, - mems=mems, head_mask=head_mask) - ff_output = self.pos_ff(attn_outputs[0]) - - outputs = [ff_output] + attn_outputs[1:] - - return outputs class RelPartialLearnableDecoderLayer(nn.Module): def __init__(self, n_head, d_model, d_head, d_inner, dropout, @@ -643,7 +380,6 @@ class RelPartialLearnableDecoderLayer(nn.Module): return outputs - class AdaptiveEmbedding(nn.Module): def __init__(self, n_token, d_embed, d_proj, cutoffs, div_val=1, sample_softmax=False): @@ -767,9 +503,6 @@ class TransfoXLPreTrainedModel(PreTrainedModel): if hasattr(m, 'r_bias'): self._init_bias(m.r_bias) - def set_num_special_tokens(self, num_special_tokens): - pass - TRANSFO_XL_START_DOCSTRING = r""" The Transformer-XL model was proposed in `Transformer-XL: Attentive Language Models Beyond a Fixed-Length Context`_ @@ -882,43 +615,16 @@ class TransfoXLModel(TransfoXLPreTrainedModel): r_r_bias=None if config.untie_r else self.r_r_bias, output_attentions=self.output_attentions) ) - elif config.attn_type == 1: # learnable embeddings - for i in range(config.n_layer): - self.layers.append( - RelLearnableDecoderLayer( - config.n_head, config.d_model, config.d_head, config.d_inner, config.dropout, - tgt_len=config.tgt_len, ext_len=config.ext_len, mem_len=config.mem_len, - dropatt=config.dropatt, pre_lnorm=config.pre_lnorm, - r_w_bias=None if config.untie_r else self.r_w_bias, - r_r_bias=None if config.untie_r else self.r_r_bias, - output_attentions=self.output_attentions) - ) - elif config.attn_type in [2, 3]: # absolute embeddings - for i in range(config.n_layer): - self.layers.append( - DecoderLayer( - config.n_head, config.d_model, config.d_head, config.d_inner, config.dropout, - dropatt=config.dropatt, pre_lnorm=config.pre_lnorm, - r_w_bias=None if config.untie_r else self.r_w_bias, - r_r_bias=None if config.untie_r else self.r_r_bias, - output_attentions=self.output_attentions) - ) + else: # learnable embeddings and absolute embeddings are not used in our pretrained checkpoints + raise NotImplementedError # Removed them to avoid maintaining dead code self.same_length = config.same_length self.clamp_len = config.clamp_len if self.attn_type == 0: # default attention self.pos_emb = PositionalEmbedding(self.d_model) - elif self.attn_type == 1: # learnable - self.r_emb = nn.Parameter(torch.FloatTensor( - self.n_layer, self.max_klen, self.n_head, self.d_head)) - self.r_bias = nn.Parameter(torch.FloatTensor( - self.n_layer, self.max_klen, self.n_head)) - elif self.attn_type == 2: # absolute standard - self.pos_emb = PositionalEmbedding(self.d_model) - elif self.attn_type == 3: # absolute deeper SA - self.r_emb = nn.Parameter(torch.FloatTensor( - self.n_layer, self.max_klen, self.n_head, self.d_head)) + else: # learnable embeddings and absolute embeddings + raise NotImplementedError # Removed these to avoid maintaining dead code - They are not used in our pretrained checkpoint self.init_weights() @@ -973,8 +679,15 @@ class TransfoXLModel(TransfoXLPreTrainedModel): return new_mems - def _forward(self, dec_inp, mems=None, head_mask=None): - qlen, bsz = dec_inp.size() + def forward(self, input_ids, mems=None, head_mask=None): + # the original code for Transformer-XL used shapes [len, bsz] but we want a unified interface in the library + # so we transpose here from shape [bsz, len] to shape [len, bsz] + input_ids = input_ids.transpose(0, 1).contiguous() + + if mems is None: + mems = self.init_mems(input_ids) + + qlen, bsz = input_ids.size() # Prepare head mask if needed # 1.0 in head_mask indicate we keep the head @@ -991,7 +704,7 @@ class TransfoXLModel(TransfoXLPreTrainedModel): else: head_mask = [None] * self.n_layer - word_emb = self.word_emb(dec_inp) + word_emb = self.word_emb(input_ids) mlen = mems[0].size(0) if mems is not None else 0 klen = mlen + qlen @@ -1028,64 +741,8 @@ class TransfoXLModel(TransfoXLPreTrainedModel): core_out = layer_outputs[0] if self.output_attentions: attentions.append(layer_outputs[1]) - elif self.attn_type == 1: # learnable - core_out = self.drop(word_emb) - for i, layer in enumerate(self.layers): - hids.append(core_out) - if self.clamp_len > 0: - r_emb = self.r_emb[i][-self.clamp_len :] - r_bias = self.r_bias[i][-self.clamp_len :] - else: - r_emb, r_bias = self.r_emb[i], self.r_bias[i] - - mems_i = None if mems is None else mems[i] - layer_outputs = layer(core_out, r_emb, self.r_w_bias[i], - r_bias, dec_attn_mask=dec_attn_mask, - mems=mems_i, head_mask=head_mask[i]) - core_out = layer_outputs[0] - if self.output_attentions: - attentions.append(layer_outputs[1]) - elif self.attn_type == 2: # absolute - pos_seq = torch.arange(klen - 1, -1, -1.0, device=word_emb.device, - dtype=word_emb.dtype) - if self.clamp_len > 0: - pos_seq.clamp_(max=self.clamp_len) - pos_emb = self.pos_emb(pos_seq) - - core_out = self.drop(word_emb + pos_emb[-qlen:]) - - for i, layer in enumerate(self.layers): - hids.append(core_out) - mems_i = None if mems is None else mems[i] - if mems_i is not None and i == 0: - mems_i += pos_emb[:mlen] - layer_outputs = layer(core_out, dec_attn_mask=dec_attn_mask, - mems=mems_i, head_mask=head_mask[i]) - core_out = layer_outputs[0] - if self.output_attentions: - attentions.append(layer_outputs[1]) - elif self.attn_type == 3: - core_out = self.drop(word_emb) - - for i, layer in enumerate(self.layers): - hids.append(core_out) - mems_i = None if mems is None else mems[i] - if mems_i is not None and mlen > 0: - cur_emb = self.r_emb[i][:-qlen] - cur_size = cur_emb.size(0) - if cur_size < mlen: - cur_emb_pad = cur_emb[0:1].expand(mlen-cur_size, -1, -1) - cur_emb = torch.cat([cur_emb_pad, cur_emb], 0) - else: - cur_emb = cur_emb[-mlen:] - mems_i += cur_emb.view(mlen, 1, -1) - core_out += self.r_emb[i][-qlen:].view(qlen, 1, -1) - - layer_outputs = layer(core_out, dec_attn_mask=dec_attn_mask, - mems=mems_i, head_mask=head_mask[i]) - core_out = layer_outputs[0] - if self.output_attentions: - attentions.append(layer_outputs[1]) + else: # learnable embeddings and absolute embeddings + raise NotImplementedError # Removed these to avoid maintaining dead code - They are not used in our pretrained checkpoint core_out = self.drop(core_out) @@ -1102,16 +759,6 @@ class TransfoXLModel(TransfoXLPreTrainedModel): # Transpose to library standard shape [bsz, n_heads, query_seq_len, key_seq_len] attentions = list(t.permute(2, 3, 0, 1).contiguous() for t in attentions) outputs.append(attentions) - return outputs # last hidden state, new_mems, (all hidden states), (all attentions) - - def forward(self, input_ids, mems=None, head_mask=None): - # the original code for Transformer-XL used shapes [len, bsz] but we want a unified interface in the library - # so we transpose here from shape [bsz, len] to shape [len, bsz] - input_ids = input_ids.transpose(0, 1).contiguous() - - if mems is None: - mems = self.init_mems(input_ids) - outputs = self._forward(input_ids, mems=mems, head_mask=head_mask) return outputs # last hidden state, new_mems, (all hidden states), (all attentions) diff --git a/pytorch_transformers/tests/modeling_tf_bert_test.py b/pytorch_transformers/tests/modeling_tf_bert_test.py index 55bbe36feb..a1bf9bf794 100644 --- a/pytorch_transformers/tests/modeling_tf_bert_test.py +++ b/pytorch_transformers/tests/modeling_tf_bert_test.py @@ -131,10 +131,14 @@ class TFBertModelTest(TFCommonTestCases.TFCommonModelTester): def create_and_check_bert_model(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): model = TFBertModel(config=config) + # inputs = {'input_ids': input_ids, + # 'attention_mask': input_mask, + # 'token_type_ids': token_type_ids} + # sequence_output, pooled_output = model(**inputs) inputs = {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids} - sequence_output, pooled_output = model(inputs) + sequence_output, pooled_output = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids) inputs = [input_ids, input_mask] sequence_output, pooled_output = model(inputs) diff --git a/pytorch_transformers/tests/modeling_tf_transfo_xl_test.py b/pytorch_transformers/tests/modeling_tf_transfo_xl_test.py new file mode 100644 index 0000000000..58a4343dfd --- /dev/null +++ b/pytorch_transformers/tests/modeling_tf_transfo_xl_test.py @@ -0,0 +1,217 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import unittest +import random +import shutil +import pytest + +from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) +from .configuration_common_test import ConfigTester + +from pytorch_transformers import TransfoXLConfig, is_tf_available + +if is_tf_available(): + import tensorflow as tf + from pytorch_transformers.modeling_tf_transfo_xl import (TFTransfoXLModel, + TFTransfoXLLMHeadModel, + TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP) +else: + pytestmark = pytest.mark.skip("Require TensorFlow") + + +class TFTransfoXLModelTest(TFCommonTestCases.TFCommonModelTester): + + all_model_classes = (TFTransfoXLModel, TFTransfoXLLMHeadModel) if is_tf_available() else () + test_pruning = False + test_torchscript = False + test_resize_embeddings = False + + class TFTransfoXLModelTester(object): + + def __init__(self, + parent, + batch_size=13, + seq_length=7, + mem_len=30, + clamp_len=15, + is_training=True, + use_labels=True, + vocab_size=99, + cutoffs=[10, 50, 80], + hidden_size=32, + d_embed=32, + num_attention_heads=4, + d_head=8, + d_inner=128, + div_val=2, + num_hidden_layers=5, + scope=None, + seed=1, + ): + self.parent = parent + self.batch_size = batch_size + self.seq_length = seq_length + self.mem_len = mem_len + self.key_len = seq_length + mem_len + self.clamp_len = clamp_len + self.is_training = is_training + self.use_labels = use_labels + self.vocab_size = vocab_size + self.cutoffs = cutoffs + self.hidden_size = hidden_size + self.d_embed = d_embed + self.num_attention_heads = num_attention_heads + self.d_head = d_head + self.d_inner = d_inner + self.div_val = div_val + self.num_hidden_layers = num_hidden_layers + self.scope = scope + self.seed = seed + + def prepare_config_and_inputs(self): + input_ids_1 = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) + input_ids_2 = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) + + lm_labels = None + if self.use_labels: + lm_labels = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) + + config = TransfoXLConfig( + vocab_size_or_config_json_file=self.vocab_size, + mem_len=self.mem_len, + clamp_len=self.clamp_len, + cutoffs=self.cutoffs, + d_model=self.hidden_size, + d_embed=self.d_embed, + n_head=self.num_attention_heads, + d_head=self.d_head, + d_inner=self.d_inner, + div_val=self.div_val, + n_layer=self.num_hidden_layers) + + return (config, input_ids_1, input_ids_2, lm_labels) + + def set_seed(self): + random.seed(self.seed) + tf.random.set_seed(self.seed) + + def create_and_check_transfo_xl_model(self, config, input_ids_1, input_ids_2, lm_labels): + model = TFTransfoXLModel(config) + + hidden_states_1, mems_1 = model(input_ids_1) + + inputs = {'input_ids': input_ids_2, + 'mems': mems_1} + + hidden_states_2, mems_2 = model(inputs) + + result = { + "hidden_states_1": hidden_states_1.numpy(), + "mems_1": [mem.numpy() for mem in mems_1], + "hidden_states_2": hidden_states_2.numpy(), + "mems_2": [mem.numpy() for mem in mems_2], + } + + self.parent.assertListEqual( + list(result["hidden_states_1"].shape), + [self.batch_size, self.seq_length, self.hidden_size]) + self.parent.assertListEqual( + list(result["hidden_states_2"].shape), + [self.batch_size, self.seq_length, self.hidden_size]) + self.parent.assertListEqual( + list(list(mem.shape) for mem in result["mems_1"]), + [[self.mem_len, self.batch_size, self.hidden_size]] * self.num_hidden_layers) + self.parent.assertListEqual( + list(list(mem.shape) for mem in result["mems_2"]), + [[self.mem_len, self.batch_size, self.hidden_size]] * self.num_hidden_layers) + + + def create_and_check_transfo_xl_lm_head(self, config, input_ids_1, input_ids_2, lm_labels): + model = TFTransfoXLLMHeadModel(config) + + lm_logits_1, mems_1 = model(input_ids_1) + + inputs = {'input_ids': input_ids_1, + 'labels': lm_labels} + _, mems_1 = model(inputs) + + lm_logits_2, mems_2 = model([input_ids_2, mems_1]) + + inputs = {'input_ids': input_ids_1, + 'mems': mems_1, + 'labels': lm_labels} + + _, mems_2 = model(inputs) + + result = { + "mems_1": [mem.numpy() for mem in mems_1], + "lm_logits_1": lm_logits_1.numpy(), + "mems_2": [mem.numpy() for mem in mems_2], + "lm_logits_2": lm_logits_2.numpy(), + } + + self.parent.assertListEqual( + list(result["lm_logits_1"].shape), + [self.batch_size, self.seq_length, self.vocab_size]) + self.parent.assertListEqual( + list(list(mem.shape) for mem in result["mems_1"]), + [[self.mem_len, self.batch_size, self.hidden_size]] * self.num_hidden_layers) + + self.parent.assertListEqual( + list(result["lm_logits_2"].shape), + [self.batch_size, self.seq_length, self.vocab_size]) + self.parent.assertListEqual( + list(list(mem.shape) for mem in result["mems_2"]), + [[self.mem_len, self.batch_size, self.hidden_size]] * self.num_hidden_layers) + + def prepare_config_and_inputs_for_common(self): + config_and_inputs = self.prepare_config_and_inputs() + (config, input_ids_1, input_ids_2, lm_labels) = config_and_inputs + inputs_dict = {'input_ids': input_ids_1} + return config, inputs_dict + + + def setUp(self): + self.model_tester = TFTransfoXLModelTest.TFTransfoXLModelTester(self) + self.config_tester = ConfigTester(self, config_class=TransfoXLConfig, d_embed=37) + + def test_config(self): + self.config_tester.run_common_tests() + + def test_transfo_xl_model(self): + self.model_tester.set_seed() + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_transfo_xl_model(*config_and_inputs) + + def test_transfo_xl_lm_head(self): + self.model_tester.set_seed() + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_transfo_xl_lm_head(*config_and_inputs) + + @pytest.mark.slow + def test_model_from_pretrained(self): + cache_dir = "/tmp/pytorch_transformers_test/" + for model_name in list(TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: + model = TFTransfoXLModel.from_pretrained(model_name, cache_dir=cache_dir) + shutil.rmtree(cache_dir) + self.assertIsNotNone(model) + + +if __name__ == "__main__": + unittest.main() From b97af8cce90aa2d147ca3fd9543ca372d3cb2ae3 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Fri, 13 Sep 2019 16:43:49 +0200 Subject: [PATCH 051/219] skip finetuned checkpoints --- pytorch_transformers/__init__.py | 5 + .../convert_pytorch_checkpoint_to_tf2.py | 14 ++- .../modeling_tf_transfo_xl_utilities.py | 106 +----------------- 3 files changed, 17 insertions(+), 108 deletions(-) diff --git a/pytorch_transformers/__init__.py b/pytorch_transformers/__init__.py index 2f7bdb3de4..40c4bab7e1 100644 --- a/pytorch_transformers/__init__.py +++ b/pytorch_transformers/__init__.py @@ -113,6 +113,11 @@ if _tf_available: load_gpt2_pt_weights_in_tf2, TF_GPT2_PRETRAINED_MODEL_ARCHIVE_MAP) + from .modeling_tf_transfo_xl import (TFTransfoXLPreTrainedModel, TFTransfoXLMainLayer, + TFTransfoXLModel, TFTransfoXLLMHeadModel, + load_transfo_xl_pt_weights_in_tf2, + TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP) + from .modeling_tf_xlnet import (TFXLNetPreTrainedModel, TFXLNetMainLayer, TFXLNetModel, TFXLNetLMHeadModel, TFXLNetForSequenceClassification, diff --git a/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py b/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py index d586fecbc5..ee26ac0a89 100644 --- a/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py +++ b/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py @@ -27,7 +27,8 @@ from pytorch_transformers import is_torch_available, cached_path from pytorch_transformers import (BertConfig, TFBertForPreTraining, load_bert_pt_weights_in_tf2, GPT2Config, TFGPT2LMHeadModel, load_gpt2_pt_weights_in_tf2, XLNetConfig, TFXLNetLMHeadModel, load_xlnet_pt_weights_in_tf2, - XLMConfig, TFXLMWithLMHeadModel, load_xlm_pt_weights_in_tf2,) + XLMConfig, TFXLMWithLMHeadModel, load_xlm_pt_weights_in_tf2, + TransfoXLConfig, TFTransfoXLLMHeadModel, load_transfo_xl_pt_weights_in_tf2,) if is_torch_available(): import torch @@ -35,12 +36,15 @@ if is_torch_available(): from pytorch_transformers import (BertForPreTraining, BERT_PRETRAINED_MODEL_ARCHIVE_MAP, BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, GPT2LMHeadModel, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP, GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP, XLNetLMHeadModel, XLNET_PRETRAINED_MODEL_ARCHIVE_MAP, XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP, - XLMWithLMHeadModel, XLM_PRETRAINED_MODEL_ARCHIVE_MAP, XLM_PRETRAINED_CONFIG_ARCHIVE_MAP,) + XLMWithLMHeadModel, XLM_PRETRAINED_MODEL_ARCHIVE_MAP, XLM_PRETRAINED_CONFIG_ARCHIVE_MAP, + TransfoXLLMHeadModel, TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP, TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP,) else: (BertForPreTraining, BERT_PRETRAINED_MODEL_ARCHIVE_MAP, BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, GPT2LMHeadModel, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP, GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP, XLNetLMHeadModel, XLNET_PRETRAINED_MODEL_ARCHIVE_MAP, XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP, - XLMWithLMHeadModel, XLM_PRETRAINED_MODEL_ARCHIVE_MAP, XLM_PRETRAINED_CONFIG_ARCHIVE_MAP,) = ( + XLMWithLMHeadModel, XLM_PRETRAINED_MODEL_ARCHIVE_MAP, XLM_PRETRAINED_CONFIG_ARCHIVE_MAP, + TransfoXLLMHeadModel, TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP, TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP,) = ( + None, None, None, None, None, None, None, None, None, None, None, None, @@ -55,6 +59,7 @@ MODEL_CLASSES = { 'gpt2': (GPT2Config, TFGPT2LMHeadModel, load_gpt2_pt_weights_in_tf2, GPT2LMHeadModel, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP, GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP), 'xlnet': (XLNetConfig, TFXLNetLMHeadModel, load_xlnet_pt_weights_in_tf2, XLNetLMHeadModel, XLNET_PRETRAINED_MODEL_ARCHIVE_MAP, XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP), 'xlm': (XLMConfig, TFXLMWithLMHeadModel, load_xlm_pt_weights_in_tf2, XLMWithLMHeadModel, XLM_PRETRAINED_MODEL_ARCHIVE_MAP, XLM_PRETRAINED_CONFIG_ARCHIVE_MAP), + 'transfo-xl': (TransfoXLConfig, TFTransfoXLLMHeadModel, load_transfo_xl_pt_weights_in_tf2, TransfoXLLMHeadModel, TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP, TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP), } def convert_pt_checkpoint_to_tf(model_type, pytorch_checkpoint_path, config_file, tf_dump_path, compare_with_pt_model=False): @@ -118,6 +123,9 @@ def convert_all_pt_checkpoints_to_tf(args_model_type, tf_dump_path, compare_with print("-" * 100) print(" Converting checkpoint {}/{}: {}".format(i, len(aws_config_map), shortcut_name)) print("-" * 100) + if 'finetuned' in shortcut_name: + print(" Skipping fintenued checkpoint ") + continue config_file = cached_path(aws_config_map[shortcut_name], force_download=True) model_file = cached_path(aws_model_maps[shortcut_name], force_download=True) diff --git a/pytorch_transformers/modeling_tf_transfo_xl_utilities.py b/pytorch_transformers/modeling_tf_transfo_xl_utilities.py index d313ba177e..d7666a650e 100644 --- a/pytorch_transformers/modeling_tf_transfo_xl_utilities.py +++ b/pytorch_transformers/modeling_tf_transfo_xl_utilities.py @@ -13,8 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" Utilities for PyTorch Transformer XL model. - Directly adapted from https://github.com/kimiyoung/transformer-xl. +""" A TF 2.0 Adaptive Softmax for Transformer XL model. """ from collections import defaultdict @@ -174,106 +173,3 @@ class TFAdaptiveSoftmaxMask(tf.keras.layers.Layer): self.add_metric(loss, name=self.name, aggregation='mean' if return_mean else '') return out - - -def mul_adaptive_logsoftmax(hidden, target, n_token, d_embed, d_proj, cutoffs, - params, tie_projs, - initializer=None, proj_initializer=None, - div_val=1, perms=None, proj_same_dim=True, - scope='adaptive_softmax', - **kwargs): - def _logit(x, W, b, proj): - y = x - if x.shape.ndims == 3: - if proj is not None: - y = tf.einsum('ibd,ed->ibe', y, proj) - return tf.einsum('ibd,nd->ibn', y, W) + b - else: - if proj is not None: - y = tf.einsum('id,ed->ie', y, proj) - return tf.einsum('id,nd->in', y, W) + b - - params_W, params_projs = params[0], params[1] - - with tf.variable_scope(scope): - if len(cutoffs) == 0: - softmax_b = tf.get_variable('bias', [n_token], - initializer=tf.zeros_initializer()) - output = _logit(hidden, params_W, softmax_b, params_projs) - nll = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=target, - logits=output) - nll = tf.reduce_mean(nll) - else: - total_loss, total_cnt = 0, 0 - cutoff_ends = [0] + cutoffs + [n_token] - for i in range(len(cutoff_ends) - 1): - with tf.variable_scope('cutoff_{}'.format(i)): - l_idx, r_idx = cutoff_ends[i], cutoff_ends[i + 1] - - cur_d_embed = d_embed // (div_val ** i) - - if div_val == 1: - cur_W = params_W[l_idx: r_idx] - else: - cur_W = params_W[i] - cur_b = tf.get_variable('b', [r_idx - l_idx], - initializer=tf.zeros_initializer()) - if tie_projs[i]: - if div_val == 1: - cur_proj = params_projs - else: - cur_proj = params_projs[i] - else: - if (div_val == 1 or not proj_same_dim) and d_proj == cur_d_embed: - cur_proj = None - else: - cur_proj = tf.get_variable('proj', [cur_d_embed, d_proj], - initializer=proj_initializer) - - if i == 0: - cluster_W = tf.get_variable('cluster_W', [len(cutoffs), d_embed], - initializer=tf.zeros_initializer()) - cluster_b = tf.get_variable('cluster_b', [len(cutoffs)], - initializer=tf.zeros_initializer()) - cur_W = tf.concat([cur_W, cluster_W], 0) - cur_b = tf.concat([cur_b, cluster_b], 0) - - head_logit = _logit(hidden, cur_W, cur_b, cur_proj) - - head_target = kwargs.get("head_target") - head_nll = tf.nn.sparse_softmax_cross_entropy_with_logits( - labels=head_target, - logits=head_logit) - - masked_loss = head_nll * perms[i] - total_loss += tf.reduce_sum(masked_loss) - total_cnt += tf.reduce_sum(perms[i]) - - # head_logprob = tf.nn.log_softmax(head_logit) - - # final_logprob = head_logprob * perms[i][:, :, None] - # final_target = tf.one_hot(target, tf.shape(head_logprob)[2]) - # total_loss -= tf.einsum('ibn,ibn->', final_logprob, final_target) - # total_cnt += tf.reduce_sum(perms[i]) - else: - cur_head_nll = tf.einsum('ib,ibk->k', head_nll, perms[i]) - - cur_hidden = tf.einsum('ibd,ibk->kd', hidden, perms[i]) - tail_logit = _logit(cur_hidden, cur_W, cur_b, cur_proj) - - tail_target = tf.einsum('ib,ibk->k', tf.to_float(target - l_idx), - perms[i]) - tail_nll = tf.nn.sparse_softmax_cross_entropy_with_logits( - labels=tf.to_int32(tail_target), - logits=tail_logit) - - sum_nll = cur_head_nll + tail_nll - mask = tf.reduce_sum(perms[i], [0, 1]) - - masked_loss = sum_nll * mask - total_loss += tf.reduce_sum(masked_loss) - total_cnt += tf.reduce_sum(mask) - - nll = total_loss / total_cnt - - return nll \ No newline at end of file From 4b956b2a6bfec5532f8df8de0b7b37b5f1120258 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Fri, 13 Sep 2019 17:09:20 +0200 Subject: [PATCH 052/219] add layer_norm_epsilon configuration for transformer xl --- .../configuration_transfo_xl.py | 67 ++++++++++--------- .../convert_pytorch_checkpoint_to_tf2.py | 6 +- .../modeling_tf_pytorch_utils.py | 4 +- .../modeling_tf_transfo_xl.py | 22 +++--- pytorch_transformers/modeling_transfo_xl.py | 19 +++--- 5 files changed, 65 insertions(+), 53 deletions(-) diff --git a/pytorch_transformers/configuration_transfo_xl.py b/pytorch_transformers/configuration_transfo_xl.py index 2e966ee55c..d55a6adbe6 100644 --- a/pytorch_transformers/configuration_transfo_xl.py +++ b/pytorch_transformers/configuration_transfo_xl.py @@ -95,10 +95,43 @@ class TransfoXLConfig(PretrainedConfig): init_range=0.01, proj_init_std=0.01, init_std=0.02, + layer_norm_epsilon=1e-5, **kwargs): """Constructs TransfoXLConfig. """ super(TransfoXLConfig, self).__init__(**kwargs) + self.n_token = vocab_size_or_config_json_file if isinstance(vocab_size_or_config_json_file, int) else -1 + self.cutoffs = [] + self.cutoffs.extend(cutoffs) + self.tie_weight = tie_weight + if proj_share_all_but_first: + self.tie_projs = [False] + [True] * len(self.cutoffs) + else: + self.tie_projs = [False] + [False] * len(self.cutoffs) + self.d_model = d_model + self.d_embed = d_embed + self.d_head = d_head + self.d_inner = d_inner + self.div_val = div_val + self.pre_lnorm = pre_lnorm + self.n_layer = n_layer + self.n_head = n_head + self.tgt_len = tgt_len + self.ext_len = ext_len + self.mem_len = mem_len + self.same_length = same_length + self.attn_type = attn_type + self.clamp_len = clamp_len + self.sample_softmax = sample_softmax + self.adaptive = adaptive + self.dropout = dropout + self.dropatt = dropatt + self.untie_r = untie_r + self.init = init + self.init_range = init_range + self.proj_init_std = proj_init_std + self.init_std = init_std + self.layer_norm_epsilon = layer_norm_epsilon if isinstance(vocab_size_or_config_json_file, str) or (sys.version_info[0] == 2 and isinstance(vocab_size_or_config_json_file, unicode)): @@ -106,39 +139,7 @@ class TransfoXLConfig(PretrainedConfig): json_config = json.loads(reader.read()) for key, value in json_config.items(): self.__dict__[key] = value - elif isinstance(vocab_size_or_config_json_file, int): - self.n_token = vocab_size_or_config_json_file - self.cutoffs = [] - self.cutoffs.extend(cutoffs) - self.tie_weight = tie_weight - if proj_share_all_but_first: - self.tie_projs = [False] + [True] * len(self.cutoffs) - else: - self.tie_projs = [False] + [False] * len(self.cutoffs) - self.d_model = d_model - self.d_embed = d_embed - self.d_head = d_head - self.d_inner = d_inner - self.div_val = div_val - self.pre_lnorm = pre_lnorm - self.n_layer = n_layer - self.n_head = n_head - self.tgt_len = tgt_len - self.ext_len = ext_len - self.mem_len = mem_len - self.same_length = same_length - self.attn_type = attn_type - self.clamp_len = clamp_len - self.sample_softmax = sample_softmax - self.adaptive = adaptive - self.dropout = dropout - self.dropatt = dropatt - self.untie_r = untie_r - self.init = init - self.init_range = init_range - self.proj_init_std = proj_init_std - self.init_std = init_std - else: + elif not isinstance(vocab_size_or_config_json_file, int): raise ValueError("First argument must be either a vocabulary size (int)" " or the path to a pretrained model config file (str)") diff --git a/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py b/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py index ee26ac0a89..bc51491ce7 100644 --- a/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py +++ b/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py @@ -84,8 +84,8 @@ def convert_pt_checkpoint_to_tf(model_type, pytorch_checkpoint_path, config_file tfo = tf_model(tf_inputs, training=False) # build the network pt_model = pt_model_class.from_pretrained(None, - config=config, - state_dict=torch.load(pytorch_checkpoint_path, + config=config, + state_dict=torch.load(pytorch_checkpoint_path, map_location='cpu')) pt_inputs = torch.tensor(inputs_list) with torch.no_grad(): @@ -124,7 +124,7 @@ def convert_all_pt_checkpoints_to_tf(args_model_type, tf_dump_path, compare_with print(" Converting checkpoint {}/{}: {}".format(i, len(aws_config_map), shortcut_name)) print("-" * 100) if 'finetuned' in shortcut_name: - print(" Skipping fintenued checkpoint ") + print(" Skipping finetuned checkpoint ") continue config_file = cached_path(aws_config_map[shortcut_name], force_download=True) model_file = cached_path(aws_model_maps[shortcut_name], force_download=True) diff --git a/pytorch_transformers/modeling_tf_pytorch_utils.py b/pytorch_transformers/modeling_tf_pytorch_utils.py index 13cd6dd14d..2420b69fb9 100644 --- a/pytorch_transformers/modeling_tf_pytorch_utils.py +++ b/pytorch_transformers/modeling_tf_pytorch_utils.py @@ -91,8 +91,10 @@ def load_pytorch_state_dict_in_tf2_model(tf_model, pt_state_dict, tf_inputs=None name = name.split('/') # Convert from TF2.0 '/' separators to PyTorch '.' separators name = name[1:] # Remove level zero + # When should we transpose the weights + transpose = bool(name[-1] == 'kernel' or 'emb_projs' in name or 'out_projs' in name) + # Convert standard TF2.0 names in PyTorch names - transpose = bool(name[-1] == 'kernel') if name[-1] == 'kernel' or name[-1] == 'embeddings' or name[-1] == 'gamma': name[-1] = 'weight' if name[-1] == 'beta': diff --git a/pytorch_transformers/modeling_tf_transfo_xl.py b/pytorch_transformers/modeling_tf_transfo_xl.py index a4a333d969..ded06923e8 100644 --- a/pytorch_transformers/modeling_tf_transfo_xl.py +++ b/pytorch_transformers/modeling_tf_transfo_xl.py @@ -66,7 +66,7 @@ class TFPositionalEmbedding(tf.keras.layers.Layer): class TFPositionwiseFF(tf.keras.layers.Layer): - def __init__(self, d_model, d_inner, dropout, pre_lnorm=False, **kwargs): + def __init__(self, d_model, d_inner, dropout, pre_lnorm=False, layer_norm_epsilon=1e-5, **kwargs): super(TFPositionwiseFF, self).__init__(**kwargs) self.d_model = d_model @@ -75,10 +75,10 @@ class TFPositionwiseFF(tf.keras.layers.Layer): self.layer_1 = tf.keras.layers.Dense(d_inner, activation=tf.nn.relu, name='CoreNet_._0') self.drop_1 = tf.keras.layers.Dropout(dropout) - self.layer_2 = tf.keras.layers.Dense(d_model, name='CoreNet_._2') + self.layer_2 = tf.keras.layers.Dense(d_model, name='CoreNet_._3') self.drop_2 = tf.keras.layers.Dropout(dropout) - self.layer_norm = tf.keras.layers.LayerNormalization(epsilon=1e-12, name='layer_norm') + self.layer_norm = tf.keras.layers.LayerNormalization(epsilon=layer_norm_epsilon, name='layer_norm') self.pre_lnorm = pre_lnorm @@ -109,7 +109,8 @@ class TFPositionwiseFF(tf.keras.layers.Layer): class TFRelPartialLearnableMultiHeadAttn(tf.keras.layers.Layer): def __init__(self, n_head, d_model, d_head, dropout, dropatt=0, tgt_len=None, ext_len=None, mem_len=None, pre_lnorm=False, - r_r_bias=None, r_w_bias=None, output_attentions=False, **kwargs): + r_r_bias=None, r_w_bias=None, output_attentions=False, + layer_norm_epsilon=1e-5, **kwargs): super(TFRelPartialLearnableMultiHeadAttn, self).__init__(**kwargs) self.output_attentions = output_attentions @@ -124,7 +125,7 @@ class TFRelPartialLearnableMultiHeadAttn(tf.keras.layers.Layer): self.dropatt = tf.keras.layers.Dropout(dropatt) self.o_net = tf.keras.layers.Dense(d_model, use_bias=False, name='o_net') - self.layer_norm = tf.keras.layers.LayerNormalization(epsilon=1e-12, name='layer_norm') + self.layer_norm = tf.keras.layers.LayerNormalization(epsilon=layer_norm_epsilon, name='layer_norm') self.scale = 1 / (d_head ** 0.5) @@ -247,6 +248,7 @@ class TFRelPartialLearnableDecoderLayer(tf.keras.layers.Layer): r_w_bias=None, r_r_bias=None, output_attentions=False, + layer_norm_epsilon=1e-5, **kwargs): super(TFRelPartialLearnableDecoderLayer, self).__init__(**kwargs) @@ -254,9 +256,12 @@ class TFRelPartialLearnableDecoderLayer(tf.keras.layers.Layer): d_head, dropout, tgt_len=tgt_len, ext_len=ext_len, mem_len=mem_len, dropatt=dropatt, pre_lnorm=pre_lnorm, r_w_bias=r_w_bias, r_r_bias=r_r_bias, - output_attentions=output_attentions, name='dec_attn') + output_attentions=output_attentions, + layer_norm_epsilon=layer_norm_epsilon, name='dec_attn') self.pos_ff = TFPositionwiseFF(d_model, d_inner, dropout, - pre_lnorm=pre_lnorm, name='pos_ff') + pre_lnorm=pre_lnorm, + layer_norm_epsilon=layer_norm_epsilon, + name='pos_ff') def call(self, inputs, training=False): dec_inp, r, dec_attn_mask, mems, head_mask = inputs @@ -300,7 +305,7 @@ class TFAdaptiveEmbedding(tf.keras.layers.Layer): d_emb_i = self.d_embed // (self.div_val ** i) self.emb_projs.append(self.add_weight(shape=(d_emb_i, self.d_proj), trainable=True, - name='emb_projs._{}'.format(i))) + name='emb_projs_._{}'.format(i))) super(TFAdaptiveEmbedding, self).build(input_shape) def call(self, inp): @@ -368,6 +373,7 @@ class TFTransfoXLMainLayer(tf.keras.layers.Layer): r_w_bias=None if self.untie_r else self.r_w_bias, r_r_bias=None if self.untie_r else self.r_r_bias, output_attentions=self.output_attentions, + layer_norm_epsilon=config.layer_norm_epsilon, name='layers_._{}'.format(i)) ) else: # learnable embeddings and absolute embeddings diff --git a/pytorch_transformers/modeling_transfo_xl.py b/pytorch_transformers/modeling_transfo_xl.py index 557694e84f..c0919c8543 100644 --- a/pytorch_transformers/modeling_transfo_xl.py +++ b/pytorch_transformers/modeling_transfo_xl.py @@ -194,7 +194,7 @@ class PositionalEmbedding(nn.Module): class PositionwiseFF(nn.Module): - def __init__(self, d_model, d_inner, dropout, pre_lnorm=False): + def __init__(self, d_model, d_inner, dropout, pre_lnorm=False, layer_norm_epsilon=1e-5): super(PositionwiseFF, self).__init__() self.d_model = d_model @@ -208,7 +208,7 @@ class PositionwiseFF(nn.Module): nn.Dropout(dropout), ) - self.layer_norm = nn.LayerNorm(d_model) + self.layer_norm = nn.LayerNorm(d_model, eps=layer_norm_epsilon) self.pre_lnorm = pre_lnorm @@ -232,7 +232,8 @@ class PositionwiseFF(nn.Module): class RelPartialLearnableMultiHeadAttn(nn.Module): def __init__(self, n_head, d_model, d_head, dropout, dropatt=0, tgt_len=None, ext_len=None, mem_len=None, pre_lnorm=False, - r_r_bias=None, r_w_bias=None, output_attentions=False): + r_r_bias=None, r_w_bias=None, output_attentions=False, + layer_norm_epsilon=1e-5): super(RelPartialLearnableMultiHeadAttn, self).__init__() self.output_attentions = output_attentions @@ -247,7 +248,7 @@ class RelPartialLearnableMultiHeadAttn(nn.Module): self.dropatt = nn.Dropout(dropatt) self.o_net = nn.Linear(n_head * d_head, d_model, bias=False) - self.layer_norm = nn.LayerNorm(d_model) + self.layer_norm = nn.LayerNorm(d_model, eps=layer_norm_epsilon) self.scale = 1 / (d_head ** 0.5) @@ -359,14 +360,15 @@ class RelPartialLearnableMultiHeadAttn(nn.Module): class RelPartialLearnableDecoderLayer(nn.Module): - def __init__(self, n_head, d_model, d_head, d_inner, dropout, + def __init__(self, n_head, d_model, d_head, d_inner, dropout, layer_norm_epsilon=1e-5, **kwargs): super(RelPartialLearnableDecoderLayer, self).__init__() self.dec_attn = RelPartialLearnableMultiHeadAttn(n_head, d_model, - d_head, dropout, **kwargs) + d_head, dropout, layer_norm_epsilon=layer_norm_epsilon, **kwargs) self.pos_ff = PositionwiseFF(d_model, d_inner, dropout, - pre_lnorm=kwargs.get('pre_lnorm')) + pre_lnorm=kwargs.get('pre_lnorm'), + layer_norm_epsilon=layer_norm_epsilon) def forward(self, dec_inp, r, dec_attn_mask=None, mems=None, head_mask=None): @@ -613,7 +615,8 @@ class TransfoXLModel(TransfoXLPreTrainedModel): dropatt=config.dropatt, pre_lnorm=config.pre_lnorm, r_w_bias=None if config.untie_r else self.r_w_bias, r_r_bias=None if config.untie_r else self.r_r_bias, - output_attentions=self.output_attentions) + output_attentions=self.output_attentions, + layer_norm_epsilon=config.layer_norm_epsilon) ) else: # learnable embeddings and absolute embeddings are not used in our pretrained checkpoints raise NotImplementedError # Removed them to avoid maintaining dead code From f6969cc12b90c8f6c76f4819ea5f2f2e909f70b3 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Wed, 18 Sep 2019 11:12:02 +0200 Subject: [PATCH 053/219] upgrade max model difference to 2e-2 (for transfo-xl adaptive softmax + inputs) --- pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py b/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py index bc51491ce7..ca6089bd04 100644 --- a/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py +++ b/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py @@ -95,7 +95,7 @@ def convert_pt_checkpoint_to_tf(model_type, pytorch_checkpoint_path, config_file np_tf = tfo[0].numpy() diff = np.amax(np.abs(np_pt - np_tf)) print("Max absolute difference between models outputs {}".format(diff)) - assert diff <= 1e-3, "Error, model absolute difference is >1e-3" + assert diff <= 2e-2, "Error, model absolute difference is >2e-2" # Save pytorch-model print("Save TensorFlow model to {}".format(tf_dump_path)) From 6a083fd4478be2f5163c08c93edd6b65dcbd0a98 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Wed, 18 Sep 2019 12:11:32 +0200 Subject: [PATCH 054/219] update pt-tf conversion script --- .../convert_pytorch_checkpoint_to_tf2.py | 12 ++++++++---- .../modeling_tf_pytorch_utils.py | 16 ++++++++++++++++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py b/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py index ca6089bd04..25ed70c2db 100644 --- a/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py +++ b/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py @@ -102,7 +102,7 @@ def convert_pt_checkpoint_to_tf(model_type, pytorch_checkpoint_path, config_file tf_model.save_weights(tf_dump_path, save_format='h5') -def convert_all_pt_checkpoints_to_tf(args_model_type, tf_dump_path, compare_with_pt_model=False): +def convert_all_pt_checkpoints_to_tf(args_model_type, tf_dump_path, compare_with_pt_model=False, use_cached_models=False): assert os.path.isdir(args.tf_dump_path), "--tf_dump_path should be a directory" if args_model_type is None: @@ -126,8 +126,8 @@ def convert_all_pt_checkpoints_to_tf(args_model_type, tf_dump_path, compare_with if 'finetuned' in shortcut_name: print(" Skipping finetuned checkpoint ") continue - config_file = cached_path(aws_config_map[shortcut_name], force_download=True) - model_file = cached_path(aws_model_maps[shortcut_name], force_download=True) + config_file = cached_path(aws_config_map[shortcut_name], force_download=not use_cached_models) + model_file = cached_path(aws_model_maps[shortcut_name], force_download=not use_cached_models) convert_pt_checkpoint_to_tf(model_type, model_file, @@ -165,6 +165,9 @@ if __name__ == "__main__": parser.add_argument("--compare_with_pt_model", action='store_true', help = "Compare Tensorflow and PyTorch model predictions.") + parser.add_argument("--use_cached_models", + action='store_true', + help = "Use cached models if possible instead of updating to latest checkpoint versions.") args = parser.parse_args() if args.pytorch_checkpoint_path is not None: @@ -176,4 +179,5 @@ if __name__ == "__main__": else: convert_all_pt_checkpoints_to_tf(args.model_type.lower() if args.model_type is not None else None, args.tf_dump_path, - compare_with_pt_model=args.compare_with_pt_model) + compare_with_pt_model=args.compare_with_pt_model, + use_cached_models=args.use_cached_models) diff --git a/pytorch_transformers/modeling_tf_pytorch_utils.py b/pytorch_transformers/modeling_tf_pytorch_utils.py index 2420b69fb9..9950a5a73f 100644 --- a/pytorch_transformers/modeling_tf_pytorch_utils.py +++ b/pytorch_transformers/modeling_tf_pytorch_utils.py @@ -78,6 +78,12 @@ def load_pytorch_state_dict_in_tf2_model(tf_model, pt_state_dict, tf_inputs=None for old_key, new_key in zip(old_keys, new_keys): pt_state_dict[new_key] = pt_state_dict.pop(old_key) + # Make sure we are able to load PyTorch base models as well as derived models (with heads) + # TF models always have a prefix, some of PyTorch models (base ones) don't + start_prefix_to_remove = '' + if not any(s.startswith(tf_model.base_model_prefix) for s in pt_state_dict.keys()): + start_prefix_to_remove = tf_model.base_model_prefix + '.' + symbolic_weights = tf_model.trainable_weights + tf_model.non_trainable_weights weight_value_tuples = [] @@ -100,13 +106,23 @@ def load_pytorch_state_dict_in_tf2_model(tf_model, pt_state_dict, tf_inputs=None if name[-1] == 'beta': name[-1] = 'bias' + # Remove prefix if needed name = '.'.join(name) + if start_prefix_to_remove: + name = name.replace(start_prefix_to_remove, '', 1) + + # Find associated numpy array in pytorch model state dict assert name in pt_state_dict, "{} not found in PyTorch model".format(name) array = pt_state_dict[name].numpy() if transpose: array = numpy.transpose(array) + if len(symbolic_weight.shape) < len(array.shape): + array = numpy.squeeze(array) + elif len(symbolic_weight.shape) > len(array.shape): + array = numpy.expand_dims(array, axis=0) + try: assert list(symbolic_weight.shape) == list(array.shape) except AssertionError as e: From 26497d119911d0ab481c68cc2517ae121ce1db9a Mon Sep 17 00:00:00 2001 From: thomwolf Date: Wed, 18 Sep 2019 12:17:21 +0200 Subject: [PATCH 055/219] fix tests --- pytorch_transformers/tests/modeling_tf_common_test.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pytorch_transformers/tests/modeling_tf_common_test.py b/pytorch_transformers/tests/modeling_tf_common_test.py index 3dae24b283..46360d1dd4 100644 --- a/pytorch_transformers/tests/modeling_tf_common_test.py +++ b/pytorch_transformers/tests/modeling_tf_common_test.py @@ -262,7 +262,7 @@ class TFCommonTestCases: # self.assertEqual(len(params_tied_2), len(params_tied)) -def ids_tensor(shape, vocab_size, rng=None, name=None, dtype=tf.int32): +def ids_tensor(shape, vocab_size, rng=None, name=None, dtype=None): """Creates a random int32 tensor of the shape within the vocab size.""" if rng is None: rng = random.Random() @@ -275,7 +275,11 @@ def ids_tensor(shape, vocab_size, rng=None, name=None, dtype=tf.int32): for _ in range(total_dims): values.append(rng.randint(0, vocab_size - 1)) - return tf.constant(values, shape=shape, dtype=dtype) + output = tf.constant(values, + shape=shape, + dtype=dtype if dtype is not None else tf.int32) + + return output class TFModelUtilsTest(unittest.TestCase): From 160b5d60807afbe89b578e6db844ebe63400a1c7 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Wed, 18 Sep 2019 14:10:20 +0200 Subject: [PATCH 056/219] fix xlm lang_embeddings loading --- pytorch_transformers/modeling_tf_xlm.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pytorch_transformers/modeling_tf_xlm.py b/pytorch_transformers/modeling_tf_xlm.py index 7d04443745..1bbad55a9d 100644 --- a/pytorch_transformers/modeling_tf_xlm.py +++ b/pytorch_transformers/modeling_tf_xlm.py @@ -47,10 +47,13 @@ TF_XLM_PRETRAINED_MODEL_ARCHIVE_MAP = { def load_xlm_pt_weights_in_tf2(tf_model, pytorch_checkpoint_path): # build the network - inputs_list = [[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]] - attns_list = [[1, 1, 0, 0, 1], [1, 1, 1, 0, 0], [1, 0, 0, 1, 1]] - langs_list = [[1, 1, 0, 0, 1], [1, 1, 1, 0, 0], [1, 0, 0, 1, 1]] - tf_inputs = [tf.constant(inputs_list), tf.constant(attns_list), tf.constant(langs_list)] + inputs_list = tf.constant([[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]]) + attns_list = tf.constant([[1, 1, 0, 0, 1], [1, 1, 1, 0, 0], [1, 0, 0, 1, 1]]) + if tf_model.config.use_lang_emb and tf_model.config.n_langs > 1: + langs_list = tf.constant([[1, 1, 0, 0, 1], [1, 1, 1, 0, 0], [1, 0, 0, 1, 1]]) + else: + langs_list = None + tf_inputs = [inputs_list, attns_list), langs_list] tfo = tf_model(tf_inputs, training=False) return load_pytorch_checkpoint_in_tf2_model(tf_model, pytorch_checkpoint_path, tf_inputs=tf_inputs) From 556442afb3005d837a7b46e0f50ca396488542d2 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Wed, 18 Sep 2019 14:12:41 +0200 Subject: [PATCH 057/219] hot fix --- pytorch_transformers/modeling_tf_xlm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_transformers/modeling_tf_xlm.py b/pytorch_transformers/modeling_tf_xlm.py index 1bbad55a9d..94daa1f04b 100644 --- a/pytorch_transformers/modeling_tf_xlm.py +++ b/pytorch_transformers/modeling_tf_xlm.py @@ -53,7 +53,7 @@ def load_xlm_pt_weights_in_tf2(tf_model, pytorch_checkpoint_path): langs_list = tf.constant([[1, 1, 0, 0, 1], [1, 1, 1, 0, 0], [1, 0, 0, 1, 1]]) else: langs_list = None - tf_inputs = [inputs_list, attns_list), langs_list] + tf_inputs = [inputs_list, attns_list, langs_list] tfo = tf_model(tf_inputs, training=False) return load_pytorch_checkpoint_in_tf2_model(tf_model, pytorch_checkpoint_path, tf_inputs=tf_inputs) From 3a527fa820d6ef99cff637c2e2923442805f5fdf Mon Sep 17 00:00:00 2001 From: thomwolf Date: Wed, 18 Sep 2019 14:15:48 +0200 Subject: [PATCH 058/219] OpenAI GPT tests ok --- pytorch_transformers/configuration_xlm.py | 3 - pytorch_transformers/configuration_xlnet.py | 4 +- pytorch_transformers/modeling_tf_gpt2.py | 7 +- pytorch_transformers/modeling_tf_openai.py | 558 ++++++++++++++++++ .../tests/modeling_tf_openai_gpt_test.py | 231 ++++++++ 5 files changed, 793 insertions(+), 10 deletions(-) create mode 100644 pytorch_transformers/modeling_tf_openai.py create mode 100644 pytorch_transformers/tests/modeling_tf_openai_gpt_test.py diff --git a/pytorch_transformers/configuration_xlm.py b/pytorch_transformers/configuration_xlm.py index ab251c8939..fa3a5f40f6 100644 --- a/pytorch_transformers/configuration_xlm.py +++ b/pytorch_transformers/configuration_xlm.py @@ -56,8 +56,6 @@ class XLMConfig(PretrainedConfig): dropout: The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler. - dropatt: The dropout ratio for the attention - probabilities. max_position_embeddings: The maximum sequence length that this model might ever be used with. Typically set this to something large just in case (e.g., 512 or 1024 or 2048). @@ -66,7 +64,6 @@ class XLMConfig(PretrainedConfig): layer_norm_eps: The epsilon used by LayerNorm. dropout: float, dropout rate. - dropatt: float, dropout rate on attention probabilities. init: str, the initialization scheme, either "normal" or "uniform". init_range: float, initialize the parameters with a uniform distribution in [-init_range, init_range]. Only effective when init="uniform". diff --git a/pytorch_transformers/configuration_xlnet.py b/pytorch_transformers/configuration_xlnet.py index cb325dfe17..0dbf518849 100644 --- a/pytorch_transformers/configuration_xlnet.py +++ b/pytorch_transformers/configuration_xlnet.py @@ -49,14 +49,11 @@ class XLNetConfig(PretrainedConfig): dropout: The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler. - dropatt: The dropout ratio for the attention - probabilities. initializer_range: The sttdev of the truncated_normal_initializer for initializing all weight matrices. layer_norm_eps: The epsilon used by LayerNorm. dropout: float, dropout rate. - dropatt: float, dropout rate on attention probabilities. init: str, the initialization scheme, either "normal" or "uniform". init_range: float, initialize the parameters with a uniform distribution in [-init_range, init_range]. Only effective when init="uniform". @@ -80,6 +77,7 @@ class XLNetConfig(PretrainedConfig): n_layer=24, n_head=16, d_inner=4096, + max_position_embeddings=512, ff_activation="gelu", untie_r=True, attn_type="bi", diff --git a/pytorch_transformers/modeling_tf_gpt2.py b/pytorch_transformers/modeling_tf_gpt2.py index 8bb5eb52f9..e0a0a16799 100644 --- a/pytorch_transformers/modeling_tf_gpt2.py +++ b/pytorch_transformers/modeling_tf_gpt2.py @@ -249,7 +249,7 @@ class TFGPT2MainLayer(tf.keras.layers.Layer): token_type_ids = inputs.get('token_type_ids', None) position_ids = inputs.get('position_ids', None) head_mask = inputs.get('head_mask', None) - assert len(inputs) <= 5, "Too many inputs." + assert len(inputs) <= 6, "Too many inputs." if past is None: past_length = 0 @@ -551,7 +551,6 @@ class TFGPT2DoubleHeadsModel(TFGPT2PreTrainedModel): self.transformer = TFGPT2MainLayer(config, name='transformer') self.multiple_choice_head = TFSequenceSummary(config, name='multiple_choice_head') - def call(self, inputs, training=False): if not isinstance(inputs, (dict, tuple, list)): input_ids = inputs @@ -573,7 +572,7 @@ class TFGPT2DoubleHeadsModel(TFGPT2PreTrainedModel): token_type_ids = inputs.get('token_type_ids', None) position_ids = inputs.get('position_ids', None) head_mask = inputs.get('head_mask', None) - assert len(inputs) <= 5, "Too many inputs." + assert len(inputs) <= 7, "Too many inputs." input_shapes = shape_list(input_ids) @@ -598,4 +597,4 @@ class TFGPT2DoubleHeadsModel(TFGPT2PreTrainedModel): outputs = (lm_logits, mc_logits) + transformer_outputs[1:] - return outputs # (lm loss), (mc loss), lm logits, mc logits, presents, (all hidden_states), (attentions) + return outputs # lm logits, mc logits, presents, (all hidden_states), (attentions) diff --git a/pytorch_transformers/modeling_tf_openai.py b/pytorch_transformers/modeling_tf_openai.py new file mode 100644 index 0000000000..2292d4cab8 --- /dev/null +++ b/pytorch_transformers/modeling_tf_openai.py @@ -0,0 +1,558 @@ +# coding=utf-8 +# Copyright 2018 The OpenAI Team Authors and HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" TF 2.0 OpenAI GPT model.""" + +from __future__ import absolute_import, division, print_function, unicode_literals + +import collections +import json +import logging +import math +import os +import sys +from io import open + +import numpy as np +import tensorflow as tf + +from .modeling_tf_utils import (TFPreTrainedModel, TFConv1D, TFSharedEmbeddings, + TFSequenceSummary, shape_list) +from .configuration_openai import OpenAIGPTConfig +from .file_utils import add_start_docstrings +from .modeling_tf_pytorch_utils import load_pytorch_checkpoint_in_tf2_model + +logger = logging.getLogger(__name__) + +TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP = {"openai-gpt": "https://s3.amazonaws.com/models.huggingface.co/bert/openai-gpt-tf_model.h5"} + + +def load_openai_gpt_pt_weights_in_tf2(tf_model, pytorch_checkpoint_path): + # build the network + inputs_list = [[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]] + tf_inputs = tf.constant(inputs_list) + tfo = tf_model(tf_inputs, training=False) + return load_pytorch_checkpoint_in_tf2_model(tf_model, pytorch_checkpoint_path, tf_inputs=tf_inputs) + + +def gelu(x): + """Gaussian Error Linear Unit. + This is a smoother version of the RELU. + Original paper: https://arxiv.org/abs/1606.08415 + Args: + x: float Tensor to perform activation. + Returns: + `x` with the GELU activation applied. + """ + cdf = 0.5 * (1.0 + tf.tanh( + (np.sqrt(2 / np.pi) * (x + 0.044715 * tf.pow(x, 3))))) + return x * cdf + + +def swish(x): + return x * tf.math.sigmoid(x) + + +ACT_FNS = {"gelu": tf.keras.layers.Activation(gelu), + "relu": tf.keras.activations.relu, + "swish": tf.keras.layers.Activation(swish)} + + +class TFAttention(tf.keras.layers.Layer): + def __init__(self, nx, n_ctx, config, scale=False, **kwargs): + super(TFAttention, self).__init__(**kwargs) + self.output_attentions = config.output_attentions + + n_state = nx # in Attention: n_state=768 (nx=n_embd) + # [switch nx => n_state from Block to Attention to keep identical to TF implem] + assert n_state % config.n_head == 0 + self.n_ctx = n_ctx + self.n_head = config.n_head + self.split_size = n_state + self.scale = scale + + self.c_attn = TFConv1D(n_state * 3, nx, name='c_attn') + self.c_proj = TFConv1D(n_state, nx, name='c_proj') + self.attn_dropout = tf.keras.layers.Dropout(config.attn_pdrop) + self.resid_dropout = tf.keras.layers.Dropout(config.resid_pdrop) + self.pruned_heads = set() + + def prune_heads(self, heads): + pass + + @staticmethod + def causal_attention_mask(nd, ns, dtype): + """1's in the lower triangle, counting from the lower right corner. + Same as tf.matrix_band_part(tf.ones([nd, ns]), -1, ns-nd), but doesn't produce garbage on TPUs. + """ + i = tf.range(nd)[:,None] + j = tf.range(ns) + m = i >= j - ns + nd + return tf.cast(m, dtype) + + def _attn(self, inputs, training=False): + q, k, v, attention_mask, head_mask = inputs + # q, k, v have shape [batch, heads, sequence, features] + w = tf.matmul(q, k, transpose_b=True) + if self.scale: + dk = tf.cast(tf.shape(k)[-1], tf.float32) # scale attention_scores + w = w / tf.math.sqrt(dk) + + # w has shape [batch, heads, dst_sequence, src_sequence], where information flows from src to dst. + _, _, nd, ns = shape_list(w) + b = self.causal_attention_mask(nd, ns, dtype=w.dtype) + b = tf.reshape(b, [1, 1, nd, ns]) + w = w * b - 1e4 * (1 - b) + + if attention_mask is not None: + # Apply the attention mask + w = w + attention_mask + + w = tf.nn.softmax(w, axis=-1) + w = self.attn_dropout(w, training=training) + + # Mask heads if we want to + if head_mask is not None: + w = w * head_mask + + outputs = [tf.matmul(w, v)] + if self.output_attentions: + outputs.append(w) + return outputs + + def merge_heads(self, x): + x = tf.transpose(x, [0, 2, 1, 3]) + x_shape = shape_list(x) + new_x_shape = x_shape[:-2] + [x_shape[-2] * x_shape[-1]] + return tf.reshape(x, new_x_shape) + + def split_heads(self, x): + x_shape = shape_list(x) + new_x_shape = x_shape[:-1] + [self.n_head, x_shape[-1] // self.n_head] + x = tf.reshape(x, new_x_shape) + return tf.transpose(x, (0, 2, 1, 3)) # (batch, head, seq_length, head_features) + + def call(self, inputs, training=False): + x, attention_mask, head_mask = inputs + + x = self.c_attn(x) + query, key, value = tf.split(x, 3, axis=2) + query = self.split_heads(query) + key = self.split_heads(key) + value = self.split_heads(value) + + attn_outputs = self._attn([query, key, value, attention_mask, head_mask], training=training) + a = attn_outputs[0] + + a = self.merge_heads(a) + a = self.c_proj(a) + a = self.resid_dropout(a, training=training) + + outputs = [a] + attn_outputs[1:] + return outputs # a, (attentions) + + +class TFMLP(tf.keras.layers.Layer): + def __init__(self, n_state, config, **kwargs): + super(TFMLP, self).__init__(**kwargs) + nx = config.n_embd + self.c_fc = TFConv1D(n_state, nx, name='c_fc') + self.c_proj = TFConv1D(nx, n_state, name='c_proj') + self.act = gelu + self.dropout = tf.keras.layers.Dropout(config.resid_pdrop) + + def call(self, x, training=False): + h = self.act(self.c_fc(x)) + h2 = self.c_proj(h) + h2 = self.dropout(h2, training=training) + return h2 + + +class TFBlock(tf.keras.layers.Layer): + def __init__(self, n_ctx, config, scale=False, **kwargs): + super(TFBlock, self).__init__(**kwargs) + nx = config.n_embd + self.attn = TFAttention(nx, n_ctx, config, scale, name='attn') + self.ln_1 = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_epsilon, name='ln_1') + self.mlp = TFMLP(4 * nx, config, name='mlp') + self.ln_2 = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_epsilon, name='ln_2') + + def call(self, inputs, training=False): + x, attention_mask, head_mask = inputs + + output_attn = self.attn([x, attention_mask, head_mask], training=training) + a = output_attn[0] # output_attn: a, (attentions) + + n = self.ln_1(x + a) + m = self.mlp(n, training=training) + h = self.ln_2(n + m) + + outputs = [h] + output_attn[1:] + return outputs # x, (attentions) + + +class TFOpenAIGPTMainLayer(tf.keras.layers.Layer): + def __init__(self, config, *inputs, **kwargs): + super(TFOpenAIGPTMainLayer, self).__init__(config, *inputs, **kwargs) + self.output_hidden_states = config.output_hidden_states + self.output_attentions = config.output_attentions + self.num_hidden_layers = config.n_layer + self.vocab_size = config.vocab_size + self.n_embd = config.n_embd + + self.tokens_embed = TFSharedEmbeddings(config.vocab_size, config.n_embd, name='tokens_embed') + self.positions_embed = tf.keras.layers.Embedding(config.n_positions, config.n_embd, name='positions_embed') + self.drop = tf.keras.layers.Dropout(config.embd_pdrop) + self.h = [TFBlock(config.n_ctx, + config, + scale=True, + name='h_._{}'.format(i)) for i in range(config.n_layer)] + + def _resize_token_embeddings(self, new_num_tokens): + raise NotImplementedError + + def _prune_heads(self, heads_to_prune): + """ Prunes heads of the model. + heads_to_prune: dict of {layer_num: list of heads to prune in this layer} + """ + raise NotImplementedError + + def call(self, inputs, training=False): + if not isinstance(inputs, (dict, tuple, list)): + input_ids = inputs + attention_mask, token_type_ids, position_ids, head_mask = None, None, None, None + elif isinstance(inputs, (tuple, list)): + input_ids = inputs[0] + attention_mask = inputs[1] if len(inputs) > 1 else None + token_type_ids = inputs[2] if len(inputs) > 2 else None + position_ids = inputs[3] if len(inputs) > 3 else None + head_mask = inputs[4] if len(inputs) > 4 else None + assert len(inputs) <= 5, "Too many inputs." + else: + input_ids = inputs.get('input_ids') + attention_mask = inputs.get('attention_mask', None) + token_type_ids = inputs.get('token_type_ids', None) + position_ids = inputs.get('position_ids', None) + head_mask = inputs.get('head_mask', None) + assert len(inputs) <= 5, "Too many inputs." + + if position_ids is None: + position_ids = tf.range(shape_list(input_ids)[-1], dtype=tf.int32)[tf.newaxis, :] + + if attention_mask is not None: + # We create a 3D attention mask from a 2D tensor mask. + # Sizes are [batch_size, 1, 1, to_seq_length] + # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length] + # this attention mask is more simple than the triangular masking of causal attention + # used in OpenAI GPT, we just need to prepare the broadcast dimension here. + attention_mask = attention_mask[:, tf.newaxis, tf.newaxis, :] + + # Since attention_mask is 1.0 for positions we want to attend and 0.0 for + # masked positions, this operation will create a tensor which is 0.0 for + # positions we want to attend and -10000.0 for masked positions. + # Since we are adding it to the raw scores before the softmax, this is + # effectively the same as removing these entirely. + + attention_mask = tf.cast(attention_mask, tf.float32) + attention_mask = (1.0 - attention_mask) * -10000.0 + else: + attention_mask = None + + # Prepare head mask if needed + # 1.0 in head_mask indicate we keep the head + # attention_probs has shape bsz x n_heads x N x N + # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] + # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length] + if not head_mask is None: + raise NotImplementedError + else: + head_mask = [None] * self.num_hidden_layers + # head_mask = tf.constant([0] * self.num_hidden_layers) + + input_shape = shape_list(input_ids) + input_ids = tf.reshape(input_ids, [-1, input_shape[-1]]) + position_ids = tf.reshape(position_ids, [-1, shape_list(position_ids)[-1]]) + + inputs_embeds = self.tokens_embed(input_ids, mode='embedding') + position_embeds = self.positions_embed(position_ids) + if token_type_ids is not None: + token_type_ids = tf.reshape(token_type_ids, [-1, shape_list(token_type_ids)[-1]]) + token_type_embeds = self.tokens_embed(token_type_ids, mode='embedding') + else: + token_type_embeds = 0 + hidden_states = inputs_embeds + position_embeds + token_type_embeds + hidden_states = self.drop(hidden_states, training=training) + + output_shape = input_shape + [shape_list(hidden_states)[-1]] + + all_attentions = [] + all_hidden_states = () + for i, block in enumerate(self.h): + if self.output_hidden_states: + all_hidden_states = all_hidden_states + (tf.reshape(hidden_states, output_shape),) + + outputs = block([hidden_states, attention_mask, head_mask[i]], training=training) + hidden_states = outputs[0] + if self.output_attentions: + all_attentions.append(outputs[1]) + + hidden_states = tf.reshape(hidden_states, output_shape) + # Add last hidden state + if self.output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + outputs = (hidden_states,) + if self.output_hidden_states: + outputs = outputs + (all_hidden_states,) + if self.output_attentions: + # let the number of heads free (-1) so we can extract attention even after head pruning + attention_output_shape = input_shape[:-1] + [-1] + shape_list(all_attentions[0])[-2:] + all_attentions = tuple(tf.reshape(t, attention_output_shape) for t in all_attentions) + outputs = outputs + (all_attentions,) + return outputs # last hidden state, (all hidden_states), (attentions) + + +class TFOpenAIGPTPreTrainedModel(TFPreTrainedModel): + """ An abstract class to handle weights initialization and + a simple interface for dowloading and loading pretrained models. + """ + config_class = OpenAIGPTConfig + pretrained_model_archive_map = TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP + load_pt_weights = load_openai_gpt_pt_weights_in_tf2 + base_model_prefix = "transformer" + + +OPENAI_GPT_START_DOCSTRING = r""" OpenAI GPT model was proposed in + `Improving Language Understanding by Generative Pre-Training`_ + by Alec Radford, Karthik Narasimhan, Tim Salimans and Ilya Sutskever. + It's a causal (unidirectional) transformer pre-trained using language modeling on a large + corpus will long range dependencies, the Toronto Book Corpus. + + This model is a tf.keras.Model `tf.keras.Model`_ sub-class. Use it as a regular TF 2.0 Keras Model and + refer to the TF 2.0 documentation for all matter related to general usage and behavior. + + .. _`Improving Language Understanding by Generative Pre-Training`: + https://openai.com/blog/language-unsupervised/ + + .. _`tf.keras.Model`: + https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/Model + + Important note on the model inputs: + The inputs of the TF 2.0 models are slightly different from the PyTorch ones since + TF 2.0 Keras doesn't accept named arguments with defaults values for input Tensor. + More precisely, input Tensors are gathered in the first arguments of the model call function: `model(inputs)`. + There are three possibilities to gather and feed the inputs to the model: + + - a single Tensor with input_ids only and nothing else: `model(inputs_ids) + - a list of varying length with one or several input Tensors IN THE ORDER given in the docstring: + `model([input_ids, attention_mask])` or `model([input_ids, attention_mask, token_type_ids])` + - a dictionary with one or several input Tensors associaed to the input names given in the docstring: + `model({'input_ids': input_ids, 'token_type_ids': token_type_ids})` + + Parameters: + config (:class:`~pytorch_transformers.OpenAIGPTConfig`): Model configuration class with all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the configuration. + Check out the :meth:`~pytorch_transformers.PreTrainedModel.from_pretrained` method to load the model weights. +""" + +OPENAI_GPT_INPUTS_DOCSTRING = r""" Inputs: + **input_ids**: ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Indices of input sequence tokens in the vocabulary. + GPT is a model with absolute position embeddings so it's usually advised to pad the inputs on + the right rather than the left. + Indices can be obtained using :class:`pytorch_transformers.BPT2Tokenizer`. + See :func:`pytorch_transformers.PreTrainedTokenizer.encode` and + :func:`pytorch_transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``: + Mask to avoid performing attention on padding token indices. + Mask values selected in ``[0, 1]``: + ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. + **token_type_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + A parallel sequence of tokens (can be used to indicate various portions of the inputs). + The embeddings from these tokens will be summed with the respective token embeddings. + Indices are selected in the vocabulary (unlike BERT which has a specific vocabulary for segment indices) + **position_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Indices of positions of each input sequence tokens in the position embeddings. + Selected in the range ``[0, config.max_position_embeddings - 1]``. + **head_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: + Mask to nullify selected heads of the self-attention modules. + Mask values selected in ``[0, 1]``: + ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. +""" + +@add_start_docstrings("The bare OpenAI GPT transformer model outputing raw hidden-states without any specific head on top.", + OPENAI_GPT_START_DOCSTRING, OPENAI_GPT_INPUTS_DOCSTRING) +class TFOpenAIGPTModel(TFOpenAIGPTPreTrainedModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)`` + Sequence of hidden-states at the last layer of the model. + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = OpenAIGPTTokenizer.from_pretrained('openai-gpt') + model = OpenAIGPTModel.from_pretrained('openai-gpt') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids) + last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple + + """ + def __init__(self, config, *inputs, **kwargs): + super(TFOpenAIGPTModel, self).__init__(config, *inputs, **kwargs) + self.transformer = TFOpenAIGPTMainLayer(config, name='transformer') + + def call(self, inputs, training=False): + outputs = self.transformer(inputs, training=training) + return outputs + + +@add_start_docstrings("""OpenAI GPT Model transformer with a language modeling head on top +(linear layer with weights tied to the input embeddings). """, OPENAI_GPT_START_DOCSTRING, OPENAI_GPT_INPUTS_DOCSTRING) +class TFOpenAIGPTLMHeadModel(TFOpenAIGPTPreTrainedModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Language modeling loss. + **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = OpenAIGPTTokenizer.from_pretrained('openai-gpt') + model = OpenAIGPTLMHeadModel.from_pretrained('openai-gpt') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, labels=input_ids) + loss, logits = outputs[:2] + + """ + def __init__(self, config, *inputs, **kwargs): + super(TFOpenAIGPTLMHeadModel, self).__init__(config, *inputs, **kwargs) + self.transformer = TFOpenAIGPTMainLayer(config, name='transformer') + + def call(self, inputs, training=False): + transformer_outputs = self.transformer(inputs, training=training) + hidden_states = transformer_outputs[0] + + lm_logits = self.transformer.tokens_embed(hidden_states, mode="linear") + + outputs = (lm_logits,) + transformer_outputs[1:] + + return outputs # lm_logits, (all hidden_states), (attentions) + + +@add_start_docstrings("""OpenAI GPT Model transformer with a language modeling and a multiple-choice classification +head on top e.g. for RocStories/SWAG tasks. The two heads are two linear layers. +The language modeling head has its weights tied to the input embeddings, +the classification head takes as input the input of a specified classification token index in the input sequence). +""", OPENAI_GPT_START_DOCSTRING, OPENAI_GPT_INPUTS_DOCSTRING) +class TFOpenAIGPTDoubleHeadsModel(TFOpenAIGPTPreTrainedModel): + r""" + **mc_token_ids**: (`optional`, default to index of the last token of the input) ``torch.LongTensor`` of shape ``(batch_size, num_choices)``: + Index of the classification token in each input sequence. + Selected in the range ``[0, input_ids.size(-1) - 1[``. + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **lm_loss**: (`optional`, returned when ``lm_labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Language modeling loss. + **mc_loss**: (`optional`, returned when ``multiple_choice_labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Multiple choice classification loss. + **lm_prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, num_choices, sequence_length, config.vocab_size)`` + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + **mc_prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, num_choices)`` + Prediction scores of the multiplechoice classification head (scores for each choice before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = OpenAIGPTTokenizer.from_pretrained('openai-gpt') + model = OpenAIGPTDoubleHeadsModel.from_pretrained('openai-gpt') + tokenizer.add_special_tokens({'cls_token': '[CLS]'}) # Add a [CLS] to the vocabulary (we should train it also!) + choices = ["Hello, my dog is cute [CLS]", "Hello, my cat is cute [CLS]"] + input_ids = torch.tensor([tokenizer.encode(s) for s in choices]).unsqueeze(0) # Batch size 1, 2 choices + mc_token_ids = torch.tensor([input_ids.size(-1), input_ids.size(-1)]).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, mc_token_ids=mc_token_ids) + lm_prediction_scores, mc_prediction_scores = outputs[:2] + + """ + def __init__(self, config, *inputs, **kwargs): + super(TFOpenAIGPTDoubleHeadsModel, self).__init__(config, *inputs, **kwargs) + self.transformer = TFOpenAIGPTMainLayer(config, name='transformer') + self.multiple_choice_head = TFSequenceSummary(config, name='multiple_choice_head') + + def call(self, inputs, training=False): + if not isinstance(inputs, (dict, tuple, list)): + input_ids = inputs + mc_token_ids, attention_mask, token_type_ids, position_ids, head_mask = None, None, None, None + elif isinstance(inputs, (tuple, list)): + input_ids = inputs[0] + mc_token_ids = inputs[1] if len(inputs) > 1 else None + attention_mask = inputs[2] if len(inputs) > 2 else None + token_type_ids = inputs[3] if len(inputs) > 3 else None + position_ids = inputs[4] if len(inputs) > 4 else None + head_mask = inputs[5] if len(inputs) > 5 else None + assert len(inputs) <= 6, "Too many inputs." + else: + input_ids = inputs.get('input_ids') + mc_token_ids = inputs.get('mc_token_ids', None) + attention_mask = inputs.get('attention_mask', None) + token_type_ids = inputs.get('token_type_ids', None) + position_ids = inputs.get('position_ids', None) + head_mask = inputs.get('head_mask', None) + assert len(inputs) <= 6, "Too many inputs." + + input_shapes = shape_list(input_ids) + + seq_length = input_shapes[-1] + + flat_input_ids = tf.reshape(input_ids, (-1, seq_length)) + flat_attention_mask = tf.reshape(attention_mask, (-1, seq_length)) if attention_mask is not None else None + flat_token_type_ids = tf.reshape(token_type_ids, (-1, seq_length)) if token_type_ids is not None else None + flat_position_ids = tf.reshape(position_ids, (-1, seq_length)) if position_ids is not None else None + + flat_inputs = [flat_input_ids, flat_attention_mask, flat_token_type_ids, flat_position_ids, head_mask] + + transformer_outputs = self.transformer(flat_inputs, training=training) + hidden_states = transformer_outputs[0] + + hidden_states = tf.reshape(hidden_states, input_shapes + shape_list(hidden_states)[-1:]) + + lm_logits = self.transformer.tokens_embed(hidden_states, mode="linear") + mc_logits = self.multiple_choice_head([hidden_states, mc_token_ids], training=training) + + mc_logits = tf.squeeze(mc_logits, axis=-1) + + outputs = (lm_logits, mc_logits) + transformer_outputs[1:] + + return outputs # lm logits, mc logits, (all hidden_states), (attentions) diff --git a/pytorch_transformers/tests/modeling_tf_openai_gpt_test.py b/pytorch_transformers/tests/modeling_tf_openai_gpt_test.py new file mode 100644 index 0000000000..c553209db3 --- /dev/null +++ b/pytorch_transformers/tests/modeling_tf_openai_gpt_test.py @@ -0,0 +1,231 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import unittest +import shutil +import pytest +import sys + +from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) +from .configuration_common_test import ConfigTester + +from pytorch_transformers import OpenAIGPTConfig, is_tf_available + +if is_tf_available(): + import tensorflow as tf + from pytorch_transformers.modeling_tf_openai import (TFOpenAIGPTModel, TFOpenAIGPTLMHeadModel, + TFOpenAIGPTDoubleHeadsModel, + TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP) +else: + pytestmark = pytest.mark.skip("Require TensorFlow") + + +class TFOpenAIGPTModelTest(TFCommonTestCases.TFCommonModelTester): + + all_model_classes = (TFOpenAIGPTModel, TFOpenAIGPTLMHeadModel, + TFOpenAIGPTDoubleHeadsModel) if is_tf_available() else () + + class TFOpenAIGPTModelTester(object): + + def __init__(self, + parent, + batch_size=13, + seq_length=7, + is_training=True, + use_token_type_ids=True, + use_input_mask=True, + use_labels=True, + use_mc_token_ids=True, + vocab_size=99, + hidden_size=32, + num_hidden_layers=5, + num_attention_heads=4, + intermediate_size=37, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + type_sequence_label_size=2, + initializer_range=0.02, + num_labels=3, + num_choices=4, + scope=None, + ): + self.parent = parent + self.batch_size = batch_size + self.seq_length = seq_length + self.is_training = is_training + self.use_token_type_ids = use_token_type_ids + self.use_input_mask = use_input_mask + self.use_labels = use_labels + self.use_mc_token_ids = use_mc_token_ids + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.intermediate_size = intermediate_size + self.hidden_act = hidden_act + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.type_sequence_label_size = type_sequence_label_size + self.initializer_range = initializer_range + self.num_labels = num_labels + self.num_choices = num_choices + self.scope = scope + + def prepare_config_and_inputs(self): + input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) + + input_mask = None + if self.use_input_mask: + input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2) + + token_type_ids = None + if self.use_token_type_ids: + token_type_ids = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size) + + mc_token_ids = None + if self.use_mc_token_ids: + mc_token_ids = ids_tensor([self.batch_size, self.num_choices], self.seq_length) + + sequence_labels = None + token_labels = None + choice_labels = None + if self.use_labels: + sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size) + token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels) + choice_labels = ids_tensor([self.batch_size], self.num_choices) + + config = OpenAIGPTConfig( + vocab_size_or_config_json_file=self.vocab_size, + n_embd=self.hidden_size, + n_layer=self.num_hidden_layers, + n_head=self.num_attention_heads, + # intermediate_size=self.intermediate_size, + # hidden_act=self.hidden_act, + # hidden_dropout_prob=self.hidden_dropout_prob, + # attention_probs_dropout_prob=self.attention_probs_dropout_prob, + n_positions=self.max_position_embeddings, + n_ctx=self.max_position_embeddings + # type_vocab_size=self.type_vocab_size, + # initializer_range=self.initializer_range + ) + + head_mask = ids_tensor([self.num_hidden_layers, self.num_attention_heads], 2) + + return config, input_ids, input_mask, head_mask, token_type_ids, mc_token_ids, sequence_labels, token_labels, choice_labels + + def create_and_check_openai_gpt_model(self, config, input_ids, input_mask, head_mask, token_type_ids, *args): + model = TFOpenAIGPTModel(config=config) + inputs = {'input_ids': input_ids, + 'attention_mask': input_mask, + 'token_type_ids': token_type_ids} + sequence_output = model(inputs)[0] + + inputs = [input_ids, input_mask] + sequence_output = model(inputs)[0] + + sequence_output = model(input_ids)[0] + + result = { + "sequence_output": sequence_output.numpy(), + } + self.parent.assertListEqual( + list(result["sequence_output"].shape), + [self.batch_size, self.seq_length, self.hidden_size]) + + + def create_and_check_openai_gpt_lm_head(self, config, input_ids, input_mask, head_mask, token_type_ids, *args): + model = TFOpenAIGPTLMHeadModel(config=config) + inputs = {'input_ids': input_ids, + 'attention_mask': input_mask, + 'token_type_ids': token_type_ids} + prediction_scores = model(inputs)[0] + result = { + "prediction_scores": prediction_scores.numpy(), + } + self.parent.assertListEqual( + list(result["prediction_scores"].shape), + [self.batch_size, self.seq_length, self.vocab_size]) + + + def create_and_check_openai_gpt_double_head(self, config, input_ids, input_mask, head_mask, token_type_ids, mc_token_ids, *args): + model = TFOpenAIGPTDoubleHeadsModel(config=config) + + multiple_choice_inputs_ids = tf.tile(tf.expand_dims(input_ids, 1), (1, self.num_choices, 1)) + multiple_choice_input_mask = tf.tile(tf.expand_dims(input_mask, 1), (1, self.num_choices, 1)) + multiple_choice_token_type_ids = tf.tile(tf.expand_dims(token_type_ids, 1), (1, self.num_choices, 1)) + + inputs = {'input_ids': multiple_choice_inputs_ids, + 'mc_token_ids': mc_token_ids, + 'attention_mask': multiple_choice_input_mask, + 'token_type_ids': multiple_choice_token_type_ids} + lm_logits, mc_logits = model(inputs)[:2] + result = { + "lm_logits": lm_logits.numpy(), + "mc_logits": mc_logits.numpy() + } + self.parent.assertListEqual( + list(result["lm_logits"].shape), + [self.batch_size, self.num_choices, self.seq_length, self.vocab_size]) + self.parent.assertListEqual( + list(result["mc_logits"].shape), + [self.batch_size, self.num_choices]) + + def prepare_config_and_inputs_for_common(self): + config_and_inputs = self.prepare_config_and_inputs() + + (config, input_ids, input_mask, head_mask, token_type_ids, + mc_token_ids, sequence_labels, token_labels, choice_labels) = config_and_inputs + + inputs_dict = {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask} + return config, inputs_dict + + def setUp(self): + self.model_tester = TFOpenAIGPTModelTest.TFOpenAIGPTModelTester(self) + self.config_tester = ConfigTester(self, config_class=OpenAIGPTConfig, n_embd=37) + + def test_config(self): + self.config_tester.run_common_tests() + + def test_openai_gpt_model(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_openai_gpt_model(*config_and_inputs) + + def test_openai_gpt_lm_head(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_openai_gpt_lm_head(*config_and_inputs) + + def test_openai_gpt_double_head(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_openai_gpt_double_head(*config_and_inputs) + + @pytest.mark.slow + def test_model_from_pretrained(self): + cache_dir = "/tmp/pytorch_transformers_test/" + for model_name in list(TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: + model = TFOpenAIGPTModel.from_pretrained(model_name, cache_dir=cache_dir) + shutil.rmtree(cache_dir) + self.assertIsNotNone(model) + +if __name__ == "__main__": + unittest.main() + From ec94f4e0f80d33433cbb2c14fd694af33656b779 Mon Sep 17 00:00:00 2001 From: Simon Layton Date: Wed, 18 Sep 2019 09:30:58 -0400 Subject: [PATCH 059/219] Fix fp16 masking in PoolerEndLogits Necessary to run xlnet (at least in squad) with `--fp16 --fp16_opt_level="O2"`, otherwise loss is immediately `NaN` and fine-tuning cannot proceed. --- pytorch_transformers/modeling_utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pytorch_transformers/modeling_utils.py b/pytorch_transformers/modeling_utils.py index 25aeefe10f..fdc8415fa6 100644 --- a/pytorch_transformers/modeling_utils.py +++ b/pytorch_transformers/modeling_utils.py @@ -478,7 +478,10 @@ class PoolerEndLogits(nn.Module): x = self.dense_1(x).squeeze(-1) if p_mask is not None: - x = x * (1 - p_mask) - 1e30 * p_mask + if next(self.parameters()).dtype == torch.float16: + x = x * (1 - p_mask) - 65500 * p_mask + else: + x = x * (1 - p_mask) - 1e30 * p_mask return x From f0340eccf90a81462ca025ef21350bdb13f22b42 Mon Sep 17 00:00:00 2001 From: Erik Chan Date: Wed, 18 Sep 2019 13:42:11 -0700 Subject: [PATCH 060/219] Typo Typo --- examples/run_lm_finetuning.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/run_lm_finetuning.py b/examples/run_lm_finetuning.py index 4d14fe7ebb..dad9fab83f 100644 --- a/examples/run_lm_finetuning.py +++ b/examples/run_lm_finetuning.py @@ -57,7 +57,7 @@ class TextDataset(Dataset): def __init__(self, tokenizer, file_path='train', block_size=512): assert os.path.isfile(file_path) directory, filename = os.path.split(file_path) - cached_features_file = os.path.join(directory, f'cached_lm_{block_size}_{filename}') + cached_features_file = os.path.join(directory, 'cached_lm_{block_size}_{filename}') if os.path.exists(cached_features_file): logger.info("Loading features from cached file %s", cached_features_file) From 08e4ad5eea1d07c5c1287bf365f994acdb1645c7 Mon Sep 17 00:00:00 2001 From: sshleifer Date: Wed, 18 Sep 2019 16:35:01 -0700 Subject: [PATCH 061/219] Remove documentation for unused kwarg --- pytorch_transformers/configuration_openai.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pytorch_transformers/configuration_openai.py b/pytorch_transformers/configuration_openai.py index b27df56899..886b7f5bc5 100644 --- a/pytorch_transformers/configuration_openai.py +++ b/pytorch_transformers/configuration_openai.py @@ -36,7 +36,6 @@ class OpenAIGPTConfig(PretrainedConfig): Args: vocab_size_or_config_json_file: Vocabulary size of `inputs_ids` in `OpenAIGPTModel` or a configuration json file. - n_special: The number of special tokens to learn during fine-tuning ('[SEP]', '[CLF]', ...) n_positions: Number of positional embeddings. n_ctx: Size of the causal mask (usually same as n_positions). n_embd: Dimensionality of the embeddings and hidden states. From e391d4735eb02023dd74fb0f3f751d9acc8b6c7f Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Mon, 2 Sep 2019 16:42:32 -0400 Subject: [PATCH 062/219] Tokenizers' encode function can output binary masks --- pytorch_transformers/tokenization_bert.py | 10 ++++++++-- pytorch_transformers/tokenization_roberta.py | 10 ++++++++-- pytorch_transformers/tokenization_utils.py | 8 +++++--- pytorch_transformers/tokenization_xlm.py | 11 +++++++++-- pytorch_transformers/tokenization_xlnet.py | 11 +++++++++-- 5 files changed, 39 insertions(+), 11 deletions(-) diff --git a/pytorch_transformers/tokenization_bert.py b/pytorch_transformers/tokenization_bert.py index b85a4ccf9c..f720d530f8 100644 --- a/pytorch_transformers/tokenization_bert.py +++ b/pytorch_transformers/tokenization_bert.py @@ -194,14 +194,20 @@ class BertTokenizer(PreTrainedTokenizer): """ return [self.cls_token_id] + token_ids + [self.sep_token_id] - def add_special_tokens_sentences_pair(self, token_ids_0, token_ids_1): + def add_special_tokens_sentences_pair(self, token_ids_0, token_ids_1, output_mask=False): """ Adds special tokens to a sequence pair for sequence classification tasks. A BERT sequence pair has the following format: [CLS] A [SEP] B [SEP] """ sep = [self.sep_token_id] cls = [self.cls_token_id] - return cls + token_ids_0 + sep + token_ids_1 + sep + if output_mask: + return ( + cls + token_ids_0 + sep + token_ids_1 + sep, + [0] * len(cls + token_ids_0 + sep) + [1] * len(token_ids_1 + sep) + ) + else: + return cls + token_ids_0 + sep + token_ids_1 + sep def save_vocabulary(self, vocab_path): """Save the tokenizer vocabulary to a directory or file.""" diff --git a/pytorch_transformers/tokenization_roberta.py b/pytorch_transformers/tokenization_roberta.py index 67808752d5..db4cbe967f 100644 --- a/pytorch_transformers/tokenization_roberta.py +++ b/pytorch_transformers/tokenization_roberta.py @@ -88,11 +88,17 @@ class RobertaTokenizer(GPT2Tokenizer): """ return [self.cls_token_id] + token_ids + [self.sep_token_id] - def add_special_tokens_sentences_pair(self, token_ids_0, token_ids_1): + def add_special_tokens_sentences_pair(self, token_ids_0, token_ids_1, output_mask=False): """ Adds special tokens to a sequence pair for sequence classification tasks. A RoBERTa sequence pair has the following format: A B """ sep = [self.sep_token_id] cls = [self.cls_token_id] - return cls + token_ids_0 + sep + sep + token_ids_1 + sep + if output_mask: + return ( + cls + token_ids_0 + sep + sep + token_ids_1 + sep, + [0] * len(cls + token_ids_0 + sep) + [1] * len(sep + token_ids_1 + sep) + ) + else: + return cls + token_ids_0 + sep + sep + token_ids_1 + sep diff --git a/pytorch_transformers/tokenization_utils.py b/pytorch_transformers/tokenization_utils.py index 1e2cd59648..22ab04ac6a 100644 --- a/pytorch_transformers/tokenization_utils.py +++ b/pytorch_transformers/tokenization_utils.py @@ -663,7 +663,7 @@ class PreTrainedTokenizer(object): def _convert_token_to_id(self, token): raise NotImplementedError - def encode(self, text, text_pair=None, add_special_tokens=False, **kwargs): + def encode(self, text, text_pair=None, add_special_tokens=False, output_mask=False, **kwargs): """ Converts a string in a sequence of ids (integer), using the tokenizer and vocabulary. @@ -674,6 +674,8 @@ class PreTrainedTokenizer(object): text_pair: Optional second sequence to be encoded. add_special_tokens: if set to ``True``, the sequences will be encoded with the special tokens relative to their model. + output_mask: if set to ``True``, returns the text pair corresponding mask with 0 for the first sequence, + and 1 for the second. **kwargs: passed to the `self.tokenize()` method """ if text_pair is None: @@ -686,7 +688,7 @@ class PreTrainedTokenizer(object): second_sentence_tokens = [self._convert_token_to_id(token) for token in self.tokenize(text_pair, **kwargs)] if add_special_tokens: - return self.add_special_tokens_sentences_pair(first_sentence_tokens, second_sentence_tokens) + return self.add_special_tokens_sentences_pair(first_sentence_tokens, second_sentence_tokens, output_mask) else: return first_sentence_tokens, second_sentence_tokens @@ -694,7 +696,7 @@ class PreTrainedTokenizer(object): logger.warning("This tokenizer does not make use of special tokens. The sequence has been returned with no modification.") return token_ids - def add_special_tokens_sentences_pair(self, token_ids_0, token_ids_1): + def add_special_tokens_sentences_pair(self, token_ids_0, token_ids_1, output_mask=False): logger.warning("This tokenizer does not make use of special tokens. The two sequences have been concatenated.") return token_ids_0 + token_ids_1 diff --git a/pytorch_transformers/tokenization_xlm.py b/pytorch_transformers/tokenization_xlm.py index f7231384b3..167860f15f 100644 --- a/pytorch_transformers/tokenization_xlm.py +++ b/pytorch_transformers/tokenization_xlm.py @@ -761,14 +761,21 @@ class XLMTokenizer(PreTrainedTokenizer): """ return [self.cls_token_id] + token_ids + [self.sep_token_id] - def add_special_tokens_sentences_pair(self, token_ids_0, token_ids_1): + def add_special_tokens_sentences_pair(self, token_ids_0, token_ids_1, output_mask=False): """ Adds special tokens to a sequence pair for sequence classification tasks. An XLM sequence pair has the following format: [CLS] A [SEP] B [SEP] """ sep = [self.sep_token_id] cls = [self.cls_token_id] - return cls + token_ids_0 + sep + token_ids_1 + sep + + if output_mask: + return ( + cls + token_ids_0 + sep + token_ids_1 + sep, + [0] * len(cls + token_ids_0 + sep) + [1] * len(token_ids_1 + sep) + ) + else: + return cls + token_ids_0 + sep + token_ids_1 + sep def save_vocabulary(self, save_directory): """Save the tokenizer vocabulary and merge files to a directory.""" diff --git a/pytorch_transformers/tokenization_xlnet.py b/pytorch_transformers/tokenization_xlnet.py index 230095daa9..cbf312bd38 100644 --- a/pytorch_transformers/tokenization_xlnet.py +++ b/pytorch_transformers/tokenization_xlnet.py @@ -190,14 +190,21 @@ class XLNetTokenizer(PreTrainedTokenizer): cls = [self.cls_token_id] return token_ids + sep + cls - def add_special_tokens_sentences_pair(self, token_ids_0, token_ids_1): + def add_special_tokens_sentences_pair(self, token_ids_0, token_ids_1, output_mask=False): """ Adds special tokens to a sequence for sequence classification tasks. An XLNet sequence has the following format: X [SEP][CLS] """ + sep = [self.sep_token_id] cls = [self.cls_token_id] - return token_ids_0 + sep + token_ids_1 + sep + cls + if output_mask: + return ( + token_ids_0 + sep + token_ids_1 + sep + cls, + [0] * len(token_ids_0 + sep) + [1] * len(token_ids_1 + sep + cls) + ) + else: + return token_ids_0 + sep + token_ids_1 + sep + cls def save_vocabulary(self, save_directory): """ Save the sentencepiece vocabulary (copy original file) and special tokens file From c3df2136e13512811507b36a108065d85328cff9 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Mon, 2 Sep 2019 17:47:16 -0400 Subject: [PATCH 063/219] Added binary masking tests --- .../tests/tokenization_tests_commons.py | 12 ++++++++++++ pytorch_transformers/tokenization_utils.py | 2 ++ 2 files changed, 14 insertions(+) diff --git a/pytorch_transformers/tests/tokenization_tests_commons.py b/pytorch_transformers/tests/tokenization_tests_commons.py index 3da0494ac4..aa6746a758 100644 --- a/pytorch_transformers/tests/tokenization_tests_commons.py +++ b/pytorch_transformers/tests/tokenization_tests_commons.py @@ -186,3 +186,15 @@ class CommonTestCases: for weights_list_2 in weights_lists_2: self.assertListEqual(weights_list, weights_list_2) + + def test_mask_output(self): + if sys.version_info <= (3, 0): + return + + tokenizer = self.get_tokenizer() + + if tokenizer.add_special_tokens_sentences_pair.__qualname__.split('.')[0] != "PreTrainedTokenizer": + seq_0 = "Test this method." + seq_1 = "With these inputs." + sequences, mask = tokenizer.encode(seq_0, seq_1, add_special_tokens=True, output_mask=True) + assert len(sequences) == len(mask) diff --git a/pytorch_transformers/tokenization_utils.py b/pytorch_transformers/tokenization_utils.py index 22ab04ac6a..be49d7eab5 100644 --- a/pytorch_transformers/tokenization_utils.py +++ b/pytorch_transformers/tokenization_utils.py @@ -690,6 +690,8 @@ class PreTrainedTokenizer(object): if add_special_tokens: return self.add_special_tokens_sentences_pair(first_sentence_tokens, second_sentence_tokens, output_mask) else: + if output_mask: + logger.warning("Can't output mask if no special tokens are involved. Please call the method with add_special_tokens set to True.") return first_sentence_tokens, second_sentence_tokens def add_special_tokens_single_sentence(self, token_ids): From bac332fec06c843c8c3436091bb53343c109202d Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Mon, 2 Sep 2019 21:46:11 -0400 Subject: [PATCH 064/219] Updated the GLUE data processor. Corrections to RoBERTa and XLNet. --- examples/utils_glue.py | 55 +------------------- pytorch_transformers/tokenization_roberta.py | 2 +- pytorch_transformers/tokenization_xlnet.py | 3 +- 3 files changed, 4 insertions(+), 56 deletions(-) diff --git a/examples/utils_glue.py b/examples/utils_glue.py index 3e3f104672..841d7376cf 100644 --- a/examples/utils_glue.py +++ b/examples/utils_glue.py @@ -415,58 +415,7 @@ def convert_examples_to_features(examples, label_list, max_seq_length, if ex_index % 10000 == 0: logger.info("Writing example %d of %d" % (ex_index, len(examples))) - tokens_a = tokenizer.tokenize(example.text_a) - - tokens_b = None - if example.text_b: - tokens_b = tokenizer.tokenize(example.text_b) - # Modifies `tokens_a` and `tokens_b` in place so that the total - # length is less than the specified length. - # Account for [CLS], [SEP], [SEP] with "- 3". " -4" for RoBERTa. - special_tokens_count = 4 if sep_token_extra else 3 - _truncate_seq_pair(tokens_a, tokens_b, max_seq_length - special_tokens_count) - else: - # Account for [CLS] and [SEP] with "- 2" and with "- 3" for RoBERTa. - special_tokens_count = 3 if sep_token_extra else 2 - if len(tokens_a) > max_seq_length - special_tokens_count: - tokens_a = tokens_a[:(max_seq_length - special_tokens_count)] - - # The convention in BERT is: - # (a) For sequence pairs: - # tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP] - # type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1 - # (b) For single sequences: - # tokens: [CLS] the dog is hairy . [SEP] - # type_ids: 0 0 0 0 0 0 0 - # - # Where "type_ids" are used to indicate whether this is the first - # sequence or the second sequence. The embedding vectors for `type=0` and - # `type=1` were learned during pre-training and are added to the wordpiece - # embedding vector (and position vector). This is not *strictly* necessary - # since the [SEP] token unambiguously separates the sequences, but it makes - # it easier for the model to learn the concept of sequences. - # - # For classification tasks, the first vector (corresponding to [CLS]) is - # used as as the "sentence vector". Note that this only makes sense because - # the entire model is fine-tuned. - tokens = tokens_a + [sep_token] - if sep_token_extra: - # roberta uses an extra separator b/w pairs of sentences - tokens += [sep_token] - segment_ids = [sequence_a_segment_id] * len(tokens) - - if tokens_b: - tokens += tokens_b + [sep_token] - segment_ids += [sequence_b_segment_id] * (len(tokens_b) + 1) - - if cls_token_at_end: - tokens = tokens + [cls_token] - segment_ids = segment_ids + [cls_token_segment_id] - else: - tokens = [cls_token] + tokens - segment_ids = [cls_token_segment_id] + segment_ids - - input_ids = tokenizer.convert_tokens_to_ids(tokens) + input_ids, input_mask = tokenizer.encode(example.text_a, example.text_b, add_special_tokens=True, output_mask=True) # The mask has 1 for real tokens and 0 for padding tokens. Only real # tokens are attended to. @@ -497,8 +446,6 @@ def convert_examples_to_features(examples, label_list, max_seq_length, if ex_index < 5: logger.info("*** Example ***") logger.info("guid: %s" % (example.guid)) - logger.info("tokens: %s" % " ".join( - [str(x) for x in tokens])) logger.info("input_ids: %s" % " ".join([str(x) for x in input_ids])) logger.info("input_mask: %s" % " ".join([str(x) for x in input_mask])) logger.info("segment_ids: %s" % " ".join([str(x) for x in segment_ids])) diff --git a/pytorch_transformers/tokenization_roberta.py b/pytorch_transformers/tokenization_roberta.py index db4cbe967f..b41d85e7e2 100644 --- a/pytorch_transformers/tokenization_roberta.py +++ b/pytorch_transformers/tokenization_roberta.py @@ -98,7 +98,7 @@ class RobertaTokenizer(GPT2Tokenizer): if output_mask: return ( cls + token_ids_0 + sep + sep + token_ids_1 + sep, - [0] * len(cls + token_ids_0 + sep) + [1] * len(sep + token_ids_1 + sep) + [0] * len(cls + token_ids_0 + sep + sep) + [1] * len(token_ids_1 + sep) ) else: return cls + token_ids_0 + sep + sep + token_ids_1 + sep diff --git a/pytorch_transformers/tokenization_xlnet.py b/pytorch_transformers/tokenization_xlnet.py index cbf312bd38..9faa3c7e59 100644 --- a/pytorch_transformers/tokenization_xlnet.py +++ b/pytorch_transformers/tokenization_xlnet.py @@ -198,10 +198,11 @@ class XLNetTokenizer(PreTrainedTokenizer): sep = [self.sep_token_id] cls = [self.cls_token_id] + cls_segment_ids = [2] if output_mask: return ( token_ids_0 + sep + token_ids_1 + sep + cls, - [0] * len(token_ids_0 + sep) + [1] * len(token_ids_1 + sep + cls) + [0] * len(token_ids_0 + sep) + [1] * len(token_ids_1 + sep) + cls_segment_ids ) else: return token_ids_0 + sep + token_ids_1 + sep + cls From 59057abe523297711bd3edcab64b6c20a32c46de Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Mon, 2 Sep 2019 22:03:39 -0400 Subject: [PATCH 065/219] typo --- examples/utils_glue.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/utils_glue.py b/examples/utils_glue.py index 841d7376cf..96023e149c 100644 --- a/examples/utils_glue.py +++ b/examples/utils_glue.py @@ -415,7 +415,7 @@ def convert_examples_to_features(examples, label_list, max_seq_length, if ex_index % 10000 == 0: logger.info("Writing example %d of %d" % (ex_index, len(examples))) - input_ids, input_mask = tokenizer.encode(example.text_a, example.text_b, add_special_tokens=True, output_mask=True) + input_ids, segment_ids = tokenizer.encode(example.text_a, example.text_b, add_special_tokens=True, output_mask=True) # The mask has 1 for real tokens and 0 for padding tokens. Only real # tokens are attended to. From 92a9976e919664268afa5e6a8de38c72cefc1efd Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Fri, 6 Sep 2019 17:19:31 -0400 Subject: [PATCH 066/219] Distilbert sequence builder w/ mask --- pytorch_transformers/tokenization_distilbert.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pytorch_transformers/tokenization_distilbert.py b/pytorch_transformers/tokenization_distilbert.py index 5a6d02f98d..f91989d2bd 100644 --- a/pytorch_transformers/tokenization_distilbert.py +++ b/pytorch_transformers/tokenization_distilbert.py @@ -60,3 +60,16 @@ class DistilBertTokenizer(BertTokenizer): vocab_files_names = VOCAB_FILES_NAMES pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES + + def add_special_tokens_single_sentence(self, token_ids): + return token_ids + + def add_special_tokens_sentences_pair(self, token_ids_0, token_ids_1, output_mask=False): + sep = [self.sep_token_id] + if output_mask: + return ( + token_ids_0 + sep + token_ids_1, + [0] * len(token_ids_0 + sep) + [1] * len(token_ids_1) + ) + else: + return token_ids_0 + sep + token_ids_1 From 75635072e11afbf716011464d30869efbe226886 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Fri, 6 Sep 2019 17:23:47 -0400 Subject: [PATCH 067/219] Updated GLUE script to add DistilBERT. Cleaned up unused args in the utils file. --- examples/run_glue.py | 15 +++++++-------- examples/utils_glue.py | 14 ++------------ 2 files changed, 9 insertions(+), 20 deletions(-) diff --git a/examples/run_glue.py b/examples/run_glue.py index e20f6d84c4..e8c2edc833 100644 --- a/examples/run_glue.py +++ b/examples/run_glue.py @@ -39,7 +39,10 @@ from pytorch_transformers import (WEIGHTS_NAME, BertConfig, XLMConfig, XLMForSequenceClassification, XLMTokenizer, XLNetConfig, XLNetForSequenceClassification, - XLNetTokenizer) + XLNetTokenizer, + DistilBertConfig, + DistilBertForSequenceClassification, + DistilBertTokenizer) from pytorch_transformers import AdamW, WarmupLinearSchedule @@ -55,6 +58,7 @@ MODEL_CLASSES = { 'xlnet': (XLNetConfig, XLNetForSequenceClassification, XLNetTokenizer), 'xlm': (XLMConfig, XLMForSequenceClassification, XLMTokenizer), 'roberta': (RobertaConfig, RobertaForSequenceClassification, RobertaTokenizer), + 'distilbert': (DistilBertConfig, DistilBertForSequenceClassification, DistilBertTokenizer) } @@ -128,7 +132,7 @@ def train(args, train_dataset, model, tokenizer): batch = tuple(t.to(args.device) for t in batch) inputs = {'input_ids': batch[0], 'attention_mask': batch[1], - 'token_type_ids': batch[2] if args.model_type in ['bert', 'xlnet'] else None, # XLM and RoBERTa don't use segment_ids + 'token_type_ids': batch[2] if args.model_type in ['bert', 'xlnet'] else None, # XLM, DistilBERT and RoBERTa don't use segment_ids 'labels': batch[3]} outputs = model(**inputs) loss = outputs[0] # model outputs are always tuple in pytorch-transformers (see doc) @@ -218,7 +222,7 @@ def evaluate(args, model, tokenizer, prefix=""): with torch.no_grad(): inputs = {'input_ids': batch[0], 'attention_mask': batch[1], - 'token_type_ids': batch[2] if args.model_type in ['bert', 'xlnet'] else None, # XLM and RoBERTa don't use segment_ids + 'token_type_ids': batch[2] if args.model_type in ['bert', 'xlnet'] else None, # XLM, DistilBERT and RoBERTa don't use segment_ids 'labels': batch[3]} outputs = model(**inputs) tmp_eval_loss, logits = outputs[:2] @@ -273,11 +277,6 @@ def load_and_cache_examples(args, task, tokenizer, evaluate=False): label_list[1], label_list[2] = label_list[2], label_list[1] examples = processor.get_dev_examples(args.data_dir) if evaluate else processor.get_train_examples(args.data_dir) features = convert_examples_to_features(examples, label_list, args.max_seq_length, tokenizer, output_mode, - cls_token_at_end=bool(args.model_type in ['xlnet']), # xlnet has a cls token at the end - cls_token=tokenizer.cls_token, - cls_token_segment_id=2 if args.model_type in ['xlnet'] else 0, - sep_token=tokenizer.sep_token, - sep_token_extra=bool(args.model_type in ['roberta']), # roberta uses an extra separator b/w pairs of sentences, cf. github.com/pytorch/fairseq/commit/1684e166e3da03f5b600dbb7855cb98ddfcd0805 pad_on_left=bool(args.model_type in ['xlnet']), # pad on the left for xlnet pad_token=tokenizer.convert_tokens_to_ids([tokenizer.pad_token])[0], pad_token_segment_id=4 if args.model_type in ['xlnet'] else 0, diff --git a/examples/utils_glue.py b/examples/utils_glue.py index 96023e149c..8d0ed737d7 100644 --- a/examples/utils_glue.py +++ b/examples/utils_glue.py @@ -390,22 +390,12 @@ class WnliProcessor(DataProcessor): def convert_examples_to_features(examples, label_list, max_seq_length, tokenizer, output_mode, - cls_token_at_end=False, - cls_token='[CLS]', - cls_token_segment_id=1, - sep_token='[SEP]', - sep_token_extra=False, pad_on_left=False, pad_token=0, pad_token_segment_id=0, - sequence_a_segment_id=0, - sequence_b_segment_id=1, mask_padding_with_zero=True): - """ Loads a data file into a list of `InputBatch`s - `cls_token_at_end` define the location of the CLS token: - - False (Default, BERT/XLM pattern): [CLS] + A + [SEP] + B + [SEP] - - True (XLNet/GPT pattern): A + [SEP] + B + [SEP] + [CLS] - `cls_token_segment_id` define the segment id associated to the CLS token (0 for BERT, 2 for XLNet) + """ + Loads a data file into a list of `InputBatch`s """ label_map = {label : i for i, label in enumerate(label_list)} From 2d8ec5a684806d9a0d427645dcafbdf5de17eaed Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Fri, 6 Sep 2019 17:51:55 -0400 Subject: [PATCH 068/219] Changed warning to be more explicit Co-authored by: julien_c --- pytorch_transformers/tokenization_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_transformers/tokenization_utils.py b/pytorch_transformers/tokenization_utils.py index be49d7eab5..6a4f8a2dc9 100644 --- a/pytorch_transformers/tokenization_utils.py +++ b/pytorch_transformers/tokenization_utils.py @@ -691,7 +691,7 @@ class PreTrainedTokenizer(object): return self.add_special_tokens_sentences_pair(first_sentence_tokens, second_sentence_tokens, output_mask) else: if output_mask: - logger.warning("Can't output mask if no special tokens are involved. Please call the method with add_special_tokens set to True.") + logger.warning("Can't output mask if you're not joining two sequences.") return first_sentence_tokens, second_sentence_tokens def add_special_tokens_single_sentence(self, token_ids): From 88368c2a16d26bc2d00dc28f79196c81373d3a71 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Fri, 6 Sep 2019 18:05:56 -0400 Subject: [PATCH 069/219] Added DistilBERT to `run_lm_finetuning` --- examples/run_lm_finetuning.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/examples/run_lm_finetuning.py b/examples/run_lm_finetuning.py index 4d14fe7ebb..974a84c34e 100644 --- a/examples/run_lm_finetuning.py +++ b/examples/run_lm_finetuning.py @@ -39,7 +39,8 @@ from pytorch_transformers import (WEIGHTS_NAME, AdamW, WarmupLinearSchedule, BertConfig, BertForMaskedLM, BertTokenizer, GPT2Config, GPT2LMHeadModel, GPT2Tokenizer, OpenAIGPTConfig, OpenAIGPTLMHeadModel, OpenAIGPTTokenizer, - RobertaConfig, RobertaForMaskedLM, RobertaTokenizer) + RobertaConfig, RobertaForMaskedLM, RobertaTokenizer, + DistilBertConfig, DistilBertForMaskedLM, DistilBertTokenizer) logger = logging.getLogger(__name__) @@ -49,7 +50,8 @@ MODEL_CLASSES = { 'gpt2': (GPT2Config, GPT2LMHeadModel, GPT2Tokenizer), 'openai-gpt': (OpenAIGPTConfig, OpenAIGPTLMHeadModel, OpenAIGPTTokenizer), 'bert': (BertConfig, BertForMaskedLM, BertTokenizer), - 'roberta': (RobertaConfig, RobertaForMaskedLM, RobertaTokenizer) + 'roberta': (RobertaConfig, RobertaForMaskedLM, RobertaTokenizer), + 'distilbert': (DistilBertConfig, DistilBertForMaskedLM, DistilBertTokenizer) } @@ -380,7 +382,7 @@ def main(): parser.add_argument('--server_port', type=str, default='', help="For distant debugging.") args = parser.parse_args() - if args.model_type in ["bert", "roberta"] and not args.mlm: + if args.model_type in ["bert", "roberta", "distilbert"] and not args.mlm: raise ValueError("BERT and RoBERTa do not have LM heads but masked LM heads. They must be run using the --mlm " "flag (masked language modeling).") if args.eval_data_file is None and args.do_eval: From de8e14b6c0d1a9c573835972221c0bda883d7f2a Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Wed, 11 Sep 2019 10:21:18 +0200 Subject: [PATCH 070/219] Added DistilBERT to run_squad script --- examples/run_squad.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/run_squad.py b/examples/run_squad.py index cc4eda306c..affef90ca9 100644 --- a/examples/run_squad.py +++ b/examples/run_squad.py @@ -37,7 +37,8 @@ from pytorch_transformers import (WEIGHTS_NAME, BertConfig, XLMConfig, XLMForQuestionAnswering, XLMTokenizer, XLNetConfig, XLNetForQuestionAnswering, - XLNetTokenizer) + XLNetTokenizer, + DistilBertConfig, DistilBertForQuestionAnswering, DistilBertTokenizer) from pytorch_transformers import AdamW, WarmupLinearSchedule @@ -59,6 +60,7 @@ MODEL_CLASSES = { 'bert': (BertConfig, BertForQuestionAnswering, BertTokenizer), 'xlnet': (XLNetConfig, XLNetForQuestionAnswering, XLNetTokenizer), 'xlm': (XLMConfig, XLMForQuestionAnswering, XLMTokenizer), + 'distilbert': (DistilBertConfig, DistilBertForQuestionAnswering, DistilBertTokenizer) } def set_seed(args): From d572d7027b098072e090248f6c13d3cde40c926b Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Wed, 11 Sep 2019 11:20:07 +0200 Subject: [PATCH 071/219] Number of added tokens calculator --- .../tests/tokenization_tests_commons.py | 13 ++++++++ pytorch_transformers/tokenization_utils.py | 30 +++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/pytorch_transformers/tests/tokenization_tests_commons.py b/pytorch_transformers/tests/tokenization_tests_commons.py index aa6746a758..7500741cee 100644 --- a/pytorch_transformers/tests/tokenization_tests_commons.py +++ b/pytorch_transformers/tests/tokenization_tests_commons.py @@ -198,3 +198,16 @@ class CommonTestCases: seq_1 = "With these inputs." sequences, mask = tokenizer.encode(seq_0, seq_1, add_special_tokens=True, output_mask=True) assert len(sequences) == len(mask) + + def test_number_of_added_tokens(self): + tokenizer = self.get_tokenizer() + + seq_0 = "Test this method." + seq_1 = "With these inputs." + + sequences = tokenizer.encode(seq_0, seq_1) + attached_sequences = tokenizer.encode(seq_0, seq_1, add_special_tokens=True) + + # Method is implemented (e.g. not GPT-2) + if len(attached_sequences) != 2: + assert tokenizer.num_added_tokens(pair=True) == len(attached_sequences) - sum([len(seq) for seq in sequences]) diff --git a/pytorch_transformers/tokenization_utils.py b/pytorch_transformers/tokenization_utils.py index 6a4f8a2dc9..a22f15fa3e 100644 --- a/pytorch_transformers/tokenization_utils.py +++ b/pytorch_transformers/tokenization_utils.py @@ -518,6 +518,36 @@ class PreTrainedTokenizer(object): return len(to_add_tokens) + def num_added_tokens(self, pair=False): + """ + Returns the number of added tokens when encoding a sequence with special tokens. + + Note: + This encodes inputs and checks the number of added tokens, and is therefore not efficient. Do not put this + inside your training loop. + + Args: + pair: Returns the number of added tokens in the case of a sequence pair if set to True, returns the + number of added tokens in the case of a single sequence if set to False. + + Returns: + Number of tokens added to sequences + """ + + if pair: + initial_tokens_len = sum([len(encoded) for encoded in self.encode("This is a sequence", "This is another")]) + final_tokens = self.encode("This is a sequence", "This is another", add_special_tokens=True) + + # In some models (e.g. GPT-2), there is no sequence pair encoding. + if len(final_tokens) == 2: + return 0 + else: + final_tokens_len = len(final_tokens) + else: + initial_tokens_len = len(self.encode("This is a sequence")) + final_tokens_len = len(self.encode("This is a sequence", add_special_tokens=True)) + + return final_tokens_len - initial_tokens_len def add_special_tokens(self, special_tokens_dict): """ From c4d4f3ec8cdea133112fa986f222f5d8faedb59d Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Wed, 11 Sep 2019 11:22:04 +0200 Subject: [PATCH 072/219] Updated DistilBERT test to reflect the sequence encoding --- ...zation_dilbert_test.py => tokenization_distilbert_test.py} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename pytorch_transformers/tests/{tokenization_dilbert_test.py => tokenization_distilbert_test.py} (92%) diff --git a/pytorch_transformers/tests/tokenization_dilbert_test.py b/pytorch_transformers/tests/tokenization_distilbert_test.py similarity index 92% rename from pytorch_transformers/tests/tokenization_dilbert_test.py rename to pytorch_transformers/tests/tokenization_distilbert_test.py index 42f8060998..30160fc98e 100644 --- a/pytorch_transformers/tests/tokenization_dilbert_test.py +++ b/pytorch_transformers/tests/tokenization_distilbert_test.py @@ -39,8 +39,8 @@ class DistilBertTokenizationTest(BertTokenizationTest): encoded_sentence = tokenizer.add_special_tokens_single_sentence(text) encoded_pair = tokenizer.add_special_tokens_sentences_pair(text, text_2) - assert encoded_sentence == [101] + text + [102] - assert encoded_pair == [101] + text + [102] + text_2 + [102] + assert encoded_sentence == text + assert encoded_pair == text + [102] + text_2 if __name__ == '__main__': unittest.main() From af23b626c8dcf16827dae08a9ba5ed3b8a8d6d97 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Wed, 11 Sep 2019 18:20:24 +0200 Subject: [PATCH 073/219] Max encoding length + corresponding tests --- .../tests/tokenization_tests_commons.py | 29 +++++++++++++++++++ pytorch_transformers/tokenization_utils.py | 22 ++++++++++++-- 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/pytorch_transformers/tests/tokenization_tests_commons.py b/pytorch_transformers/tests/tokenization_tests_commons.py index 7500741cee..e077333c19 100644 --- a/pytorch_transformers/tests/tokenization_tests_commons.py +++ b/pytorch_transformers/tests/tokenization_tests_commons.py @@ -211,3 +211,32 @@ class CommonTestCases: # Method is implemented (e.g. not GPT-2) if len(attached_sequences) != 2: assert tokenizer.num_added_tokens(pair=True) == len(attached_sequences) - sum([len(seq) for seq in sequences]) + + def test_maximum_encoding_length_single_input(self): + tokenizer = self.get_tokenizer() + + seq_0 = "This is a sentence to be encoded." + + sequence = tokenizer.encode(seq_0) + num_added_tokens = tokenizer.num_added_tokens() + total_length = len(sequence) + num_added_tokens + truncated_sequence = tokenizer.encode(seq_0, max_length=total_length - 2, add_special_tokens=True) + + assert len(truncated_sequence) == total_length - 2 + assert truncated_sequence == tokenizer.add_special_tokens_single_sentence(sequence[:-2]) + + def test_maximum_encoding_length_pair_input(self): + tokenizer = self.get_tokenizer() + + seq_0 = "This is a sentence to be encoded." + seq_1 = "This is another sentence to be encoded." + + sequence = tokenizer.encode(seq_0, seq_1, add_special_tokens=True) + truncated_second_sequence = tokenizer.add_special_tokens_sentences_pair( + tokenizer.encode(seq_0), + tokenizer.encode(seq_1)[:-2] + ) + truncated_sequence = tokenizer.encode(seq_0, seq_1, max_length=len(sequence) - 2, add_special_tokens=True) + + assert len(truncated_sequence) == len(sequence) - 2 + assert truncated_sequence == truncated_second_sequence diff --git a/pytorch_transformers/tokenization_utils.py b/pytorch_transformers/tokenization_utils.py index a22f15fa3e..97cf242511 100644 --- a/pytorch_transformers/tokenization_utils.py +++ b/pytorch_transformers/tokenization_utils.py @@ -693,7 +693,7 @@ class PreTrainedTokenizer(object): def _convert_token_to_id(self, token): raise NotImplementedError - def encode(self, text, text_pair=None, add_special_tokens=False, output_mask=False, **kwargs): + def encode(self, text, text_pair=None, add_special_tokens=False, output_mask=False, max_length=None, **kwargs): """ Converts a string in a sequence of ids (integer), using the tokenizer and vocabulary. @@ -706,20 +706,36 @@ class PreTrainedTokenizer(object): to their model. output_mask: if set to ``True``, returns the text pair corresponding mask with 0 for the first sequence, and 1 for the second. + max_length: if set to a number, will limit the total sequence returned so that it has a maximum length. **kwargs: passed to the `self.tokenize()` method """ if text_pair is None: if add_special_tokens: - return self.add_special_tokens_single_sentence(self.convert_tokens_to_ids(self.tokenize(text, **kwargs))) + sequence_tokens = self.convert_tokens_to_ids(self.tokenize(text, **kwargs)) + if max_length: + sequence_tokens = sequence_tokens[:max_length - self.num_added_tokens()] + return self.add_special_tokens_single_sentence(sequence_tokens) else: - return self.convert_tokens_to_ids(self.tokenize(text, **kwargs)) + ids = self.convert_tokens_to_ids(self.tokenize(text, **kwargs)) + return ids[:max_length] if max_length != -1 else ids first_sentence_tokens = [self._convert_token_to_id(token) for token in self.tokenize(text, **kwargs)] second_sentence_tokens = [self._convert_token_to_id(token) for token in self.tokenize(text_pair, **kwargs)] if add_special_tokens: + if max_length: + if len(first_sentence_tokens) + self.num_added_tokens(pair=True) >= max_length: + logger.warning("The first sequence is longer than the maximum specified length. This sequence will not be truncated.") + else: + if len(second_sentence_tokens) + len(first_sentence_tokens) + self.num_added_tokens(pair=True) > max_length: + second_sentence_tokens = second_sentence_tokens[:max_length - len(first_sentence_tokens) - self.num_added_tokens(pair=True)] + return self.add_special_tokens_sentences_pair(first_sentence_tokens, second_sentence_tokens, output_mask) else: + if max_length: + first_sentence_tokens = first_sentence_tokens[:max_length] + second_sentence_tokens = second_sentence_tokens[:max_length] + if output_mask: logger.warning("Can't output mask if you're not joining two sequences.") return first_sentence_tokens, second_sentence_tokens From dcc9bb3252fdad12897851f22f57c3130ba38a7a Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Thu, 19 Sep 2019 09:29:48 +0200 Subject: [PATCH 074/219] Modified encode to return only lists. Added a more complete encode_plus method --- pytorch_transformers/tokenization_utils.py | 111 +++++++++++++++++++-- 1 file changed, 104 insertions(+), 7 deletions(-) diff --git a/pytorch_transformers/tokenization_utils.py b/pytorch_transformers/tokenization_utils.py index 97cf242511..3a3ebd49be 100644 --- a/pytorch_transformers/tokenization_utils.py +++ b/pytorch_transformers/tokenization_utils.py @@ -535,7 +535,7 @@ class PreTrainedTokenizer(object): """ if pair: - initial_tokens_len = sum([len(encoded) for encoded in self.encode("This is a sequence", "This is another")]) + initial_tokens_len = len(self.encode("This is a sequence") + self.encode("This is another")) final_tokens = self.encode("This is a sequence", "This is another", add_special_tokens=True) # In some models (e.g. GPT-2), there is no sequence pair encoding. @@ -693,10 +693,39 @@ class PreTrainedTokenizer(object): def _convert_token_to_id(self, token): raise NotImplementedError - def encode(self, text, text_pair=None, add_special_tokens=False, output_mask=False, max_length=None, **kwargs): + def encode(self, text, text_pair=None, add_special_tokens=False, **kwargs): """ Converts a string in a sequence of ids (integer), using the tokenizer and vocabulary. - + + Same as doing ``self.convert_tokens_to_ids(self.tokenize(text))``. + + Args: + text: The first sequence to be encoded. + text_pair: Optional second sequence to be encoded. + add_special_tokens: if set to ``True``, the sequences will be encoded with the special tokens relative + to their model. + """ + if text_pair is None: + if add_special_tokens: + sequence_tokens = self.convert_tokens_to_ids(self.tokenize(text, **kwargs)) + return self.add_special_tokens_single_sentence(sequence_tokens) + else: + ids = self.convert_tokens_to_ids(self.tokenize(text, **kwargs)) + return ids + + first_sentence_tokens = [self._convert_token_to_id(token) for token in self.tokenize(text, **kwargs)] + second_sentence_tokens = [self._convert_token_to_id(token) for token in self.tokenize(text_pair, **kwargs)] + + if add_special_tokens: + return self.add_special_tokens_sentences_pair(first_sentence_tokens, second_sentence_tokens) + else: + logger.warning("No special tokens were added. The two sequences have been concatenated.") + return first_sentence_tokens + second_sentence_tokens + + def encode_plus(self, text, text_pair=None, add_special_tokens=False, output_mask=False, max_length=None, **kwargs): + """ + Converts a string in a sequence of ids (integer), using the tokenizer and vocabulary. + Same as doing ``self.convert_tokens_to_ids(self.tokenize(text))``. Args: @@ -709,6 +738,69 @@ class PreTrainedTokenizer(object): max_length: if set to a number, will limit the total sequence returned so that it has a maximum length. **kwargs: passed to the `self.tokenize()` method """ + + information = {} + + if text_pair is None: + n_added_tokens = self.num_added_tokens() + if add_special_tokens: + sequence_tokens = self.convert_tokens_to_ids(self.tokenize(text, **kwargs)) + if max_length: + information["overflowing_tokens"] = sequence_tokens[max_length - n_added_tokens:] + sequence_tokens = sequence_tokens[:max_length - n_added_tokens] + sequence = self.add_special_tokens_single_sentence(sequence_tokens) + else: + sequence_tokens = self.convert_tokens_to_ids(self.tokenize(text, **kwargs)) + if max_length: + information["overflowing_tokens"] = sequence_tokens[max_length:] + sequence_tokens = sequence_tokens[:max_length] + sequence = sequence_tokens + + if output_mask: + information["mask"] = [0] * len(sequence) + + information["sequence"] = sequence + else: + first_sentence_tokens = [self._convert_token_to_id(token) for token in self.tokenize(text, **kwargs)] + second_sentence_tokens = [self._convert_token_to_id(token) for token in self.tokenize(text_pair, **kwargs)] + f_len, s_len = len(first_sentence_tokens), len(second_sentence_tokens) + n_added_tokens = self.num_added_tokens(pair=True) + + if add_special_tokens: + if max_length: + if len(first_sentence_tokens) + n_added_tokens >= max_length: + logger.warning("The first sequence is longer than the maximum specified length. This sequence will not be truncated.") + else: + if f_len + s_len + self.num_added_tokens(pair=True) > max_length: + information["overflowing_tokens"] = second_sentence_tokens[max_length - f_len - n_added_tokens:] + second_sentence_tokens = second_sentence_tokens[:max_length - f_len - n_added_tokens] + + encoded_sequence = self.add_special_tokens_sentences_pair( + first_sentence_tokens, + second_sentence_tokens, + output_mask + ) + + if output_mask: + sequence, information["mask"] = encoded_sequence + else: + sequence = encoded_sequence + + information["sequence"] = sequence + else: + logger.warning("No special tokens were added. The two sequences have been concatenated.") + sequence = first_sentence_tokens + second_sentence_tokens + + if max_length: + information["overflowing_tokens"] = sequence[max_length:] + sequence = sequence[:max_length] + if output_mask: + information["mask"] = [0] * len(sequence) + + information["sequence"] = sequence + + return information + if text_pair is None: if add_special_tokens: sequence_tokens = self.convert_tokens_to_ids(self.tokenize(text, **kwargs)) @@ -725,12 +817,17 @@ class PreTrainedTokenizer(object): if add_special_tokens: if max_length: if len(first_sentence_tokens) + self.num_added_tokens(pair=True) >= max_length: - logger.warning("The first sequence is longer than the maximum specified length. This sequence will not be truncated.") + logger.warning( + "The first sequence is longer than the maximum specified length. This sequence will not be truncated.") else: - if len(second_sentence_tokens) + len(first_sentence_tokens) + self.num_added_tokens(pair=True) > max_length: - second_sentence_tokens = second_sentence_tokens[:max_length - len(first_sentence_tokens) - self.num_added_tokens(pair=True)] + if len(second_sentence_tokens) + len(first_sentence_tokens) + self.num_added_tokens( + pair=True) > max_length: + second_sentence_tokens = second_sentence_tokens[ + :max_length - len(first_sentence_tokens) - self.num_added_tokens( + pair=True)] - return self.add_special_tokens_sentences_pair(first_sentence_tokens, second_sentence_tokens, output_mask) + return self.add_special_tokens_sentences_pair(first_sentence_tokens, second_sentence_tokens, + output_mask) else: if max_length: first_sentence_tokens = first_sentence_tokens[:max_length] From 6393261e41df5dd0babbbeba52d90f1c19a0e89a Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Thu, 19 Sep 2019 09:30:09 +0200 Subject: [PATCH 075/219] encode + encode_plus tests modified --- .../tests/tokenization_tests_commons.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/pytorch_transformers/tests/tokenization_tests_commons.py b/pytorch_transformers/tests/tokenization_tests_commons.py index e077333c19..4f85e2bdd6 100644 --- a/pytorch_transformers/tests/tokenization_tests_commons.py +++ b/pytorch_transformers/tests/tokenization_tests_commons.py @@ -196,7 +196,8 @@ class CommonTestCases: if tokenizer.add_special_tokens_sentences_pair.__qualname__.split('.')[0] != "PreTrainedTokenizer": seq_0 = "Test this method." seq_1 = "With these inputs." - sequences, mask = tokenizer.encode(seq_0, seq_1, add_special_tokens=True, output_mask=True) + information = tokenizer.encode_plus(seq_0, seq_1, add_special_tokens=True, output_mask=True) + sequences, mask = information["sequence"], information["mask"] assert len(sequences) == len(mask) def test_number_of_added_tokens(self): @@ -210,7 +211,7 @@ class CommonTestCases: # Method is implemented (e.g. not GPT-2) if len(attached_sequences) != 2: - assert tokenizer.num_added_tokens(pair=True) == len(attached_sequences) - sum([len(seq) for seq in sequences]) + assert tokenizer.num_added_tokens(pair=True) == len(attached_sequences) - len(sequences) def test_maximum_encoding_length_single_input(self): tokenizer = self.get_tokenizer() @@ -220,8 +221,12 @@ class CommonTestCases: sequence = tokenizer.encode(seq_0) num_added_tokens = tokenizer.num_added_tokens() total_length = len(sequence) + num_added_tokens - truncated_sequence = tokenizer.encode(seq_0, max_length=total_length - 2, add_special_tokens=True) + information = tokenizer.encode_plus(seq_0, max_length=total_length - 2, add_special_tokens=True) + truncated_sequence = information["sequence"] + overflowing_tokens = information["overflowing_tokens"] + + assert len(overflowing_tokens) == 2 assert len(truncated_sequence) == total_length - 2 assert truncated_sequence == tokenizer.add_special_tokens_single_sentence(sequence[:-2]) @@ -236,7 +241,10 @@ class CommonTestCases: tokenizer.encode(seq_0), tokenizer.encode(seq_1)[:-2] ) - truncated_sequence = tokenizer.encode(seq_0, seq_1, max_length=len(sequence) - 2, add_special_tokens=True) + information = tokenizer.encode_plus(seq_0, seq_1, max_length=len(sequence) - 2, add_special_tokens=True) + + truncated_sequence = information["sequence"] + overflowing_tokens = information["overflowing_tokens"] assert len(truncated_sequence) == len(sequence) - 2 assert truncated_sequence == truncated_second_sequence From 8cba0572603516f4f0d7fcd52fb76dce885b1358 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Thu, 19 Sep 2019 09:42:13 +0200 Subject: [PATCH 076/219] Doc + remove artefacts --- pytorch_transformers/tokenization_utils.py | 41 ++-------------------- 1 file changed, 2 insertions(+), 39 deletions(-) diff --git a/pytorch_transformers/tokenization_utils.py b/pytorch_transformers/tokenization_utils.py index 3a3ebd49be..2a31aec887 100644 --- a/pytorch_transformers/tokenization_utils.py +++ b/pytorch_transformers/tokenization_utils.py @@ -724,9 +724,8 @@ class PreTrainedTokenizer(object): def encode_plus(self, text, text_pair=None, add_special_tokens=False, output_mask=False, max_length=None, **kwargs): """ - Converts a string in a sequence of ids (integer), using the tokenizer and vocabulary. - - Same as doing ``self.convert_tokens_to_ids(self.tokenize(text))``. + Returns a dictionary containing the encoded sequence or sequence pair. Other values can be returned by this + method: the mask for sequence classification and the overflowing elements if a ``max_length`` is specified. Args: text: The first sequence to be encoded. @@ -801,42 +800,6 @@ class PreTrainedTokenizer(object): return information - if text_pair is None: - if add_special_tokens: - sequence_tokens = self.convert_tokens_to_ids(self.tokenize(text, **kwargs)) - if max_length: - sequence_tokens = sequence_tokens[:max_length - self.num_added_tokens()] - return self.add_special_tokens_single_sentence(sequence_tokens) - else: - ids = self.convert_tokens_to_ids(self.tokenize(text, **kwargs)) - return ids[:max_length] if max_length != -1 else ids - - first_sentence_tokens = [self._convert_token_to_id(token) for token in self.tokenize(text, **kwargs)] - second_sentence_tokens = [self._convert_token_to_id(token) for token in self.tokenize(text_pair, **kwargs)] - - if add_special_tokens: - if max_length: - if len(first_sentence_tokens) + self.num_added_tokens(pair=True) >= max_length: - logger.warning( - "The first sequence is longer than the maximum specified length. This sequence will not be truncated.") - else: - if len(second_sentence_tokens) + len(first_sentence_tokens) + self.num_added_tokens( - pair=True) > max_length: - second_sentence_tokens = second_sentence_tokens[ - :max_length - len(first_sentence_tokens) - self.num_added_tokens( - pair=True)] - - return self.add_special_tokens_sentences_pair(first_sentence_tokens, second_sentence_tokens, - output_mask) - else: - if max_length: - first_sentence_tokens = first_sentence_tokens[:max_length] - second_sentence_tokens = second_sentence_tokens[:max_length] - - if output_mask: - logger.warning("Can't output mask if you're not joining two sequences.") - return first_sentence_tokens, second_sentence_tokens - def add_special_tokens_single_sentence(self, token_ids): logger.warning("This tokenizer does not make use of special tokens. The sequence has been returned with no modification.") return token_ids From bf503158c558d11cf6ab6ce8cf2e91e3e81b479a Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Thu, 19 Sep 2019 09:55:36 +0200 Subject: [PATCH 077/219] Sentence -> Sequence. Removed output_mask from the special token addition methods. --- examples/run_lm_finetuning.py | 2 +- .../tests/tokenization_bert_test.py | 4 +-- .../tests/tokenization_distilbert_test.py | 4 +-- .../tests/tokenization_roberta_test.py | 4 +-- .../tests/tokenization_tests_commons.py | 28 +++++++++---------- .../tests/tokenization_xlm_test.py | 4 +-- .../tests/tokenization_xlnet_test.py | 4 +-- pytorch_transformers/tokenization_bert.py | 13 +++------ .../tokenization_distilbert.py | 4 +-- pytorch_transformers/tokenization_roberta.py | 12 ++------ pytorch_transformers/tokenization_utils.py | 21 ++++++-------- pytorch_transformers/tokenization_xlm.py | 13 ++------- pytorch_transformers/tokenization_xlnet.py | 12 ++------ 13 files changed, 49 insertions(+), 76 deletions(-) diff --git a/examples/run_lm_finetuning.py b/examples/run_lm_finetuning.py index 974a84c34e..556cce6753 100644 --- a/examples/run_lm_finetuning.py +++ b/examples/run_lm_finetuning.py @@ -75,7 +75,7 @@ class TextDataset(Dataset): tokenized_text = tokenizer.convert_tokens_to_ids(tokenizer.tokenize(text)) while len(tokenized_text) >= block_size: # Truncate in block of block_size - self.examples.append(tokenizer.add_special_tokens_single_sentence(tokenized_text[:block_size])) + self.examples.append(tokenizer.add_special_tokens_single_sequence(tokenized_text[:block_size])) tokenized_text = tokenized_text[block_size:] # Note that we are loosing the last truncated example here for the sake of simplicity (no padding) # If your dataset is small, first you should loook for a bigger one :-) and second you diff --git a/pytorch_transformers/tests/tokenization_bert_test.py b/pytorch_transformers/tests/tokenization_bert_test.py index 1111683ecc..4cfdfed136 100644 --- a/pytorch_transformers/tests/tokenization_bert_test.py +++ b/pytorch_transformers/tests/tokenization_bert_test.py @@ -131,8 +131,8 @@ class BertTokenizationTest(CommonTestCases.CommonTokenizerTester): text = tokenizer.encode("sequence builders") text_2 = tokenizer.encode("multi-sequence build") - encoded_sentence = tokenizer.add_special_tokens_single_sentence(text) - encoded_pair = tokenizer.add_special_tokens_sentences_pair(text, text_2) + encoded_sentence = tokenizer.add_special_tokens_single_sequence(text) + encoded_pair = tokenizer.add_special_tokens_sequence_pair(text, text_2) assert encoded_sentence == [101] + text + [102] assert encoded_pair == [101] + text + [102] + text_2 + [102] diff --git a/pytorch_transformers/tests/tokenization_distilbert_test.py b/pytorch_transformers/tests/tokenization_distilbert_test.py index 30160fc98e..ef3cac10be 100644 --- a/pytorch_transformers/tests/tokenization_distilbert_test.py +++ b/pytorch_transformers/tests/tokenization_distilbert_test.py @@ -36,8 +36,8 @@ class DistilBertTokenizationTest(BertTokenizationTest): text = tokenizer.encode("sequence builders") text_2 = tokenizer.encode("multi-sequence build") - encoded_sentence = tokenizer.add_special_tokens_single_sentence(text) - encoded_pair = tokenizer.add_special_tokens_sentences_pair(text, text_2) + encoded_sentence = tokenizer.add_special_tokens_single_sequence(text) + encoded_pair = tokenizer.add_special_tokens_sequence_pair(text, text_2) assert encoded_sentence == text assert encoded_pair == text + [102] + text_2 diff --git a/pytorch_transformers/tests/tokenization_roberta_test.py b/pytorch_transformers/tests/tokenization_roberta_test.py index 8add2529a5..ba2651c7f2 100644 --- a/pytorch_transformers/tests/tokenization_roberta_test.py +++ b/pytorch_transformers/tests/tokenization_roberta_test.py @@ -87,8 +87,8 @@ class RobertaTokenizationTest(CommonTestCases.CommonTokenizerTester): encoded_text_from_decode = tokenizer.encode("sequence builders", add_special_tokens=True) encoded_pair_from_decode = tokenizer.encode("sequence builders", "multi-sequence build", add_special_tokens=True) - encoded_sentence = tokenizer.add_special_tokens_single_sentence(text) - encoded_pair = tokenizer.add_special_tokens_sentences_pair(text, text_2) + encoded_sentence = tokenizer.add_special_tokens_single_sequence(text) + encoded_pair = tokenizer.add_special_tokens_sequence_pair(text, text_2) assert encoded_sentence == encoded_text_from_decode assert encoded_pair == encoded_pair_from_decode diff --git a/pytorch_transformers/tests/tokenization_tests_commons.py b/pytorch_transformers/tests/tokenization_tests_commons.py index 4f85e2bdd6..e16891a5db 100644 --- a/pytorch_transformers/tests/tokenization_tests_commons.py +++ b/pytorch_transformers/tests/tokenization_tests_commons.py @@ -187,18 +187,18 @@ class CommonTestCases: for weights_list_2 in weights_lists_2: self.assertListEqual(weights_list, weights_list_2) - def test_mask_output(self): - if sys.version_info <= (3, 0): - return - - tokenizer = self.get_tokenizer() - - if tokenizer.add_special_tokens_sentences_pair.__qualname__.split('.')[0] != "PreTrainedTokenizer": - seq_0 = "Test this method." - seq_1 = "With these inputs." - information = tokenizer.encode_plus(seq_0, seq_1, add_special_tokens=True, output_mask=True) - sequences, mask = information["sequence"], information["mask"] - assert len(sequences) == len(mask) + # def test_mask_output(self): + # if sys.version_info <= (3, 0): + # return + # + # tokenizer = self.get_tokenizer() + # + # if tokenizer.add_special_tokens_sequence_pair.__qualname__.split('.')[0] != "PreTrainedTokenizer": + # seq_0 = "Test this method." + # seq_1 = "With these inputs." + # information = tokenizer.encode_plus(seq_0, seq_1, add_special_tokens=True, output_mask=True) + # sequences, mask = information["sequence"], information["mask"] + # assert len(sequences) == len(mask) def test_number_of_added_tokens(self): tokenizer = self.get_tokenizer() @@ -228,7 +228,7 @@ class CommonTestCases: assert len(overflowing_tokens) == 2 assert len(truncated_sequence) == total_length - 2 - assert truncated_sequence == tokenizer.add_special_tokens_single_sentence(sequence[:-2]) + assert truncated_sequence == tokenizer.add_special_tokens_single_sequence(sequence[:-2]) def test_maximum_encoding_length_pair_input(self): tokenizer = self.get_tokenizer() @@ -237,7 +237,7 @@ class CommonTestCases: seq_1 = "This is another sentence to be encoded." sequence = tokenizer.encode(seq_0, seq_1, add_special_tokens=True) - truncated_second_sequence = tokenizer.add_special_tokens_sentences_pair( + truncated_second_sequence = tokenizer.add_special_tokens_sequence_pair( tokenizer.encode(seq_0), tokenizer.encode(seq_1)[:-2] ) diff --git a/pytorch_transformers/tests/tokenization_xlm_test.py b/pytorch_transformers/tests/tokenization_xlm_test.py index 43f1e0c5dd..13fdb4a8bb 100644 --- a/pytorch_transformers/tests/tokenization_xlm_test.py +++ b/pytorch_transformers/tests/tokenization_xlm_test.py @@ -72,8 +72,8 @@ class XLMTokenizationTest(CommonTestCases.CommonTokenizerTester): text = tokenizer.encode("sequence builders") text_2 = tokenizer.encode("multi-sequence build") - encoded_sentence = tokenizer.add_special_tokens_single_sentence(text) - encoded_pair = tokenizer.add_special_tokens_sentences_pair(text, text_2) + encoded_sentence = tokenizer.add_special_tokens_single_sequence(text) + encoded_pair = tokenizer.add_special_tokens_sequence_pair(text, text_2) assert encoded_sentence == [1] + text + [1] assert encoded_pair == [1] + text + [1] + text_2 + [1] diff --git a/pytorch_transformers/tests/tokenization_xlnet_test.py b/pytorch_transformers/tests/tokenization_xlnet_test.py index c603ce55f9..a6e9f23fe7 100644 --- a/pytorch_transformers/tests/tokenization_xlnet_test.py +++ b/pytorch_transformers/tests/tokenization_xlnet_test.py @@ -95,8 +95,8 @@ class XLNetTokenizationTest(CommonTestCases.CommonTokenizerTester): text = tokenizer.encode("sequence builders") text_2 = tokenizer.encode("multi-sequence build") - encoded_sentence = tokenizer.add_special_tokens_single_sentence(text) - encoded_pair = tokenizer.add_special_tokens_sentences_pair(text, text_2) + encoded_sentence = tokenizer.add_special_tokens_single_sequence(text) + encoded_pair = tokenizer.add_special_tokens_sequence_pair(text, text_2) assert encoded_sentence == text + [4, 3] assert encoded_pair == text + [4] + text_2 + [4, 3] diff --git a/pytorch_transformers/tokenization_bert.py b/pytorch_transformers/tokenization_bert.py index f720d530f8..0aae2d6a5f 100644 --- a/pytorch_transformers/tokenization_bert.py +++ b/pytorch_transformers/tokenization_bert.py @@ -187,27 +187,22 @@ class BertTokenizer(PreTrainedTokenizer): out_string = ' '.join(tokens).replace(' ##', '').strip() return out_string - def add_special_tokens_single_sentence(self, token_ids): + def add_special_tokens_single_sequence(self, token_ids): """ Adds special tokens to the a sequence for sequence classification tasks. A BERT sequence has the following format: [CLS] X [SEP] """ return [self.cls_token_id] + token_ids + [self.sep_token_id] - def add_special_tokens_sentences_pair(self, token_ids_0, token_ids_1, output_mask=False): + def add_special_tokens_sequence_pair(self, token_ids_0, token_ids_1): """ Adds special tokens to a sequence pair for sequence classification tasks. A BERT sequence pair has the following format: [CLS] A [SEP] B [SEP] """ sep = [self.sep_token_id] cls = [self.cls_token_id] - if output_mask: - return ( - cls + token_ids_0 + sep + token_ids_1 + sep, - [0] * len(cls + token_ids_0 + sep) + [1] * len(token_ids_1 + sep) - ) - else: - return cls + token_ids_0 + sep + token_ids_1 + sep + + return cls + token_ids_0 + sep + token_ids_1 + sep def save_vocabulary(self, vocab_path): """Save the tokenizer vocabulary to a directory or file.""" diff --git a/pytorch_transformers/tokenization_distilbert.py b/pytorch_transformers/tokenization_distilbert.py index f91989d2bd..ce2ace35d8 100644 --- a/pytorch_transformers/tokenization_distilbert.py +++ b/pytorch_transformers/tokenization_distilbert.py @@ -61,10 +61,10 @@ class DistilBertTokenizer(BertTokenizer): pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES - def add_special_tokens_single_sentence(self, token_ids): + def add_special_tokens_single_sequence(self, token_ids): return token_ids - def add_special_tokens_sentences_pair(self, token_ids_0, token_ids_1, output_mask=False): + def add_special_tokens_sequence_pair(self, token_ids_0, token_ids_1, output_mask=False): sep = [self.sep_token_id] if output_mask: return ( diff --git a/pytorch_transformers/tokenization_roberta.py b/pytorch_transformers/tokenization_roberta.py index b41d85e7e2..b39904711a 100644 --- a/pytorch_transformers/tokenization_roberta.py +++ b/pytorch_transformers/tokenization_roberta.py @@ -81,24 +81,18 @@ class RobertaTokenizer(GPT2Tokenizer): sep_token=sep_token, cls_token=cls_token, pad_token=pad_token, mask_token=mask_token, **kwargs) - def add_special_tokens_single_sentence(self, token_ids): + def add_special_tokens_single_sequence(self, token_ids): """ Adds special tokens to a sequence for sequence classification tasks. A RoBERTa sequence has the following format: X """ return [self.cls_token_id] + token_ids + [self.sep_token_id] - def add_special_tokens_sentences_pair(self, token_ids_0, token_ids_1, output_mask=False): + def add_special_tokens_sequence_pair(self, token_ids_0, token_ids_1): """ Adds special tokens to a sequence pair for sequence classification tasks. A RoBERTa sequence pair has the following format: A B """ sep = [self.sep_token_id] cls = [self.cls_token_id] - if output_mask: - return ( - cls + token_ids_0 + sep + sep + token_ids_1 + sep, - [0] * len(cls + token_ids_0 + sep + sep) + [1] * len(token_ids_1 + sep) - ) - else: - return cls + token_ids_0 + sep + sep + token_ids_1 + sep + return cls + token_ids_0 + sep + sep + token_ids_1 + sep diff --git a/pytorch_transformers/tokenization_utils.py b/pytorch_transformers/tokenization_utils.py index 2a31aec887..185ca09576 100644 --- a/pytorch_transformers/tokenization_utils.py +++ b/pytorch_transformers/tokenization_utils.py @@ -708,7 +708,7 @@ class PreTrainedTokenizer(object): if text_pair is None: if add_special_tokens: sequence_tokens = self.convert_tokens_to_ids(self.tokenize(text, **kwargs)) - return self.add_special_tokens_single_sentence(sequence_tokens) + return self.add_special_tokens_single_sequence(sequence_tokens) else: ids = self.convert_tokens_to_ids(self.tokenize(text, **kwargs)) return ids @@ -717,7 +717,7 @@ class PreTrainedTokenizer(object): second_sentence_tokens = [self._convert_token_to_id(token) for token in self.tokenize(text_pair, **kwargs)] if add_special_tokens: - return self.add_special_tokens_sentences_pair(first_sentence_tokens, second_sentence_tokens) + return self.add_special_tokens_sequence_pair(first_sentence_tokens, second_sentence_tokens) else: logger.warning("No special tokens were added. The two sequences have been concatenated.") return first_sentence_tokens + second_sentence_tokens @@ -747,7 +747,7 @@ class PreTrainedTokenizer(object): if max_length: information["overflowing_tokens"] = sequence_tokens[max_length - n_added_tokens:] sequence_tokens = sequence_tokens[:max_length - n_added_tokens] - sequence = self.add_special_tokens_single_sentence(sequence_tokens) + sequence = self.add_special_tokens_single_sequence(sequence_tokens) else: sequence_tokens = self.convert_tokens_to_ids(self.tokenize(text, **kwargs)) if max_length: @@ -774,16 +774,13 @@ class PreTrainedTokenizer(object): information["overflowing_tokens"] = second_sentence_tokens[max_length - f_len - n_added_tokens:] second_sentence_tokens = second_sentence_tokens[:max_length - f_len - n_added_tokens] - encoded_sequence = self.add_special_tokens_sentences_pair( + sequence = self.add_special_tokens_sequence_pair( first_sentence_tokens, - second_sentence_tokens, - output_mask + second_sentence_tokens ) - if output_mask: - sequence, information["mask"] = encoded_sequence - else: - sequence = encoded_sequence + # if output_mask: + # sequence, information["mask"] = encoded_sequence information["sequence"] = sequence else: @@ -800,11 +797,11 @@ class PreTrainedTokenizer(object): return information - def add_special_tokens_single_sentence(self, token_ids): + def add_special_tokens_single_sequence(self, token_ids): logger.warning("This tokenizer does not make use of special tokens. The sequence has been returned with no modification.") return token_ids - def add_special_tokens_sentences_pair(self, token_ids_0, token_ids_1, output_mask=False): + def add_special_tokens_sequence_pair(self, token_ids_0, token_ids_1): logger.warning("This tokenizer does not make use of special tokens. The two sequences have been concatenated.") return token_ids_0 + token_ids_1 diff --git a/pytorch_transformers/tokenization_xlm.py b/pytorch_transformers/tokenization_xlm.py index 167860f15f..5a29944d10 100644 --- a/pytorch_transformers/tokenization_xlm.py +++ b/pytorch_transformers/tokenization_xlm.py @@ -754,28 +754,21 @@ class XLMTokenizer(PreTrainedTokenizer): out_string = ''.join(tokens).replace('', ' ').strip() return out_string - def add_special_tokens_single_sentence(self, token_ids): + def add_special_tokens_single_sequence(self, token_ids): """ Adds special tokens to a sequence for sequence classification tasks. An XLM sequence has the following format: [CLS] X [SEP] """ return [self.cls_token_id] + token_ids + [self.sep_token_id] - def add_special_tokens_sentences_pair(self, token_ids_0, token_ids_1, output_mask=False): + def add_special_tokens_sequence_pair(self, token_ids_0, token_ids_1): """ Adds special tokens to a sequence pair for sequence classification tasks. An XLM sequence pair has the following format: [CLS] A [SEP] B [SEP] """ sep = [self.sep_token_id] cls = [self.cls_token_id] - - if output_mask: - return ( - cls + token_ids_0 + sep + token_ids_1 + sep, - [0] * len(cls + token_ids_0 + sep) + [1] * len(token_ids_1 + sep) - ) - else: - return cls + token_ids_0 + sep + token_ids_1 + sep + return cls + token_ids_0 + sep + token_ids_1 + sep def save_vocabulary(self, save_directory): """Save the tokenizer vocabulary and merge files to a directory.""" diff --git a/pytorch_transformers/tokenization_xlnet.py b/pytorch_transformers/tokenization_xlnet.py index 9faa3c7e59..fecc35b070 100644 --- a/pytorch_transformers/tokenization_xlnet.py +++ b/pytorch_transformers/tokenization_xlnet.py @@ -181,7 +181,7 @@ class XLNetTokenizer(PreTrainedTokenizer): out_string = ''.join(tokens).replace(SPIECE_UNDERLINE, ' ').strip() return out_string - def add_special_tokens_single_sentence(self, token_ids): + def add_special_tokens_single_sequence(self, token_ids): """ Adds special tokens to a sequence pair for sequence classification tasks. An XLNet sequence pair has the following format: A [SEP] B [SEP][CLS] @@ -190,7 +190,7 @@ class XLNetTokenizer(PreTrainedTokenizer): cls = [self.cls_token_id] return token_ids + sep + cls - def add_special_tokens_sentences_pair(self, token_ids_0, token_ids_1, output_mask=False): + def add_special_tokens_sequence_pair(self, token_ids_0, token_ids_1): """ Adds special tokens to a sequence for sequence classification tasks. An XLNet sequence has the following format: X [SEP][CLS] @@ -199,13 +199,7 @@ class XLNetTokenizer(PreTrainedTokenizer): sep = [self.sep_token_id] cls = [self.cls_token_id] cls_segment_ids = [2] - if output_mask: - return ( - token_ids_0 + sep + token_ids_1 + sep + cls, - [0] * len(token_ids_0 + sep) + [1] * len(token_ids_1 + sep) + cls_segment_ids - ) - else: - return token_ids_0 + sep + token_ids_1 + sep + cls + return token_ids_0 + sep + token_ids_1 + sep + cls def save_vocabulary(self, save_directory): """ Save the sentencepiece vocabulary (copy original file) and special tokens file From c10c7d59e7a306609c44d1523346dbbc3fd7121d Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Thu, 19 Sep 2019 10:13:10 +0200 Subject: [PATCH 078/219] Mask computing in standalone method. Tests. --- .../tests/tokenization_tests_commons.py | 24 +++++++++---------- pytorch_transformers/tokenization_bert.py | 12 ++++++++++ .../tokenization_distilbert.py | 22 ++++++++++------- pytorch_transformers/tokenization_roberta.py | 12 ++++++++++ pytorch_transformers/tokenization_utils.py | 8 +++++-- pytorch_transformers/tokenization_xlm.py | 12 ++++++++++ pytorch_transformers/tokenization_xlnet.py | 14 ++++++++++- 7 files changed, 81 insertions(+), 23 deletions(-) diff --git a/pytorch_transformers/tests/tokenization_tests_commons.py b/pytorch_transformers/tests/tokenization_tests_commons.py index e16891a5db..e1c5ccb10a 100644 --- a/pytorch_transformers/tests/tokenization_tests_commons.py +++ b/pytorch_transformers/tests/tokenization_tests_commons.py @@ -187,18 +187,18 @@ class CommonTestCases: for weights_list_2 in weights_lists_2: self.assertListEqual(weights_list, weights_list_2) - # def test_mask_output(self): - # if sys.version_info <= (3, 0): - # return - # - # tokenizer = self.get_tokenizer() - # - # if tokenizer.add_special_tokens_sequence_pair.__qualname__.split('.')[0] != "PreTrainedTokenizer": - # seq_0 = "Test this method." - # seq_1 = "With these inputs." - # information = tokenizer.encode_plus(seq_0, seq_1, add_special_tokens=True, output_mask=True) - # sequences, mask = information["sequence"], information["mask"] - # assert len(sequences) == len(mask) + def test_mask_output(self): + if sys.version_info <= (3, 0): + return + + tokenizer = self.get_tokenizer() + + if tokenizer.add_special_tokens_sequence_pair.__qualname__.split('.')[0] != "PreTrainedTokenizer": + seq_0 = "Test this method." + seq_1 = "With these inputs." + information = tokenizer.encode_plus(seq_0, seq_1, add_special_tokens=True, output_mask=True) + sequences, mask = information["sequence"], information["mask"] + assert len(sequences) == len(mask) def test_number_of_added_tokens(self): tokenizer = self.get_tokenizer() diff --git a/pytorch_transformers/tokenization_bert.py b/pytorch_transformers/tokenization_bert.py index 0aae2d6a5f..211b8fe93b 100644 --- a/pytorch_transformers/tokenization_bert.py +++ b/pytorch_transformers/tokenization_bert.py @@ -204,6 +204,18 @@ class BertTokenizer(PreTrainedTokenizer): return cls + token_ids_0 + sep + token_ids_1 + sep + def create_mask_from_sequences(self, sequence_0, sequence_1): + """ + Creates a mask from the two sequences passed to be used in a sequence-pair classification task. + A BERT sequence pair mask has the following format: + 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 + | first sequence | second sequence + """ + sep = [self.sep_token_id] + cls = [self.cls_token_id] + + return len(cls + self.encode(sequence_0) + sep) * [0] + len(self.encode(sequence_1) + sep) * [1] + def save_vocabulary(self, vocab_path): """Save the tokenizer vocabulary to a directory or file.""" index = 0 diff --git a/pytorch_transformers/tokenization_distilbert.py b/pytorch_transformers/tokenization_distilbert.py index ce2ace35d8..cb716594a2 100644 --- a/pytorch_transformers/tokenization_distilbert.py +++ b/pytorch_transformers/tokenization_distilbert.py @@ -64,12 +64,18 @@ class DistilBertTokenizer(BertTokenizer): def add_special_tokens_single_sequence(self, token_ids): return token_ids - def add_special_tokens_sequence_pair(self, token_ids_0, token_ids_1, output_mask=False): + def add_special_tokens_sequence_pair(self, token_ids_0, token_ids_1): sep = [self.sep_token_id] - if output_mask: - return ( - token_ids_0 + sep + token_ids_1, - [0] * len(token_ids_0 + sep) + [1] * len(token_ids_1) - ) - else: - return token_ids_0 + sep + token_ids_1 + return token_ids_0 + sep + token_ids_1 + + def create_mask_from_sequences(self, sequence_0, sequence_1): + """ + Creates a mask from the two sequences passed to be used in a sequence-pair classification task. + A BERT sequence pair mask has the following format: + 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 + | first sequence | second sequence + """ + sep = [self.sep_token_id] + cls = [self.cls_token_id] + + return len(self.encode(sequence_0) + sep) * [0] + len(self.encode(sequence_1)) * [1] diff --git a/pytorch_transformers/tokenization_roberta.py b/pytorch_transformers/tokenization_roberta.py index b39904711a..161c4e6870 100644 --- a/pytorch_transformers/tokenization_roberta.py +++ b/pytorch_transformers/tokenization_roberta.py @@ -96,3 +96,15 @@ class RobertaTokenizer(GPT2Tokenizer): sep = [self.sep_token_id] cls = [self.cls_token_id] return cls + token_ids_0 + sep + sep + token_ids_1 + sep + + def create_mask_from_sequences(self, sequence_0, sequence_1): + """ + Creates a mask from the two sequences passed to be used in a sequence-pair classification task. + A RoBERTa sequence pair mask has the following format: + 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 + | first sequence | second sequence + """ + sep = [self.sep_token_id] + cls = [self.cls_token_id] + + return len(cls + self.encode(sequence_0) + sep + sep) * [0] + len(self.encode(sequence_1) + sep) * [1] \ No newline at end of file diff --git a/pytorch_transformers/tokenization_utils.py b/pytorch_transformers/tokenization_utils.py index 185ca09576..490b2611e1 100644 --- a/pytorch_transformers/tokenization_utils.py +++ b/pytorch_transformers/tokenization_utils.py @@ -779,8 +779,8 @@ class PreTrainedTokenizer(object): second_sentence_tokens ) - # if output_mask: - # sequence, information["mask"] = encoded_sequence + if output_mask: + information["mask"] = self.create_mask_from_sequences(text, text_pair) information["sequence"] = sequence else: @@ -797,6 +797,10 @@ class PreTrainedTokenizer(object): return information + def create_mask_from_sequences(self, sequence_0, sequence_1): + logger.warning("This tokenizer does not make use of special tokens.") + return [0] * len(self.encode(sequence_0)) + [1] * len(self.encode(sequence_1)) + def add_special_tokens_single_sequence(self, token_ids): logger.warning("This tokenizer does not make use of special tokens. The sequence has been returned with no modification.") return token_ids diff --git a/pytorch_transformers/tokenization_xlm.py b/pytorch_transformers/tokenization_xlm.py index 5a29944d10..3b177b6a17 100644 --- a/pytorch_transformers/tokenization_xlm.py +++ b/pytorch_transformers/tokenization_xlm.py @@ -770,6 +770,18 @@ class XLMTokenizer(PreTrainedTokenizer): cls = [self.cls_token_id] return cls + token_ids_0 + sep + token_ids_1 + sep + def create_mask_from_sequences(self, sequence_0, sequence_1): + """ + Creates a mask from the two sequences passed to be used in a sequence-pair classification task. + An XLM sequence pair mask has the following format: + 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 + | first sequence | second sequence + """ + sep = [self.sep_token_id] + cls = [self.cls_token_id] + + return len(cls + self.encode(sequence_0) + sep) * [0] + len(self.encode(sequence_1) + sep) * [1] + def save_vocabulary(self, save_directory): """Save the tokenizer vocabulary and merge files to a directory.""" if not os.path.isdir(save_directory): diff --git a/pytorch_transformers/tokenization_xlnet.py b/pytorch_transformers/tokenization_xlnet.py index fecc35b070..1464743759 100644 --- a/pytorch_transformers/tokenization_xlnet.py +++ b/pytorch_transformers/tokenization_xlnet.py @@ -198,9 +198,21 @@ class XLNetTokenizer(PreTrainedTokenizer): sep = [self.sep_token_id] cls = [self.cls_token_id] - cls_segment_ids = [2] return token_ids_0 + sep + token_ids_1 + sep + cls + def create_mask_from_sequences(self, sequence_0, sequence_1): + """ + Creates a mask from the two sequences passed to be used in a sequence-pair classification task. + A BERT sequence pair mask has the following format: + 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 2 + | first sequence | second sequence | CLS segment ID + """ + sep = [self.sep_token_id] + cls = [self.cls_token_id] + cls_segment_id = [2] + + return len(self.encode(sequence_0) + sep) * [0] + len(self.encode(sequence_1) + sep) * [1] + cls_segment_id + def save_vocabulary(self, save_directory): """ Save the sentencepiece vocabulary (copy original file) and special tokens file to a directory. From baa74326ab4d0443ae22f998e494d6306ec738d8 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Thu, 19 Sep 2019 10:42:32 +0200 Subject: [PATCH 079/219] Stride + tests + small fixes --- .../tests/tokenization_tests_commons.py | 6 ++++-- pytorch_transformers/tokenization_distilbert.py | 1 - pytorch_transformers/tokenization_utils.py | 11 +++++++---- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/pytorch_transformers/tests/tokenization_tests_commons.py b/pytorch_transformers/tests/tokenization_tests_commons.py index e1c5ccb10a..65a2938378 100644 --- a/pytorch_transformers/tests/tokenization_tests_commons.py +++ b/pytorch_transformers/tests/tokenization_tests_commons.py @@ -217,16 +217,18 @@ class CommonTestCases: tokenizer = self.get_tokenizer() seq_0 = "This is a sentence to be encoded." + stride = 2 sequence = tokenizer.encode(seq_0) num_added_tokens = tokenizer.num_added_tokens() total_length = len(sequence) + num_added_tokens - information = tokenizer.encode_plus(seq_0, max_length=total_length - 2, add_special_tokens=True) + information = tokenizer.encode_plus(seq_0, max_length=total_length - 2, add_special_tokens=True, stride=stride) truncated_sequence = information["sequence"] overflowing_tokens = information["overflowing_tokens"] - assert len(overflowing_tokens) == 2 + assert len(overflowing_tokens) == 2 + stride + assert overflowing_tokens == sequence[-(2 + stride):] assert len(truncated_sequence) == total_length - 2 assert truncated_sequence == tokenizer.add_special_tokens_single_sequence(sequence[:-2]) diff --git a/pytorch_transformers/tokenization_distilbert.py b/pytorch_transformers/tokenization_distilbert.py index cb716594a2..0af782beb1 100644 --- a/pytorch_transformers/tokenization_distilbert.py +++ b/pytorch_transformers/tokenization_distilbert.py @@ -76,6 +76,5 @@ class DistilBertTokenizer(BertTokenizer): | first sequence | second sequence """ sep = [self.sep_token_id] - cls = [self.cls_token_id] return len(self.encode(sequence_0) + sep) * [0] + len(self.encode(sequence_1)) * [1] diff --git a/pytorch_transformers/tokenization_utils.py b/pytorch_transformers/tokenization_utils.py index 490b2611e1..c4a9c71917 100644 --- a/pytorch_transformers/tokenization_utils.py +++ b/pytorch_transformers/tokenization_utils.py @@ -722,7 +722,7 @@ class PreTrainedTokenizer(object): logger.warning("No special tokens were added. The two sequences have been concatenated.") return first_sentence_tokens + second_sentence_tokens - def encode_plus(self, text, text_pair=None, add_special_tokens=False, output_mask=False, max_length=None, **kwargs): + def encode_plus(self, text, text_pair=None, add_special_tokens=False, output_mask=False, max_length=None, stride=0, **kwargs): """ Returns a dictionary containing the encoded sequence or sequence pair. Other values can be returned by this method: the mask for sequence classification and the overflowing elements if a ``max_length`` is specified. @@ -735,6 +735,9 @@ class PreTrainedTokenizer(object): output_mask: if set to ``True``, returns the text pair corresponding mask with 0 for the first sequence, and 1 for the second. max_length: if set to a number, will limit the total sequence returned so that it has a maximum length. + If there are overflowing tokens, those will be added to the returned dictionary + stride: if set to a number along with max_length, the overflowing tokens returned will contain some tokens + from the main sequence returned. The value of this argument defined the number of additional tokens. **kwargs: passed to the `self.tokenize()` method """ @@ -745,13 +748,13 @@ class PreTrainedTokenizer(object): if add_special_tokens: sequence_tokens = self.convert_tokens_to_ids(self.tokenize(text, **kwargs)) if max_length: - information["overflowing_tokens"] = sequence_tokens[max_length - n_added_tokens:] + information["overflowing_tokens"] = sequence_tokens[max_length - n_added_tokens - stride:] sequence_tokens = sequence_tokens[:max_length - n_added_tokens] sequence = self.add_special_tokens_single_sequence(sequence_tokens) else: sequence_tokens = self.convert_tokens_to_ids(self.tokenize(text, **kwargs)) if max_length: - information["overflowing_tokens"] = sequence_tokens[max_length:] + information["overflowing_tokens"] = sequence_tokens[max_length - stride:] sequence_tokens = sequence_tokens[:max_length] sequence = sequence_tokens @@ -788,7 +791,7 @@ class PreTrainedTokenizer(object): sequence = first_sentence_tokens + second_sentence_tokens if max_length: - information["overflowing_tokens"] = sequence[max_length:] + information["overflowing_tokens"] = sequence[max_length - stride:] sequence = sequence[:max_length] if output_mask: information["mask"] = [0] * len(sequence) From 60414f31a9a67ade65d90191c0b8349a189978a6 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Thu, 19 Sep 2019 10:53:45 +0200 Subject: [PATCH 080/219] GLUE updated with new methods --- examples/utils_glue.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/examples/utils_glue.py b/examples/utils_glue.py index 8d0ed737d7..e2fc3a119a 100644 --- a/examples/utils_glue.py +++ b/examples/utils_glue.py @@ -405,7 +405,14 @@ def convert_examples_to_features(examples, label_list, max_seq_length, if ex_index % 10000 == 0: logger.info("Writing example %d of %d" % (ex_index, len(examples))) - input_ids, segment_ids = tokenizer.encode(example.text_a, example.text_b, add_special_tokens=True, output_mask=True) + inputs = tokenizer.encode_plus( + example.text_a, + example.text_b, + add_special_tokens=True, + output_mask=True, + max_length=max_seq_length + ) + input_ids, segment_ids = inputs["sequence"], inputs["mask"] # The mask has 1 for real tokens and 0 for padding tokens. Only real # tokens are attended to. From 66ea76b8a9e867494993a96a671a94735d99a317 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Thu, 19 Sep 2019 13:50:51 +0200 Subject: [PATCH 081/219] prepare_for_model and prepare_pair_for_model methods. Added an option to select which sequence will be truncated. --- .../tests/tokenization_tests_commons.py | 15 +++- pytorch_transformers/tokenization_utils.py | 79 +++++++++++++------ 2 files changed, 67 insertions(+), 27 deletions(-) diff --git a/pytorch_transformers/tests/tokenization_tests_commons.py b/pytorch_transformers/tests/tokenization_tests_commons.py index 65a2938378..5da19bb660 100644 --- a/pytorch_transformers/tests/tokenization_tests_commons.py +++ b/pytorch_transformers/tests/tokenization_tests_commons.py @@ -237,16 +237,29 @@ class CommonTestCases: seq_0 = "This is a sentence to be encoded." seq_1 = "This is another sentence to be encoded." + stride = 2 + + sequence_0_no_special_tokens = tokenizer.encode(seq_0) + sequence_1_no_special_tokens = tokenizer.encode(seq_1) sequence = tokenizer.encode(seq_0, seq_1, add_special_tokens=True) truncated_second_sequence = tokenizer.add_special_tokens_sequence_pair( tokenizer.encode(seq_0), tokenizer.encode(seq_1)[:-2] ) - information = tokenizer.encode_plus(seq_0, seq_1, max_length=len(sequence) - 2, add_special_tokens=True) + + information = tokenizer.encode_plus(seq_0, seq_1, max_length=len(sequence) - 2, add_special_tokens=True, + stride=stride) + information_first_truncated = tokenizer.encode_plus(seq_0, seq_1, max_length=len(sequence) - 2, + add_special_tokens=True, stride=stride, + truncate_second_sequence_first=False) truncated_sequence = information["sequence"] overflowing_tokens = information["overflowing_tokens"] + overflowing_tokens_first_truncated = information_first_truncated["overflowing_tokens"] + assert len(overflowing_tokens) == 2 + stride + assert overflowing_tokens == sequence_1_no_special_tokens[-(2 + stride):] + assert overflowing_tokens_first_truncated == sequence_0_no_special_tokens[-(2 + stride):] assert len(truncated_sequence) == len(sequence) - 2 assert truncated_sequence == truncated_second_sequence diff --git a/pytorch_transformers/tokenization_utils.py b/pytorch_transformers/tokenization_utils.py index c4a9c71917..0b23d00b8a 100644 --- a/pytorch_transformers/tokenization_utils.py +++ b/pytorch_transformers/tokenization_utils.py @@ -722,7 +722,15 @@ class PreTrainedTokenizer(object): logger.warning("No special tokens were added. The two sequences have been concatenated.") return first_sentence_tokens + second_sentence_tokens - def encode_plus(self, text, text_pair=None, add_special_tokens=False, output_mask=False, max_length=None, stride=0, **kwargs): + def encode_plus(self, + text, + text_pair=None, + add_special_tokens=False, + output_mask=False, + max_length=None, + stride=0, + truncate_second_sequence_first=True, + **kwargs): """ Returns a dictionary containing the encoded sequence or sequence pair. Other values can be returned by this method: the mask for sequence classification and the overflowing elements if a ``max_length`` is specified. @@ -738,54 +746,40 @@ class PreTrainedTokenizer(object): If there are overflowing tokens, those will be added to the returned dictionary stride: if set to a number along with max_length, the overflowing tokens returned will contain some tokens from the main sequence returned. The value of this argument defined the number of additional tokens. + truncate_second_sequence_first: if there is a specified max_length, this flag will choose which sequence + will be truncated. **kwargs: passed to the `self.tokenize()` method """ information = {} if text_pair is None: - n_added_tokens = self.num_added_tokens() + sequence_tokens = self.convert_tokens_to_ids(self.tokenize(text, **kwargs)) if add_special_tokens: - sequence_tokens = self.convert_tokens_to_ids(self.tokenize(text, **kwargs)) - if max_length: - information["overflowing_tokens"] = sequence_tokens[max_length - n_added_tokens - stride:] - sequence_tokens = sequence_tokens[:max_length - n_added_tokens] - sequence = self.add_special_tokens_single_sequence(sequence_tokens) + information = self.prepare_for_model(sequence_tokens, max_length, stride) else: - sequence_tokens = self.convert_tokens_to_ids(self.tokenize(text, **kwargs)) if max_length: information["overflowing_tokens"] = sequence_tokens[max_length - stride:] sequence_tokens = sequence_tokens[:max_length] - sequence = sequence_tokens + information["sequence"] = sequence_tokens if output_mask: - information["mask"] = [0] * len(sequence) - - information["sequence"] = sequence + information["mask"] = [0] * len(information["sequence"]) else: first_sentence_tokens = [self._convert_token_to_id(token) for token in self.tokenize(text, **kwargs)] second_sentence_tokens = [self._convert_token_to_id(token) for token in self.tokenize(text_pair, **kwargs)] - f_len, s_len = len(first_sentence_tokens), len(second_sentence_tokens) - n_added_tokens = self.num_added_tokens(pair=True) if add_special_tokens: - if max_length: - if len(first_sentence_tokens) + n_added_tokens >= max_length: - logger.warning("The first sequence is longer than the maximum specified length. This sequence will not be truncated.") - else: - if f_len + s_len + self.num_added_tokens(pair=True) > max_length: - information["overflowing_tokens"] = second_sentence_tokens[max_length - f_len - n_added_tokens:] - second_sentence_tokens = second_sentence_tokens[:max_length - f_len - n_added_tokens] - - sequence = self.add_special_tokens_sequence_pair( + information = self.prepare_pair_for_model( first_sentence_tokens, - second_sentence_tokens + second_sentence_tokens, + max_length, + truncate_second_sequence_first, + stride ) if output_mask: information["mask"] = self.create_mask_from_sequences(text, text_pair) - - information["sequence"] = sequence else: logger.warning("No special tokens were added. The two sequences have been concatenated.") sequence = first_sentence_tokens + second_sentence_tokens @@ -800,6 +794,39 @@ class PreTrainedTokenizer(object): return information + def prepare_for_model(self, ids, max_length=None, stride=0): + information = {} + n_added_tokens = self.num_added_tokens() + if max_length: + information["overflowing_tokens"] = ids[max_length - n_added_tokens - stride:] + ids = ids[:max_length - n_added_tokens] + information["sequence"] = self.add_special_tokens_single_sequence(ids) + + return information + + def prepare_pair_for_model(self, ids_0, ids_1, max_length=None, truncate_second_sequence_first=True, stride=0): + f_len, s_len = len(ids_0), len(ids_1) + n_added_tokens = self.num_added_tokens(pair=True) + information = {} + + if max_length: + if len(ids_0) + n_added_tokens >= max_length: + logger.warning( + "The first sequence is longer than the maximum specified length. This sequence will not be truncated.") + else: + if f_len + s_len + self.num_added_tokens(pair=True) > max_length: + if truncate_second_sequence_first: + information["overflowing_tokens"] = ids_1[max_length - f_len - n_added_tokens - stride:] + ids_1 = ids_1[:max_length - f_len - n_added_tokens] + else: + information["overflowing_tokens"] = ids_0[max_length - s_len - n_added_tokens - stride:] + ids_0 = ids_0[:max_length - s_len - n_added_tokens] + + sequence = self.add_special_tokens_sequence_pair(ids_0, ids_1) + information["sequence"] = sequence + + return information + def create_mask_from_sequences(self, sequence_0, sequence_1): logger.warning("This tokenizer does not make use of special tokens.") return [0] * len(self.encode(sequence_0)) + [1] * len(self.encode(sequence_1)) From 3df208c93a2ce06d64fc22083837d5b74991d0f6 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Thu, 19 Sep 2019 14:47:52 +0200 Subject: [PATCH 082/219] Tokenizer accepts token list as well as string --- .../tests/tokenization_tests_commons.py | 7 +++++++ pytorch_transformers/tokenization_utils.py | 14 +++++++------- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/pytorch_transformers/tests/tokenization_tests_commons.py b/pytorch_transformers/tests/tokenization_tests_commons.py index 5da19bb660..8a3b56a058 100644 --- a/pytorch_transformers/tests/tokenization_tests_commons.py +++ b/pytorch_transformers/tests/tokenization_tests_commons.py @@ -263,3 +263,10 @@ class CommonTestCases: assert overflowing_tokens_first_truncated == sequence_0_no_special_tokens[-(2 + stride):] assert len(truncated_sequence) == len(sequence) - 2 assert truncated_sequence == truncated_second_sequence + + def test_tokens_sent_to_encode(self): + tokenizer = self.get_tokenizer() + + sequence = "Let's encode this sequence" + tokens = tokenizer.encode(sequence) + tokenizer.encode(tokens, add_special_tokens=True) diff --git a/pytorch_transformers/tokenization_utils.py b/pytorch_transformers/tokenization_utils.py index 0b23d00b8a..b32d6daeb6 100644 --- a/pytorch_transformers/tokenization_utils.py +++ b/pytorch_transformers/tokenization_utils.py @@ -707,14 +707,14 @@ class PreTrainedTokenizer(object): """ if text_pair is None: if add_special_tokens: - sequence_tokens = self.convert_tokens_to_ids(self.tokenize(text, **kwargs)) + sequence_tokens = self.convert_tokens_to_ids(self.tokenize(text, **kwargs)) if isinstance(text, str) else text return self.add_special_tokens_single_sequence(sequence_tokens) else: - ids = self.convert_tokens_to_ids(self.tokenize(text, **kwargs)) + ids = self.convert_tokens_to_ids(self.tokenize(text, **kwargs)) if isinstance(text, str) else text return ids - first_sentence_tokens = [self._convert_token_to_id(token) for token in self.tokenize(text, **kwargs)] - second_sentence_tokens = [self._convert_token_to_id(token) for token in self.tokenize(text_pair, **kwargs)] + first_sentence_tokens = [self._convert_token_to_id(token) for token in self.tokenize(text, **kwargs)] if isinstance(text, str) else text + second_sentence_tokens = [self._convert_token_to_id(token) for token in self.tokenize(text_pair, **kwargs)] if isinstance(text_pair, str) else text_pair if add_special_tokens: return self.add_special_tokens_sequence_pair(first_sentence_tokens, second_sentence_tokens) @@ -754,7 +754,7 @@ class PreTrainedTokenizer(object): information = {} if text_pair is None: - sequence_tokens = self.convert_tokens_to_ids(self.tokenize(text, **kwargs)) + sequence_tokens = self.convert_tokens_to_ids(self.tokenize(text, **kwargs)) if isinstance(text, str) else text if add_special_tokens: information = self.prepare_for_model(sequence_tokens, max_length, stride) else: @@ -766,8 +766,8 @@ class PreTrainedTokenizer(object): if output_mask: information["mask"] = [0] * len(information["sequence"]) else: - first_sentence_tokens = [self._convert_token_to_id(token) for token in self.tokenize(text, **kwargs)] - second_sentence_tokens = [self._convert_token_to_id(token) for token in self.tokenize(text_pair, **kwargs)] + first_sentence_tokens = [self._convert_token_to_id(token) for token in self.tokenize(text, **kwargs)] if isinstance(text, str) else text + second_sentence_tokens = [self._convert_token_to_id(token) for token in self.tokenize(text_pair, **kwargs)] if isinstance(text_pair, str) else text_pair if add_special_tokens: information = self.prepare_pair_for_model( From ab984a8b720972651dbe1f48161f4df8bcc09178 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Thu, 19 Sep 2019 15:01:33 +0200 Subject: [PATCH 083/219] Python 2 compatibility --- pytorch_transformers/tokenization_utils.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pytorch_transformers/tokenization_utils.py b/pytorch_transformers/tokenization_utils.py index b32d6daeb6..f2cb383143 100644 --- a/pytorch_transformers/tokenization_utils.py +++ b/pytorch_transformers/tokenization_utils.py @@ -707,14 +707,14 @@ class PreTrainedTokenizer(object): """ if text_pair is None: if add_special_tokens: - sequence_tokens = self.convert_tokens_to_ids(self.tokenize(text, **kwargs)) if isinstance(text, str) else text + sequence_tokens = self.convert_tokens_to_ids(self.tokenize(text, **kwargs)) if isinstance(text, six.string_types) else text return self.add_special_tokens_single_sequence(sequence_tokens) else: - ids = self.convert_tokens_to_ids(self.tokenize(text, **kwargs)) if isinstance(text, str) else text + ids = self.convert_tokens_to_ids(self.tokenize(text, **kwargs)) if isinstance(text, six.string_types) else text return ids - first_sentence_tokens = [self._convert_token_to_id(token) for token in self.tokenize(text, **kwargs)] if isinstance(text, str) else text - second_sentence_tokens = [self._convert_token_to_id(token) for token in self.tokenize(text_pair, **kwargs)] if isinstance(text_pair, str) else text_pair + first_sentence_tokens = [self._convert_token_to_id(token) for token in self.tokenize(text, **kwargs)] if isinstance(text, six.string_types) else text + second_sentence_tokens = [self._convert_token_to_id(token) for token in self.tokenize(text_pair, **kwargs)] if isinstance(text_pair, six.string_types) else text_pair if add_special_tokens: return self.add_special_tokens_sequence_pair(first_sentence_tokens, second_sentence_tokens) @@ -754,7 +754,7 @@ class PreTrainedTokenizer(object): information = {} if text_pair is None: - sequence_tokens = self.convert_tokens_to_ids(self.tokenize(text, **kwargs)) if isinstance(text, str) else text + sequence_tokens = self.convert_tokens_to_ids(self.tokenize(text, **kwargs)) if isinstance(text, six.string_types) else text if add_special_tokens: information = self.prepare_for_model(sequence_tokens, max_length, stride) else: @@ -766,8 +766,8 @@ class PreTrainedTokenizer(object): if output_mask: information["mask"] = [0] * len(information["sequence"]) else: - first_sentence_tokens = [self._convert_token_to_id(token) for token in self.tokenize(text, **kwargs)] if isinstance(text, str) else text - second_sentence_tokens = [self._convert_token_to_id(token) for token in self.tokenize(text_pair, **kwargs)] if isinstance(text_pair, str) else text_pair + first_sentence_tokens = [self._convert_token_to_id(token) for token in self.tokenize(text, **kwargs)] if isinstance(text, six.string_types) else text + second_sentence_tokens = [self._convert_token_to_id(token) for token in self.tokenize(text_pair, **kwargs)] if isinstance(text_pair, six.string_types) else text_pair if add_special_tokens: information = self.prepare_pair_for_model( From 2e6797cc7d467bc2242c54fe61ae61891d19677f Mon Sep 17 00:00:00 2001 From: danai-antoniou Date: Thu, 19 Sep 2019 15:40:42 +0100 Subject: [PATCH 084/219] Added valuerror for duplicate added tokens --- pytorch_transformers/tokenization_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pytorch_transformers/tokenization_utils.py b/pytorch_transformers/tokenization_utils.py index 1e2cd59648..bdc0ec7d3c 100644 --- a/pytorch_transformers/tokenization_utils.py +++ b/pytorch_transformers/tokenization_utils.py @@ -503,6 +503,9 @@ class PreTrainedTokenizer(object): if not new_tokens: return 0 + if len(new_tokens) != len(set(new_tokens)): + raise ValueError("The provided list of tokens contains duplicates.") + to_add_tokens = [] for token in new_tokens: assert isinstance(token, str) or (six.PY2 and isinstance(token, unicode)) From 68a3e0223a79e1cc2de02b01527b095e3a627e64 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Fri, 20 Sep 2019 23:14:51 +0200 Subject: [PATCH 085/219] roberta and distilbert --- pytorch_transformers/modeling_tf_bert.py | 1 - .../modeling_tf_distilbert.py | 714 ++++++++++++++++++ pytorch_transformers/modeling_tf_roberta.py | 357 +++++++++ .../tests/modeling_tf_roberta_test.py | 246 ++++++ 4 files changed, 1317 insertions(+), 1 deletion(-) create mode 100644 pytorch_transformers/modeling_tf_distilbert.py create mode 100644 pytorch_transformers/modeling_tf_roberta.py create mode 100644 pytorch_transformers/tests/modeling_tf_roberta_test.py diff --git a/pytorch_transformers/modeling_tf_bert.py b/pytorch_transformers/modeling_tf_bert.py index 9aa9e21c99..edf4c30db9 100644 --- a/pytorch_transformers/modeling_tf_bert.py +++ b/pytorch_transformers/modeling_tf_bert.py @@ -145,7 +145,6 @@ class TFBertEmbeddings(tf.keras.layers.Layer): def _embedding(self, inputs, training=False): """Applies embedding based on inputs tensor.""" - # Create binary mask of size [batch_size, length] input_ids, position_ids, token_type_ids = inputs seq_length = tf.shape(input_ids)[1] diff --git a/pytorch_transformers/modeling_tf_distilbert.py b/pytorch_transformers/modeling_tf_distilbert.py new file mode 100644 index 0000000000..51f1a50aa2 --- /dev/null +++ b/pytorch_transformers/modeling_tf_distilbert.py @@ -0,0 +1,714 @@ +# coding=utf-8 +# Copyright 2019-present, the HuggingFace Inc. team, The Google AI Language Team and Facebook, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" TF 2.0 DistilBERT model +""" +from __future__ import absolute_import, division, print_function, unicode_literals + +import json +import logging +import math +import copy +import sys +from io import open + +import itertools + +import numpy as np +import tensorflow as tf + +from .configuration_distilbert import DistilBertConfig +from .modeling_tf_utils import TFPreTrainedModel, TFSharedEmbeddings, shape_list +from .file_utils import add_start_docstrings +from .modeling_tf_pytorch_utils import load_pytorch_checkpoint_in_tf2_model + +logger = logging.getLogger(__name__) + + +TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP = { + 'distilbert-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-tf_model.h5", + 'distilbert-base-uncased-distilled-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-tf_model.h5" +} + + +### UTILS AND BUILDING BLOCKS OF THE ARCHITECTURE ### +def gelu(x): + """ Gaussian Error Linear Unit. + Original Implementation of the gelu activation function in Google Bert repo when initialy created. + For information: OpenAI GPT's gelu is slightly different (and gives slightly different results): + 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) + Also see https://arxiv.org/abs/1606.08415 + """ + cdf = 0.5 * (1.0 + tf.math.erf(x / tf.math.sqrt(2.0))) + return x * cdf + +def gelu_new(x): + """Gaussian Error Linear Unit. + This is a smoother version of the RELU. + Original paper: https://arxiv.org/abs/1606.08415 + Args: + x: float Tensor to perform activation. + Returns: + `x` with the GELU activation applied. + """ + cdf = 0.5 * (1.0 + tf.tanh( + (np.sqrt(2 / np.pi) * (x + 0.044715 * tf.pow(x, 3))))) + return x * cdf + +def load_distilbert_pt_weights_in_tf2(tf_model, pytorch_checkpoint_path): + # build the network + inputs_list = tf.constant([[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]]) + attns_list = tf.constant([[1, 1, 0, 0, 1], [1, 1, 1, 0, 0], [1, 0, 0, 1, 1]]) + tf_inputs = [inputs_list, attns_list] + tfo = tf_model(tf_inputs, training=False) + return load_pytorch_checkpoint_in_tf2_model(tf_model, pytorch_checkpoint_path, tf_inputs=tf_inputs) + +class TFEmbeddings(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFEmbeddings, self).__init__(**kwargs) + self.vocab_size = config.vocab_size + self.dim = config.dim + self.word_embeddings = TFSharedEmbeddings(, name='word_embeddings') # padding_idx=0) + self.position_embeddings = tf.keras.layers.Embedding(config.max_position_embeddings, config.dim, name='position_embeddings') + if config.sinusoidal_embeddings: + raise NotImplementedError + + self.LayerNorm = tf.keras.layers.LayerNormalization(epsilon=1e-12, name="LayerNorm") + self.dropout = tf.keras.layers.Dropout(config.dropout) + + def build(self, input_shape): + """Build shared word embedding layer """ + with tf.name_scope("word_embeddings"): + # Create and initialize weights. The random normal initializer was chosen + # arbitrarily, and works well. + self.word_embeddings = self.add_weight( + "weight", + shape=[self.vocab_size, self.hidden_size], + initializer=tf.random_normal_initializer( + mean=0., stddev=self.hidden_size**-0.5)) + super(TFEmbeddings, self).build(input_shape) + + def call(self, inputs, mode="embedding", training=False): + """Get token embeddings of inputs. + Args: + inputs: list of three int64 tensors with shape [batch_size, length]: (input_ids, position_ids, token_type_ids) + mode: string, a valid value is one of "embedding" and "linear". + Returns: + outputs: (1) If mode == "embedding", output embedding tensor, float32 with + shape [batch_size, length, embedding_size]; (2) mode == "linear", output + linear tensor, float32 with shape [batch_size, length, vocab_size]. + Raises: + ValueError: if mode is not valid. + + Shared weights logic adapted from + https://github.com/tensorflow/models/blob/a009f4fb9d2fc4949e32192a944688925ef78659/official/transformer/v2/embedding_layer.py#L24 + """ + if mode == "embedding": + return self._embedding(inputs, training=training) + elif mode == "linear": + return self._linear(inputs) + else: + raise ValueError("mode {} is not valid.".format(mode)) + + def _embedding(self, inputs, training=False): + """ + Parameters + ---------- + input_ids: torch.tensor(bs, max_seq_length) + The token ids to embed. + + Outputs + ------- + embeddings: torch.tensor(bs, max_seq_length, dim) + The embedded tokens (plus position embeddings, no token_type embeddings) + """ + input_ids, position_ids = inputs + + seq_length = tf.shape(input_ids)[1] + if position_ids is None: + position_ids = tf.range(seq_length, dtype=tf.int32)[tf.newaxis, :] + + words_embeddings = tf.gather(self.word_embeddings, input_ids) + position_embeddings = self.position_embeddings(position_ids) # (bs, max_seq_length, dim) + + embeddings = word_embeddings + position_embeddings # (bs, max_seq_length, dim) + embeddings = self.LayerNorm(embeddings) # (bs, max_seq_length, dim) + embeddings = self.dropout(embeddings, training=training) # (bs, max_seq_length, dim) + return embeddings + + def _linear(self, inputs): + """Computes logits by running inputs through a linear layer. + Args: + inputs: A float32 tensor with shape [batch_size, length, hidden_size] + Returns: + float32 tensor with shape [batch_size, length, vocab_size]. + """ + batch_size = tf.shape(inputs)[0] + length = tf.shape(inputs)[1] + + x = tf.reshape(inputs, [-1, self.hidden_size]) + logits = tf.matmul(x, self.word_embeddings, transpose_b=True) + + return tf.reshape(logits, [batch_size, length, self.vocab_size]) + + +class TFMultiHeadSelfAttention(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFMultiHeadSelfAttention, self).__init__(**kwargs) + + self.n_heads = config.n_heads + self.dim = config.dim + self.dropout = nn.Dropout(p=config.attention_dropout) + self.output_attentions = config.output_attentions + + assert self.dim % self.n_heads == 0 + + self.q_lin = tf.keras.layers.Dense(config.dim, name="q_lin") + self.k_lin = tf.keras.layers.Dense(config.dim, name="k_lin") + self.v_lin = tf.keras.layers.Dense(config.dim, name="v_lin") + self.out_lin = tf.keras.layers.Dense(config.dim, name="out_lin") + + self.pruned_heads = set() + + def prune_heads(self, heads): + raise NotImplementedError + + def call(self, inputs, training=False): + """ + Parameters + ---------- + query: torch.tensor(bs, seq_length, dim) + key: torch.tensor(bs, seq_length, dim) + value: torch.tensor(bs, seq_length, dim) + mask: torch.tensor(bs, seq_length) + + Outputs + ------- + weights: torch.tensor(bs, n_heads, seq_length, seq_length) + Attention weights + context: torch.tensor(bs, seq_length, dim) + Contextualized layer. Optional: only if `output_attentions=True` + """ + query, key, value, mask, head_mask = inputs + bs, q_length, dim = shape_list(query) + k_length = shape_list(key)[1] + # assert dim == self.dim, 'Dimensions do not match: %s input vs %s configured' % (dim, self.dim) + # assert key.size() == value.size() + + dim_per_head = self.dim // self.n_heads + + assert 2 <= len(tf.shape(mask)) <= 3 + causal = (len(tf.shape(mask)) == 3) + mask_reshp = [bs, 1, 1, k_length] + + def shape(x): + """ separate heads """ + return tf.transpose(tf.reshape(x, (bs, -1, self.n_heads, dim_per_head)), perm=(0, 2, 1, 3)) + + def unshape(x): + """ group heads """ + return tf.reshape(tf.transpose(x, perm=(0, 2, 1, 3)), (bs, -1, self.n_heads * dim_per_head)) + + q = shape(self.q_lin(query)) # (bs, n_heads, q_length, dim_per_head) + k = shape(self.k_lin(key)) # (bs, n_heads, k_length, dim_per_head) + v = shape(self.v_lin(value)) # (bs, n_heads, k_length, dim_per_head) + + q = q / math.sqrt(dim_per_head) # (bs, n_heads, q_length, dim_per_head) + scores = tf.matmul(q, k, transpose_b=True) # (bs, n_heads, q_length, k_length) + mask = tf.reshape(mask, mask_reshape) # (bs, n_heads, qlen, klen) + # scores.masked_fill_(mask, -float('inf')) # (bs, n_heads, q_length, k_length) + scores = scores - 1e30 * (1.0 - mask) + + weights = tf.nn.softmax(scores, axis=-1) # (bs, n_heads, qlen, klen) + weights = self.dropout(weights, training=training) # (bs, n_heads, qlen, klen) + + # Mask heads if we want to + if head_mask is not None: + weights = weights * head_mask + + context = tf.matmul(weights, v) # (bs, n_heads, qlen, dim_per_head) + context = unshape(context) # (bs, q_length, dim) + context = self.out_lin(context) # (bs, q_length, dim) + + if self.output_attentions: + return (context, weights) + else: + return (context,) + +class TFFFN(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFFFN, self).__init__(**kwargs) + self.dropout = tf.keras.layers.Dropout(config.dropout) + self.lin1 = tf.keras.layers.Dense(config.hidden_dim, name="lin1") + self.lin2 = tf.keras.layers.Dense(config.dim, name="lin2") + assert config.activation in ['relu', 'gelu'], "activation ({}) must be in ['relu', 'gelu']".format(config.activation) + self.activation = tf.keras.layers.Activation(gelu) if config.activation=='gelu' else tf.keras.activations.relu + + def call(self, input, training=False): + x = self.lin1(input) + x = self.activation(x) + x = self.lin2(x) + x = self.dropout(x, training=training) + return x + + +class TFTransformerBlock(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFTransformerBlock, self).__init__(**kwargs) + + self.n_heads = config.n_heads + self.dim = config.dim + self.hidden_dim = config.hidden_dim + self.dropout = tf.keras.layers.Dropout(config.dropout) + self.activation = config.activation + self.output_attentions = config.output_attentions + + assert config.dim % config.n_heads == 0 + + self.attention = TFMultiHeadSelfAttention(config, name="attention") + self.sa_layer_norm = tf.keras.layers.LayerNormalization(epsilon=1e-12, name="sa_layer_norm") + + self.ffn = TFFFN(config, name="ffn") + self.output_layer_norm = tf.keras.layers.LayerNormalization(epsilon=1e-12, name="output_layer_norm") + + def call(self, inputs, training=False): # removed: src_enc=None, src_len=None + """ + Parameters + ---------- + x: torch.tensor(bs, seq_length, dim) + attn_mask: torch.tensor(bs, seq_length) + + Outputs + ------- + sa_weights: torch.tensor(bs, n_heads, seq_length, seq_length) + The attention weights + ffn_output: torch.tensor(bs, seq_length, dim) + The output of the transformer block contextualization. + """ + x, attn_mask, head_mask = inputs + + # Self-Attention + sa_output = self.attention([x, x, x, attn_mask, head_mask], training=training) + if self.output_attentions: + sa_output, sa_weights = sa_output # (bs, seq_length, dim), (bs, n_heads, seq_length, seq_length) + else: # To handle these `output_attention` or `output_hidden_states` cases returning tuples + # assert type(sa_output) == tuple + sa_output = sa_output[0] + sa_output = self.sa_layer_norm(sa_output + x) # (bs, seq_length, dim) + + # Feed Forward Network + ffn_output = self.ffn(sa_output, training=training) # (bs, seq_length, dim) + ffn_output = self.output_layer_norm(ffn_output + sa_output) # (bs, seq_length, dim) + + output = (ffn_output,) + if self.output_attentions: + output = (sa_weights,) + output + return output + + +class TFTransformer(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFTransformer, self).__init__(**kwargs) + self.n_layers = config.n_layers + self.output_attentions = config.output_attentions + self.output_hidden_states = config.output_hidden_states + + self.layer = [TFTransformerBlock(config, name='layer_._{}'.format(i)) + for i in range(config.n_layers)] + + def forward(self, inputs, training=False): + """ + Parameters + ---------- + x: torch.tensor(bs, seq_length, dim) + Input sequence embedded. + attn_mask: torch.tensor(bs, seq_length) + Attention mask on the sequence. + + Outputs + ------- + hidden_state: torch.tensor(bs, seq_length, dim) + Sequence of hiddens states in the last (top) layer + all_hidden_states: Tuple[torch.tensor(bs, seq_length, dim)] + Tuple of length n_layers with the hidden states from each layer. + Optional: only if output_hidden_states=True + all_attentions: Tuple[torch.tensor(bs, n_heads, seq_length, seq_length)] + Tuple of length n_layers with the attention weights from each layer + Optional: only if output_attentions=True + """ + x, attn_mask, head_mask = inputs + + all_hidden_states = () + all_attentions = () + + hidden_state = x + for i, layer_module in enumerate(self.layer): + if self.output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_state,) + + layer_outputs = layer_module([hidden_state, attn_mask, head_mask[i]], training=training) + hidden_state = layer_outputs[-1] + + if self.output_attentions: + assert len(layer_outputs) == 2 + attentions = layer_outputs[0] + all_attentions = all_attentions + (attentions,) + else: + assert len(layer_outputs) == 1 + + # Add last layer + if self.output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_state,) + + outputs = (hidden_state,) + if self.output_hidden_states: + outputs = outputs + (all_hidden_states,) + if self.output_attentions: + outputs = outputs + (all_attentions,) + return outputs # last-layer hidden state, (all hidden states), (all attentions) + + +class TFDistilBertMainLayer(tf.keras.layers.Layer): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)`` + Sequence of hidden-states at the output of the last layer of the model. + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased') + model = DistilBertModel.from_pretrained('distilbert-base-uncased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids) + last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple + + """ + def __init__(self, config, **kwargs): + super(TFDistilBertMainLayer, self).__init__(**kwargs) + + self.embeddings = TFEmbeddings(config, name="embeddings") # Embeddings + self.transformer = TFTransformer(config, name="transformer") # Encoder + + def _resize_token_embeddings(self, new_num_tokens): + raise NotImplementedError + + def _prune_heads(self, heads_to_prune): + raise NotImplementedError + + def call(self, inputs, training=False): + if not isinstance(inputs, (dict, tuple, list)): + input_ids = inputs + (attention_mask, head_mask) = None, None + elif isinstance(inputs, (tuple, list)): + input_ids = inputs[0] + attention_mask = inputs[1] if len(inputs) > 1 else None + head_mask = inputs[2] if len(inputs) > 2 else None + assert len(inputs) <= 3, "Too many inputs." + else: + input_ids = inputs.get('input_ids') + attention_mask = inputs.get('attention_mask', None) + head_mask = inputs.get('head_mask', None) + assert len(inputs) <= 3, "Too many inputs." + + if attention_mask is None: + attention_mask = tf.ones(shape_list(input_ids)) # (bs, seq_length) + + # Prepare head mask if needed + # 1.0 in head_mask indicate we keep the head + # attention_probs has shape bsz x n_heads x N x N + # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] + # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length] + if head_mask is not None: + raise NotImplementedError + else: + head_mask = [None] * self.config.num_hidden_layers + + embedding_output = self.embeddings(input_ids) # (bs, seq_length, dim) + tfmr_output = self.transformer([embedding_output, attention_mask, head_mask], training=training) + + hidden_state = tfmr_output[0] + output = (hidden_state, ) + tfmr_output[1:] + + return output # last-layer hidden-state, (all hidden_states), (all attentions) + + +### INTERFACE FOR ENCODER AND TASK SPECIFIC MODEL ### +class TFDistilBertPreTrainedModel(TFPreTrainedModel): + """ An abstract class to handle weights initialization and + a simple interface for downloading and loading pretrained models. + """ + config_class = DistilBertConfig + pretrained_model_archive_map = TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP + load_pt_weights = load_distilbert_pt_weights_in_tf2 + base_model_prefix = "distilbert" + + +DISTILBERT_START_DOCSTRING = r""" + DistilBERT is a small, fast, cheap and light Transformer model + trained by distilling Bert base. It has 40% less parameters than + `bert-base-uncased`, runs 60% faster while preserving over 95% of + Bert's performances as measured on the GLUE language understanding benchmark. + + Here are the differences between the interface of Bert and DistilBert: + + - DistilBert doesn't have `token_type_ids`, you don't need to indicate which token belongs to which segment. Just separate your segments with the separation token `tokenizer.sep_token` (or `[SEP]`) + - DistilBert doesn't have options to select the input positions (`position_ids` input). This could be added if necessary though, just let's us know if you need this option. + + For more information on DistilBERT, please refer to our + `detailed blog post`_ + + .. _`detailed blog post`: + https://medium.com/huggingface/distilbert-8cf3380435b5 + + Parameters: + config (:class:`~pytorch_transformers.DistilBertConfig`): Model configuration class with all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the configuration. + Check out the :meth:`~pytorch_transformers.PreTrainedModel.from_pretrained` method to load the model weights. +""" + +DISTILBERT_INPUTS_DOCSTRING = r""" + Inputs: + **input_ids** ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Indices of input sequence tokens in the vocabulary. + The input sequences should start with `[CLS]` and end with `[SEP]` tokens. + + For now, ONLY BertTokenizer(`bert-base-uncased`) is supported and you should use this tokenizer when using DistilBERT. + **attention_mask**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Mask to avoid performing attention on padding token indices. + Mask values selected in ``[0, 1]``: + ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. + **head_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: + Mask to nullify selected heads of the self-attention modules. + Mask values selected in ``[0, 1]``: + ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. +""" + +@add_start_docstrings("The bare DistilBERT encoder/transformer outputing raw hidden-states without any specific head on top.", + DISTILBERT_START_DOCSTRING, DISTILBERT_INPUTS_DOCSTRING) +class DistilBertModel(DistilBertPreTrainedModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)`` + Sequence of hidden-states at the output of the last layer of the model. + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased') + model = DistilBertModel.from_pretrained('distilbert-base-uncased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids) + last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple + + """ + def __init__(self, config, *inputs, **kwargs): + super(DistilBertModel, self).__init__(config, *inputs, **kwargs) + self.distilbert = TFDistilBertMainLayer(config, name="distilbert") # Embeddings + + def call(self, inputs, training=False): + outputs = self.distilbert(inputs, training=training) + return outputs + + +@add_start_docstrings("""DistilBert Model with a `masked language modeling` head on top. """, + DISTILBERT_START_DOCSTRING, DISTILBERT_INPUTS_DOCSTRING) +class TFDistilBertForMaskedLM(TFDistilBertPreTrainedModel): + r""" + **masked_lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Labels for computing the masked language modeling loss. + Indices should be in ``[-1, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring) + Tokens with indices set to ``-1`` are ignored (masked), the loss is only computed for the tokens with labels + in ``[0, ..., config.vocab_size]`` + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``masked_lm_labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Masked language modeling loss. + **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased') + model = DistilBertForMaskedLM.from_pretrained('distilbert-base-uncased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, masked_lm_labels=input_ids) + loss, prediction_scores = outputs[:2] + + """ + def __init__(self, config, *inputs, **kwargs): + super(TFDistilBertForMaskedLM, self).__init__(config, *inputs, **kwargs) + self.output_attentions = config.output_attentions + self.output_hidden_states = config.output_hidden_states + + self.distilbert = TFDistilBertMainLayer(config, name="distilbert") + self.vocab_transform = tf.keras.layers.Dense(config.dim, name="vocab_transform") + self.act = tf.keras.layers.Activation(gelu) + self.vocab_layer_norm = tf.keras.layers.LayerNormalization(epsilon=1e-12, name="vocab_layer_norm") + self.vocab_projector_weight = self.distilbert.embeddings + + def build(self, input_shape): + self.vocab_projector_bias = self.add_weight(shape=(self.vocab_size,), + initializer='zeros', + trainable=True, + name='vocab_projector_._bias') + super(TFDistilBertForMaskedLM, self).build(input_shape) + + def call(self, inputs, training=False): + dlbrt_output = self.distilbert(inputs, training=training) + hidden_states = dlbrt_output[0] # (bs, seq_length, dim) + prediction_logits = self.vocab_transform(hidden_states) # (bs, seq_length, dim) + prediction_logits = self.act(prediction_logits) # (bs, seq_length, dim) + prediction_logits = self.vocab_layer_norm(prediction_logits) # (bs, seq_length, dim) + prediction_logits = self.vocab_projector_weight(prediction_logits, mode='linear') + self.vocab_projector_bias + + outputs = (prediction_logits, ) + dlbrt_output[1:] + + return outputs # prediction_logits, (all hidden_states), (all attentions) + + +@add_start_docstrings("""DistilBert Model transformer with a sequence classification/regression head on top (a linear layer on top of + the pooled output) e.g. for GLUE tasks. """, + DISTILBERT_START_DOCSTRING, DISTILBERT_INPUTS_DOCSTRING) +class TFDistilBertForSequenceClassification(TFDistilBertPreTrainedModel): + r""" + **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for computing the sequence classification/regression loss. + Indices should be in ``[0, ..., config.num_labels - 1]``. + If ``config.num_labels == 1`` a regression loss is computed (Mean-Square loss), + If ``config.num_labels > 1`` a classification loss is computed (Cross-Entropy). + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Classification (or regression if config.num_labels==1) loss. + **logits**: ``torch.FloatTensor`` of shape ``(batch_size, config.num_labels)`` + Classification (or regression if config.num_labels==1) scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased') + model = DistilBertForSequenceClassification.from_pretrained('distilbert-base-uncased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + labels = torch.tensor([1]).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, labels=labels) + loss, logits = outputs[:2] + + """ + def __init__(self, config, *inputs, **kwargs): + super(TFDistilBertForSequenceClassification, self).__init__(config, *inputs, **kwargs) + self.num_labels = config.num_labels + + self.distilbert = TFDistilBertModel(config, name="distilbert") + self.pre_classifier = tf.keras.layers.Dense(config.dim, activation='relu', name="pre_classifier") + self.classifier = tf.keras.layers.Dense(config.num_labels, name="classifier") + self.dropout = tf.keras.layers.Dropout(config.seq_classif_dropout) + + def call(self, inputs, training=False): + distilbert_output = self.distilbert(inputs, training=training) + hidden_state = distilbert_output[0] # (bs, seq_len, dim) + pooled_output = hidden_state[:, 0] # (bs, dim) + pooled_output = self.pre_classifier(pooled_output) # (bs, dim) + pooled_output = self.dropout(pooled_output, training=training) # (bs, dim) + logits = self.classifier(pooled_output) # (bs, dim) + + outputs = (logits,) + distilbert_output[1:] + return outputs # logits, (hidden_states), (attentions) + + +@add_start_docstrings("""DistilBert Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layers on top of + the hidden-states output to compute `span start logits` and `span end logits`). """, + DISTILBERT_START_DOCSTRING, DISTILBERT_INPUTS_DOCSTRING) +class TFDistilBertForQuestionAnswering(TFDistilBertPreTrainedModel): + r""" + **start_positions**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for position (index) of the start of the labelled span for computing the token classification loss. + Positions are clamped to the length of the sequence (`sequence_length`). + Position outside of the sequence are not taken into account for computing the loss. + **end_positions**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for position (index) of the end of the labelled span for computing the token classification loss. + Positions are clamped to the length of the sequence (`sequence_length`). + Position outside of the sequence are not taken into account for computing the loss. + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Total span extraction loss is the sum of a Cross-Entropy for the start and end positions. + **start_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length,)`` + Span-start scores (before SoftMax). + **end_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length,)`` + Span-end scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased') + model = DistilBertForQuestionAnswering.from_pretrained('distilbert-base-uncased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + start_positions = torch.tensor([1]) + end_positions = torch.tensor([3]) + outputs = model(input_ids, start_positions=start_positions, end_positions=end_positions) + loss, start_scores, end_scores = outputs[:2] + + """ + def __init__(self, config, *inputs, **kwargs): + super(TFDistilBertForQuestionAnswering, self).__init__(config, *inputs, **kwargs) + + self.distilbert = TFDistilBertModel(config, name="distilbert") + self.qa_outputs = tf.keras.layers.Dense(config.num_labels, name='qa_output') + assert config.num_labels == 2 + self.dropout = tf.keras.layers.Dropout(config.qa_dropout) + + def call(self, inputs, training=False): + distilbert_output = self.distilbert(inputs, training=training) + hidden_states = distilbert_output[0] # (bs, max_query_len, dim) + + hidden_states = self.dropout(hidden_states, training=training) # (bs, max_query_len, dim) + logits = self.qa_outputs(hidden_states) # (bs, max_query_len, 2) + start_logits, end_logits = tf.split(logits, 2, axis=-1) + start_logits = tf.squeeze(start_logits, axis=-1) + end_logits = tf.squeeze(end_logits, axis=-1) + + outputs = (start_logits, end_logits,) + distilbert_output[1:] + return outputs # start_logits, end_logits, (hidden_states), (attentions) diff --git a/pytorch_transformers/modeling_tf_roberta.py b/pytorch_transformers/modeling_tf_roberta.py new file mode 100644 index 0000000000..285dfb5b98 --- /dev/null +++ b/pytorch_transformers/modeling_tf_roberta.py @@ -0,0 +1,357 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" TF 2.0 RoBERTa model. """ + +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +import logging + +import numpy as np +import tensorflow as tf + +from .configuration_roberta import RobertaConfig +from .modeling_tf_utils import TFPreTrainedModel +from .file_utils import add_start_docstrings +from .modeling_tf_pytorch_utils import load_pytorch_checkpoint_in_tf2_model + +from .modeling_tf_bert import TFBertEmbeddings, TFBertMainLayer, gelu, gelu_new + +logger = logging.getLogger(__name__) + +TF_ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP = { + 'roberta-base': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-tf_model.h5", + 'roberta-large': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-tf_model.h5", + 'roberta-large-mnli': "https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-mnli-tf_model.h5", +} + +def load_roberta_pt_weights_in_tf2(tf_model, pytorch_checkpoint_path): + # build the network + inputs_list = [[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]] + tf_inputs = tf.constant(inputs_list) + tfo = tf_model(tf_inputs, training=False) + return load_pytorch_checkpoint_in_tf2_model(tf_model, pytorch_checkpoint_path, tf_inputs=tf_inputs) + + +class TFRobertaEmbeddings(TFBertEmbeddings): + """ + Same as BertEmbeddings with a tiny tweak for positional embeddings indexing. + """ + def __init__(self, config, **kwargs): + super(TFRobertaEmbeddings, self).__init__(config, **kwargs) + self.padding_idx = 1 + + def _embedding(self, inputs, training=False): + """Applies embedding based on inputs tensor.""" + input_ids, position_ids, token_type_ids = inputs + + seq_length = tf.shape(input_ids)[1] + if position_ids is None: + position_ids = tf.range(self.padding_idx+1, seq_length+self.padding_idx+1, dtype=tf.int32)[tf.newaxis, :] + + return super(TFRobertaEmbeddings, self)._embedding([input_ids, position_ids, token_type_ids], training=training) + + +class TFRobertaMainLayer(TFBertMainLayer): + """ + Same as TFBertMainLayer but uses TFRobertaEmbeddings. + """ + def __init__(self, config, **kwargs): + super(TFRobertaMainLayer, self).__init__(config, **kwargs) + self.embeddings = TFRobertaEmbeddings(config, name='embeddings') + + def call(self, inputs, training=False): + # Check that input_ids starts with control token + if not isinstance(inputs, (dict, tuple, list)): + input_ids = inputs + elif isinstance(inputs, (tuple, list)): + input_ids = inputs[0] + else: + input_ids = inputs.get('input_ids') + + if tf.not_equal(tf.reduce_sum(input_ids[:, 0]), 0): + logger.warning("A sequence with no special tokens has been passed to the RoBERTa model. " + "This model requires special tokens in order to work. " + "Please specify add_special_tokens=True in your encoding.") + + return super(TFRobertaMainLayer, self).call(inputs, training=training) + + +class TFRobertaPreTrainedModel(TFPreTrainedModel): + """ An abstract class to handle weights initialization and + a simple interface for dowloading and loading pretrained models. + """ + config_class = RobertaConfig + pretrained_model_archive_map = TF_ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP + load_pt_weights = load_roberta_pt_weights_in_tf2 + base_model_prefix = "roberta" + + +ROBERTA_START_DOCSTRING = r""" The RoBERTa model was proposed in + `RoBERTa: A Robustly Optimized BERT Pretraining Approach`_ + by Yinhan Liu, Myle Ott, Naman Goyal, Jingfei Du, Mandar Joshi, Danqi Chen, Omer Levy, Mike Lewis, Luke Zettlemoyer, + Veselin Stoyanov. It is based on Google's BERT model released in 2018. + + It builds on BERT and modifies key hyperparameters, removing the next-sentence pretraining + objective and training with much larger mini-batches and learning rates. + + This implementation is the same as BertModel with a tiny embeddings tweak as well as a setup for Roberta pretrained + models. + + This model is a PyTorch `torch.nn.Module`_ sub-class. Use it as a regular PyTorch Module and + refer to the PyTorch documentation for all matter related to general usage and behavior. + + .. _`RoBERTa: A Robustly Optimized BERT Pretraining Approach`: + https://arxiv.org/abs/1907.11692 + + .. _`torch.nn.Module`: + https://pytorch.org/docs/stable/nn.html#module + + Parameters: + config (:class:`~pytorch_transformers.RobertaConfig`): Model configuration class with all the parameters of the + model. Initializing with a config file does not load the weights associated with the model, only the configuration. + Check out the :meth:`~pytorch_transformers.PreTrainedModel.from_pretrained` method to load the model weights. +""" + +ROBERTA_INPUTS_DOCSTRING = r""" + Inputs: + **input_ids**: ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Indices of input sequence tokens in the vocabulary. + To match pre-training, RoBERTa input sequence should be formatted with and tokens as follows: + + (a) For sequence pairs: + + ``tokens: Is this Jacksonville ? No it is not . `` + + (b) For single sequences: + + ``tokens: the dog is hairy . `` + + Fully encoded sequences or sequence pairs can be obtained using the RobertaTokenizer.encode function with + the ``add_special_tokens`` parameter set to ``True``. + + RoBERTa is a model with absolute position embeddings so it's usually advised to pad the inputs on + the right rather than the left. + + See :func:`pytorch_transformers.PreTrainedTokenizer.encode` and + :func:`pytorch_transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``: + Mask to avoid performing attention on padding token indices. + Mask values selected in ``[0, 1]``: + ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. + **token_type_ids**: (`optional` need to be trained) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Optional segment token indices to indicate first and second portions of the inputs. + This embedding matrice is not trained (not pretrained during RoBERTa pretraining), you will have to train it + during finetuning. + Indices are selected in ``[0, 1]``: ``0`` corresponds to a `sentence A` token, ``1`` + corresponds to a `sentence B` token + (see `BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ for more details). + **position_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Indices of positions of each input sequence tokens in the position embeddings. + Selected in the range ``[0, config.max_position_embeddings - 1[``. + **head_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: + Mask to nullify selected heads of the self-attention modules. + Mask values selected in ``[0, 1]``: + ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. +""" + +@add_start_docstrings("The bare RoBERTa Model transformer outputing raw hidden-states without any specific head on top.", + ROBERTA_START_DOCSTRING, ROBERTA_INPUTS_DOCSTRING) +class TFRobertaModel(TFRobertaPreTrainedModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)`` + Sequence of hidden-states at the output of the last layer of the model. + **pooler_output**: ``torch.FloatTensor`` of shape ``(batch_size, hidden_size)`` + Last layer hidden-state of the first token of the sequence (classification token) + further processed by a Linear layer and a Tanh activation function. The Linear + layer weights are trained from the next sentence prediction (classification) + objective during Bert pretraining. This output is usually *not* a good summary + of the semantic content of the input, you're often better with averaging or pooling + the sequence of hidden-states for the whole input sequence. + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = RobertaTokenizer.from_pretrained('roberta-base') + model = RobertaModel.from_pretrained('roberta-base') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids) + last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple + + """ + def __init__(self, config, *inputs, **kwargs): + super(TFRobertaModel, self).__init__(config, *inputs, **kwargs) + self.roberta = TFRobertaMainLayer(config, name='roberta') + + def call(self, inputs, training=False): + outputs = self.roberta(inputs, training=training) + return outputs + + +class TFRobertaLMHead(tf.keras.layers.Layer): + """Roberta Head for masked language modeling.""" + def __init__(self, config, input_embeddings, **kwargs): + super(TFRobertaLMHead, self).__init__(**kwargs) + self.vocab_size = config.vocab_size + self.dense = tf.keras.layers.Dense(config.hidden_size, name='dense') + self.layer_norm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name='layer_norm') + self.act = tf.keras.layers.Activation(gelu) + + # The output weights are the same as the input embeddings, but there is + # an output-only bias for each token. + self.decoder = input_embeddings + + def build(self, input_shape): + self.bias = self.add_weight(shape=(self.vocab_size,), + initializer='zeros', + trainable=True, + name='bias') + super(TFRobertaLMHead, self).build(input_shape) + + def call(self, features): + x = self.dense(features) + x = self.act(x) + x = self.layer_norm(x) + + # project back to size of vocabulary with bias + x = self.decoder(x, mode="linear") + self.bias + + return x + + +@add_start_docstrings("""RoBERTa Model with a `language modeling` head on top. """, + ROBERTA_START_DOCSTRING, ROBERTA_INPUTS_DOCSTRING) +class TFRobertaForMaskedLM(TFRobertaPreTrainedModel): + r""" + **masked_lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Labels for computing the masked language modeling loss. + Indices should be in ``[-1, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring) + Tokens with indices set to ``-1`` are ignored (masked), the loss is only computed for the tokens with labels + in ``[0, ..., config.vocab_size]`` + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``masked_lm_labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Masked language modeling loss. + **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = RobertaTokenizer.from_pretrained('roberta-base') + model = RobertaForMaskedLM.from_pretrained('roberta-base') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, masked_lm_labels=input_ids) + loss, prediction_scores = outputs[:2] + + """ + def __init__(self, config, *inputs, **kwargs): + super(TFRobertaForMaskedLM, self).__init__(config, *inputs, **kwargs) + + self.roberta = TFRobertaMainLayer(config, name="roberta") + self.lm_head = TFRobertaLMHead(config, self.roberta.embeddings, name="lm_head") + + def call(self, inputs, training=False): + outputs = self.roberta(inputs, training=training) + + sequence_output = outputs[0] + prediction_scores = self.lm_head(sequence_output) + + outputs = (prediction_scores,) + outputs[2:] # Add hidden states and attention if they are here + + return outputs # prediction_scores, (hidden_states), (attentions) + + +class TFRobertaClassificationHead(tf.keras.layers.Layer): + """Head for sentence-level classification tasks.""" + + def __init__(self, config, **kwargs): + super(TFRobertaClassificationHead, self).__init__(config, **kwargs) + self.dense = tf.keras.layers.Dense(config.hidden_size, activation='tanh', name="dense") + self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) + self.out_proj = tf.keras.layers.Dense(config.num_labels, name="out_proj") + + def call(self, features, training=False): + x = features[:, 0, :] # take token (equiv. to [CLS]) + x = self.dropout(x, training=training) + x = self.dense(x) + x = self.dropout(x, training=training) + x = self.out_proj(x) + return x + + +@add_start_docstrings("""RoBERTa Model transformer with a sequence classification/regression head on top (a linear layer + on top of the pooled output) e.g. for GLUE tasks. """, + ROBERTA_START_DOCSTRING, ROBERTA_INPUTS_DOCSTRING) +class TFRobertaForSequenceClassification(TFRobertaPreTrainedModel): + r""" + **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for computing the sequence classification/regression loss. + Indices should be in ``[0, ..., config.num_labels]``. + If ``config.num_labels == 1`` a regression loss is computed (Mean-Square loss), + If ``config.num_labels > 1`` a classification loss is computed (Cross-Entropy). + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Classification (or regression if config.num_labels==1) loss. + **logits**: ``torch.FloatTensor`` of shape ``(batch_size, config.num_labels)`` + Classification (or regression if config.num_labels==1) scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = RoertaTokenizer.from_pretrained('roberta-base') + model = RobertaForSequenceClassification.from_pretrained('roberta-base') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + labels = torch.tensor([1]).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, labels=labels) + loss, logits = outputs[:2] + + """ + def __init__(self, config, *inputs, **kwargs): + super(TFRobertaForSequenceClassification, self).__init__(config, *inputs, **kwargs) + self.num_labels = config.num_labels + + self.roberta = TFRobertaMainLayer(config, name="roberta") + self.classifier = TFRobertaClassificationHead(config, name="classifier") + + def call(self, inputs, training=False): + outputs = self.roberta(inputs, training=training) + sequence_output = outputs[0] + logits = self.classifier(sequence_output, training=training) + + outputs = (logits,) + outputs[2:] + + return outputs # logits, (hidden_states), (attentions) diff --git a/pytorch_transformers/tests/modeling_tf_roberta_test.py b/pytorch_transformers/tests/modeling_tf_roberta_test.py new file mode 100644 index 0000000000..1fbd5d0f5f --- /dev/null +++ b/pytorch_transformers/tests/modeling_tf_roberta_test.py @@ -0,0 +1,246 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import unittest +import shutil +import pytest + +from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) +from .configuration_common_test import ConfigTester + +from pytorch_transformers import RobertaConfig, is_tf_available + +if is_tf_available(): + import tensorflow as tf + import numpy + from pytorch_transformers.modeling_tf_roberta import (TFRobertaModel, TFRobertaForMaskedLM, + TFRobertaForSequenceClassification, + TF_ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP) +else: + pytestmark = pytest.mark.skip("Require TensorFlow") + + +class TFRobertaModelTest(TFCommonTestCases.TFCommonModelTester): + + all_model_classes = (TFRobertaModel,TFRobertaForMaskedLM, + TFRobertaForSequenceClassification) if is_tf_available() else () + + class TFRobertaModelTester(object): + + def __init__(self, + parent, + batch_size=13, + seq_length=7, + is_training=True, + use_input_mask=True, + use_token_type_ids=True, + use_labels=True, + vocab_size=99, + hidden_size=32, + num_hidden_layers=5, + num_attention_heads=4, + intermediate_size=37, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + type_sequence_label_size=2, + initializer_range=0.02, + num_labels=3, + num_choices=4, + scope=None, + ): + self.parent = parent + self.batch_size = batch_size + self.seq_length = seq_length + self.is_training = is_training + self.use_input_mask = use_input_mask + self.use_token_type_ids = use_token_type_ids + self.use_labels = use_labels + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.intermediate_size = intermediate_size + self.hidden_act = hidden_act + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.type_sequence_label_size = type_sequence_label_size + self.initializer_range = initializer_range + self.num_labels = num_labels + self.num_choices = num_choices + self.scope = scope + + def prepare_config_and_inputs(self): + input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) + + input_mask = None + if self.use_input_mask: + input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2) + + token_type_ids = None + if self.use_token_type_ids: + token_type_ids = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size) + + sequence_labels = None + token_labels = None + choice_labels = None + if self.use_labels: + sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size) + token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels) + choice_labels = ids_tensor([self.batch_size], self.num_choices) + + config = RobertaConfig( + vocab_size_or_config_json_file=self.vocab_size, + hidden_size=self.hidden_size, + num_hidden_layers=self.num_hidden_layers, + num_attention_heads=self.num_attention_heads, + intermediate_size=self.intermediate_size, + hidden_act=self.hidden_act, + hidden_dropout_prob=self.hidden_dropout_prob, + attention_probs_dropout_prob=self.attention_probs_dropout_prob, + max_position_embeddings=self.max_position_embeddings, + type_vocab_size=self.type_vocab_size, + initializer_range=self.initializer_range) + + return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels + + def create_and_check_roberta_model(self, config, input_ids, token_type_ids, input_mask, sequence_labels, + token_labels, choice_labels): + model = TFRobertaModel(config=config) + inputs = {'input_ids': input_ids, + 'attention_mask': input_mask, + 'token_type_ids': token_type_ids} + sequence_output = model(inputs)[0] + + inputs = [input_ids, input_mask] + sequence_output = model(inputs)[0] + + sequence_output = model(input_ids)[0] + + result = { + "sequence_output": sequence_output.numpy(), + } + self.parent.assertListEqual( + list(result["sequence_output"].shape), + [self.batch_size, self.seq_length, self.hidden_size]) + + def create_and_check_roberta_for_masked_lm(self, config, input_ids, token_type_ids, input_mask, sequence_labels, + token_labels, choice_labels): + model = TFRobertaForMaskedLM(config=config) + prediction_scores = model([input_ids, input_mask, token_type_ids])[0] + result = { + "prediction_scores": prediction_scores.numpy(), + } + self.parent.assertListEqual( + list(result["prediction_scores"].shape), + [self.batch_size, self.seq_length, self.vocab_size]) + + def prepare_config_and_inputs_for_common(self): + config_and_inputs = self.prepare_config_and_inputs() + (config, input_ids, token_type_ids, input_mask, + sequence_labels, token_labels, choice_labels) = config_and_inputs + inputs_dict = {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask} + return config, inputs_dict + + def setUp(self): + self.model_tester = TFRobertaModelTest.TFRobertaModelTester(self) + self.config_tester = ConfigTester(self, config_class=RobertaConfig, hidden_size=37) + + def test_config(self): + self.config_tester.run_common_tests() + + def test_roberta_model(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_roberta_model(*config_and_inputs) + + def test_for_masked_lm(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_roberta_for_masked_lm(*config_and_inputs) + + @pytest.mark.slow + def test_model_from_pretrained(self): + cache_dir = "/tmp/pytorch_transformers_test/" + for model_name in list(TF_ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: + model = TFRobertaModel.from_pretrained(model_name, cache_dir=cache_dir) + shutil.rmtree(cache_dir) + self.assertIsNotNone(model) + + + +class TFRobertaModelIntegrationTest(unittest.TestCase): + + @pytest.mark.slow + def test_inference_masked_lm(self): + model = TFRobertaForMaskedLM.from_pretrained('roberta-base') + + input_ids = tf.constant([[ 0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]]) + output = model(input_ids)[0] + expected_shape = [1, 11, 50265] + self.assertEqual( + list(output.numpy().shape), + expected_shape + ) + # compare the actual values for a slice. + expected_slice = tf.constant( + [[[33.8843, -4.3107, 22.7779], + [ 4.6533, -2.8099, 13.6252], + [ 1.8222, -3.6898, 8.8600]]] + ) + self.assertTrue( + numpy.allclose(output[:, :3, :3].numpy(), expected_slice.numpy(), atol=1e-3) + ) + + @pytest.mark.slow + def test_inference_no_head(self): + model = TFRobertaModel.from_pretrained('roberta-base') + + input_ids = tf.constant([[ 0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]]) + output = model(input_ids)[0] + # compare the actual values for a slice. + expected_slice = tf.constant( + [[[-0.0231, 0.0782, 0.0074], + [-0.1854, 0.0539, -0.0174], + [ 0.0548, 0.0799, 0.1687]]] + ) + self.assertTrue( + numpy.allclose(output[:, :3, :3].numpy(), expected_slice.numpy(), atol=1e-3) + ) + + @pytest.mark.slow + def test_inference_classification_head(self): + model = TFRobertaForSequenceClassification.from_pretrained('roberta-large-mnli') + + input_ids = tf.constant([[ 0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]]) + output = model(input_ids)[0] + expected_shape = [1, 3] + self.assertEqual( + list(output.numpy().shape), + expected_shape + ) + expected_tensor = tf.constant([[-0.9469, 0.3913, 0.5118]]) + self.assertTrue( + numpy.allclose(output.numpy(), expected_tensor.numpy(), atol=1e-3) + ) + + +if __name__ == "__main__": + unittest.main() From 98dd19b96b351f481e1268ab6c7b035bb21d106e Mon Sep 17 00:00:00 2001 From: Santiago Castro Date: Sun, 22 Sep 2019 20:31:36 -0400 Subject: [PATCH 086/219] Remove unnecessary use of FusedLayerNorm --- pytorch_transformers/modeling_bert.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pytorch_transformers/modeling_bert.py b/pytorch_transformers/modeling_bert.py index ca12df427c..dc3700d26b 100644 --- a/pytorch_transformers/modeling_bert.py +++ b/pytorch_transformers/modeling_bert.py @@ -133,11 +133,7 @@ def swish(x): ACT2FN = {"gelu": gelu, "relu": torch.nn.functional.relu, "swish": swish} -try: - from apex.normalization.fused_layer_norm import FusedLayerNorm as BertLayerNorm -except (ImportError, AttributeError) as e: - logger.info("Better speed can be achieved with apex installed from https://www.github.com/nvidia/apex .") - BertLayerNorm = torch.nn.LayerNorm +BertLayerNorm = torch.nn.LayerNorm class BertEmbeddings(nn.Module): """Construct the embeddings from word, position and token_type embeddings. From 447de34ddedfcb7caa2a1e03a5fe74e82f2e377f Mon Sep 17 00:00:00 2001 From: thomwolf Date: Mon, 23 Sep 2019 15:38:29 +0200 Subject: [PATCH 087/219] tests for distilbert and roberta --- .../configuration_distilbert.py | 2 +- .../modeling_tf_distilbert.py | 80 ++++--- .../tests/modeling_tf_distilbert_test.py | 222 ++++++++++++++++++ 3 files changed, 273 insertions(+), 31 deletions(-) create mode 100644 pytorch_transformers/tests/modeling_tf_distilbert_test.py diff --git a/pytorch_transformers/configuration_distilbert.py b/pytorch_transformers/configuration_distilbert.py index b8929eedec..2a8a149acf 100644 --- a/pytorch_transformers/configuration_distilbert.py +++ b/pytorch_transformers/configuration_distilbert.py @@ -37,7 +37,7 @@ class DistilBertConfig(PretrainedConfig): def __init__(self, vocab_size_or_config_json_file=30522, max_position_embeddings=512, - sinusoidal_pos_embds=True, + sinusoidal_pos_embds=False, n_layers=6, n_heads=12, dim=768, diff --git a/pytorch_transformers/modeling_tf_distilbert.py b/pytorch_transformers/modeling_tf_distilbert.py index 51f1a50aa2..48bf3f377c 100644 --- a/pytorch_transformers/modeling_tf_distilbert.py +++ b/pytorch_transformers/modeling_tf_distilbert.py @@ -79,9 +79,9 @@ class TFEmbeddings(tf.keras.layers.Layer): super(TFEmbeddings, self).__init__(**kwargs) self.vocab_size = config.vocab_size self.dim = config.dim - self.word_embeddings = TFSharedEmbeddings(, name='word_embeddings') # padding_idx=0) + self.word_embeddings = TFSharedEmbeddings(config.vocab_size, config.dim, name='word_embeddings') # padding_idx=0) self.position_embeddings = tf.keras.layers.Embedding(config.max_position_embeddings, config.dim, name='position_embeddings') - if config.sinusoidal_embeddings: + if config.sinusoidal_pos_embds: raise NotImplementedError self.LayerNorm = tf.keras.layers.LayerNormalization(epsilon=1e-12, name="LayerNorm") @@ -94,9 +94,9 @@ class TFEmbeddings(tf.keras.layers.Layer): # arbitrarily, and works well. self.word_embeddings = self.add_weight( "weight", - shape=[self.vocab_size, self.hidden_size], + shape=[self.vocab_size, self.dim], initializer=tf.random_normal_initializer( - mean=0., stddev=self.hidden_size**-0.5)) + mean=0., stddev=self.dim**-0.5)) super(TFEmbeddings, self).build(input_shape) def call(self, inputs, mode="embedding", training=False): @@ -133,13 +133,17 @@ class TFEmbeddings(tf.keras.layers.Layer): embeddings: torch.tensor(bs, max_seq_length, dim) The embedded tokens (plus position embeddings, no token_type embeddings) """ - input_ids, position_ids = inputs + if not isinstance(inputs, (tuple, list)): + input_ids = inputs + position_ids = None + else: + input_ids, position_ids = inputs seq_length = tf.shape(input_ids)[1] if position_ids is None: position_ids = tf.range(seq_length, dtype=tf.int32)[tf.newaxis, :] - words_embeddings = tf.gather(self.word_embeddings, input_ids) + word_embeddings = tf.gather(self.word_embeddings, input_ids) position_embeddings = self.position_embeddings(position_ids) # (bs, max_seq_length, dim) embeddings = word_embeddings + position_embeddings # (bs, max_seq_length, dim) @@ -157,7 +161,7 @@ class TFEmbeddings(tf.keras.layers.Layer): batch_size = tf.shape(inputs)[0] length = tf.shape(inputs)[1] - x = tf.reshape(inputs, [-1, self.hidden_size]) + x = tf.reshape(inputs, [-1, self.dim]) logits = tf.matmul(x, self.word_embeddings, transpose_b=True) return tf.reshape(logits, [batch_size, length, self.vocab_size]) @@ -169,7 +173,7 @@ class TFMultiHeadSelfAttention(tf.keras.layers.Layer): self.n_heads = config.n_heads self.dim = config.dim - self.dropout = nn.Dropout(p=config.attention_dropout) + self.dropout = tf.keras.layers.Dropout(config.attention_dropout) self.output_attentions = config.output_attentions assert self.dim % self.n_heads == 0 @@ -210,7 +214,7 @@ class TFMultiHeadSelfAttention(tf.keras.layers.Layer): assert 2 <= len(tf.shape(mask)) <= 3 causal = (len(tf.shape(mask)) == 3) - mask_reshp = [bs, 1, 1, k_length] + mask_reshape = [bs, 1, 1, k_length] def shape(x): """ separate heads """ @@ -327,7 +331,7 @@ class TFTransformer(tf.keras.layers.Layer): self.layer = [TFTransformerBlock(config, name='layer_._{}'.format(i)) for i in range(config.n_layers)] - def forward(self, inputs, training=False): + def call(self, inputs, training=False): """ Parameters ---------- @@ -403,6 +407,7 @@ class TFDistilBertMainLayer(tf.keras.layers.Layer): """ def __init__(self, config, **kwargs): super(TFDistilBertMainLayer, self).__init__(**kwargs) + self.num_hidden_layers = config.num_hidden_layers self.embeddings = TFEmbeddings(config, name="embeddings") # Embeddings self.transformer = TFTransformer(config, name="transformer") # Encoder @@ -430,6 +435,7 @@ class TFDistilBertMainLayer(tf.keras.layers.Layer): if attention_mask is None: attention_mask = tf.ones(shape_list(input_ids)) # (bs, seq_length) + attention_mask = tf.cast(attention_mask, dtype=tf.float32) # Prepare head mask if needed # 1.0 in head_mask indicate we keep the head @@ -439,15 +445,12 @@ class TFDistilBertMainLayer(tf.keras.layers.Layer): if head_mask is not None: raise NotImplementedError else: - head_mask = [None] * self.config.num_hidden_layers + head_mask = [None] * self.num_hidden_layers embedding_output = self.embeddings(input_ids) # (bs, seq_length, dim) tfmr_output = self.transformer([embedding_output, attention_mask, head_mask], training=training) - hidden_state = tfmr_output[0] - output = (hidden_state, ) + tfmr_output[1:] - - return output # last-layer hidden-state, (all hidden_states), (all attentions) + return tfmr_output # last-layer hidden-state, (all hidden_states), (all attentions) ### INTERFACE FOR ENCODER AND TASK SPECIFIC MODEL ### @@ -503,7 +506,7 @@ DISTILBERT_INPUTS_DOCSTRING = r""" @add_start_docstrings("The bare DistilBERT encoder/transformer outputing raw hidden-states without any specific head on top.", DISTILBERT_START_DOCSTRING, DISTILBERT_INPUTS_DOCSTRING) -class DistilBertModel(DistilBertPreTrainedModel): +class TFDistilBertModel(TFDistilBertPreTrainedModel): r""" Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)`` @@ -526,7 +529,7 @@ class DistilBertModel(DistilBertPreTrainedModel): """ def __init__(self, config, *inputs, **kwargs): - super(DistilBertModel, self).__init__(config, *inputs, **kwargs) + super(TFDistilBertModel, self).__init__(config, *inputs, **kwargs) self.distilbert = TFDistilBertMainLayer(config, name="distilbert") # Embeddings def call(self, inputs, training=False): @@ -534,6 +537,28 @@ class DistilBertModel(DistilBertPreTrainedModel): return outputs +class TFDistilBertLMHead(tf.keras.layers.Layer): + def __init__(self, config, input_embeddings, **kwargs): + super(TFDistilBertLMHead, self).__init__(**kwargs) + self.vocab_size = config.vocab_size + + # The output weights are the same as the input embeddings, but there is + # an output-only bias for each token. + self.input_embeddings = input_embeddings + + def build(self, input_shape): + self.bias = self.add_weight(shape=(self.vocab_size,), + initializer='zeros', + trainable=True, + name='bias') + super(TFDistilBertLMHead, self).build(input_shape) + + def call(self, hidden_states): + hidden_states = self.input_embeddings(hidden_states, mode="linear") + hidden_states = hidden_states + self.bias + return hidden_states + + @add_start_docstrings("""DistilBert Model with a `masked language modeling` head on top. """, DISTILBERT_START_DOCSTRING, DISTILBERT_INPUTS_DOCSTRING) class TFDistilBertForMaskedLM(TFDistilBertPreTrainedModel): @@ -570,27 +595,22 @@ class TFDistilBertForMaskedLM(TFDistilBertPreTrainedModel): super(TFDistilBertForMaskedLM, self).__init__(config, *inputs, **kwargs) self.output_attentions = config.output_attentions self.output_hidden_states = config.output_hidden_states + self.vocab_size = config.vocab_size self.distilbert = TFDistilBertMainLayer(config, name="distilbert") self.vocab_transform = tf.keras.layers.Dense(config.dim, name="vocab_transform") self.act = tf.keras.layers.Activation(gelu) self.vocab_layer_norm = tf.keras.layers.LayerNormalization(epsilon=1e-12, name="vocab_layer_norm") - self.vocab_projector_weight = self.distilbert.embeddings - - def build(self, input_shape): - self.vocab_projector_bias = self.add_weight(shape=(self.vocab_size,), - initializer='zeros', - trainable=True, - name='vocab_projector_._bias') - super(TFDistilBertForMaskedLM, self).build(input_shape) + self.vocab_projector = TFDistilBertLMHead(config, self.distilbert.embeddings, name="vocab_projector") def call(self, inputs, training=False): dlbrt_output = self.distilbert(inputs, training=training) - hidden_states = dlbrt_output[0] # (bs, seq_length, dim) - prediction_logits = self.vocab_transform(hidden_states) # (bs, seq_length, dim) - prediction_logits = self.act(prediction_logits) # (bs, seq_length, dim) - prediction_logits = self.vocab_layer_norm(prediction_logits) # (bs, seq_length, dim) - prediction_logits = self.vocab_projector_weight(prediction_logits, mode='linear') + self.vocab_projector_bias + + hidden_states = dlbrt_output[0] # (bs, seq_length, dim) + prediction_logits = self.vocab_transform(hidden_states) # (bs, seq_length, dim) + prediction_logits = self.act(prediction_logits) # (bs, seq_length, dim) + prediction_logits = self.vocab_layer_norm(prediction_logits) # (bs, seq_length, dim) + prediction_logits = self.vocab_projector(prediction_logits) outputs = (prediction_logits, ) + dlbrt_output[1:] diff --git a/pytorch_transformers/tests/modeling_tf_distilbert_test.py b/pytorch_transformers/tests/modeling_tf_distilbert_test.py new file mode 100644 index 0000000000..5d7f0f7a31 --- /dev/null +++ b/pytorch_transformers/tests/modeling_tf_distilbert_test.py @@ -0,0 +1,222 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import unittest +import pytest + +from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) +from .configuration_common_test import ConfigTester + +from pytorch_transformers import DistilBertConfig, is_tf_available + +if is_tf_available(): + import tensorflow as tf + from pytorch_transformers.modeling_tf_distilbert import (TFDistilBertModel, + TFDistilBertForMaskedLM, + TFDistilBertForQuestionAnswering, + TFDistilBertForSequenceClassification) +else: + pytestmark = pytest.mark.skip("Require TensorFlow") + + +class TFDistilBertModelTest(TFCommonTestCases.TFCommonModelTester): + + all_model_classes = (TFDistilBertModel, TFDistilBertForMaskedLM, TFDistilBertForQuestionAnswering, + TFDistilBertForSequenceClassification) if is_tf_available() else None + test_pruning = True + test_torchscript = True + test_resize_embeddings = True + test_head_masking = True + + class TFDistilBertModelTester(object): + + def __init__(self, + parent, + batch_size=13, + seq_length=7, + is_training=True, + use_input_mask=True, + use_token_type_ids=False, + use_labels=True, + vocab_size=99, + hidden_size=32, + num_hidden_layers=5, + num_attention_heads=4, + intermediate_size=37, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + type_sequence_label_size=2, + initializer_range=0.02, + num_labels=3, + num_choices=4, + scope=None, + ): + self.parent = parent + self.batch_size = batch_size + self.seq_length = seq_length + self.is_training = is_training + self.use_input_mask = use_input_mask + self.use_token_type_ids = use_token_type_ids + self.use_labels = use_labels + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.intermediate_size = intermediate_size + self.hidden_act = hidden_act + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.type_sequence_label_size = type_sequence_label_size + self.initializer_range = initializer_range + self.num_labels = num_labels + self.num_choices = num_choices + self.scope = scope + + def prepare_config_and_inputs(self): + input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) + + input_mask = None + if self.use_input_mask: + input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2) + + sequence_labels = None + token_labels = None + choice_labels = None + if self.use_labels: + sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size) + token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels) + choice_labels = ids_tensor([self.batch_size], self.num_choices) + + config = DistilBertConfig( + vocab_size_or_config_json_file=self.vocab_size, + dim=self.hidden_size, + n_layers=self.num_hidden_layers, + n_heads=self.num_attention_heads, + hidden_dim=self.intermediate_size, + hidden_act=self.hidden_act, + dropout=self.hidden_dropout_prob, + attention_dropout=self.attention_probs_dropout_prob, + max_position_embeddings=self.max_position_embeddings, + initializer_range=self.initializer_range) + + return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels + + def create_and_check_distilbert_model(self, config, input_ids, input_mask, sequence_labels, token_labels, choice_labels): + model = TFDistilBertModel(config=config) + inputs = {'input_ids': input_ids, + 'attention_mask': input_mask} + + outputs = model(inputs) + sequence_output = outputs[0] + + inputs = [input_ids, input_mask] + + (sequence_output,) = model(inputs) + + result = { + "sequence_output": sequence_output.numpy(), + } + self.parent.assertListEqual( + list(result["sequence_output"].shape), + [self.batch_size, self.seq_length, self.hidden_size]) + + def create_and_check_distilbert_for_masked_lm(self, config, input_ids, input_mask, sequence_labels, token_labels, choice_labels): + model = TFDistilBertForMaskedLM(config=config) + inputs = {'input_ids': input_ids, + 'attention_mask': input_mask} + (prediction_scores,) = model(inputs) + result = { + "prediction_scores": prediction_scores.numpy(), + } + self.parent.assertListEqual( + list(result["prediction_scores"].shape), + [self.batch_size, self.seq_length, self.vocab_size]) + + def create_and_check_distilbert_for_question_answering(self, config, input_ids, input_mask, sequence_labels, token_labels, choice_labels): + model = TFDistilBertForQuestionAnswering(config=config) + inputs = {'input_ids': input_ids, + 'attention_mask': input_mask} + start_logits, end_logits = model(inputs) + result = { + "start_logits": start_logits.numpy(), + "end_logits": end_logits.numpy(), + } + self.parent.assertListEqual( + list(result["start_logits"].shape), + [self.batch_size, self.seq_length]) + self.parent.assertListEqual( + list(result["end_logits"].shape), + [self.batch_size, self.seq_length]) + + def create_and_check_distilbert_for_sequence_classification(self, config, input_ids, input_mask, sequence_labels, token_labels, choice_labels): + config.num_labels = self.num_labels + model = TFDistilBertForSequenceClassification(config) + inputs = {'input_ids': input_ids, + 'attention_mask': input_mask} + (logits,) = model(inputs) + result = { + "logits": logits.numpy(), + } + self.parent.assertListEqual( + list(result["logits"].shape), + [self.batch_size, self.num_labels]) + + def prepare_config_and_inputs_for_common(self): + config_and_inputs = self.prepare_config_and_inputs() + (config, input_ids, input_mask, sequence_labels, token_labels, choice_labels) = config_and_inputs + inputs_dict = {'input_ids': input_ids, 'attention_mask': input_mask} + return config, inputs_dict + + def setUp(self): + self.model_tester = TFDistilBertModelTest.TFDistilBertModelTester(self) + self.config_tester = ConfigTester(self, config_class=DistilBertConfig, dim=37) + + def test_config(self): + self.config_tester.run_common_tests() + + def test_distilbert_model(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_distilbert_model(*config_and_inputs) + + def test_for_masked_lm(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_distilbert_for_masked_lm(*config_and_inputs) + + def test_for_question_answering(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_distilbert_for_question_answering(*config_and_inputs) + + def test_for_sequence_classification(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_distilbert_for_sequence_classification(*config_and_inputs) + + # @pytest.mark.slow + # def test_model_from_pretrained(self): + # cache_dir = "/tmp/pytorch_transformers_test/" + # for model_name in list(DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: + # model = DistilBertModel.from_pretrained(model_name, cache_dir=cache_dir) + # shutil.rmtree(cache_dir) + # self.assertIsNotNone(model) + +if __name__ == "__main__": + unittest.main() From a31e591d27a099ca6bd30949ecd7cc61213b8327 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Mon, 23 Sep 2019 15:54:10 +0200 Subject: [PATCH 088/219] fix XLM tests --- pytorch_transformers/modeling_xlm.py | 2 +- pytorch_transformers/tests/modeling_xlm_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pytorch_transformers/modeling_xlm.py b/pytorch_transformers/modeling_xlm.py index 6516bd2aa1..92febd296d 100644 --- a/pytorch_transformers/modeling_xlm.py +++ b/pytorch_transformers/modeling_xlm.py @@ -566,7 +566,7 @@ class XLMPredLayer(nn.Module): scores = self.proj(x) outputs = (scores,) + outputs if y is not None: - loss = F.cross_entropy(scores.view(-1, self.n_words), y, reduction='elementwise_mean') + loss = F.cross_entropy(scores.view(-1, self.n_words), y.view(-1), reduction='elementwise_mean') outputs = (loss,) + outputs else: scores = self.proj.log_prob(x) diff --git a/pytorch_transformers/tests/modeling_xlm_test.py b/pytorch_transformers/tests/modeling_xlm_test.py index 841bea94b7..4f7f81c002 100644 --- a/pytorch_transformers/tests/modeling_xlm_test.py +++ b/pytorch_transformers/tests/modeling_xlm_test.py @@ -185,7 +185,7 @@ class XLMModelTest(CommonTestCases.CommonModelTester): model.eval() outputs = model(input_ids) - start_top_log_probs, start_top_index, end_top_log_probs, end_top_index, cls_logits, mems = outputs + start_top_log_probs, start_top_index, end_top_log_probs, end_top_index, cls_logits = outputs outputs = model(input_ids, start_positions=sequence_labels, end_positions=sequence_labels, From 830d212be74dad2c41d39b0e94ffa4b2ac8c7b22 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Mon, 23 Sep 2019 16:26:06 +0200 Subject: [PATCH 089/219] test circleCI h5py version --- .../tests/modeling_tf_auto_test.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/pytorch_transformers/tests/modeling_tf_auto_test.py b/pytorch_transformers/tests/modeling_tf_auto_test.py index 4617ab817b..3a63039cc8 100644 --- a/pytorch_transformers/tests/modeling_tf_auto_test.py +++ b/pytorch_transformers/tests/modeling_tf_auto_test.py @@ -39,46 +39,49 @@ else: class TFAutoModelTest(unittest.TestCase): def test_model_from_pretrained(self): + import h5py + self.assertTrue(h5py.version.hdf5_version.startswith("1.10")) + logging.basicConfig(level=logging.INFO) for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: - config = AutoConfig.from_pretrained(model_name) + config = AutoConfig.from_pretrained(model_name, force_download=True) self.assertIsNotNone(config) self.assertIsInstance(config, BertConfig) - model = TFAutoModel.from_pretrained(model_name) + model = TFAutoModel.from_pretrained(model_name, force_download=True) self.assertIsNotNone(model) self.assertIsInstance(model, TFBertModel) def test_lmhead_model_from_pretrained(self): logging.basicConfig(level=logging.INFO) for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: - config = AutoConfig.from_pretrained(model_name) + config = AutoConfig.from_pretrained(model_name, force_download=True) self.assertIsNotNone(config) self.assertIsInstance(config, BertConfig) - model = TFAutoModelWithLMHead.from_pretrained(model_name) + model = TFAutoModelWithLMHead.from_pretrained(model_name, force_download=True) self.assertIsNotNone(model) self.assertIsInstance(model, TFBertForMaskedLM) def test_sequence_classification_model_from_pretrained(self): logging.basicConfig(level=logging.INFO) for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: - config = AutoConfig.from_pretrained(model_name) + config = AutoConfig.from_pretrained(model_name, force_download=True) self.assertIsNotNone(config) self.assertIsInstance(config, BertConfig) - model = TFAutoModelForSequenceClassification.from_pretrained(model_name) + model = TFAutoModelForSequenceClassification.from_pretrained(model_name, force_download=True) self.assertIsNotNone(model) self.assertIsInstance(model, TFBertForSequenceClassification) def test_question_answering_model_from_pretrained(self): logging.basicConfig(level=logging.INFO) for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: - config = AutoConfig.from_pretrained(model_name) + config = AutoConfig.from_pretrained(model_name, force_download=True) self.assertIsNotNone(config) self.assertIsInstance(config, BertConfig) - model = TFAutoModelForQuestionAnswering.from_pretrained(model_name) + model = TFAutoModelForQuestionAnswering.from_pretrained(model_name, force_download=True) self.assertIsNotNone(model) self.assertIsInstance(model, TFBertForQuestionAnswering) From 0b22e47a4016635a750c0eb4bbc41bdfd93e14d7 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Mon, 23 Sep 2019 16:38:03 +0200 Subject: [PATCH 090/219] skipping pretrained TF model tests for now --- pytorch_transformers/tests/modeling_tf_bert_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_transformers/tests/modeling_tf_bert_test.py b/pytorch_transformers/tests/modeling_tf_bert_test.py index a1bf9bf794..d6c10ade22 100644 --- a/pytorch_transformers/tests/modeling_tf_bert_test.py +++ b/pytorch_transformers/tests/modeling_tf_bert_test.py @@ -26,7 +26,7 @@ from .configuration_common_test import ConfigTester from pytorch_transformers import BertConfig, is_tf_available -if is_tf_available(): +if False and is_tf_available(): import tensorflow as tf from pytorch_transformers.modeling_tf_bert import (TFBertModel, TFBertForMaskedLM, TFBertForNextSentencePrediction, From c014d1f0c64a318ef288ab8dca47aa3f29052540 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Mon, 23 Sep 2019 16:39:57 +0200 Subject: [PATCH 091/219] fix the skipping --- pytorch_transformers/tests/modeling_tf_auto_test.py | 3 ++- pytorch_transformers/tests/modeling_tf_bert_test.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pytorch_transformers/tests/modeling_tf_auto_test.py b/pytorch_transformers/tests/modeling_tf_auto_test.py index 3a63039cc8..70b13c54a8 100644 --- a/pytorch_transformers/tests/modeling_tf_auto_test.py +++ b/pytorch_transformers/tests/modeling_tf_auto_test.py @@ -23,7 +23,8 @@ import logging from pytorch_transformers import is_tf_available -if is_tf_available(): +# if is_tf_available(): +if False: from pytorch_transformers import (AutoConfig, BertConfig, TFAutoModel, TFBertModel, TFAutoModelWithLMHead, TFBertForMaskedLM, diff --git a/pytorch_transformers/tests/modeling_tf_bert_test.py b/pytorch_transformers/tests/modeling_tf_bert_test.py index d6c10ade22..a1bf9bf794 100644 --- a/pytorch_transformers/tests/modeling_tf_bert_test.py +++ b/pytorch_transformers/tests/modeling_tf_bert_test.py @@ -26,7 +26,7 @@ from .configuration_common_test import ConfigTester from pytorch_transformers import BertConfig, is_tf_available -if False and is_tf_available(): +if is_tf_available(): import tensorflow as tf from pytorch_transformers.modeling_tf_bert import (TFBertModel, TFBertForMaskedLM, TFBertForNextSentencePrediction, From 798da627ebb24cf729bb55575e69e5d8caf91332 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Mon, 23 Sep 2019 12:06:10 -0400 Subject: [PATCH 092/219] Fix TFBert tests in Python 3.5 --- .../tests/modeling_tf_auto_test.py | 15 +++++++++------ .../tests/modeling_tf_bert_test.py | 3 ++- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/pytorch_transformers/tests/modeling_tf_auto_test.py b/pytorch_transformers/tests/modeling_tf_auto_test.py index 70b13c54a8..7b080bafcd 100644 --- a/pytorch_transformers/tests/modeling_tf_auto_test.py +++ b/pytorch_transformers/tests/modeling_tf_auto_test.py @@ -23,8 +23,7 @@ import logging from pytorch_transformers import is_tf_available -# if is_tf_available(): -if False: +if is_tf_available(): from pytorch_transformers import (AutoConfig, BertConfig, TFAutoModel, TFBertModel, TFAutoModelWithLMHead, TFBertForMaskedLM, @@ -44,7 +43,8 @@ class TFAutoModelTest(unittest.TestCase): self.assertTrue(h5py.version.hdf5_version.startswith("1.10")) logging.basicConfig(level=logging.INFO) - for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: + # for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: + for model_name in ['bert-base-uncased']: config = AutoConfig.from_pretrained(model_name, force_download=True) self.assertIsNotNone(config) self.assertIsInstance(config, BertConfig) @@ -55,7 +55,8 @@ class TFAutoModelTest(unittest.TestCase): def test_lmhead_model_from_pretrained(self): logging.basicConfig(level=logging.INFO) - for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: + # for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: + for model_name in ['bert-base-uncased']: config = AutoConfig.from_pretrained(model_name, force_download=True) self.assertIsNotNone(config) self.assertIsInstance(config, BertConfig) @@ -66,7 +67,8 @@ class TFAutoModelTest(unittest.TestCase): def test_sequence_classification_model_from_pretrained(self): logging.basicConfig(level=logging.INFO) - for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: + # for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: + for model_name in ['bert-base-uncased']: config = AutoConfig.from_pretrained(model_name, force_download=True) self.assertIsNotNone(config) self.assertIsInstance(config, BertConfig) @@ -77,7 +79,8 @@ class TFAutoModelTest(unittest.TestCase): def test_question_answering_model_from_pretrained(self): logging.basicConfig(level=logging.INFO) - for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: + # for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: + for model_name in ['bert-base-uncased']: config = AutoConfig.from_pretrained(model_name, force_download=True) self.assertIsNotNone(config) self.assertIsInstance(config, BertConfig) diff --git a/pytorch_transformers/tests/modeling_tf_bert_test.py b/pytorch_transformers/tests/modeling_tf_bert_test.py index a1bf9bf794..b12f113c0f 100644 --- a/pytorch_transformers/tests/modeling_tf_bert_test.py +++ b/pytorch_transformers/tests/modeling_tf_bert_test.py @@ -316,7 +316,8 @@ class TFBertModelTest(TFCommonTestCases.TFCommonModelTester): @pytest.mark.slow def test_model_from_pretrained(self): cache_dir = "/tmp/pytorch_transformers_test/" - for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: + # for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: + for model_name in ['bert-base-uncased']: model = TFBertModel.from_pretrained(model_name, cache_dir=cache_dir) shutil.rmtree(cache_dir) self.assertIsNotNone(model) From c9591f6fac688aabe0aff62d0b68df2f95367a01 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Mon, 23 Sep 2019 22:08:08 +0200 Subject: [PATCH 093/219] updated models input format + tests --- pytorch_transformers/modeling_tf_bert.py | 94 +++++++++---------- .../modeling_tf_distilbert.py | 51 +++++----- pytorch_transformers/modeling_tf_gpt2.py | 78 ++++++++------- pytorch_transformers/modeling_tf_openai.py | 70 +++++++------- pytorch_transformers/modeling_tf_roberta.py | 25 ++--- .../modeling_tf_transfo_xl.py | 50 +++++----- pytorch_transformers/modeling_tf_xlm.py | 60 ++++++------ pytorch_transformers/modeling_tf_xlnet.py | 59 ++++++------ .../tests/modeling_tf_bert_test.py | 2 +- .../tests/modeling_tf_common_test.py | 17 ++++ 10 files changed, 257 insertions(+), 249 deletions(-) diff --git a/pytorch_transformers/modeling_tf_bert.py b/pytorch_transformers/modeling_tf_bert.py index edf4c30db9..b00bd09296 100644 --- a/pytorch_transformers/modeling_tf_bert.py +++ b/pytorch_transformers/modeling_tf_bert.py @@ -456,24 +456,23 @@ class TFBertMainLayer(tf.keras.layers.Layer): # def call(self, input_ids, attention_mask=None, token_type_ids=None, # position_ids=None, head_mask=None, training=False): - def call(self, inputs, training=False): - if not isinstance(inputs, (dict, tuple, list)): - input_ids = inputs - attention_mask, head_mask, position_ids, token_type_ids = None, None, None, None - elif isinstance(inputs, (tuple, list)): + def call(self, inputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, training=False): + if isinstance(inputs, (tuple, list)): input_ids = inputs[0] - attention_mask = inputs[1] if len(inputs) > 1 else None - token_type_ids = inputs[2] if len(inputs) > 2 else None - position_ids = inputs[3] if len(inputs) > 3 else None - head_mask = inputs[4] if len(inputs) > 4 else None + attention_mask = inputs[1] if len(inputs) > 1 else attention_mask + token_type_ids = inputs[2] if len(inputs) > 2 else token_type_ids + position_ids = inputs[3] if len(inputs) > 3 else position_ids + head_mask = inputs[4] if len(inputs) > 4 else head_mask + assert len(inputs) <= 5, "Too many inputs." + elif isinstance(inputs, dict): + input_ids = inputs.get('input_ids') + attention_mask = inputs.get('attention_mask', attention_mask) + token_type_ids = inputs.get('token_type_ids', token_type_ids) + position_ids = inputs.get('position_ids', position_ids) + head_mask = inputs.get('head_mask', head_mask) assert len(inputs) <= 5, "Too many inputs." else: - input_ids = inputs.get('input_ids') - attention_mask = inputs.get('attention_mask', None) - token_type_ids = inputs.get('token_type_ids', None) - position_ids = inputs.get('position_ids', None) - head_mask = inputs.get('head_mask', None) - assert len(inputs) <= 5, "Too many inputs." + input_ids = inputs if attention_mask is None: attention_mask = tf.fill(tf.shape(input_ids), 1) @@ -637,8 +636,8 @@ class TFBertModel(TFBertPreTrainedModel): super(TFBertModel, self).__init__(config, *inputs, **kwargs) self.bert = TFBertMainLayer(config, name='bert') - def call(self, inputs, training=False): - outputs = self.bert(inputs, training=training) + def call(self, inputs, **kwargs): + outputs = self.bert(inputs, **kwargs) return outputs @@ -676,11 +675,11 @@ class TFBertForPreTraining(TFBertPreTrainedModel): self.nsp = TFBertNSPHead(config, name='nsp___cls') self.mlm = TFBertMLMHead(config, self.bert.embeddings, name='mlm___cls') - def call(self, inputs, training=False): - outputs = self.bert(inputs, training=training) + def call(self, inputs, **kwargs): + outputs = self.bert(inputs, **kwargs) sequence_output, pooled_output = outputs[:2] - prediction_scores = self.mlm(sequence_output, training=training) + prediction_scores = self.mlm(sequence_output, training=kwargs.get('training', False)) seq_relationship_score = self.nsp(pooled_output) outputs = (prediction_scores, seq_relationship_score,) + outputs[2:] # add hidden states and attention if they are here @@ -718,11 +717,11 @@ class TFBertForMaskedLM(TFBertPreTrainedModel): self.bert = TFBertMainLayer(config, name='bert') self.mlm = TFBertMLMHead(config, self.bert.embeddings, name='mlm___cls') - def call(self, inputs, training=False): - outputs = self.bert(inputs, training=training) + def call(self, inputs, **kwargs): + outputs = self.bert(inputs, **kwargs) sequence_output = outputs[0] - prediction_scores = self.mlm(sequence_output, training=training) + prediction_scores = self.mlm(sequence_output, training=kwargs.get('training', False)) outputs = (prediction_scores,) + outputs[2:] # Add hidden states and attention if they are here @@ -761,8 +760,8 @@ class TFBertForNextSentencePrediction(TFBertPreTrainedModel): self.bert = TFBertMainLayer(config, name='bert') self.nsp = TFBertNSPHead(config, name='nsp___cls') - def call(self, inputs, training=False): - outputs = self.bert(inputs, training=training) + def call(self, inputs, **kwargs): + outputs = self.bert(inputs, **kwargs) pooled_output = outputs[1] seq_relationship_score = self.nsp(pooled_output) @@ -805,12 +804,12 @@ class TFBertForSequenceClassification(TFBertPreTrainedModel): self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) self.classifier = tf.keras.layers.Dense(config.num_labels, name='classifier') - def call(self, inputs, training=False): - outputs = self.bert(inputs, training=training) + def call(self, inputs, **kwargs): + outputs = self.bert(inputs, **kwargs) pooled_output = outputs[1] - pooled_output = self.dropout(pooled_output, training=training) + pooled_output = self.dropout(pooled_output, training=kwargs.get('training', False)) logits = self.classifier(pooled_output) outputs = (logits,) + outputs[2:] # add hidden states and attention if they are here @@ -852,24 +851,23 @@ class TFBertForMultipleChoice(TFBertPreTrainedModel): self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) self.classifier = tf.keras.layers.Dense(1, name='classifier') - def call(self, inputs, training=False): - if not isinstance(inputs, (dict, tuple, list)): - input_ids = inputs - attention_mask, head_mask, position_ids, token_type_ids = None, None, None, None - elif isinstance(inputs, (tuple, list)): + def call(self, inputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, training=False): + if isinstance(inputs, (tuple, list)): input_ids = inputs[0] - attention_mask = inputs[1] if len(inputs) > 1 else None - token_type_ids = inputs[2] if len(inputs) > 2 else None - position_ids = inputs[3] if len(inputs) > 3 else None - head_mask = inputs[4] if len(inputs) > 4 else None + attention_mask = inputs[1] if len(inputs) > 1 else attention_mask + token_type_ids = inputs[2] if len(inputs) > 2 else token_type_ids + position_ids = inputs[3] if len(inputs) > 3 else position_ids + head_mask = inputs[4] if len(inputs) > 4 else head_mask + assert len(inputs) <= 5, "Too many inputs." + elif isinstance(inputs, dict): + input_ids = inputs.get('input_ids') + attention_mask = inputs.get('attention_mask', attention_mask) + token_type_ids = inputs.get('token_type_ids', token_type_ids) + position_ids = inputs.get('position_ids', position_ids) + head_mask = inputs.get('head_mask', head_mask) assert len(inputs) <= 5, "Too many inputs." else: - input_ids = inputs.get('input_ids') - attention_mask = inputs.get('attention_mask', None) - token_type_ids = inputs.get('token_type_ids', None) - position_ids = inputs.get('position_ids', None) - head_mask = inputs.get('head_mask', None) - assert len(inputs) <= 5, "Too many inputs." + input_ids = inputs num_choices = tf.shape(input_ids)[1] seq_length = tf.shape(input_ids)[2] @@ -927,12 +925,12 @@ class TFBertForTokenClassification(TFBertPreTrainedModel): self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) self.classifier = tf.keras.layers.Dense(config.num_labels, name='classifier') - def call(self, inputs, training=False): - outputs = self.bert(inputs, training=training) + def call(self, inputs, **kwargs): + outputs = self.bert(inputs, **kwargs) sequence_output = outputs[0] - sequence_output = self.dropout(sequence_output, training=training) + sequence_output = self.dropout(sequence_output, training=kwargs.get('training', False)) logits = self.classifier(sequence_output) outputs = (logits,) + outputs[2:] # add hidden states and attention if they are here @@ -976,8 +974,8 @@ class TFBertForQuestionAnswering(TFBertPreTrainedModel): self.bert = TFBertMainLayer(config, name='bert') self.qa_outputs = tf.keras.layers.Dense(config.num_labels, name='qa_outputs') - def call(self, inputs, training=False): - outputs = self.bert(inputs, training=training) + def call(self, inputs, **kwargs): + outputs = self.bert(inputs, **kwargs) sequence_output = outputs[0] diff --git a/pytorch_transformers/modeling_tf_distilbert.py b/pytorch_transformers/modeling_tf_distilbert.py index 48bf3f377c..706d2fc02d 100644 --- a/pytorch_transformers/modeling_tf_distilbert.py +++ b/pytorch_transformers/modeling_tf_distilbert.py @@ -418,20 +418,19 @@ class TFDistilBertMainLayer(tf.keras.layers.Layer): def _prune_heads(self, heads_to_prune): raise NotImplementedError - def call(self, inputs, training=False): - if not isinstance(inputs, (dict, tuple, list)): - input_ids = inputs - (attention_mask, head_mask) = None, None - elif isinstance(inputs, (tuple, list)): + def call(self, inputs, attention_mask=None, head_mask=None, training=False): + if isinstance(inputs, (tuple, list)): input_ids = inputs[0] - attention_mask = inputs[1] if len(inputs) > 1 else None - head_mask = inputs[2] if len(inputs) > 2 else None + attention_mask = inputs[1] if len(inputs) > 1 else attention_mask + head_mask = inputs[2] if len(inputs) > 2 else head_mask + assert len(inputs) <= 3, "Too many inputs." + elif isinstance(inputs, dict): + input_ids = inputs.get('input_ids') + attention_mask = inputs.get('attention_mask', attention_mask) + head_mask = inputs.get('head_mask', head_mask) assert len(inputs) <= 3, "Too many inputs." else: - input_ids = inputs.get('input_ids') - attention_mask = inputs.get('attention_mask', None) - head_mask = inputs.get('head_mask', None) - assert len(inputs) <= 3, "Too many inputs." + input_ids = inputs if attention_mask is None: attention_mask = tf.ones(shape_list(input_ids)) # (bs, seq_length) @@ -532,8 +531,8 @@ class TFDistilBertModel(TFDistilBertPreTrainedModel): super(TFDistilBertModel, self).__init__(config, *inputs, **kwargs) self.distilbert = TFDistilBertMainLayer(config, name="distilbert") # Embeddings - def call(self, inputs, training=False): - outputs = self.distilbert(inputs, training=training) + def call(self, inputs, **kwargs): + outputs = self.distilbert(inputs, **kwargs) return outputs @@ -603,18 +602,17 @@ class TFDistilBertForMaskedLM(TFDistilBertPreTrainedModel): self.vocab_layer_norm = tf.keras.layers.LayerNormalization(epsilon=1e-12, name="vocab_layer_norm") self.vocab_projector = TFDistilBertLMHead(config, self.distilbert.embeddings, name="vocab_projector") - def call(self, inputs, training=False): - dlbrt_output = self.distilbert(inputs, training=training) + def call(self, inputs, **kwargs): + distilbert_output = self.distilbert(inputs, **kwargs) - hidden_states = dlbrt_output[0] # (bs, seq_length, dim) + hidden_states = distilbert_output[0] # (bs, seq_length, dim) prediction_logits = self.vocab_transform(hidden_states) # (bs, seq_length, dim) prediction_logits = self.act(prediction_logits) # (bs, seq_length, dim) prediction_logits = self.vocab_layer_norm(prediction_logits) # (bs, seq_length, dim) prediction_logits = self.vocab_projector(prediction_logits) - outputs = (prediction_logits, ) + dlbrt_output[1:] - - return outputs # prediction_logits, (all hidden_states), (all attentions) + outputs = (prediction_logits,) + distilbert_output[1:] + return outputs # logits, (hidden_states), (attentions) @add_start_docstrings("""DistilBert Model transformer with a sequence classification/regression head on top (a linear layer on top of @@ -660,12 +658,13 @@ class TFDistilBertForSequenceClassification(TFDistilBertPreTrainedModel): self.classifier = tf.keras.layers.Dense(config.num_labels, name="classifier") self.dropout = tf.keras.layers.Dropout(config.seq_classif_dropout) - def call(self, inputs, training=False): - distilbert_output = self.distilbert(inputs, training=training) + def call(self, inputs, **kwargs): + distilbert_output = self.distilbert(inputs, **kwargs) + hidden_state = distilbert_output[0] # (bs, seq_len, dim) pooled_output = hidden_state[:, 0] # (bs, dim) pooled_output = self.pre_classifier(pooled_output) # (bs, dim) - pooled_output = self.dropout(pooled_output, training=training) # (bs, dim) + pooled_output = self.dropout(pooled_output, training=kwargs.get('training', False)) # (bs, dim) logits = self.classifier(pooled_output) # (bs, dim) outputs = (logits,) + distilbert_output[1:] @@ -720,11 +719,11 @@ class TFDistilBertForQuestionAnswering(TFDistilBertPreTrainedModel): assert config.num_labels == 2 self.dropout = tf.keras.layers.Dropout(config.qa_dropout) - def call(self, inputs, training=False): - distilbert_output = self.distilbert(inputs, training=training) - hidden_states = distilbert_output[0] # (bs, max_query_len, dim) + def call(self, inputs, **kwargs): + distilbert_output = self.distilbert(inputs, **kwargs) - hidden_states = self.dropout(hidden_states, training=training) # (bs, max_query_len, dim) + hidden_states = distilbert_output[0] # (bs, max_query_len, dim) + hidden_states = self.dropout(hidden_states, training=kwargs.get('training', False)) # (bs, max_query_len, dim) logits = self.qa_outputs(hidden_states) # (bs, max_query_len, 2) start_logits, end_logits = tf.split(logits, 2, axis=-1) start_logits = tf.squeeze(start_logits, axis=-1) diff --git a/pytorch_transformers/modeling_tf_gpt2.py b/pytorch_transformers/modeling_tf_gpt2.py index e0a0a16799..858ea7dd42 100644 --- a/pytorch_transformers/modeling_tf_gpt2.py +++ b/pytorch_transformers/modeling_tf_gpt2.py @@ -230,26 +230,25 @@ class TFGPT2MainLayer(tf.keras.layers.Layer): """ raise NotImplementedError - def call(self, inputs, training=False): - if not isinstance(inputs, (dict, tuple, list)): - input_ids = inputs - past, attention_mask, token_type_ids, position_ids, head_mask = None, None, None, None, None - elif isinstance(inputs, (tuple, list)): + def call(self, inputs, past=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, training=False): + if isinstance(inputs, (tuple, list)): input_ids = inputs[0] - past = inputs[1] if len(inputs) > 1 else None - attention_mask = inputs[2] if len(inputs) > 2 else None - token_type_ids = inputs[3] if len(inputs) > 3 else None - position_ids = inputs[4] if len(inputs) > 4 else None - head_mask = inputs[5] if len(inputs) > 5 else None + past = inputs[1] if len(inputs) > 1 else past + attention_mask = inputs[2] if len(inputs) > 2 else attention_mask + token_type_ids = inputs[3] if len(inputs) > 3 else token_type_ids + position_ids = inputs[4] if len(inputs) > 4 else position_ids + head_mask = inputs[5] if len(inputs) > 5 else head_mask + assert len(inputs) <= 6, "Too many inputs." + elif isinstance(inputs, dict): + input_ids = inputs.get('input_ids') + past = inputs.get('past', past) + attention_mask = inputs.get('attention_mask', attention_mask) + token_type_ids = inputs.get('token_type_ids', token_type_ids) + position_ids = inputs.get('position_ids', position_ids) + head_mask = inputs.get('head_mask', head_mask) assert len(inputs) <= 6, "Too many inputs." else: - input_ids = inputs.get('input_ids') - past = inputs.get('past', None) - attention_mask = inputs.get('attention_mask', None) - token_type_ids = inputs.get('token_type_ids', None) - position_ids = inputs.get('position_ids', None) - head_mask = inputs.get('head_mask', None) - assert len(inputs) <= 6, "Too many inputs." + input_ids = inputs if past is None: past_length = 0 @@ -442,8 +441,8 @@ class TFGPT2Model(TFGPT2PreTrainedModel): super(TFGPT2Model, self).__init__(config, *inputs, **kwargs) self.transformer = TFGPT2MainLayer(config, name='transformer') - def call(self, inputs, training=False): - outputs = self.transformer(inputs, training=training) + def call(self, inputs, **kwargs): + outputs = self.transformer(inputs, **kwargs) return outputs @@ -483,8 +482,8 @@ class TFGPT2LMHeadModel(TFGPT2PreTrainedModel): super(TFGPT2LMHeadModel, self).__init__(config, *inputs, **kwargs) self.transformer = TFGPT2MainLayer(config, name='transformer') - def call(self, inputs, training=False): - transformer_outputs = self.transformer(inputs, training=training) + def call(self, inputs, **kwargs): + transformer_outputs = self.transformer(inputs, **kwargs) hidden_states = transformer_outputs[0] lm_logits = self.transformer.wte(hidden_states, mode="linear") @@ -551,28 +550,27 @@ class TFGPT2DoubleHeadsModel(TFGPT2PreTrainedModel): self.transformer = TFGPT2MainLayer(config, name='transformer') self.multiple_choice_head = TFSequenceSummary(config, name='multiple_choice_head') - def call(self, inputs, training=False): - if not isinstance(inputs, (dict, tuple, list)): - input_ids = inputs - mc_token_ids, past, attention_mask, token_type_ids, position_ids, head_mask = None, None, None, None, None - elif isinstance(inputs, (tuple, list)): + def call(self, inputs, past=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, mc_token_ids=None, training=False): + if isinstance(inputs, (tuple, list)): input_ids = inputs[0] - mc_token_ids = inputs[1] if len(inputs) > 1 else None - past = inputs[2] if len(inputs) > 2 else None - attention_mask = inputs[3] if len(inputs) > 3 else None - token_type_ids = inputs[4] if len(inputs) > 4 else None - position_ids = inputs[5] if len(inputs) > 5 else None - head_mask = inputs[6] if len(inputs) > 6 else None + past = inputs[1] if len(inputs) > 1 else past + attention_mask = inputs[2] if len(inputs) > 2 else attention_mask + token_type_ids = inputs[3] if len(inputs) > 3 else token_type_ids + position_ids = inputs[4] if len(inputs) > 4 else position_ids + head_mask = inputs[5] if len(inputs) > 5 else head_mask + mc_token_ids = inputs[6] if len(inputs) > 6 else mc_token_ids + assert len(inputs) <= 7, "Too many inputs." + elif isinstance(inputs, dict): + input_ids = inputs.get('input_ids') + past = inputs.get('past', past) + attention_mask = inputs.get('attention_mask', attention_mask) + token_type_ids = inputs.get('token_type_ids', token_type_ids) + position_ids = inputs.get('position_ids', position_ids) + head_mask = inputs.get('head_mask', head_mask) + mc_token_ids = inputs.get('mc_token_ids', mc_token_ids) assert len(inputs) <= 7, "Too many inputs." else: - input_ids = inputs.get('input_ids') - mc_token_ids = inputs.get('mc_token_ids', None) - past = inputs.get('past', None) - attention_mask = inputs.get('attention_mask', None) - token_type_ids = inputs.get('token_type_ids', None) - position_ids = inputs.get('position_ids', None) - head_mask = inputs.get('head_mask', None) - assert len(inputs) <= 7, "Too many inputs." + input_ids = inputs input_shapes = shape_list(input_ids) diff --git a/pytorch_transformers/modeling_tf_openai.py b/pytorch_transformers/modeling_tf_openai.py index 2292d4cab8..5bfe5b2abc 100644 --- a/pytorch_transformers/modeling_tf_openai.py +++ b/pytorch_transformers/modeling_tf_openai.py @@ -229,24 +229,23 @@ class TFOpenAIGPTMainLayer(tf.keras.layers.Layer): """ raise NotImplementedError - def call(self, inputs, training=False): - if not isinstance(inputs, (dict, tuple, list)): - input_ids = inputs - attention_mask, token_type_ids, position_ids, head_mask = None, None, None, None - elif isinstance(inputs, (tuple, list)): + def call(self, inputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, training=False): + if isinstance(inputs, (tuple, list)): input_ids = inputs[0] - attention_mask = inputs[1] if len(inputs) > 1 else None - token_type_ids = inputs[2] if len(inputs) > 2 else None - position_ids = inputs[3] if len(inputs) > 3 else None - head_mask = inputs[4] if len(inputs) > 4 else None + attention_mask = inputs[1] if len(inputs) > 1 else attention_mask + token_type_ids = inputs[2] if len(inputs) > 2 else token_type_ids + position_ids = inputs[3] if len(inputs) > 3 else position_ids + head_mask = inputs[4] if len(inputs) > 4 else head_mask + assert len(inputs) <= 5, "Too many inputs." + elif isinstance(inputs, dict): + input_ids = inputs.get('input_ids') + attention_mask = inputs.get('attention_mask', attention_mask) + token_type_ids = inputs.get('token_type_ids', token_type_ids) + position_ids = inputs.get('position_ids', position_ids) + head_mask = inputs.get('head_mask', head_mask) assert len(inputs) <= 5, "Too many inputs." else: - input_ids = inputs.get('input_ids') - attention_mask = inputs.get('attention_mask', None) - token_type_ids = inputs.get('token_type_ids', None) - position_ids = inputs.get('position_ids', None) - head_mask = inputs.get('head_mask', None) - assert len(inputs) <= 5, "Too many inputs." + input_ids = inputs if position_ids is None: position_ids = tf.range(shape_list(input_ids)[-1], dtype=tf.int32)[tf.newaxis, :] @@ -420,8 +419,8 @@ class TFOpenAIGPTModel(TFOpenAIGPTPreTrainedModel): super(TFOpenAIGPTModel, self).__init__(config, *inputs, **kwargs) self.transformer = TFOpenAIGPTMainLayer(config, name='transformer') - def call(self, inputs, training=False): - outputs = self.transformer(inputs, training=training) + def call(self, inputs, **kwargs): + outputs = self.transformer(inputs, **kwargs) return outputs @@ -455,8 +454,8 @@ class TFOpenAIGPTLMHeadModel(TFOpenAIGPTPreTrainedModel): super(TFOpenAIGPTLMHeadModel, self).__init__(config, *inputs, **kwargs) self.transformer = TFOpenAIGPTMainLayer(config, name='transformer') - def call(self, inputs, training=False): - transformer_outputs = self.transformer(inputs, training=training) + def call(self, inputs, **kwargs): + transformer_outputs = self.transformer(inputs, **kwargs) hidden_states = transformer_outputs[0] lm_logits = self.transformer.tokens_embed(hidden_states, mode="linear") @@ -511,26 +510,25 @@ class TFOpenAIGPTDoubleHeadsModel(TFOpenAIGPTPreTrainedModel): self.transformer = TFOpenAIGPTMainLayer(config, name='transformer') self.multiple_choice_head = TFSequenceSummary(config, name='multiple_choice_head') - def call(self, inputs, training=False): - if not isinstance(inputs, (dict, tuple, list)): - input_ids = inputs - mc_token_ids, attention_mask, token_type_ids, position_ids, head_mask = None, None, None, None - elif isinstance(inputs, (tuple, list)): + def call(self, inputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, mc_token_ids=None, training=False): + if isinstance(inputs, (tuple, list)): input_ids = inputs[0] - mc_token_ids = inputs[1] if len(inputs) > 1 else None - attention_mask = inputs[2] if len(inputs) > 2 else None - token_type_ids = inputs[3] if len(inputs) > 3 else None - position_ids = inputs[4] if len(inputs) > 4 else None - head_mask = inputs[5] if len(inputs) > 5 else None + attention_mask = inputs[1] if len(inputs) > 1 else attention_mask + token_type_ids = inputs[2] if len(inputs) > 2 else token_type_ids + position_ids = inputs[3] if len(inputs) > 3 else position_ids + head_mask = inputs[4] if len(inputs) > 4 else head_mask + mc_token_ids = inputs[5] if len(inputs) > 5 else mc_token_ids + assert len(inputs) <= 6, "Too many inputs." + elif isinstance(inputs, dict): + input_ids = inputs.get('input_ids') + attention_mask = inputs.get('attention_mask', attention_mask) + token_type_ids = inputs.get('token_type_ids', token_type_ids) + position_ids = inputs.get('position_ids', position_ids) + head_mask = inputs.get('head_mask', head_mask) + mc_token_ids = inputs.get('mc_token_ids', mc_token_ids) assert len(inputs) <= 6, "Too many inputs." else: - input_ids = inputs.get('input_ids') - mc_token_ids = inputs.get('mc_token_ids', None) - attention_mask = inputs.get('attention_mask', None) - token_type_ids = inputs.get('token_type_ids', None) - position_ids = inputs.get('position_ids', None) - head_mask = inputs.get('head_mask', None) - assert len(inputs) <= 6, "Too many inputs." + input_ids = inputs input_shapes = shape_list(input_ids) diff --git a/pytorch_transformers/modeling_tf_roberta.py b/pytorch_transformers/modeling_tf_roberta.py index 285dfb5b98..2d912eb572 100644 --- a/pytorch_transformers/modeling_tf_roberta.py +++ b/pytorch_transformers/modeling_tf_roberta.py @@ -73,21 +73,21 @@ class TFRobertaMainLayer(TFBertMainLayer): super(TFRobertaMainLayer, self).__init__(config, **kwargs) self.embeddings = TFRobertaEmbeddings(config, name='embeddings') - def call(self, inputs, training=False): + def call(self, inputs, **kwargs): # Check that input_ids starts with control token - if not isinstance(inputs, (dict, tuple, list)): - input_ids = inputs - elif isinstance(inputs, (tuple, list)): + if isinstance(inputs, (tuple, list)): input_ids = inputs[0] - else: + elif isinstance(inputs, dict): input_ids = inputs.get('input_ids') + else: + input_ids = inputs if tf.not_equal(tf.reduce_sum(input_ids[:, 0]), 0): logger.warning("A sequence with no special tokens has been passed to the RoBERTa model. " "This model requires special tokens in order to work. " "Please specify add_special_tokens=True in your encoding.") - return super(TFRobertaMainLayer, self).call(inputs, training=training) + return super(TFRobertaMainLayer, self).call(inputs, **kwargs) class TFRobertaPreTrainedModel(TFPreTrainedModel): @@ -203,8 +203,8 @@ class TFRobertaModel(TFRobertaPreTrainedModel): super(TFRobertaModel, self).__init__(config, *inputs, **kwargs) self.roberta = TFRobertaMainLayer(config, name='roberta') - def call(self, inputs, training=False): - outputs = self.roberta(inputs, training=training) + def call(self, inputs, **kwargs): + outputs = self.roberta(inputs, **kwargs) return outputs @@ -277,8 +277,8 @@ class TFRobertaForMaskedLM(TFRobertaPreTrainedModel): self.roberta = TFRobertaMainLayer(config, name="roberta") self.lm_head = TFRobertaLMHead(config, self.roberta.embeddings, name="lm_head") - def call(self, inputs, training=False): - outputs = self.roberta(inputs, training=training) + def call(self, inputs, **kwargs): + outputs = self.roberta(inputs, **kwargs) sequence_output = outputs[0] prediction_scores = self.lm_head(sequence_output) @@ -347,8 +347,9 @@ class TFRobertaForSequenceClassification(TFRobertaPreTrainedModel): self.roberta = TFRobertaMainLayer(config, name="roberta") self.classifier = TFRobertaClassificationHead(config, name="classifier") - def call(self, inputs, training=False): - outputs = self.roberta(inputs, training=training) + def call(self, inputs, **kwargs): + outputs = self.roberta(inputs, **kwargs) + sequence_output = outputs[0] logits = self.classifier(sequence_output, training=training) diff --git a/pytorch_transformers/modeling_tf_transfo_xl.py b/pytorch_transformers/modeling_tf_transfo_xl.py index ded06923e8..57bb61869e 100644 --- a/pytorch_transformers/modeling_tf_transfo_xl.py +++ b/pytorch_transformers/modeling_tf_transfo_xl.py @@ -447,20 +447,19 @@ class TFTransfoXLMainLayer(tf.keras.layers.Layer): return new_mems - def call(self, inputs, training=False): - if not isinstance(inputs, (dict, tuple, list)): - input_ids = inputs - mems, head_mask = None, None - elif isinstance(inputs, (tuple, list)): + def call(self, inputs, mems=None, head_mask=None, training=False): + if isinstance(inputs, (tuple, list)): input_ids = inputs[0] - mems = inputs[1] if len(inputs) > 1 else None - head_mask = inputs[2] if len(inputs) > 2 else None + mems = inputs[1] if len(inputs) > 1 else mems + head_mask = inputs[2] if len(inputs) > 2 else head_mask + assert len(inputs) <= 3, "Too many inputs." + elif isinstance(inputs, dict): + input_ids = inputs.get('input_ids') + mems = inputs.get('mems', mems) + head_mask = inputs.get('head_mask', head_mask) assert len(inputs) <= 3, "Too many inputs." else: - input_ids = inputs.get('input_ids') - mems = inputs.get('mems', None) - head_mask = inputs.get('head_mask', None) - assert len(inputs) <= 3, "Too many inputs." + input_ids = inputs # the original code for Transformer-XL used shapes [len, bsz] but we want a unified interface in the library # so we transpose here from shape [bsz, len] to shape [len, bsz] @@ -632,8 +631,8 @@ class TFTransfoXLModel(TFTransfoXLPreTrainedModel): super(TFTransfoXLModel, self).__init__(config, *inputs, **kwargs) self.transformer = TFTransfoXLMainLayer(config, name='transformer') - def call(self, inputs, training=False, **kwargs): - outputs = self.transformer(inputs, training=training, **kwargs) + def call(self, inputs, **kwargs): + outputs = self.transformer(inputs, **kwargs) return outputs @@ -694,22 +693,21 @@ class TFTransfoXLLMHeadModel(TFTransfoXLPreTrainedModel): def init_mems(self, data): return self.transformer.init_mems(data) - def call(self, inputs, training=False): - if not isinstance(inputs, (dict, tuple, list)): - input_ids = inputs - mems, head_mask, labels = None, None, None - elif isinstance(inputs, (tuple, list)): + def call(self, inputs, mems=None, head_mask=None, labels=None, training=False): + if isinstance(inputs, (tuple, list)): input_ids = inputs[0] - mems = inputs[1] if len(inputs) > 1 else None - head_mask = inputs[2] if len(inputs) > 2 else None - labels = inputs[3] if len(inputs) > 3 else None + mems = inputs[1] if len(inputs) > 1 else mems + head_mask = inputs[2] if len(inputs) > 2 else head_mask + labels = inputs[3] if len(inputs) > 3 else labels + assert len(inputs) <= 4, "Too many inputs." + elif isinstance(inputs, dict): + input_ids = inputs.get('input_ids') + mems = inputs.get('mems', mems) + head_mask = inputs.get('head_mask', head_mask) + labels = inputs.get('labels', labels) assert len(inputs) <= 4, "Too many inputs." else: - input_ids = inputs.get('input_ids') - mems = inputs.get('mems', None) - head_mask = inputs.get('head_mask', None) - labels = inputs.get('labels', None) - assert len(inputs) <= 4, "Too many inputs." + input_ids = inputs bsz, tgt_len = shape_list(input_ids)[:2] diff --git a/pytorch_transformers/modeling_tf_xlm.py b/pytorch_transformers/modeling_tf_xlm.py index 94daa1f04b..a3915b6c47 100644 --- a/pytorch_transformers/modeling_tf_xlm.py +++ b/pytorch_transformers/modeling_tf_xlm.py @@ -294,31 +294,31 @@ class TFXLMMainLayer(tf.keras.layers.Layer): """ raise NotImplementedError - def call(self, inputs, training=False): # removed: src_enc=None, src_len=None - if not isinstance(inputs, (dict, tuple, list)): - input_ids = inputs - (attention_mask, langs, token_type_ids, position_ids, - lengths, cache, head_mask) = None, None, None, None, None, None, None - elif isinstance(inputs, (tuple, list)): + def call(self, inputs, attention_mask=None, langs=None, token_type_ids=None, + position_ids=None, lengths=None, cache=None, head_mask=None, + training=False): # removed: src_enc=None, src_len=None + if isinstance(inputs, (tuple, list)): input_ids = inputs[0] - attention_mask = inputs[1] if len(inputs) > 1 else None - langs = inputs[2] if len(inputs) > 2 else None - token_type_ids = inputs[3] if len(inputs) > 3 else None - position_ids = inputs[4] if len(inputs) > 4 else None - lengths = inputs[5] if len(inputs) > 5 else None - cache = inputs[6] if len(inputs) > 6 else None - head_mask = inputs[7] if len(inputs) > 7 else None + attention_mask = inputs[1] if len(inputs) > 1 else attention_mask + langs = inputs[2] if len(inputs) > 2 else langs + token_type_ids = inputs[3] if len(inputs) > 3 else token_type_ids + position_ids = inputs[4] if len(inputs) > 4 else position_ids + lengths = inputs[5] if len(inputs) > 5 else lengths + cache = inputs[6] if len(inputs) > 6 else cache + head_mask = inputs[7] if len(inputs) > 7 else head_mask + assert len(inputs) <= 8, "Too many inputs." + elif isinstance(inputs, dict): + input_ids = inputs.get('input_ids') + attention_mask = inputs.get('attention_mask', attention_mask) + langs = inputs.get('langs', langs) + token_type_ids = inputs.get('token_type_ids', token_type_ids) + position_ids = inputs.get('position_ids', position_ids) + lengths = inputs.get('lengths', lengths) + cache = inputs.get('cache', cache) + head_mask = inputs.get('head_mask', head_mask) assert len(inputs) <= 8, "Too many inputs." else: - input_ids = inputs.get('input_ids') - attention_mask = inputs.get('attention_mask', None) - langs = inputs.get('langs', None) - token_type_ids = inputs.get('token_type_ids', None) - position_ids = inputs.get('position_ids', None) - lengths = inputs.get('lengths', None) - cache = inputs.get('cache', None) - head_mask = inputs.get('head_mask', None) - assert len(inputs) <= 8, "Too many inputs." + input_ids = inputs if lengths is None: lengths = tf.reduce_sum(tf.cast(tf.not_equal(input_ids, self.pad_index), dtype=tf.int32), axis=1) @@ -538,8 +538,8 @@ class TFXLMModel(TFXLMPreTrainedModel): super(TFXLMModel, self).__init__(config, *inputs, **kwargs) self.transformer = TFXLMMainLayer(config, name='transformer') - def call(self, inputs, training=False): - outputs = self.transformer(inputs, training=training) + def call(self, inputs, **kwargs): + outputs = self.transformer(inputs, **kwargs) return outputs @@ -619,8 +619,8 @@ class TFXLMWithLMHeadModel(TFXLMPreTrainedModel): self.pred_layer = TFXLMPredLayer(config, self.transformer.embeddings, name='pred_layer_._proj') - def call(self, inputs, training=False): - transformer_outputs = self.transformer(inputs, training=training) + def call(self, inputs, **kwargs): + transformer_outputs = self.transformer(inputs, **kwargs) output = transformer_outputs[0] outputs = self.pred_layer(output) @@ -670,8 +670,8 @@ class TFXLMForSequenceClassification(TFXLMPreTrainedModel): self.transformer = TFXLMMainLayer(config, name='transformer') self.sequence_summary = TFSequenceSummary(config, name='sequence_summary') - def call(self, inputs, training=False): - transformer_outputs = self.transformer(inputs, training=training) + def call(self, inputs, **kwargs): + transformer_outputs = self.transformer(inputs, **kwargs) output = transformer_outputs[0] logits = self.sequence_summary(output) @@ -731,8 +731,8 @@ class TFXLMForQuestionAnsweringSimple(TFXLMPreTrainedModel): self.transformer = TFXLMMainLayer(config, name='transformer') self.qa_outputs = tf.keras.layers.Dense(config.num_labels, name='qa_outputs') - def call(self, inputs, training=False): - transformer_outputs = self.transformer(inputs, training=training) + def call(self, inputs, **kwargs): + transformer_outputs = self.transformer(inputs, **kwargs) sequence_output = transformer_outputs[0] diff --git a/pytorch_transformers/modeling_tf_xlnet.py b/pytorch_transformers/modeling_tf_xlnet.py index 1605564802..f15ac4899f 100644 --- a/pytorch_transformers/modeling_tf_xlnet.py +++ b/pytorch_transformers/modeling_tf_xlnet.py @@ -489,31 +489,30 @@ class TFXLNetMainLayer(tf.keras.layers.Layer): return pos_emb - def call(self, inputs, training=False): - if not isinstance(inputs, (dict, tuple, list)): - input_ids = inputs - (attention_mask, mems, perm_mask, target_mapping, - token_type_ids, input_mask, head_mask) = None, None, None, None, None, None, None - elif isinstance(inputs, (tuple, list)): + def call(self, inputs, attention_mask=None, mems=None, perm_mask=None, target_mapping=None, + token_type_ids=None, input_mask=None, head_mask=None, training=False): + if isinstance(inputs, (tuple, list)): input_ids = inputs[0] - attention_mask = inputs[1] if len(inputs) > 1 else None - mems = inputs[2] if len(inputs) > 2 else None - perm_mask = inputs[3] if len(inputs) > 3 else None - target_mapping = inputs[4] if len(inputs) > 4 else None - token_type_ids = inputs[5] if len(inputs) > 5 else None - input_mask = inputs[6] if len(inputs) > 6 else None - head_mask = inputs[7] if len(inputs) > 7 else None + attention_mask = inputs[1] if len(inputs) > 1 else attention_mask + mems = inputs[2] if len(inputs) > 2 else mems + perm_mask = inputs[3] if len(inputs) > 3 else perm_mask + target_mapping = inputs[4] if len(inputs) > 4 else target_mapping + token_type_ids = inputs[5] if len(inputs) > 5 else token_type_ids + input_mask = inputs[6] if len(inputs) > 6 else input_mask + head_mask = inputs[7] if len(inputs) > 7 else head_mask + assert len(inputs) <= 8, "Too many inputs." + elif isinstance(inputs, dict): + input_ids = inputs.get('input_ids') + attention_mask = inputs.get('attention_mask', attention_mask) + mems = inputs.get('mems', mems) + perm_mask = inputs.get('perm_mask', perm_mask) + target_mapping = inputs.get('target_mapping', target_mapping) + token_type_ids = inputs.get('token_type_ids', token_type_ids) + input_mask = inputs.get('input_mask', input_mask) + head_mask = inputs.get('head_mask', head_mask) assert len(inputs) <= 8, "Too many inputs." else: - input_ids = inputs.get('input_ids') - attention_mask = inputs.get('attention_mask', None) - mems = inputs.get('mems', None) - perm_mask = inputs.get('perm_mask', None) - target_mapping = inputs.get('target_mapping', None) - token_type_ids = inputs.get('token_type_ids', None) - input_mask = inputs.get('input_mask', None) - head_mask = inputs.get('head_mask', None) - assert len(inputs) <= 8, "Too many inputs." + input_ids = inputs # the original code for XLNet uses shapes [len, bsz] with the batch dimension at the end # but we want a unified interface in the library with the batch size on the first dimension @@ -784,8 +783,8 @@ class TFXLNetModel(TFXLNetPreTrainedModel): super(TFXLNetModel, self).__init__(config, *inputs, **kwargs) self.transformer = TFXLNetMainLayer(config, name='transformer') - def call(self, inputs, training=False): - outputs = self.transformer(inputs, training=training) + def call(self, inputs, **kwargs): + outputs = self.transformer(inputs, **kwargs) return outputs @@ -829,8 +828,8 @@ class TFXLNetLMHeadModel(TFXLNetPreTrainedModel): self.transformer = TFXLNetMainLayer(config, name='transformer') self.lm_loss = TFXLNetLMHead(config, self.transformer.word_embedding, name='lm_loss') - def call(self, inputs, training=False): - transformer_outputs = self.transformer(inputs, training=training) + def call(self, inputs, **kwargs): + transformer_outputs = self.transformer(inputs, **kwargs) hidden_state = transformer_outputs[0] logits = self.lm_loss(hidden_state) @@ -886,8 +885,8 @@ class TFXLNetForSequenceClassification(TFXLNetPreTrainedModel): self.sequence_summary = TFSequenceSummary(config, name='sequence_summary') self.logits_proj = tf.keras.layers.Dense(config.num_labels, name='logits_proj') - def call(self, inputs, training=False): - transformer_outputs = self.transformer(inputs, training=training) + def call(self, inputs, **kwargs): + transformer_outputs = self.transformer(inputs, **kwargs) output = transformer_outputs[0] output = self.sequence_summary(output) @@ -933,8 +932,8 @@ class TFXLNetForQuestionAnsweringSimple(TFXLNetPreTrainedModel): self.transformer = TFXLNetMainLayer(config, name='transformer') self.qa_outputs = tf.keras.layers.Dense(config.num_labels, name='qa_outputs') - def call(self, inputs, training=False): - transformer_outputs = self.transformer(inputs, training=training) + def call(self, inputs, **kwargs): + transformer_outputs = self.transformer(inputs, **kwargs) sequence_output = transformer_outputs[0] diff --git a/pytorch_transformers/tests/modeling_tf_bert_test.py b/pytorch_transformers/tests/modeling_tf_bert_test.py index a1bf9bf794..200fac2c05 100644 --- a/pytorch_transformers/tests/modeling_tf_bert_test.py +++ b/pytorch_transformers/tests/modeling_tf_bert_test.py @@ -138,7 +138,7 @@ class TFBertModelTest(TFCommonTestCases.TFCommonModelTester): inputs = {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids} - sequence_output, pooled_output = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids) + sequence_output, pooled_output = model(inputs) inputs = [input_ids, input_mask] sequence_output, pooled_output = model(inputs) diff --git a/pytorch_transformers/tests/modeling_tf_common_test.py b/pytorch_transformers/tests/modeling_tf_common_test.py index 46360d1dd4..332db01408 100644 --- a/pytorch_transformers/tests/modeling_tf_common_test.py +++ b/pytorch_transformers/tests/modeling_tf_common_test.py @@ -29,6 +29,7 @@ from pytorch_transformers import is_tf_available if is_tf_available(): import tensorflow as tf + import numpy as np from pytorch_transformers import TFPreTrainedModel # from pytorch_transformers.modeling_bert import BertModel, BertConfig, BERT_PRETRAINED_MODEL_ARCHIVE_MAP else: @@ -65,6 +66,22 @@ class TFCommonTestCases: # msg="Parameter {} of model {} seems not properly initialized".format(name, model_class)) + def test_keyword_and_dict_args(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + model = model_class(config) + outputs_dict = model(inputs_dict) + + inputs_keywords = copy.deepcopy(inputs_dict) + input_ids = inputs_keywords.pop('input_ids') + outputs_keywords = model(input_ids, **inputs_keywords) + + output_dict = outputs_dict[0].numpy() + output_keywords = outputs_keywords[0].numpy() + + self.assertLess(np.sum(np.abs(output_dict - output_keywords)), 1e-6) + def test_attention_outputs(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() From 6448396d5473a4b55b5bae81c5654e9f951676ab Mon Sep 17 00:00:00 2001 From: thomwolf Date: Mon, 23 Sep 2019 22:27:13 +0200 Subject: [PATCH 094/219] fix roberta test --- pytorch_transformers/modeling_tf_roberta.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_transformers/modeling_tf_roberta.py b/pytorch_transformers/modeling_tf_roberta.py index 2d912eb572..4172dc4e6e 100644 --- a/pytorch_transformers/modeling_tf_roberta.py +++ b/pytorch_transformers/modeling_tf_roberta.py @@ -351,7 +351,7 @@ class TFRobertaForSequenceClassification(TFRobertaPreTrainedModel): outputs = self.roberta(inputs, **kwargs) sequence_output = outputs[0] - logits = self.classifier(sequence_output, training=training) + logits = self.classifier(sequence_output, training=kwargs.get('training', False)) outputs = (logits,) + outputs[2:] From 2b11fa51742e900c7afb2e698006bbb7ada24272 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Mon, 23 Sep 2019 22:35:45 +0200 Subject: [PATCH 095/219] update __init__ and conversion script --- pytorch_transformers/__init__.py | 18 ++++++ .../convert_pytorch_checkpoint_to_tf2.py | 55 ++++++++++++------- 2 files changed, 53 insertions(+), 20 deletions(-) diff --git a/pytorch_transformers/__init__.py b/pytorch_transformers/__init__.py index 40c4bab7e1..befe5710e8 100644 --- a/pytorch_transformers/__init__.py +++ b/pytorch_transformers/__init__.py @@ -113,6 +113,11 @@ if _tf_available: load_gpt2_pt_weights_in_tf2, TF_GPT2_PRETRAINED_MODEL_ARCHIVE_MAP) + from .modeling_tf_openai import (TFOpenAIGPTPreTrainedModel, TFOpenAIGPTMainLayer, + TFOpenAIGPTModel, TFOpenAIGPTLMHeadModel, TFOpenAIGPTDoubleHeadsModel, + load_openai_gpt_pt_weights_in_tf2, + TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP) + from .modeling_tf_transfo_xl import (TFTransfoXLPreTrainedModel, TFTransfoXLMainLayer, TFTransfoXLModel, TFTransfoXLLMHeadModel, load_transfo_xl_pt_weights_in_tf2, @@ -132,6 +137,19 @@ if _tf_available: load_xlm_pt_weights_in_tf2, TF_XLM_PRETRAINED_MODEL_ARCHIVE_MAP) + from .modeling_tf_roberta import (TFRobertaPreTrainedModel, TFRobertaMainLayer, + TFRobertaModel, TFRobertaLMHead, + TFRobertaForSequenceClassification, + load_roberta_pt_weights_in_tf2, + TF_ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP) + + from .modeling_tf_distilbert import (TFDistilBertPreTrainedModel, TFDistilBertMainLayer, + TFDistilBertModel, TFDistilBertForMaskedLM, + TFDistilBertForSequenceClassification, + TFDistilBertForSequenceClassification, + load_distilbert_pt_weights_in_tf2, + TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP) + # Files and general utilities from .file_utils import (PYTORCH_TRANSFORMERS_CACHE, PYTORCH_PRETRAINED_BERT_CACHE, cached_path, add_start_docstrings, add_end_docstrings, diff --git a/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py b/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py index 25ed70c2db..52a65ab607 100644 --- a/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py +++ b/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py @@ -24,31 +24,43 @@ import tensorflow as tf from pytorch_transformers import is_torch_available, cached_path -from pytorch_transformers import (BertConfig, TFBertForPreTraining, load_bert_pt_weights_in_tf2, - GPT2Config, TFGPT2LMHeadModel, load_gpt2_pt_weights_in_tf2, - XLNetConfig, TFXLNetLMHeadModel, load_xlnet_pt_weights_in_tf2, - XLMConfig, TFXLMWithLMHeadModel, load_xlm_pt_weights_in_tf2, - TransfoXLConfig, TFTransfoXLLMHeadModel, load_transfo_xl_pt_weights_in_tf2,) +from pytorch_transformers import (BertConfig, TFBertForPreTraining, load_bert_pt_weights_in_tf2, BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, + GPT2Config, TFGPT2LMHeadModel, load_gpt2_pt_weights_in_tf2, GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP, + XLNetConfig, TFXLNetLMHeadModel, load_xlnet_pt_weights_in_tf2, XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP, + XLMConfig, TFXLMWithLMHeadModel, load_xlm_pt_weights_in_tf2, XLM_PRETRAINED_CONFIG_ARCHIVE_MAP, + TransfoXLConfig, TFTransfoXLLMHeadModel, load_transfo_xl_pt_weights_in_tf2, TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP, + OpenAIGPTConfig, TFOpenAIGPTLMHeadModel, load_openai_gpt_pt_weights_in_tf2, OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP, + RobertaConfig, TFRobertaLMHead, load_roberta_pt_weights_in_tf2, ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, + DistilBertConfig, TFDistilBertForMaskedLM, load_distilbert_pt_weights_in_tf2, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP) if is_torch_available(): import torch import numpy as np - from pytorch_transformers import (BertForPreTraining, BERT_PRETRAINED_MODEL_ARCHIVE_MAP, BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, - GPT2LMHeadModel, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP, GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP, - XLNetLMHeadModel, XLNET_PRETRAINED_MODEL_ARCHIVE_MAP, XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP, - XLMWithLMHeadModel, XLM_PRETRAINED_MODEL_ARCHIVE_MAP, XLM_PRETRAINED_CONFIG_ARCHIVE_MAP, - TransfoXLLMHeadModel, TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP, TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP,) + from pytorch_transformers import (BertForPreTraining, BERT_PRETRAINED_MODEL_ARCHIVE_MAP, + GPT2LMHeadModel, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP, + XLNetLMHeadModel, XLNET_PRETRAINED_MODEL_ARCHIVE_MAP, + XLMWithLMHeadModel, XLM_PRETRAINED_MODEL_ARCHIVE_MAP, + TransfoXLLMHeadModel, TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP, + OpenAIGPTLMHeadModel, OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP, + RobertaForMaskedLM, ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP, + DistilBertForMaskedLM, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP) else: - (BertForPreTraining, BERT_PRETRAINED_MODEL_ARCHIVE_MAP, BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, - GPT2LMHeadModel, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP, GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP, - XLNetLMHeadModel, XLNET_PRETRAINED_MODEL_ARCHIVE_MAP, XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP, - XLMWithLMHeadModel, XLM_PRETRAINED_MODEL_ARCHIVE_MAP, XLM_PRETRAINED_CONFIG_ARCHIVE_MAP, - TransfoXLLMHeadModel, TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP, TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP,) = ( - None, None, None, - None, None, None, - None, None, None, - None, None, None, - None, None, None,) + (BertForPreTraining, BERT_PRETRAINED_MODEL_ARCHIVE_MAP, + GPT2LMHeadModel, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP, + XLNetLMHeadModel, XLNET_PRETRAINED_MODEL_ARCHIVE_MAP, + XLMWithLMHeadModel, XLM_PRETRAINED_MODEL_ARCHIVE_MAP, + TransfoXLLMHeadModel, TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP, + OpenAIGPTLMHeadModel, OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP, + RobertaForMaskedLM, ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP, + DistilBertForMaskedLM, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP,) = ( + None, None, + None, None, + None, None, + None, None, + None, None, + None, None, + None, None, + None, None,) import logging @@ -60,6 +72,9 @@ MODEL_CLASSES = { 'xlnet': (XLNetConfig, TFXLNetLMHeadModel, load_xlnet_pt_weights_in_tf2, XLNetLMHeadModel, XLNET_PRETRAINED_MODEL_ARCHIVE_MAP, XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP), 'xlm': (XLMConfig, TFXLMWithLMHeadModel, load_xlm_pt_weights_in_tf2, XLMWithLMHeadModel, XLM_PRETRAINED_MODEL_ARCHIVE_MAP, XLM_PRETRAINED_CONFIG_ARCHIVE_MAP), 'transfo-xl': (TransfoXLConfig, TFTransfoXLLMHeadModel, load_transfo_xl_pt_weights_in_tf2, TransfoXLLMHeadModel, TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP, TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP), + 'openai-gpt': (OpenAIGPTConfig, TFOpenAIGPTLMHeadModel, load_openai_gpt_pt_weights_in_tf2, OpenAIGPTLMHeadModel, OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP, OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP), + 'roberta': (RobertaConfig, TFRobertaLMHead, load_roberta_pt_weights_in_tf2, RobertaForMaskedLM, ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP, ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP), + 'distilbert': (DistilBertConfig, TFDistilBertForMaskedLM, load_distilbert_pt_weights_in_tf2, DistilBertForMaskedLM, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP), } def convert_pt_checkpoint_to_tf(model_type, pytorch_checkpoint_path, config_file, tf_dump_path, compare_with_pt_model=False): From 8ba44ced95a4b2d9e3680ed8ea84bb13d06b2dd9 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 24 Sep 2019 09:48:23 +0200 Subject: [PATCH 096/219] fix roberta conversion script --- pytorch_transformers/__init__.py | 2 +- pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pytorch_transformers/__init__.py b/pytorch_transformers/__init__.py index befe5710e8..e12f4b2650 100644 --- a/pytorch_transformers/__init__.py +++ b/pytorch_transformers/__init__.py @@ -138,7 +138,7 @@ if _tf_available: TF_XLM_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_tf_roberta import (TFRobertaPreTrainedModel, TFRobertaMainLayer, - TFRobertaModel, TFRobertaLMHead, + TFRobertaModel, TFRobertaForMaskedLM, TFRobertaForSequenceClassification, load_roberta_pt_weights_in_tf2, TF_ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP) diff --git a/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py b/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py index 52a65ab607..567f1d0b5b 100644 --- a/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py +++ b/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py @@ -30,7 +30,7 @@ from pytorch_transformers import (BertConfig, TFBertForPreTraining, load_bert_pt XLMConfig, TFXLMWithLMHeadModel, load_xlm_pt_weights_in_tf2, XLM_PRETRAINED_CONFIG_ARCHIVE_MAP, TransfoXLConfig, TFTransfoXLLMHeadModel, load_transfo_xl_pt_weights_in_tf2, TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP, OpenAIGPTConfig, TFOpenAIGPTLMHeadModel, load_openai_gpt_pt_weights_in_tf2, OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP, - RobertaConfig, TFRobertaLMHead, load_roberta_pt_weights_in_tf2, ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, + RobertaConfig, TFRobertaForMaskedLM, load_roberta_pt_weights_in_tf2, ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, DistilBertConfig, TFDistilBertForMaskedLM, load_distilbert_pt_weights_in_tf2, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP) if is_torch_available(): @@ -73,7 +73,7 @@ MODEL_CLASSES = { 'xlm': (XLMConfig, TFXLMWithLMHeadModel, load_xlm_pt_weights_in_tf2, XLMWithLMHeadModel, XLM_PRETRAINED_MODEL_ARCHIVE_MAP, XLM_PRETRAINED_CONFIG_ARCHIVE_MAP), 'transfo-xl': (TransfoXLConfig, TFTransfoXLLMHeadModel, load_transfo_xl_pt_weights_in_tf2, TransfoXLLMHeadModel, TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP, TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP), 'openai-gpt': (OpenAIGPTConfig, TFOpenAIGPTLMHeadModel, load_openai_gpt_pt_weights_in_tf2, OpenAIGPTLMHeadModel, OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP, OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP), - 'roberta': (RobertaConfig, TFRobertaLMHead, load_roberta_pt_weights_in_tf2, RobertaForMaskedLM, ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP, ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP), + 'roberta': (RobertaConfig, TFRobertaForMaskedLM, load_roberta_pt_weights_in_tf2, RobertaForMaskedLM, ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP, ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP), 'distilbert': (DistilBertConfig, TFDistilBertForMaskedLM, load_distilbert_pt_weights_in_tf2, DistilBertForMaskedLM, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP), } From a7e01a248b4232fa1bfa62c3a0d86d3d09efb281 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 24 Sep 2019 10:58:52 +0200 Subject: [PATCH 097/219] converting distilled/fine-tuned models --- pytorch_transformers/__init__.py | 2 +- .../convert_pytorch_checkpoint_to_tf2.py | 103 ++++++++++++------ .../modeling_tf_distilbert.py | 6 +- .../modeling_tf_pytorch_utils.py | 18 ++- 4 files changed, 90 insertions(+), 39 deletions(-) diff --git a/pytorch_transformers/__init__.py b/pytorch_transformers/__init__.py index e12f4b2650..907115f70d 100644 --- a/pytorch_transformers/__init__.py +++ b/pytorch_transformers/__init__.py @@ -146,7 +146,7 @@ if _tf_available: from .modeling_tf_distilbert import (TFDistilBertPreTrainedModel, TFDistilBertMainLayer, TFDistilBertModel, TFDistilBertForMaskedLM, TFDistilBertForSequenceClassification, - TFDistilBertForSequenceClassification, + TFDistilBertForQuestionAnswering, load_distilbert_pt_weights_in_tf2, TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP) diff --git a/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py b/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py index 567f1d0b5b..6c0043d6a7 100644 --- a/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py +++ b/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py @@ -24,43 +24,43 @@ import tensorflow as tf from pytorch_transformers import is_torch_available, cached_path -from pytorch_transformers import (BertConfig, TFBertForPreTraining, load_bert_pt_weights_in_tf2, BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, +from pytorch_transformers import (BertConfig, TFBertForPreTraining, TFBertForQuestionAnswering, TFBertForSequenceClassification, load_bert_pt_weights_in_tf2, BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, GPT2Config, TFGPT2LMHeadModel, load_gpt2_pt_weights_in_tf2, GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP, XLNetConfig, TFXLNetLMHeadModel, load_xlnet_pt_weights_in_tf2, XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP, XLMConfig, TFXLMWithLMHeadModel, load_xlm_pt_weights_in_tf2, XLM_PRETRAINED_CONFIG_ARCHIVE_MAP, TransfoXLConfig, TFTransfoXLLMHeadModel, load_transfo_xl_pt_weights_in_tf2, TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP, OpenAIGPTConfig, TFOpenAIGPTLMHeadModel, load_openai_gpt_pt_weights_in_tf2, OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP, - RobertaConfig, TFRobertaForMaskedLM, load_roberta_pt_weights_in_tf2, ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, - DistilBertConfig, TFDistilBertForMaskedLM, load_distilbert_pt_weights_in_tf2, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP) + RobertaConfig, TFRobertaForMaskedLM, TFRobertaForSequenceClassification, load_roberta_pt_weights_in_tf2, ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, + DistilBertConfig, TFDistilBertForMaskedLM, TFDistilBertForQuestionAnswering, load_distilbert_pt_weights_in_tf2, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP) if is_torch_available(): import torch import numpy as np - from pytorch_transformers import (BertForPreTraining, BERT_PRETRAINED_MODEL_ARCHIVE_MAP, + from pytorch_transformers import (BertForPreTraining, BertForQuestionAnswering, BertForSequenceClassification, BERT_PRETRAINED_MODEL_ARCHIVE_MAP, GPT2LMHeadModel, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP, XLNetLMHeadModel, XLNET_PRETRAINED_MODEL_ARCHIVE_MAP, XLMWithLMHeadModel, XLM_PRETRAINED_MODEL_ARCHIVE_MAP, TransfoXLLMHeadModel, TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP, OpenAIGPTLMHeadModel, OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP, - RobertaForMaskedLM, ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP, - DistilBertForMaskedLM, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP) + RobertaForMaskedLM, RobertaForSequenceClassification, ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP, + DistilBertForMaskedLM, DistilBertForQuestionAnswering, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP) else: - (BertForPreTraining, BERT_PRETRAINED_MODEL_ARCHIVE_MAP, + (BertForPreTraining, BertForQuestionAnswering, BertForSequenceClassification, BERT_PRETRAINED_MODEL_ARCHIVE_MAP, GPT2LMHeadModel, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP, XLNetLMHeadModel, XLNET_PRETRAINED_MODEL_ARCHIVE_MAP, XLMWithLMHeadModel, XLM_PRETRAINED_MODEL_ARCHIVE_MAP, TransfoXLLMHeadModel, TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP, OpenAIGPTLMHeadModel, OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP, - RobertaForMaskedLM, ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP, - DistilBertForMaskedLM, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP,) = ( + RobertaForMaskedLM, RobertaForSequenceClassification, ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP, + DistilBertForMaskedLM, DistilBertForQuestionAnswering, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP,) = ( + None, None, None, None, None, None, None, None, None, None, None, None, None, None, - None, None, - None, None, - None, None,) + None, None, None, + None, None, None,) import logging @@ -68,22 +68,29 @@ logging.basicConfig(level=logging.INFO) MODEL_CLASSES = { 'bert': (BertConfig, TFBertForPreTraining, load_bert_pt_weights_in_tf2, BertForPreTraining, BERT_PRETRAINED_MODEL_ARCHIVE_MAP, BERT_PRETRAINED_CONFIG_ARCHIVE_MAP), + 'bert-large-uncased-whole-word-masking-finetuned-squad': (BertConfig, TFBertForQuestionAnswering, load_bert_pt_weights_in_tf2, BertForQuestionAnswering, BERT_PRETRAINED_MODEL_ARCHIVE_MAP, BERT_PRETRAINED_CONFIG_ARCHIVE_MAP), + 'bert-large-cased-whole-word-masking-finetuned-squad': (BertConfig, TFBertForQuestionAnswering, load_bert_pt_weights_in_tf2, BertForQuestionAnswering, BERT_PRETRAINED_MODEL_ARCHIVE_MAP, BERT_PRETRAINED_CONFIG_ARCHIVE_MAP), + 'bert-base-cased-finetuned-mrpc': (BertConfig, TFBertForSequenceClassification, load_bert_pt_weights_in_tf2, BertForSequenceClassification, BERT_PRETRAINED_MODEL_ARCHIVE_MAP, BERT_PRETRAINED_CONFIG_ARCHIVE_MAP), 'gpt2': (GPT2Config, TFGPT2LMHeadModel, load_gpt2_pt_weights_in_tf2, GPT2LMHeadModel, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP, GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP), 'xlnet': (XLNetConfig, TFXLNetLMHeadModel, load_xlnet_pt_weights_in_tf2, XLNetLMHeadModel, XLNET_PRETRAINED_MODEL_ARCHIVE_MAP, XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP), 'xlm': (XLMConfig, TFXLMWithLMHeadModel, load_xlm_pt_weights_in_tf2, XLMWithLMHeadModel, XLM_PRETRAINED_MODEL_ARCHIVE_MAP, XLM_PRETRAINED_CONFIG_ARCHIVE_MAP), 'transfo-xl': (TransfoXLConfig, TFTransfoXLLMHeadModel, load_transfo_xl_pt_weights_in_tf2, TransfoXLLMHeadModel, TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP, TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP), 'openai-gpt': (OpenAIGPTConfig, TFOpenAIGPTLMHeadModel, load_openai_gpt_pt_weights_in_tf2, OpenAIGPTLMHeadModel, OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP, OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP), 'roberta': (RobertaConfig, TFRobertaForMaskedLM, load_roberta_pt_weights_in_tf2, RobertaForMaskedLM, ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP, ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP), + 'roberta-large-mnli': (RobertaConfig, TFRobertaForSequenceClassification, load_roberta_pt_weights_in_tf2, RobertaForSequenceClassification, ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP, ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP), 'distilbert': (DistilBertConfig, TFDistilBertForMaskedLM, load_distilbert_pt_weights_in_tf2, DistilBertForMaskedLM, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP), + 'distilbert-base-uncased-distilled-squad': (DistilBertConfig, TFDistilBertForQuestionAnswering, load_distilbert_pt_weights_in_tf2, DistilBertForQuestionAnswering, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP), } -def convert_pt_checkpoint_to_tf(model_type, pytorch_checkpoint_path, config_file, tf_dump_path, compare_with_pt_model=False): +def convert_pt_checkpoint_to_tf(model_type, pytorch_checkpoint_path, config_file, tf_dump_path, compare_with_pt_model=False, use_cached_models=True): if model_type not in MODEL_CLASSES: raise ValueError("Unrecognized model type, should be one of {}.".format(list(MODEL_CLASSES.keys()))) config_class, model_class, loading_fct, pt_model_class, aws_model_maps, aws_config_map = MODEL_CLASSES[model_type] # Initialise TF model + if config_file in aws_config_map: + config_file = cached_path(aws_config_map[config_file], force_download=not use_cached_models) config = config_class.from_json_file(config_file) config.output_hidden_states = True config.output_attentions = True @@ -91,6 +98,8 @@ def convert_pt_checkpoint_to_tf(model_type, pytorch_checkpoint_path, config_file tf_model = model_class(config) # Load weights from tf checkpoint + if pytorch_checkpoint_path in aws_model_maps: + pytorch_checkpoint_path = cached_path(aws_model_maps[pytorch_checkpoint_path], force_download=not use_cached_models) tf_model = loading_fct(tf_model, pytorch_checkpoint_path) if compare_with_pt_model: @@ -117,7 +126,8 @@ def convert_pt_checkpoint_to_tf(model_type, pytorch_checkpoint_path, config_file tf_model.save_weights(tf_dump_path, save_format='h5') -def convert_all_pt_checkpoints_to_tf(args_model_type, tf_dump_path, compare_with_pt_model=False, use_cached_models=False): +def convert_all_pt_checkpoints_to_tf(args_model_type, tf_dump_path, model_shortcut_names_or_path=None, config_shortcut_names_or_path=None, + compare_with_pt_model=False, use_cached_models=False, only_convert_finetuned_models=False): assert os.path.isdir(args.tf_dump_path), "--tf_dump_path should be a directory" if args_model_type is None: @@ -134,20 +144,39 @@ def convert_all_pt_checkpoints_to_tf(args_model_type, tf_dump_path, compare_with config_class, model_class, loading_fct, pt_model_class, aws_model_maps, aws_config_map = MODEL_CLASSES[model_type] - for i, shortcut_name in enumerate(aws_config_map.keys(), start=1): + if model_shortcut_names_or_path is None: + model_shortcut_names_or_path = list(aws_model_maps.keys()) + if config_shortcut_names_or_path is None: + config_shortcut_names_or_path = model_shortcut_names_or_path + + for i, (model_shortcut_name, config_shortcut_name) in enumerate( + zip(model_shortcut_names_or_path, config_shortcut_names_or_path), start=1): print("-" * 100) - print(" Converting checkpoint {}/{}: {}".format(i, len(aws_config_map), shortcut_name)) - print("-" * 100) - if 'finetuned' in shortcut_name: - print(" Skipping finetuned checkpoint ") + if '-squad' in model_shortcut_name or '-mrpc' in model_shortcut_name or '-mnli' in model_shortcut_name: + if not only_convert_finetuned_models: + print(" Skipping finetuned checkpoint {}".format(model_shortcut_name)) + continue + model_type = model_shortcut_name + elif only_convert_finetuned_models: + print(" Skipping not finetuned checkpoint {}".format(model_shortcut_name)) continue - config_file = cached_path(aws_config_map[shortcut_name], force_download=not use_cached_models) - model_file = cached_path(aws_model_maps[shortcut_name], force_download=not use_cached_models) + print(" Converting checkpoint {}/{}: {} - model_type {}".format(i, len(aws_config_map), model_shortcut_name, model_type)) + print("-" * 100) + + if config_shortcut_name in aws_config_map: + config_file = cached_path(aws_config_map[config_shortcut_name], force_download=not use_cached_models) + else: + config_file = cached_path(config_shortcut_name, force_download=not use_cached_models) + + if model_shortcut_name in aws_model_maps: + model_file = cached_path(aws_model_maps[model_shortcut_name], force_download=not use_cached_models) + else: + model_file = cached_path(model_shortcut_name, force_download=not use_cached_models) convert_pt_checkpoint_to_tf(model_type, model_file, config_file, - os.path.join(tf_dump_path, shortcut_name + '-tf_model.h5'), + os.path.join(tf_dump_path, model_shortcut_name + '-tf_model.h5'), compare_with_pt_model=compare_with_pt_model) os.remove(config_file) os.remove(model_file) @@ -176,23 +205,29 @@ if __name__ == "__main__": help = "The config json file corresponding to the pre-trained model. \n" "This specifies the model architecture. If not given and " "--pytorch_checkpoint_path is not given or is a shortcut name" - "use the configuration associated to teh shortcut name on the AWS") + "use the configuration associated to the shortcut name on the AWS") parser.add_argument("--compare_with_pt_model", action='store_true', help = "Compare Tensorflow and PyTorch model predictions.") parser.add_argument("--use_cached_models", action='store_true', help = "Use cached models if possible instead of updating to latest checkpoint versions.") + parser.add_argument("--only_convert_finetuned_models", + action='store_true', + help = "Only convert finetuned models.") args = parser.parse_args() - if args.pytorch_checkpoint_path is not None: - convert_pt_checkpoint_to_tf(args.model_type.lower(), - args.pytorch_checkpoint_path, - args.config_file, - args.tf_dump_path, - compare_with_pt_model=args.compare_with_pt_model) - else: - convert_all_pt_checkpoints_to_tf(args.model_type.lower() if args.model_type is not None else None, - args.tf_dump_path, - compare_with_pt_model=args.compare_with_pt_model, - use_cached_models=args.use_cached_models) + # if args.pytorch_checkpoint_path is not None: + # convert_pt_checkpoint_to_tf(args.model_type.lower(), + # args.pytorch_checkpoint_path, + # args.config_file if args.config_file is not None else args.pytorch_checkpoint_path, + # args.tf_dump_path, + # compare_with_pt_model=args.compare_with_pt_model, + # use_cached_models=args.use_cached_models) + # else: + convert_all_pt_checkpoints_to_tf(args.model_type.lower() if args.model_type is not None else None, + args.tf_dump_path, + model_shortcut_names_or_path=[args.pytorch_checkpoint_path] if args.pytorch_checkpoint_path is not None else None, + compare_with_pt_model=args.compare_with_pt_model, + use_cached_models=args.use_cached_models, + only_convert_finetuned_models=args.only_convert_finetuned_models) diff --git a/pytorch_transformers/modeling_tf_distilbert.py b/pytorch_transformers/modeling_tf_distilbert.py index 706d2fc02d..1811573bbf 100644 --- a/pytorch_transformers/modeling_tf_distilbert.py +++ b/pytorch_transformers/modeling_tf_distilbert.py @@ -653,7 +653,7 @@ class TFDistilBertForSequenceClassification(TFDistilBertPreTrainedModel): super(TFDistilBertForSequenceClassification, self).__init__(config, *inputs, **kwargs) self.num_labels = config.num_labels - self.distilbert = TFDistilBertModel(config, name="distilbert") + self.distilbert = TFDistilBertMainLayer(config, name="distilbert") self.pre_classifier = tf.keras.layers.Dense(config.dim, activation='relu', name="pre_classifier") self.classifier = tf.keras.layers.Dense(config.num_labels, name="classifier") self.dropout = tf.keras.layers.Dropout(config.seq_classif_dropout) @@ -714,8 +714,8 @@ class TFDistilBertForQuestionAnswering(TFDistilBertPreTrainedModel): def __init__(self, config, *inputs, **kwargs): super(TFDistilBertForQuestionAnswering, self).__init__(config, *inputs, **kwargs) - self.distilbert = TFDistilBertModel(config, name="distilbert") - self.qa_outputs = tf.keras.layers.Dense(config.num_labels, name='qa_output') + self.distilbert = TFDistilBertMainLayer(config, name="distilbert") + self.qa_outputs = tf.keras.layers.Dense(config.num_labels, name='qa_outputs') assert config.num_labels == 2 self.dropout = tf.keras.layers.Dropout(config.qa_dropout) diff --git a/pytorch_transformers/modeling_tf_pytorch_utils.py b/pytorch_transformers/modeling_tf_pytorch_utils.py index 9950a5a73f..12e5023802 100644 --- a/pytorch_transformers/modeling_tf_pytorch_utils.py +++ b/pytorch_transformers/modeling_tf_pytorch_utils.py @@ -148,8 +148,24 @@ def load_tf2_checkpoint_in_pytorch_model(pt_model, tf_checkpoint_path): """ Load TF 2.0 HDF5 checkpoint in a PyTorch model We use HDF5 to easily do transfer learning (see https://github.com/tensorflow/tensorflow/blob/ee16fcac960ae660e0e4496658a366e2f745e1f0/tensorflow/python/keras/engine/network.py#L1352-L1357). + Conventions for TF2.0 scopes -> PyTorch attribute names conversions: + - '$1___$2' is replaced by $2 (can be used to duplicate or remove layers in TF2.0 vs PyTorch) + - '_._' is replaced by a new level separation (can be used to convert TF2.0 lists in PyTorch nn.ModulesList) """ - raise NotImplementedError + try: + import tensorflow as tf + import torch + except ImportError as e: + logger.error("Loading a TensorFlow model in PyTorch, requires both PyTorch and TensorFlow to be installed. Please see " + "https://pytorch.org/ and https://www.tensorflow.org/install/ for installation instructions.") + raise e + + tf_path = os.path.abspath(tf_checkpoint_path) + logger.info("Loading TensorFlow weights from {}".format(tf_path)) + + tf_state_dict = torch.load(tf_path, map_location='cpu') + + return load_tf2_weights_in_pytorch_model(pt_model, tf_state_dict) def load_tf2_weights_in_pytorch_model(pt_model, tf_model): """ Load TF2.0 symbolic weights in a PyTorch model From 9d44236f7021028a0581345b3917589320e479b8 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Tue, 24 Sep 2019 07:03:24 -0400 Subject: [PATCH 098/219] Updated DistilBERT --- examples/utils_glue.py | 2 +- .../tests/tokenization_tests_commons.py | 12 +- pytorch_transformers/tokenization_utils.py | 146 +++++++++++------- 3 files changed, 98 insertions(+), 62 deletions(-) diff --git a/examples/utils_glue.py b/examples/utils_glue.py index e2fc3a119a..efe42189e4 100644 --- a/examples/utils_glue.py +++ b/examples/utils_glue.py @@ -412,7 +412,7 @@ def convert_examples_to_features(examples, label_list, max_seq_length, output_mask=True, max_length=max_seq_length ) - input_ids, segment_ids = inputs["sequence"], inputs["mask"] + input_ids, segment_ids = inputs["input_ids"], inputs["output_token_type"] # The mask has 1 for real tokens and 0 for padding tokens. Only real # tokens are attended to. diff --git a/pytorch_transformers/tests/tokenization_tests_commons.py b/pytorch_transformers/tests/tokenization_tests_commons.py index 8a3b56a058..1f84d36e7d 100644 --- a/pytorch_transformers/tests/tokenization_tests_commons.py +++ b/pytorch_transformers/tests/tokenization_tests_commons.py @@ -196,8 +196,8 @@ class CommonTestCases: if tokenizer.add_special_tokens_sequence_pair.__qualname__.split('.')[0] != "PreTrainedTokenizer": seq_0 = "Test this method." seq_1 = "With these inputs." - information = tokenizer.encode_plus(seq_0, seq_1, add_special_tokens=True, output_mask=True) - sequences, mask = information["sequence"], information["mask"] + information = tokenizer.encode_plus(seq_0, seq_1, add_special_tokens=True, output_token_type=True) + sequences, mask = information["input_ids"], information["output_token_type"] assert len(sequences) == len(mask) def test_number_of_added_tokens(self): @@ -224,7 +224,7 @@ class CommonTestCases: total_length = len(sequence) + num_added_tokens information = tokenizer.encode_plus(seq_0, max_length=total_length - 2, add_special_tokens=True, stride=stride) - truncated_sequence = information["sequence"] + truncated_sequence = information["input_ids"] overflowing_tokens = information["overflowing_tokens"] assert len(overflowing_tokens) == 2 + stride @@ -249,12 +249,12 @@ class CommonTestCases: ) information = tokenizer.encode_plus(seq_0, seq_1, max_length=len(sequence) - 2, add_special_tokens=True, - stride=stride) + stride=stride, truncate_first_sequence=False) information_first_truncated = tokenizer.encode_plus(seq_0, seq_1, max_length=len(sequence) - 2, add_special_tokens=True, stride=stride, - truncate_second_sequence_first=False) + truncate_first_sequence=True) - truncated_sequence = information["sequence"] + truncated_sequence = information["input_ids"] overflowing_tokens = information["overflowing_tokens"] overflowing_tokens_first_truncated = information_first_truncated["overflowing_tokens"] diff --git a/pytorch_transformers/tokenization_utils.py b/pytorch_transformers/tokenization_utils.py index f2cb383143..1209c60de5 100644 --- a/pytorch_transformers/tokenization_utils.py +++ b/pytorch_transformers/tokenization_utils.py @@ -536,13 +536,7 @@ class PreTrainedTokenizer(object): if pair: initial_tokens_len = len(self.encode("This is a sequence") + self.encode("This is another")) - final_tokens = self.encode("This is a sequence", "This is another", add_special_tokens=True) - - # In some models (e.g. GPT-2), there is no sequence pair encoding. - if len(final_tokens) == 2: - return 0 - else: - final_tokens_len = len(final_tokens) + final_tokens_len = len(self.encode("This is a sequence", "This is another", add_special_tokens=True)) else: initial_tokens_len = len(self.encode("This is a sequence")) final_tokens_len = len(self.encode("This is a sequence", add_special_tokens=True)) @@ -700,86 +694,93 @@ class PreTrainedTokenizer(object): Same as doing ``self.convert_tokens_to_ids(self.tokenize(text))``. Args: - text: The first sequence to be encoded. - text_pair: Optional second sequence to be encoded. + text: The first sequence to be encoded. This can be a string, a list of strings (tokenized string using + the `tokenize` method) or a list of integers (tokenized string ids using the `convert_tokens_to_ids` + method) + text_pair: Optional second sequence to be encoded. This can be a string, a list of strings (tokenized + string using the `tokenize` method) or a list of integers (tokenized string ids using the + `convert_tokens_to_ids` method) add_special_tokens: if set to ``True``, the sequences will be encoded with the special tokens relative to their model. + **kwargs: passed to the `self.tokenize()` method """ - if text_pair is None: - if add_special_tokens: - sequence_tokens = self.convert_tokens_to_ids(self.tokenize(text, **kwargs)) if isinstance(text, six.string_types) else text - return self.add_special_tokens_single_sequence(sequence_tokens) - else: - ids = self.convert_tokens_to_ids(self.tokenize(text, **kwargs)) if isinstance(text, six.string_types) else text - return ids - - first_sentence_tokens = [self._convert_token_to_id(token) for token in self.tokenize(text, **kwargs)] if isinstance(text, six.string_types) else text - second_sentence_tokens = [self._convert_token_to_id(token) for token in self.tokenize(text_pair, **kwargs)] if isinstance(text_pair, six.string_types) else text_pair - - if add_special_tokens: - return self.add_special_tokens_sequence_pair(first_sentence_tokens, second_sentence_tokens) - else: - logger.warning("No special tokens were added. The two sequences have been concatenated.") - return first_sentence_tokens + second_sentence_tokens + return self.encode_plus(text, text_pair, add_special_tokens, **kwargs)["input_ids"] def encode_plus(self, text, text_pair=None, add_special_tokens=False, - output_mask=False, + output_token_type=False, max_length=None, stride=0, - truncate_second_sequence_first=True, + truncate_first_sequence=True, **kwargs): """ Returns a dictionary containing the encoded sequence or sequence pair. Other values can be returned by this method: the mask for sequence classification and the overflowing elements if a ``max_length`` is specified. Args: - text: The first sequence to be encoded. - text_pair: Optional second sequence to be encoded. + text: The first sequence to be encoded. This can be a string, a list of strings (tokenized string using + the `tokenize` method) or a list of integers (tokenized string ids using the `convert_tokens_to_ids` + method) + text_pair: Optional second sequence to be encoded. This can be a string, a list of strings (tokenized + string using the `tokenize` method) or a list of integers (tokenized string ids using the + `convert_tokens_to_ids` method) add_special_tokens: if set to ``True``, the sequences will be encoded with the special tokens relative to their model. - output_mask: if set to ``True``, returns the text pair corresponding mask with 0 for the first sequence, + output_token_type: if set to ``True``, returns the text pair corresponding mask with 0 for the first sequence, and 1 for the second. max_length: if set to a number, will limit the total sequence returned so that it has a maximum length. If there are overflowing tokens, those will be added to the returned dictionary stride: if set to a number along with max_length, the overflowing tokens returned will contain some tokens from the main sequence returned. The value of this argument defined the number of additional tokens. - truncate_second_sequence_first: if there is a specified max_length, this flag will choose which sequence + truncate_first_sequence: if there is a specified max_length, this flag will choose which sequence will be truncated. **kwargs: passed to the `self.tokenize()` method """ information = {} + def get_input_ids(text): + if isinstance(text, six.string_types): + input_ids = self.convert_tokens_to_ids(self.tokenize(text, **kwargs)) + elif isinstance(text, (list, tuple)) and len(text) > 0 and isinstance(text[0], str): + input_ids = self.convert_tokens_to_ids(text) + elif isinstance(text, (list, tuple)) and len(text) > 0 and isinstance(text[0], int): + input_ids = text + else: + raise ValueError("Input is not valid. Should be a string, a list/tuple of strings or a list/tuple of integers.") + + return input_ids + if text_pair is None: - sequence_tokens = self.convert_tokens_to_ids(self.tokenize(text, **kwargs)) if isinstance(text, six.string_types) else text + sequence_tokens = get_input_ids(text) + if add_special_tokens: - information = self.prepare_for_model(sequence_tokens, max_length, stride) + information = self.prepare_for_model(sequence_tokens, max_length=max_length, stride=stride) else: if max_length: information["overflowing_tokens"] = sequence_tokens[max_length - stride:] sequence_tokens = sequence_tokens[:max_length] - information["sequence"] = sequence_tokens + information["input_ids"] = sequence_tokens - if output_mask: - information["mask"] = [0] * len(information["sequence"]) + if output_token_type: + information["output_token_type"] = [0] * len(information["input_ids"]) else: - first_sentence_tokens = [self._convert_token_to_id(token) for token in self.tokenize(text, **kwargs)] if isinstance(text, six.string_types) else text - second_sentence_tokens = [self._convert_token_to_id(token) for token in self.tokenize(text_pair, **kwargs)] if isinstance(text_pair, six.string_types) else text_pair + first_sentence_tokens = get_input_ids(text) + second_sentence_tokens = get_input_ids(text_pair) if add_special_tokens: information = self.prepare_pair_for_model( first_sentence_tokens, second_sentence_tokens, - max_length, - truncate_second_sequence_first, - stride + max_length=max_length, + truncate_first_sequence=truncate_first_sequence, + stride=stride ) - if output_mask: - information["mask"] = self.create_mask_from_sequences(text, text_pair) + if output_token_type: + information["output_token_type"] = self.create_mask_from_sequences(text, text_pair) else: logger.warning("No special tokens were added. The two sequences have been concatenated.") sequence = first_sentence_tokens + second_sentence_tokens @@ -787,43 +788,78 @@ class PreTrainedTokenizer(object): if max_length: information["overflowing_tokens"] = sequence[max_length - stride:] sequence = sequence[:max_length] - if output_mask: - information["mask"] = [0] * len(sequence) + if output_token_type: + information["output_token_type"] = [0] * len(sequence) - information["sequence"] = sequence + information["input_ids"] = sequence return information def prepare_for_model(self, ids, max_length=None, stride=0): + """ + Prepares a list of tokenized input ids so that it can be used by the model. It adds special tokens, truncates + sequences if overflowing while taking into account the special tokens and manages a window stride for + overflowing tokens + + Args: + ids: list of tokenized input ids. Can be obtained from a string by chaining the + `tokenize` and `convert_tokens_to_ids` methods. + max_length: maximum length of the returned list. Will truncate by taking into account the special tokens. + stride: window stride for overflowing tokens. Can be useful for edge effect removal when using sequential + list of inputs. + + Return: + a dictionary containing the `input_ids` as well as the `overflowing_tokens` if a `max_length` was given. + """ information = {} - n_added_tokens = self.num_added_tokens() if max_length: + n_added_tokens = self.num_added_tokens() information["overflowing_tokens"] = ids[max_length - n_added_tokens - stride:] ids = ids[:max_length - n_added_tokens] - information["sequence"] = self.add_special_tokens_single_sequence(ids) + information["input_ids"] = self.add_special_tokens_single_sequence(ids) return information - def prepare_pair_for_model(self, ids_0, ids_1, max_length=None, truncate_second_sequence_first=True, stride=0): + def prepare_pair_for_model(self, ids_0, ids_1, max_length=None, truncate_first_sequence=True, stride=0): + """ + Prepares a list of tokenized input ids pair so that it can be used by the model. It adds special tokens, + truncates sequences if overflowing while taking into account the special tokens and manages a window stride for + overflowing tokens + + Args: + ids_0: list of tokenized input ids. Can be obtained from a string by chaining the + `tokenize` and `convert_tokens_to_ids` methods. + ids_1: second list of tokenized input ids. Can be obtained from a string by chaining the + `tokenize` and `convert_tokens_to_ids` methods. + max_length: maximum length of the returned list. Will truncate by taking into account the special tokens. + truncate_first_sequence: if set to `True`, alongside a specified `max_length`, will truncate the first + sequence if the total size is superior than the specified `max_length`. If set to `False`, will + truncate the second sequence instead. + stride: window stride for overflowing tokens. Can be useful for edge effect removal when using sequential + list of inputs. + + Return: + a dictionary containing the `input_ids` as well as the `overflowing_tokens` if a `max_length` was given. + """ f_len, s_len = len(ids_0), len(ids_1) - n_added_tokens = self.num_added_tokens(pair=True) information = {} if max_length: + n_added_tokens = self.num_added_tokens(pair=True) if len(ids_0) + n_added_tokens >= max_length: logger.warning( "The first sequence is longer than the maximum specified length. This sequence will not be truncated.") else: if f_len + s_len + self.num_added_tokens(pair=True) > max_length: - if truncate_second_sequence_first: - information["overflowing_tokens"] = ids_1[max_length - f_len - n_added_tokens - stride:] - ids_1 = ids_1[:max_length - f_len - n_added_tokens] - else: + if truncate_first_sequence: information["overflowing_tokens"] = ids_0[max_length - s_len - n_added_tokens - stride:] ids_0 = ids_0[:max_length - s_len - n_added_tokens] + else: + information["overflowing_tokens"] = ids_1[max_length - f_len - n_added_tokens - stride:] + ids_1 = ids_1[:max_length - f_len - n_added_tokens] sequence = self.add_special_tokens_sequence_pair(ids_0, ids_1) - information["sequence"] = sequence + information["input_ids"] = sequence return information From 0ea82b246f4a587295939b3621ce78b3d8e2ee60 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Tue, 24 Sep 2019 07:10:09 -0400 Subject: [PATCH 099/219] Updated tests --- .../tests/tokenization_tests_commons.py | 11 ++++++++--- pytorch_transformers/tokenization_utils.py | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/pytorch_transformers/tests/tokenization_tests_commons.py b/pytorch_transformers/tests/tokenization_tests_commons.py index 1f84d36e7d..323e558310 100644 --- a/pytorch_transformers/tests/tokenization_tests_commons.py +++ b/pytorch_transformers/tests/tokenization_tests_commons.py @@ -264,9 +264,14 @@ class CommonTestCases: assert len(truncated_sequence) == len(sequence) - 2 assert truncated_sequence == truncated_second_sequence - def test_tokens_sent_to_encode(self): + def test_encode_input_type(self): tokenizer = self.get_tokenizer() sequence = "Let's encode this sequence" - tokens = tokenizer.encode(sequence) - tokenizer.encode(tokens, add_special_tokens=True) + + tokens = tokenizer.tokenize(sequence) + input_ids = tokenizer.convert_tokens_to_ids(tokens) + formatted_input = tokenizer.encode(sequence, add_special_tokens=True) + + assert tokenizer.encode(tokens, add_special_tokens=True) == formatted_input + assert tokenizer.encode(input_ids, add_special_tokens=True) == formatted_input diff --git a/pytorch_transformers/tokenization_utils.py b/pytorch_transformers/tokenization_utils.py index 1209c60de5..478ba6da87 100644 --- a/pytorch_transformers/tokenization_utils.py +++ b/pytorch_transformers/tokenization_utils.py @@ -744,7 +744,7 @@ class PreTrainedTokenizer(object): def get_input_ids(text): if isinstance(text, six.string_types): input_ids = self.convert_tokens_to_ids(self.tokenize(text, **kwargs)) - elif isinstance(text, (list, tuple)) and len(text) > 0 and isinstance(text[0], str): + elif isinstance(text, (list, tuple)) and len(text) > 0 and isinstance(text[0], six.string_types): input_ids = self.convert_tokens_to_ids(text) elif isinstance(text, (list, tuple)) and len(text) > 0 and isinstance(text[0], int): input_ids = text From 3927d7756cb8e45f54a8c080a61fc37a9ba524ca Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Tue, 24 Sep 2019 07:15:11 -0400 Subject: [PATCH 100/219] Updated the GLUE pre-processing method --- examples/utils_glue.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/utils_glue.py b/examples/utils_glue.py index efe42189e4..225b496029 100644 --- a/examples/utils_glue.py +++ b/examples/utils_glue.py @@ -409,8 +409,9 @@ def convert_examples_to_features(examples, label_list, max_seq_length, example.text_a, example.text_b, add_special_tokens=True, - output_mask=True, - max_length=max_seq_length + output_token_type=True, + max_length=max_seq_length, + truncate_first_sequence=True # We're truncating the first sequence as a priority ) input_ids, segment_ids = inputs["input_ids"], inputs["output_token_type"] From c832f43a4dc01fbf55fa63eda1554912f01ac57a Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Tue, 24 Sep 2019 07:21:38 -0400 Subject: [PATCH 101/219] `output_token_type` -> `token_type_ids` --- examples/utils_glue.py | 2 +- pytorch_transformers/tests/tokenization_tests_commons.py | 2 +- pytorch_transformers/tokenization_utils.py | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/utils_glue.py b/examples/utils_glue.py index 225b496029..2557540cc6 100644 --- a/examples/utils_glue.py +++ b/examples/utils_glue.py @@ -413,7 +413,7 @@ def convert_examples_to_features(examples, label_list, max_seq_length, max_length=max_seq_length, truncate_first_sequence=True # We're truncating the first sequence as a priority ) - input_ids, segment_ids = inputs["input_ids"], inputs["output_token_type"] + input_ids, segment_ids = inputs["input_ids"], inputs["token_type_ids"] # The mask has 1 for real tokens and 0 for padding tokens. Only real # tokens are attended to. diff --git a/pytorch_transformers/tests/tokenization_tests_commons.py b/pytorch_transformers/tests/tokenization_tests_commons.py index 323e558310..4ad92c8192 100644 --- a/pytorch_transformers/tests/tokenization_tests_commons.py +++ b/pytorch_transformers/tests/tokenization_tests_commons.py @@ -197,7 +197,7 @@ class CommonTestCases: seq_0 = "Test this method." seq_1 = "With these inputs." information = tokenizer.encode_plus(seq_0, seq_1, add_special_tokens=True, output_token_type=True) - sequences, mask = information["input_ids"], information["output_token_type"] + sequences, mask = information["input_ids"], information["token_type_ids"] assert len(sequences) == len(mask) def test_number_of_added_tokens(self): diff --git a/pytorch_transformers/tokenization_utils.py b/pytorch_transformers/tokenization_utils.py index 478ba6da87..02b4bef699 100644 --- a/pytorch_transformers/tokenization_utils.py +++ b/pytorch_transformers/tokenization_utils.py @@ -765,7 +765,7 @@ class PreTrainedTokenizer(object): information["input_ids"] = sequence_tokens if output_token_type: - information["output_token_type"] = [0] * len(information["input_ids"]) + information["token_type_ids"] = [0] * len(information["input_ids"]) else: first_sentence_tokens = get_input_ids(text) second_sentence_tokens = get_input_ids(text_pair) @@ -780,7 +780,7 @@ class PreTrainedTokenizer(object): ) if output_token_type: - information["output_token_type"] = self.create_mask_from_sequences(text, text_pair) + information["token_type_ids"] = self.create_mask_from_sequences(text, text_pair) else: logger.warning("No special tokens were added. The two sequences have been concatenated.") sequence = first_sentence_tokens + second_sentence_tokens @@ -789,7 +789,7 @@ class PreTrainedTokenizer(object): information["overflowing_tokens"] = sequence[max_length - stride:] sequence = sequence[:max_length] if output_token_type: - information["output_token_type"] = [0] * len(sequence) + information["token_type_ids"] = [0] * len(sequence) information["input_ids"] = sequence From e9a103c17ac86ac45c7a64ef4e1410b933dd0ae3 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 24 Sep 2019 13:25:50 +0200 Subject: [PATCH 102/219] bidirectional conversion TF <=> PT - extended tests --- .circleci/config.yml | 15 ++ pytorch_transformers/__init__.py | 12 +- .../modeling_tf_pytorch_utils.py | 180 ++++++++++++++---- pytorch_transformers/modeling_xlm.py | 95 +++++++++ .../tests/modeling_tf_common_test.py | 21 +- .../tests/modeling_tf_xlm_test.py | 2 +- .../tests/modeling_xlm_test.py | 32 +++- 7 files changed, 313 insertions(+), 44 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c49cf7df8a..30a4458807 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,5 +1,20 @@ version: 2 jobs: + build_py3_torch_and_tf: + working_directory: ~/pytorch-transformers + docker: + - image: circleci/python:3.5 + resource_class: xlarge + parallelism: 1 + steps: + - checkout + - run: sudo pip install torch + - run: sudo pip install tensorflow==2.0.0-rc0 + - run: sudo pip install --progress-bar off . + - run: sudo pip install pytest codecov pytest-cov + - run: sudo pip install tensorboardX scikit-learn + - run: python -m pytest -sv ./pytorch_transformers/tests/ --cov + - run: codecov build_py3_torch: working_directory: ~/pytorch-transformers docker: diff --git a/pytorch_transformers/__init__.py b/pytorch_transformers/__init__.py index 907115f70d..b8c7eccfe7 100644 --- a/pytorch_transformers/__init__.py +++ b/pytorch_transformers/__init__.py @@ -73,7 +73,8 @@ if _torch_available: load_tf_weights_in_xlnet, XLNET_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_xlm import (XLMPreTrainedModel , XLMModel, XLMWithLMHeadModel, XLMForSequenceClassification, - XLMForQuestionAnswering, XLM_PRETRAINED_MODEL_ARCHIVE_MAP) + XLMForQuestionAnswering, XLMForQuestionAnsweringSimple, + XLM_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_roberta import (RobertaForMaskedLM, RobertaModel, RobertaForSequenceClassification, ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_distilbert import (DistilBertForMaskedLM, DistilBertModel, @@ -150,6 +151,15 @@ if _tf_available: load_distilbert_pt_weights_in_tf2, TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP) +if _tf_available and _torch_available: + from .modeling_tf_pytorch_utils import (convert_tf_weight_name_to_pt_weight_name, + load_pytorch_checkpoint_in_tf2_model, + load_pytorch_weights_in_tf2_model, + load_pytorch_model_in_tf2_model, + load_tf2_checkpoint_in_pytorch_model, + load_tf2_weights_in_pytorch_model, + load_tf2_model_in_pytorch_model) + # Files and general utilities from .file_utils import (PYTORCH_TRANSFORMERS_CACHE, PYTORCH_PRETRAINED_BERT_CACHE, cached_path, add_start_docstrings, add_end_docstrings, diff --git a/pytorch_transformers/modeling_tf_pytorch_utils.py b/pytorch_transformers/modeling_tf_pytorch_utils.py index 12e5023802..8e879e1447 100644 --- a/pytorch_transformers/modeling_tf_pytorch_utils.py +++ b/pytorch_transformers/modeling_tf_pytorch_utils.py @@ -20,15 +20,49 @@ from __future__ import (absolute_import, division, print_function, import logging import os +import re +import numpy logger = logging.getLogger(__name__) -def load_pytorch_checkpoint_in_tf2_model(tf_model, pytorch_checkpoint_path, tf_inputs=None): - """ Load pytorch checkpoints in a TF 2.0 model +def convert_tf_weight_name_to_pt_weight_name(tf_name, start_prefix_to_remove=''): + """ Convert a TF 2.0 model variable name in a pytorch model weight name. + Conventions for TF2.0 scopes -> PyTorch attribute names conversions: - '$1___$2' is replaced by $2 (can be used to duplicate or remove layers in TF2.0 vs PyTorch) - '_._' is replaced by a new level separation (can be used to convert TF2.0 lists in PyTorch nn.ModulesList) + + return tuple with: + - pytorch model weight name + - transpose: boolean indicating weither TF2.0 and PyTorch weights matrices are transposed with regards to each other + """ + tf_name = tf_name.replace(':0', '') # device ids + tf_name = re.sub(r'/[^/]*___([^/]*)/', r'/\1/', tf_name) # '$1___$2' is replaced by $2 (can be used to duplicate or remove layers in TF2.0 vs PyTorch) + tf_name = tf_name.replace('_._', '/') # '_._' is replaced by a level separation (can be used to convert TF2.0 lists in PyTorch nn.ModulesList) + tf_name = re.sub(r'//+', '/', tf_name) # Remove empty levels at the end + tf_name = tf_name.split('/') # Convert from TF2.0 '/' separators to PyTorch '.' separators + tf_name = tf_name[1:] # Remove level zero + + # When should we transpose the weights + transpose = bool(tf_name[-1] == 'kernel' or 'emb_projs' in tf_name or 'out_projs' in tf_name) + + # Convert standard TF2.0 names in PyTorch names + if tf_name[-1] == 'kernel' or tf_name[-1] == 'embeddings' or tf_name[-1] == 'gamma': + tf_name[-1] = 'weight' + if tf_name[-1] == 'beta': + tf_name[-1] = 'bias' + + # Remove prefix if needed + tf_name = '.'.join(tf_name) + if start_prefix_to_remove: + tf_name = tf_name.replace(start_prefix_to_remove, '', 1) + + return tf_name, transpose + + +def load_pytorch_checkpoint_in_tf2_model(tf_model, pytorch_checkpoint_path, tf_inputs=None): + """ Load pytorch checkpoints in a TF 2.0 model """ try: import tensorflow as tf @@ -43,25 +77,31 @@ def load_pytorch_checkpoint_in_tf2_model(tf_model, pytorch_checkpoint_path, tf_i pt_state_dict = torch.load(pt_path, map_location='cpu') - return load_pytorch_state_dict_in_tf2_model(tf_model, pt_state_dict, tf_inputs=tf_inputs) + return load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, tf_inputs=tf_inputs) -def load_pytorch_state_dict_in_tf2_model(tf_model, pt_state_dict, tf_inputs=None): +def load_pytorch_model_in_tf2_model(tf_model, pt_model, tf_inputs=None): + """ Load pytorch checkpoints in a TF 2.0 model + """ + pt_state_dict = pt_model.state_dict() + + return load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, tf_inputs=tf_inputs) + + +def load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, tf_inputs=None): """ Load pytorch state_dict in a TF 2.0 model. - Conventions for TF2.0 scopes -> PyTorch attribute names conversions: - - '$1___$2' is replaced by $2 (can be used to duplicate or remove layers in TF2.0 vs PyTorch) - - '_._' is replaced by a level separation (can be used to convert TF2.0 lists in PyTorch nn.ModulesList) """ try: - import re import torch - import numpy from tensorflow.python.keras import backend as K except ImportError as e: logger.error("Loading a PyTorch model in TensorFlow, requires both PyTorch and TensorFlow to be installed. Please see " "https://pytorch.org/ and https://www.tensorflow.org/install/ for installation instructions.") raise e + if tf_inputs is not None: + tfo = tf_model(tf_inputs, training=False) # Make sure model is built + # Adapt state dict - TODO remove this and update the AWS weights files instead # Convert old format to new format if needed from a PyTorch state_dict old_keys = [] @@ -89,27 +129,8 @@ def load_pytorch_state_dict_in_tf2_model(tf_model, pt_state_dict, tf_inputs=None weight_value_tuples = [] all_pytorch_weights = set(list(pt_state_dict.keys())) for symbolic_weight in symbolic_weights: - name = symbolic_weight.name - name = name.replace(':0', '') # device ids - name = re.sub(r'/[^/]*___([^/]*)/', r'/\1/', name) # '$1___$2' is replaced by $2 (can be used to duplicate or remove layers in TF2.0 vs PyTorch) - name = name.replace('_._', '/') # '_._' is replaced by a level separation (can be used to convert TF2.0 lists in PyTorch nn.ModulesList) - name = re.sub(r'//+', '/', name) # Remove empty levels at the end - name = name.split('/') # Convert from TF2.0 '/' separators to PyTorch '.' separators - name = name[1:] # Remove level zero - - # When should we transpose the weights - transpose = bool(name[-1] == 'kernel' or 'emb_projs' in name or 'out_projs' in name) - - # Convert standard TF2.0 names in PyTorch names - if name[-1] == 'kernel' or name[-1] == 'embeddings' or name[-1] == 'gamma': - name[-1] = 'weight' - if name[-1] == 'beta': - name[-1] = 'bias' - - # Remove prefix if needed - name = '.'.join(name) - if start_prefix_to_remove: - name = name.replace(start_prefix_to_remove, '', 1) + sw_name = symbolic_weight.name + name, transpose = convert_tf_weight_name_to_pt_weight_name(sw_name, start_prefix_to_remove=start_prefix_to_remove) # Find associated numpy array in pytorch model state dict assert name in pt_state_dict, "{} not found in PyTorch model".format(name) @@ -144,13 +165,10 @@ def load_pytorch_state_dict_in_tf2_model(tf_model, pt_state_dict, tf_inputs=None return tf_model -def load_tf2_checkpoint_in_pytorch_model(pt_model, tf_checkpoint_path): +def load_tf2_checkpoint_in_pytorch_model(pt_model, tf_checkpoint_path, tf_inputs=None): """ Load TF 2.0 HDF5 checkpoint in a PyTorch model We use HDF5 to easily do transfer learning (see https://github.com/tensorflow/tensorflow/blob/ee16fcac960ae660e0e4496658a366e2f745e1f0/tensorflow/python/keras/engine/network.py#L1352-L1357). - Conventions for TF2.0 scopes -> PyTorch attribute names conversions: - - '$1___$2' is replaced by $2 (can be used to duplicate or remove layers in TF2.0 vs PyTorch) - - '_._' is replaced by a new level separation (can be used to convert TF2.0 lists in PyTorch nn.ModulesList) """ try: import tensorflow as tf @@ -161,13 +179,97 @@ def load_tf2_checkpoint_in_pytorch_model(pt_model, tf_checkpoint_path): raise e tf_path = os.path.abspath(tf_checkpoint_path) - logger.info("Loading TensorFlow weights from {}".format(tf_path)) + logger.info("Loading TensorFlow weights from {}".format(tf_checkpoint_path)) - tf_state_dict = torch.load(tf_path, map_location='cpu') + # Instantiate and load the associated TF 2.0 model + tf_model_class_name = "TF" + model_class.__name__ # Add "TF" at the beggining + tf_model_class = getattr(pytorch_transformers, tf_model_class_name) + tf_model = tf_model_class(pt_model.config) - return load_tf2_weights_in_pytorch_model(pt_model, tf_state_dict) + if tf_inputs is not None: + tfo = tf_model(tf_inputs, training=False) # Make sure model is built -def load_tf2_weights_in_pytorch_model(pt_model, tf_model): + tf_model.load_weights(tf_checkpoint_path, by_name=True) + + return load_tf2_model_in_pytorch_model(pt_model, tf_model) + +def load_tf2_model_in_pytorch_model(pt_model, tf_model): + """ Load TF 2.0 model in a pytorch model + """ + weights = tf_model.weights + + return load_tf2_weights_in_pytorch_model(pt_model, weights) + + +def load_tf2_weights_in_pytorch_model(pt_model, tf_weights): """ Load TF2.0 symbolic weights in a PyTorch model """ - raise NotImplementedError + try: + import tensorflow as tf + import torch + except ImportError as e: + logger.error("Loading a TensorFlow model in PyTorch, requires both PyTorch and TensorFlow to be installed. Please see " + "https://pytorch.org/ and https://www.tensorflow.org/install/ for installation instructions.") + raise e + + new_pt_params_dict = {} + current_pt_params_dict = dict(pt_model.named_parameters()) + + # Make sure we are able to load PyTorch base models as well as derived models (with heads) + # TF models always have a prefix, some of PyTorch models (base ones) don't + start_prefix_to_remove = '' + if not any(s.startswith(pt_model.base_model_prefix) for s in current_pt_params_dict.keys()): + start_prefix_to_remove = pt_model.base_model_prefix + '.' + + # Build a map from potential PyTorch weight names to TF 2.0 Variables + tf_weights_map = {} + for tf_weight in tf_weights: + pt_name, transpose = convert_tf_weight_name_to_pt_weight_name(tf_weight.name, start_prefix_to_remove=start_prefix_to_remove) + tf_weights_map[pt_name] = (tf_weight.numpy(), transpose) + + all_tf_weights = set(list(tf_weights_map.keys())) + loaded_pt_weights_data_ptr = {} + for pt_weight_name, pt_weight in current_pt_params_dict.items(): + # Handle PyTorch shared weight ()not duplicated in TF 2.0 + if pt_weight.data_ptr() in loaded_pt_weights_data_ptr: + new_pt_params_dict[pt_weight_name] = loaded_pt_weights_data_ptr[pt_weight.data_ptr()] + continue + + # Find associated numpy array in pytorch model state dict + if pt_weight_name not in tf_weights_map: + raise ValueError("{} not found in TF 2.0 model".format(pt_weight_name)) + + array, transpose = tf_weights_map[pt_weight_name] + + if transpose: + array = numpy.transpose(array) + + if len(pt_weight.shape) < len(array.shape): + array = numpy.squeeze(array) + elif len(pt_weight.shape) > len(array.shape): + array = numpy.expand_dims(array, axis=0) + + try: + assert list(pt_weight.shape) == list(array.shape) + except AssertionError as e: + e.args += (pt_weight.shape, array.shape) + raise e + + logger.info("Initialize PyTorch weight {}".format(pt_weight_name)) + + new_pt_params_dict[pt_weight_name] = torch.from_numpy(array) + loaded_pt_weights_data_ptr[pt_weight.data_ptr()] = torch.from_numpy(array) + all_tf_weights.discard(pt_weight_name) + + missing_keys, unexpected_keys = pt_model.load_state_dict(new_pt_params_dict, strict=False) + + if len(missing_keys) > 0: + logger.info("Weights of {} not initialized from TF 2.0 model: {}".format( + pt_model.__class__.__name__, missing_keys)) + if len(unexpected_keys) > 0: + logger.info("Weights from TF 2.0 model not used in {}: {}".format( + pt_model.__class__.__name__, unexpected_keys)) + + logger.info("Weights or buffers not loaded from TF 2.0 model: {}".format(all_tf_weights)) + + return pt_model diff --git a/pytorch_transformers/modeling_xlm.py b/pytorch_transformers/modeling_xlm.py index 92febd296d..95629ba535 100644 --- a/pytorch_transformers/modeling_xlm.py +++ b/pytorch_transformers/modeling_xlm.py @@ -718,6 +718,101 @@ class XLMForSequenceClassification(XLMPreTrainedModel): @add_start_docstrings("""XLM Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`). """, XLM_START_DOCSTRING, XLM_INPUTS_DOCSTRING) +class XLMForQuestionAnsweringSimple(XLMPreTrainedModel): + r""" + **start_positions**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for position (index) of the start of the labelled span for computing the token classification loss. + Positions are clamped to the length of the sequence (`sequence_length`). + Position outside of the sequence are not taken into account for computing the loss. + **end_positions**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for position (index) of the end of the labelled span for computing the token classification loss. + Positions are clamped to the length of the sequence (`sequence_length`). + Position outside of the sequence are not taken into account for computing the loss. + **is_impossible**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels whether a question has an answer or no answer (SQuAD 2.0) + **cls_index**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for position (index) of the classification token to use as input for computing plausibility of the answer. + **p_mask**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Optional mask of tokens which can't be in answers (e.g. [CLS], [PAD], ...) + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Total span extraction loss is the sum of a Cross-Entropy for the start and end positions. + **start_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length,)`` + Span-start scores (before SoftMax). + **end_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length,)`` + Span-end scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = XLMTokenizer.from_pretrained('xlm-mlm-en-2048') + model = XLMForQuestionAnsweringSimple.from_pretrained('xlm-mlm-en-2048') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + start_positions = torch.tensor([1]) + end_positions = torch.tensor([3]) + outputs = model(input_ids, start_positions=start_positions, end_positions=end_positions) + loss, start_scores, end_scores = outputs[:2] + + """ + def __init__(self, config): + super(XLMForQuestionAnsweringSimple, self).__init__(config) + + self.transformer = XLMModel(config) + self.qa_outputs = nn.Linear(config.hidden_size, config.num_labels) + + self.init_weights() + + def forward(self, input_ids, attention_mask=None, langs=None, token_type_ids=None, position_ids=None, + lengths=None, cache=None, head_mask=None, start_positions=None, end_positions=None): + transformer_outputs = self.transformer(input_ids, + attention_mask=attention_mask, + langs=langs, + token_type_ids=token_type_ids, + position_ids=position_ids, + lengths=lengths, + cache=cache, + head_mask=head_mask) + + sequence_output = transformer_outputs[0] + + logits = self.qa_outputs(sequence_output) + start_logits, end_logits = logits.split(1, dim=-1) + start_logits = start_logits.squeeze(-1) + end_logits = end_logits.squeeze(-1) + + outputs = (start_logits, end_logits,) + if start_positions is not None and end_positions is not None: + # If we are on multi-GPU, split add a dimension + if len(start_positions.size()) > 1: + start_positions = start_positions.squeeze(-1) + if len(end_positions.size()) > 1: + end_positions = end_positions.squeeze(-1) + # sometimes the start/end positions are outside our model inputs, we ignore these terms + ignored_index = start_logits.size(1) + start_positions.clamp_(0, ignored_index) + end_positions.clamp_(0, ignored_index) + + loss_fct = CrossEntropyLoss(ignore_index=ignored_index) + start_loss = loss_fct(start_logits, start_positions) + end_loss = loss_fct(end_logits, end_positions) + total_loss = (start_loss + end_loss) / 2 + outputs = (total_loss,) + outputs + + outputs = outputs + transformer_outputs[1:] # Keep new_mems and attention/hidden states if they are here + + return outputs + + +@add_start_docstrings("""XLM Model with a beam-search span classification head on top for extractive question-answering tasks like SQuAD (a linear layers on top of + the hidden-states output to compute `span start logits` and `span end logits`). """, + XLM_START_DOCSTRING, XLM_INPUTS_DOCSTRING) class XLMForQuestionAnswering(XLMPreTrainedModel): r""" **start_positions**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: diff --git a/pytorch_transformers/tests/modeling_tf_common_test.py b/pytorch_transformers/tests/modeling_tf_common_test.py index 332db01408..ac25320189 100644 --- a/pytorch_transformers/tests/modeling_tf_common_test.py +++ b/pytorch_transformers/tests/modeling_tf_common_test.py @@ -17,6 +17,7 @@ from __future__ import absolute_import, division, print_function import copy import json import logging +import importlib import random import shutil import unittest @@ -25,7 +26,7 @@ import uuid import pytest import sys -from pytorch_transformers import is_tf_available +from pytorch_transformers import is_tf_available, is_torch_available if is_tf_available(): import tensorflow as tf @@ -66,6 +67,24 @@ class TFCommonTestCases: # msg="Parameter {} of model {} seems not properly initialized".format(name, model_class)) + def test_pt_tf_model_equivalence(self): + if not is_torch_available(): + pass + import pytorch_transformers + + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + pt_model_class_name = model_class.__name__[2:] # Skip the "TF" at the beggining + pt_model_class = getattr(pytorch_transformers, pt_model_class_name) + + tf_model = model_class(config) + pt_model = pt_model_class(config) + + tf_model = pytorch_transformers.load_pytorch_model_in_tf2_model(tf_model, pt_model, tf_inputs=inputs_dict) + pt_model = pytorch_transformers.load_tf2_model_in_pytorch_model(pt_model, tf_model) + + def test_keyword_and_dict_args(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() diff --git a/pytorch_transformers/tests/modeling_tf_xlm_test.py b/pytorch_transformers/tests/modeling_tf_xlm_test.py index 5aed818ddd..26329bebb6 100644 --- a/pytorch_transformers/tests/modeling_tf_xlm_test.py +++ b/pytorch_transformers/tests/modeling_tf_xlm_test.py @@ -225,7 +225,7 @@ class TFXLMModelTest(TFCommonTestCases.TFCommonModelTester): config_and_inputs = self.prepare_config_and_inputs() (config, input_ids, token_type_ids, input_lengths, sequence_labels, token_labels, is_impossible_labels, input_mask) = config_and_inputs - inputs_dict = {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'lengths': input_lengths} + inputs_dict = {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'langs': token_type_ids, 'lengths': input_lengths} return config, inputs_dict def setUp(self): diff --git a/pytorch_transformers/tests/modeling_xlm_test.py b/pytorch_transformers/tests/modeling_xlm_test.py index 4f7f81c002..c3f75e2623 100644 --- a/pytorch_transformers/tests/modeling_xlm_test.py +++ b/pytorch_transformers/tests/modeling_xlm_test.py @@ -24,7 +24,7 @@ from pytorch_transformers import is_torch_available if is_torch_available(): from pytorch_transformers import (XLMConfig, XLMModel, XLMWithLMHeadModel, XLMForQuestionAnswering, - XLMForSequenceClassification) + XLMForSequenceClassification, XLMForQuestionAnsweringSimple) from pytorch_transformers.modeling_xlm import XLM_PRETRAINED_MODEL_ARCHIVE_MAP else: pytestmark = pytest.mark.skip("Require Torch") @@ -36,7 +36,7 @@ from .configuration_common_test import ConfigTester class XLMModelTest(CommonTestCases.CommonModelTester): all_model_classes = (XLMModel, XLMWithLMHeadModel, XLMForQuestionAnswering, - XLMForSequenceClassification) if is_torch_available() else () + XLMForSequenceClassification, XLMForQuestionAnsweringSimple) if is_torch_available() else () class XLMModelTester(object): @@ -180,6 +180,30 @@ class XLMModelTest(CommonTestCases.CommonModelTester): [self.batch_size, self.seq_length, self.vocab_size]) + def create_and_check_xlm_simple_qa(self, config, input_ids, token_type_ids, input_lengths, sequence_labels, token_labels, is_impossible_labels, input_mask): + model = XLMForQuestionAnsweringSimple(config) + model.eval() + + outputs = model(input_ids) + + outputs = model(input_ids, start_positions=sequence_labels, + end_positions=sequence_labels) + loss, start_logits, end_logits = outputs + + result = { + "loss": loss, + "start_logits": start_logits, + "end_logits": end_logits, + } + self.parent.assertListEqual( + list(result["start_logits"].size()), + [self.batch_size, self.seq_length]) + self.parent.assertListEqual( + list(result["end_logits"].size()), + [self.batch_size, self.seq_length]) + self.check_loss_output(result) + + def create_and_check_xlm_qa(self, config, input_ids, token_type_ids, input_lengths, sequence_labels, token_labels, is_impossible_labels, input_mask): model = XLMForQuestionAnswering(config) model.eval() @@ -276,6 +300,10 @@ class XLMModelTest(CommonTestCases.CommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_xlm_lm_head(*config_and_inputs) + def test_xlm_simple_qa(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_xlm_simple_qa(*config_and_inputs) + def test_xlm_qa(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_xlm_qa(*config_and_inputs) From 2167e366ba8cf6bc10023e1f10ffe74e698fc75a Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 24 Sep 2019 13:27:45 +0200 Subject: [PATCH 103/219] update circleCi --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 30a4458807..e2a758e6eb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -91,6 +91,7 @@ workflows: version: 2 build_and_test: jobs: + - build_py3_torch_and_tf - build_py3_torch - build_py3_tf - build_py2_torch From cf9c1cbb60af3881b11f30dd691396fcc462e1e6 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 24 Sep 2019 13:32:47 +0200 Subject: [PATCH 104/219] fix tests chen only using tf --- pytorch_transformers/tests/modeling_tf_common_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pytorch_transformers/tests/modeling_tf_common_test.py b/pytorch_transformers/tests/modeling_tf_common_test.py index ac25320189..ecd1e387f9 100644 --- a/pytorch_transformers/tests/modeling_tf_common_test.py +++ b/pytorch_transformers/tests/modeling_tf_common_test.py @@ -69,7 +69,8 @@ class TFCommonTestCases: def test_pt_tf_model_equivalence(self): if not is_torch_available(): - pass + return + import pytorch_transformers config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() From 271f213621211c6f9c344ba6cbfb85aaa3a7ab96 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 24 Sep 2019 13:51:28 +0200 Subject: [PATCH 105/219] updating to load tf model in pt - fixing headmasking test --- .../modeling_tf_pytorch_utils.py | 26 ++-- pytorch_transformers/modeling_tf_utils.py | 4 +- pytorch_transformers/modeling_utils.py | 117 ++++++++++-------- .../tests/modeling_common_test.py | 3 + 4 files changed, 85 insertions(+), 65 deletions(-) diff --git a/pytorch_transformers/modeling_tf_pytorch_utils.py b/pytorch_transformers/modeling_tf_pytorch_utils.py index 8e879e1447..24f0373d9c 100644 --- a/pytorch_transformers/modeling_tf_pytorch_utils.py +++ b/pytorch_transformers/modeling_tf_pytorch_utils.py @@ -61,7 +61,10 @@ def convert_tf_weight_name_to_pt_weight_name(tf_name, start_prefix_to_remove='') return tf_name, transpose -def load_pytorch_checkpoint_in_tf2_model(tf_model, pytorch_checkpoint_path, tf_inputs=None): +##################### +### PyTorch => TF 2.0 + +def load_pytorch_checkpoint_in_tf2_model(tf_model, pytorch_checkpoint_path, tf_inputs=None, allow_missing_keys=False): """ Load pytorch checkpoints in a TF 2.0 model """ try: @@ -77,18 +80,18 @@ def load_pytorch_checkpoint_in_tf2_model(tf_model, pytorch_checkpoint_path, tf_i pt_state_dict = torch.load(pt_path, map_location='cpu') - return load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, tf_inputs=tf_inputs) + return load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, tf_inputs=tf_inputs, allow_missing_keys=allow_missing_keys) -def load_pytorch_model_in_tf2_model(tf_model, pt_model, tf_inputs=None): +def load_pytorch_model_in_tf2_model(tf_model, pt_model, tf_inputs=None, allow_missing_keys=False): """ Load pytorch checkpoints in a TF 2.0 model """ pt_state_dict = pt_model.state_dict() - return load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, tf_inputs=tf_inputs) + return load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, tf_inputs=tf_inputs, allow_missing_keys=allow_missing_keys) -def load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, tf_inputs=None): +def load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, tf_inputs=None, allow_missing_keys=False): """ Load pytorch state_dict in a TF 2.0 model. """ try: @@ -165,7 +168,10 @@ def load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, tf_inputs=None): return tf_model -def load_tf2_checkpoint_in_pytorch_model(pt_model, tf_checkpoint_path, tf_inputs=None): +##################### +### TF 2.0 => PyTorch + +def load_tf2_checkpoint_in_pytorch_model(pt_model, tf_checkpoint_path, tf_inputs=None, allow_missing_keys=False): """ Load TF 2.0 HDF5 checkpoint in a PyTorch model We use HDF5 to easily do transfer learning (see https://github.com/tensorflow/tensorflow/blob/ee16fcac960ae660e0e4496658a366e2f745e1f0/tensorflow/python/keras/engine/network.py#L1352-L1357). @@ -191,17 +197,17 @@ def load_tf2_checkpoint_in_pytorch_model(pt_model, tf_checkpoint_path, tf_inputs tf_model.load_weights(tf_checkpoint_path, by_name=True) - return load_tf2_model_in_pytorch_model(pt_model, tf_model) + return load_tf2_model_in_pytorch_model(pt_model, tf_model, allow_missing_keys=allow_missing_keys) -def load_tf2_model_in_pytorch_model(pt_model, tf_model): +def load_tf2_model_in_pytorch_model(pt_model, tf_model, allow_missing_keys=False): """ Load TF 2.0 model in a pytorch model """ weights = tf_model.weights - return load_tf2_weights_in_pytorch_model(pt_model, weights) + return load_tf2_weights_in_pytorch_model(pt_model, weights, allow_missing_keys=allow_missing_keys) -def load_tf2_weights_in_pytorch_model(pt_model, tf_weights): +def load_tf2_weights_in_pytorch_model(pt_model, tf_weights, allow_missing_keys=False): """ Load TF2.0 symbolic weights in a PyTorch model """ try: diff --git a/pytorch_transformers/modeling_tf_utils.py b/pytorch_transformers/modeling_tf_utils.py index 704de3b62f..f2b3623a3c 100644 --- a/pytorch_transformers/modeling_tf_utils.py +++ b/pytorch_transformers/modeling_tf_utils.py @@ -129,7 +129,7 @@ class TFPreTrainedModel(tf.keras.Model): @classmethod def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): - r"""Instantiate a pretrained pytorch model from a pre-trained model configuration. + r"""Instantiate a pretrained TF 2.0 model from a pre-trained model configuration. The model is set in evaluation mode by default using ``model.eval()`` (Dropout modules are deactivated) To train the model, you should first set it back in training mode with ``model.train()`` @@ -243,7 +243,7 @@ class TFPreTrainedModel(tf.keras.Model): if from_pt: # Load from a PyTorch checkpoint - return cls.load_pt_weights(model, config, resolved_archive_file) + return cls.load_pt_weights(model, resolved_archive_file) inputs = tf.constant([[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]]) ret = model(inputs, training=False) # build the network with dummy inputs diff --git a/pytorch_transformers/modeling_utils.py b/pytorch_transformers/modeling_utils.py index c316b66bc9..aadf410a19 100644 --- a/pytorch_transformers/modeling_utils.py +++ b/pytorch_transformers/modeling_utils.py @@ -299,12 +299,12 @@ class PreTrainedModel(nn.Module): archive_file = os.path.join(pretrained_model_name_or_path, TF_WEIGHTS_NAME + ".index") else: archive_file = os.path.join(pretrained_model_name_or_path, WEIGHTS_NAME) + elif os.path.isfile(pretrained_model_name_or_path): + archive_file = pretrained_model_name_or_path else: - if from_tf: - # Directly load from a TensorFlow checkpoint - archive_file = pretrained_model_name_or_path + ".index" - else: - archive_file = pretrained_model_name_or_path + assert from_tf, "Error finding file {}, no file or TF 1.X checkpoint found".format(pretrained_model_name_or_path) + archive_file = pretrained_model_name_or_path + ".index" + # redirect to the cache, if necessary try: resolved_archive_file = cached_path(archive_file, cache_dir=cache_dir, force_download=force_download, proxies=proxies) @@ -335,61 +335,72 @@ class PreTrainedModel(nn.Module): if state_dict is None and not from_tf: state_dict = torch.load(resolved_archive_file, map_location='cpu') - if from_tf: - # Directly load from a TensorFlow checkpoint - return cls.load_tf_weights(model, config, resolved_archive_file[:-6]) # Remove the '.index' - # Convert old format to new format if needed from a PyTorch state_dict - old_keys = [] - new_keys = [] - for key in state_dict.keys(): - new_key = None - if 'gamma' in key: - new_key = key.replace('gamma', 'weight') - if 'beta' in key: - new_key = key.replace('beta', 'bias') - if new_key: - old_keys.append(key) - new_keys.append(new_key) - for old_key, new_key in zip(old_keys, new_keys): - state_dict[new_key] = state_dict.pop(old_key) - - # Load from a PyTorch state_dict missing_keys = [] unexpected_keys = [] error_msgs = [] - # copy state_dict so _load_from_state_dict can modify it - metadata = getattr(state_dict, '_metadata', None) - state_dict = state_dict.copy() - if metadata is not None: - state_dict._metadata = metadata - def load(module, prefix=''): - local_metadata = {} if metadata is None else metadata.get(prefix[:-1], {}) - module._load_from_state_dict( - state_dict, prefix, local_metadata, True, missing_keys, unexpected_keys, error_msgs) - for name, child in module._modules.items(): - if child is not None: - load(child, prefix + name + '.') + if from_tf: + if resolved_archive_file.endswith('.index'): + # Load from a TensorFlow 1.X checkpoint - provided by original authors + model = cls.load_tf_weights(model, config, resolved_archive_file[:-6]) # Remove the '.index' + else: + # Load from our TensorFlow 2.0 checkpoints + try: + from pytorch_transformers import load_tf2_checkpoint_in_pytorch_model + model = load_tf2_checkpoint_in_pytorch_model(model, resolved_archive_file, allow_missing_keys=True) + except ImportError as e: + logger.error("Loading a TensorFlow model in PyTorch, requires both PyTorch and TensorFlow to be installed. Please see " + "https://pytorch.org/ and https://www.tensorflow.org/install/ for installation instructions.") + raise e + else: + # Convert old format to new format if needed from a PyTorch state_dict + old_keys = [] + new_keys = [] + for key in state_dict.keys(): + new_key = None + if 'gamma' in key: + new_key = key.replace('gamma', 'weight') + if 'beta' in key: + new_key = key.replace('beta', 'bias') + if new_key: + old_keys.append(key) + new_keys.append(new_key) + for old_key, new_key in zip(old_keys, new_keys): + state_dict[new_key] = state_dict.pop(old_key) - # Make sure we are able to load base models as well as derived models (with heads) - start_prefix = '' - model_to_load = model - if not hasattr(model, cls.base_model_prefix) and any(s.startswith(cls.base_model_prefix) for s in state_dict.keys()): - start_prefix = cls.base_model_prefix + '.' - if hasattr(model, cls.base_model_prefix) and not any(s.startswith(cls.base_model_prefix) for s in state_dict.keys()): - model_to_load = getattr(model, cls.base_model_prefix) + # copy state_dict so _load_from_state_dict can modify it + metadata = getattr(state_dict, '_metadata', None) + state_dict = state_dict.copy() + if metadata is not None: + state_dict._metadata = metadata - load(model_to_load, prefix=start_prefix) - if len(missing_keys) > 0: - logger.info("Weights of {} not initialized from pretrained model: {}".format( - model.__class__.__name__, missing_keys)) - if len(unexpected_keys) > 0: - logger.info("Weights from pretrained model not used in {}: {}".format( - model.__class__.__name__, unexpected_keys)) - if len(error_msgs) > 0: - raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format( - model.__class__.__name__, "\n\t".join(error_msgs))) + def load(module, prefix=''): + local_metadata = {} if metadata is None else metadata.get(prefix[:-1], {}) + module._load_from_state_dict( + state_dict, prefix, local_metadata, True, missing_keys, unexpected_keys, error_msgs) + for name, child in module._modules.items(): + if child is not None: + load(child, prefix + name + '.') + + # Make sure we are able to load base models as well as derived models (with heads) + start_prefix = '' + model_to_load = model + if not hasattr(model, cls.base_model_prefix) and any(s.startswith(cls.base_model_prefix) for s in state_dict.keys()): + start_prefix = cls.base_model_prefix + '.' + if hasattr(model, cls.base_model_prefix) and not any(s.startswith(cls.base_model_prefix) for s in state_dict.keys()): + model_to_load = getattr(model, cls.base_model_prefix) + + load(model_to_load, prefix=start_prefix) + if len(missing_keys) > 0: + logger.info("Weights of {} not initialized from pretrained model: {}".format( + model.__class__.__name__, missing_keys)) + if len(unexpected_keys) > 0: + logger.info("Weights from pretrained model not used in {}: {}".format( + model.__class__.__name__, unexpected_keys)) + if len(error_msgs) > 0: + raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format( + model.__class__.__name__, "\n\t".join(error_msgs))) if hasattr(model, 'tie_weights'): model.tie_weights() # make sure word embedding weights are still tied diff --git a/pytorch_transformers/tests/modeling_common_test.py b/pytorch_transformers/tests/modeling_common_test.py index a7d56041a3..ff321193a5 100644 --- a/pytorch_transformers/tests/modeling_common_test.py +++ b/pytorch_transformers/tests/modeling_common_test.py @@ -200,6 +200,9 @@ class CommonTestCases: hidden_states = outputs[-2] # Remove Nan + for t in attentions: + self.assertLess(torch.sum(torch.isnan(t)), t.numel() / 4) # Check we don't have more than 25% nans (arbitrary) + attentions = [t.masked_fill(torch.isnan(t), 0.0) for t in attentions] # remove them (the test is less complete) self.assertIsNotNone(multihead_outputs) self.assertEqual(len(multihead_outputs), self.model_tester.num_hidden_layers) From f5397ffc3bf444e814b4234526dccba146be0347 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 24 Sep 2019 14:03:58 +0200 Subject: [PATCH 106/219] update loading logics --- pytorch_transformers/__init__.py | 2 +- pytorch_transformers/file_utils.py | 1 + pytorch_transformers/modeling_tf_utils.py | 71 +++++++++++++---------- pytorch_transformers/modeling_utils.py | 16 +++-- 4 files changed, 55 insertions(+), 35 deletions(-) diff --git a/pytorch_transformers/__init__.py b/pytorch_transformers/__init__.py index b8c7eccfe7..508d0f84c4 100644 --- a/pytorch_transformers/__init__.py +++ b/pytorch_transformers/__init__.py @@ -163,7 +163,7 @@ if _tf_available and _torch_available: # Files and general utilities from .file_utils import (PYTORCH_TRANSFORMERS_CACHE, PYTORCH_PRETRAINED_BERT_CACHE, cached_path, add_start_docstrings, add_end_docstrings, - WEIGHTS_NAME, TF_WEIGHTS_NAME, CONFIG_NAME) + WEIGHTS_NAME, TF2_WEIGHTS_NAME, TF_WEIGHTS_NAME, CONFIG_NAME) def is_torch_available(): return _torch_available diff --git a/pytorch_transformers/file_utils.py b/pytorch_transformers/file_utils.py index a656e757b5..34333aaafb 100644 --- a/pytorch_transformers/file_utils.py +++ b/pytorch_transformers/file_utils.py @@ -49,6 +49,7 @@ except (AttributeError, ImportError): PYTORCH_TRANSFORMERS_CACHE = PYTORCH_PRETRAINED_BERT_CACHE # Kept for backward compatibility WEIGHTS_NAME = "pytorch_model.bin" +TF2_WEIGHTS_NAME = 'tf_model.h5' TF_WEIGHTS_NAME = 'model.ckpt' CONFIG_NAME = "config.json" diff --git a/pytorch_transformers/modeling_tf_utils.py b/pytorch_transformers/modeling_tf_utils.py index f2b3623a3c..4a695e2864 100644 --- a/pytorch_transformers/modeling_tf_utils.py +++ b/pytorch_transformers/modeling_tf_utils.py @@ -24,7 +24,7 @@ import os import tensorflow as tf from .configuration_utils import PretrainedConfig -from .file_utils import cached_path, WEIGHTS_NAME, TF_WEIGHTS_NAME +from .file_utils import cached_path, WEIGHTS_NAME, TF_WEIGHTS_NAME, TF2_WEIGHTS_NAME logger = logging.getLogger(__name__) @@ -205,38 +205,49 @@ class TFPreTrainedModel(tf.keras.Model): model_kwargs = kwargs # Load model - if pretrained_model_name_or_path in cls.pretrained_model_archive_map: - archive_file = cls.pretrained_model_archive_map[pretrained_model_name_or_path] - elif os.path.isdir(pretrained_model_name_or_path): - if from_pt: - # Load from a PyTorch checkpoint - archive_file = os.path.join(pretrained_model_name_or_path, WEIGHTS_NAME) - else: - archive_file = os.path.join(pretrained_model_name_or_path, TF_WEIGHTS_NAME) - else: - archive_file = pretrained_model_name_or_path - # redirect to the cache, if necessary - try: - resolved_archive_file = cached_path(archive_file, cache_dir=cache_dir, force_download=force_download, proxies=proxies) - except EnvironmentError: + if pretrained_model_name_or_path is not None: if pretrained_model_name_or_path in cls.pretrained_model_archive_map: - logger.error( - "Couldn't reach server at '{}' to download pretrained weights.".format( - archive_file)) + archive_file = cls.pretrained_model_archive_map[pretrained_model_name_or_path] + elif os.path.isdir(pretrained_model_name_or_path): + if os.path.isfile(os.path.join(pretrained_model_name_or_path, TF2_WEIGHTS_NAME)): + # Load from a TF 2.0 checkpoint + archive_file = os.path.join(pretrained_model_name_or_path, TF2_WEIGHTS_NAME) + elif from_pt and os.path.isfile(os.path.join(pretrained_model_name_or_path, WEIGHTS_NAME)): + # Load from a PyTorch checkpoint + archive_file = os.path.join(pretrained_model_name_or_path, WEIGHTS_NAME) + else: + raise EnvironmentError("Error no file named {} found in directory {}".format( + tuple(WEIGHTS_NAME, TF2_WEIGHTS_NAME), + pretrained_model_name_or_path)) + elif os.path.isfile(pretrained_model_name_or_path): + archive_file = pretrained_model_name_or_path else: - logger.error( - "Model name '{}' was not found in model name list ({}). " - "We assumed '{}' was a path or url but couldn't find any file " - "associated to this path or url.".format( - pretrained_model_name_or_path, - ', '.join(cls.pretrained_model_archive_map.keys()), - archive_file)) - return None - if resolved_archive_file == archive_file: - logger.info("loading weights file {}".format(archive_file)) + raise EnvironmentError("Error file {} not found".format(pretrained_model_name_or_path)) + + # redirect to the cache, if necessary + try: + resolved_archive_file = cached_path(archive_file, cache_dir=cache_dir, force_download=force_download, proxies=proxies) + except EnvironmentError as e: + if pretrained_model_name_or_path in cls.pretrained_model_archive_map: + logger.error( + "Couldn't reach server at '{}' to download pretrained weights.".format( + archive_file)) + else: + logger.error( + "Model name '{}' was not found in model name list ({}). " + "We assumed '{}' was a path or url but couldn't find any file " + "associated to this path or url.".format( + pretrained_model_name_or_path, + ', '.join(cls.pretrained_model_archive_map.keys()), + archive_file)) + raise e + if resolved_archive_file == archive_file: + logger.info("loading weights file {}".format(archive_file)) + else: + logger.info("loading weights file {} from cache at {}".format( + archive_file, resolved_archive_file)) else: - logger.info("loading weights file {} from cache at {}".format( - archive_file, resolved_archive_file)) + resolved_archive_file = None # Instantiate model. model = cls(config, *model_args, **model_kwargs) diff --git a/pytorch_transformers/modeling_utils.py b/pytorch_transformers/modeling_utils.py index aadf410a19..790d4dcd54 100644 --- a/pytorch_transformers/modeling_utils.py +++ b/pytorch_transformers/modeling_utils.py @@ -31,7 +31,7 @@ from torch.nn import CrossEntropyLoss from torch.nn import functional as F from .configuration_utils import PretrainedConfig -from .file_utils import cached_path, WEIGHTS_NAME, TF_WEIGHTS_NAME +from .file_utils import cached_path, WEIGHTS_NAME, TF_WEIGHTS_NAME, TF2_WEIGHTS_NAME logger = logging.getLogger(__name__) @@ -294,11 +294,19 @@ class PreTrainedModel(nn.Module): if pretrained_model_name_or_path in cls.pretrained_model_archive_map: archive_file = cls.pretrained_model_archive_map[pretrained_model_name_or_path] elif os.path.isdir(pretrained_model_name_or_path): - if from_tf: - # Directly load from a TensorFlow checkpoint + if from_tf and os.path.isfile(os.path.join(pretrained_model_name_or_path, TF_WEIGHTS_NAME + ".index")): + # Load from a TF 1.0 checkpoint archive_file = os.path.join(pretrained_model_name_or_path, TF_WEIGHTS_NAME + ".index") - else: + elif from_tf and os.path.isfile(os.path.join(pretrained_model_name_or_path, TF2_WEIGHTS_NAME)): + # Load from a TF 2.0 checkpoint + archive_file = os.path.join(pretrained_model_name_or_path, TF2_WEIGHTS_NAME) + elif os.path.isfile(os.path.join(pretrained_model_name_or_path, WEIGHTS_NAME)): + # Load from a PyTorch checkpoint archive_file = os.path.join(pretrained_model_name_or_path, WEIGHTS_NAME) + else: + raise EnvironmentError("Error no file named {} found in directory {}".format( + tuple(WEIGHTS_NAME, TF2_WEIGHTS_NAME, TF_WEIGHTS_NAME + ".index"), + pretrained_model_name_or_path)) elif os.path.isfile(pretrained_model_name_or_path): archive_file = pretrained_model_name_or_path else: From 29bb3e4eb0e7f275dcf270eb351c5e871f6f3312 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 24 Sep 2019 14:23:46 +0200 Subject: [PATCH 107/219] double loading ok --- .../modeling_tf_pytorch_utils.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/pytorch_transformers/modeling_tf_pytorch_utils.py b/pytorch_transformers/modeling_tf_pytorch_utils.py index 24f0373d9c..f4b41c3b8e 100644 --- a/pytorch_transformers/modeling_tf_pytorch_utils.py +++ b/pytorch_transformers/modeling_tf_pytorch_utils.py @@ -25,6 +25,7 @@ import numpy logger = logging.getLogger(__name__) +DUMMY_INPUTS = [[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]] def convert_tf_weight_name_to_pt_weight_name(tf_name, start_prefix_to_remove=''): """ Convert a TF 2.0 model variable name in a pytorch model weight name. @@ -64,7 +65,7 @@ def convert_tf_weight_name_to_pt_weight_name(tf_name, start_prefix_to_remove='') ##################### ### PyTorch => TF 2.0 -def load_pytorch_checkpoint_in_tf2_model(tf_model, pytorch_checkpoint_path, tf_inputs=None, allow_missing_keys=False): +def load_pytorch_checkpoint_in_tf2_model(tf_model, pytorch_checkpoint_path, tf_inputs=DUMMY_INPUTS, allow_missing_keys=False): """ Load pytorch checkpoints in a TF 2.0 model """ try: @@ -83,7 +84,7 @@ def load_pytorch_checkpoint_in_tf2_model(tf_model, pytorch_checkpoint_path, tf_i return load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, tf_inputs=tf_inputs, allow_missing_keys=allow_missing_keys) -def load_pytorch_model_in_tf2_model(tf_model, pt_model, tf_inputs=None, allow_missing_keys=False): +def load_pytorch_model_in_tf2_model(tf_model, pt_model, tf_inputs=DUMMY_INPUTS, allow_missing_keys=False): """ Load pytorch checkpoints in a TF 2.0 model """ pt_state_dict = pt_model.state_dict() @@ -91,17 +92,21 @@ def load_pytorch_model_in_tf2_model(tf_model, pt_model, tf_inputs=None, allow_mi return load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, tf_inputs=tf_inputs, allow_missing_keys=allow_missing_keys) -def load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, tf_inputs=None, allow_missing_keys=False): +def load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, tf_inputs=DUMMY_INPUTS, allow_missing_keys=False): """ Load pytorch state_dict in a TF 2.0 model. """ try: import torch + import tensorflow as tf from tensorflow.python.keras import backend as K except ImportError as e: logger.error("Loading a PyTorch model in TensorFlow, requires both PyTorch and TensorFlow to be installed. Please see " "https://pytorch.org/ and https://www.tensorflow.org/install/ for installation instructions.") raise e + if tf_inputs is not None and not isinstance(tf_inputs, tf.Tensor): + tf_inputs = tf.constant(tf_inputs) + if tf_inputs is not None: tfo = tf_model(tf_inputs, training=False) # Make sure model is built @@ -171,7 +176,7 @@ def load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, tf_inputs=None, a ##################### ### TF 2.0 => PyTorch -def load_tf2_checkpoint_in_pytorch_model(pt_model, tf_checkpoint_path, tf_inputs=None, allow_missing_keys=False): +def load_tf2_checkpoint_in_pytorch_model(pt_model, tf_checkpoint_path, tf_inputs=DUMMY_INPUTS, allow_missing_keys=False): """ Load TF 2.0 HDF5 checkpoint in a PyTorch model We use HDF5 to easily do transfer learning (see https://github.com/tensorflow/tensorflow/blob/ee16fcac960ae660e0e4496658a366e2f745e1f0/tensorflow/python/keras/engine/network.py#L1352-L1357). @@ -184,15 +189,19 @@ def load_tf2_checkpoint_in_pytorch_model(pt_model, tf_checkpoint_path, tf_inputs "https://pytorch.org/ and https://www.tensorflow.org/install/ for installation instructions.") raise e + import pytorch_transformers + tf_path = os.path.abspath(tf_checkpoint_path) logger.info("Loading TensorFlow weights from {}".format(tf_checkpoint_path)) # Instantiate and load the associated TF 2.0 model - tf_model_class_name = "TF" + model_class.__name__ # Add "TF" at the beggining + tf_model_class_name = "TF" + pt_model.__class__.__name__ # Add "TF" at the beggining tf_model_class = getattr(pytorch_transformers, tf_model_class_name) tf_model = tf_model_class(pt_model.config) if tf_inputs is not None: + if tf_inputs is not None and not isinstance(tf_inputs, tf.Tensor): + tf_inputs = tf.constant(tf_inputs) tfo = tf_model(tf_inputs, training=False) # Make sure model is built tf_model.load_weights(tf_checkpoint_path, by_name=True) From ee261439a9f7913f652c23eca134deed5e04e8b6 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 24 Sep 2019 14:30:28 +0200 Subject: [PATCH 108/219] add save_pretrained --- pytorch_transformers/modeling_tf_utils.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pytorch_transformers/modeling_tf_utils.py b/pytorch_transformers/modeling_tf_utils.py index 4a695e2864..bdbc034c4a 100644 --- a/pytorch_transformers/modeling_tf_utils.py +++ b/pytorch_transformers/modeling_tf_utils.py @@ -125,7 +125,15 @@ class TFPreTrainedModel(tf.keras.Model): """ Save a model and its configuration file to a directory, so that it can be re-loaded using the `:func:`~pytorch_transformers.PreTrainedModel.from_pretrained`` class method. """ - raise NotImplementedError + assert os.path.isdir(save_directory), "Saving path should be a directory where the model and configuration can be saved" + + # Save configuration file + self.config.save_pretrained(save_directory) + + # If we save using the predefined names, we can load using `from_pretrained` + output_model_file = os.path.join(save_directory, TF2_WEIGHTS_NAME) + + self.save_weights(output_model_file) @classmethod def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): From dd2d90f344c0e6dcda4a8aca0c2ab951bfce7da7 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 24 Sep 2019 14:39:41 +0200 Subject: [PATCH 109/219] update automodels --- pytorch_transformers/modeling_tf_auto.py | 169 ++++++++++++----------- 1 file changed, 91 insertions(+), 78 deletions(-) diff --git a/pytorch_transformers/modeling_tf_auto.py b/pytorch_transformers/modeling_tf_auto.py index 3b10d700a1..23abab901c 100644 --- a/pytorch_transformers/modeling_tf_auto.py +++ b/pytorch_transformers/modeling_tf_auto.py @@ -19,6 +19,13 @@ from __future__ import absolute_import, division, print_function, unicode_litera import logging from .modeling_tf_bert import TFBertModel, TFBertForMaskedLM, TFBertForSequenceClassification, TFBertForQuestionAnswering +from .modeling_openai import TFOpenAIGPTModel, TFOpenAIGPTLMHeadModel +from .modeling_gpt2 import TFGPT2Model, TFGPT2LMHeadModel +from .modeling_transfo_xl import TFTransfoXLModel, TFTransfoXLLMHeadModel +from .modeling_xlnet import TFXLNetModel, TFXLNetLMHeadModel, TFXLNetForSequenceClassification, TFXLNetForQuestionAnswering +from .modeling_xlm import TFXLMModel, TFXLMWithLMHeadModel, TFXLMForSequenceClassification, TFXLMForQuestionAnsweringSimple +from .modeling_roberta import TFRobertaModel, TFRobertaForMaskedLM, TFRobertaForSequenceClassification +from .modeling_distilbert import TFDistilBertModel, TFDistilBertForQuestionAnswering, TFDistilBertForMaskedLM, TFDistilBertForSequenceClassification from .file_utils import add_start_docstrings @@ -37,14 +44,14 @@ class TFAutoModel(object): The base model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - - contains `distilbert`: DistilBertModel (DistilBERT model) - - contains `roberta`: RobertaModel (RoBERTa model) + - contains `distilbert`: TFDistilBertModel (DistilBERT model) + - contains `roberta`: TFRobertaModel (RoBERTa model) - contains `bert`: TFBertModel (Bert model) - - contains `openai-gpt`: OpenAIGPTModel (OpenAI GPT model) - - contains `gpt2`: GPT2Model (OpenAI GPT-2 model) - - contains `transfo-xl`: TransfoXLModel (Transformer-XL model) - - contains `xlnet`: XLNetModel (XLNet model) - - contains `xlm`: XLMModel (XLM model) + - contains `openai-gpt`: TFOpenAIGPTModel (OpenAI GPT model) + - contains `gpt2`: TFGPT2Model (OpenAI GPT-2 model) + - contains `transfo-xl`: TFTransfoXLModel (Transformer-XL model) + - contains `xlnet`: TFXLNetModel (XLNet model) + - contains `xlm`: TFXLMModel (XLM model) This class cannot be instantiated using `__init__()` (throws an error). """ @@ -59,24 +66,24 @@ class TFAutoModel(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - - contains `distilbert`: DistilBertModel (DistilBERT model) - - contains `roberta`: RobertaModel (RoBERTa model) - - contains `bert`: TFBertModel (Bert model) - - contains `openai-gpt`: OpenAIGPTModel (OpenAI GPT model) - - contains `gpt2`: GPT2Model (OpenAI GPT-2 model) - - contains `transfo-xl`: TransfoXLModel (Transformer-XL model) - - contains `xlnet`: XLNetModel (XLNet model) - - contains `xlm`: XLMModel (XLM model) - - The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated) - To train the model, you should first set it back in training mode with `model.train()` + - contains `distilbert`: TFDistilBertModel (DistilBERT model) + - contains `roberta`: TFRobertaModel (RoBERTa model) + - contains `bert`: TFTFBertModel (Bert model) + - contains `openai-gpt`: TFOpenAIGPTModel (OpenAI GPT model) + - contains `gpt2`: TFGPT2Model (OpenAI GPT-2 model) + - contains `transfo-xl`: TFTransfoXLModel (Transformer-XL model) + - contains `xlnet`: TFXLNetModel (XLNet model) + - contains `xlm`: TFXLMModel (XLM model) Params: pretrained_model_name_or_path: either: - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. - a path to a `directory` containing model weights saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. - - a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. + - a path or url to a `PyTorch, TF 1.X or TF 2.0 checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In the case of a PyTorch checkpoint, ``from_pt`` should be set to True and a configuration object should be provided as ``config`` argument. + + from_pt: (`Optional`) Boolean + Set to True if the Checkpoint is a PyTorch checkpoint. model_args: (`optional`) Sequence of positional arguments: All remaning positional arguments will be passed to the underlying model's ``__init__`` method @@ -121,25 +128,25 @@ class TFAutoModel(object): assert model.config.output_attention == True # Loading from a TF checkpoint file instead of a PyTorch model (slower) config = AutoConfig.from_json_file('./tf_model/bert_tf_model_config.json') - model = TFAutoModel.from_pretrained('./tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config) + model = TFAutoModel.from_pretrained('./pt_model/bert_pytorch_model.bin', from_pt=True, config=config) """ if 'distilbert' in pretrained_model_name_or_path: - raise NotImplementedError + return TFDistilBertModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'roberta' in pretrained_model_name_or_path: - raise NotImplementedError + return TFRobertaModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'bert' in pretrained_model_name_or_path: return TFBertModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'openai-gpt' in pretrained_model_name_or_path: - raise NotImplementedError + return TFOpenAIGPTModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'gpt2' in pretrained_model_name_or_path: - raise NotImplementedError + return TFGPT2Model.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'transfo-xl' in pretrained_model_name_or_path: - raise NotImplementedError + return TFTransfoXLModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'xlnet' in pretrained_model_name_or_path: - raise NotImplementedError + return TFXLNetModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'xlm' in pretrained_model_name_or_path: - raise NotImplementedError + return TFXLMModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) raise ValueError("Unrecognized model identifier in {}. Should contains one of " "'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', " @@ -158,14 +165,14 @@ class TFAutoModelWithLMHead(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - - contains `distilbert`: DistilBertForMaskedLM (DistilBERT model) - - contains `roberta`: RobertaForMaskedLM (RoBERTa model) + - contains `distilbert`: TFDistilBertForMaskedLM (DistilBERT model) + - contains `roberta`: TFRobertaForMaskedLM (RoBERTa model) - contains `bert`: TFBertForMaskedLM (Bert model) - - contains `openai-gpt`: OpenAIGPTLMHeadModel (OpenAI GPT model) - - contains `gpt2`: GPT2LMHeadModel (OpenAI GPT-2 model) - - contains `transfo-xl`: TransfoXLLMHeadModel (Transformer-XL model) - - contains `xlnet`: XLNetLMHeadModel (XLNet model) - - contains `xlm`: XLMWithLMHeadModel (XLM model) + - contains `openai-gpt`: TFOpenAIGPTLMHeadModel (OpenAI GPT model) + - contains `gpt2`: TFGPT2LMHeadModel (OpenAI GPT-2 model) + - contains `transfo-xl`: TFTransfoXLLMHeadModel (Transformer-XL model) + - contains `xlnet`: TFXLNetLMHeadModel (XLNet model) + - contains `xlm`: TFXLMWithLMHeadModel (XLM model) This class cannot be instantiated using `__init__()` (throws an error). """ @@ -183,24 +190,24 @@ class TFAutoModelWithLMHead(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - - contains `distilbert`: DistilBertForMaskedLM (DistilBERT model) - - contains `roberta`: RobertaForMaskedLM (RoBERTa model) + - contains `distilbert`: TFDistilBertForMaskedLM (DistilBERT model) + - contains `roberta`: TFRobertaForMaskedLM (RoBERTa model) - contains `bert`: TFBertForMaskedLM (Bert model) - - contains `openai-gpt`: OpenAIGPTLMHeadModel (OpenAI GPT model) - - contains `gpt2`: GPT2LMHeadModel (OpenAI GPT-2 model) - - contains `transfo-xl`: TransfoXLLMHeadModel (Transformer-XL model) - - contains `xlnet`: XLNetLMHeadModel (XLNet model) - - contains `xlm`: XLMWithLMHeadModel (XLM model) - - The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated) - To train the model, you should first set it back in training mode with `model.train()` + - contains `openai-gpt`: TFOpenAIGPTLMHeadModel (OpenAI GPT model) + - contains `gpt2`: TFGPT2LMHeadModel (OpenAI GPT-2 model) + - contains `transfo-xl`: TFTransfoXLLMHeadModel (Transformer-XL model) + - contains `xlnet`: TFXLNetLMHeadModel (XLNet model) + - contains `xlm`: TFXLMWithLMHeadModel (XLM model) Params: pretrained_model_name_or_path: either: - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. - a path to a `directory` containing model weights saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. - - a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. + - a path or url to a `PyTorch, TF 1.X or TF 2.0 checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In the case of a PyTorch checkpoint, ``from_pt`` should be set to True and a configuration object should be provided as ``config`` argument. + + from_pt: (`Optional`) Boolean + Set to True if the Checkpoint is a PyTorch checkpoint. model_args: (`optional`) Sequence of positional arguments: All remaning positional arguments will be passed to the underlying model's ``__init__`` method @@ -245,25 +252,25 @@ class TFAutoModelWithLMHead(object): assert model.config.output_attention == True # Loading from a TF checkpoint file instead of a PyTorch model (slower) config = AutoConfig.from_json_file('./tf_model/bert_tf_model_config.json') - model = TFAutoModelWithLMHead.from_pretrained('./tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config) + model = TFAutoModelWithLMHead.from_pretrained('./pt_model/bert_pytorch_model.bin', from_pt=True, config=config) """ if 'distilbert' in pretrained_model_name_or_path: - raise NotImplementedError + return TFDistilBertForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'roberta' in pretrained_model_name_or_path: - raise NotImplementedError + return TFRobertaForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'bert' in pretrained_model_name_or_path: return TFBertForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'openai-gpt' in pretrained_model_name_or_path: - raise NotImplementedError + return TFOpenAIGPTLMHeadModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'gpt2' in pretrained_model_name_or_path: - raise NotImplementedError + return TFGPT2LMHeadModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'transfo-xl' in pretrained_model_name_or_path: - raise NotImplementedError + return TFTransfoXLLMHeadModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'xlnet' in pretrained_model_name_or_path: - raise NotImplementedError + return TFXLNetLMHeadModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'xlm' in pretrained_model_name_or_path: - raise NotImplementedError + return TFXLMWithLMHeadModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) raise ValueError("Unrecognized model identifier in {}. Should contains one of " "'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', " @@ -282,11 +289,11 @@ class TFAutoModelForSequenceClassification(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - - contains `distilbert`: DistilBertForSequenceClassification (DistilBERT model) - - contains `roberta`: RobertaForSequenceClassification (RoBERTa model) + - contains `distilbert`: TFDistilBertForSequenceClassification (DistilBERT model) + - contains `roberta`: TFRobertaForSequenceClassification (RoBERTa model) - contains `bert`: TFBertForSequenceClassification (Bert model) - - contains `xlnet`: XLNetForSequenceClassification (XLNet model) - - contains `xlm`: XLMForSequenceClassification (XLM model) + - contains `xlnet`: TFXLNetForSequenceClassification (XLNet model) + - contains `xlm`: TFXLMForSequenceClassification (XLM model) This class cannot be instantiated using `__init__()` (throws an error). """ @@ -304,11 +311,11 @@ class TFAutoModelForSequenceClassification(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - - contains `distilbert`: DistilBertForSequenceClassification (DistilBERT model) - - contains `roberta`: RobertaForSequenceClassification (RoBERTa model) + - contains `distilbert`: TFDistilBertForSequenceClassification (DistilBERT model) + - contains `roberta`: TFRobertaForSequenceClassification (RoBERTa model) - contains `bert`: TFBertForSequenceClassification (Bert model) - - contains `xlnet`: XLNetForSequenceClassification (XLNet model) - - contains `xlm`: XLMForSequenceClassification (XLM model) + - contains `xlnet`: TFXLNetForSequenceClassification (XLNet model) + - contains `xlm`: TFXLMForSequenceClassification (XLM model) The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated) To train the model, you should first set it back in training mode with `model.train()` @@ -318,7 +325,10 @@ class TFAutoModelForSequenceClassification(object): - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. - a path to a `directory` containing model weights saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. - - a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. + - a path or url to a `PyTorch, TF 1.X or TF 2.0 checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In the case of a PyTorch checkpoint, ``from_pt`` should be set to True and a configuration object should be provided as ``config`` argument. + + from_pt: (`Optional`) Boolean + Set to True if the Checkpoint is a PyTorch checkpoint. model_args: (`optional`) Sequence of positional arguments: All remaning positional arguments will be passed to the underlying model's ``__init__`` method @@ -363,19 +373,19 @@ class TFAutoModelForSequenceClassification(object): assert model.config.output_attention == True # Loading from a TF checkpoint file instead of a PyTorch model (slower) config = AutoConfig.from_json_file('./tf_model/bert_tf_model_config.json') - model = TFAutoModelForSequenceClassification.from_pretrained('./tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config) + model = TFAutoModelForSequenceClassification.from_pretrained('./pt_model/bert_pytorch_model.bin', from_pt=True, config=config) """ if 'distilbert' in pretrained_model_name_or_path: - raise NotImplementedError + return TFDistilBertForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'roberta' in pretrained_model_name_or_path: - raise NotImplementedError + return TFRobertaForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'bert' in pretrained_model_name_or_path: return TFBertForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'xlnet' in pretrained_model_name_or_path: - raise NotImplementedError + return TFXLNetForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'xlm' in pretrained_model_name_or_path: - raise NotImplementedError + return TFXLMForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) raise ValueError("Unrecognized model identifier in {}. Should contains one of " "'bert', 'xlnet', 'xlm', 'roberta'".format(pretrained_model_name_or_path)) @@ -393,10 +403,10 @@ class TFAutoModelForQuestionAnswering(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - - contains `distilbert`: DistilBertForQuestionAnswering (DistilBERT model) + - contains `distilbert`: TFDistilBertForQuestionAnswering (DistilBERT model) - contains `bert`: TFBertForQuestionAnswering (Bert model) - - contains `xlnet`: XLNetForQuestionAnswering (XLNet model) - - contains `xlm`: XLMForQuestionAnswering (XLM model) + - contains `xlnet`: TFXLNetForQuestionAnswering (XLNet model) + - contains `xlm`: TFXLMForQuestionAnswering (XLM model) This class cannot be instantiated using `__init__()` (throws an error). """ @@ -414,10 +424,10 @@ class TFAutoModelForQuestionAnswering(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - - contains `distilbert`: DistilBertForQuestionAnswering (DistilBERT model) + - contains `distilbert`: TFDistilBertForQuestionAnswering (DistilBERT model) - contains `bert`: TFBertForQuestionAnswering (Bert model) - - contains `xlnet`: XLNetForQuestionAnswering (XLNet model) - - contains `xlm`: XLMForQuestionAnswering (XLM model) + - contains `xlnet`: TFXLNetForQuestionAnswering (XLNet model) + - contains `xlm`: TFXLMForQuestionAnswering (XLM model) The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated) To train the model, you should first set it back in training mode with `model.train()` @@ -427,7 +437,10 @@ class TFAutoModelForQuestionAnswering(object): - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. - a path to a `directory` containing model weights saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. - - a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. + - a path or url to a `PyTorch, TF 1.X or TF 2.0 checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In the case of a PyTorch checkpoint, ``from_pt`` should be set to True and a configuration object should be provided as ``config`` argument. + + from_pt: (`Optional`) Boolean + Set to True if the Checkpoint is a PyTorch checkpoint. model_args: (`optional`) Sequence of positional arguments: All remaning positional arguments will be passed to the underlying model's ``__init__`` method @@ -472,17 +485,17 @@ class TFAutoModelForQuestionAnswering(object): assert model.config.output_attention == True # Loading from a TF checkpoint file instead of a PyTorch model (slower) config = AutoConfig.from_json_file('./tf_model/bert_tf_model_config.json') - model = TFAutoModelForQuestionAnswering.from_pretrained('./tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config) + model = TFAutoModelForQuestionAnswering.from_pretrained('./pt_model/bert_pytorch_model.bin', from_pt=True, config=config) """ if 'distilbert' in pretrained_model_name_or_path: - raise NotImplementedError + return TFDistilBertForQuestionAnswering.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'bert' in pretrained_model_name_or_path: return TFBertForQuestionAnswering.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'xlnet' in pretrained_model_name_or_path: - raise NotImplementedError + return TFXLNetForQuestionAnswering.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'xlm' in pretrained_model_name_or_path: - raise NotImplementedError + return TFXLMForQuestionAnswering.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) raise ValueError("Unrecognized model identifier in {}. Should contains one of " "'bert', 'xlnet', 'xlm'".format(pretrained_model_name_or_path)) From f3d1511b5bb667aa1126478d892ac3c8919ac8e7 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 24 Sep 2019 14:42:09 +0200 Subject: [PATCH 110/219] fix imports --- pytorch_transformers/modeling_tf_auto.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pytorch_transformers/modeling_tf_auto.py b/pytorch_transformers/modeling_tf_auto.py index 23abab901c..1db81c7ab4 100644 --- a/pytorch_transformers/modeling_tf_auto.py +++ b/pytorch_transformers/modeling_tf_auto.py @@ -19,13 +19,13 @@ from __future__ import absolute_import, division, print_function, unicode_litera import logging from .modeling_tf_bert import TFBertModel, TFBertForMaskedLM, TFBertForSequenceClassification, TFBertForQuestionAnswering -from .modeling_openai import TFOpenAIGPTModel, TFOpenAIGPTLMHeadModel -from .modeling_gpt2 import TFGPT2Model, TFGPT2LMHeadModel -from .modeling_transfo_xl import TFTransfoXLModel, TFTransfoXLLMHeadModel -from .modeling_xlnet import TFXLNetModel, TFXLNetLMHeadModel, TFXLNetForSequenceClassification, TFXLNetForQuestionAnswering -from .modeling_xlm import TFXLMModel, TFXLMWithLMHeadModel, TFXLMForSequenceClassification, TFXLMForQuestionAnsweringSimple -from .modeling_roberta import TFRobertaModel, TFRobertaForMaskedLM, TFRobertaForSequenceClassification -from .modeling_distilbert import TFDistilBertModel, TFDistilBertForQuestionAnswering, TFDistilBertForMaskedLM, TFDistilBertForSequenceClassification +from .modeling_tf_openai import TFOpenAIGPTModel, TFOpenAIGPTLMHeadModel +from .modeling_tf_gpt2 import TFGPT2Model, TFGPT2LMHeadModel +from .modeling_tf_transfo_xl import TFTransfoXLModel, TFTransfoXLLMHeadModel +from .modeling_tf_xlnet import TFXLNetModel, TFXLNetLMHeadModel, TFXLNetForSequenceClassification, TFXLNetForQuestionAnswering +from .modeling_tf_xlm import TFXLMModel, TFXLMWithLMHeadModel, TFXLMForSequenceClassification, TFXLMForQuestionAnsweringSimple +from .modeling_tf_roberta import TFRobertaModel, TFRobertaForMaskedLM, TFRobertaForSequenceClassification +from .modeling_tf_distilbert import TFDistilBertModel, TFDistilBertForQuestionAnswering, TFDistilBertForMaskedLM, TFDistilBertForSequenceClassification from .file_utils import add_start_docstrings From 9678c49419ca81b854d80f77be2a9c027f9b5513 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 24 Sep 2019 14:57:05 +0200 Subject: [PATCH 111/219] docstrings for bert --- pytorch_transformers/modeling_tf_bert.py | 140 ++++++++++++++--------- 1 file changed, 84 insertions(+), 56 deletions(-) diff --git a/pytorch_transformers/modeling_tf_bert.py b/pytorch_transformers/modeling_tf_bert.py index b00bd09296..583d403963 100644 --- a/pytorch_transformers/modeling_tf_bert.py +++ b/pytorch_transformers/modeling_tf_bert.py @@ -541,11 +541,15 @@ BERT_START_DOCSTRING = r""" The BERT model was proposed in .. _`tf.keras.Model`: https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/Model - Important note on the model inputs: - The inputs of the TF 2.0 models are slightly different from the PyTorch ones since - TF 2.0 Keras doesn't accept named arguments with defaults values for input Tensor. - More precisely, input Tensors are gathered in the first arguments of the model call function: `model(inputs)`. - There are three possibilities to gather and feed the inputs to the model: + Note on the model inputs: + TF 2.0 models accepts two formats as inputs: + + - having all inputs as keyword arguments (like PyTorch models), or + - having all inputs as a list, tuple or dict in the first positional arguments. + + This second option is usefull when using `tf.keras.Model.fit()` method which currently requires having all the tensors in the first argument of the model call function: `model(inputs)`. + + If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the first positional argument : - a single Tensor with input_ids only and nothing else: `model(inputs_ids) - a list of varying length with one or several input Tensors IN THE ORDER given in the docstring: @@ -561,7 +565,7 @@ BERT_START_DOCSTRING = r""" The BERT model was proposed in BERT_INPUTS_DOCSTRING = r""" Inputs: - **input_ids**: ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + **input_ids**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: Indices of input sequence tokens in the vocabulary. To match pre-training, BERT input sequence should be formatted with [CLS] and [SEP] tokens as follows: @@ -583,19 +587,19 @@ BERT_INPUTS_DOCSTRING = r""" Indices can be obtained using :class:`pytorch_transformers.BertTokenizer`. See :func:`pytorch_transformers.PreTrainedTokenizer.encode` and :func:`pytorch_transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. - **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``: + **attention_mask**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``: ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. - **token_type_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + **token_type_ids**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: Segment token indices to indicate first and second portions of the inputs. Indices are selected in ``[0, 1]``: ``0`` corresponds to a `sentence A` token, ``1`` corresponds to a `sentence B` token (see `BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ for more details). - **position_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + **position_ids**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: Indices of positions of each input sequence tokens in the position embeddings. Selected in the range ``[0, config.max_position_embeddings - 1]``. - **head_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: + **head_mask**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. @@ -606,9 +610,9 @@ BERT_INPUTS_DOCSTRING = r""" class TFBertModel(TFBertPreTrainedModel): r""" Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: - **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)`` + **last_hidden_state**: ``tf.Tensor`` of shape ``(batch_size, sequence_length, hidden_size)`` Sequence of hidden-states at the output of the last layer of the model. - **pooler_output**: ``torch.FloatTensor`` of shape ``(batch_size, hidden_size)`` + **pooler_output**: ``tf.Tensor`` of shape ``(batch_size, hidden_size)`` Last layer hidden-state of the first token of the sequence (classification token) further processed by a Linear layer and a Tanh activation function. The Linear layer weights are trained from the next sentence prediction (classification) @@ -616,18 +620,21 @@ class TFBertModel(TFBertPreTrainedModel): of the semantic content of the input, you're often better with averaging or pooling the sequence of hidden-states for the whole input sequence. **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) - list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + list of ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) of shape ``(batch_size, sequence_length, hidden_size)``: Hidden-states of the model at the output of each layer plus the initial embedding outputs. **attentions**: (`optional`, returned when ``config.output_attentions=True``) - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Examples:: + import tensorflow as tf + from pytorch_transformers import BertTokenizer, TFBertModel + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = TFBertModel.from_pretrained('bert-base-uncased') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -647,23 +654,26 @@ class TFBertModel(TFBertPreTrainedModel): class TFBertForPreTraining(TFBertPreTrainedModel): r""" Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: - **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` + **prediction_scores**: ```tf.Tensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). - **seq_relationship_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, 2)`` + **seq_relationship_scores**: ```tf.Tensor`` of shape ``(batch_size, sequence_length, 2)`` Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation before SoftMax). **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) - list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + list of ```tf.Tensor`` (one for the output of each layer + the output of the embeddings) of shape ``(batch_size, sequence_length, hidden_size)``: Hidden-states of the model at the output of each layer plus the initial embedding outputs. **attentions**: (`optional`, returned when ``config.output_attentions=True``) - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ```tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Examples:: + import tensorflow as tf + from pytorch_transformers import BertTokenizer, TFBertForPreTraining + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = TFBertForPreTraining.from_pretrained('bert-base-uncased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 outputs = model(input_ids) prediction_scores, seq_relationship_scores = outputs[:2] @@ -692,23 +702,26 @@ class TFBertForPreTraining(TFBertPreTrainedModel): class TFBertForMaskedLM(TFBertPreTrainedModel): r""" Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: - **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` + **prediction_scores**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) - list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + list of ``Numpy array`` or ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) of shape ``(batch_size, sequence_length, hidden_size)``: Hidden-states of the model at the output of each layer plus the initial embedding outputs. **attentions**: (`optional`, returned when ``config.output_attentions=True``) - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``Numpy array`` or ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Examples:: + import tensorflow as tf + from pytorch_transformers import BertTokenizer, TFBertForMaskedLM + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = TFBertForMaskedLM.from_pretrained('bert-base-uncased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 outputs = model(input_ids) - prediction_scores = outputs[:2] + prediction_scores = outputs[0] """ def __init__(self, config, *inputs, **kwargs): @@ -733,23 +746,26 @@ class TFBertForMaskedLM(TFBertPreTrainedModel): class TFBertForNextSentencePrediction(TFBertPreTrainedModel): r""" Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: - **loss**: (`optional`, returned when ``next_sentence_label`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + **loss**: (`optional`, returned when ``next_sentence_label`` is provided) ``Numpy array`` or ``tf.Tensor`` of shape ``(1,)``: Next sequence prediction (classification) loss. - **seq_relationship_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, 2)`` + **seq_relationship_scores**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length, 2)`` Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation before SoftMax). **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) - list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + list of ``Numpy array`` or ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) of shape ``(batch_size, sequence_length, hidden_size)``: Hidden-states of the model at the output of each layer plus the initial embedding outputs. **attentions**: (`optional`, returned when ``config.output_attentions=True``) - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``Numpy array`` or ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Examples:: + import tensorflow as tf + from pytorch_transformers import BertTokenizer, TFBertForNextSentencePrediction + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = TFBertForNextSentencePrediction.from_pretrained('bert-base-uncased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 outputs = model(input_ids) seq_relationship_scores = outputs[0] @@ -777,23 +793,26 @@ class TFBertForNextSentencePrediction(TFBertPreTrainedModel): class TFBertForSequenceClassification(TFBertPreTrainedModel): r""" Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: - **logits**: ``torch.FloatTensor`` of shape ``(batch_size, config.num_labels)`` + **logits**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, config.num_labels)`` Classification (or regression if config.num_labels==1) scores (before SoftMax). **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) - list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + list of ``Numpy array`` or ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) of shape ``(batch_size, sequence_length, hidden_size)``: Hidden-states of the model at the output of each layer plus the initial embedding outputs. **attentions**: (`optional`, returned when ``config.output_attentions=True``) - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``Numpy array`` or ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Examples:: + import tensorflow as tf + from pytorch_transformers import BertTokenizer, TFBertForSequenceClassification + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = TFBertForSequenceClassification.from_pretrained('bert-base-uncased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 outputs = model(input_ids) - loss, logits = outputs[:2] + logits = outputs[0] """ def __init__(self, config, *inputs, **kwargs): @@ -823,25 +842,28 @@ class TFBertForSequenceClassification(TFBertPreTrainedModel): class TFBertForMultipleChoice(TFBertPreTrainedModel): r""" Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: - **classification_scores**: ``torch.FloatTensor`` of shape ``(batch_size, num_choices)`` where `num_choices` is the size of the second dimension + **classification_scores**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, num_choices)`` where `num_choices` is the size of the second dimension of the input tensors. (see `input_ids` above). Classification scores (before SoftMax). **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) - list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + list of ``Numpy array`` or ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) of shape ``(batch_size, sequence_length, hidden_size)``: Hidden-states of the model at the output of each layer plus the initial embedding outputs. **attentions**: (`optional`, returned when ``config.output_attentions=True``) - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``Numpy array`` or ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Examples:: + import tensorflow as tf + from pytorch_transformers import BertTokenizer, TFBertForMultipleChoice + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') - model = BertForMultipleChoice.from_pretrained('bert-base-uncased') + model = TFBertForMultipleChoice.from_pretrained('bert-base-uncased') choices = ["Hello, my dog is cute", "Hello, my cat is amazing"] - input_ids = torch.tensor([tokenizer.encode(s) for s in choices]).unsqueeze(0) # Batch size 1, 2 choices + input_ids = tf.constant([tokenizer.encode(s) for s in choices])[None, :] # Batch size 1, 2 choices outputs = model(input_ids) - loss, classification_scores = outputs[:2] + classification_scores = outputs[0] """ def __init__(self, config, *inputs, **kwargs): @@ -898,23 +920,26 @@ class TFBertForMultipleChoice(TFBertPreTrainedModel): class TFBertForTokenClassification(TFBertPreTrainedModel): r""" Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: - **scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.num_labels)`` + **scores**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length, config.num_labels)`` Classification scores (before SoftMax). **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) - list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + list of ``Numpy array`` or ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) of shape ``(batch_size, sequence_length, hidden_size)``: Hidden-states of the model at the output of each layer plus the initial embedding outputs. **attentions**: (`optional`, returned when ``config.output_attentions=True``) - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``Numpy array`` or ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Examples:: + import tensorflow as tf + from pytorch_transformers import BertTokenizer, TFBertForTokenClassification + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') - model = BertForTokenClassification.from_pretrained('bert-base-uncased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + model = TFBertForTokenClassification.from_pretrained('bert-base-uncased') + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 outputs = model(input_ids) - loss, scores = outputs[:2] + scores = outputs[0] """ def __init__(self, config, *inputs, **kwargs): @@ -944,27 +969,30 @@ class TFBertForTokenClassification(TFBertPreTrainedModel): class TFBertForQuestionAnswering(TFBertPreTrainedModel): r""" Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: - **start_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length,)`` + **start_scores**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length,)`` Span-start scores (before SoftMax). - **end_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length,)`` + **end_scores**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length,)`` Span-end scores (before SoftMax). **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) - list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + list of ``Numpy array`` or ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) of shape ``(batch_size, sequence_length, hidden_size)``: Hidden-states of the model at the output of each layer plus the initial embedding outputs. **attentions**: (`optional`, returned when ``config.output_attentions=True``) - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``Numpy array`` or ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Examples:: + import tensorflow as tf + from pytorch_transformers import BertTokenizer, TFBertForQuestionAnswering + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') - model = BertForQuestionAnswering.from_pretrained('bert-base-uncased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 - start_positions = torch.tensor([1]) - end_positions = torch.tensor([3]) + model = TFBertForQuestionAnswering.from_pretrained('bert-base-uncased') + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + start_positions = tf.constant([1]) + end_positions = tf.constant([3]) outputs = model(input_ids, start_positions=start_positions, end_positions=end_positions) - loss, start_scores, end_scores = outputs[:2] + start_scores, end_scores = outputs[:2] """ def __init__(self, config, *inputs, **kwargs): From b94f73bab76fcd10b0681469d5c2b9f23059d76b Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 24 Sep 2019 15:06:51 +0200 Subject: [PATCH 112/219] distilbert docstring --- pytorch_transformers/modeling_tf_bert.py | 2 - .../modeling_tf_distilbert.py | 177 ++++++++---------- 2 files changed, 81 insertions(+), 98 deletions(-) diff --git a/pytorch_transformers/modeling_tf_bert.py b/pytorch_transformers/modeling_tf_bert.py index 583d403963..f9d6cd9990 100644 --- a/pytorch_transformers/modeling_tf_bert.py +++ b/pytorch_transformers/modeling_tf_bert.py @@ -746,8 +746,6 @@ class TFBertForMaskedLM(TFBertPreTrainedModel): class TFBertForNextSentencePrediction(TFBertPreTrainedModel): r""" Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: - **loss**: (`optional`, returned when ``next_sentence_label`` is provided) ``Numpy array`` or ``tf.Tensor`` of shape ``(1,)``: - Next sequence prediction (classification) loss. **seq_relationship_scores**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length, 2)`` Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation before SoftMax). **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) diff --git a/pytorch_transformers/modeling_tf_distilbert.py b/pytorch_transformers/modeling_tf_distilbert.py index 1811573bbf..7ca8d8c815 100644 --- a/pytorch_transformers/modeling_tf_distilbert.py +++ b/pytorch_transformers/modeling_tf_distilbert.py @@ -125,12 +125,12 @@ class TFEmbeddings(tf.keras.layers.Layer): """ Parameters ---------- - input_ids: torch.tensor(bs, max_seq_length) + input_ids: tf.Tensor(bs, max_seq_length) The token ids to embed. Outputs ------- - embeddings: torch.tensor(bs, max_seq_length, dim) + embeddings: tf.Tensor(bs, max_seq_length, dim) The embedded tokens (plus position embeddings, no token_type embeddings) """ if not isinstance(inputs, (tuple, list)): @@ -192,16 +192,16 @@ class TFMultiHeadSelfAttention(tf.keras.layers.Layer): """ Parameters ---------- - query: torch.tensor(bs, seq_length, dim) - key: torch.tensor(bs, seq_length, dim) - value: torch.tensor(bs, seq_length, dim) - mask: torch.tensor(bs, seq_length) + query: tf.Tensor(bs, seq_length, dim) + key: tf.Tensor(bs, seq_length, dim) + value: tf.Tensor(bs, seq_length, dim) + mask: tf.Tensor(bs, seq_length) Outputs ------- - weights: torch.tensor(bs, n_heads, seq_length, seq_length) + weights: tf.Tensor(bs, n_heads, seq_length, seq_length) Attention weights - context: torch.tensor(bs, seq_length, dim) + context: tf.Tensor(bs, seq_length, dim) Contextualized layer. Optional: only if `output_attentions=True` """ query, key, value, mask, head_mask = inputs @@ -290,14 +290,14 @@ class TFTransformerBlock(tf.keras.layers.Layer): """ Parameters ---------- - x: torch.tensor(bs, seq_length, dim) - attn_mask: torch.tensor(bs, seq_length) + x: tf.Tensor(bs, seq_length, dim) + attn_mask: tf.Tensor(bs, seq_length) Outputs ------- - sa_weights: torch.tensor(bs, n_heads, seq_length, seq_length) + sa_weights: tf.Tensor(bs, n_heads, seq_length, seq_length) The attention weights - ffn_output: torch.tensor(bs, seq_length, dim) + ffn_output: tf.Tensor(bs, seq_length, dim) The output of the transformer block contextualization. """ x, attn_mask, head_mask = inputs @@ -335,19 +335,19 @@ class TFTransformer(tf.keras.layers.Layer): """ Parameters ---------- - x: torch.tensor(bs, seq_length, dim) + x: tf.Tensor(bs, seq_length, dim) Input sequence embedded. - attn_mask: torch.tensor(bs, seq_length) + attn_mask: tf.Tensor(bs, seq_length) Attention mask on the sequence. Outputs ------- - hidden_state: torch.tensor(bs, seq_length, dim) + hidden_state: tf.Tensor(bs, seq_length, dim) Sequence of hiddens states in the last (top) layer - all_hidden_states: Tuple[torch.tensor(bs, seq_length, dim)] + all_hidden_states: Tuple[tf.Tensor(bs, seq_length, dim)] Tuple of length n_layers with the hidden states from each layer. Optional: only if output_hidden_states=True - all_attentions: Tuple[torch.tensor(bs, n_heads, seq_length, seq_length)] + all_attentions: Tuple[tf.Tensor(bs, n_heads, seq_length, seq_length)] Tuple of length n_layers with the attention weights from each layer Optional: only if output_attentions=True """ @@ -384,27 +384,6 @@ class TFTransformer(tf.keras.layers.Layer): class TFDistilBertMainLayer(tf.keras.layers.Layer): - r""" - Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: - **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)`` - Sequence of hidden-states at the output of the last layer of the model. - **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) - list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) - of shape ``(batch_size, sequence_length, hidden_size)``: - Hidden-states of the model at the output of each layer plus the initial embedding outputs. - **attentions**: (`optional`, returned when ``config.output_attentions=True``) - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: - Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. - - Examples:: - - tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased') - model = DistilBertModel.from_pretrained('distilbert-base-uncased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 - outputs = model(input_ids) - last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple - - """ def __init__(self, config, **kwargs): super(TFDistilBertMainLayer, self).__init__(**kwargs) self.num_hidden_layers = config.num_hidden_layers @@ -477,9 +456,31 @@ DISTILBERT_START_DOCSTRING = r""" For more information on DistilBERT, please refer to our `detailed blog post`_ + This model is a tf.keras.Model `tf.keras.Model`_ sub-class. Use it as a regular TF 2.0 Keras Model and + refer to the TF 2.0 documentation for all matter related to general usage and behavior. + .. _`detailed blog post`: https://medium.com/huggingface/distilbert-8cf3380435b5 + .. _`tf.keras.Model`: + https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/Model + + Note on the model inputs: + TF 2.0 models accepts two formats as inputs: + + - having all inputs as keyword arguments (like PyTorch models), or + - having all inputs as a list, tuple or dict in the first positional arguments. + + This second option is usefull when using `tf.keras.Model.fit()` method which currently requires having all the tensors in the first argument of the model call function: `model(inputs)`. + + If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the first positional argument : + + - a single Tensor with input_ids only and nothing else: `model(inputs_ids) + - a list of varying length with one or several input Tensors IN THE ORDER given in the docstring: + `model([input_ids, attention_mask])` or `model([input_ids, attention_mask, token_type_ids])` + - a dictionary with one or several input Tensors associaed to the input names given in the docstring: + `model({'input_ids': input_ids, 'token_type_ids': token_type_ids})` + Parameters: config (:class:`~pytorch_transformers.DistilBertConfig`): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. @@ -488,16 +489,16 @@ DISTILBERT_START_DOCSTRING = r""" DISTILBERT_INPUTS_DOCSTRING = r""" Inputs: - **input_ids** ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + **input_ids** ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: Indices of input sequence tokens in the vocabulary. The input sequences should start with `[CLS]` and end with `[SEP]` tokens. For now, ONLY BertTokenizer(`bert-base-uncased`) is supported and you should use this tokenizer when using DistilBERT. - **attention_mask**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + **attention_mask**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``: ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. - **head_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: + **head_mask**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. @@ -508,21 +509,24 @@ DISTILBERT_INPUTS_DOCSTRING = r""" class TFDistilBertModel(TFDistilBertPreTrainedModel): r""" Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: - **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)`` + **last_hidden_state**: ``tf.Tensor`` of shape ``(batch_size, sequence_length, hidden_size)`` Sequence of hidden-states at the output of the last layer of the model. **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) - list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + list of ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) of shape ``(batch_size, sequence_length, hidden_size)``: Hidden-states of the model at the output of each layer plus the initial embedding outputs. **attentions**: (`optional`, returned when ``config.output_attentions=True``) - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Examples:: + import tensorflow as tf + from pytorch_transformers import DistilBertTokenizer, TFDistilBertModel + tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased') - model = DistilBertModel.from_pretrained('distilbert-base-uncased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + model = TFDistilBertModel.from_pretrained('distilbert-base-uncased') + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -562,32 +566,27 @@ class TFDistilBertLMHead(tf.keras.layers.Layer): DISTILBERT_START_DOCSTRING, DISTILBERT_INPUTS_DOCSTRING) class TFDistilBertForMaskedLM(TFDistilBertPreTrainedModel): r""" - **masked_lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: - Labels for computing the masked language modeling loss. - Indices should be in ``[-1, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring) - Tokens with indices set to ``-1`` are ignored (masked), the loss is only computed for the tokens with labels - in ``[0, ..., config.vocab_size]`` - Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: - **loss**: (`optional`, returned when ``masked_lm_labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: - Masked language modeling loss. - **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` + **prediction_scores**: ``tf.Tensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) - list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + list of ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) of shape ``(batch_size, sequence_length, hidden_size)``: Hidden-states of the model at the output of each layer plus the initial embedding outputs. **attentions**: (`optional`, returned when ``config.output_attentions=True``) - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Examples:: + import tensorflow as tf + from pytorch_transformers import DistilBertTokenizer, TFDistilBertForMaskedLM + tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased') - model = DistilBertForMaskedLM.from_pretrained('distilbert-base-uncased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + model = TFDistilBertForMaskedLM.from_pretrained('distilbert-base-uncased') + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 outputs = model(input_ids, masked_lm_labels=input_ids) - loss, prediction_scores = outputs[:2] + prediction_scores = outputs[0] """ def __init__(self, config, *inputs, **kwargs): @@ -620,33 +619,27 @@ class TFDistilBertForMaskedLM(TFDistilBertPreTrainedModel): DISTILBERT_START_DOCSTRING, DISTILBERT_INPUTS_DOCSTRING) class TFDistilBertForSequenceClassification(TFDistilBertPreTrainedModel): r""" - **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: - Labels for computing the sequence classification/regression loss. - Indices should be in ``[0, ..., config.num_labels - 1]``. - If ``config.num_labels == 1`` a regression loss is computed (Mean-Square loss), - If ``config.num_labels > 1`` a classification loss is computed (Cross-Entropy). - Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: - **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: - Classification (or regression if config.num_labels==1) loss. - **logits**: ``torch.FloatTensor`` of shape ``(batch_size, config.num_labels)`` + **logits**: ``tf.Tensor`` of shape ``(batch_size, config.num_labels)`` Classification (or regression if config.num_labels==1) scores (before SoftMax). **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) - list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + list of ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) of shape ``(batch_size, sequence_length, hidden_size)``: Hidden-states of the model at the output of each layer plus the initial embedding outputs. **attentions**: (`optional`, returned when ``config.output_attentions=True``) - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Examples:: + import tensorflow as tf + from pytorch_transformers import BertTokenizer, TFDistilBertForSequenceClassification + tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased') - model = DistilBertForSequenceClassification.from_pretrained('distilbert-base-uncased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 - labels = torch.tensor([1]).unsqueeze(0) # Batch size 1 - outputs = model(input_ids, labels=labels) - loss, logits = outputs[:2] + model = TFDistilBertForSequenceClassification.from_pretrained('distilbert-base-uncased') + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + outputs = model(input_ids) + logits = outputs[0] """ def __init__(self, config, *inputs, **kwargs): @@ -676,39 +669,31 @@ class TFDistilBertForSequenceClassification(TFDistilBertPreTrainedModel): DISTILBERT_START_DOCSTRING, DISTILBERT_INPUTS_DOCSTRING) class TFDistilBertForQuestionAnswering(TFDistilBertPreTrainedModel): r""" - **start_positions**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). - Position outside of the sequence are not taken into account for computing the loss. - **end_positions**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). - Position outside of the sequence are not taken into account for computing the loss. - Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: - **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: - Total span extraction loss is the sum of a Cross-Entropy for the start and end positions. - **start_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length,)`` + **start_scores**: ``tf.Tensor`` of shape ``(batch_size, sequence_length,)`` Span-start scores (before SoftMax). - **end_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length,)`` + **end_scores**: ``tf.Tensor`` of shape ``(batch_size, sequence_length,)`` Span-end scores (before SoftMax). **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) - list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + list of ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) of shape ``(batch_size, sequence_length, hidden_size)``: Hidden-states of the model at the output of each layer plus the initial embedding outputs. **attentions**: (`optional`, returned when ``config.output_attentions=True``) - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Examples:: + import tensorflow as tf + from pytorch_transformers import BertTokenizer, TFDistilBertForQuestionAnswering + tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased') - model = DistilBertForQuestionAnswering.from_pretrained('distilbert-base-uncased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 - start_positions = torch.tensor([1]) - end_positions = torch.tensor([3]) + model = TFDistilBertForQuestionAnswering.from_pretrained('distilbert-base-uncased') + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + start_positions = tf.constant([1]) + end_positions = tf.constant([3]) outputs = model(input_ids, start_positions=start_positions, end_positions=end_positions) - loss, start_scores, end_scores = outputs[:2] + start_scores, end_scores = outputs[:2] """ def __init__(self, config, *inputs, **kwargs): From d340e2329e13674e61bbe763ea577769ef1984c8 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Tue, 24 Sep 2019 09:09:28 -0400 Subject: [PATCH 113/219] create_mask_from_sequences -> create_token_type_ids_from_sequences --- pytorch_transformers/tokenization_bert.py | 2 +- pytorch_transformers/tokenization_distilbert.py | 11 ----------- pytorch_transformers/tokenization_roberta.py | 2 +- pytorch_transformers/tokenization_utils.py | 4 ++-- pytorch_transformers/tokenization_xlm.py | 2 +- pytorch_transformers/tokenization_xlnet.py | 2 +- 6 files changed, 6 insertions(+), 17 deletions(-) diff --git a/pytorch_transformers/tokenization_bert.py b/pytorch_transformers/tokenization_bert.py index 211b8fe93b..7eca60a140 100644 --- a/pytorch_transformers/tokenization_bert.py +++ b/pytorch_transformers/tokenization_bert.py @@ -204,7 +204,7 @@ class BertTokenizer(PreTrainedTokenizer): return cls + token_ids_0 + sep + token_ids_1 + sep - def create_mask_from_sequences(self, sequence_0, sequence_1): + def create_token_type_ids_from_sequences(self, sequence_0, sequence_1): """ Creates a mask from the two sequences passed to be used in a sequence-pair classification task. A BERT sequence pair mask has the following format: diff --git a/pytorch_transformers/tokenization_distilbert.py b/pytorch_transformers/tokenization_distilbert.py index 0af782beb1..547ea29981 100644 --- a/pytorch_transformers/tokenization_distilbert.py +++ b/pytorch_transformers/tokenization_distilbert.py @@ -67,14 +67,3 @@ class DistilBertTokenizer(BertTokenizer): def add_special_tokens_sequence_pair(self, token_ids_0, token_ids_1): sep = [self.sep_token_id] return token_ids_0 + sep + token_ids_1 - - def create_mask_from_sequences(self, sequence_0, sequence_1): - """ - Creates a mask from the two sequences passed to be used in a sequence-pair classification task. - A BERT sequence pair mask has the following format: - 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 - | first sequence | second sequence - """ - sep = [self.sep_token_id] - - return len(self.encode(sequence_0) + sep) * [0] + len(self.encode(sequence_1)) * [1] diff --git a/pytorch_transformers/tokenization_roberta.py b/pytorch_transformers/tokenization_roberta.py index 161c4e6870..475aee47fa 100644 --- a/pytorch_transformers/tokenization_roberta.py +++ b/pytorch_transformers/tokenization_roberta.py @@ -97,7 +97,7 @@ class RobertaTokenizer(GPT2Tokenizer): cls = [self.cls_token_id] return cls + token_ids_0 + sep + sep + token_ids_1 + sep - def create_mask_from_sequences(self, sequence_0, sequence_1): + def create_token_type_ids_from_sequences(self, sequence_0, sequence_1): """ Creates a mask from the two sequences passed to be used in a sequence-pair classification task. A RoBERTa sequence pair mask has the following format: diff --git a/pytorch_transformers/tokenization_utils.py b/pytorch_transformers/tokenization_utils.py index 02b4bef699..c5efd37a53 100644 --- a/pytorch_transformers/tokenization_utils.py +++ b/pytorch_transformers/tokenization_utils.py @@ -780,7 +780,7 @@ class PreTrainedTokenizer(object): ) if output_token_type: - information["token_type_ids"] = self.create_mask_from_sequences(text, text_pair) + information["token_type_ids"] = self.create_token_type_ids_from_sequences(text, text_pair) else: logger.warning("No special tokens were added. The two sequences have been concatenated.") sequence = first_sentence_tokens + second_sentence_tokens @@ -863,7 +863,7 @@ class PreTrainedTokenizer(object): return information - def create_mask_from_sequences(self, sequence_0, sequence_1): + def create_token_type_ids_from_sequences(self, sequence_0, sequence_1): logger.warning("This tokenizer does not make use of special tokens.") return [0] * len(self.encode(sequence_0)) + [1] * len(self.encode(sequence_1)) diff --git a/pytorch_transformers/tokenization_xlm.py b/pytorch_transformers/tokenization_xlm.py index 3b177b6a17..833a8d8be6 100644 --- a/pytorch_transformers/tokenization_xlm.py +++ b/pytorch_transformers/tokenization_xlm.py @@ -770,7 +770,7 @@ class XLMTokenizer(PreTrainedTokenizer): cls = [self.cls_token_id] return cls + token_ids_0 + sep + token_ids_1 + sep - def create_mask_from_sequences(self, sequence_0, sequence_1): + def create_token_type_ids_from_sequences(self, sequence_0, sequence_1): """ Creates a mask from the two sequences passed to be used in a sequence-pair classification task. An XLM sequence pair mask has the following format: diff --git a/pytorch_transformers/tokenization_xlnet.py b/pytorch_transformers/tokenization_xlnet.py index 1464743759..5febf16418 100644 --- a/pytorch_transformers/tokenization_xlnet.py +++ b/pytorch_transformers/tokenization_xlnet.py @@ -200,7 +200,7 @@ class XLNetTokenizer(PreTrainedTokenizer): cls = [self.cls_token_id] return token_ids_0 + sep + token_ids_1 + sep + cls - def create_mask_from_sequences(self, sequence_0, sequence_1): + def create_token_type_ids_from_sequences(self, sequence_0, sequence_1): """ Creates a mask from the two sequences passed to be used in a sequence-pair classification task. A BERT sequence pair mask has the following format: From e7ba5bc85b52f3107ea2e1670491f6e6faa39718 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 24 Sep 2019 15:12:36 +0200 Subject: [PATCH 114/219] docstring for GPT2 --- pytorch_transformers/modeling_tf_gpt2.py | 85 +++++++++++++----------- 1 file changed, 47 insertions(+), 38 deletions(-) diff --git a/pytorch_transformers/modeling_tf_gpt2.py b/pytorch_transformers/modeling_tf_gpt2.py index 858ea7dd42..5d686ae704 100644 --- a/pytorch_transformers/modeling_tf_gpt2.py +++ b/pytorch_transformers/modeling_tf_gpt2.py @@ -362,11 +362,15 @@ GPT2_START_DOCSTRING = r""" OpenAI GPT-2 model was proposed in .. _`tf.keras.Model`: https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/Model - Important note on the model inputs: - The inputs of the TF 2.0 models are slightly different from the PyTorch ones since - TF 2.0 Keras doesn't accept named arguments with defaults values for input Tensor. - More precisely, input Tensors are gathered in the first arguments of the model call function: `model(inputs)`. - There are three possibilities to gather and feed the inputs to the model: + Note on the model inputs: + TF 2.0 models accepts two formats as inputs: + + - having all inputs as keyword arguments (like PyTorch models), or + - having all inputs as a list, tuple or dict in the first positional arguments. + + This second option is usefull when using `tf.keras.Model.fit()` method which currently requires having all the tensors in the first argument of the model call function: `model(inputs)`. + + If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the first positional argument : - a single Tensor with input_ids only and nothing else: `model(inputs_ids) - a list of varying length with one or several input Tensors IN THE ORDER given in the docstring: @@ -381,7 +385,7 @@ GPT2_START_DOCSTRING = r""" OpenAI GPT-2 model was proposed in """ GPT2_INPUTS_DOCSTRING = r""" Inputs: - **input_ids**: ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + **input_ids**: ```Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: Indices of input sequence tokens in the vocabulary. GPT-2 is a model with absolute position embeddings so it's usually advised to pad the inputs on the right rather than the left. @@ -389,21 +393,21 @@ GPT2_INPUTS_DOCSTRING = r""" Inputs: See :func:`pytorch_transformers.PreTrainedTokenizer.encode` and :func:`pytorch_transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. **past**: - list of ``torch.FloatTensor`` (one for each layer): + list of ``Numpy array`` or ``tf.Tensor`` (one for each layer): that contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model (see `past` output below). Can be used to speed up sequential decoding. - **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``: + **attention_mask**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``: ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. - **token_type_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + **token_type_ids**: (`optional`) ```Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: A parallel sequence of tokens (can be used to indicate various portions of the inputs). The embeddings from these tokens will be summed with the respective token embeddings. Indices are selected in the vocabulary (unlike BERT which has a specific vocabulary for segment indices). - **position_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + **position_ids**: (`optional`) ```Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: Indices of positions of each input sequence tokens in the position embeddings. Selected in the range ``[0, config.max_position_embeddings - 1]``. - **head_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: + **head_mask**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. @@ -414,25 +418,28 @@ GPT2_INPUTS_DOCSTRING = r""" Inputs: class TFGPT2Model(TFGPT2PreTrainedModel): r""" Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: - **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)`` + **last_hidden_state**: `tf.Tensor`` of shape ``(batch_size, sequence_length, hidden_size)`` Sequence of hidden-states at the last layer of the model. **past**: - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of `tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: that contains pre-computed hidden-states (key and values in the attention blocks). Can be used (see `past` input) to speed up sequential decoding. **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) - list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + list of `tf.Tensor`` (one for the output of each layer + the output of the embeddings) of shape ``(batch_size, sequence_length, hidden_size)``: Hidden-states of the model at the output of each layer plus the initial embedding outputs. **attentions**: (`optional`, returned when ``config.output_attentions=True``) - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of `tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Examples:: + import tensorflow as tf + from pytorch_transformers import GPT2Tokenizer, TFGPT2Model + tokenizer = GPT2Tokenizer.from_pretrained('gpt2') - model = GPT2Model.from_pretrained('gpt2') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + model = TFGPT2Model.from_pretrained('gpt2') + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -451,31 +458,31 @@ class TFGPT2Model(TFGPT2PreTrainedModel): class TFGPT2LMHeadModel(TFGPT2PreTrainedModel): r""" Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: - **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` + **prediction_scores**: `tf.Tensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). **past**: - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of `tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: that contains pre-computed hidden-states (key and values in the attention blocks). Can be used (see `past` input) to speed up sequential decoding. **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) - list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + list of `tf.Tensor`` (one for the output of each layer + the output of the embeddings) of shape ``(batch_size, sequence_length, hidden_size)``: Hidden-states of the model at the output of each layer plus the initial embedding outputs. **attentions**: (`optional`, returned when ``config.output_attentions=True``) - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of `tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Examples:: - import torch - from pytorch_transformers import GPT2Tokenizer, GPT2LMHeadModel + import tensorflow as tf + from pytorch_transformers import GPT2Tokenizer, TFGPT2LMHeadModel tokenizer = GPT2Tokenizer.from_pretrained('gpt2') - model = GPT2LMHeadModel.from_pretrained('gpt2') + model = TFGPT2LMHeadModel.from_pretrained('gpt2') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 outputs = model(input_ids) - logits = outputs[:2] + logits = outputs[0] """ def __init__(self, config, *inputs, **kwargs): @@ -500,36 +507,38 @@ the classification head takes as input the input of a specified classification t """, GPT2_START_DOCSTRING, GPT2_INPUTS_DOCSTRING) class TFGPT2DoubleHeadsModel(TFGPT2PreTrainedModel): r""" - **mc_token_ids**: (`optional`, default to index of the last token of the input) ``torch.LongTensor`` of shape ``(batch_size, num_choices)``: + **mc_token_ids**: (`optional`, default to index of the last token of the input) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, num_choices)``: Index of the classification token in each input sequence. Selected in the range ``[0, input_ids.size(-1) - 1[``. Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: - **lm_prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, num_choices, sequence_length, config.vocab_size)`` + **lm_prediction_scores**: `tf.Tensor`` of shape ``(batch_size, num_choices, sequence_length, config.vocab_size)`` Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). - **mc_prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, num_choices)`` + **mc_prediction_scores**: `tf.Tensor`` of shape ``(batch_size, num_choices)`` Prediction scores of the multiplechoice classification head (scores for each choice before SoftMax). **past**: - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of `tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: that contains pre-computed hidden-states (key and values in the attention blocks). Can be used (see `past` input) to speed up sequential decoding. **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) - list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + list of `tf.Tensor`` (one for the output of each layer + the output of the embeddings) of shape ``(batch_size, sequence_length, hidden_size)``: Hidden-states of the model at the output of each layer plus the initial embedding outputs. **attentions**: (`optional`, returned when ``config.output_attentions=True``) - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of `tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Examples:: - import torch - from pytorch_transformers import GPT2Tokenizer, GPT2DoubleHeadsModel - + import tensorflow as tf + from pytorch_transformers import GPT2Tokenizer, TFGPT2DoubleHeadsModel + tokenizer = GPT2Tokenizer.from_pretrained('gpt2') - model = GPT2DoubleHeadsModel.from_pretrained('gpt2') + model = TFGPT2DoubleHeadsModel.from_pretrained('gpt2') # Add a [CLS] to the vocabulary (we should train it also!) + # This option is currently not implemented in TF 2.0 + raise NotImplementedError tokenizer.add_special_tokens({'cls_token': '[CLS]'}) model.resize_token_embeddings(len(tokenizer)) # Update the model embeddings with the new vocabulary size print(tokenizer.cls_token_id, len(tokenizer)) # The newly token the last token of the vocabulary @@ -538,8 +547,8 @@ class TFGPT2DoubleHeadsModel(TFGPT2PreTrainedModel): encoded_choices = [tokenizer.encode(s) for s in choices] cls_token_location = [tokens.index(tokenizer.cls_token_id) for tokens in encoded_choices] - input_ids = torch.tensor(encoded_choices).unsqueeze(0) # Batch size: 1, number of choices: 2 - mc_token_ids = torch.tensor([cls_token_location]) # Batch size: 1 + input_ids = tf.constant(encoded_choices)[None, :] # Batch size: 1, number of choices: 2 + mc_token_ids = tf.constant([cls_token_location]) # Batch size: 1 outputs = model(input_ids, mc_token_ids=mc_token_ids) lm_prediction_scores, mc_prediction_scores = outputs[:2] From 45a6f2edd920765aad5ef46e5c06330d95170c8c Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 24 Sep 2019 15:15:47 +0200 Subject: [PATCH 115/219] docstring for GPT --- pytorch_transformers/modeling_tf_openai.py | 68 +++++++++++++--------- 1 file changed, 41 insertions(+), 27 deletions(-) diff --git a/pytorch_transformers/modeling_tf_openai.py b/pytorch_transformers/modeling_tf_openai.py index 5bfe5b2abc..18b07e0637 100644 --- a/pytorch_transformers/modeling_tf_openai.py +++ b/pytorch_transformers/modeling_tf_openai.py @@ -348,11 +348,15 @@ OPENAI_GPT_START_DOCSTRING = r""" OpenAI GPT model was proposed in .. _`tf.keras.Model`: https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/Model - Important note on the model inputs: - The inputs of the TF 2.0 models are slightly different from the PyTorch ones since - TF 2.0 Keras doesn't accept named arguments with defaults values for input Tensor. - More precisely, input Tensors are gathered in the first arguments of the model call function: `model(inputs)`. - There are three possibilities to gather and feed the inputs to the model: + Note on the model inputs: + TF 2.0 models accepts two formats as inputs: + + - having all inputs as keyword arguments (like PyTorch models), or + - having all inputs as a list, tuple or dict in the first positional arguments. + + This second option is usefull when using `tf.keras.Model.fit()` method which currently requires having all the tensors in the first argument of the model call function: `model(inputs)`. + + If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the first positional argument : - a single Tensor with input_ids only and nothing else: `model(inputs_ids) - a list of varying length with one or several input Tensors IN THE ORDER given in the docstring: @@ -367,25 +371,25 @@ OPENAI_GPT_START_DOCSTRING = r""" OpenAI GPT model was proposed in """ OPENAI_GPT_INPUTS_DOCSTRING = r""" Inputs: - **input_ids**: ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + **input_ids**: ```Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: Indices of input sequence tokens in the vocabulary. GPT is a model with absolute position embeddings so it's usually advised to pad the inputs on the right rather than the left. Indices can be obtained using :class:`pytorch_transformers.BPT2Tokenizer`. See :func:`pytorch_transformers.PreTrainedTokenizer.encode` and :func:`pytorch_transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. - **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``: + **attention_mask**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``: ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. - **token_type_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + **token_type_ids**: (`optional`) ```Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: A parallel sequence of tokens (can be used to indicate various portions of the inputs). The embeddings from these tokens will be summed with the respective token embeddings. Indices are selected in the vocabulary (unlike BERT which has a specific vocabulary for segment indices) - **position_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + **position_ids**: (`optional`) ```Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: Indices of positions of each input sequence tokens in the position embeddings. Selected in the range ``[0, config.max_position_embeddings - 1]``. - **head_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: + **head_mask**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. @@ -408,9 +412,12 @@ class TFOpenAIGPTModel(TFOpenAIGPTPreTrainedModel): Examples:: + import tensorflow as tf + from pytorch_transformers import OpenAIGPTTokenizer, TFOpenAIGPTModel + tokenizer = OpenAIGPTTokenizer.from_pretrained('openai-gpt') - model = OpenAIGPTModel.from_pretrained('openai-gpt') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + model = TFOpenAIGPTModel.from_pretrained('openai-gpt') + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -429,8 +436,6 @@ class TFOpenAIGPTModel(TFOpenAIGPTPreTrainedModel): class TFOpenAIGPTLMHeadModel(TFOpenAIGPTPreTrainedModel): r""" Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: - **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: - Language modeling loss. **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) @@ -443,11 +448,14 @@ class TFOpenAIGPTLMHeadModel(TFOpenAIGPTPreTrainedModel): Examples:: + import tensorflow as tf + from pytorch_transformers import OpenAIGPTTokenizer, TFOpenAIGPTLMHeadModel + tokenizer = OpenAIGPTTokenizer.from_pretrained('openai-gpt') - model = OpenAIGPTLMHeadModel.from_pretrained('openai-gpt') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 - outputs = model(input_ids, labels=input_ids) - loss, logits = outputs[:2] + model = TFOpenAIGPTLMHeadModel.from_pretrained('openai-gpt') + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + outputs = model(input_ids) + logits = outputs[0] """ def __init__(self, config, *inputs, **kwargs): @@ -472,15 +480,11 @@ the classification head takes as input the input of a specified classification t """, OPENAI_GPT_START_DOCSTRING, OPENAI_GPT_INPUTS_DOCSTRING) class TFOpenAIGPTDoubleHeadsModel(TFOpenAIGPTPreTrainedModel): r""" - **mc_token_ids**: (`optional`, default to index of the last token of the input) ``torch.LongTensor`` of shape ``(batch_size, num_choices)``: + **mc_token_ids**: (`optional`, default to index of the last token of the input) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, num_choices)``: Index of the classification token in each input sequence. Selected in the range ``[0, input_ids.size(-1) - 1[``. Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: - **lm_loss**: (`optional`, returned when ``lm_labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: - Language modeling loss. - **mc_loss**: (`optional`, returned when ``multiple_choice_labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: - Multiple choice classification loss. **lm_prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, num_choices, sequence_length, config.vocab_size)`` Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). **mc_prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, num_choices)`` @@ -495,12 +499,22 @@ class TFOpenAIGPTDoubleHeadsModel(TFOpenAIGPTPreTrainedModel): Examples:: + import tensorflow as tf + from pytorch_transformers import OpenAIGPTTokenizer, TFOpenAIGPTDoubleHeadsModel + tokenizer = OpenAIGPTTokenizer.from_pretrained('openai-gpt') - model = OpenAIGPTDoubleHeadsModel.from_pretrained('openai-gpt') - tokenizer.add_special_tokens({'cls_token': '[CLS]'}) # Add a [CLS] to the vocabulary (we should train it also!) + model = TFOpenAIGPTDoubleHeadsModel.from_pretrained('openai-gpt') + + # Add a [CLS] to the vocabulary (we should train it also!) + # This option is currently not implemented in TF 2.0 + raise NotImplementedError + tokenizer.add_special_tokens({'cls_token': '[CLS]'}) + model.resize_token_embeddings(len(tokenizer)) # Update the model embeddings with the new vocabulary size + print(tokenizer.cls_token_id, len(tokenizer)) # The newly token the last token of the vocabulary + choices = ["Hello, my dog is cute [CLS]", "Hello, my cat is cute [CLS]"] - input_ids = torch.tensor([tokenizer.encode(s) for s in choices]).unsqueeze(0) # Batch size 1, 2 choices - mc_token_ids = torch.tensor([input_ids.size(-1), input_ids.size(-1)]).unsqueeze(0) # Batch size 1 + input_ids = tf.constant([tokenizer.encode(s) for s in choices])[None, :] # Batch size 1, 2 choices + mc_token_ids = tf.constant([input_ids.size(-1), input_ids.size(-1)])[None, :] # Batch size 1 outputs = model(input_ids, mc_token_ids=mc_token_ids) lm_prediction_scores, mc_prediction_scores = outputs[:2] From 4761a39781bdd2bd0fb812c84b8e7b808d2c7664 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 24 Sep 2019 15:19:09 +0200 Subject: [PATCH 116/219] doctring roberta --- pytorch_transformers/modeling_tf_gpt2.py | 10 +-- pytorch_transformers/modeling_tf_roberta.py | 95 ++++++++++++--------- 2 files changed, 61 insertions(+), 44 deletions(-) diff --git a/pytorch_transformers/modeling_tf_gpt2.py b/pytorch_transformers/modeling_tf_gpt2.py index 5d686ae704..cb63b95794 100644 --- a/pytorch_transformers/modeling_tf_gpt2.py +++ b/pytorch_transformers/modeling_tf_gpt2.py @@ -385,7 +385,7 @@ GPT2_START_DOCSTRING = r""" OpenAI GPT-2 model was proposed in """ GPT2_INPUTS_DOCSTRING = r""" Inputs: - **input_ids**: ```Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: + **input_ids**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: Indices of input sequence tokens in the vocabulary. GPT-2 is a model with absolute position embeddings so it's usually advised to pad the inputs on the right rather than the left. @@ -418,18 +418,18 @@ GPT2_INPUTS_DOCSTRING = r""" Inputs: class TFGPT2Model(TFGPT2PreTrainedModel): r""" Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: - **last_hidden_state**: `tf.Tensor`` of shape ``(batch_size, sequence_length, hidden_size)`` + **last_hidden_state**: ``tf.Tensor`` of shape ``(batch_size, sequence_length, hidden_size)`` Sequence of hidden-states at the last layer of the model. **past**: - list of `tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: that contains pre-computed hidden-states (key and values in the attention blocks). Can be used (see `past` input) to speed up sequential decoding. **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) - list of `tf.Tensor`` (one for the output of each layer + the output of the embeddings) + list of ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) of shape ``(batch_size, sequence_length, hidden_size)``: Hidden-states of the model at the output of each layer plus the initial embedding outputs. **attentions**: (`optional`, returned when ``config.output_attentions=True``) - list of `tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Examples:: diff --git a/pytorch_transformers/modeling_tf_roberta.py b/pytorch_transformers/modeling_tf_roberta.py index 4172dc4e6e..488ea566dc 100644 --- a/pytorch_transformers/modeling_tf_roberta.py +++ b/pytorch_transformers/modeling_tf_roberta.py @@ -111,14 +111,30 @@ ROBERTA_START_DOCSTRING = r""" The RoBERTa model was proposed in This implementation is the same as BertModel with a tiny embeddings tweak as well as a setup for Roberta pretrained models. - This model is a PyTorch `torch.nn.Module`_ sub-class. Use it as a regular PyTorch Module and - refer to the PyTorch documentation for all matter related to general usage and behavior. + This model is a tf.keras.Model `tf.keras.Model`_ sub-class. Use it as a regular TF 2.0 Keras Model and + refer to the TF 2.0 documentation for all matter related to general usage and behavior. .. _`RoBERTa: A Robustly Optimized BERT Pretraining Approach`: https://arxiv.org/abs/1907.11692 - .. _`torch.nn.Module`: - https://pytorch.org/docs/stable/nn.html#module + .. _`tf.keras.Model`: + https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/Model + + Note on the model inputs: + TF 2.0 models accepts two formats as inputs: + + - having all inputs as keyword arguments (like PyTorch models), or + - having all inputs as a list, tuple or dict in the first positional arguments. + + This second option is usefull when using `tf.keras.Model.fit()` method which currently requires having all the tensors in the first argument of the model call function: `model(inputs)`. + + If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the first positional argument : + + - a single Tensor with input_ids only and nothing else: `model(inputs_ids) + - a list of varying length with one or several input Tensors IN THE ORDER given in the docstring: + `model([input_ids, attention_mask])` or `model([input_ids, attention_mask, token_type_ids])` + - a dictionary with one or several input Tensors associaed to the input names given in the docstring: + `model({'input_ids': input_ids, 'token_type_ids': token_type_ids})` Parameters: config (:class:`~pytorch_transformers.RobertaConfig`): Model configuration class with all the parameters of the @@ -128,7 +144,7 @@ ROBERTA_START_DOCSTRING = r""" The RoBERTa model was proposed in ROBERTA_INPUTS_DOCSTRING = r""" Inputs: - **input_ids**: ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + **input_ids**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: Indices of input sequence tokens in the vocabulary. To match pre-training, RoBERTa input sequence should be formatted with and tokens as follows: @@ -148,21 +164,21 @@ ROBERTA_INPUTS_DOCSTRING = r""" See :func:`pytorch_transformers.PreTrainedTokenizer.encode` and :func:`pytorch_transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. - **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``: + **attention_mask**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``: ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. - **token_type_ids**: (`optional` need to be trained) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + **token_type_ids**: (`optional` need to be trained) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: Optional segment token indices to indicate first and second portions of the inputs. This embedding matrice is not trained (not pretrained during RoBERTa pretraining), you will have to train it during finetuning. Indices are selected in ``[0, 1]``: ``0`` corresponds to a `sentence A` token, ``1`` corresponds to a `sentence B` token (see `BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ for more details). - **position_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + **position_ids**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: Indices of positions of each input sequence tokens in the position embeddings. Selected in the range ``[0, config.max_position_embeddings - 1[``. - **head_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: + **head_mask**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. @@ -173,9 +189,9 @@ ROBERTA_INPUTS_DOCSTRING = r""" class TFRobertaModel(TFRobertaPreTrainedModel): r""" Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: - **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)`` + **last_hidden_state**: ``tf.Tensor`` of shape ``(batch_size, sequence_length, hidden_size)`` Sequence of hidden-states at the output of the last layer of the model. - **pooler_output**: ``torch.FloatTensor`` of shape ``(batch_size, hidden_size)`` + **pooler_output**: ``tf.Tensor`` of shape ``(batch_size, hidden_size)`` Last layer hidden-state of the first token of the sequence (classification token) further processed by a Linear layer and a Tanh activation function. The Linear layer weights are trained from the next sentence prediction (classification) @@ -183,18 +199,21 @@ class TFRobertaModel(TFRobertaPreTrainedModel): of the semantic content of the input, you're often better with averaging or pooling the sequence of hidden-states for the whole input sequence. **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) - list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + list of ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) of shape ``(batch_size, sequence_length, hidden_size)``: Hidden-states of the model at the output of each layer plus the initial embedding outputs. **attentions**: (`optional`, returned when ``config.output_attentions=True``) - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Examples:: + import tensorflow as tf + from pytorch_transformers import RobertaTokenizer, TFRobertaModel + tokenizer = RobertaTokenizer.from_pretrained('roberta-base') - model = RobertaModel.from_pretrained('roberta-base') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + model = TFRobertaModel.from_pretrained('roberta-base') + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -243,32 +262,35 @@ class TFRobertaLMHead(tf.keras.layers.Layer): ROBERTA_START_DOCSTRING, ROBERTA_INPUTS_DOCSTRING) class TFRobertaForMaskedLM(TFRobertaPreTrainedModel): r""" - **masked_lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + **masked_lm_labels**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: Labels for computing the masked language modeling loss. Indices should be in ``[-1, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring) Tokens with indices set to ``-1`` are ignored (masked), the loss is only computed for the tokens with labels in ``[0, ..., config.vocab_size]`` Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: - **loss**: (`optional`, returned when ``masked_lm_labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + **loss**: (`optional`, returned when ``masked_lm_labels`` is provided) ``tf.Tensor`` of shape ``(1,)``: Masked language modeling loss. - **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` + **prediction_scores**: ``tf.Tensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) - list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + list of ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) of shape ``(batch_size, sequence_length, hidden_size)``: Hidden-states of the model at the output of each layer plus the initial embedding outputs. **attentions**: (`optional`, returned when ``config.output_attentions=True``) - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Examples:: + import tensorflow as tf + from pytorch_transformers import RobertaTokenizer, TFRobertaForMaskedLM + tokenizer = RobertaTokenizer.from_pretrained('roberta-base') - model = RobertaForMaskedLM.from_pretrained('roberta-base') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + model = TFRobertaForMaskedLM.from_pretrained('roberta-base') + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 outputs = model(input_ids, masked_lm_labels=input_ids) - loss, prediction_scores = outputs[:2] + prediction_scores = outputs[0] """ def __init__(self, config, *inputs, **kwargs): @@ -311,33 +333,28 @@ class TFRobertaClassificationHead(tf.keras.layers.Layer): ROBERTA_START_DOCSTRING, ROBERTA_INPUTS_DOCSTRING) class TFRobertaForSequenceClassification(TFRobertaPreTrainedModel): r""" - **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: - Labels for computing the sequence classification/regression loss. - Indices should be in ``[0, ..., config.num_labels]``. - If ``config.num_labels == 1`` a regression loss is computed (Mean-Square loss), - If ``config.num_labels > 1`` a classification loss is computed (Cross-Entropy). - Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: - **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: - Classification (or regression if config.num_labels==1) loss. - **logits**: ``torch.FloatTensor`` of shape ``(batch_size, config.num_labels)`` + **logits**: ``tf.Tensor`` of shape ``(batch_size, config.num_labels)`` Classification (or regression if config.num_labels==1) scores (before SoftMax). **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) - list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + list of ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) of shape ``(batch_size, sequence_length, hidden_size)``: Hidden-states of the model at the output of each layer plus the initial embedding outputs. **attentions**: (`optional`, returned when ``config.output_attentions=True``) - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Examples:: + import tensorflow as tf + from pytorch_transformers import RobertaTokenizer, TFRobertaForSequenceClassification + tokenizer = RoertaTokenizer.from_pretrained('roberta-base') - model = RobertaForSequenceClassification.from_pretrained('roberta-base') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 - labels = torch.tensor([1]).unsqueeze(0) # Batch size 1 - outputs = model(input_ids, labels=labels) - loss, logits = outputs[:2] + model = TFRobertaForSequenceClassification.from_pretrained('roberta-base') + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + labels = tf.constant([1])[None, :] # Batch size 1 + outputs = model(input_ids) + logits = outputs[0] """ def __init__(self, config, *inputs, **kwargs): From b3087ddde8c5774cb4c150916ef224f0bd3b1d87 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 24 Sep 2019 15:21:51 +0200 Subject: [PATCH 117/219] docstring t-xl --- .../modeling_tf_transfo_xl.py | 69 +++++++++++-------- 1 file changed, 41 insertions(+), 28 deletions(-) diff --git a/pytorch_transformers/modeling_tf_transfo_xl.py b/pytorch_transformers/modeling_tf_transfo_xl.py index 57bb61869e..bd2854ac0a 100644 --- a/pytorch_transformers/modeling_tf_transfo_xl.py +++ b/pytorch_transformers/modeling_tf_transfo_xl.py @@ -565,14 +565,30 @@ TRANSFO_XL_START_DOCSTRING = r""" The Transformer-XL model was proposed in previously computed hidden-states to attend to longer context (memory). This model also uses adaptive softmax inputs and outputs (tied). - This model is a PyTorch `torch.tf.keras.layers.Layer`_ sub-class. Use it as a regular PyTorch Module and - refer to the PyTorch documentation for all matter related to general usage and behavior. + This model is a tf.keras.Model `tf.keras.Model`_ sub-class. Use it as a regular TF 2.0 Keras Model and + refer to the TF 2.0 documentation for all matter related to general usage and behavior. .. _`Transformer-XL: Attentive Language Models Beyond a Fixed-Length Context`: https://arxiv.org/abs/1901.02860 - .. _`torch.tf.keras.layers.Layer`: - https://pytorch.org/docs/stable/nn.html#module + .. _`tf.keras.Model`: + https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/Model + + Note on the model inputs: + TF 2.0 models accepts two formats as inputs: + + - having all inputs as keyword arguments (like PyTorch models), or + - having all inputs as a list, tuple or dict in the first positional arguments. + + This second option is usefull when using `tf.keras.Model.fit()` method which currently requires having all the tensors in the first argument of the model call function: `model(inputs)`. + + If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the first positional argument : + + - a single Tensor with input_ids only and nothing else: `model(inputs_ids) + - a list of varying length with one or several input Tensors IN THE ORDER given in the docstring: + `model([input_ids, attention_mask])` or `model([input_ids, attention_mask, token_type_ids])` + - a dictionary with one or several input Tensors associaed to the input names given in the docstring: + `model({'input_ids': input_ids, 'token_type_ids': token_type_ids})` Parameters: config (:class:`~pytorch_transformers.TransfoXLConfig`): Model configuration class with all the parameters of the model. @@ -582,7 +598,7 @@ TRANSFO_XL_START_DOCSTRING = r""" The Transformer-XL model was proposed in TRANSFO_XL_INPUTS_DOCSTRING = r""" Inputs: - **input_ids**: ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + **input_ids**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: Indices of input sequence tokens in the vocabulary. Transformer-XL is a model with relative position embeddings so you can either pad the inputs on the right or on the left. @@ -590,10 +606,10 @@ TRANSFO_XL_INPUTS_DOCSTRING = r""" See :func:`pytorch_transformers.PreTrainedTokenizer.encode` and :func:`pytorch_transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. **mems**: (`optional`) - list of ``torch.FloatTensor`` (one for each layer): + list of ``Numpy array`` or ``tf.Tensor`` (one for each layer): that contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model (see `mems` output below). Can be used to speed up sequential decoding and attend to longer context. - **head_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: + **head_mask**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. @@ -604,25 +620,28 @@ TRANSFO_XL_INPUTS_DOCSTRING = r""" class TFTransfoXLModel(TFTransfoXLPreTrainedModel): r""" Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: - **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)`` + **last_hidden_state**: ``tf.Tensor`` of shape ``(batch_size, sequence_length, hidden_size)`` Sequence of hidden-states at the last layer of the model. **mems**: - list of ``torch.FloatTensor`` (one for each layer): + list of ``tf.Tensor`` (one for each layer): that contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model (see `mems` input above). Can be used to speed up sequential decoding and attend to longer context. **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) - list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + list of ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) of shape ``(batch_size, sequence_length, hidden_size)``: Hidden-states of the model at the output of each layer plus the initial embedding outputs. **attentions**: (`optional`, returned when ``config.output_attentions=True``) - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Examples:: + import tensorflow as tf + from pytorch_transformers import TransfoXLTokenizer, TFTransfoXLModel + tokenizer = TransfoXLTokenizer.from_pretrained('transfo-xl-wt103') - model = TransfoXLModel.from_pretrained('transfo-xl-wt103') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + model = TFTransfoXLModel.from_pretrained('transfo-xl-wt103') + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 outputs = model(input_ids) last_hidden_states, mems = outputs[:2] @@ -641,36 +660,30 @@ class TFTransfoXLModel(TFTransfoXLPreTrainedModel): TRANSFO_XL_START_DOCSTRING, TRANSFO_XL_INPUTS_DOCSTRING) class TFTransfoXLLMHeadModel(TFTransfoXLPreTrainedModel): r""" - **lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: - Labels for language modeling. - Note that the labels **are shifted** inside the model, i.e. you can set ``lm_labels = input_ids`` - Indices are selected in ``[-1, 0, ..., config.vocab_size]`` - All labels set to ``-1`` are ignored (masked), the loss is only - computed for labels in ``[0, ..., config.vocab_size]`` - Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: - **loss**: (`optional`, returned when ``lm_labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: - Language modeling loss. - **prediction_scores**: ``None`` if ``lm_labels`` is provided else ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` + **prediction_scores**: ``None`` if ``lm_labels`` is provided else ``tf.Tensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). We don't output them when the loss is computed to speedup adaptive softmax decoding. **mems**: - list of ``torch.FloatTensor`` (one for each layer): + list of ``tf.Tensor`` (one for each layer): that contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model (see `mems` input above). Can be used to speed up sequential decoding and attend to longer context. **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) - list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + list of ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) of shape ``(batch_size, sequence_length, hidden_size)``: Hidden-states of the model at the output of each layer plus the initial embedding outputs. **attentions**: (`optional`, returned when ``config.output_attentions=True``) - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Examples:: + import tensorflow as tf + from pytorch_transformers import TransfoXLTokenizer, TFTransfoXLLMHeadModel + tokenizer = TransfoXLTokenizer.from_pretrained('transfo-xl-wt103') - model = TransfoXLLMHeadModel.from_pretrained('transfo-xl-wt103') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + model = TFTransfoXLLMHeadModel.from_pretrained('transfo-xl-wt103') + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 outputs = model(input_ids) prediction_scores, mems = outputs[:2] From 559790f9e4576b4a21b3efb01711abba6b5ac1c4 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 24 Sep 2019 15:26:57 +0200 Subject: [PATCH 118/219] docstring for xlm --- pytorch_transformers/modeling_tf_bert.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pytorch_transformers/modeling_tf_bert.py b/pytorch_transformers/modeling_tf_bert.py index f9d6cd9990..d5bbac6258 100644 --- a/pytorch_transformers/modeling_tf_bert.py +++ b/pytorch_transformers/modeling_tf_bert.py @@ -987,9 +987,7 @@ class TFBertForQuestionAnswering(TFBertPreTrainedModel): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = TFBertForQuestionAnswering.from_pretrained('bert-base-uncased') input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 - start_positions = tf.constant([1]) - end_positions = tf.constant([3]) - outputs = model(input_ids, start_positions=start_positions, end_positions=end_positions) + outputs = model(input_ids) start_scores, end_scores = outputs[:2] """ From de203853cc0b47faee1dbab5fb3eb393fdade0dd Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 24 Sep 2019 15:30:55 +0200 Subject: [PATCH 119/219] docstring for xlnet --- pytorch_transformers/modeling_tf_utils.py | 6 +- pytorch_transformers/modeling_tf_xlm.py | 142 ++++++++++----------- pytorch_transformers/modeling_tf_xlnet.py | 145 ++++++++++++---------- 3 files changed, 149 insertions(+), 144 deletions(-) diff --git a/pytorch_transformers/modeling_tf_utils.py b/pytorch_transformers/modeling_tf_utils.py index bdbc034c4a..2186e2d488 100644 --- a/pytorch_transformers/modeling_tf_utils.py +++ b/pytorch_transformers/modeling_tf_utils.py @@ -74,7 +74,7 @@ class TFPreTrainedModel(tf.keras.Model): Increasing the size will add newly initialized vectors at the end Reducing the size will remove vectors from the end If not provided or None: return the provided token Embedding Module. - Return: ``torch.nn.Embeddings`` + Return: ``tf.Variable`` Pointer to the resized Embedding Module or the old Embedding Module if new_num_tokens is None """ # if new_num_tokens is None: @@ -105,9 +105,9 @@ class TFPreTrainedModel(tf.keras.Model): new_num_tokens: (`optional`) int: New number of tokens in the embedding matrix. Increasing the size will add newly initialized vectors at the end. Reducing the size will remove vectors from the end. - If not provided or None: does nothing and just returns a pointer to the input tokens ``torch.nn.Embeddings`` Module of the model. + If not provided or None: does nothing and just returns a pointer to the input tokens ``tf.Variable`` Module of the model. - Return: ``torch.nn.Embeddings`` + Return: ``tf.Variable`` Pointer to the input tokens Embeddings Module of the model """ raise NotImplementedError diff --git a/pytorch_transformers/modeling_tf_xlm.py b/pytorch_transformers/modeling_tf_xlm.py index a3915b6c47..1cd2f355be 100644 --- a/pytorch_transformers/modeling_tf_xlm.py +++ b/pytorch_transformers/modeling_tf_xlm.py @@ -449,18 +449,34 @@ XLM_START_DOCSTRING = r""" The XLM model was proposed in Original code can be found `here`_. - This model is a PyTorch `torch.tf.keras.layers.Layer`_ sub-class. Use it as a regular PyTorch Module and - refer to the PyTorch documentation for all matter related to general usage and behavior. + This model is a tf.keras.Model `tf.keras.Model`_ sub-class. Use it as a regular TF 2.0 Keras Model and + refer to the TF 2.0 documentation for all matter related to general usage and behavior. .. _`Cross-lingual Language Model Pretraining`: https://arxiv.org/abs/1901.07291 - .. _`torch.tf.keras.layers.Layer`: - https://pytorch.org/docs/stable/nn.html#module - .. _`here`: https://github.com/facebookresearch/XLM + .. _`tf.keras.Model`: + https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/Model + + Note on the model inputs: + TF 2.0 models accepts two formats as inputs: + + - having all inputs as keyword arguments (like PyTorch models), or + - having all inputs as a list, tuple or dict in the first positional arguments. + + This second option is usefull when using `tf.keras.Model.fit()` method which currently requires having all the tensors in the first argument of the model call function: `model(inputs)`. + + If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the first positional argument : + + - a single Tensor with input_ids only and nothing else: `model(inputs_ids) + - a list of varying length with one or several input Tensors IN THE ORDER given in the docstring: + `model([input_ids, attention_mask])` or `model([input_ids, attention_mask, token_type_ids])` + - a dictionary with one or several input Tensors associaed to the input names given in the docstring: + `model({'input_ids': input_ids, 'token_type_ids': token_type_ids})` + Parameters: config (:class:`~pytorch_transformers.XLMConfig`): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. @@ -469,7 +485,7 @@ XLM_START_DOCSTRING = r""" The XLM model was proposed in XLM_INPUTS_DOCSTRING = r""" Inputs: - **input_ids**: ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + **input_ids**: ```Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: Indices of input sequence tokens in the vocabulary. XLM is a model with absolute position embeddings so it's usually advised to pad the inputs on @@ -478,33 +494,33 @@ XLM_INPUTS_DOCSTRING = r""" Indices can be obtained using :class:`pytorch_transformers.XLMTokenizer`. See :func:`pytorch_transformers.PreTrainedTokenizer.encode` and :func:`pytorch_transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. - **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``: + **attention_mask**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``: ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. - **langs**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + **langs**: (`optional`) ```Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: A parallel sequence of tokens to be used to indicate the language of each token in the input. Indices are languages ids which can be obtained from the language names by using two conversion mappings provided in the configuration of the model (only provided for multilingual models). More precisely, the `language name -> language id` mapping is in `model.config.lang2id` (dict str -> int) and the `language id -> language name` mapping is `model.config.id2lang` (dict int -> str). - **token_type_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + **token_type_ids**: (`optional`) ```Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: A parallel sequence of tokens (can be used to indicate various portions of the inputs). The embeddings from these tokens will be summed with the respective token embeddings. Indices are selected in the vocabulary (unlike BERT which has a specific vocabulary for segment indices). - **position_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + **position_ids**: (`optional`) ```Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: Indices of positions of each input sequence tokens in the position embeddings. Selected in the range ``[0, config.max_position_embeddings - 1]``. - **lengths**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + **lengths**: (`optional`) ```Numpy array`` or ``tf.Tensor`` of shape ``(batch_size,)``: Length of each sentence that can be used to avoid performing attention on padding token indices. You can also use `attention_mask` for the same result (see above), kept here for compatbility. Indices selected in ``[0, ..., input_ids.size(-1)]``: **cache**: - dictionary with ``torch.FloatTensor`` that contains pre-computed + dictionary with ``Numpy array`` or ``tf.Tensor`` that contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model (see `cache` output below). Can be used to speed up sequential decoding. The dictionary object will be modified in-place during the forward pass to add newly computed hidden-states. - **head_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: + **head_mask**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. @@ -515,21 +531,24 @@ XLM_INPUTS_DOCSTRING = r""" class TFXLMModel(TFXLMPreTrainedModel): r""" Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: - **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)`` + **last_hidden_state**: ``tf.Tensor`` of shape ``(batch_size, sequence_length, hidden_size)`` Sequence of hidden-states at the last layer of the model. **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) - list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + list of ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) of shape ``(batch_size, sequence_length, hidden_size)``: Hidden-states of the model at the output of each layer plus the initial embedding outputs. **attentions**: (`optional`, returned when ``config.output_attentions=True``) - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Examples:: + import tensorflow as tf + from pytorch_transformers import XLMTokenizer, TFXLMModel + tokenizer = XLMTokenizer.from_pretrained('xlm-mlm-en-2048') - model = XLMModel.from_pretrained('xlm-mlm-en-2048') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + model = TFXLMModel.from_pretrained('xlm-mlm-en-2048') + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -584,31 +603,25 @@ class TFXLMPredLayer(tf.keras.layers.Layer): XLM_START_DOCSTRING, XLM_INPUTS_DOCSTRING) class TFXLMWithLMHeadModel(TFXLMPreTrainedModel): r""" - **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: - Labels for language modeling. - Note that the labels **are shifted** inside the model, i.e. you can set ``lm_labels = input_ids`` - Indices are selected in ``[-1, 0, ..., config.vocab_size]`` - All labels set to ``-1`` are ignored (masked), the loss is only - computed for labels in ``[0, ..., config.vocab_size]`` - Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: - **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: - Language modeling loss. - **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` + **prediction_scores**: ``tf.Tensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) - list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + list of ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) of shape ``(batch_size, sequence_length, hidden_size)``: Hidden-states of the model at the output of each layer plus the initial embedding outputs. **attentions**: (`optional`, returned when ``config.output_attentions=True``) - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Examples:: + import tensorflow as tf + from pytorch_transformers import XLMTokenizer, TFXLMWithLMHeadModel + tokenizer = XLMTokenizer.from_pretrained('xlm-mlm-en-2048') - model = XLMWithLMHeadModel.from_pretrained('xlm-mlm-en-2048') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + model = TFXLMWithLMHeadModel.from_pretrained('xlm-mlm-en-2048') + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -634,33 +647,28 @@ class TFXLMWithLMHeadModel(TFXLMPreTrainedModel): XLM_START_DOCSTRING, XLM_INPUTS_DOCSTRING) class TFXLMForSequenceClassification(TFXLMPreTrainedModel): r""" - **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: - Labels for computing the sequence classification/regression loss. - Indices should be in ``[0, ..., config.num_labels - 1]``. - If ``config.num_labels == 1`` a regression loss is computed (Mean-Square loss), - If ``config.num_labels > 1`` a classification loss is computed (Cross-Entropy). - Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: - **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: - Classification (or regression if config.num_labels==1) loss. - **logits**: ``torch.FloatTensor`` of shape ``(batch_size, config.num_labels)`` + **logits**: ``tf.Tensor`` of shape ``(batch_size, config.num_labels)`` Classification (or regression if config.num_labels==1) scores (before SoftMax). **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) - list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + list of ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) of shape ``(batch_size, sequence_length, hidden_size)``: Hidden-states of the model at the output of each layer plus the initial embedding outputs. **attentions**: (`optional`, returned when ``config.output_attentions=True``) - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Examples:: + import tensorflow as tf + from pytorch_transformers import XLMTokenizer, TFXLMForSequenceClassification + tokenizer = XLMTokenizer.from_pretrained('xlm-mlm-en-2048') - model = XLMForSequenceClassification.from_pretrained('xlm-mlm-en-2048') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 - labels = torch.tensor([1]).unsqueeze(0) # Batch size 1 - outputs = model(input_ids, labels=labels) - loss, logits = outputs[:2] + model = TFXLMForSequenceClassification.from_pretrained('xlm-mlm-en-2048') + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + labels = tf.constant([1])[None, :] # Batch size 1 + outputs = model(input_ids) + logits = outputs[0] """ def __init__(self, config, *inputs, **kwargs): @@ -685,45 +693,29 @@ class TFXLMForSequenceClassification(TFXLMPreTrainedModel): XLM_START_DOCSTRING, XLM_INPUTS_DOCSTRING) class TFXLMForQuestionAnsweringSimple(TFXLMPreTrainedModel): r""" - **start_positions**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). - Position outside of the sequence are not taken into account for computing the loss. - **end_positions**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (`sequence_length`). - Position outside of the sequence are not taken into account for computing the loss. - **is_impossible**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: - Labels whether a question has an answer or no answer (SQuAD 2.0) - **cls_index**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: - Labels for position (index) of the classification token to use as input for computing plausibility of the answer. - **p_mask**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: - Optional mask of tokens which can't be in answers (e.g. [CLS], [PAD], ...) - Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: - **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: - Total span extraction loss is the sum of a Cross-Entropy for the start and end positions. - **start_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length,)`` + **start_scores**: ``tf.Tensor`` of shape ``(batch_size, sequence_length,)`` Span-start scores (before SoftMax). - **end_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length,)`` + **end_scores**: ``tf.Tensor`` of shape ``(batch_size, sequence_length,)`` Span-end scores (before SoftMax). **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) - list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + list of ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) of shape ``(batch_size, sequence_length, hidden_size)``: Hidden-states of the model at the output of each layer plus the initial embedding outputs. **attentions**: (`optional`, returned when ``config.output_attentions=True``) - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Examples:: + import tensorflow as tf + from pytorch_transformers import XLMTokenizer, TFXLMForQuestionAnsweringSimple + tokenizer = XLMTokenizer.from_pretrained('xlm-mlm-en-2048') - model = XLMForQuestionAnswering.from_pretrained('xlm-mlm-en-2048') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 - start_positions = torch.tensor([1]) - end_positions = torch.tensor([3]) - outputs = model(input_ids, start_positions=start_positions, end_positions=end_positions) - loss, start_scores, end_scores = outputs[:2] + model = TFXLMForQuestionAnsweringSimple.from_pretrained('xlm-mlm-en-2048') + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + outputs = model(input_ids) + start_scores, end_scores = outputs[:2] """ def __init__(self, config, *inputs, **kwargs): diff --git a/pytorch_transformers/modeling_tf_xlnet.py b/pytorch_transformers/modeling_tf_xlnet.py index f15ac4899f..a6ffb5b681 100644 --- a/pytorch_transformers/modeling_tf_xlnet.py +++ b/pytorch_transformers/modeling_tf_xlnet.py @@ -686,14 +686,30 @@ XLNET_START_DOCSTRING = r""" The XLNet model was proposed in To use XLNet for sequential decoding (i.e. not in fully bi-directional setting), use the `perm_mask` and `target_mapping` inputs to control the attention span and outputs (see examples in `examples/run_generation.py`) - This model is a PyTorch `torch.tf.keras.layers.Layer`_ sub-class. Use it as a regular PyTorch Module and - refer to the PyTorch documentation for all matter related to general usage and behavior. + This model is a tf.keras.Model `tf.keras.Model`_ sub-class. Use it as a regular TF 2.0 Keras Model and + refer to the TF 2.0 documentation for all matter related to general usage and behavior. .. _`XLNet: Generalized Autoregressive Pretraining for Language Understanding`: http://arxiv.org/abs/1906.08237 - .. _`torch.tf.keras.layers.Layer`: - https://pytorch.org/docs/stable/nn.html#module + .. _`tf.keras.Model`: + https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/Model + + Note on the model inputs: + TF 2.0 models accepts two formats as inputs: + + - having all inputs as keyword arguments (like PyTorch models), or + - having all inputs as a list, tuple or dict in the first positional arguments. + + This second option is usefull when using `tf.keras.Model.fit()` method which currently requires having all the tensors in the first argument of the model call function: `model(inputs)`. + + If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the first positional argument : + + - a single Tensor with input_ids only and nothing else: `model(inputs_ids) + - a list of varying length with one or several input Tensors IN THE ORDER given in the docstring: + `model([input_ids, attention_mask])` or `model([input_ids, attention_mask, token_type_ids])` + - a dictionary with one or several input Tensors associaed to the input names given in the docstring: + `model({'input_ids': input_ids, 'token_type_ids': token_type_ids})` Parameters: config (:class:`~pytorch_transformers.XLNetConfig`): Model configuration class with all the parameters of the model. @@ -703,48 +719,48 @@ XLNET_START_DOCSTRING = r""" The XLNet model was proposed in XLNET_INPUTS_DOCSTRING = r""" Inputs: - **input_ids**: ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + **input_ids**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: Indices of input sequence tokens in the vocabulary. XLNet is a model with relative position embeddings so you can either pad the inputs on the right or on the left. Indices can be obtained using :class:`pytorch_transformers.XLNetTokenizer`. See :func:`pytorch_transformers.PreTrainedTokenizer.encode` and :func:`pytorch_transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. - **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``: + **attention_mask**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``: ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. **mems**: (`optional`) - list of ``torch.FloatTensor`` (one for each layer): + list of ``Numpy array`` or ``tf.Tensor`` (one for each layer): that contains pre-computed hidden-states (key and values in the attention blocks) as output by the model (see `mems` output below). Can be used to speed up sequential decoding and attend to longer context. To activate mems you need to set up config.mem_len to a positive value which will be the max number of tokens in the memory output by the model. E.g. `model = XLNetModel.from_pretrained('xlnet-base-case, mem_len=1024)` will instantiate a model which can use up to 1024 tokens of memory (in addition to the input it self). - **perm_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, sequence_length)``: + **perm_mask**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length, sequence_length)``: Mask to indicate the attention pattern for each input token with values selected in ``[0, 1]``: If ``perm_mask[k, i, j] = 0``, i attend to j in batch k; if ``perm_mask[k, i, j] = 1``, i does not attend to j in batch k. If None, each token attends to all the others (full bidirectional attention). Only used during pretraining (to define factorization order) or for sequential decoding (generation). - **target_mapping**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, num_predict, sequence_length)``: + **target_mapping**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, num_predict, sequence_length)``: Mask to indicate the output tokens to use. If ``target_mapping[k, i, j] = 1``, the i-th predict in batch k is on the j-th token. Only used during pretraining for partial prediction or for sequential decoding (generation). - **token_type_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + **token_type_ids**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: A parallel sequence of tokens (can be used to indicate various portions of the inputs). The type indices in XLNet are NOT selected in the vocabulary, they can be arbitrary numbers and the important thing is that they should be different for tokens which belong to different segments. The model will compute relative segment differences from the given type indices: 0 if the segment id of two tokens are the same, 1 if not. - **input_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``: + **input_mask**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: Mask to avoid performing attention on padding token indices. Negative of `attention_mask`, i.e. with 0 for real tokens and 1 for padding. Kept for compatibility with the original code base. You can only uses one of `input_mask` and `attention_mask` Mask values selected in ``[0, 1]``: ``1`` for tokens that are MASKED, ``0`` for tokens that are NOT MASKED. - **head_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: + **head_mask**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. @@ -755,26 +771,29 @@ XLNET_INPUTS_DOCSTRING = r""" class TFXLNetModel(TFXLNetPreTrainedModel): r""" Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: - **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)`` + **last_hidden_state**: ``tf.Tensor`` of shape ``(batch_size, sequence_length, hidden_size)`` Sequence of hidden-states at the last layer of the model. **mems**: - list of ``torch.FloatTensor`` (one for each layer): + list of ``tf.Tensor`` (one for each layer): that contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model if config.mem_len > 0 else tuple of None. Can be used to speed up sequential decoding and attend to longer context. See details in the docstring of the `mems` input above. **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) - list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + list of ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) of shape ``(batch_size, sequence_length, hidden_size)``: Hidden-states of the model at the output of each layer plus the initial embedding outputs. **attentions**: (`optional`, returned when ``config.output_attentions=True``) - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Examples:: + import tensorflow as tf + from pytorch_transformers import XLNetTokenizer, TFXLNetModel + tokenizer = XLNetTokenizer.from_pretrained('xlnet-large-cased') - model = XLNetModel.from_pretrained('xlnet-large-cased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + model = TFXLNetModel.from_pretrained('xlnet-large-cased') + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -794,32 +813,37 @@ class TFXLNetModel(TFXLNetPreTrainedModel): class TFXLNetLMHeadModel(TFXLNetPreTrainedModel): r""" Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: - **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` + **prediction_scores**: ``tf.Tensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). **mems**: - list of ``torch.FloatTensor`` (one for each layer): + list of ``tf.Tensor`` (one for each layer): that contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model if config.mem_len > 0 else tuple of None. Can be used to speed up sequential decoding and attend to longer context. See details in the docstring of the `mems` input above. **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) - list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + list of ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) of shape ``(batch_size, sequence_length, hidden_size)``: Hidden-states of the model at the output of each layer plus the initial embedding outputs. **attentions**: (`optional`, returned when ``config.output_attentions=True``) - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Examples:: + import tensorflow as tf + from pytorch_transformers import XLNetTokenizer, TFXLNetLMHeadModel + tokenizer = XLNetTokenizer.from_pretrained('xlnet-large-cased') model = TFXLNetLMHeadModel.from_pretrained('xlnet-large-cased') + # We show how to setup inputs to predict a next token using a bi-directional context. - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is very ")).unsqueeze(0) # We will predict the masked token - perm_mask = torch.zeros((1, input_ids.shape[1], input_ids.shape[1]), dtype=torch.float) + input_ids = tf.constant(tokenizer.encode("Hello, my dog is very "))[None, :] # We will predict the masked token + perm_mask = tf.zeros((1, input_ids.shape[1], input_ids.shape[1])) perm_mask[:, :, -1] = 1.0 # Previous tokens don't see last token - target_mapping = torch.zeros((1, 1, input_ids.shape[1]), dtype=torch.float) # Shape [1, 1, seq_length] => let's predict one token + target_mapping = tf.zeros((1, 1, input_ids.shape[1])) # Shape [1, 1, seq_length] => let's predict one token target_mapping[0, 0, -1] = 1.0 # Our first (and only) prediction will be the last token of the sequence (the masked token) outputs = model(input_ids, perm_mask=perm_mask, target_mapping=target_mapping) + next_token_logits = outputs[0] # Output has shape [target_mapping.size(0), target_mapping.size(1), config.vocab_size] """ @@ -843,38 +867,32 @@ class TFXLNetLMHeadModel(TFXLNetPreTrainedModel): XLNET_START_DOCSTRING, XLNET_INPUTS_DOCSTRING) class TFXLNetForSequenceClassification(TFXLNetPreTrainedModel): r""" - **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: - Labels for computing the sequence classification/regression loss. - Indices should be in ``[0, ..., config.num_labels - 1]``. - If ``config.num_labels == 1`` a regression loss is computed (Mean-Square loss), - If ``config.num_labels > 1`` a classification loss is computed (Cross-Entropy). - Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: - **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: - Classification (or regression if config.num_labels==1) loss. - **logits**: ``torch.FloatTensor`` of shape ``(batch_size, config.num_labels)`` + **logits**: ``tf.Tensor`` of shape ``(batch_size, config.num_labels)`` Classification (or regression if config.num_labels==1) scores (before SoftMax). **mems**: - list of ``torch.FloatTensor`` (one for each layer): + list of ``tf.Tensor`` (one for each layer): that contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model if config.mem_len > 0 else tuple of None. Can be used to speed up sequential decoding and attend to longer context. See details in the docstring of the `mems` input above. **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) - list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + list of ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) of shape ``(batch_size, sequence_length, hidden_size)``: Hidden-states of the model at the output of each layer plus the initial embedding outputs. **attentions**: (`optional`, returned when ``config.output_attentions=True``) - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Examples:: + import tensorflow as tf + from pytorch_transformers import XLNetTokenizer, TFXLNetForSequenceClassification + tokenizer = XLNetTokenizer.from_pretrained('xlnet-large-cased') - model = XLNetForSequenceClassification.from_pretrained('xlnet-large-cased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 - labels = torch.tensor([1]).unsqueeze(0) # Batch size 1 - outputs = model(input_ids, labels=labels) - loss, logits = outputs[:2] + model = TFXLNetForSequenceClassification.from_pretrained('xlnet-large-cased') + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + outputs = model(input_ids) + logits = outputs[0] """ def __init__(self, config, *inputs, **kwargs): @@ -904,27 +922,28 @@ class TFXLNetForSequenceClassification(TFXLNetPreTrainedModel): class TFXLNetForQuestionAnsweringSimple(TFXLNetPreTrainedModel): r""" Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: - **start_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length,)`` + **start_scores**: ``tf.Tensor`` of shape ``(batch_size, sequence_length,)`` Span-start scores (before SoftMax). - **end_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length,)`` + **end_scores**: ``tf.Tensor`` of shape ``(batch_size, sequence_length,)`` Span-end scores (before SoftMax). **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) - list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + list of ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) of shape ``(batch_size, sequence_length, hidden_size)``: Hidden-states of the model at the output of each layer plus the initial embedding outputs. **attentions**: (`optional`, returned when ``config.output_attentions=True``) - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Examples:: + import tensorflow as tf + from pytorch_transformers import XLNetTokenizer, TFXLNetForQuestionAnsweringSimple + tokenizer = XLNetTokenizer.from_pretrained('xlnet-base-cased') model = TFXLNetForQuestionAnsweringSimple.from_pretrained('xlnet-base-cased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 - start_positions = torch.tensor([1]) - end_positions = torch.tensor([3]) - outputs = model(input_ids, start_positions=start_positions, end_positions=end_positions) - loss, start_scores, end_scores = outputs[:2] + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + outputs = model(input_ids) + start_scores, end_scores = outputs[:2] """ def __init__(self, config, *inputs, **kwargs): @@ -951,48 +970,42 @@ class TFXLNetForQuestionAnsweringSimple(TFXLNetPreTrainedModel): # XLNET_START_DOCSTRING, XLNET_INPUTS_DOCSTRING) # class TFXLNetForQuestionAnswering(TFXLNetPreTrainedModel): # r""" -# **p_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``: -# Optional mask of tokens which can't be in answers (e.g. [CLS], [PAD], ...). -# 1.0 means token should be masked. 0.0 mean token is not masked. - # Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: -# **loss**: (`optional`, returned if both ``start_positions`` and ``end_positions`` are provided) ``torch.FloatTensor`` of shape ``(1,)``: -# Classification loss as the sum of start token, end token (and is_impossible if provided) classification losses. # **start_top_log_probs**: (`optional`, returned if ``start_positions`` or ``end_positions`` is not provided) -# ``torch.FloatTensor`` of shape ``(batch_size, config.start_n_top)`` +# ``tf.Tensor`` of shape ``(batch_size, config.start_n_top)`` # Log probabilities for the top config.start_n_top start token possibilities (beam-search). # **start_top_index**: (`optional`, returned if ``start_positions`` or ``end_positions`` is not provided) # ``torch.LongTensor`` of shape ``(batch_size, config.start_n_top)`` # Indices for the top config.start_n_top start token possibilities (beam-search). # **end_top_log_probs**: (`optional`, returned if ``start_positions`` or ``end_positions`` is not provided) -# ``torch.FloatTensor`` of shape ``(batch_size, config.start_n_top * config.end_n_top)`` +# ``tf.Tensor`` of shape ``(batch_size, config.start_n_top * config.end_n_top)`` # Log probabilities for the top ``config.start_n_top * config.end_n_top`` end token possibilities (beam-search). # **end_top_index**: (`optional`, returned if ``start_positions`` or ``end_positions`` is not provided) # ``torch.LongTensor`` of shape ``(batch_size, config.start_n_top * config.end_n_top)`` # Indices for the top ``config.start_n_top * config.end_n_top`` end token possibilities (beam-search). # **cls_logits**: (`optional`, returned if ``start_positions`` or ``end_positions`` is not provided) -# ``torch.FloatTensor`` of shape ``(batch_size,)`` +# ``tf.Tensor`` of shape ``(batch_size,)`` # Log probabilities for the ``is_impossible`` label of the answers. # **mems**: -# list of ``torch.FloatTensor`` (one for each layer): +# list of ``tf.Tensor`` (one for each layer): # that contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model # if config.mem_len > 0 else tuple of None. Can be used to speed up sequential decoding and attend to longer context. # See details in the docstring of the `mems` input above. # **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) -# list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) +# list of ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) # of shape ``(batch_size, sequence_length, hidden_size)``: # Hidden-states of the model at the output of each layer plus the initial embedding outputs. # **attentions**: (`optional`, returned when ``config.output_attentions=True``) -# list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: +# list of ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: # Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. # Examples:: # tokenizer = XLMTokenizer.from_pretrained('xlm-mlm-en-2048') # model = XLMForQuestionAnswering.from_pretrained('xlnet-large-cased') -# input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 -# start_positions = torch.tensor([1]) -# end_positions = torch.tensor([3]) +# input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 +# start_positions = tf.constant([1]) +# end_positions = tf.constant([3]) # outputs = model(input_ids, start_positions=start_positions, end_positions=end_positions) # loss, start_scores, end_scores = outputs[:2] From 28a30af6d184c84b5d844add96c15bf023a53f7f Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 24 Sep 2019 15:33:39 +0200 Subject: [PATCH 120/219] fix auto models --- pytorch_transformers/modeling_tf_auto.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pytorch_transformers/modeling_tf_auto.py b/pytorch_transformers/modeling_tf_auto.py index 1db81c7ab4..3d3cc6306e 100644 --- a/pytorch_transformers/modeling_tf_auto.py +++ b/pytorch_transformers/modeling_tf_auto.py @@ -22,7 +22,7 @@ from .modeling_tf_bert import TFBertModel, TFBertForMaskedLM, TFBertForSequenceC from .modeling_tf_openai import TFOpenAIGPTModel, TFOpenAIGPTLMHeadModel from .modeling_tf_gpt2 import TFGPT2Model, TFGPT2LMHeadModel from .modeling_tf_transfo_xl import TFTransfoXLModel, TFTransfoXLLMHeadModel -from .modeling_tf_xlnet import TFXLNetModel, TFXLNetLMHeadModel, TFXLNetForSequenceClassification, TFXLNetForQuestionAnswering +from .modeling_tf_xlnet import TFXLNetModel, TFXLNetLMHeadModel, TFXLNetForSequenceClassification, TFXLNetForQuestionAnsweringSimple from .modeling_tf_xlm import TFXLMModel, TFXLMWithLMHeadModel, TFXLMForSequenceClassification, TFXLMForQuestionAnsweringSimple from .modeling_tf_roberta import TFRobertaModel, TFRobertaForMaskedLM, TFRobertaForSequenceClassification from .modeling_tf_distilbert import TFDistilBertModel, TFDistilBertForQuestionAnswering, TFDistilBertForMaskedLM, TFDistilBertForSequenceClassification @@ -493,9 +493,9 @@ class TFAutoModelForQuestionAnswering(object): elif 'bert' in pretrained_model_name_or_path: return TFBertForQuestionAnswering.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'xlnet' in pretrained_model_name_or_path: - return TFXLNetForQuestionAnswering.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + return TFXLNetForQuestionAnsweringSimple.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'xlm' in pretrained_model_name_or_path: - return TFXLMForQuestionAnswering.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + return TFXLMForQuestionAnsweringSimple.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) raise ValueError("Unrecognized model identifier in {}. Should contains one of " "'bert', 'xlnet', 'xlm'".format(pretrained_model_name_or_path)) From 72402d1acdc4fed6ddea75d6f29a1d45552ba250 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Tue, 24 Sep 2019 09:41:14 -0400 Subject: [PATCH 121/219] Fixed DistilBERT tokenizer --- pytorch_transformers/tests/tokenization_distilbert_test.py | 6 ++++-- pytorch_transformers/tokenization_distilbert.py | 7 ------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/pytorch_transformers/tests/tokenization_distilbert_test.py b/pytorch_transformers/tests/tokenization_distilbert_test.py index ef3cac10be..674c78d104 100644 --- a/pytorch_transformers/tests/tokenization_distilbert_test.py +++ b/pytorch_transformers/tests/tokenization_distilbert_test.py @@ -39,8 +39,10 @@ class DistilBertTokenizationTest(BertTokenizationTest): encoded_sentence = tokenizer.add_special_tokens_single_sequence(text) encoded_pair = tokenizer.add_special_tokens_sequence_pair(text, text_2) - assert encoded_sentence == text - assert encoded_pair == text + [102] + text_2 + assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + \ + text_2 + [tokenizer.sep_token_id] + if __name__ == '__main__': unittest.main() diff --git a/pytorch_transformers/tokenization_distilbert.py b/pytorch_transformers/tokenization_distilbert.py index 547ea29981..5a6d02f98d 100644 --- a/pytorch_transformers/tokenization_distilbert.py +++ b/pytorch_transformers/tokenization_distilbert.py @@ -60,10 +60,3 @@ class DistilBertTokenizer(BertTokenizer): vocab_files_names = VOCAB_FILES_NAMES pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES - - def add_special_tokens_single_sequence(self, token_ids): - return token_ids - - def add_special_tokens_sequence_pair(self, token_ids_0, token_ids_1): - sep = [self.sep_token_id] - return token_ids_0 + sep + token_ids_1 From 128bdd4c3549e2a1401af87493ff6be467c79c14 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 24 Sep 2019 15:43:39 +0200 Subject: [PATCH 122/219] fix tests pt/tf --- .../modeling_tf_pytorch_utils.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/pytorch_transformers/modeling_tf_pytorch_utils.py b/pytorch_transformers/modeling_tf_pytorch_utils.py index f4b41c3b8e..25a0cf3bf6 100644 --- a/pytorch_transformers/modeling_tf_pytorch_utils.py +++ b/pytorch_transformers/modeling_tf_pytorch_utils.py @@ -65,7 +65,7 @@ def convert_tf_weight_name_to_pt_weight_name(tf_name, start_prefix_to_remove='') ##################### ### PyTorch => TF 2.0 -def load_pytorch_checkpoint_in_tf2_model(tf_model, pytorch_checkpoint_path, tf_inputs=DUMMY_INPUTS, allow_missing_keys=False): +def load_pytorch_checkpoint_in_tf2_model(tf_model, pytorch_checkpoint_path, tf_inputs=None, allow_missing_keys=False): """ Load pytorch checkpoints in a TF 2.0 model """ try: @@ -84,7 +84,7 @@ def load_pytorch_checkpoint_in_tf2_model(tf_model, pytorch_checkpoint_path, tf_i return load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, tf_inputs=tf_inputs, allow_missing_keys=allow_missing_keys) -def load_pytorch_model_in_tf2_model(tf_model, pt_model, tf_inputs=DUMMY_INPUTS, allow_missing_keys=False): +def load_pytorch_model_in_tf2_model(tf_model, pt_model, tf_inputs=None, allow_missing_keys=False): """ Load pytorch checkpoints in a TF 2.0 model """ pt_state_dict = pt_model.state_dict() @@ -92,7 +92,7 @@ def load_pytorch_model_in_tf2_model(tf_model, pt_model, tf_inputs=DUMMY_INPUTS, return load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, tf_inputs=tf_inputs, allow_missing_keys=allow_missing_keys) -def load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, tf_inputs=DUMMY_INPUTS, allow_missing_keys=False): +def load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, tf_inputs=None, allow_missing_keys=False): """ Load pytorch state_dict in a TF 2.0 model. """ try: @@ -104,8 +104,8 @@ def load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, tf_inputs=DUMMY_I "https://pytorch.org/ and https://www.tensorflow.org/install/ for installation instructions.") raise e - if tf_inputs is not None and not isinstance(tf_inputs, tf.Tensor): - tf_inputs = tf.constant(tf_inputs) + if tf_inputs is None: + tf_inputs = tf.constant(DUMMY_INPUTS) if tf_inputs is not None: tfo = tf_model(tf_inputs, training=False) # Make sure model is built @@ -176,7 +176,7 @@ def load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, tf_inputs=DUMMY_I ##################### ### TF 2.0 => PyTorch -def load_tf2_checkpoint_in_pytorch_model(pt_model, tf_checkpoint_path, tf_inputs=DUMMY_INPUTS, allow_missing_keys=False): +def load_tf2_checkpoint_in_pytorch_model(pt_model, tf_checkpoint_path, tf_inputs=None, allow_missing_keys=False): """ Load TF 2.0 HDF5 checkpoint in a PyTorch model We use HDF5 to easily do transfer learning (see https://github.com/tensorflow/tensorflow/blob/ee16fcac960ae660e0e4496658a366e2f745e1f0/tensorflow/python/keras/engine/network.py#L1352-L1357). @@ -199,9 +199,10 @@ def load_tf2_checkpoint_in_pytorch_model(pt_model, tf_checkpoint_path, tf_inputs tf_model_class = getattr(pytorch_transformers, tf_model_class_name) tf_model = tf_model_class(pt_model.config) + if tf_inputs is None: + tf_inputs = tf.constant(DUMMY_INPUTS) + if tf_inputs is not None: - if tf_inputs is not None and not isinstance(tf_inputs, tf.Tensor): - tf_inputs = tf.constant(tf_inputs) tfo = tf_model(tf_inputs, training=False) # Make sure model is built tf_model.load_weights(tf_checkpoint_path, by_name=True) From f09e5ecef0566ce485c7ee913602dd502fcaeeb8 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Tue, 24 Sep 2019 09:47:34 -0400 Subject: [PATCH 123/219] [Proposal] GLUE processors included in library --- examples/run_glue.py | 5 +- .../preprocessing/__init__.py | 56 ++++ .../preprocessing/glue.py | 274 +++++------------- pytorch_transformers/preprocessing/utils.py | 99 +++++++ 4 files changed, 230 insertions(+), 204 deletions(-) create mode 100644 pytorch_transformers/preprocessing/__init__.py rename examples/utils_glue.py => pytorch_transformers/preprocessing/glue.py (75%) create mode 100644 pytorch_transformers/preprocessing/utils.py diff --git a/examples/run_glue.py b/examples/run_glue.py index e8c2edc833..3748c1a13a 100644 --- a/examples/run_glue.py +++ b/examples/run_glue.py @@ -46,8 +46,7 @@ from pytorch_transformers import (WEIGHTS_NAME, BertConfig, from pytorch_transformers import AdamW, WarmupLinearSchedule -from utils_glue import (compute_metrics, convert_examples_to_features, - output_modes, processors) +from pytorch_transformers.preprocessing import (compute_metrics, output_modes, processors, convert_examples_to_glue_features) logger = logging.getLogger(__name__) @@ -276,7 +275,7 @@ def load_and_cache_examples(args, task, tokenizer, evaluate=False): # HACK(label indices are swapped in RoBERTa pretrained model) label_list[1], label_list[2] = label_list[2], label_list[1] examples = processor.get_dev_examples(args.data_dir) if evaluate else processor.get_train_examples(args.data_dir) - features = convert_examples_to_features(examples, label_list, args.max_seq_length, tokenizer, output_mode, + features = convert_examples_to_glue_features(examples, label_list, args.max_seq_length, tokenizer, output_mode, pad_on_left=bool(args.model_type in ['xlnet']), # pad on the left for xlnet pad_token=tokenizer.convert_tokens_to_ids([tokenizer.pad_token])[0], pad_token_segment_id=4 if args.model_type in ['xlnet'] else 0, diff --git a/pytorch_transformers/preprocessing/__init__.py b/pytorch_transformers/preprocessing/__init__.py new file mode 100644 index 0000000000..33426f06d6 --- /dev/null +++ b/pytorch_transformers/preprocessing/__init__.py @@ -0,0 +1,56 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from glue import (ColaProcessor, + MnliProcessor, + MnliMismatchedProcessor, + MrpcProcessor, + Sst2Processor, + StsbProcessor, + QqpProcessor, + QnliProcessor, + RteProcessor, + WnliProcessor, + convert_examples_to_glue_features, + ) + +from utils import DataProcessor, simple_accuracy, acc_and_f1, pearson_and_spearman, compute_metrics + +processors = { + "cola": ColaProcessor, + "mnli": MnliProcessor, + "mnli-mm": MnliMismatchedProcessor, + "mrpc": MrpcProcessor, + "sst-2": Sst2Processor, + "sts-b": StsbProcessor, + "qqp": QqpProcessor, + "qnli": QnliProcessor, + "rte": RteProcessor, + "wnli": WnliProcessor, +} + +output_modes = { + "cola": "classification", + "mnli": "classification", + "mnli-mm": "classification", + "mrpc": "classification", + "sst-2": "classification", + "sts-b": "regression", + "qqp": "classification", + "qnli": "classification", + "rte": "classification", + "wnli": "classification", +} diff --git a/examples/utils_glue.py b/pytorch_transformers/preprocessing/glue.py similarity index 75% rename from examples/utils_glue.py rename to pytorch_transformers/preprocessing/glue.py index 2557540cc6..b36ecebed6 100644 --- a/examples/utils_glue.py +++ b/pytorch_transformers/preprocessing/glue.py @@ -13,22 +13,84 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" BERT classification fine-tuning: utilities to work with GLUE tasks """ +""" GLUE processors and helpers """ -from __future__ import absolute_import, division, print_function - -import csv +from utils import DataProcessor import logging import os -import sys -from io import open - -from scipy.stats import pearsonr, spearmanr -from sklearn.metrics import matthews_corrcoef, f1_score logger = logging.getLogger(__name__) +def convert_examples_to_glue_features(examples, label_list, max_seq_length, + tokenizer, output_mode, + pad_on_left=False, + pad_token=0, + pad_token_segment_id=0, + mask_padding_with_zero=True): + """ + Loads a data file into a list of `InputBatch`s + """ + + label_map = {label: i for i, label in enumerate(label_list)} + + features = [] + for (ex_index, example) in enumerate(examples): + if ex_index % 10000 == 0: + logger.info("Writing example %d of %d" % (ex_index, len(examples))) + + inputs = tokenizer.encode_plus( + example.text_a, + example.text_b, + add_special_tokens=True, + output_token_type=True, + max_length=max_seq_length, + truncate_first_sequence=True # We're truncating the first sequence as a priority + ) + input_ids, segment_ids = inputs["input_ids"], inputs["token_type_ids"] + + # The mask has 1 for real tokens and 0 for padding tokens. Only real + # tokens are attended to. + input_mask = [1 if mask_padding_with_zero else 0] * len(input_ids) + + # Zero-pad up to the sequence length. + padding_length = max_seq_length - len(input_ids) + if pad_on_left: + input_ids = ([pad_token] * padding_length) + input_ids + input_mask = ([0 if mask_padding_with_zero else 1] * padding_length) + input_mask + segment_ids = ([pad_token_segment_id] * padding_length) + segment_ids + else: + input_ids = input_ids + ([pad_token] * padding_length) + input_mask = input_mask + ([0 if mask_padding_with_zero else 1] * padding_length) + segment_ids = segment_ids + ([pad_token_segment_id] * padding_length) + + assert len(input_ids) == max_seq_length + assert len(input_mask) == max_seq_length + assert len(segment_ids) == max_seq_length + + if output_mode == "classification": + label_id = label_map[example.label] + elif output_mode == "regression": + label_id = float(example.label) + else: + raise KeyError(output_mode) + + if ex_index < 5: + logger.info("*** Example ***") + logger.info("guid: %s" % (example.guid)) + logger.info("input_ids: %s" % " ".join([str(x) for x in input_ids])) + logger.info("input_mask: %s" % " ".join([str(x) for x in input_mask])) + logger.info("segment_ids: %s" % " ".join([str(x) for x in segment_ids])) + logger.info("label: %s (id = %d)" % (example.label, label_id)) + + features.append( + InputFeatures(input_ids=input_ids, + input_mask=input_mask, + segment_ids=segment_ids, + label_id=label_id)) + return features + + class InputExample(object): """A single training/test example for simple sequence classification.""" @@ -60,34 +122,6 @@ class InputFeatures(object): self.label_id = label_id -class DataProcessor(object): - """Base class for data converters for sequence classification data sets.""" - - def get_train_examples(self, data_dir): - """Gets a collection of `InputExample`s for the train set.""" - raise NotImplementedError() - - def get_dev_examples(self, data_dir): - """Gets a collection of `InputExample`s for the dev set.""" - raise NotImplementedError() - - def get_labels(self): - """Gets the list of labels for this data set.""" - raise NotImplementedError() - - @classmethod - def _read_tsv(cls, input_file, quotechar=None): - """Reads a tab separated value file.""" - with open(input_file, "r", encoding="utf-8-sig") as f: - reader = csv.reader(f, delimiter="\t", quotechar=quotechar) - lines = [] - for line in reader: - if sys.version_info[0] == 2: - line = list(unicode(cell, 'utf-8') for cell in line) - lines.append(line) - return lines - - class MrpcProcessor(DataProcessor): """Processor for the MRPC data set (GLUE version).""" @@ -302,7 +336,7 @@ class QnliProcessor(DataProcessor): def get_dev_examples(self, data_dir): """See base class.""" return self._create_examples( - self._read_tsv(os.path.join(data_dir, "dev.tsv")), + self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev_matched") def get_labels(self): @@ -387,168 +421,6 @@ class WnliProcessor(DataProcessor): InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) return examples - -def convert_examples_to_features(examples, label_list, max_seq_length, - tokenizer, output_mode, - pad_on_left=False, - pad_token=0, - pad_token_segment_id=0, - mask_padding_with_zero=True): - """ - Loads a data file into a list of `InputBatch`s - """ - - label_map = {label : i for i, label in enumerate(label_list)} - - features = [] - for (ex_index, example) in enumerate(examples): - if ex_index % 10000 == 0: - logger.info("Writing example %d of %d" % (ex_index, len(examples))) - - inputs = tokenizer.encode_plus( - example.text_a, - example.text_b, - add_special_tokens=True, - output_token_type=True, - max_length=max_seq_length, - truncate_first_sequence=True # We're truncating the first sequence as a priority - ) - input_ids, segment_ids = inputs["input_ids"], inputs["token_type_ids"] - - # The mask has 1 for real tokens and 0 for padding tokens. Only real - # tokens are attended to. - input_mask = [1 if mask_padding_with_zero else 0] * len(input_ids) - - # Zero-pad up to the sequence length. - padding_length = max_seq_length - len(input_ids) - if pad_on_left: - input_ids = ([pad_token] * padding_length) + input_ids - input_mask = ([0 if mask_padding_with_zero else 1] * padding_length) + input_mask - segment_ids = ([pad_token_segment_id] * padding_length) + segment_ids - else: - input_ids = input_ids + ([pad_token] * padding_length) - input_mask = input_mask + ([0 if mask_padding_with_zero else 1] * padding_length) - segment_ids = segment_ids + ([pad_token_segment_id] * padding_length) - - assert len(input_ids) == max_seq_length - assert len(input_mask) == max_seq_length - assert len(segment_ids) == max_seq_length - - if output_mode == "classification": - label_id = label_map[example.label] - elif output_mode == "regression": - label_id = float(example.label) - else: - raise KeyError(output_mode) - - if ex_index < 5: - logger.info("*** Example ***") - logger.info("guid: %s" % (example.guid)) - logger.info("input_ids: %s" % " ".join([str(x) for x in input_ids])) - logger.info("input_mask: %s" % " ".join([str(x) for x in input_mask])) - logger.info("segment_ids: %s" % " ".join([str(x) for x in segment_ids])) - logger.info("label: %s (id = %d)" % (example.label, label_id)) - - features.append( - InputFeatures(input_ids=input_ids, - input_mask=input_mask, - segment_ids=segment_ids, - label_id=label_id)) - return features - - -def _truncate_seq_pair(tokens_a, tokens_b, max_length): - """Truncates a sequence pair in place to the maximum length.""" - - # This is a simple heuristic which will always truncate the longer sequence - # one token at a time. This makes more sense than truncating an equal percent - # of tokens from each, since if one sequence is very short then each token - # that's truncated likely contains more information than a longer sequence. - while True: - total_length = len(tokens_a) + len(tokens_b) - if total_length <= max_length: - break - if len(tokens_a) > len(tokens_b): - tokens_a.pop() - else: - tokens_b.pop() - - -def simple_accuracy(preds, labels): - return (preds == labels).mean() - - -def acc_and_f1(preds, labels): - acc = simple_accuracy(preds, labels) - f1 = f1_score(y_true=labels, y_pred=preds) - return { - "acc": acc, - "f1": f1, - "acc_and_f1": (acc + f1) / 2, - } - - -def pearson_and_spearman(preds, labels): - pearson_corr = pearsonr(preds, labels)[0] - spearman_corr = spearmanr(preds, labels)[0] - return { - "pearson": pearson_corr, - "spearmanr": spearman_corr, - "corr": (pearson_corr + spearman_corr) / 2, - } - - -def compute_metrics(task_name, preds, labels): - assert len(preds) == len(labels) - if task_name == "cola": - return {"mcc": matthews_corrcoef(labels, preds)} - elif task_name == "sst-2": - return {"acc": simple_accuracy(preds, labels)} - elif task_name == "mrpc": - return acc_and_f1(preds, labels) - elif task_name == "sts-b": - return pearson_and_spearman(preds, labels) - elif task_name == "qqp": - return acc_and_f1(preds, labels) - elif task_name == "mnli": - return {"acc": simple_accuracy(preds, labels)} - elif task_name == "mnli-mm": - return {"acc": simple_accuracy(preds, labels)} - elif task_name == "qnli": - return {"acc": simple_accuracy(preds, labels)} - elif task_name == "rte": - return {"acc": simple_accuracy(preds, labels)} - elif task_name == "wnli": - return {"acc": simple_accuracy(preds, labels)} - else: - raise KeyError(task_name) - -processors = { - "cola": ColaProcessor, - "mnli": MnliProcessor, - "mnli-mm": MnliMismatchedProcessor, - "mrpc": MrpcProcessor, - "sst-2": Sst2Processor, - "sts-b": StsbProcessor, - "qqp": QqpProcessor, - "qnli": QnliProcessor, - "rte": RteProcessor, - "wnli": WnliProcessor, -} - -output_modes = { - "cola": "classification", - "mnli": "classification", - "mnli-mm": "classification", - "mrpc": "classification", - "sst-2": "classification", - "sts-b": "regression", - "qqp": "classification", - "qnli": "classification", - "rte": "classification", - "wnli": "classification", -} - GLUE_TASKS_NUM_LABELS = { "cola": 2, "mnli": 3, @@ -559,4 +431,4 @@ GLUE_TASKS_NUM_LABELS = { "qnli": 2, "rte": 2, "wnli": 2, -} +} \ No newline at end of file diff --git a/pytorch_transformers/preprocessing/utils.py b/pytorch_transformers/preprocessing/utils.py new file mode 100644 index 0000000000..b4a3d0d968 --- /dev/null +++ b/pytorch_transformers/preprocessing/utils.py @@ -0,0 +1,99 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import csv +import sys + +from scipy.stats import pearsonr, spearmanr +from sklearn.metrics import matthews_corrcoef, f1_score + + +class DataProcessor(object): + """Base class for data converters for sequence classification data sets.""" + + def get_train_examples(self, data_dir): + """Gets a collection of `InputExample`s for the train set.""" + raise NotImplementedError() + + def get_dev_examples(self, data_dir): + """Gets a collection of `InputExample`s for the dev set.""" + raise NotImplementedError() + + def get_labels(self): + """Gets the list of labels for this data set.""" + raise NotImplementedError() + + @classmethod + def _read_tsv(cls, input_file, quotechar=None): + """Reads a tab separated value file.""" + with open(input_file, "r", encoding="utf-8-sig") as f: + reader = csv.reader(f, delimiter="\t", quotechar=quotechar) + lines = [] + for line in reader: + if sys.version_info[0] == 2: + line = list(unicode(cell, 'utf-8') for cell in line) + lines.append(line) + return lines + + +def simple_accuracy(preds, labels): + return (preds == labels).mean() + + +def acc_and_f1(preds, labels): + acc = simple_accuracy(preds, labels) + f1 = f1_score(y_true=labels, y_pred=preds) + return { + "acc": acc, + "f1": f1, + "acc_and_f1": (acc + f1) / 2, + } + + +def pearson_and_spearman(preds, labels): + pearson_corr = pearsonr(preds, labels)[0] + spearman_corr = spearmanr(preds, labels)[0] + return { + "pearson": pearson_corr, + "spearmanr": spearman_corr, + "corr": (pearson_corr + spearman_corr) / 2, + } + + +def compute_metrics(task_name, preds, labels): + assert len(preds) == len(labels) + if task_name == "cola": + return {"mcc": matthews_corrcoef(labels, preds)} + elif task_name == "sst-2": + return {"acc": simple_accuracy(preds, labels)} + elif task_name == "mrpc": + return acc_and_f1(preds, labels) + elif task_name == "sts-b": + return pearson_and_spearman(preds, labels) + elif task_name == "qqp": + return acc_and_f1(preds, labels) + elif task_name == "mnli": + return {"acc": simple_accuracy(preds, labels)} + elif task_name == "mnli-mm": + return {"acc": simple_accuracy(preds, labels)} + elif task_name == "qnli": + return {"acc": simple_accuracy(preds, labels)} + elif task_name == "rte": + return {"acc": simple_accuracy(preds, labels)} + elif task_name == "wnli": + return {"acc": simple_accuracy(preds, labels)} + else: + raise KeyError(task_name) \ No newline at end of file From 0b82e3d0d913a3aee36bb4239aca38a6be476c5a Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Tue, 24 Sep 2019 09:52:25 -0400 Subject: [PATCH 124/219] Relative imports --- pytorch_transformers/preprocessing/__init__.py | 4 ++-- pytorch_transformers/preprocessing/glue.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pytorch_transformers/preprocessing/__init__.py b/pytorch_transformers/preprocessing/__init__.py index 33426f06d6..8550b80488 100644 --- a/pytorch_transformers/preprocessing/__init__.py +++ b/pytorch_transformers/preprocessing/__init__.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from glue import (ColaProcessor, +from .glue import (ColaProcessor, MnliProcessor, MnliMismatchedProcessor, MrpcProcessor, @@ -27,7 +27,7 @@ from glue import (ColaProcessor, convert_examples_to_glue_features, ) -from utils import DataProcessor, simple_accuracy, acc_and_f1, pearson_and_spearman, compute_metrics +from .utils import DataProcessor, simple_accuracy, acc_and_f1, pearson_and_spearman, compute_metrics processors = { "cola": ColaProcessor, diff --git a/pytorch_transformers/preprocessing/glue.py b/pytorch_transformers/preprocessing/glue.py index b36ecebed6..ed03e7ef4a 100644 --- a/pytorch_transformers/preprocessing/glue.py +++ b/pytorch_transformers/preprocessing/glue.py @@ -15,7 +15,7 @@ # limitations under the License. """ GLUE processors and helpers """ -from utils import DataProcessor +from .utils import DataProcessor import logging import os From a6981076eca5494b9d230f13217c14b93443888a Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 24 Sep 2019 16:46:26 +0200 Subject: [PATCH 125/219] various updates --- examples/utils_glue.py | 1 - .../tests/tokenization_tests_commons.py | 2 +- pytorch_transformers/tokenization_bert.py | 4 +- pytorch_transformers/tokenization_roberta.py | 4 +- pytorch_transformers/tokenization_utils.py | 153 +++++++----------- pytorch_transformers/tokenization_xlm.py | 4 +- pytorch_transformers/tokenization_xlnet.py | 4 +- 7 files changed, 63 insertions(+), 109 deletions(-) diff --git a/examples/utils_glue.py b/examples/utils_glue.py index 2557540cc6..e0ca9caa0a 100644 --- a/examples/utils_glue.py +++ b/examples/utils_glue.py @@ -409,7 +409,6 @@ def convert_examples_to_features(examples, label_list, max_seq_length, example.text_a, example.text_b, add_special_tokens=True, - output_token_type=True, max_length=max_seq_length, truncate_first_sequence=True # We're truncating the first sequence as a priority ) diff --git a/pytorch_transformers/tests/tokenization_tests_commons.py b/pytorch_transformers/tests/tokenization_tests_commons.py index 4ad92c8192..b71ba44436 100644 --- a/pytorch_transformers/tests/tokenization_tests_commons.py +++ b/pytorch_transformers/tests/tokenization_tests_commons.py @@ -196,7 +196,7 @@ class CommonTestCases: if tokenizer.add_special_tokens_sequence_pair.__qualname__.split('.')[0] != "PreTrainedTokenizer": seq_0 = "Test this method." seq_1 = "With these inputs." - information = tokenizer.encode_plus(seq_0, seq_1, add_special_tokens=True, output_token_type=True) + information = tokenizer.encode_plus(seq_0, seq_1, add_special_tokens=True) sequences, mask = information["input_ids"], information["token_type_ids"] assert len(sequences) == len(mask) diff --git a/pytorch_transformers/tokenization_bert.py b/pytorch_transformers/tokenization_bert.py index 7eca60a140..225152e065 100644 --- a/pytorch_transformers/tokenization_bert.py +++ b/pytorch_transformers/tokenization_bert.py @@ -204,7 +204,7 @@ class BertTokenizer(PreTrainedTokenizer): return cls + token_ids_0 + sep + token_ids_1 + sep - def create_token_type_ids_from_sequences(self, sequence_0, sequence_1): + def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1): """ Creates a mask from the two sequences passed to be used in a sequence-pair classification task. A BERT sequence pair mask has the following format: @@ -214,7 +214,7 @@ class BertTokenizer(PreTrainedTokenizer): sep = [self.sep_token_id] cls = [self.cls_token_id] - return len(cls + self.encode(sequence_0) + sep) * [0] + len(self.encode(sequence_1) + sep) * [1] + return len(cls + token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1] def save_vocabulary(self, vocab_path): """Save the tokenizer vocabulary to a directory or file.""" diff --git a/pytorch_transformers/tokenization_roberta.py b/pytorch_transformers/tokenization_roberta.py index 475aee47fa..ee8e97d6bf 100644 --- a/pytorch_transformers/tokenization_roberta.py +++ b/pytorch_transformers/tokenization_roberta.py @@ -97,7 +97,7 @@ class RobertaTokenizer(GPT2Tokenizer): cls = [self.cls_token_id] return cls + token_ids_0 + sep + sep + token_ids_1 + sep - def create_token_type_ids_from_sequences(self, sequence_0, sequence_1): + def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1): """ Creates a mask from the two sequences passed to be used in a sequence-pair classification task. A RoBERTa sequence pair mask has the following format: @@ -107,4 +107,4 @@ class RobertaTokenizer(GPT2Tokenizer): sep = [self.sep_token_id] cls = [self.cls_token_id] - return len(cls + self.encode(sequence_0) + sep + sep) * [0] + len(self.encode(sequence_1) + sep) * [1] \ No newline at end of file + return len(cls + token_ids_0 + sep + sep) * [0] + len(token_ids_1 + sep) * [1] \ No newline at end of file diff --git a/pytorch_transformers/tokenization_utils.py b/pytorch_transformers/tokenization_utils.py index c5efd37a53..136429b7d1 100644 --- a/pytorch_transformers/tokenization_utils.py +++ b/pytorch_transformers/tokenization_utils.py @@ -704,13 +704,14 @@ class PreTrainedTokenizer(object): to their model. **kwargs: passed to the `self.tokenize()` method """ - return self.encode_plus(text, text_pair, add_special_tokens, **kwargs)["input_ids"] + encoded_inputs = self.encode_plus(text, text_pair=text_pair, add_special_tokens=add_special_tokens, **kwargs) + + return encoded_inputs["input_ids"] def encode_plus(self, text, text_pair=None, add_special_tokens=False, - output_token_type=False, max_length=None, stride=0, truncate_first_sequence=True, @@ -728,8 +729,6 @@ class PreTrainedTokenizer(object): `convert_tokens_to_ids` method) add_special_tokens: if set to ``True``, the sequences will be encoded with the special tokens relative to their model. - output_token_type: if set to ``True``, returns the text pair corresponding mask with 0 for the first sequence, - and 1 for the second. max_length: if set to a number, will limit the total sequence returned so that it has a maximum length. If there are overflowing tokens, those will be added to the returned dictionary stride: if set to a number along with max_length, the overflowing tokens returned will contain some tokens @@ -739,133 +738,89 @@ class PreTrainedTokenizer(object): **kwargs: passed to the `self.tokenize()` method """ - information = {} - def get_input_ids(text): if isinstance(text, six.string_types): - input_ids = self.convert_tokens_to_ids(self.tokenize(text, **kwargs)) + return self.convert_tokens_to_ids(self.tokenize(text, **kwargs)) elif isinstance(text, (list, tuple)) and len(text) > 0 and isinstance(text[0], six.string_types): - input_ids = self.convert_tokens_to_ids(text) + return self.convert_tokens_to_ids(text) elif isinstance(text, (list, tuple)) and len(text) > 0 and isinstance(text[0], int): - input_ids = text + return text else: raise ValueError("Input is not valid. Should be a string, a list/tuple of strings or a list/tuple of integers.") - return input_ids + first_ids = get_input_ids(text) + second_ids = get_input_ids(text_pair) if text_pair is not None else None - if text_pair is None: - sequence_tokens = get_input_ids(text) + return self.prepare_for_model(first_ids, + pair_ids=second_ids, + max_length=max_length, + add_special_tokens=add_special_tokens, + stride=stride, + truncate_first_sequence=truncate_first_sequence) - if add_special_tokens: - information = self.prepare_for_model(sequence_tokens, max_length=max_length, stride=stride) - else: - if max_length: - information["overflowing_tokens"] = sequence_tokens[max_length - stride:] - sequence_tokens = sequence_tokens[:max_length] - information["input_ids"] = sequence_tokens - if output_token_type: - information["token_type_ids"] = [0] * len(information["input_ids"]) - else: - first_sentence_tokens = get_input_ids(text) - second_sentence_tokens = get_input_ids(text_pair) - - if add_special_tokens: - information = self.prepare_pair_for_model( - first_sentence_tokens, - second_sentence_tokens, - max_length=max_length, - truncate_first_sequence=truncate_first_sequence, - stride=stride - ) - - if output_token_type: - information["token_type_ids"] = self.create_token_type_ids_from_sequences(text, text_pair) - else: - logger.warning("No special tokens were added. The two sequences have been concatenated.") - sequence = first_sentence_tokens + second_sentence_tokens - - if max_length: - information["overflowing_tokens"] = sequence[max_length - stride:] - sequence = sequence[:max_length] - if output_token_type: - information["token_type_ids"] = [0] * len(sequence) - - information["input_ids"] = sequence - - return information - - def prepare_for_model(self, ids, max_length=None, stride=0): + def prepare_for_model(self, ids, pair_ids=None, max_length=None, add_special_tokens=False, stride=0, truncate_first_sequence=True): """ - Prepares a list of tokenized input ids so that it can be used by the model. It adds special tokens, truncates + Prepares a sequence of input id, or a pair of sequences of inputs ids so that it can be used by the model. + It adds special tokens, truncates sequences if overflowing while taking into account the special tokens and manages a window stride for overflowing tokens Args: ids: list of tokenized input ids. Can be obtained from a string by chaining the `tokenize` and `convert_tokens_to_ids` methods. + pair_ids: Optional second list of input ids. Can be obtained from a string by chaining the + `tokenize` and `convert_tokens_to_ids` methods. max_length: maximum length of the returned list. Will truncate by taking into account the special tokens. + add_special_tokens: if set to ``True``, the sequences will be encoded with the special tokens relative + to their model. stride: window stride for overflowing tokens. Can be useful for edge effect removal when using sequential list of inputs. + truncate_first_sequence: if set to `True` and an optional second list of input ids is provided, + alongside a specified `max_length`, will truncate the first sequence if the total size is superior + than the specified `max_length`. If set to `False`, will truncate the second sequence instead. Return: a dictionary containing the `input_ids` as well as the `overflowing_tokens` if a `max_length` was given. """ - information = {} + pair = bool(pair_ids is not None) + len_ids = len(ids) + len_pair_ids = len(pair_ids) if pair else 0 + + encoded_inputs = {} if max_length: - n_added_tokens = self.num_added_tokens() - information["overflowing_tokens"] = ids[max_length - n_added_tokens - stride:] - ids = ids[:max_length - n_added_tokens] - information["input_ids"] = self.add_special_tokens_single_sequence(ids) - - return information - - def prepare_pair_for_model(self, ids_0, ids_1, max_length=None, truncate_first_sequence=True, stride=0): - """ - Prepares a list of tokenized input ids pair so that it can be used by the model. It adds special tokens, - truncates sequences if overflowing while taking into account the special tokens and manages a window stride for - overflowing tokens - - Args: - ids_0: list of tokenized input ids. Can be obtained from a string by chaining the - `tokenize` and `convert_tokens_to_ids` methods. - ids_1: second list of tokenized input ids. Can be obtained from a string by chaining the - `tokenize` and `convert_tokens_to_ids` methods. - max_length: maximum length of the returned list. Will truncate by taking into account the special tokens. - truncate_first_sequence: if set to `True`, alongside a specified `max_length`, will truncate the first - sequence if the total size is superior than the specified `max_length`. If set to `False`, will - truncate the second sequence instead. - stride: window stride for overflowing tokens. Can be useful for edge effect removal when using sequential - list of inputs. - - Return: - a dictionary containing the `input_ids` as well as the `overflowing_tokens` if a `max_length` was given. - """ - f_len, s_len = len(ids_0), len(ids_1) - information = {} - - if max_length: - n_added_tokens = self.num_added_tokens(pair=True) - if len(ids_0) + n_added_tokens >= max_length: + n_added_tokens = self.num_added_tokens(pair=pair) if add_special_tokens else 0 + if pair and n_added_tokens + (len_pair_ids if truncate_first_sequence else len_ids) >= max_length: logger.warning( - "The first sequence is longer than the maximum specified length. This sequence will not be truncated.") + "You supplied a pair of sequence in which the sequence that will not be truncated is longer than the maximum specified length." + "This pair of sequences will not be truncated.") else: - if f_len + s_len + self.num_added_tokens(pair=True) > max_length: - if truncate_first_sequence: - information["overflowing_tokens"] = ids_0[max_length - s_len - n_added_tokens - stride:] - ids_0 = ids_0[:max_length - s_len - n_added_tokens] + if n_added_tokens + len_ids + len_pair_ids > max_length: + if truncate_first_sequence or not pair: + encoded_inputs["overflowing_tokens"] = ids[max_length - len_pair_ids - n_added_tokens - stride:] + ids = ids[:max_length - len_pair_ids - n_added_tokens] + elif not truncate_first_sequence and pair: + encoded_inputs["overflowing_tokens"] = pair_ids[max_length - len_ids - n_added_tokens - stride:] + pair_ids = pair_ids[:max_length - len_ids - n_added_tokens] else: - information["overflowing_tokens"] = ids_1[max_length - f_len - n_added_tokens - stride:] - ids_1 = ids_1[:max_length - f_len - n_added_tokens] + logger.warning( + "Cannot truncate second sequence as it is not provided. No truncation.") - sequence = self.add_special_tokens_sequence_pair(ids_0, ids_1) - information["input_ids"] = sequence + if add_special_tokens: + sequence = self.add_special_tokens_sequence_pair(ids, pair_ids) if pair else self.add_special_tokens_single_sequence(ids) + token_type_ids = self.create_token_type_ids_from_sequences(ids, pair_ids) if pair else [0] * len(sequence) + else: + sequence = ids + pair_ids if pair else ids + token_type_ids = [0] * len(ids) + ([1] * len(pair_ids) if pair else []) - return information + encoded_inputs["input_ids"] = sequence + encoded_inputs["token_type_ids"] = token_type_ids - def create_token_type_ids_from_sequences(self, sequence_0, sequence_1): + return encoded_inputs + + def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1): logger.warning("This tokenizer does not make use of special tokens.") - return [0] * len(self.encode(sequence_0)) + [1] * len(self.encode(sequence_1)) + return [0] * len(token_ids_0) + [1] * len(token_ids_1) def add_special_tokens_single_sequence(self, token_ids): logger.warning("This tokenizer does not make use of special tokens. The sequence has been returned with no modification.") diff --git a/pytorch_transformers/tokenization_xlm.py b/pytorch_transformers/tokenization_xlm.py index 833a8d8be6..f1e49416a4 100644 --- a/pytorch_transformers/tokenization_xlm.py +++ b/pytorch_transformers/tokenization_xlm.py @@ -770,7 +770,7 @@ class XLMTokenizer(PreTrainedTokenizer): cls = [self.cls_token_id] return cls + token_ids_0 + sep + token_ids_1 + sep - def create_token_type_ids_from_sequences(self, sequence_0, sequence_1): + def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1): """ Creates a mask from the two sequences passed to be used in a sequence-pair classification task. An XLM sequence pair mask has the following format: @@ -780,7 +780,7 @@ class XLMTokenizer(PreTrainedTokenizer): sep = [self.sep_token_id] cls = [self.cls_token_id] - return len(cls + self.encode(sequence_0) + sep) * [0] + len(self.encode(sequence_1) + sep) * [1] + return len(cls + token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1] def save_vocabulary(self, save_directory): """Save the tokenizer vocabulary and merge files to a directory.""" diff --git a/pytorch_transformers/tokenization_xlnet.py b/pytorch_transformers/tokenization_xlnet.py index 5febf16418..941c6c5bc3 100644 --- a/pytorch_transformers/tokenization_xlnet.py +++ b/pytorch_transformers/tokenization_xlnet.py @@ -200,7 +200,7 @@ class XLNetTokenizer(PreTrainedTokenizer): cls = [self.cls_token_id] return token_ids_0 + sep + token_ids_1 + sep + cls - def create_token_type_ids_from_sequences(self, sequence_0, sequence_1): + def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1): """ Creates a mask from the two sequences passed to be used in a sequence-pair classification task. A BERT sequence pair mask has the following format: @@ -211,7 +211,7 @@ class XLNetTokenizer(PreTrainedTokenizer): cls = [self.cls_token_id] cls_segment_id = [2] - return len(self.encode(sequence_0) + sep) * [0] + len(self.encode(sequence_1) + sep) * [1] + cls_segment_id + return len(token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1] + cls_segment_id def save_vocabulary(self, save_directory): """ Save the sentencepiece vocabulary (copy original file) and special tokens file From b5ec526f8576daedfd4bfc7614ef5a860b83fd36 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 24 Sep 2019 17:10:50 +0200 Subject: [PATCH 126/219] updated data processor and metrics --- .gitignore | 2 +- examples/run_glue.py | 7 +- pytorch_transformers/__init__.py | 7 ++ pytorch_transformers/data/__init__.py | 6 ++ pytorch_transformers/data/metrics/__init__.py | 83 +++++++++++++++++++ .../data/processors/__init__.py | 2 + .../processors}/glue.py | 83 +++++++++---------- .../processors}/utils.py | 80 +++++++----------- .../preprocessing/__init__.py | 56 ------------- 9 files changed, 171 insertions(+), 155 deletions(-) create mode 100644 pytorch_transformers/data/__init__.py create mode 100644 pytorch_transformers/data/metrics/__init__.py create mode 100644 pytorch_transformers/data/processors/__init__.py rename pytorch_transformers/{preprocessing => data/processors}/glue.py (92%) rename pytorch_transformers/{preprocessing => data/processors}/utils.py (52%) delete mode 100644 pytorch_transformers/preprocessing/__init__.py diff --git a/.gitignore b/.gitignore index d285d0ded9..e673ce5f47 100644 --- a/.gitignore +++ b/.gitignore @@ -130,5 +130,5 @@ runs examples/runs # data -data +/data serialization_dir \ No newline at end of file diff --git a/examples/run_glue.py b/examples/run_glue.py index 3748c1a13a..b39c6bf054 100644 --- a/examples/run_glue.py +++ b/examples/run_glue.py @@ -46,7 +46,10 @@ from pytorch_transformers import (WEIGHTS_NAME, BertConfig, from pytorch_transformers import AdamW, WarmupLinearSchedule -from pytorch_transformers.preprocessing import (compute_metrics, output_modes, processors, convert_examples_to_glue_features) +from pytorch_transformers import glue_compute_metrics as compute_metrics +from pytorch_transformers import glue_output_modes as output_modes +from pytorch_transformers import glue_processors as processors +from pytorch_transformers import glue_convert_examples_to_features as convert_examples_to_features logger = logging.getLogger(__name__) @@ -275,7 +278,7 @@ def load_and_cache_examples(args, task, tokenizer, evaluate=False): # HACK(label indices are swapped in RoBERTa pretrained model) label_list[1], label_list[2] = label_list[2], label_list[1] examples = processor.get_dev_examples(args.data_dir) if evaluate else processor.get_train_examples(args.data_dir) - features = convert_examples_to_glue_features(examples, label_list, args.max_seq_length, tokenizer, output_mode, + features = convert_examples_to_features(examples, label_list, args.max_seq_length, tokenizer, output_mode, pad_on_left=bool(args.model_type in ['xlnet']), # pad on the left for xlnet pad_token=tokenizer.convert_tokens_to_ids([tokenizer.pad_token])[0], pad_token_segment_id=4 if args.model_type in ['xlnet'] else 0, diff --git a/pytorch_transformers/__init__.py b/pytorch_transformers/__init__.py index f12a5ebea7..93de6a982b 100644 --- a/pytorch_transformers/__init__.py +++ b/pytorch_transformers/__init__.py @@ -73,3 +73,10 @@ from .optimization import (AdamW, ConstantLRSchedule, WarmupConstantSchedule, Wa from .file_utils import (PYTORCH_TRANSFORMERS_CACHE, PYTORCH_PRETRAINED_BERT_CACHE, cached_path, add_start_docstrings, add_end_docstrings, WEIGHTS_NAME, TF_WEIGHTS_NAME, CONFIG_NAME) + +from .data import (is_sklearn_available, + InputExample, InputFeatures, DataProcessor, + glue_output_modes, glue_convert_examples_to_features, glue_processors) + +if is_sklearn_available(): + from .data import glue_compute_metrics diff --git a/pytorch_transformers/data/__init__.py b/pytorch_transformers/data/__init__.py new file mode 100644 index 0000000000..4522b802ab --- /dev/null +++ b/pytorch_transformers/data/__init__.py @@ -0,0 +1,6 @@ +from .processors import (InputExample, InputFeatures, DataProcessor, + glue_output_modes, glue_convert_examples_to_features, glue_processors) +from .metrics import is_sklearn_available + +if is_sklearn_available(): + from .metrics import glue_compute_metrics diff --git a/pytorch_transformers/data/metrics/__init__.py b/pytorch_transformers/data/metrics/__init__.py new file mode 100644 index 0000000000..4e0c088b90 --- /dev/null +++ b/pytorch_transformers/data/metrics/__init__.py @@ -0,0 +1,83 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import csv +import sys +import logging + +logger = logging.getLogger(__name__) + +try: + from scipy.stats import pearsonr, spearmanr + from sklearn.metrics import matthews_corrcoef, f1_score + _has_sklearn = True +except e: + logger.warning("To use data.metrics please install scikit-learn. See https://scikit-learn.org/stable/index.html") + _has_sklearn = False + +def is_sklearn_available(): + return _has_sklearn + +if _has_sklearn: + + def simple_accuracy(preds, labels): + return (preds == labels).mean() + + + def acc_and_f1(preds, labels): + acc = simple_accuracy(preds, labels) + f1 = f1_score(y_true=labels, y_pred=preds) + return { + "acc": acc, + "f1": f1, + "acc_and_f1": (acc + f1) / 2, + } + + + def pearson_and_spearman(preds, labels): + pearson_corr = pearsonr(preds, labels)[0] + spearman_corr = spearmanr(preds, labels)[0] + return { + "pearson": pearson_corr, + "spearmanr": spearman_corr, + "corr": (pearson_corr + spearman_corr) / 2, + } + + + def glue_compute_metrics(task_name, preds, labels): + assert len(preds) == len(labels) + if task_name == "cola": + return {"mcc": matthews_corrcoef(labels, preds)} + elif task_name == "sst-2": + return {"acc": simple_accuracy(preds, labels)} + elif task_name == "mrpc": + return acc_and_f1(preds, labels) + elif task_name == "sts-b": + return pearson_and_spearman(preds, labels) + elif task_name == "qqp": + return acc_and_f1(preds, labels) + elif task_name == "mnli": + return {"acc": simple_accuracy(preds, labels)} + elif task_name == "mnli-mm": + return {"acc": simple_accuracy(preds, labels)} + elif task_name == "qnli": + return {"acc": simple_accuracy(preds, labels)} + elif task_name == "rte": + return {"acc": simple_accuracy(preds, labels)} + elif task_name == "wnli": + return {"acc": simple_accuracy(preds, labels)} + else: + raise KeyError(task_name) diff --git a/pytorch_transformers/data/processors/__init__.py b/pytorch_transformers/data/processors/__init__.py new file mode 100644 index 0000000000..1a442bf839 --- /dev/null +++ b/pytorch_transformers/data/processors/__init__.py @@ -0,0 +1,2 @@ +from .utils import InputExample, InputFeatures, DataProcessor +from .glue import output_modes, processors, convert_examples_to_glue_features diff --git a/pytorch_transformers/preprocessing/glue.py b/pytorch_transformers/data/processors/glue.py similarity index 92% rename from pytorch_transformers/preprocessing/glue.py rename to pytorch_transformers/data/processors/glue.py index ed03e7ef4a..e1376816de 100644 --- a/pytorch_transformers/preprocessing/glue.py +++ b/pytorch_transformers/data/processors/glue.py @@ -15,12 +15,50 @@ # limitations under the License. """ GLUE processors and helpers """ -from .utils import DataProcessor import logging import os +from .utils import DataProcessor, InputExample, InputFeatures + logger = logging.getLogger(__name__) +GLUE_TASKS_NUM_LABELS = { + "cola": 2, + "mnli": 3, + "mrpc": 2, + "sst-2": 2, + "sts-b": 1, + "qqp": 2, + "qnli": 2, + "rte": 2, + "wnli": 2, +} + +processors = { + "cola": ColaProcessor, + "mnli": MnliProcessor, + "mnli-mm": MnliMismatchedProcessor, + "mrpc": MrpcProcessor, + "sst-2": Sst2Processor, + "sts-b": StsbProcessor, + "qqp": QqpProcessor, + "qnli": QnliProcessor, + "rte": RteProcessor, + "wnli": WnliProcessor, +} + +output_modes = { + "cola": "classification", + "mnli": "classification", + "mnli-mm": "classification", + "mrpc": "classification", + "sst-2": "classification", + "sts-b": "regression", + "qqp": "classification", + "qnli": "classification", + "rte": "classification", + "wnli": "classification", +} def convert_examples_to_glue_features(examples, label_list, max_seq_length, tokenizer, output_mode, @@ -91,37 +129,6 @@ def convert_examples_to_glue_features(examples, label_list, max_seq_length, return features -class InputExample(object): - """A single training/test example for simple sequence classification.""" - - def __init__(self, guid, text_a, text_b=None, label=None): - """Constructs a InputExample. - - Args: - guid: Unique id for the example. - text_a: string. The untokenized text of the first sequence. For single - sequence tasks, only this sequence must be specified. - text_b: (Optional) string. The untokenized text of the second sequence. - Only must be specified for sequence pair tasks. - label: (Optional) string. The label of the example. This should be - specified for train and dev examples, but not for test examples. - """ - self.guid = guid - self.text_a = text_a - self.text_b = text_b - self.label = label - - -class InputFeatures(object): - """A single set of features of data.""" - - def __init__(self, input_ids, input_mask, segment_ids, label_id): - self.input_ids = input_ids - self.input_mask = input_mask - self.segment_ids = segment_ids - self.label_id = label_id - - class MrpcProcessor(DataProcessor): """Processor for the MRPC data set (GLUE version).""" @@ -420,15 +427,3 @@ class WnliProcessor(DataProcessor): examples.append( InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) return examples - -GLUE_TASKS_NUM_LABELS = { - "cola": 2, - "mnli": 3, - "mrpc": 2, - "sst-2": 2, - "sts-b": 1, - "qqp": 2, - "qnli": 2, - "rte": 2, - "wnli": 2, -} \ No newline at end of file diff --git a/pytorch_transformers/preprocessing/utils.py b/pytorch_transformers/data/processors/utils.py similarity index 52% rename from pytorch_transformers/preprocessing/utils.py rename to pytorch_transformers/data/processors/utils.py index b4a3d0d968..af90e7a47c 100644 --- a/pytorch_transformers/preprocessing/utils.py +++ b/pytorch_transformers/data/processors/utils.py @@ -17,8 +17,34 @@ import csv import sys -from scipy.stats import pearsonr, spearmanr -from sklearn.metrics import matthews_corrcoef, f1_score +class InputExample(object): + """A single training/test example for simple sequence classification.""" + def __init__(self, guid, text_a, text_b=None, label=None): + """Constructs a InputExample. + + Args: + guid: Unique id for the example. + text_a: string. The untokenized text of the first sequence. For single + sequence tasks, only this sequence must be specified. + text_b: (Optional) string. The untokenized text of the second sequence. + Only must be specified for sequence pair tasks. + label: (Optional) string. The label of the example. This should be + specified for train and dev examples, but not for test examples. + """ + self.guid = guid + self.text_a = text_a + self.text_b = text_b + self.label = label + + +class InputFeatures(object): + """A single set of features of data.""" + + def __init__(self, input_ids, input_mask, segment_ids, label_id): + self.input_ids = input_ids + self.input_mask = input_mask + self.segment_ids = segment_ids + self.label_id = label_id class DataProcessor(object): @@ -47,53 +73,3 @@ class DataProcessor(object): line = list(unicode(cell, 'utf-8') for cell in line) lines.append(line) return lines - - -def simple_accuracy(preds, labels): - return (preds == labels).mean() - - -def acc_and_f1(preds, labels): - acc = simple_accuracy(preds, labels) - f1 = f1_score(y_true=labels, y_pred=preds) - return { - "acc": acc, - "f1": f1, - "acc_and_f1": (acc + f1) / 2, - } - - -def pearson_and_spearman(preds, labels): - pearson_corr = pearsonr(preds, labels)[0] - spearman_corr = spearmanr(preds, labels)[0] - return { - "pearson": pearson_corr, - "spearmanr": spearman_corr, - "corr": (pearson_corr + spearman_corr) / 2, - } - - -def compute_metrics(task_name, preds, labels): - assert len(preds) == len(labels) - if task_name == "cola": - return {"mcc": matthews_corrcoef(labels, preds)} - elif task_name == "sst-2": - return {"acc": simple_accuracy(preds, labels)} - elif task_name == "mrpc": - return acc_and_f1(preds, labels) - elif task_name == "sts-b": - return pearson_and_spearman(preds, labels) - elif task_name == "qqp": - return acc_and_f1(preds, labels) - elif task_name == "mnli": - return {"acc": simple_accuracy(preds, labels)} - elif task_name == "mnli-mm": - return {"acc": simple_accuracy(preds, labels)} - elif task_name == "qnli": - return {"acc": simple_accuracy(preds, labels)} - elif task_name == "rte": - return {"acc": simple_accuracy(preds, labels)} - elif task_name == "wnli": - return {"acc": simple_accuracy(preds, labels)} - else: - raise KeyError(task_name) \ No newline at end of file diff --git a/pytorch_transformers/preprocessing/__init__.py b/pytorch_transformers/preprocessing/__init__.py deleted file mode 100644 index 8550b80488..0000000000 --- a/pytorch_transformers/preprocessing/__init__.py +++ /dev/null @@ -1,56 +0,0 @@ -# coding=utf-8 -# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. -# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from .glue import (ColaProcessor, - MnliProcessor, - MnliMismatchedProcessor, - MrpcProcessor, - Sst2Processor, - StsbProcessor, - QqpProcessor, - QnliProcessor, - RteProcessor, - WnliProcessor, - convert_examples_to_glue_features, - ) - -from .utils import DataProcessor, simple_accuracy, acc_and_f1, pearson_and_spearman, compute_metrics - -processors = { - "cola": ColaProcessor, - "mnli": MnliProcessor, - "mnli-mm": MnliMismatchedProcessor, - "mrpc": MrpcProcessor, - "sst-2": Sst2Processor, - "sts-b": StsbProcessor, - "qqp": QqpProcessor, - "qnli": QnliProcessor, - "rte": RteProcessor, - "wnli": WnliProcessor, -} - -output_modes = { - "cola": "classification", - "mnli": "classification", - "mnli-mm": "classification", - "mrpc": "classification", - "sst-2": "classification", - "sts-b": "regression", - "qqp": "classification", - "qnli": "classification", - "rte": "classification", - "wnli": "classification", -} From 99a90e43d421369357815b21771f5211c2528667 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 24 Sep 2019 17:16:46 +0200 Subject: [PATCH 127/219] update data processors __init__ --- pytorch_transformers/__init__.py | 2 +- pytorch_transformers/data/__init__.py | 6 +- .../data/processors/__init__.py | 3 +- pytorch_transformers/data/processors/glue.py | 78 +++++++++---------- 4 files changed, 45 insertions(+), 44 deletions(-) diff --git a/pytorch_transformers/__init__.py b/pytorch_transformers/__init__.py index 93de6a982b..130a32885a 100644 --- a/pytorch_transformers/__init__.py +++ b/pytorch_transformers/__init__.py @@ -76,7 +76,7 @@ from .file_utils import (PYTORCH_TRANSFORMERS_CACHE, PYTORCH_PRETRAINED_BERT_CAC from .data import (is_sklearn_available, InputExample, InputFeatures, DataProcessor, - glue_output_modes, glue_convert_examples_to_features, glue_processors) + glue_output_modes, glue_convert_examples_to_features, glue_processors, glue_tasks_num_labels) if is_sklearn_available(): from .data import glue_compute_metrics diff --git a/pytorch_transformers/data/__init__.py b/pytorch_transformers/data/__init__.py index 4522b802ab..e910d6da2e 100644 --- a/pytorch_transformers/data/__init__.py +++ b/pytorch_transformers/data/__init__.py @@ -1,6 +1,6 @@ -from .processors import (InputExample, InputFeatures, DataProcessor, - glue_output_modes, glue_convert_examples_to_features, glue_processors) -from .metrics import is_sklearn_available +from .processors import InputExample, InputFeatures, DataProcessor +from .processors import glue_output_modes, glue_processors, glue_tasks_num_labels, glue_convert_examples_to_features +from .metrics import is_sklearn_available if is_sklearn_available(): from .metrics import glue_compute_metrics diff --git a/pytorch_transformers/data/processors/__init__.py b/pytorch_transformers/data/processors/__init__.py index 1a442bf839..af38c54beb 100644 --- a/pytorch_transformers/data/processors/__init__.py +++ b/pytorch_transformers/data/processors/__init__.py @@ -1,2 +1,3 @@ from .utils import InputExample, InputFeatures, DataProcessor -from .glue import output_modes, processors, convert_examples_to_glue_features +from .glue import glue_output_modes, glue_processors, glue_tasks_num_labels, glue_convert_examples_to_features + diff --git a/pytorch_transformers/data/processors/glue.py b/pytorch_transformers/data/processors/glue.py index e1376816de..a7d56491e2 100644 --- a/pytorch_transformers/data/processors/glue.py +++ b/pytorch_transformers/data/processors/glue.py @@ -22,45 +22,7 @@ from .utils import DataProcessor, InputExample, InputFeatures logger = logging.getLogger(__name__) -GLUE_TASKS_NUM_LABELS = { - "cola": 2, - "mnli": 3, - "mrpc": 2, - "sst-2": 2, - "sts-b": 1, - "qqp": 2, - "qnli": 2, - "rte": 2, - "wnli": 2, -} - -processors = { - "cola": ColaProcessor, - "mnli": MnliProcessor, - "mnli-mm": MnliMismatchedProcessor, - "mrpc": MrpcProcessor, - "sst-2": Sst2Processor, - "sts-b": StsbProcessor, - "qqp": QqpProcessor, - "qnli": QnliProcessor, - "rte": RteProcessor, - "wnli": WnliProcessor, -} - -output_modes = { - "cola": "classification", - "mnli": "classification", - "mnli-mm": "classification", - "mrpc": "classification", - "sst-2": "classification", - "sts-b": "regression", - "qqp": "classification", - "qnli": "classification", - "rte": "classification", - "wnli": "classification", -} - -def convert_examples_to_glue_features(examples, label_list, max_seq_length, +def glue_convert_examples_to_features(examples, label_list, max_seq_length, tokenizer, output_mode, pad_on_left=False, pad_token=0, @@ -427,3 +389,41 @@ class WnliProcessor(DataProcessor): examples.append( InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) return examples + +glue_tasks_num_labels = { + "cola": 2, + "mnli": 3, + "mrpc": 2, + "sst-2": 2, + "sts-b": 1, + "qqp": 2, + "qnli": 2, + "rte": 2, + "wnli": 2, +} + +glue_processors = { + "cola": ColaProcessor, + "mnli": MnliProcessor, + "mnli-mm": MnliMismatchedProcessor, + "mrpc": MrpcProcessor, + "sst-2": Sst2Processor, + "sts-b": StsbProcessor, + "qqp": QqpProcessor, + "qnli": QnliProcessor, + "rte": RteProcessor, + "wnli": WnliProcessor, +} + +glue_output_modes = { + "cola": "classification", + "mnli": "classification", + "mnli-mm": "classification", + "mrpc": "classification", + "sst-2": "classification", + "sts-b": "regression", + "qqp": "classification", + "qnli": "classification", + "rte": "classification", + "wnli": "classification", +} From 743e383d4b04eff248572722ed014516eb08caa8 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 24 Sep 2019 17:21:54 +0200 Subject: [PATCH 128/219] py2 fix --- pytorch_transformers/data/metrics/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_transformers/data/metrics/__init__.py b/pytorch_transformers/data/metrics/__init__.py index 4e0c088b90..c9ebaac38d 100644 --- a/pytorch_transformers/data/metrics/__init__.py +++ b/pytorch_transformers/data/metrics/__init__.py @@ -24,7 +24,7 @@ try: from scipy.stats import pearsonr, spearmanr from sklearn.metrics import matthews_corrcoef, f1_score _has_sklearn = True -except e: +except (AttributeError, ImportError) as e: logger.warning("To use data.metrics please install scikit-learn. See https://scikit-learn.org/stable/index.html") _has_sklearn = False From 789ea72037219332e20e66e626b6c4c32851f0dc Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 24 Sep 2019 17:32:01 +0200 Subject: [PATCH 129/219] fix output_token_type in glue --- pytorch_transformers/data/processors/glue.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pytorch_transformers/data/processors/glue.py b/pytorch_transformers/data/processors/glue.py index a7d56491e2..cb89ccf6c6 100644 --- a/pytorch_transformers/data/processors/glue.py +++ b/pytorch_transformers/data/processors/glue.py @@ -43,7 +43,6 @@ def glue_convert_examples_to_features(examples, label_list, max_seq_length, example.text_a, example.text_b, add_special_tokens=True, - output_token_type=True, max_length=max_seq_length, truncate_first_sequence=True # We're truncating the first sequence as a priority ) From 1761d2091a898103a07fc2cd4c02e28bc5b72be0 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Tue, 24 Sep 2019 14:59:10 -0400 Subject: [PATCH 130/219] Check to see if the models have the same results when in eval mode (pt) or when training=False (tf) --- pytorch_transformers/tests/modeling_common_test.py | 10 ++++++++++ pytorch_transformers/tests/modeling_tf_common_test.py | 8 ++++++++ 2 files changed, 18 insertions(+) diff --git a/pytorch_transformers/tests/modeling_common_test.py b/pytorch_transformers/tests/modeling_common_test.py index ff321193a5..5e30cd1e32 100644 --- a/pytorch_transformers/tests/modeling_common_test.py +++ b/pytorch_transformers/tests/modeling_common_test.py @@ -68,6 +68,16 @@ class CommonTestCases: self.assertIn(param.data.mean().item(), [0.0, 1.0], msg="Parameter {} of model {} seems not properly initialized".format(name, model_class)) + def test_determinism(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + model = model_class(config) + model.eval() + first, second = model(inputs_dict["input_ids"])[0], model(inputs_dict["input_ids"])[0] + self.assertEqual(first.ne(second).sum().item(), 0) + + def test_attention_outputs(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() diff --git a/pytorch_transformers/tests/modeling_tf_common_test.py b/pytorch_transformers/tests/modeling_tf_common_test.py index ecd1e387f9..5e7d29cb7f 100644 --- a/pytorch_transformers/tests/modeling_tf_common_test.py +++ b/pytorch_transformers/tests/modeling_tf_common_test.py @@ -298,6 +298,14 @@ class TFCommonTestCases: # self.assertGreater(len(params_not_tied), len(params_tied)) # self.assertEqual(len(params_tied_2), len(params_tied)) + def test_determinism(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + model = model_class(config) + first, second = model(inputs_dict, training=False)[0], model(inputs_dict, training=False)[0] + self.assertTrue(tf.math.equal(first, second).numpy().all()) + def ids_tensor(shape, vocab_size, rng=None, name=None, dtype=None): """Creates a random int32 tensor of the shape within the vocab size.""" From c4acc3a8e96d7cb1d69b72899c4e730719cfe498 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Wed, 25 Sep 2019 10:19:14 +0200 Subject: [PATCH 131/219] let encode accept tensor inputs --- pytorch_transformers/__init__.py | 9 ++---- pytorch_transformers/file_utils.py | 20 ++++++++++++ pytorch_transformers/tokenization_utils.py | 36 ++++++++++++++++------ 3 files changed, 48 insertions(+), 17 deletions(-) diff --git a/pytorch_transformers/__init__.py b/pytorch_transformers/__init__.py index 508d0f84c4..5efbece795 100644 --- a/pytorch_transformers/__init__.py +++ b/pytorch_transformers/__init__.py @@ -163,10 +163,5 @@ if _tf_available and _torch_available: # Files and general utilities from .file_utils import (PYTORCH_TRANSFORMERS_CACHE, PYTORCH_PRETRAINED_BERT_CACHE, cached_path, add_start_docstrings, add_end_docstrings, - WEIGHTS_NAME, TF2_WEIGHTS_NAME, TF_WEIGHTS_NAME, CONFIG_NAME) - -def is_torch_available(): - return _torch_available - -def is_tf_available(): - return _tf_available + WEIGHTS_NAME, TF2_WEIGHTS_NAME, TF_WEIGHTS_NAME, CONFIG_NAME, + is_tf_available, is_torch_available) \ No newline at end of file diff --git a/pytorch_transformers/file_utils.py b/pytorch_transformers/file_utils.py index 34333aaafb..2c761ef51d 100644 --- a/pytorch_transformers/file_utils.py +++ b/pytorch_transformers/file_utils.py @@ -23,6 +23,20 @@ from botocore.exceptions import ClientError import requests from tqdm import tqdm +try: + import tensorflow as tf + assert int(tf.__version__[0]) >= 2 + _tf_available = True # pylint: disable=invalid-name +except (ImportError, AssertionError): + _tf_available = False # pylint: disable=invalid-name + +try: + import torch + _torch_available = True # pylint: disable=invalid-name +except ImportError: + _torch_available = False # pylint: disable=invalid-name + + try: from torch.hub import _get_torch_home torch_cache_home = _get_torch_home() @@ -55,6 +69,12 @@ CONFIG_NAME = "config.json" logger = logging.getLogger(__name__) # pylint: disable=invalid-name +def is_torch_available(): + return _torch_available + +def is_tf_available(): + return _tf_available + if not six.PY2: def add_start_docstrings(*docstr): def docstring_decorator(fn): diff --git a/pytorch_transformers/tokenization_utils.py b/pytorch_transformers/tokenization_utils.py index 53b8d245b8..5a307c5979 100644 --- a/pytorch_transformers/tokenization_utils.py +++ b/pytorch_transformers/tokenization_utils.py @@ -23,7 +23,10 @@ import six import copy from io import open -from .file_utils import cached_path +from .file_utils import cached_path, is_tf_available + +if is_tf_available(): + import tensorflow as tf logger = logging.getLogger(__name__) @@ -686,19 +689,32 @@ class PreTrainedTokenizer(object): to their model. **kwargs: passed to the `self.tokenize()` method """ + if is_tf_available(): + is_tf_tensor = False + if isinstance(text, tf.Tensor): + text = text.numpy() + is_tf_tensor = True + if isinstance(text, bytes): + text = text.decode('utf-8') + if text_pair is None: if add_special_tokens: - return self.add_special_tokens_single_sentence(self.convert_tokens_to_ids(self.tokenize(text, **kwargs))) + output = self.add_special_tokens_single_sentence(self.convert_tokens_to_ids(self.tokenize(text, **kwargs))) else: - return self.convert_tokens_to_ids(self.tokenize(text, **kwargs)) - - first_sentence_tokens = [self._convert_token_to_id(token) for token in self.tokenize(text, **kwargs)] - second_sentence_tokens = [self._convert_token_to_id(token) for token in self.tokenize(text_pair, **kwargs)] - - if add_special_tokens: - return self.add_special_tokens_sentences_pair(first_sentence_tokens, second_sentence_tokens) + output = self.convert_tokens_to_ids(self.tokenize(text, **kwargs)) else: - return first_sentence_tokens, second_sentence_tokens + first_sentence_tokens = [self._convert_token_to_id(token) for token in self.tokenize(text, **kwargs)] + second_sentence_tokens = [self._convert_token_to_id(token) for token in self.tokenize(text_pair, **kwargs)] + + if add_special_tokens: + output = self.add_special_tokens_sentences_pair(first_sentence_tokens, second_sentence_tokens) + else: + output = first_sentence_tokens, second_sentence_tokens + + if is_tf_available() and is_tf_tensor: + output = tf.constant(output) + + return output def add_special_tokens_single_sentence(self, token_ids): logger.warning("This tokenizer does not make use of special tokens. The sequence has been returned with no modification.") From f71758f7a47c29943918fdf01a9c757a9de0524f Mon Sep 17 00:00:00 2001 From: thomwolf Date: Wed, 25 Sep 2019 12:00:50 +0200 Subject: [PATCH 132/219] update internal glue processors --- examples/run_glue.py | 22 +++-- pytorch_transformers/data/processors/glue.py | 89 ++++++++++++++----- pytorch_transformers/data/processors/utils.py | 33 ++++++- 3 files changed, 108 insertions(+), 36 deletions(-) diff --git a/examples/run_glue.py b/examples/run_glue.py index b39c6bf054..496d4e937a 100644 --- a/examples/run_glue.py +++ b/examples/run_glue.py @@ -278,10 +278,14 @@ def load_and_cache_examples(args, task, tokenizer, evaluate=False): # HACK(label indices are swapped in RoBERTa pretrained model) label_list[1], label_list[2] = label_list[2], label_list[1] examples = processor.get_dev_examples(args.data_dir) if evaluate else processor.get_train_examples(args.data_dir) - features = convert_examples_to_features(examples, label_list, args.max_seq_length, tokenizer, output_mode, - pad_on_left=bool(args.model_type in ['xlnet']), # pad on the left for xlnet - pad_token=tokenizer.convert_tokens_to_ids([tokenizer.pad_token])[0], - pad_token_segment_id=4 if args.model_type in ['xlnet'] else 0, + features = convert_examples_to_features(examples, + label_list, + args.max_seq_length, + tokenizer, + output_mode, + pad_on_left=bool(args.model_type in ['xlnet']), # pad on the left for xlnet + pad_token=tokenizer.convert_tokens_to_ids([tokenizer.pad_token])[0], + pad_token_segment_id=4 if args.model_type in ['xlnet'] else 0, ) if args.local_rank in [-1, 0]: logger.info("Saving features into cached file %s", cached_features_file) @@ -292,14 +296,14 @@ def load_and_cache_examples(args, task, tokenizer, evaluate=False): # Convert to Tensors and build dataset all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long) - all_input_mask = torch.tensor([f.input_mask for f in features], dtype=torch.long) - all_segment_ids = torch.tensor([f.segment_ids for f in features], dtype=torch.long) + all_attention_mask = torch.tensor([f.attention_mask for f in features], dtype=torch.long) + all_token_type_ids = torch.tensor([f.token_type_ids for f in features], dtype=torch.long) if output_mode == "classification": - all_label_ids = torch.tensor([f.label_id for f in features], dtype=torch.long) + all_labels = torch.tensor([f.label for f in features], dtype=torch.long) elif output_mode == "regression": - all_label_ids = torch.tensor([f.label_id for f in features], dtype=torch.float) + all_labels = torch.tensor([f.label for f in features], dtype=torch.float) - dataset = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids) + dataset = TensorDataset(all_input_ids, all_attention_mask, all_token_type_ids, all_labels) return dataset diff --git a/pytorch_transformers/data/processors/glue.py b/pytorch_transformers/data/processors/glue.py index cb89ccf6c6..3010ce9840 100644 --- a/pytorch_transformers/data/processors/glue.py +++ b/pytorch_transformers/data/processors/glue.py @@ -19,11 +19,18 @@ import logging import os from .utils import DataProcessor, InputExample, InputFeatures +from ...file_utils import is_tf_available + +if is_tf_available(): + import tensorflow as tf logger = logging.getLogger(__name__) -def glue_convert_examples_to_features(examples, label_list, max_seq_length, - tokenizer, output_mode, +def glue_convert_examples_to_features(examples, tokenizer, + max_length=512, + task=None, + label_list=None, + output_mode=None, pad_on_left=False, pad_token=0, pad_token_segment_id=0, @@ -31,46 +38,63 @@ def glue_convert_examples_to_features(examples, label_list, max_seq_length, """ Loads a data file into a list of `InputBatch`s """ + is_tf_dataset = False + if is_tf_available() and isinstance(examples, tf.data.Dataset): + is_tf_dataset = True + + if task is not None: + processor = glue_processors[task]() + if label_list is None: + label_list = processor.get_labels() + logger.info("Using label list %s for task %s" % (label_list, task)) + if output_mode is None: + output_mode = glue_output_modes[task] + logger.info("Using output mode %s for task %s" % (output_mode, task)) label_map = {label: i for i, label in enumerate(label_list)} features = [] for (ex_index, example) in enumerate(examples): if ex_index % 10000 == 0: - logger.info("Writing example %d of %d" % (ex_index, len(examples))) + logger.info("Writing example %d" % (ex_index)) + if is_tf_dataset: + example = InputExample(example['idx'].numpy(), + example['sentence1'].numpy().decode('utf-8'), + example['sentence2'].numpy().decode('utf-8'), + str(example['label'].numpy())) inputs = tokenizer.encode_plus( example.text_a, example.text_b, add_special_tokens=True, - max_length=max_seq_length, - truncate_first_sequence=True # We're truncating the first sequence as a priority + max_length=max_length, + truncate_first_sequence=True # We're truncating the first sequence in priority ) - input_ids, segment_ids = inputs["input_ids"], inputs["token_type_ids"] + input_ids, token_type_ids = inputs["input_ids"], inputs["token_type_ids"] # The mask has 1 for real tokens and 0 for padding tokens. Only real # tokens are attended to. - input_mask = [1 if mask_padding_with_zero else 0] * len(input_ids) + attention_mask = [1 if mask_padding_with_zero else 0] * len(input_ids) # Zero-pad up to the sequence length. - padding_length = max_seq_length - len(input_ids) + padding_length = max_length - len(input_ids) if pad_on_left: input_ids = ([pad_token] * padding_length) + input_ids - input_mask = ([0 if mask_padding_with_zero else 1] * padding_length) + input_mask - segment_ids = ([pad_token_segment_id] * padding_length) + segment_ids + attention_mask = ([0 if mask_padding_with_zero else 1] * padding_length) + attention_mask + token_type_ids = ([pad_token_segment_id] * padding_length) + token_type_ids else: input_ids = input_ids + ([pad_token] * padding_length) - input_mask = input_mask + ([0 if mask_padding_with_zero else 1] * padding_length) - segment_ids = segment_ids + ([pad_token_segment_id] * padding_length) + attention_mask = attention_mask + ([0 if mask_padding_with_zero else 1] * padding_length) + token_type_ids = token_type_ids + ([pad_token_segment_id] * padding_length) - assert len(input_ids) == max_seq_length - assert len(input_mask) == max_seq_length - assert len(segment_ids) == max_seq_length + assert len(input_ids) == max_length, "Error with input length {} vs {}".format(len(input_ids), max_length) + assert len(attention_mask) == max_length, "Error with input length {} vs {}".format(len(attention_mask), max_length) + assert len(token_type_ids) == max_length, "Error with input length {} vs {}".format(len(token_type_ids), max_length) if output_mode == "classification": - label_id = label_map[example.label] + label = label_map[example.label] elif output_mode == "regression": - label_id = float(example.label) + label = float(example.label) else: raise KeyError(output_mode) @@ -78,15 +102,34 @@ def glue_convert_examples_to_features(examples, label_list, max_seq_length, logger.info("*** Example ***") logger.info("guid: %s" % (example.guid)) logger.info("input_ids: %s" % " ".join([str(x) for x in input_ids])) - logger.info("input_mask: %s" % " ".join([str(x) for x in input_mask])) - logger.info("segment_ids: %s" % " ".join([str(x) for x in segment_ids])) - logger.info("label: %s (id = %d)" % (example.label, label_id)) + logger.info("attention_mask: %s" % " ".join([str(x) for x in attention_mask])) + logger.info("token_type_ids: %s" % " ".join([str(x) for x in token_type_ids])) + logger.info("label: %s (id = %d)" % (example.label, label)) features.append( InputFeatures(input_ids=input_ids, - input_mask=input_mask, - segment_ids=segment_ids, - label_id=label_id)) + attention_mask=attention_mask, + token_type_ids=token_type_ids, + label=label)) + + if is_tf_available() and is_tf_dataset: + def gen(): + for ex in features: + yield ({'input_ids': ex.input_ids, + 'attention_mask': ex.attention_mask, + 'token_type_ids': ex.token_type_ids}, + ex.label) + + return tf.data.Dataset.from_generator(gen, + ({'input_ids': tf.int32, + 'attention_mask': tf.int32, + 'token_type_ids': tf.int32}, + tf.int64), + ({'input_ids': tf.TensorShape([None]), + 'attention_mask': tf.TensorShape([None]), + 'token_type_ids': tf.TensorShape([None])}, + tf.TensorShape([]))) + return features diff --git a/pytorch_transformers/data/processors/utils.py b/pytorch_transformers/data/processors/utils.py index af90e7a47c..ed85f4a1f4 100644 --- a/pytorch_transformers/data/processors/utils.py +++ b/pytorch_transformers/data/processors/utils.py @@ -16,6 +16,7 @@ import csv import sys +import copy class InputExample(object): """A single training/test example for simple sequence classification.""" @@ -36,15 +37,39 @@ class InputExample(object): self.text_b = text_b self.label = label + def __repr__(self): + return str(self.to_json_string()) + + def to_dict(self): + """Serializes this instance to a Python dictionary.""" + output = copy.deepcopy(self.__dict__) + return output + + def to_json_string(self): + """Serializes this instance to a JSON string.""" + return json.dumps(self.to_dict(), indent=2, sort_keys=True) + "\n" + class InputFeatures(object): """A single set of features of data.""" - def __init__(self, input_ids, input_mask, segment_ids, label_id): + def __init__(self, input_ids, attention_mask, token_type_ids, label): self.input_ids = input_ids - self.input_mask = input_mask - self.segment_ids = segment_ids - self.label_id = label_id + self.attention_mask = attention_mask + self.token_type_ids = token_type_ids + self.label = label + + def __repr__(self): + return str(self.to_json_string()) + + def to_dict(self): + """Serializes this instance to a Python dictionary.""" + output = copy.deepcopy(self.__dict__) + return output + + def to_json_string(self): + """Serializes this instance to a JSON string.""" + return json.dumps(self.to_dict(), indent=2, sort_keys=True) + "\n" class DataProcessor(object): From 5def3302f455434e4aeb1aa225bacda63d9817c9 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Wed, 25 Sep 2019 12:38:08 +0200 Subject: [PATCH 133/219] update run_glue --- examples/run_glue.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/run_glue.py b/examples/run_glue.py index 496d4e937a..278f5c723a 100644 --- a/examples/run_glue.py +++ b/examples/run_glue.py @@ -279,10 +279,10 @@ def load_and_cache_examples(args, task, tokenizer, evaluate=False): label_list[1], label_list[2] = label_list[2], label_list[1] examples = processor.get_dev_examples(args.data_dir) if evaluate else processor.get_train_examples(args.data_dir) features = convert_examples_to_features(examples, - label_list, - args.max_seq_length, tokenizer, - output_mode, + label_list=label_list, + max_length=args.max_seq_length, + output_mode=output_mode, pad_on_left=bool(args.model_type in ['xlnet']), # pad on the left for xlnet pad_token=tokenizer.convert_tokens_to_ids([tokenizer.pad_token])[0], pad_token_segment_id=4 if args.model_type in ['xlnet'] else 0, From a049c8043b65ffed887e661b3b0506eb7a8c8c50 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Wed, 25 Sep 2019 17:33:16 +0200 Subject: [PATCH 134/219] push fix to training --- examples/run_glue.py | 2 +- examples/run_tf_glue.py | 69 +++++++++++++++++++ pytorch_transformers/configuration_utils.py | 1 + pytorch_transformers/data/processors/utils.py | 1 + pytorch_transformers/modeling_tf_utils.py | 2 +- pytorch_transformers/modeling_utils.py | 4 +- 6 files changed, 75 insertions(+), 4 deletions(-) create mode 100644 examples/run_tf_glue.py diff --git a/examples/run_glue.py b/examples/run_glue.py index 278f5c723a..99821f454d 100644 --- a/examples/run_glue.py +++ b/examples/run_glue.py @@ -154,8 +154,8 @@ def train(args, train_dataset, model, tokenizer): tr_loss += loss.item() if (step + 1) % args.gradient_accumulation_steps == 0: - scheduler.step() # Update learning rate schedule optimizer.step() + scheduler.step() # Update learning rate schedule model.zero_grad() global_step += 1 diff --git a/examples/run_tf_glue.py b/examples/run_tf_glue.py new file mode 100644 index 0000000000..6f59d15286 --- /dev/null +++ b/examples/run_tf_glue.py @@ -0,0 +1,69 @@ +import tensorflow as tf +import tensorflow_datasets +from pytorch_transformers import BertTokenizer, BertForSequenceClassification, TFBertForSequenceClassification, glue_convert_examples_to_features + +# Load tokenizer, model, dataset +tokenizer = BertTokenizer.from_pretrained('bert-base-cased') +tf_model = TFBertForSequenceClassification.from_pretrained('bert-base-cased') +dataset = tensorflow_datasets.load("glue/mrpc") + +# Prepare dataset for GLUE +train_dataset = glue_convert_examples_to_features(dataset['train'], tokenizer, task='mrpc', max_length=128) +valid_dataset = glue_convert_examples_to_features(dataset['validation'], tokenizer, task='mrpc', max_length=128) +train_dataset = train_dataset.shuffle(100).batch(32).repeat(3) +valid_dataset = valid_dataset.batch(64) + +# Compile tf.keras model for training +learning_rate = tf.keras.optimizers.schedules.PolynomialDecay(2e-5, 345, end_learning_rate=0) +loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) +tf_model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=learning_rate, epsilon=1e-08, clipnorm=1.0), + loss=loss, metrics=['sparse_categorical_accuracy']) + +# Train and evaluate using tf.keras.Model.fit() +tf_model.fit(train_dataset, epochs=3, steps_per_epoch=115, validation_data=valid_dataset, validation_steps=7) + +# Save the model and load it in PyTorch +tf_model.save_pretrained('./runs/') +pt_model = BertForSequenceClassification.from_pretrained('./runs/') + +# Quickly inspect a few predictions + + +# Divers +import torch + +import tensorflow as tf +import tensorflow_datasets +from pytorch_transformers import BertTokenizer, BertForSequenceClassification, TFBertForSequenceClassification, glue_convert_examples_to_features + +# Load tokenizer, model, dataset +tokenizer = BertTokenizer.from_pretrained('bert-base-cased') +model = TFBertForSequenceClassification.from_pretrained('bert-base-cased') + +pt_train_dataset = torch.load('../../data/glue_data//MRPC/cached_train_bert-base-cased_128_mrpc') + +def gen(): + for el in pt_train_dataset: + yield ((el.input_ids, el.attention_mask, el.token_type_ids), (el.label,)) + +dataset = tf.data.Dataset.from_generator(gen, + ((tf.int32, tf.int32, tf.int32), (tf.int64,)), + ((tf.TensorShape([None]), tf.TensorShape([None]), tf.TensorShape([None])), + (tf.TensorShape([]),))) + +dataset = dataset.shuffle(100).batch(32) +next(iter(dataset)) + +learning_rate = tf.keras.optimizers.schedules.PolynomialDecay(2e-5, 345, 0) +loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) +model.compile(optimizer=tf.keras.optimizers.Adam( + learning_rate=learning_rate, + epsilon=1e-08, + clipnorm=1.0), + loss=loss, + metrics=[['sparse_categorical_accuracy']]) + +tensorboard_cbk = tf.keras.callbacks.TensorBoard(log_dir='./runs/', update_freq=10, histogram_freq=1) + +# Train model +model.fit(dataset, epochs=3, callbacks=[tensorboard_cbk]) diff --git a/pytorch_transformers/configuration_utils.py b/pytorch_transformers/configuration_utils.py index fb1fe82f43..649a94e28c 100644 --- a/pytorch_transformers/configuration_utils.py +++ b/pytorch_transformers/configuration_utils.py @@ -67,6 +67,7 @@ class PretrainedConfig(object): output_config_file = os.path.join(save_directory, CONFIG_NAME) self.to_json_file(output_config_file) + logger.info("Configuration saved in {}".format(output_config_file)) @classmethod def from_pretrained(cls, pretrained_model_name_or_path, **kwargs): diff --git a/pytorch_transformers/data/processors/utils.py b/pytorch_transformers/data/processors/utils.py index ed85f4a1f4..a616372054 100644 --- a/pytorch_transformers/data/processors/utils.py +++ b/pytorch_transformers/data/processors/utils.py @@ -17,6 +17,7 @@ import csv import sys import copy +import json class InputExample(object): """A single training/test example for simple sequence classification.""" diff --git a/pytorch_transformers/modeling_tf_utils.py b/pytorch_transformers/modeling_tf_utils.py index 2186e2d488..21faee6616 100644 --- a/pytorch_transformers/modeling_tf_utils.py +++ b/pytorch_transformers/modeling_tf_utils.py @@ -132,8 +132,8 @@ class TFPreTrainedModel(tf.keras.Model): # If we save using the predefined names, we can load using `from_pretrained` output_model_file = os.path.join(save_directory, TF2_WEIGHTS_NAME) - self.save_weights(output_model_file) + logger.info("Model weights saved in {}".format(output_model_file)) @classmethod def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): diff --git a/pytorch_transformers/modeling_utils.py b/pytorch_transformers/modeling_utils.py index af33c22d6e..00e1156125 100644 --- a/pytorch_transformers/modeling_utils.py +++ b/pytorch_transformers/modeling_utils.py @@ -201,8 +201,8 @@ class PreTrainedModel(nn.Module): # If we save using the predefined names, we can load using `from_pretrained` output_model_file = os.path.join(save_directory, WEIGHTS_NAME) - torch.save(model_to_save.state_dict(), output_model_file) + logger.info("Model weights saved in {}".format(output_model_file)) @classmethod def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): @@ -305,7 +305,7 @@ class PreTrainedModel(nn.Module): archive_file = os.path.join(pretrained_model_name_or_path, WEIGHTS_NAME) else: raise EnvironmentError("Error no file named {} found in directory {}".format( - tuple(WEIGHTS_NAME, TF2_WEIGHTS_NAME, TF_WEIGHTS_NAME + ".index"), + [WEIGHTS_NAME, TF2_WEIGHTS_NAME, TF_WEIGHTS_NAME + ".index"], pretrained_model_name_or_path)) elif os.path.isfile(pretrained_model_name_or_path): archive_file = pretrained_model_name_or_path From 3b7fb48c3b963a88809263140e56108ac15225f8 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Wed, 25 Sep 2019 17:46:16 +0200 Subject: [PATCH 135/219] fix loading from tf/pt --- examples/run_tf_glue.py | 3 ++- pytorch_transformers/modeling_tf_utils.py | 4 ++-- pytorch_transformers/modeling_utils.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/examples/run_tf_glue.py b/examples/run_tf_glue.py index 6f59d15286..4328ff5170 100644 --- a/examples/run_tf_glue.py +++ b/examples/run_tf_glue.py @@ -27,7 +27,8 @@ tf_model.save_pretrained('./runs/') pt_model = BertForSequenceClassification.from_pretrained('./runs/') # Quickly inspect a few predictions - +inputs = tokenizer.encode_plus("I said the company is doing great", "The company has good results", add_special_tokens=True) +pred = pt_model(torch.tensor([tokens])) # Divers import torch diff --git a/pytorch_transformers/modeling_tf_utils.py b/pytorch_transformers/modeling_tf_utils.py index 21faee6616..e9db995b4d 100644 --- a/pytorch_transformers/modeling_tf_utils.py +++ b/pytorch_transformers/modeling_tf_utils.py @@ -224,8 +224,8 @@ class TFPreTrainedModel(tf.keras.Model): # Load from a PyTorch checkpoint archive_file = os.path.join(pretrained_model_name_or_path, WEIGHTS_NAME) else: - raise EnvironmentError("Error no file named {} found in directory {}".format( - tuple(WEIGHTS_NAME, TF2_WEIGHTS_NAME), + raise EnvironmentError("Error no file named {} found in directory {} or `from_pt` set to False".format( + [WEIGHTS_NAME, TF2_WEIGHTS_NAME], pretrained_model_name_or_path)) elif os.path.isfile(pretrained_model_name_or_path): archive_file = pretrained_model_name_or_path diff --git a/pytorch_transformers/modeling_utils.py b/pytorch_transformers/modeling_utils.py index 00e1156125..541ef7c741 100644 --- a/pytorch_transformers/modeling_utils.py +++ b/pytorch_transformers/modeling_utils.py @@ -304,7 +304,7 @@ class PreTrainedModel(nn.Module): # Load from a PyTorch checkpoint archive_file = os.path.join(pretrained_model_name_or_path, WEIGHTS_NAME) else: - raise EnvironmentError("Error no file named {} found in directory {}".format( + raise EnvironmentError("Error no file named {} found in directory {} or `from_tf` set to False".format( [WEIGHTS_NAME, TF2_WEIGHTS_NAME, TF_WEIGHTS_NAME + ".index"], pretrained_model_name_or_path)) elif os.path.isfile(pretrained_model_name_or_path): From 8a618e0af5140a91e237f7758657a78319b03a2e Mon Sep 17 00:00:00 2001 From: thomwolf Date: Wed, 25 Sep 2019 21:04:52 +0200 Subject: [PATCH 136/219] clean up __init__ --- pytorch_transformers/__init__.py | 50 +++++------ pytorch_transformers/tokenization_utils.py | 97 +++++++++++++++------- 2 files changed, 85 insertions(+), 62 deletions(-) diff --git a/pytorch_transformers/__init__.py b/pytorch_transformers/__init__.py index 68f81be085..5744537cba 100644 --- a/pytorch_transformers/__init__.py +++ b/pytorch_transformers/__init__.py @@ -16,7 +16,21 @@ import logging logger = logging.getLogger(__name__) # pylint: disable=invalid-name -# Tokenizer +# Files and general utilities +from .file_utils import (PYTORCH_TRANSFORMERS_CACHE, PYTORCH_PRETRAINED_BERT_CACHE, + cached_path, add_start_docstrings, add_end_docstrings, + WEIGHTS_NAME, TF2_WEIGHTS_NAME, TF_WEIGHTS_NAME, CONFIG_NAME, + is_tf_available, is_torch_available) + +from .data import (is_sklearn_available, + InputExample, InputFeatures, DataProcessor, + glue_output_modes, glue_convert_examples_to_features, + glue_processors, glue_tasks_num_labels) + +if is_sklearn_available(): + from .data import glue_compute_metrics + +# Tokenizers from .tokenization_utils import (PreTrainedTokenizer) from .tokenization_auto import AutoTokenizer from .tokenization_bert import BertTokenizer, BasicTokenizer, WordpieceTokenizer @@ -41,13 +55,7 @@ from .configuration_roberta import RobertaConfig, ROBERTA_PRETRAINED_CONFIG_ARCH from .configuration_distilbert import DistilBertConfig, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP # Modeling -try: - import torch - _torch_available = True # pylint: disable=invalid-name -except ImportError: - _torch_available = False # pylint: disable=invalid-name - -if _torch_available: +if is_torch_available(): logger.info("PyTorch version {} available.".format(torch.__version__)) from .modeling_utils import (PreTrainedModel, prune_layer, Conv1D) @@ -87,14 +95,7 @@ if _torch_available: # TensorFlow -try: - import tensorflow as tf - assert int(tf.__version__[0]) >= 2 - _tf_available = True # pylint: disable=invalid-name -except (ImportError, AssertionError): - _tf_available = False # pylint: disable=invalid-name - -if _tf_available: +if is_tf_available(): logger.info("TensorFlow version {} available.".format(tf.__version__)) from .modeling_tf_utils import TFPreTrainedModel, TFSharedEmbeddings, TFSequenceSummary @@ -151,7 +152,8 @@ if _tf_available: load_distilbert_pt_weights_in_tf2, TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP) -if _tf_available and _torch_available: +# TF 2.0 <=> PyTorch conversion utilities +if is_tf_available() and is_torch_available(): from .modeling_tf_pytorch_utils import (convert_tf_weight_name_to_pt_weight_name, load_pytorch_checkpoint_in_tf2_model, load_pytorch_weights_in_tf2_model, @@ -159,17 +161,3 @@ if _tf_available and _torch_available: load_tf2_checkpoint_in_pytorch_model, load_tf2_weights_in_pytorch_model, load_tf2_model_in_pytorch_model) - -# Files and general utilities -from .file_utils import (PYTORCH_TRANSFORMERS_CACHE, PYTORCH_PRETRAINED_BERT_CACHE, - cached_path, add_start_docstrings, add_end_docstrings, - WEIGHTS_NAME, TF2_WEIGHTS_NAME, TF_WEIGHTS_NAME, CONFIG_NAME, - is_tf_available, is_torch_available) - -from .data import (is_sklearn_available, - InputExample, InputFeatures, DataProcessor, - glue_output_modes, glue_convert_examples_to_features, - glue_processors, glue_tasks_num_labels) - -if is_sklearn_available(): - from .data import glue_compute_metrics diff --git a/pytorch_transformers/tokenization_utils.py b/pytorch_transformers/tokenization_utils.py index 01f7532386..f7c0e976ab 100644 --- a/pytorch_transformers/tokenization_utils.py +++ b/pytorch_transformers/tokenization_utils.py @@ -23,7 +23,7 @@ import six import copy from io import open -from .file_utils import cached_path, is_tf_available +from .file_utils import cached_path, is_tf_available, is_torch_available if is_tf_available(): import tensorflow as tf @@ -690,39 +690,20 @@ class PreTrainedTokenizer(object): def _convert_token_to_id(self, token): raise NotImplementedError - def encode(self, text, text_pair=None, add_special_tokens=False, **kwargs): + def encode(self, + text, + text_pair=None, + add_special_tokens=False, + max_length=None, + stride=0, + truncate_first_sequence=True, + return_tensors=None, + **kwargs): """ Converts a string in a sequence of ids (integer), using the tokenizer and vocabulary. Same as doing ``self.convert_tokens_to_ids(self.tokenize(text))``. - Args: - text: The first sequence to be encoded. This can be a string, a list of strings (tokenized string using - the `tokenize` method) or a list of integers (tokenized string ids using the `convert_tokens_to_ids` - method) - text_pair: Optional second sequence to be encoded. This can be a string, a list of strings (tokenized - string using the `tokenize` method) or a list of integers (tokenized string ids using the - `convert_tokens_to_ids` method) - add_special_tokens: if set to ``True``, the sequences will be encoded with the special tokens relative - to their model. - **kwargs: passed to the `self.tokenize()` method - """ - encoded_inputs = self.encode_plus(text, text_pair=text_pair, add_special_tokens=add_special_tokens, **kwargs) - - return encoded_inputs["input_ids"] - - def encode_plus(self, - text, - text_pair=None, - add_special_tokens=False, - max_length=None, - stride=0, - truncate_first_sequence=True, - **kwargs): - """ - Returns a dictionary containing the encoded sequence or sequence pair. Other values can be returned by this - method: the mask for sequence classification and the overflowing elements if a ``max_length`` is specified. - Args: text: The first sequence to be encoded. This can be a string, a list of strings (tokenized string using the `tokenize` method) or a list of integers (tokenized string ids using the `convert_tokens_to_ids` @@ -738,6 +719,51 @@ class PreTrainedTokenizer(object): from the main sequence returned. The value of this argument defined the number of additional tokens. truncate_first_sequence: if there is a specified max_length, this flag will choose which sequence will be truncated. + return_tensors: (optional) can be set to 'tf' or 'pt' to return respectively TensorFlow tf.constant + or PyTorch torch.Tensor instead of a list of python integers. + **kwargs: passed to the `self.tokenize()` method + """ + encoded_inputs = self.encode_plus(text, + text_pair=text_pair, + max_length=max_length, + add_special_tokens=add_special_tokens, + stride=stride, + truncate_first_sequence=truncate_first_sequence, + return_tensors=return_tensors, + **kwargs) + + return encoded_inputs["input_ids"] + + def encode_plus(self, + text, + text_pair=None, + add_special_tokens=False, + max_length=None, + stride=0, + truncate_first_sequence=True, + return_tensors=None, + **kwargs): + """ + Returns a dictionary containing the encoded sequence or sequence pair and additional informations: + the mask for sequence classification and the overflowing elements if a ``max_length`` is specified. + + Args: + text: The first sequence to be encoded. This can be a string, a list of strings (tokenized string using + the `tokenize` method) or a list of integers (tokenized string ids using the `convert_tokens_to_ids` + method) + text_pair: Optional second sequence to be encoded. This can be a string, a list of strings (tokenized + string using the `tokenize` method) or a list of integers (tokenized string ids using the + `convert_tokens_to_ids` method) + add_special_tokens: if set to ``True``, the sequences will be encoded with the special tokens relative + to their model. + max_length: if set to a number, will limit the total sequence returned so that it has a maximum length. + If there are overflowing tokens, those will be added to the returned dictionary + stride: if set to a number along with max_length, the overflowing tokens returned will contain some tokens + from the main sequence returned. The value of this argument defined the number of additional tokens. + truncate_first_sequence: if there is a specified max_length, this flag will choose which sequence + will be truncated. + return_tensors: (optional) can be set to 'tf' or 'pt' to return respectively TensorFlow tf.constant + or PyTorch torch.Tensor instead of a list of python integers. **kwargs: passed to the `self.tokenize()` method """ @@ -759,10 +785,12 @@ class PreTrainedTokenizer(object): max_length=max_length, add_special_tokens=add_special_tokens, stride=stride, - truncate_first_sequence=truncate_first_sequence) + truncate_first_sequence=truncate_first_sequence, + return_tensors=return_tensors) - def prepare_for_model(self, ids, pair_ids=None, max_length=None, add_special_tokens=False, stride=0, truncate_first_sequence=True): + def prepare_for_model(self, ids, pair_ids=None, max_length=None, add_special_tokens=False, stride=0, + truncate_first_sequence=True, return_tensors=None): """ Prepares a sequence of input id, or a pair of sequences of inputs ids so that it can be used by the model. It adds special tokens, truncates @@ -782,6 +810,8 @@ class PreTrainedTokenizer(object): truncate_first_sequence: if set to `True` and an optional second list of input ids is provided, alongside a specified `max_length`, will truncate the first sequence if the total size is superior than the specified `max_length`. If set to `False`, will truncate the second sequence instead. + return_tensors: (optional) can be set to 'tf' or 'pt' to return respectively TensorFlow tf.constant + or PyTorch torch.Tensor instead of a list of python integers. Return: a dictionary containing the `input_ids` as well as the `overflowing_tokens` if a `max_length` was given. @@ -816,6 +846,11 @@ class PreTrainedTokenizer(object): sequence = ids + pair_ids if pair else ids token_type_ids = [0] * len(ids) + ([1] * len(pair_ids) if pair else []) + if return_tensors == 'tf' and is_tf_available(): + sequence = tf.constant(sequence) + token_type_ids = tf.constant(token_type_ids) + elif return_tensors = 'pt' and is + encoded_inputs["input_ids"] = sequence encoded_inputs["token_type_ids"] = token_type_ids From 78863f6b36e975f718eeae01a6cea3681ff735aa Mon Sep 17 00:00:00 2001 From: thomwolf Date: Wed, 25 Sep 2019 21:09:46 +0200 Subject: [PATCH 137/219] fix tokenizer to tensors --- pytorch_transformers/tokenization_utils.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pytorch_transformers/tokenization_utils.py b/pytorch_transformers/tokenization_utils.py index f7c0e976ab..74797ea206 100644 --- a/pytorch_transformers/tokenization_utils.py +++ b/pytorch_transformers/tokenization_utils.py @@ -27,6 +27,8 @@ from .file_utils import cached_path, is_tf_available, is_torch_available if is_tf_available(): import tensorflow as tf +if is_torch_available() + import torch logger = logging.getLogger(__name__) @@ -849,7 +851,11 @@ class PreTrainedTokenizer(object): if return_tensors == 'tf' and is_tf_available(): sequence = tf.constant(sequence) token_type_ids = tf.constant(token_type_ids) - elif return_tensors = 'pt' and is + elif return_tensors == 'pt' and is_torch_available(): + sequence = torch.tensor(sequence) + token_type_ids = torch.tensor(token_type_ids) + elif return_tensors is not None: + logger.warning("Unable to convert output to tensors format {}, PyTorch or TensorFlow is not available.".format(return_tensors)) encoded_inputs["input_ids"] = sequence encoded_inputs["token_type_ids"] = token_type_ids From a6bcfb80156fac34c40a3b8dcd973c4a990d75ca Mon Sep 17 00:00:00 2001 From: thomwolf Date: Wed, 25 Sep 2019 21:14:12 +0200 Subject: [PATCH 138/219] fix tests --- pytorch_transformers/__init__.py | 4 ---- pytorch_transformers/file_utils.py | 6 ++++-- pytorch_transformers/tokenization_utils.py | 2 +- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/pytorch_transformers/__init__.py b/pytorch_transformers/__init__.py index 5744537cba..7bcb7cafdf 100644 --- a/pytorch_transformers/__init__.py +++ b/pytorch_transformers/__init__.py @@ -56,8 +56,6 @@ from .configuration_distilbert import DistilBertConfig, DISTILBERT_PRETRAINED_CO # Modeling if is_torch_available(): - logger.info("PyTorch version {} available.".format(torch.__version__)) - from .modeling_utils import (PreTrainedModel, prune_layer, Conv1D) from .modeling_auto import (AutoModel, AutoModelForSequenceClassification, AutoModelForQuestionAnswering, AutoModelWithLMHead) @@ -96,8 +94,6 @@ if is_torch_available(): # TensorFlow if is_tf_available(): - logger.info("TensorFlow version {} available.".format(tf.__version__)) - from .modeling_tf_utils import TFPreTrainedModel, TFSharedEmbeddings, TFSequenceSummary from .modeling_tf_auto import (TFAutoModel, TFAutoModelForSequenceClassification, TFAutoModelForQuestionAnswering, TFAutoModelWithLMHead) diff --git a/pytorch_transformers/file_utils.py b/pytorch_transformers/file_utils.py index 2c761ef51d..90bdb231f1 100644 --- a/pytorch_transformers/file_utils.py +++ b/pytorch_transformers/file_utils.py @@ -23,16 +23,20 @@ from botocore.exceptions import ClientError import requests from tqdm import tqdm +logger = logging.getLogger(__name__) # pylint: disable=invalid-name + try: import tensorflow as tf assert int(tf.__version__[0]) >= 2 _tf_available = True # pylint: disable=invalid-name + logger.info("TensorFlow version {} available.".format(tf.__version__)) except (ImportError, AssertionError): _tf_available = False # pylint: disable=invalid-name try: import torch _torch_available = True # pylint: disable=invalid-name + logger.info("PyTorch version {} available.".format(torch.__version__)) except ImportError: _torch_available = False # pylint: disable=invalid-name @@ -67,8 +71,6 @@ TF2_WEIGHTS_NAME = 'tf_model.h5' TF_WEIGHTS_NAME = 'model.ckpt' CONFIG_NAME = "config.json" -logger = logging.getLogger(__name__) # pylint: disable=invalid-name - def is_torch_available(): return _torch_available diff --git a/pytorch_transformers/tokenization_utils.py b/pytorch_transformers/tokenization_utils.py index 74797ea206..9a9b141412 100644 --- a/pytorch_transformers/tokenization_utils.py +++ b/pytorch_transformers/tokenization_utils.py @@ -27,7 +27,7 @@ from .file_utils import cached_path, is_tf_available, is_torch_available if is_tf_available(): import tensorflow as tf -if is_torch_available() +if is_torch_available(): import torch logger = logging.getLogger(__name__) From 2967de06f41af543b780e5c0ad912f8316a3f80e Mon Sep 17 00:00:00 2001 From: thomwolf Date: Wed, 25 Sep 2019 22:08:38 +0200 Subject: [PATCH 139/219] adding intialization to bert --- pytorch_transformers/modeling_tf_bert.py | 71 ++++++++++++++++------- pytorch_transformers/modeling_tf_utils.py | 9 +++ 2 files changed, 60 insertions(+), 20 deletions(-) diff --git a/pytorch_transformers/modeling_tf_bert.py b/pytorch_transformers/modeling_tf_bert.py index d5bbac6258..6c5059a372 100644 --- a/pytorch_transformers/modeling_tf_bert.py +++ b/pytorch_transformers/modeling_tf_bert.py @@ -28,7 +28,7 @@ import numpy as np import tensorflow as tf from .configuration_bert import BertConfig -from .modeling_tf_utils import TFPreTrainedModel +from .modeling_tf_utils import TFPreTrainedModel, get_initializer from .file_utils import add_start_docstrings from .modeling_tf_pytorch_utils import load_pytorch_checkpoint_in_tf2_model @@ -100,9 +100,16 @@ class TFBertEmbeddings(tf.keras.layers.Layer): super(TFBertEmbeddings, self).__init__(**kwargs) self.vocab_size = config.vocab_size self.hidden_size = config.hidden_size + self.initializer_range = config.initializer_range - self.position_embeddings = tf.keras.layers.Embedding(config.max_position_embeddings, config.hidden_size, name='position_embeddings') - self.token_type_embeddings = tf.keras.layers.Embedding(config.type_vocab_size, config.hidden_size, name='token_type_embeddings') + self.position_embeddings = tf.keras.layers.Embedding(config.max_position_embeddings, + config.hidden_size, + embeddings_initializer=get_initializer(self.initializer_range), + name='position_embeddings') + self.token_type_embeddings = tf.keras.layers.Embedding(config.type_vocab_size, + config.hidden_size, + embeddings_initializer=get_initializer(self.initializer_range), + name='token_type_embeddings') # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load # any TensorFlow checkpoint file @@ -117,8 +124,7 @@ class TFBertEmbeddings(tf.keras.layers.Layer): self.word_embeddings = self.add_weight( "weight", shape=[self.vocab_size, self.hidden_size], - initializer=tf.random_normal_initializer( - mean=0., stddev=self.hidden_size**-0.5)) + initializer=get_initializer(self.initializer_range)) super(TFBertEmbeddings, self).build(input_shape) def call(self, inputs, mode="embedding", training=False): @@ -192,9 +198,15 @@ class TFBertSelfAttention(tf.keras.layers.Layer): self.attention_head_size = int(config.hidden_size / config.num_attention_heads) self.all_head_size = self.num_attention_heads * self.attention_head_size - self.query = tf.keras.layers.Dense(self.all_head_size, name='query') - self.key = tf.keras.layers.Dense(self.all_head_size, name='key') - self.value = tf.keras.layers.Dense(self.all_head_size, name='value') + self.query = tf.keras.layers.Dense(self.all_head_size, + kernel_initializer=get_initializer(self.config.initializer_range), + name='query') + self.key = tf.keras.layers.Dense(self.all_head_size, + kernel_initializer=get_initializer(self.config.initializer_range), + name='key') + self.value = tf.keras.layers.Dense(self.all_head_size, + kernel_initializer=get_initializer(self.config.initializer_range), + name='value') self.dropout = tf.keras.layers.Dropout(config.attention_probs_dropout_prob) @@ -247,7 +259,9 @@ class TFBertSelfAttention(tf.keras.layers.Layer): class TFBertSelfOutput(tf.keras.layers.Layer): def __init__(self, config, **kwargs): super(TFBertSelfOutput, self).__init__(**kwargs) - self.dense = tf.keras.layers.Dense(config.hidden_size, name='dense') + self.dense = tf.keras.layers.Dense(config.hidden_size, + kernel_initializer=get_initializer(self.config.initializer_range), + name='dense') self.LayerNorm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name='LayerNorm') self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) @@ -281,7 +295,9 @@ class TFBertAttention(tf.keras.layers.Layer): class TFBertIntermediate(tf.keras.layers.Layer): def __init__(self, config, **kwargs): super(TFBertIntermediate, self).__init__(**kwargs) - self.dense = tf.keras.layers.Dense(config.intermediate_size, name='dense') + self.dense = tf.keras.layers.Dense(config.intermediate_size, + kernel_initializer=get_initializer(self.config.initializer_range), + name='dense') if isinstance(config.hidden_act, str) or (sys.version_info[0] == 2 and isinstance(config.hidden_act, unicode)): self.intermediate_act_fn = ACT2FN[config.hidden_act] else: @@ -296,7 +312,9 @@ class TFBertIntermediate(tf.keras.layers.Layer): class TFBertOutput(tf.keras.layers.Layer): def __init__(self, config, **kwargs): super(TFBertOutput, self).__init__(**kwargs) - self.dense = tf.keras.layers.Dense(config.hidden_size, name='dense') + self.dense = tf.keras.layers.Dense(config.hidden_size, + kernel_initializer=get_initializer(self.config.initializer_range), + name='dense') self.LayerNorm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name='LayerNorm') self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) @@ -364,7 +382,10 @@ class TFBertEncoder(tf.keras.layers.Layer): class TFBertPooler(tf.keras.layers.Layer): def __init__(self, config, **kwargs): super(TFBertPooler, self).__init__(**kwargs) - self.dense = tf.keras.layers.Dense(config.hidden_size, activation='tanh', name='dense') + self.dense = tf.keras.layers.Dense(config.hidden_size, + kernel_initializer=get_initializer(self.config.initializer_range), + activation='tanh', + name='dense') def call(self, hidden_states): # We "pool" the model by simply taking the hidden state corresponding @@ -377,7 +398,9 @@ class TFBertPooler(tf.keras.layers.Layer): class TFBertPredictionHeadTransform(tf.keras.layers.Layer): def __init__(self, config, **kwargs): super(TFBertPredictionHeadTransform, self).__init__(**kwargs) - self.dense = tf.keras.layers.Dense(config.hidden_size, name='dense') + self.dense = tf.keras.layers.Dense(config.hidden_size, + kernel_initializer=get_initializer(self.config.initializer_range), + name='dense') if isinstance(config.hidden_act, str) or (sys.version_info[0] == 2 and isinstance(config.hidden_act, unicode)): self.transform_act_fn = ACT2FN[config.hidden_act] else: @@ -428,7 +451,9 @@ class TFBertMLMHead(tf.keras.layers.Layer): class TFBertNSPHead(tf.keras.layers.Layer): def __init__(self, config, **kwargs): super(TFBertNSPHead, self).__init__(**kwargs) - self.seq_relationship = tf.keras.layers.Dense(2, name='seq_relationship') + self.seq_relationship = tf.keras.layers.Dense(2, + kernel_initializer=get_initializer(self.config.initializer_range), + name='seq_relationship') def call(self, pooled_output): seq_relationship_score = self.seq_relationship(pooled_output) @@ -454,8 +479,6 @@ class TFBertMainLayer(tf.keras.layers.Layer): """ raise NotImplementedError - # def call(self, input_ids, attention_mask=None, token_type_ids=None, - # position_ids=None, head_mask=None, training=False): def call(self, inputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, training=False): if isinstance(inputs, (tuple, list)): input_ids = inputs[0] @@ -819,7 +842,9 @@ class TFBertForSequenceClassification(TFBertPreTrainedModel): self.bert = TFBertMainLayer(config, name='bert') self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) - self.classifier = tf.keras.layers.Dense(config.num_labels, name='classifier') + self.classifier = tf.keras.layers.Dense(config.num_labels, + kernel_initializer=get_initializer(self.config.initializer_range), + name='classifier') def call(self, inputs, **kwargs): outputs = self.bert(inputs, **kwargs) @@ -869,7 +894,9 @@ class TFBertForMultipleChoice(TFBertPreTrainedModel): self.bert = TFBertMainLayer(config, name='bert') self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) - self.classifier = tf.keras.layers.Dense(1, name='classifier') + self.classifier = tf.keras.layers.Dense(1, + kernel_initializer=get_initializer(self.config.initializer_range), + name='classifier') def call(self, inputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, training=False): if isinstance(inputs, (tuple, list)): @@ -946,7 +973,9 @@ class TFBertForTokenClassification(TFBertPreTrainedModel): self.bert = TFBertMainLayer(config, name='bert') self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) - self.classifier = tf.keras.layers.Dense(config.num_labels, name='classifier') + self.classifier = tf.keras.layers.Dense(config.num_labels, + kernel_initializer=get_initializer(self.config.initializer_range), + name='classifier') def call(self, inputs, **kwargs): outputs = self.bert(inputs, **kwargs) @@ -996,7 +1025,9 @@ class TFBertForQuestionAnswering(TFBertPreTrainedModel): self.num_labels = config.num_labels self.bert = TFBertMainLayer(config, name='bert') - self.qa_outputs = tf.keras.layers.Dense(config.num_labels, name='qa_outputs') + self.qa_outputs = tf.keras.layers.Dense(config.num_labels, + kernel_initializer=get_initializer(self.config.initializer_range), + name='qa_outputs') def call(self, inputs, **kwargs): outputs = self.bert(inputs, **kwargs) diff --git a/pytorch_transformers/modeling_tf_utils.py b/pytorch_transformers/modeling_tf_utils.py index e9db995b4d..00e2d41a96 100644 --- a/pytorch_transformers/modeling_tf_utils.py +++ b/pytorch_transformers/modeling_tf_utils.py @@ -474,3 +474,12 @@ def shape_list(x): static = x.shape.as_list() dynamic = tf.shape(x) return [dynamic[i] if s is None else s for i, s in enumerate(static)] + +def get_initializer(initializer_range=0.02): + """Creates a `tf.initializers.truncated_normal` with the given range. + Args: + initializer_range: float, initializer range for stddev. + Returns: + TruncatedNormal initializer with stddev = `initializer_range`. + """ + return tf.keras.initializers.TruncatedNormal(stddev=initializer_range) From 4a21c4d88d13a966ebfdbd20c73a5a9a07be0a6a Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 26 Sep 2019 01:30:06 +0200 Subject: [PATCH 140/219] add warning if neither pt nor tf are found --- examples/run_tf_glue.py | 6 +++--- pytorch_transformers/__init__.py | 5 +++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/examples/run_tf_glue.py b/examples/run_tf_glue.py index 4328ff5170..e3cf1d4033 100644 --- a/examples/run_tf_glue.py +++ b/examples/run_tf_glue.py @@ -15,12 +15,12 @@ valid_dataset = valid_dataset.batch(64) # Compile tf.keras model for training learning_rate = tf.keras.optimizers.schedules.PolynomialDecay(2e-5, 345, end_learning_rate=0) +optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate, epsilon=1e-08, clipnorm=1.0) loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) -tf_model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=learning_rate, epsilon=1e-08, clipnorm=1.0), - loss=loss, metrics=['sparse_categorical_accuracy']) +tf_model.compile(optimizer=optimizer, loss=loss, metrics=['sparse_categorical_accuracy']) # Train and evaluate using tf.keras.Model.fit() -tf_model.fit(train_dataset, epochs=3, steps_per_epoch=115, validation_data=valid_dataset, validation_steps=7) +tf_model.fit(train_dataset, epochs=1, steps_per_epoch=115, validation_data=valid_dataset, validation_steps=7) # Save the model and load it in PyTorch tf_model.save_pretrained('./runs/') diff --git a/pytorch_transformers/__init__.py b/pytorch_transformers/__init__.py index 7bcb7cafdf..e355add4ad 100644 --- a/pytorch_transformers/__init__.py +++ b/pytorch_transformers/__init__.py @@ -157,3 +157,8 @@ if is_tf_available() and is_torch_available(): load_tf2_checkpoint_in_pytorch_model, load_tf2_weights_in_pytorch_model, load_tf2_model_in_pytorch_model) + +if not is_tf_available() and not is_torch_available(): + logger.warning("Neither PyTorch nor TensorFlow >= 2.0 have been found." + "Models won't be available and only tokenizers, configuration" + "and file/data utilities can be used.") From d6dde438eaf7117fa17de0c4ede3e5126989dfdf Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 26 Sep 2019 01:45:55 +0200 Subject: [PATCH 141/219] add batch dimension in encode --- examples/run_tf_glue.py | 47 ++-------------------- pytorch_transformers/tokenization_utils.py | 8 ++-- 2 files changed, 8 insertions(+), 47 deletions(-) diff --git a/examples/run_tf_glue.py b/examples/run_tf_glue.py index e3cf1d4033..8d525842f5 100644 --- a/examples/run_tf_glue.py +++ b/examples/run_tf_glue.py @@ -20,51 +20,12 @@ loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) tf_model.compile(optimizer=optimizer, loss=loss, metrics=['sparse_categorical_accuracy']) # Train and evaluate using tf.keras.Model.fit() -tf_model.fit(train_dataset, epochs=1, steps_per_epoch=115, validation_data=valid_dataset, validation_steps=7) +tf_model.fit(train_dataset, epochs=3, steps_per_epoch=115, validation_data=valid_dataset, validation_steps=7) # Save the model and load it in PyTorch tf_model.save_pretrained('./runs/') -pt_model = BertForSequenceClassification.from_pretrained('./runs/') +pt_model = BertForSequenceClassification.from_pretrained('./runs/', from_tf=True) # Quickly inspect a few predictions -inputs = tokenizer.encode_plus("I said the company is doing great", "The company has good results", add_special_tokens=True) -pred = pt_model(torch.tensor([tokens])) - -# Divers -import torch - -import tensorflow as tf -import tensorflow_datasets -from pytorch_transformers import BertTokenizer, BertForSequenceClassification, TFBertForSequenceClassification, glue_convert_examples_to_features - -# Load tokenizer, model, dataset -tokenizer = BertTokenizer.from_pretrained('bert-base-cased') -model = TFBertForSequenceClassification.from_pretrained('bert-base-cased') - -pt_train_dataset = torch.load('../../data/glue_data//MRPC/cached_train_bert-base-cased_128_mrpc') - -def gen(): - for el in pt_train_dataset: - yield ((el.input_ids, el.attention_mask, el.token_type_ids), (el.label,)) - -dataset = tf.data.Dataset.from_generator(gen, - ((tf.int32, tf.int32, tf.int32), (tf.int64,)), - ((tf.TensorShape([None]), tf.TensorShape([None]), tf.TensorShape([None])), - (tf.TensorShape([]),))) - -dataset = dataset.shuffle(100).batch(32) -next(iter(dataset)) - -learning_rate = tf.keras.optimizers.schedules.PolynomialDecay(2e-5, 345, 0) -loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) -model.compile(optimizer=tf.keras.optimizers.Adam( - learning_rate=learning_rate, - epsilon=1e-08, - clipnorm=1.0), - loss=loss, - metrics=[['sparse_categorical_accuracy']]) - -tensorboard_cbk = tf.keras.callbacks.TensorBoard(log_dir='./runs/', update_freq=10, histogram_freq=1) - -# Train model -model.fit(dataset, epochs=3, callbacks=[tensorboard_cbk]) +inputs = tokenizer.encode_plus("I said the company is doing great", "The company has good results", add_special_tokens=True, return_tensors='pt') +pred = pt_model(torch.tensor(tokens)) diff --git a/pytorch_transformers/tokenization_utils.py b/pytorch_transformers/tokenization_utils.py index 9a9b141412..ec5de3b772 100644 --- a/pytorch_transformers/tokenization_utils.py +++ b/pytorch_transformers/tokenization_utils.py @@ -849,11 +849,11 @@ class PreTrainedTokenizer(object): token_type_ids = [0] * len(ids) + ([1] * len(pair_ids) if pair else []) if return_tensors == 'tf' and is_tf_available(): - sequence = tf.constant(sequence) - token_type_ids = tf.constant(token_type_ids) + sequence = tf.constant([sequence]) + token_type_ids = tf.constant([token_type_ids]) elif return_tensors == 'pt' and is_torch_available(): - sequence = torch.tensor(sequence) - token_type_ids = torch.tensor(token_type_ids) + sequence = torch.tensor([sequence]) + token_type_ids = torch.tensor([token_type_ids]) elif return_tensors is not None: logger.warning("Unable to convert output to tensors format {}, PyTorch or TensorFlow is not available.".format(return_tensors)) From 7c9f8f93f9e199c9599827581d5c6616ebf5b3b8 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 26 Sep 2019 01:59:53 +0200 Subject: [PATCH 142/219] fix tests --- examples/run_tf_glue.py | 38 ++++++++++++++---------- pytorch_transformers/modeling_tf_bert.py | 26 ++++++++-------- 2 files changed, 35 insertions(+), 29 deletions(-) diff --git a/examples/run_tf_glue.py b/examples/run_tf_glue.py index 8d525842f5..dfbcfd5b79 100644 --- a/examples/run_tf_glue.py +++ b/examples/run_tf_glue.py @@ -1,31 +1,37 @@ import tensorflow as tf import tensorflow_datasets -from pytorch_transformers import BertTokenizer, BertForSequenceClassification, TFBertForSequenceClassification, glue_convert_examples_to_features +from transformers import * -# Load tokenizer, model, dataset +# Load dataset, tokenizer, model from pretrained model/vocabulary tokenizer = BertTokenizer.from_pretrained('bert-base-cased') -tf_model = TFBertForSequenceClassification.from_pretrained('bert-base-cased') -dataset = tensorflow_datasets.load("glue/mrpc") +dataset = tensorflow_datasets.load('glue/mrpc') +model = TFBertForSequenceClassification.from_pretrained('bert-base-cased') -# Prepare dataset for GLUE -train_dataset = glue_convert_examples_to_features(dataset['train'], tokenizer, task='mrpc', max_length=128) -valid_dataset = glue_convert_examples_to_features(dataset['validation'], tokenizer, task='mrpc', max_length=128) +# Prepare dataset for GLUE as a tf.data.Dataset instance +train_dataset = glue_convert_examples_to_features(dataset['train'], tokenizer, task='mrpc') +valid_dataset = glue_convert_examples_to_features(dataset['validation'], tokenizer, task='mrpc') train_dataset = train_dataset.shuffle(100).batch(32).repeat(3) valid_dataset = valid_dataset.batch(64) -# Compile tf.keras model for training +# Prepare training: Compile tf.keras model with optimizer, loss and learning rate schedule learning_rate = tf.keras.optimizers.schedules.PolynomialDecay(2e-5, 345, end_learning_rate=0) optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate, epsilon=1e-08, clipnorm=1.0) loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) -tf_model.compile(optimizer=optimizer, loss=loss, metrics=['sparse_categorical_accuracy']) + +model.compile(optimizer=optimizer, loss=loss, metrics=['sparse_categorical_accuracy']) # Train and evaluate using tf.keras.Model.fit() -tf_model.fit(train_dataset, epochs=3, steps_per_epoch=115, validation_data=valid_dataset, validation_steps=7) +model.fit(train_dataset, epochs=3, steps_per_epoch=115, + validation_data=valid_dataset, validation_steps=7) -# Save the model and load it in PyTorch -tf_model.save_pretrained('./runs/') -pt_model = BertForSequenceClassification.from_pretrained('./runs/', from_tf=True) +# Save the TensorFlow model and load it in PyTorch +model.save_pretrained('./save/') +pytorch_model = BertForSequenceClassification.from_pretrained('./save/', from_tf=True) -# Quickly inspect a few predictions -inputs = tokenizer.encode_plus("I said the company is doing great", "The company has good results", add_special_tokens=True, return_tensors='pt') -pred = pt_model(torch.tensor(tokens)) +# Quickly inspect a few predictions - MRPC is a paraphrasing task +inputs = tokenizer.encode_plus("The company is doing great", + "The company has good results", + add_special_tokens=True, + return_tensors='pt') +pred = pytorch_model(**inputs) +print("Paraphrase" if pred.argmax().item() == 0 else "Not paraphrase") diff --git a/pytorch_transformers/modeling_tf_bert.py b/pytorch_transformers/modeling_tf_bert.py index 6c5059a372..51d5155d4b 100644 --- a/pytorch_transformers/modeling_tf_bert.py +++ b/pytorch_transformers/modeling_tf_bert.py @@ -199,13 +199,13 @@ class TFBertSelfAttention(tf.keras.layers.Layer): self.all_head_size = self.num_attention_heads * self.attention_head_size self.query = tf.keras.layers.Dense(self.all_head_size, - kernel_initializer=get_initializer(self.config.initializer_range), + kernel_initializer=get_initializer(config.initializer_range), name='query') self.key = tf.keras.layers.Dense(self.all_head_size, - kernel_initializer=get_initializer(self.config.initializer_range), + kernel_initializer=get_initializer(config.initializer_range), name='key') self.value = tf.keras.layers.Dense(self.all_head_size, - kernel_initializer=get_initializer(self.config.initializer_range), + kernel_initializer=get_initializer(config.initializer_range), name='value') self.dropout = tf.keras.layers.Dropout(config.attention_probs_dropout_prob) @@ -260,7 +260,7 @@ class TFBertSelfOutput(tf.keras.layers.Layer): def __init__(self, config, **kwargs): super(TFBertSelfOutput, self).__init__(**kwargs) self.dense = tf.keras.layers.Dense(config.hidden_size, - kernel_initializer=get_initializer(self.config.initializer_range), + kernel_initializer=get_initializer(config.initializer_range), name='dense') self.LayerNorm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name='LayerNorm') self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) @@ -296,7 +296,7 @@ class TFBertIntermediate(tf.keras.layers.Layer): def __init__(self, config, **kwargs): super(TFBertIntermediate, self).__init__(**kwargs) self.dense = tf.keras.layers.Dense(config.intermediate_size, - kernel_initializer=get_initializer(self.config.initializer_range), + kernel_initializer=get_initializer(config.initializer_range), name='dense') if isinstance(config.hidden_act, str) or (sys.version_info[0] == 2 and isinstance(config.hidden_act, unicode)): self.intermediate_act_fn = ACT2FN[config.hidden_act] @@ -313,7 +313,7 @@ class TFBertOutput(tf.keras.layers.Layer): def __init__(self, config, **kwargs): super(TFBertOutput, self).__init__(**kwargs) self.dense = tf.keras.layers.Dense(config.hidden_size, - kernel_initializer=get_initializer(self.config.initializer_range), + kernel_initializer=get_initializer(config.initializer_range), name='dense') self.LayerNorm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name='LayerNorm') self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) @@ -383,7 +383,7 @@ class TFBertPooler(tf.keras.layers.Layer): def __init__(self, config, **kwargs): super(TFBertPooler, self).__init__(**kwargs) self.dense = tf.keras.layers.Dense(config.hidden_size, - kernel_initializer=get_initializer(self.config.initializer_range), + kernel_initializer=get_initializer(config.initializer_range), activation='tanh', name='dense') @@ -399,7 +399,7 @@ class TFBertPredictionHeadTransform(tf.keras.layers.Layer): def __init__(self, config, **kwargs): super(TFBertPredictionHeadTransform, self).__init__(**kwargs) self.dense = tf.keras.layers.Dense(config.hidden_size, - kernel_initializer=get_initializer(self.config.initializer_range), + kernel_initializer=get_initializer(config.initializer_range), name='dense') if isinstance(config.hidden_act, str) or (sys.version_info[0] == 2 and isinstance(config.hidden_act, unicode)): self.transform_act_fn = ACT2FN[config.hidden_act] @@ -452,7 +452,7 @@ class TFBertNSPHead(tf.keras.layers.Layer): def __init__(self, config, **kwargs): super(TFBertNSPHead, self).__init__(**kwargs) self.seq_relationship = tf.keras.layers.Dense(2, - kernel_initializer=get_initializer(self.config.initializer_range), + kernel_initializer=get_initializer(config.initializer_range), name='seq_relationship') def call(self, pooled_output): @@ -843,7 +843,7 @@ class TFBertForSequenceClassification(TFBertPreTrainedModel): self.bert = TFBertMainLayer(config, name='bert') self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) self.classifier = tf.keras.layers.Dense(config.num_labels, - kernel_initializer=get_initializer(self.config.initializer_range), + kernel_initializer=get_initializer(config.initializer_range), name='classifier') def call(self, inputs, **kwargs): @@ -895,7 +895,7 @@ class TFBertForMultipleChoice(TFBertPreTrainedModel): self.bert = TFBertMainLayer(config, name='bert') self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) self.classifier = tf.keras.layers.Dense(1, - kernel_initializer=get_initializer(self.config.initializer_range), + kernel_initializer=get_initializer(config.initializer_range), name='classifier') def call(self, inputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, training=False): @@ -974,7 +974,7 @@ class TFBertForTokenClassification(TFBertPreTrainedModel): self.bert = TFBertMainLayer(config, name='bert') self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) self.classifier = tf.keras.layers.Dense(config.num_labels, - kernel_initializer=get_initializer(self.config.initializer_range), + kernel_initializer=get_initializer(config.initializer_range), name='classifier') def call(self, inputs, **kwargs): @@ -1026,7 +1026,7 @@ class TFBertForQuestionAnswering(TFBertPreTrainedModel): self.bert = TFBertMainLayer(config, name='bert') self.qa_outputs = tf.keras.layers.Dense(config.num_labels, - kernel_initializer=get_initializer(self.config.initializer_range), + kernel_initializer=get_initializer(config.initializer_range), name='qa_outputs') def call(self, inputs, **kwargs): From 7a99e4b196b8c3a958f356e58b9463e108b1d9fd Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 26 Sep 2019 08:41:02 +0200 Subject: [PATCH 143/219] fix #1196 and fix #1285 --- pytorch_transformers/tokenization_gpt2.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pytorch_transformers/tokenization_gpt2.py b/pytorch_transformers/tokenization_gpt2.py index 4ebe1ad575..3d5d3029dd 100644 --- a/pytorch_transformers/tokenization_gpt2.py +++ b/pytorch_transformers/tokenization_gpt2.py @@ -173,9 +173,15 @@ class GPT2Tokenizer(PreTrainedTokenizer): self.cache[token] = word return word - def _tokenize(self, text): - """ Tokenize a string. """ - text = ' ' + text # GPT-2 (and RoBERTa) tokenizers need at least one space to begin the sentence with. + def _tokenize(self, text, add_prefix_space=False): + """ Tokenize a string. + Args: + - add_prefix_space (boolean, default False): + Begin the sentence with at least one space toto get invariance to word order in GPT-2 (and RoBERTa) tokenizers. + """ + if add_prefix_space: + text = ' ' + text + bpe_tokens = [] for token in re.findall(self.pat, text): if sys.version_info[0] == 2: From f2a337b3ed5ecc7339e0103af74f8f54b56d22ce Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 26 Sep 2019 09:02:43 +0200 Subject: [PATCH 144/219] fix tokenization tests for gpt2 roberta --- pytorch_transformers/tests/tokenization_gpt2_test.py | 4 ++-- pytorch_transformers/tests/tokenization_roberta_test.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pytorch_transformers/tests/tokenization_gpt2_test.py b/pytorch_transformers/tests/tokenization_gpt2_test.py index 8ee9cb0b54..ecc8c7da67 100644 --- a/pytorch_transformers/tests/tokenization_gpt2_test.py +++ b/pytorch_transformers/tests/tokenization_gpt2_test.py @@ -52,14 +52,14 @@ class GPT2TokenizationTest(CommonTestCases.CommonTokenizerTester): def get_input_output_texts(self): input_text = u"lower newer" - output_text = u" lower newer" + output_text = u"lower newer" return input_text, output_text def test_full_tokenizer(self): tokenizer = GPT2Tokenizer(self.vocab_file, self.merges_file, **self.special_tokens_map) text = "lower newer" bpe_tokens = ["\u0120low", "er", "\u0120", "n", "e", "w", "er"] - tokens = tokenizer.tokenize(text) + tokens = tokenizer.tokenize(text, add_prefix_space=True) self.assertListEqual(tokens, bpe_tokens) input_tokens = tokens + [tokenizer.unk_token] diff --git a/pytorch_transformers/tests/tokenization_roberta_test.py b/pytorch_transformers/tests/tokenization_roberta_test.py index ba2651c7f2..f7807d0c80 100644 --- a/pytorch_transformers/tests/tokenization_roberta_test.py +++ b/pytorch_transformers/tests/tokenization_roberta_test.py @@ -51,14 +51,14 @@ class RobertaTokenizationTest(CommonTestCases.CommonTokenizerTester): def get_input_output_texts(self): input_text = u"lower newer" - output_text = u" lower newer" + output_text = u"lower newer" return input_text, output_text def test_full_tokenizer(self): tokenizer = RobertaTokenizer(self.vocab_file, self.merges_file, **self.special_tokens_map) text = "lower newer" bpe_tokens = ["\u0120low", "er", "\u0120", "n", "e", "w", "er"] - tokens = tokenizer.tokenize(text) + tokens = tokenizer.tokenize(text, add_prefix_space=True) self.assertListEqual(tokens, bpe_tokens) input_tokens = tokens + [tokenizer.unk_token] From 5705333441f44858d9e656a8370b6c4c2921455b Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 26 Sep 2019 10:06:20 +0200 Subject: [PATCH 145/219] add initialization for everybody --- examples/run_tf_glue.py | 49 +++++++++------- .../modeling_tf_distilbert.py | 57 ++++++++++++++----- pytorch_transformers/modeling_tf_gpt2.py | 22 ++++--- pytorch_transformers/modeling_tf_openai.py | 22 ++++--- pytorch_transformers/modeling_tf_roberta.py | 15 +++-- .../modeling_tf_transfo_xl.py | 49 +++++++++++----- pytorch_transformers/modeling_tf_utils.py | 18 +++--- pytorch_transformers/modeling_tf_xlm.py | 32 +++++++---- pytorch_transformers/modeling_tf_xlnet.py | 30 ++++++---- 9 files changed, 195 insertions(+), 99 deletions(-) diff --git a/examples/run_tf_glue.py b/examples/run_tf_glue.py index dfbcfd5b79..1301d13e08 100644 --- a/examples/run_tf_glue.py +++ b/examples/run_tf_glue.py @@ -1,37 +1,48 @@ import tensorflow as tf import tensorflow_datasets -from transformers import * +from pytorch_transformers import * # Load dataset, tokenizer, model from pretrained model/vocabulary tokenizer = BertTokenizer.from_pretrained('bert-base-cased') -dataset = tensorflow_datasets.load('glue/mrpc') model = TFBertForSequenceClassification.from_pretrained('bert-base-cased') +data = tensorflow_datasets.load('glue/mrpc') # Prepare dataset for GLUE as a tf.data.Dataset instance -train_dataset = glue_convert_examples_to_features(dataset['train'], tokenizer, task='mrpc') -valid_dataset = glue_convert_examples_to_features(dataset['validation'], tokenizer, task='mrpc') -train_dataset = train_dataset.shuffle(100).batch(32).repeat(3) +train_dataset = glue_convert_examples_to_features(data['train'], tokenizer, 128, 'mrpc') +valid_dataset = glue_convert_examples_to_features(data['validation'], tokenizer, 128, 'mrpc') +train_dataset = train_dataset.shuffle(100).batch(32).repeat(2) valid_dataset = valid_dataset.batch(64) # Prepare training: Compile tf.keras model with optimizer, loss and learning rate schedule -learning_rate = tf.keras.optimizers.schedules.PolynomialDecay(2e-5, 345, end_learning_rate=0) -optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate, epsilon=1e-08, clipnorm=1.0) +optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5, epsilon=1e-08, clipnorm=1.0) loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - -model.compile(optimizer=optimizer, loss=loss, metrics=['sparse_categorical_accuracy']) +metric = tf.keras.metrics.SparseCategoricalAccuracy('accuracy') +model.compile(optimizer=optimizer, loss=loss, metrics=[metric]) # Train and evaluate using tf.keras.Model.fit() -model.fit(train_dataset, epochs=3, steps_per_epoch=115, - validation_data=valid_dataset, validation_steps=7) +history = model.fit(train_dataset, epochs=2, steps_per_epoch=115, + validation_data=valid_dataset, validation_steps=7) -# Save the TensorFlow model and load it in PyTorch +>>> Train for 115 steps, validate for 7 steps +>>> Epoch 1/2 +>>> 115/115 [==============================] - 53s 459ms/step - loss: 0.6033 - accuracy: 0.6712 - val_loss: 0.4964 - val_accuracy: 0.7647 +>>> Epoch 2/2 +>>> 115/115 [==============================] - 33s 289ms/step - loss: 0.4141 - accuracy: 0.8160 - val_loss: 0.3914 - val_accuracy: 0.8382 + +# Load the TensorFlow model in PyTorch for inspection model.save_pretrained('./save/') pytorch_model = BertForSequenceClassification.from_pretrained('./save/', from_tf=True) -# Quickly inspect a few predictions - MRPC is a paraphrasing task -inputs = tokenizer.encode_plus("The company is doing great", - "The company has good results", - add_special_tokens=True, - return_tensors='pt') -pred = pytorch_model(**inputs) -print("Paraphrase" if pred.argmax().item() == 0 else "Not paraphrase") +# Quickly test a few predictions - MRPC is a paraphrasing task, let's see if our model learned the task +sentence_0 = "This research was consistent with his findings." +sentence_1 = "His findings were compatible with this research." +sentence_2 = "His findings were not compatible with this research." +inputs_1 = tokenizer.encode_plus(sentence_0, sentence_1, add_special_tokens=True, return_tensors='pt') +inputs_2 = tokenizer.encode_plus(sentence_0, sentence_2, add_special_tokens=True, return_tensors='pt') + +pred_1 = pytorch_model(**inputs_1)[0].argmax().item() +pred_2 = pytorch_model(**inputs_2)[0].argmax().item() +print("sentence_1 is", "a paraphrase" if pred_1 else "not a paraphrase", "of sentence_0") +print("sentence_2 is", "a paraphrase" if pred_2 else "not a paraphrase", "of sentence_0") +>>> sentence_1 is a paraphrase of sentence_0 +>>> sentence_2 is not a paraphrase of sentence_0 \ No newline at end of file diff --git a/pytorch_transformers/modeling_tf_distilbert.py b/pytorch_transformers/modeling_tf_distilbert.py index 7ca8d8c815..2ec73fc43d 100644 --- a/pytorch_transformers/modeling_tf_distilbert.py +++ b/pytorch_transformers/modeling_tf_distilbert.py @@ -29,7 +29,7 @@ import numpy as np import tensorflow as tf from .configuration_distilbert import DistilBertConfig -from .modeling_tf_utils import TFPreTrainedModel, TFSharedEmbeddings, shape_list +from .modeling_tf_utils import TFPreTrainedModel, TFSharedEmbeddings, shape_list, get_initializer from .file_utils import add_start_docstrings from .modeling_tf_pytorch_utils import load_pytorch_checkpoint_in_tf2_model @@ -79,8 +79,15 @@ class TFEmbeddings(tf.keras.layers.Layer): super(TFEmbeddings, self).__init__(**kwargs) self.vocab_size = config.vocab_size self.dim = config.dim - self.word_embeddings = TFSharedEmbeddings(config.vocab_size, config.dim, name='word_embeddings') # padding_idx=0) - self.position_embeddings = tf.keras.layers.Embedding(config.max_position_embeddings, config.dim, name='position_embeddings') + self.initializer_range = config.initializer_range + self.word_embeddings = TFSharedEmbeddings(config.vocab_size, + config.dim, + initializer_range=config.initializer_range, + name='word_embeddings') # padding_idx=0) + self.position_embeddings = tf.keras.layers.Embedding(config.max_position_embeddings, + config.dim, + embeddings_initializer=get_initializer(config.initializer_range), + name='position_embeddings') if config.sinusoidal_pos_embds: raise NotImplementedError @@ -95,8 +102,7 @@ class TFEmbeddings(tf.keras.layers.Layer): self.word_embeddings = self.add_weight( "weight", shape=[self.vocab_size, self.dim], - initializer=tf.random_normal_initializer( - mean=0., stddev=self.dim**-0.5)) + initializer=get_initializer(self.initializer_range)) super(TFEmbeddings, self).build(input_shape) def call(self, inputs, mode="embedding", training=False): @@ -178,10 +184,18 @@ class TFMultiHeadSelfAttention(tf.keras.layers.Layer): assert self.dim % self.n_heads == 0 - self.q_lin = tf.keras.layers.Dense(config.dim, name="q_lin") - self.k_lin = tf.keras.layers.Dense(config.dim, name="k_lin") - self.v_lin = tf.keras.layers.Dense(config.dim, name="v_lin") - self.out_lin = tf.keras.layers.Dense(config.dim, name="out_lin") + self.q_lin = tf.keras.layers.Dense(config.dim, + kernel_initializer=get_initializer(config.initializer_range), + name="q_lin") + self.k_lin = tf.keras.layers.Dense(config.dim, + kernel_initializer=get_initializer(config.initializer_range), + name="k_lin") + self.v_lin = tf.keras.layers.Dense(config.dim, + kernel_initializer=get_initializer(config.initializer_range), + name="v_lin") + self.out_lin = tf.keras.layers.Dense(config.dim, + kernel_initializer=get_initializer(config.initializer_range), + name="out_lin") self.pruned_heads = set() @@ -254,8 +268,12 @@ class TFFFN(tf.keras.layers.Layer): def __init__(self, config, **kwargs): super(TFFFN, self).__init__(**kwargs) self.dropout = tf.keras.layers.Dropout(config.dropout) - self.lin1 = tf.keras.layers.Dense(config.hidden_dim, name="lin1") - self.lin2 = tf.keras.layers.Dense(config.dim, name="lin2") + self.lin1 = tf.keras.layers.Dense(config.hidden_dim, + kernel_initializer=get_initializer(config.initializer_range), + name="lin1") + self.lin2 = tf.keras.layers.Dense(config.dim, + kernel_initializer=get_initializer(config.initializer_range), + name="lin2") assert config.activation in ['relu', 'gelu'], "activation ({}) must be in ['relu', 'gelu']".format(config.activation) self.activation = tf.keras.layers.Activation(gelu) if config.activation=='gelu' else tf.keras.activations.relu @@ -596,7 +614,9 @@ class TFDistilBertForMaskedLM(TFDistilBertPreTrainedModel): self.vocab_size = config.vocab_size self.distilbert = TFDistilBertMainLayer(config, name="distilbert") - self.vocab_transform = tf.keras.layers.Dense(config.dim, name="vocab_transform") + self.vocab_transform = tf.keras.layers.Dense(config.dim, + kernel_initializer=get_initializer(config.initializer_range), + name="vocab_transform") self.act = tf.keras.layers.Activation(gelu) self.vocab_layer_norm = tf.keras.layers.LayerNormalization(epsilon=1e-12, name="vocab_layer_norm") self.vocab_projector = TFDistilBertLMHead(config, self.distilbert.embeddings, name="vocab_projector") @@ -647,8 +667,13 @@ class TFDistilBertForSequenceClassification(TFDistilBertPreTrainedModel): self.num_labels = config.num_labels self.distilbert = TFDistilBertMainLayer(config, name="distilbert") - self.pre_classifier = tf.keras.layers.Dense(config.dim, activation='relu', name="pre_classifier") - self.classifier = tf.keras.layers.Dense(config.num_labels, name="classifier") + self.pre_classifier = tf.keras.layers.Dense(config.dim, + kernel_initializer=get_initializer(config.initializer_range), + activation='relu', + name="pre_classifier") + self.classifier = tf.keras.layers.Dense(config.num_labels, + kernel_initializer=get_initializer(config.initializer_range), + name="classifier") self.dropout = tf.keras.layers.Dropout(config.seq_classif_dropout) def call(self, inputs, **kwargs): @@ -700,7 +725,9 @@ class TFDistilBertForQuestionAnswering(TFDistilBertPreTrainedModel): super(TFDistilBertForQuestionAnswering, self).__init__(config, *inputs, **kwargs) self.distilbert = TFDistilBertMainLayer(config, name="distilbert") - self.qa_outputs = tf.keras.layers.Dense(config.num_labels, name='qa_outputs') + self.qa_outputs = tf.keras.layers.Dense(config.num_labels, + kernel_initializer=get_initializer(config.initializer_range), + name='qa_outputs') assert config.num_labels == 2 self.dropout = tf.keras.layers.Dropout(config.qa_dropout) diff --git a/pytorch_transformers/modeling_tf_gpt2.py b/pytorch_transformers/modeling_tf_gpt2.py index cb63b95794..77d78e7a93 100644 --- a/pytorch_transformers/modeling_tf_gpt2.py +++ b/pytorch_transformers/modeling_tf_gpt2.py @@ -29,7 +29,7 @@ import numpy as np import tensorflow as tf from .modeling_tf_utils import (TFPreTrainedModel, TFConv1D, TFSharedEmbeddings, - TFSequenceSummary, shape_list) + TFSequenceSummary, shape_list, get_initializer) from .configuration_gpt2 import GPT2Config from .file_utils import add_start_docstrings from .modeling_tf_pytorch_utils import load_pytorch_checkpoint_in_tf2_model @@ -76,8 +76,8 @@ class TFAttention(tf.keras.layers.Layer): self.split_size = n_state self.scale = scale - self.c_attn = TFConv1D(n_state * 3, nx, name='c_attn') - self.c_proj = TFConv1D(n_state, nx, name='c_proj') + self.c_attn = TFConv1D(n_state * 3, nx, initializer_range=config.initializer_range, name='c_attn') + self.c_proj = TFConv1D(n_state, nx, initializer_range=config.initializer_range, name='c_proj') self.attn_dropout = tf.keras.layers.Dropout(config.attn_pdrop) self.resid_dropout = tf.keras.layers.Dropout(config.resid_pdrop) self.pruned_heads = set() @@ -166,8 +166,8 @@ class TFMLP(tf.keras.layers.Layer): def __init__(self, n_state, config, **kwargs): super(TFMLP, self).__init__(**kwargs) nx = config.n_embd - self.c_fc = TFConv1D(n_state, nx, name='c_fc') - self.c_proj = TFConv1D(nx, n_state, name='c_proj') + self.c_fc = TFConv1D(n_state, nx, initializer_range=config.initializer_range, name='c_fc') + self.c_proj = TFConv1D(nx, n_state, initializer_range=config.initializer_range, name='c_proj') self.act = gelu self.dropout = tf.keras.layers.Dropout(config.resid_pdrop) @@ -212,8 +212,14 @@ class TFGPT2MainLayer(tf.keras.layers.Layer): self.vocab_size = config.vocab_size self.n_embd = config.n_embd - self.wte = TFSharedEmbeddings(config.vocab_size, config.hidden_size, name='wte') - self.wpe = tf.keras.layers.Embedding(config.n_positions, config.n_embd, name='wpe') + self.wte = TFSharedEmbeddings(config.vocab_size, + config.hidden_size, + initializer_range=config.initializer_range, + name='wte') + self.wpe = tf.keras.layers.Embedding(config.n_positions, + config.n_embd, + embeddings_initializer=get_initializer(config.initializer_range), + name='wpe') self.drop = tf.keras.layers.Dropout(config.embd_pdrop) self.h = [TFBlock(config.n_ctx, config, @@ -557,7 +563,7 @@ class TFGPT2DoubleHeadsModel(TFGPT2PreTrainedModel): def __init__(self, config, *inputs, **kwargs): super(TFGPT2DoubleHeadsModel, self).__init__(config, *inputs, **kwargs) self.transformer = TFGPT2MainLayer(config, name='transformer') - self.multiple_choice_head = TFSequenceSummary(config, name='multiple_choice_head') + self.multiple_choice_head = TFSequenceSummary(config, initializer_range=config.initializer_range, name='multiple_choice_head') def call(self, inputs, past=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, mc_token_ids=None, training=False): if isinstance(inputs, (tuple, list)): diff --git a/pytorch_transformers/modeling_tf_openai.py b/pytorch_transformers/modeling_tf_openai.py index 18b07e0637..0704e574a4 100644 --- a/pytorch_transformers/modeling_tf_openai.py +++ b/pytorch_transformers/modeling_tf_openai.py @@ -29,7 +29,7 @@ import numpy as np import tensorflow as tf from .modeling_tf_utils import (TFPreTrainedModel, TFConv1D, TFSharedEmbeddings, - TFSequenceSummary, shape_list) + TFSequenceSummary, shape_list, get_initializer) from .configuration_openai import OpenAIGPTConfig from .file_utils import add_start_docstrings from .modeling_tf_pytorch_utils import load_pytorch_checkpoint_in_tf2_model @@ -83,8 +83,8 @@ class TFAttention(tf.keras.layers.Layer): self.split_size = n_state self.scale = scale - self.c_attn = TFConv1D(n_state * 3, nx, name='c_attn') - self.c_proj = TFConv1D(n_state, nx, name='c_proj') + self.c_attn = TFConv1D(n_state * 3, nx, initializer_range=config.initializer_range, name='c_attn') + self.c_proj = TFConv1D(n_state, nx, initializer_range=config.initializer_range, name='c_proj') self.attn_dropout = tf.keras.layers.Dropout(config.attn_pdrop) self.resid_dropout = tf.keras.layers.Dropout(config.resid_pdrop) self.pruned_heads = set() @@ -168,8 +168,8 @@ class TFMLP(tf.keras.layers.Layer): def __init__(self, n_state, config, **kwargs): super(TFMLP, self).__init__(**kwargs) nx = config.n_embd - self.c_fc = TFConv1D(n_state, nx, name='c_fc') - self.c_proj = TFConv1D(nx, n_state, name='c_proj') + self.c_fc = TFConv1D(n_state, nx, initializer_range=config.initializer_range, name='c_fc') + self.c_proj = TFConv1D(nx, n_state, initializer_range=config.initializer_range, name='c_proj') self.act = gelu self.dropout = tf.keras.layers.Dropout(config.resid_pdrop) @@ -212,8 +212,14 @@ class TFOpenAIGPTMainLayer(tf.keras.layers.Layer): self.vocab_size = config.vocab_size self.n_embd = config.n_embd - self.tokens_embed = TFSharedEmbeddings(config.vocab_size, config.n_embd, name='tokens_embed') - self.positions_embed = tf.keras.layers.Embedding(config.n_positions, config.n_embd, name='positions_embed') + self.tokens_embed = TFSharedEmbeddings(config.vocab_size, + config.n_embd, + initializer_range=config.initializer_range, + name='tokens_embed') + self.positions_embed = tf.keras.layers.Embedding(config.n_positions, + config.n_embd, + embeddings_initializer=get_initializer(config.initializer_range), + name='positions_embed') self.drop = tf.keras.layers.Dropout(config.embd_pdrop) self.h = [TFBlock(config.n_ctx, config, @@ -522,7 +528,7 @@ class TFOpenAIGPTDoubleHeadsModel(TFOpenAIGPTPreTrainedModel): def __init__(self, config, *inputs, **kwargs): super(TFOpenAIGPTDoubleHeadsModel, self).__init__(config, *inputs, **kwargs) self.transformer = TFOpenAIGPTMainLayer(config, name='transformer') - self.multiple_choice_head = TFSequenceSummary(config, name='multiple_choice_head') + self.multiple_choice_head = TFSequenceSummary(config, initializer_range=config.initializer_range, name='multiple_choice_head') def call(self, inputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, mc_token_ids=None, training=False): if isinstance(inputs, (tuple, list)): diff --git a/pytorch_transformers/modeling_tf_roberta.py b/pytorch_transformers/modeling_tf_roberta.py index 488ea566dc..862540cf10 100644 --- a/pytorch_transformers/modeling_tf_roberta.py +++ b/pytorch_transformers/modeling_tf_roberta.py @@ -24,7 +24,7 @@ import numpy as np import tensorflow as tf from .configuration_roberta import RobertaConfig -from .modeling_tf_utils import TFPreTrainedModel +from .modeling_tf_utils import TFPreTrainedModel, get_initializer from .file_utils import add_start_docstrings from .modeling_tf_pytorch_utils import load_pytorch_checkpoint_in_tf2_model @@ -232,7 +232,9 @@ class TFRobertaLMHead(tf.keras.layers.Layer): def __init__(self, config, input_embeddings, **kwargs): super(TFRobertaLMHead, self).__init__(**kwargs) self.vocab_size = config.vocab_size - self.dense = tf.keras.layers.Dense(config.hidden_size, name='dense') + self.dense = tf.keras.layers.Dense(config.hidden_size, + kernel_initializer=get_initializer(config.initializer_range), + name='dense') self.layer_norm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name='layer_norm') self.act = tf.keras.layers.Activation(gelu) @@ -315,9 +317,14 @@ class TFRobertaClassificationHead(tf.keras.layers.Layer): def __init__(self, config, **kwargs): super(TFRobertaClassificationHead, self).__init__(config, **kwargs) - self.dense = tf.keras.layers.Dense(config.hidden_size, activation='tanh', name="dense") + self.dense = tf.keras.layers.Dense(config.hidden_size, + kernel_initializer=get_initializer(config.initializer_range), + activation='tanh', + name="dense") self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) - self.out_proj = tf.keras.layers.Dense(config.num_labels, name="out_proj") + self.out_proj = tf.keras.layers.Dense(config.num_labels, + kernel_initializer=get_initializer(config.initializer_range), + name="out_proj") def call(self, features, training=False): x = features[:, 0, :] # take token (equiv. to [CLS]) diff --git a/pytorch_transformers/modeling_tf_transfo_xl.py b/pytorch_transformers/modeling_tf_transfo_xl.py index bd2854ac0a..377599d9e5 100644 --- a/pytorch_transformers/modeling_tf_transfo_xl.py +++ b/pytorch_transformers/modeling_tf_transfo_xl.py @@ -30,7 +30,7 @@ import numpy as np import tensorflow as tf from .configuration_transfo_xl import TransfoXLConfig -from .modeling_tf_utils import TFPreTrainedModel, TFConv1D, TFSequenceSummary, shape_list +from .modeling_tf_utils import TFPreTrainedModel, TFConv1D, TFSequenceSummary, shape_list, get_initializer from .modeling_tf_transfo_xl_utilities import TFAdaptiveSoftmaxMask from .file_utils import add_start_docstrings from .modeling_tf_pytorch_utils import load_pytorch_checkpoint_in_tf2_model @@ -66,16 +66,21 @@ class TFPositionalEmbedding(tf.keras.layers.Layer): class TFPositionwiseFF(tf.keras.layers.Layer): - def __init__(self, d_model, d_inner, dropout, pre_lnorm=False, layer_norm_epsilon=1e-5, **kwargs): + def __init__(self, d_model, d_inner, dropout, pre_lnorm=False, layer_norm_epsilon=1e-5, init_std=0.02, **kwargs): super(TFPositionwiseFF, self).__init__(**kwargs) self.d_model = d_model self.d_inner = d_inner self.dropout = dropout - self.layer_1 = tf.keras.layers.Dense(d_inner, activation=tf.nn.relu, name='CoreNet_._0') + self.layer_1 = tf.keras.layers.Dense(d_inner, + kernel_initializer=get_initializer(init_std), + activation=tf.nn.relu, + name='CoreNet_._0') self.drop_1 = tf.keras.layers.Dropout(dropout) - self.layer_2 = tf.keras.layers.Dense(d_model, name='CoreNet_._3') + self.layer_2 = tf.keras.layers.Dense(d_model, + kernel_initializer=get_initializer(init_std), + name='CoreNet_._3') self.drop_2 = tf.keras.layers.Dropout(dropout) self.layer_norm = tf.keras.layers.LayerNormalization(epsilon=layer_norm_epsilon, name='layer_norm') @@ -110,7 +115,7 @@ class TFRelPartialLearnableMultiHeadAttn(tf.keras.layers.Layer): def __init__(self, n_head, d_model, d_head, dropout, dropatt=0, tgt_len=None, ext_len=None, mem_len=None, pre_lnorm=False, r_r_bias=None, r_w_bias=None, output_attentions=False, - layer_norm_epsilon=1e-5, **kwargs): + layer_norm_epsilon=1e-5, init_std=0.02, **kwargs): super(TFRelPartialLearnableMultiHeadAttn, self).__init__(**kwargs) self.output_attentions = output_attentions @@ -119,11 +124,17 @@ class TFRelPartialLearnableMultiHeadAttn(tf.keras.layers.Layer): self.d_head = d_head self.dropout = dropout - self.qkv_net = tf.keras.layers.Dense(3 * n_head * d_head, use_bias=False, name='qkv_net') + self.qkv_net = tf.keras.layers.Dense(3 * n_head * d_head, + kernel_initializer=get_initializer(init_std), + use_bias=False, + name='qkv_net') self.drop = tf.keras.layers.Dropout(dropout) self.dropatt = tf.keras.layers.Dropout(dropatt) - self.o_net = tf.keras.layers.Dense(d_model, use_bias=False, name='o_net') + self.o_net = tf.keras.layers.Dense(d_model, + kernel_initializer=get_initializer(init_std), + use_bias=False, + name='o_net') self.layer_norm = tf.keras.layers.LayerNormalization(epsilon=layer_norm_epsilon, name='layer_norm') @@ -138,14 +149,19 @@ class TFRelPartialLearnableMultiHeadAttn(tf.keras.layers.Layer): self.r_r_bias = None self.r_w_bias = None - self.r_net = tf.keras.layers.Dense(self.n_head * self.d_head, use_bias=False, name='r_net') + self.r_net = tf.keras.layers.Dense(self.n_head * self.d_head, + kernel_initializer=get_initializer(init_std), + use_bias=False, + name='r_net') def build(self, input_shape): if self.r_r_bias is None or self.r_w_bias is None: # Biases are not shared self.r_r_bias = self.add_weight(shape=(self.n_head, self.d_head), + initializer='zeros', trainable=True, name='r_r_bias') self.r_w_bias = self.add_weight(shape=(self.n_head, self.d_head), + initializer='zeros', trainable=True, name='r_w_bias') super(TFRelPartialLearnableMultiHeadAttn, self).build(input_shape) @@ -249,17 +265,18 @@ class TFRelPartialLearnableDecoderLayer(tf.keras.layers.Layer): r_r_bias=None, output_attentions=False, layer_norm_epsilon=1e-5, + init_std=0.02, **kwargs): super(TFRelPartialLearnableDecoderLayer, self).__init__(**kwargs) self.dec_attn = TFRelPartialLearnableMultiHeadAttn(n_head, d_model, d_head, dropout, tgt_len=tgt_len, ext_len=ext_len, mem_len=mem_len, dropatt=dropatt, pre_lnorm=pre_lnorm, - r_w_bias=r_w_bias, r_r_bias=r_r_bias, + r_w_bias=r_w_bias, r_r_bias=r_r_bias, init_std=init_std, output_attentions=output_attentions, layer_norm_epsilon=layer_norm_epsilon, name='dec_attn') self.pos_ff = TFPositionwiseFF(d_model, d_inner, dropout, - pre_lnorm=pre_lnorm, + pre_lnorm=pre_lnorm, init_std=init_std, layer_norm_epsilon=layer_norm_epsilon, name='pos_ff') @@ -275,12 +292,13 @@ class TFRelPartialLearnableDecoderLayer(tf.keras.layers.Layer): class TFAdaptiveEmbedding(tf.keras.layers.Layer): - def __init__(self, n_token, d_embed, d_proj, cutoffs, div_val=1, + def __init__(self, n_token, d_embed, d_proj, cutoffs, div_val=1, init_std=0.02, sample_softmax=False, **kwargs): super(TFAdaptiveEmbedding, self).__init__(**kwargs) self.n_token = n_token self.d_embed = d_embed + self.init_std = init_std self.cutoffs = cutoffs + [n_token] self.div_val = div_val @@ -298,12 +316,16 @@ class TFAdaptiveEmbedding(tf.keras.layers.Layer): for i in range(len(self.cutoffs)): l_idx, r_idx = self.cutoff_ends[i], self.cutoff_ends[i+1] d_emb_i = d_embed // (div_val ** i) - self.emb_layers.append(tf.keras.layers.Embedding(r_idx-l_idx, d_emb_i, name='emb_layers_._{}'.format(i))) + self.emb_layers.append(tf.keras.layers.Embedding(r_idx-l_idx, + d_emb_i, + embeddings_initializer=get_initializer(init_std), + name='emb_layers_._{}'.format(i))) def build(self, input_shape): for i in range(len(self.cutoffs)): d_emb_i = self.d_embed // (self.div_val ** i) self.emb_projs.append(self.add_weight(shape=(d_emb_i, self.d_proj), + initializer=get_initializer(self.init_std), trainable=True, name='emb_projs_._{}'.format(i))) super(TFAdaptiveEmbedding, self).build(input_shape) @@ -349,7 +371,7 @@ class TFTransfoXLMainLayer(tf.keras.layers.Layer): self.untie_r = config.untie_r self.word_emb = TFAdaptiveEmbedding(config.n_token, config.d_embed, config.d_model, config.cutoffs, - div_val=config.div_val, name='word_emb') + div_val=config.div_val, init_std=config.init_std, name='word_emb') self.drop = tf.keras.layers.Dropout(config.dropout) @@ -374,6 +396,7 @@ class TFTransfoXLMainLayer(tf.keras.layers.Layer): r_r_bias=None if self.untie_r else self.r_r_bias, output_attentions=self.output_attentions, layer_norm_epsilon=config.layer_norm_epsilon, + init_std=config.init_std, name='layers_._{}'.format(i)) ) else: # learnable embeddings and absolute embeddings diff --git a/pytorch_transformers/modeling_tf_utils.py b/pytorch_transformers/modeling_tf_utils.py index 00e2d41a96..e341427c36 100644 --- a/pytorch_transformers/modeling_tf_utils.py +++ b/pytorch_transformers/modeling_tf_utils.py @@ -277,20 +277,20 @@ class TFPreTrainedModel(tf.keras.Model): return model class TFConv1D(tf.keras.layers.Layer): - def __init__(self, nf, nx, *inputs, **kwargs): + def __init__(self, nf, nx, *inputs, initializer_range=0.02, **kwargs): """ TFConv1D layer as defined by Radford et al. for OpenAI GPT (and also used in GPT-2) Basically works like a Linear layer but the weights are transposed """ super(TFConv1D, self).__init__(*inputs, **kwargs) self.nf = nf self.nx = nx + self.initializer_range = initializer_range def build(self, input_shape): self.weight = self.add_weight( "weight", shape=[self.nx, self.nf], - initializer=tf.random_normal_initializer( - mean=0., stddev=0.02)) + initializer=get_initializer(self.initializer_range)) self.bias = self.add_weight( "bias", shape=[1, self.nf], @@ -314,19 +314,17 @@ class TFSharedEmbeddings(tf.keras.layers.Layer): super(TFSharedEmbeddings, self).__init__(**kwargs) self.vocab_size = vocab_size self.hidden_size = hidden_size - self.initializer_range = initializer_range + self.initializer_range = hidden_size**-0.5 if initializer_range is None else initializer_range def build(self, input_shape): """Build shared word embedding layer Shared weights logic adapted from https://github.com/tensorflow/models/blob/a009f4fb9d2fc4949e32192a944688925ef78659/official/transformer/v2/embedding_layer.py#L24 """ - initializer_range = self.hidden_size**-0.5 if self.initializer_range is None else self.initializer_range self.weight = self.add_weight( "weight", shape=[self.vocab_size, self.hidden_size], - initializer=tf.random_normal_initializer( - mean=0., stddev=initializer_range)) + initializer=get_initializer(self.initializer_range)) super(TFSharedEmbeddings, self).build(input_shape) def call(self, inputs, mode="embedding"): @@ -385,7 +383,7 @@ class TFSequenceSummary(tf.keras.layers.Layer): summary_first_dropout: Add a dropout before the projection and activation summary_last_dropout: Add a dropout after the projection and activation """ - def __init__(self, config, **kwargs): + def __init__(self, config, initializer_range=0.02, **kwargs): super(TFSequenceSummary, self).__init__(**kwargs) self.summary_type = config.summary_type if hasattr(config, 'summary_use_proj') else 'last' @@ -401,7 +399,9 @@ class TFSequenceSummary(tf.keras.layers.Layer): num_classes = config.num_labels else: num_classes = config.hidden_size - self.summary = tf.keras.layers.Dense(num_classes, name='summary') + self.summary = tf.keras.layers.Dense(num_classes, + kernel_initializer=get_initializer(initializer_range), + name='summary') self.activation = None if hasattr(config, 'summary_activation') and config.summary_activation == 'tanh': diff --git a/pytorch_transformers/modeling_tf_xlm.py b/pytorch_transformers/modeling_tf_xlm.py index 1cd2f355be..cc404b6f46 100644 --- a/pytorch_transformers/modeling_tf_xlm.py +++ b/pytorch_transformers/modeling_tf_xlm.py @@ -25,7 +25,7 @@ import numpy as np import tensorflow as tf from .configuration_xlm import XLMConfig -from .modeling_tf_utils import TFPreTrainedModel, TFSharedEmbeddings, TFSequenceSummary, shape_list +from .modeling_tf_utils import TFPreTrainedModel, TFSharedEmbeddings, TFSequenceSummary, shape_list, get_initializer from .file_utils import add_start_docstrings from .modeling_tf_pytorch_utils import load_pytorch_checkpoint_in_tf2_model @@ -119,10 +119,10 @@ class TFMultiHeadAttention(tf.keras.layers.Layer): self.n_heads = n_heads assert self.dim % self.n_heads == 0 - self.q_lin = tf.keras.layers.Dense(dim, name='q_lin') - self.k_lin = tf.keras.layers.Dense(dim, name='k_lin') - self.v_lin = tf.keras.layers.Dense(dim, name='v_lin') - self.out_lin = tf.keras.layers.Dense(dim, name='out_lin') + self.q_lin = tf.keras.layers.Dense(dim, kernel_initializer=get_initializer(config.init_std), name='q_lin') + self.k_lin = tf.keras.layers.Dense(dim, kernel_initializer=get_initializer(config.init_std), name='k_lin') + self.v_lin = tf.keras.layers.Dense(dim, kernel_initializer=get_initializer(config.init_std), name='v_lin') + self.out_lin = tf.keras.layers.Dense(dim, kernel_initializer=get_initializer(config.init_std), name='out_lin') self.dropout = tf.keras.layers.Dropout(config.attention_dropout) self.pruned_heads = set() @@ -199,8 +199,8 @@ class TFTransformerFFN(tf.keras.layers.Layer): def __init__(self, in_dim, dim_hidden, out_dim, config, **kwargs): super(TFTransformerFFN, self).__init__(**kwargs) - self.lin1 = tf.keras.layers.Dense(dim_hidden, name='lin1') - self.lin2 = tf.keras.layers.Dense(out_dim, name='lin2') + self.lin1 = tf.keras.layers.Dense(dim_hidden, kernel_initializer=get_initializer(config.init_std), name='lin1') + self.lin2 = tf.keras.layers.Dense(out_dim, kernel_initializer=get_initializer(config.init_std), name='lin2') self.act = tf.keras.layers.Activation(gelu) if config.gelu_activation else tf.keras.activations.relu self.dropout = tf.keras.layers.Dropout(config.dropout) @@ -249,13 +249,19 @@ class TFXLMMainLayer(tf.keras.layers.Layer): self.dropout = tf.keras.layers.Dropout(config.dropout) self.attention_dropout = tf.keras.layers.Dropout(config.attention_dropout) - self.position_embeddings = tf.keras.layers.Embedding(config.max_position_embeddings, self.dim, name='position_embeddings') + self.position_embeddings = tf.keras.layers.Embedding(config.max_position_embeddings, + self.dim, + embeddings_initializer=get_initializer(config.embed_init_std), + name='position_embeddings') if config.sinusoidal_embeddings: raise NotImplementedError # create_sinusoidal_embeddings(config.max_position_embeddings, self.dim, out=self.position_embeddings.weight) if config.n_langs > 1 and config.use_lang_emb: - self.lang_embeddings = tf.keras.layers.Embedding(self.n_langs, self.dim, name='lang_embeddings') - self.embeddings = TFSharedEmbeddings(self.n_words, self.dim, name='embeddings') # padding_idx=self.pad_index) + self.lang_embeddings = tf.keras.layers.Embedding(self.n_langs, + self.dim, + embeddings_initializer=get_initializer(config.embed_init_std), + name='lang_embeddings') + self.embeddings = TFSharedEmbeddings(self.n_words, self.dim, initializer_range=config.embed_init_std, name='embeddings') # padding_idx=self.pad_index) self.layer_norm_emb = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name='layer_norm_emb') # transformer layers @@ -676,7 +682,7 @@ class TFXLMForSequenceClassification(TFXLMPreTrainedModel): self.num_labels = config.num_labels self.transformer = TFXLMMainLayer(config, name='transformer') - self.sequence_summary = TFSequenceSummary(config, name='sequence_summary') + self.sequence_summary = TFSequenceSummary(config, initializer_range=config.init_std, name='sequence_summary') def call(self, inputs, **kwargs): transformer_outputs = self.transformer(inputs, **kwargs) @@ -721,7 +727,9 @@ class TFXLMForQuestionAnsweringSimple(TFXLMPreTrainedModel): def __init__(self, config, *inputs, **kwargs): super(TFXLMForQuestionAnsweringSimple, self).__init__(config, *inputs, **kwargs) self.transformer = TFXLMMainLayer(config, name='transformer') - self.qa_outputs = tf.keras.layers.Dense(config.num_labels, name='qa_outputs') + self.qa_outputs = tf.keras.layers.Dense(config.num_labels, + kernel_initializer=get_initializer(config.init_std), + name='qa_outputs') def call(self, inputs, **kwargs): transformer_outputs = self.transformer(inputs, **kwargs) diff --git a/pytorch_transformers/modeling_tf_xlnet.py b/pytorch_transformers/modeling_tf_xlnet.py index a6ffb5b681..fa9a045fd8 100644 --- a/pytorch_transformers/modeling_tf_xlnet.py +++ b/pytorch_transformers/modeling_tf_xlnet.py @@ -28,7 +28,7 @@ import numpy as np import tensorflow as tf from .configuration_xlnet import XLNetConfig -from .modeling_tf_utils import TFPreTrainedModel, TFSharedEmbeddings, TFSequenceSummary, shape_list +from .modeling_tf_utils import TFPreTrainedModel, TFSharedEmbeddings, TFSequenceSummary, shape_list, get_initializer from .file_utils import add_start_docstrings from .modeling_tf_pytorch_utils import load_pytorch_checkpoint_in_tf2_model @@ -87,7 +87,7 @@ class TFXLNetRelativeAttention(tf.keras.layers.Layer): self.dropout = tf.keras.layers.Dropout(config.dropout) def build(self, input_shape): - initializer = tf.random_normal_initializer(mean=0., stddev=self.initializer_range) + initializer = get_initializer(self.initializer_range) self.q = self.add_weight(shape=(self.d_model, self.n_head, self.d_head), initializer=initializer, trainable=True, name='q') @@ -104,13 +104,13 @@ class TFXLNetRelativeAttention(tf.keras.layers.Layer): initializer=initializer, trainable=True, name='r') self.r_r_bias = self.add_weight(shape=(self.n_head, self.d_head), - initializer=initializer, + initializer='zeros', trainable=True, name='r_r_bias') self.r_s_bias = self.add_weight(shape=(self.n_head, self.d_head), - initializer=initializer, + initializer='zeros', trainable=True, name='r_s_bias') self.r_w_bias = self.add_weight(shape=(self.n_head, self.d_head), - initializer=initializer, + initializer='zeros', trainable=True, name='r_w_bias') self.seg_embed = self.add_weight(shape=(2, self.n_head, self.d_head), initializer=initializer, @@ -294,8 +294,12 @@ class TFXLNetFeedForward(tf.keras.layers.Layer): def __init__(self, config, **kwargs): super(TFXLNetFeedForward, self).__init__(**kwargs) self.layer_norm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_eps, name='layer_norm') - self.layer_1 = tf.keras.layers.Dense(config.d_inner, name='layer_1') - self.layer_2 = tf.keras.layers.Dense(config.d_model, name='layer_2') + self.layer_1 = tf.keras.layers.Dense(config.d_inner, + kernel_initializer=get_initializer(config.initializer_range), + name='layer_1') + self.layer_2 = tf.keras.layers.Dense(config.d_model, + kernel_initializer=get_initializer(config.initializer_range), + name='layer_2') self.dropout = tf.keras.layers.Dropout(config.dropout) if isinstance(config.ff_activation, str) or \ (sys.version_info[0] == 2 and isinstance(config.ff_activation, unicode)): @@ -375,7 +379,7 @@ class TFXLNetMainLayer(tf.keras.layers.Layer): self.dropout = tf.keras.layers.Dropout(config.dropout) def build(self, input_shape): - initializer = tf.random_normal_initializer(mean=0., stddev=self.initializer_range) + initializer = get_initializer(self.initializer_range) self.mask_emb = self.add_weight(shape=(1, 1, self.d_model), initializer=initializer, trainable=True, name='mask_emb') @@ -900,8 +904,10 @@ class TFXLNetForSequenceClassification(TFXLNetPreTrainedModel): self.num_labels = config.num_labels self.transformer = TFXLNetMainLayer(config, name='transformer') - self.sequence_summary = TFSequenceSummary(config, name='sequence_summary') - self.logits_proj = tf.keras.layers.Dense(config.num_labels, name='logits_proj') + self.sequence_summary = TFSequenceSummary(config, initializer_range=config.initializer_range, name='sequence_summary') + self.logits_proj = tf.keras.layers.Dense(config.num_labels, + kernel_initializer=get_initializer(config.initializer_range), + name='logits_proj') def call(self, inputs, **kwargs): transformer_outputs = self.transformer(inputs, **kwargs) @@ -949,7 +955,9 @@ class TFXLNetForQuestionAnsweringSimple(TFXLNetPreTrainedModel): def __init__(self, config, *inputs, **kwargs): super(TFXLNetForQuestionAnsweringSimple, self).__init__(config, *inputs, **kwargs) self.transformer = TFXLNetMainLayer(config, name='transformer') - self.qa_outputs = tf.keras.layers.Dense(config.num_labels, name='qa_outputs') + self.qa_outputs = tf.keras.layers.Dense(config.num_labels, + kernel_initializer=get_initializer(config.initializer_range), + name='qa_outputs') def call(self, inputs, **kwargs): transformer_outputs = self.transformer(inputs, **kwargs) From 2f071fcb022094cdf05a59332c47468c24cf5ee8 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 26 Sep 2019 10:09:45 +0200 Subject: [PATCH 146/219] clean up TFConv1D API --- pytorch_transformers/modeling_tf_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pytorch_transformers/modeling_tf_utils.py b/pytorch_transformers/modeling_tf_utils.py index e341427c36..67402abbd4 100644 --- a/pytorch_transformers/modeling_tf_utils.py +++ b/pytorch_transformers/modeling_tf_utils.py @@ -277,11 +277,11 @@ class TFPreTrainedModel(tf.keras.Model): return model class TFConv1D(tf.keras.layers.Layer): - def __init__(self, nf, nx, *inputs, initializer_range=0.02, **kwargs): + def __init__(self, nf, nx, initializer_range=0.02, **kwargs): """ TFConv1D layer as defined by Radford et al. for OpenAI GPT (and also used in GPT-2) Basically works like a Linear layer but the weights are transposed """ - super(TFConv1D, self).__init__(*inputs, **kwargs) + super(TFConv1D, self).__init__(**kwargs) self.nf = nf self.nx = nx self.initializer_range = initializer_range From 31c23bd5ee26425a67f92fc170789656379252a6 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 26 Sep 2019 10:15:53 +0200 Subject: [PATCH 147/219] [BIG] pytorch-transformers => transformers --- .circleci/config.yml | 22 ++++---- .coveragerc | 2 +- .github/ISSUE_TEMPLATE/migration.md | 2 +- README.md | 48 ++++++++--------- docker/Dockerfile | 2 +- docs/source/_static/js/custom.js | 2 +- docs/source/bertology.rst | 2 +- docs/source/conf.py | 12 ++--- docs/source/converting_tensorflow_models.rst | 14 ++--- docs/source/index.rst | 6 +-- docs/source/installation.rst | 12 ++--- docs/source/main_classes/configuration.rst | 2 +- docs/source/main_classes/model.rst | 2 +- .../main_classes/optimizer_schedules.rst | 12 ++--- docs/source/main_classes/tokenizer.rst | 2 +- docs/source/migration.md | 14 ++--- docs/source/model_doc/auto.rst | 6 +-- docs/source/model_doc/bert.rst | 20 +++---- docs/source/model_doc/distilbert.rst | 12 ++--- docs/source/model_doc/gpt.rst | 10 ++-- docs/source/model_doc/gpt2.rst | 10 ++-- docs/source/model_doc/roberta.rst | 10 ++-- docs/source/model_doc/transformerxl.rst | 8 +-- docs/source/model_doc/xlm.rst | 12 ++--- docs/source/model_doc/xlnet.rst | 12 ++--- docs/source/notebooks.rst | 8 +-- docs/source/pretrained_models.rst | 8 +-- docs/source/quickstart.md | 10 ++-- docs/source/serialization.rst | 4 +- docs/source/torchscript.rst | 4 +- examples/README.md | 8 +-- examples/contrib/run_openai_gpt.py | 2 +- examples/contrib/run_swag.py | 8 +-- examples/contrib/run_transfo_xl.py | 2 +- examples/distillation/README.md | 4 +- examples/distillation/distiller.py | 2 +- .../distillation/scripts/binarized_data.py | 2 +- .../scripts/extract_for_distil.py | 2 +- examples/distillation/train.py | 4 +- examples/run_bertology.py | 2 +- examples/run_generation.py | 10 ++-- examples/run_glue.py | 16 +++--- examples/run_lm_finetuning.py | 6 +-- examples/run_multiple_choice.py | 10 ++-- examples/run_squad.py | 8 +-- examples/run_tf_glue.py | 2 +- examples/utils_squad.py | 2 +- hubconf.py | 50 +++++++++--------- setup.py | 10 ++-- .../__init__.py | 4 +- .../__main__.py | 32 +++++------ .../configuration_auto.py | 4 +- .../configuration_bert.py | 2 +- .../configuration_distilbert.py | 0 .../configuration_gpt2.py | 0 .../configuration_openai.py | 0 .../configuration_roberta.py | 0 .../configuration_transfo_xl.py | 0 .../configuration_utils.py | 6 +-- .../configuration_xlm.py | 0 .../configuration_xlnet.py | 0 ..._bert_original_tf_checkpoint_to_pytorch.py | 2 +- ..._bert_pytorch_checkpoint_to_original_tf.py | 2 +- ..._gpt2_original_tf_checkpoint_to_pytorch.py | 2 +- ...penai_original_tf_checkpoint_to_pytorch.py | 2 +- .../convert_pytorch_checkpoint_to_tf2.py | 6 +-- ..._original_pytorch_checkpoint_to_pytorch.py | 4 +- ...fo_xl_original_tf_checkpoint_to_pytorch.py | 8 +-- ..._original_pytorch_checkpoint_to_pytorch.py | 4 +- ...xlnet_original_tf_checkpoint_to_pytorch.py | 2 +- .../data/__init__.py | 0 .../data/metrics/__init__.py | 0 .../data/processors/__init__.py | 0 .../data/processors/glue.py | 0 .../data/processors/utils.py | 0 .../file_utils.py | 9 ++-- .../modeling_auto.py | 48 ++++++++--------- .../modeling_bert.py | 10 ++-- .../modeling_distilbert.py | 4 +- .../modeling_gpt2.py | 14 ++--- .../modeling_openai.py | 10 ++-- .../modeling_roberta.py | 14 ++--- .../modeling_tf_auto.py | 48 ++++++++--------- .../modeling_tf_bert.py | 26 ++++----- .../modeling_tf_distilbert.py | 12 ++--- .../modeling_tf_gpt2.py | 16 +++--- .../modeling_tf_openai.py | 16 +++--- .../modeling_tf_pytorch_utils.py | 4 +- .../modeling_tf_roberta.py | 14 ++--- .../modeling_tf_transfo_xl.py | 14 ++--- .../modeling_tf_transfo_xl_utilities.py | 0 .../modeling_tf_utils.py | 18 +++---- .../modeling_tf_xlm.py | 18 +++---- .../modeling_tf_xlnet.py | 18 +++---- .../modeling_transfo_xl.py | 10 ++-- .../modeling_transfo_xl_utilities.py | 0 .../modeling_utils.py | 24 ++++----- .../modeling_xlm.py | 12 ++--- .../modeling_xlnet.py | 10 ++-- .../optimization.py | 0 .../tests/__init__.py | 0 .../tests/configuration_common_test.py | 0 .../tests/conftest.py | 0 .../tests/fixtures/input.txt | 0 .../tests/fixtures/sample_text.txt | 0 .../tests/fixtures/test_sentencepiece.model | Bin .../tests/modeling_auto_test.py | 6 +-- .../tests/modeling_bert_test.py | 8 +-- .../tests/modeling_common_test.py | 6 +-- .../tests/modeling_distilbert_test.py | 6 +-- .../tests/modeling_gpt2_test.py | 6 +-- .../tests/modeling_openai_test.py | 6 +-- .../tests/modeling_roberta_test.py | 8 +-- .../tests/modeling_tf_auto_test.py | 6 +-- .../tests/modeling_tf_bert_test.py | 6 +-- .../tests/modeling_tf_common_test.py | 14 ++--- .../tests/modeling_tf_distilbert_test.py | 6 +-- .../tests/modeling_tf_gpt2_test.py | 6 +-- .../tests/modeling_tf_openai_gpt_test.py | 6 +-- .../tests/modeling_tf_roberta_test.py | 6 +-- .../tests/modeling_tf_transfo_xl_test.py | 6 +-- .../tests/modeling_tf_xlm_test.py | 6 +-- .../tests/modeling_tf_xlnet_test.py | 6 +-- .../tests/modeling_transfo_xl_test.py | 8 +-- .../tests/modeling_xlm_test.py | 8 +-- .../tests/modeling_xlnet_test.py | 8 +-- .../tests/optimization_test.py | 4 +- .../tests/tokenization_auto_test.py | 4 +- .../tests/tokenization_bert_test.py | 2 +- .../tests/tokenization_distilbert_test.py | 2 +- .../tests/tokenization_gpt2_test.py | 2 +- .../tests/tokenization_openai_test.py | 2 +- .../tests/tokenization_roberta_test.py | 2 +- .../tests/tokenization_tests_commons.py | 0 .../tests/tokenization_transfo_xl_test.py | 4 +- .../tests/tokenization_utils_test.py | 4 +- .../tests/tokenization_xlm_test.py | 2 +- .../tests/tokenization_xlnet_test.py | 2 +- .../tokenization_auto.py | 6 +-- .../tokenization_bert.py | 2 +- .../tokenization_distilbert.py | 2 +- .../tokenization_gpt2.py | 0 .../tokenization_openai.py | 0 .../tokenization_roberta.py | 0 .../tokenization_transfo_xl.py | 0 .../tokenization_utils.py | 12 ++--- .../tokenization_xlm.py | 0 .../tokenization_xlnet.py | 0 148 files changed, 540 insertions(+), 539 deletions(-) rename {pytorch_transformers => transformers}/__init__.py (98%) rename {pytorch_transformers => transformers}/__main__.py (74%) rename {pytorch_transformers => transformers}/configuration_auto.py (97%) rename {pytorch_transformers => transformers}/configuration_bert.py (98%) rename {pytorch_transformers => transformers}/configuration_distilbert.py (100%) rename {pytorch_transformers => transformers}/configuration_gpt2.py (100%) rename {pytorch_transformers => transformers}/configuration_openai.py (100%) rename {pytorch_transformers => transformers}/configuration_roberta.py (100%) rename {pytorch_transformers => transformers}/configuration_transfo_xl.py (100%) rename {pytorch_transformers => transformers}/configuration_utils.py (96%) rename {pytorch_transformers => transformers}/configuration_xlm.py (100%) rename {pytorch_transformers => transformers}/configuration_xlnet.py (100%) rename {pytorch_transformers => transformers}/convert_bert_original_tf_checkpoint_to_pytorch.py (96%) rename {pytorch_transformers => transformers}/convert_bert_pytorch_checkpoint_to_original_tf.py (99%) rename {pytorch_transformers => transformers}/convert_gpt2_original_tf_checkpoint_to_pytorch.py (98%) rename {pytorch_transformers => transformers}/convert_openai_original_tf_checkpoint_to_pytorch.py (98%) rename {pytorch_transformers => transformers}/convert_pytorch_checkpoint_to_tf2.py (97%) rename {pytorch_transformers => transformers}/convert_roberta_original_pytorch_checkpoint_to_pytorch.py (98%) rename {pytorch_transformers => transformers}/convert_transfo_xl_original_tf_checkpoint_to_pytorch.py (94%) rename {pytorch_transformers => transformers}/convert_xlm_original_pytorch_checkpoint_to_pytorch.py (96%) rename {pytorch_transformers => transformers}/convert_xlnet_original_tf_checkpoint_to_pytorch.py (98%) rename {pytorch_transformers => transformers}/data/__init__.py (100%) rename {pytorch_transformers => transformers}/data/metrics/__init__.py (100%) rename {pytorch_transformers => transformers}/data/processors/__init__.py (100%) rename {pytorch_transformers => transformers}/data/processors/glue.py (100%) rename {pytorch_transformers => transformers}/data/processors/utils.py (100%) rename {pytorch_transformers => transformers}/file_utils.py (97%) rename {pytorch_transformers => transformers}/modeling_auto.py (90%) rename {pytorch_transformers => transformers}/modeling_bert.py (99%) rename {pytorch_transformers => transformers}/modeling_distilbert.py (99%) rename {pytorch_transformers => transformers}/modeling_gpt2.py (98%) rename {pytorch_transformers => transformers}/modeling_openai.py (98%) rename {pytorch_transformers => transformers}/modeling_roberta.py (97%) rename {pytorch_transformers => transformers}/modeling_tf_auto.py (89%) rename {pytorch_transformers => transformers}/modeling_tf_bert.py (97%) rename {pytorch_transformers => transformers}/modeling_tf_distilbert.py (98%) rename {pytorch_transformers => transformers}/modeling_tf_gpt2.py (97%) rename {pytorch_transformers => transformers}/modeling_tf_openai.py (97%) rename {pytorch_transformers => transformers}/modeling_tf_pytorch_utils.py (99%) rename {pytorch_transformers => transformers}/modeling_tf_roberta.py (96%) rename {pytorch_transformers => transformers}/modeling_tf_transfo_xl.py (98%) rename {pytorch_transformers => transformers}/modeling_tf_transfo_xl_utilities.py (100%) rename {pytorch_transformers => transformers}/modeling_tf_utils.py (95%) rename {pytorch_transformers => transformers}/modeling_tf_xlm.py (97%) rename {pytorch_transformers => transformers}/modeling_tf_xlnet.py (98%) rename {pytorch_transformers => transformers}/modeling_transfo_xl.py (98%) rename {pytorch_transformers => transformers}/modeling_transfo_xl_utilities.py (100%) rename {pytorch_transformers => transformers}/modeling_utils.py (96%) rename {pytorch_transformers => transformers}/modeling_xlm.py (98%) rename {pytorch_transformers => transformers}/modeling_xlnet.py (99%) rename {pytorch_transformers => transformers}/optimization.py (100%) rename {pytorch_transformers => transformers}/tests/__init__.py (100%) rename {pytorch_transformers => transformers}/tests/configuration_common_test.py (100%) rename {pytorch_transformers => transformers}/tests/conftest.py (100%) rename {pytorch_transformers => transformers}/tests/fixtures/input.txt (100%) rename {pytorch_transformers => transformers}/tests/fixtures/sample_text.txt (100%) rename {pytorch_transformers => transformers}/tests/fixtures/test_sentencepiece.model (100%) rename {pytorch_transformers => transformers}/tests/modeling_auto_test.py (95%) rename {pytorch_transformers => transformers}/tests/modeling_bert_test.py (98%) rename {pytorch_transformers => transformers}/tests/modeling_common_test.py (99%) rename {pytorch_transformers => transformers}/tests/modeling_distilbert_test.py (97%) rename {pytorch_transformers => transformers}/tests/modeling_gpt2_test.py (98%) rename {pytorch_transformers => transformers}/tests/modeling_openai_test.py (97%) rename {pytorch_transformers => transformers}/tests/modeling_roberta_test.py (96%) rename {pytorch_transformers => transformers}/tests/modeling_tf_auto_test.py (95%) rename {pytorch_transformers => transformers}/tests/modeling_tf_bert_test.py (98%) rename {pytorch_transformers => transformers}/tests/modeling_tf_common_test.py (96%) rename {pytorch_transformers => transformers}/tests/modeling_tf_distilbert_test.py (97%) rename {pytorch_transformers => transformers}/tests/modeling_tf_gpt2_test.py (98%) rename {pytorch_transformers => transformers}/tests/modeling_tf_openai_gpt_test.py (97%) rename {pytorch_transformers => transformers}/tests/modeling_tf_roberta_test.py (97%) rename {pytorch_transformers => transformers}/tests/modeling_tf_transfo_xl_test.py (97%) rename {pytorch_transformers => transformers}/tests/modeling_tf_xlm_test.py (98%) rename {pytorch_transformers => transformers}/tests/modeling_tf_xlnet_test.py (98%) rename {pytorch_transformers => transformers}/tests/modeling_transfo_xl_test.py (96%) rename {pytorch_transformers => transformers}/tests/modeling_xlm_test.py (97%) rename {pytorch_transformers => transformers}/tests/modeling_xlnet_test.py (97%) rename {pytorch_transformers => transformers}/tests/optimization_test.py (97%) rename {pytorch_transformers => transformers}/tests/tokenization_auto_test.py (88%) rename {pytorch_transformers => transformers}/tests/tokenization_bert_test.py (98%) rename {pytorch_transformers => transformers}/tests/tokenization_distilbert_test.py (95%) rename {pytorch_transformers => transformers}/tests/tokenization_gpt2_test.py (97%) rename {pytorch_transformers => transformers}/tests/tokenization_openai_test.py (96%) rename {pytorch_transformers => transformers}/tests/tokenization_roberta_test.py (97%) rename {pytorch_transformers => transformers}/tests/tokenization_tests_commons.py (100%) rename {pytorch_transformers => transformers}/tests/tokenization_transfo_xl_test.py (94%) rename {pytorch_transformers => transformers}/tests/tokenization_utils_test.py (93%) rename {pytorch_transformers => transformers}/tests/tokenization_xlm_test.py (97%) rename {pytorch_transformers => transformers}/tests/tokenization_xlnet_test.py (98%) rename {pytorch_transformers => transformers}/tokenization_auto.py (96%) rename {pytorch_transformers => transformers}/tokenization_bert.py (99%) rename {pytorch_transformers => transformers}/tokenization_distilbert.py (93%) rename {pytorch_transformers => transformers}/tokenization_gpt2.py (100%) rename {pytorch_transformers => transformers}/tokenization_openai.py (100%) rename {pytorch_transformers => transformers}/tokenization_roberta.py (100%) rename {pytorch_transformers => transformers}/tokenization_transfo_xl.py (100%) rename {pytorch_transformers => transformers}/tokenization_utils.py (98%) rename {pytorch_transformers => transformers}/tokenization_xlm.py (100%) rename {pytorch_transformers => transformers}/tokenization_xlnet.py (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 820e8951e5..1bb3fd0877 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,7 +1,7 @@ version: 2 jobs: build_py3_torch_and_tf: - working_directory: ~/pytorch-transformers + working_directory: ~/transformers docker: - image: circleci/python:3.5 resource_class: xlarge @@ -13,10 +13,10 @@ jobs: - run: sudo pip install --progress-bar off . - run: sudo pip install pytest codecov pytest-cov - run: sudo pip install tensorboardX scikit-learn - - run: python -m pytest -sv ./pytorch_transformers/tests/ --cov + - run: python -m pytest -sv ./transformers/tests/ --cov - run: codecov build_py3_torch: - working_directory: ~/pytorch-transformers + working_directory: ~/transformers docker: - image: circleci/python:3.5 resource_class: xlarge @@ -27,11 +27,11 @@ jobs: - run: sudo pip install --progress-bar off . - run: sudo pip install pytest codecov pytest-cov - run: sudo pip install tensorboardX scikit-learn - - run: python -m pytest -sv ./pytorch_transformers/tests/ --cov + - run: python -m pytest -sv ./transformers/tests/ --cov - run: python -m pytest -sv ./examples/ - run: codecov build_py3_tf: - working_directory: ~/pytorch-transformers + working_directory: ~/transformers docker: - image: circleci/python:3.5 resource_class: xlarge @@ -42,10 +42,10 @@ jobs: - run: sudo pip install --progress-bar off . - run: sudo pip install pytest codecov pytest-cov - run: sudo pip install tensorboardX scikit-learn - - run: python -m pytest -sv ./pytorch_transformers/tests/ --cov + - run: python -m pytest -sv ./transformers/tests/ --cov - run: codecov build_py2_torch: - working_directory: ~/pytorch-transformers + working_directory: ~/transformers resource_class: large parallelism: 1 docker: @@ -55,10 +55,10 @@ jobs: - run: sudo pip install torch - run: sudo pip install --progress-bar off . - run: sudo pip install pytest codecov pytest-cov - - run: python -m pytest -sv ./pytorch_transformers/tests/ --cov + - run: python -m pytest -sv ./transformers/tests/ --cov - run: codecov build_py2_tf: - working_directory: ~/pytorch-transformers + working_directory: ~/transformers resource_class: large parallelism: 1 docker: @@ -68,10 +68,10 @@ jobs: - run: sudo pip install tensorflow==2.0.0-rc0 - run: sudo pip install --progress-bar off . - run: sudo pip install pytest codecov pytest-cov - - run: python -m pytest -sv ./pytorch_transformers/tests/ --cov + - run: python -m pytest -sv ./transformers/tests/ --cov - run: codecov deploy_doc: - working_directory: ~/pytorch-transformers + working_directory: ~/transformers docker: - image: circleci/python:3.5 steps: diff --git a/.coveragerc b/.coveragerc index fa6c165a8a..9a1103b8af 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,5 +1,5 @@ [run] -source=pytorch_transformers +source=transformers omit = # skip convertion scripts from testing for now */convert_* diff --git a/.github/ISSUE_TEMPLATE/migration.md b/.github/ISSUE_TEMPLATE/migration.md index cf0c9a4757..8ce1bc8fdd 100644 --- a/.github/ISSUE_TEMPLATE/migration.md +++ b/.github/ISSUE_TEMPLATE/migration.md @@ -1,6 +1,6 @@ --- name: "\U0001F4DA Migration from PyTorch-pretrained-Bert" -about: Report a problem when migrating from PyTorch-pretrained-Bert to PyTorch-Transformers +about: Report a problem when migrating from PyTorch-pretrained-Bert to Transformers --- ## 📚 Migration diff --git a/README.md b/README.md index 9187250c19..8acdc8eb09 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ -# 👾 PyTorch-Transformers +# 🤗 Transformers -[![CircleCI](https://circleci.com/gh/huggingface/pytorch-transformers.svg?style=svg)](https://circleci.com/gh/huggingface/pytorch-transformers) +[![CircleCI](https://circleci.com/gh/huggingface/transformers.svg?style=svg)](https://circleci.com/gh/huggingface/transformers) -PyTorch-Transformers (formerly known as `pytorch-pretrained-bert`) is a library of state-of-the-art pre-trained models for Natural Language Processing (NLP). +Transformers (formerly known as `pytorch-pretrained-bert`) is a library of state-of-the-art pre-trained models for Natural Language Processing (NLP). The library currently contains PyTorch implementations, pre-trained model weights, usage scripts and conversion utilities for the following models: @@ -13,10 +13,10 @@ The library currently contains PyTorch implementations, pre-trained model weight 5. **[XLNet](https://github.com/zihangdai/xlnet/)** (from Google/CMU) released with the paper [​XLNet: Generalized Autoregressive Pretraining for Language Understanding](https://arxiv.org/abs/1906.08237) by Zhilin Yang*, Zihang Dai*, Yiming Yang, Jaime Carbonell, Ruslan Salakhutdinov, Quoc V. Le. 6. **[XLM](https://github.com/facebookresearch/XLM/)** (from Facebook) released together with the paper [Cross-lingual Language Model Pretraining](https://arxiv.org/abs/1901.07291) by Guillaume Lample and Alexis Conneau. 7. **[RoBERTa](https://github.com/pytorch/fairseq/tree/master/examples/roberta)** (from Facebook), released together with the paper a [Robustly Optimized BERT Pretraining Approach](https://arxiv.org/abs/1907.11692) by Yinhan Liu, Myle Ott, Naman Goyal, Jingfei Du, Mandar Joshi, Danqi Chen, Omer Levy, Mike Lewis, Luke Zettlemoyer, Veselin Stoyanov. -8. **[DistilBERT](https://github.com/huggingface/pytorch-transformers/tree/master/examples/distillation)** (from HuggingFace), released together with the blogpost [Smaller, faster, cheaper, lighter: Introducing DistilBERT, a distilled version of BERT](https://medium.com/huggingface/distilbert-8cf3380435b5 +8. **[DistilBERT](https://github.com/huggingface/transformers/tree/master/examples/distillation)** (from HuggingFace), released together with the blogpost [Smaller, faster, cheaper, lighter: Introducing DistilBERT, a distilled version of BERT](https://medium.com/huggingface/distilbert-8cf3380435b5 ) by Victor Sanh, Lysandre Debut and Thomas Wolf. -These implementations have been tested on several datasets (see the example scripts) and should match the performances of the original implementations (e.g. ~93 F1 on SQuAD for BERT Whole-Word-Masking, ~88 F1 on RocStories for OpenAI GPT, ~18.3 perplexity on WikiText 103 for Transformer-XL, ~0.916 Peason R coefficient on STS-B for XLNet). You can find more details on the performances in the Examples section of the [documentation](https://huggingface.co/pytorch-transformers/examples.html). +These implementations have been tested on several datasets (see the example scripts) and should match the performances of the original implementations (e.g. ~93 F1 on SQuAD for BERT Whole-Word-Masking, ~88 F1 on RocStories for OpenAI GPT, ~18.3 perplexity on WikiText 103 for Transformer-XL, ~0.916 Peason R coefficient on STS-B for XLNet). You can find more details on the performances in the Examples section of the [documentation](https://huggingface.co/transformers/examples.html). | Section | Description | |-|-| @@ -24,8 +24,8 @@ These implementations have been tested on several datasets (see the example scri | [Online demo](#online-demo) | Experimenting with this repo’s text generation capabilities | | [Quick tour: Usage](#quick-tour) | Tokenizers & models usage: Bert and GPT-2 | | [Quick tour: Fine-tuning/usage scripts](#quick-tour-of-the-fine-tuningusage-scripts) | Using provided scripts: GLUE, SQuAD and Text generation | -| [Migrating from pytorch-pretrained-bert to pytorch-transformers](#Migrating-from-pytorch-pretrained-bert-to-pytorch-transformers) | Migrating your code from pytorch-pretrained-bert to pytorch-transformers | -| [Documentation](https://huggingface.co/pytorch-transformers/) | Full API documentation and more | +| [Migrating from pytorch-pretrained-bert to transformers](#Migrating-from-pytorch-pretrained-bert-to-transformers) | Migrating your code from pytorch-pretrained-bert to transformers | +| [Documentation](https://huggingface.co/transformers/) | Full API documentation and more | ## Installation @@ -33,10 +33,10 @@ This repo is tested on Python 2.7 and 3.5+ (examples are tested only on python 3 ### With pip -PyTorch-Transformers can be installed by pip as follows: +Transformers can be installed by pip as follows: ```bash -pip install pytorch-transformers +pip install transformers ``` ### From source @@ -49,14 +49,14 @@ pip install [--editable] . ### Tests -A series of tests is included for the library and the example scripts. Library tests can be found in the [tests folder](https://github.com/huggingface/pytorch-transformers/tree/master/pytorch_transformers/tests) and examples tests in the [examples folder](https://github.com/huggingface/pytorch-transformers/tree/master/examples). +A series of tests is included for the library and the example scripts. Library tests can be found in the [tests folder](https://github.com/huggingface/transformers/tree/master/transformers/tests) and examples tests in the [examples folder](https://github.com/huggingface/transformers/tree/master/examples). These tests can be run using `pytest` (install pytest if needed with `pip install pytest`). You can run the tests from the root of the cloned repository with the commands: ```bash -python -m pytest -sv ./pytorch_transformers/tests/ +python -m pytest -sv ./transformers/tests/ python -m pytest -sv ./examples/ ``` @@ -80,13 +80,13 @@ You can use it to experiment with completions generated by `GPT2Model`, `Transfo ## Quick tour -Let's do a very quick overview of PyTorch-Transformers. Detailed examples for each model architecture (Bert, GPT, GPT-2, Transformer-XL, XLNet and XLM) can be found in the [full documentation](https://huggingface.co/pytorch-transformers/). +Let's do a very quick overview of Transformers. Detailed examples for each model architecture (Bert, GPT, GPT-2, Transformer-XL, XLNet and XLM) can be found in the [full documentation](https://huggingface.co/transformers/). ```python import torch -from pytorch_transformers import * +from transformers import * -# PyTorch-Transformers has a unified API +# Transformers has a unified API # for 7 transformer architectures and 30 pretrained weights. # Model | Tokenizer | Pretrained weights shortcut MODELS = [(BertModel, BertTokenizer, 'bert-base-uncased'), @@ -299,19 +299,19 @@ python ./examples/run_generation.py \ --model_name_or_path=gpt2 \ ``` -## Migrating from pytorch-pretrained-bert to pytorch-transformers +## Migrating from pytorch-pretrained-bert to transformers -Here is a quick summary of what you should take care of when migrating from `pytorch-pretrained-bert` to `pytorch-transformers` +Here is a quick summary of what you should take care of when migrating from `pytorch-pretrained-bert` to `transformers` ### Models always output `tuples` -The main breaking change when migrating from `pytorch-pretrained-bert` to `pytorch-transformers` is that the models forward method always outputs a `tuple` with various elements depending on the model and the configuration parameters. +The main breaking change when migrating from `pytorch-pretrained-bert` to `transformers` is that the models forward method always outputs a `tuple` with various elements depending on the model and the configuration parameters. -The exact content of the tuples for each model are detailed in the models' docstrings and the [documentation](https://huggingface.co/pytorch-transformers/). +The exact content of the tuples for each model are detailed in the models' docstrings and the [documentation](https://huggingface.co/transformers/). In pretty much every case, you will be fine by taking the first element of the output as the output you previously used in `pytorch-pretrained-bert`. -Here is a `pytorch-pretrained-bert` to `pytorch-transformers` conversion example for a `BertForSequenceClassification` classification model: +Here is a `pytorch-pretrained-bert` to `transformers` conversion example for a `BertForSequenceClassification` classification model: ```python # Let's load our model @@ -320,11 +320,11 @@ model = BertForSequenceClassification.from_pretrained('bert-base-uncased') # If you used to have this line in pytorch-pretrained-bert: loss = model(input_ids, labels=labels) -# Now just use this line in pytorch-transformers to extract the loss from the output tuple: +# Now just use this line in transformers to extract the loss from the output tuple: outputs = model(input_ids, labels=labels) loss = outputs[0] -# In pytorch-transformers you can also have access to the logits: +# In transformers you can also have access to the logits: loss, logits = outputs[:2] # And even the attention weights if you configure the model to output them (and other outputs too, see the docstrings and documentation) @@ -339,7 +339,7 @@ Breaking change in the `from_pretrained()`method: 1. Models are now set in evaluation mode by default when instantiated with the `from_pretrained()` method. To train them don't forget to set them back in training mode (`model.train()`) to activate the dropout modules. -2. The additional `*input` and `**kwargs` arguments supplied to the `from_pretrained()` method used to be directly passed to the underlying model's class `__init__()` method. They are now used to update the model configuration attribute instead which can break derived model classes build based on the previous `BertForSequenceClassification` examples. We are working on a way to mitigate this breaking change in [#866](https://github.com/huggingface/pytorch-transformers/pull/866) by forwarding the the model `__init__()` method (i) the provided positional arguments and (ii) the keyword arguments which do not match any configuration class attributes. +2. The additional `*input` and `**kwargs` arguments supplied to the `from_pretrained()` method used to be directly passed to the underlying model's class `__init__()` method. They are now used to update the model configuration attribute instead which can break derived model classes build based on the previous `BertForSequenceClassification` examples. We are working on a way to mitigate this breaking change in [#866](https://github.com/huggingface/transformers/pull/866) by forwarding the the model `__init__()` method (i) the provided positional arguments and (ii) the keyword arguments which do not match any configuration class attributes. Also, while not a breaking change, the serialization methods have been standardized and you probably should switch to the new method `save_pretrained(save_directory)` if you were using any other serialization method before. @@ -396,7 +396,7 @@ for batch in train_data: loss.backward() optimizer.step() -### In PyTorch-Transformers, optimizer and schedules are splitted and instantiated like this: +### In Transformers, optimizer and schedules are splitted and instantiated like this: optimizer = AdamW(model.parameters(), lr=lr, correct_bias=False) # To reproduce BertAdam specific behavior set correct_bias=False scheduler = WarmupLinearSchedule(optimizer, warmup_steps=num_warmup_steps, t_total=num_total_steps) # PyTorch scheduler ### and used like this: @@ -411,4 +411,4 @@ for batch in train_data: ## Citation -At the moment, there is no paper associated to PyTorch-Transformers but we are working on preparing one. In the meantime, please include a mention of the library and a link to the present repository if you use this work in a published or open-source project. +At the moment, there is no paper associated to Transformers but we are working on preparing one. In the meantime, please include a mention of the library and a link to the present repository if you use this work in a published or open-source project. diff --git a/docker/Dockerfile b/docker/Dockerfile index 1a6c6f06f9..fed834ff88 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -2,6 +2,6 @@ FROM pytorch/pytorch:latest RUN git clone https://github.com/NVIDIA/apex.git && cd apex && python setup.py install --cuda_ext --cpp_ext -RUN pip install pytorch_transformers +RUN pip install transformers WORKDIR /workspace \ No newline at end of file diff --git a/docs/source/_static/js/custom.js b/docs/source/_static/js/custom.js index 4adf2a4672..1d5827c0fa 100644 --- a/docs/source/_static/js/custom.js +++ b/docs/source/_static/js/custom.js @@ -16,7 +16,7 @@ function addIcon() { function addCustomFooter() { const customFooter = document.createElement("div"); const questionOrIssue = document.createElement("div"); - questionOrIssue.innerHTML = "Stuck? Read our Blog posts or Create an issue"; + questionOrIssue.innerHTML = "Stuck? Read our Blog posts or Create an issue"; customFooter.appendChild(questionOrIssue); customFooter.classList.add("footer"); diff --git a/docs/source/bertology.rst b/docs/source/bertology.rst index 0ea5b53d80..c3d1b2f8b8 100644 --- a/docs/source/bertology.rst +++ b/docs/source/bertology.rst @@ -15,4 +15,4 @@ In order to help this new field develop, we have included a few additional featu * accessing all the attention weights for each head of BERT/GPT/GPT-2, * retrieving heads output values and gradients to be able to compute head importance score and prune head as explained in https://arxiv.org/abs/1905.10650. -To help you understand and use these features, we have added a specific example script: `bertology.py `_ while extract information and prune a model pre-trained on GLUE. +To help you understand and use these features, we have added a specific example script: `bertology.py `_ while extract information and prune a model pre-trained on GLUE. diff --git a/docs/source/conf.py b/docs/source/conf.py index c847dee806..ae9cc67e2c 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -19,7 +19,7 @@ sys.path.insert(0, os.path.abspath('../..')) # -- Project information ----------------------------------------------------- -project = u'pytorch-transformers' +project = u'transformers' copyright = u'2019, huggingface' author = u'huggingface' @@ -109,7 +109,7 @@ html_static_path = ['_static'] # -- Options for HTMLHelp output --------------------------------------------- # Output file base name for HTML help builder. -htmlhelp_basename = 'pytorch-transformersdoc' +htmlhelp_basename = 'transformersdoc' # -- Options for LaTeX output ------------------------------------------------ @@ -136,7 +136,7 @@ latex_elements = { # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'pytorch-transformers.tex', u'pytorch-transformers Documentation', + (master_doc, 'transformers.tex', u'transformers Documentation', u'huggingface', 'manual'), ] @@ -146,7 +146,7 @@ latex_documents = [ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - (master_doc, 'pytorch-transformers', u'pytorch-transformers Documentation', + (master_doc, 'transformers', u'transformers Documentation', [author], 1) ] @@ -157,8 +157,8 @@ man_pages = [ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'pytorch-transformers', u'pytorch-transformers Documentation', - author, 'pytorch-transformers', 'One line description of project.', + (master_doc, 'transformers', u'transformers Documentation', + author, 'transformers', 'One line description of project.', 'Miscellaneous'), ] diff --git a/docs/source/converting_tensorflow_models.rst b/docs/source/converting_tensorflow_models.rst index 8441c9b1f7..8d805b0ed1 100644 --- a/docs/source/converting_tensorflow_models.rst +++ b/docs/source/converting_tensorflow_models.rst @@ -6,7 +6,7 @@ A command-line interface is provided to convert original Bert/GPT/GPT-2/Transfor BERT ^^^^ -You can convert any TensorFlow checkpoint for BERT (in particular `the pre-trained models released by Google `_\ ) in a PyTorch save file by using the `convert_tf_checkpoint_to_pytorch.py `_ script. +You can convert any TensorFlow checkpoint for BERT (in particular `the pre-trained models released by Google `_\ ) in a PyTorch save file by using the `convert_tf_checkpoint_to_pytorch.py `_ script. This CLI takes as input a TensorFlow checkpoint (three files starting with ``bert_model.ckpt``\ ) and the associated configuration file (\ ``bert_config.json``\ ), and creates a PyTorch model for this configuration, loads the weights from the TensorFlow checkpoint in the PyTorch model and saves the resulting model in a standard PyTorch save file that can be imported using ``torch.load()`` (see examples in `run_bert_extract_features.py `_\ , `run_bert_classifier.py `_ and `run_bert_squad.py `_\ ). @@ -20,7 +20,7 @@ Here is an example of the conversion process for a pre-trained ``BERT-Base Uncas export BERT_BASE_DIR=/path/to/bert/uncased_L-12_H-768_A-12 - pytorch_transformers bert \ + transformers bert \ $BERT_BASE_DIR/bert_model.ckpt \ $BERT_BASE_DIR/bert_config.json \ $BERT_BASE_DIR/pytorch_model.bin @@ -36,7 +36,7 @@ Here is an example of the conversion process for a pre-trained OpenAI GPT model, export OPENAI_GPT_CHECKPOINT_FOLDER_PATH=/path/to/openai/pretrained/numpy/weights - pytorch_transformers gpt \ + transformers gpt \ $OPENAI_GPT_CHECKPOINT_FOLDER_PATH \ $PYTORCH_DUMP_OUTPUT \ [OPENAI_GPT_CONFIG] @@ -50,7 +50,7 @@ Here is an example of the conversion process for a pre-trained OpenAI GPT-2 mode export OPENAI_GPT2_CHECKPOINT_PATH=/path/to/gpt2/pretrained/weights - pytorch_transformers gpt2 \ + transformers gpt2 \ $OPENAI_GPT2_CHECKPOINT_PATH \ $PYTORCH_DUMP_OUTPUT \ [OPENAI_GPT2_CONFIG] @@ -64,7 +64,7 @@ Here is an example of the conversion process for a pre-trained Transformer-XL mo export TRANSFO_XL_CHECKPOINT_FOLDER_PATH=/path/to/transfo/xl/checkpoint - pytorch_transformers transfo_xl \ + transformers transfo_xl \ $TRANSFO_XL_CHECKPOINT_FOLDER_PATH \ $PYTORCH_DUMP_OUTPUT \ [TRANSFO_XL_CONFIG] @@ -80,7 +80,7 @@ Here is an example of the conversion process for a pre-trained XLNet model, fine export TRANSFO_XL_CHECKPOINT_PATH=/path/to/xlnet/checkpoint export TRANSFO_XL_CONFIG_PATH=/path/to/xlnet/config - pytorch_transformers xlnet \ + transformers xlnet \ $TRANSFO_XL_CHECKPOINT_PATH \ $TRANSFO_XL_CONFIG_PATH \ $PYTORCH_DUMP_OUTPUT \ @@ -96,6 +96,6 @@ Here is an example of the conversion process for a pre-trained XLM model: export XLM_CHECKPOINT_PATH=/path/to/xlm/checkpoint - pytorch_transformers xlm \ + transformers xlm \ $XLM_CHECKPOINT_PATH \ $PYTORCH_DUMP_OUTPUT \ diff --git a/docs/source/index.rst b/docs/source/index.rst index 5b451707d6..a205b0b314 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,7 +1,7 @@ -Pytorch-Transformers +Transformers ================================================================================================================================================ -PyTorch-Transformers is a library of state-of-the-art pre-trained models for Natural Language Processing (NLP). +Transformers is a library of state-of-the-art pre-trained models for Natural Language Processing (NLP). The library currently contains PyTorch implementations, pre-trained model weights, usage scripts and conversion utilities for the following models: @@ -12,7 +12,7 @@ The library currently contains PyTorch implementations, pre-trained model weight 5. `XLNet `_ (from Google/CMU) released with the paper `​XLNet: Generalized Autoregressive Pretraining for Language Understanding `_ by Zhilin Yang*, Zihang Dai*, Yiming Yang, Jaime Carbonell, Ruslan Salakhutdinov, Quoc V. Le. 6. `XLM `_ (from Facebook) released together with the paper `Cross-lingual Language Model Pretraining `_ by Guillaume Lample and Alexis Conneau. 7. `RoBERTa `_ (from Facebook), released together with the paper a `Robustly Optimized BERT Pretraining Approach `_ by Yinhan Liu, Myle Ott, Naman Goyal, Jingfei Du, Mandar Joshi, Danqi Chen, Omer Levy, Mike Lewis, Luke Zettlemoyer, Veselin Stoyanov. -8. `DistilBERT `_ (from HuggingFace) released together with the blog post `Smaller, faster, cheaper, lighter: Introducing DistilBERT, a distilled version of BERT `_ by Victor Sanh, Lysandre Debut and Thomas Wolf. +8. `DistilBERT `_ (from HuggingFace) released together with the blog post `Smaller, faster, cheaper, lighter: Introducing DistilBERT, a distilled version of BERT `_ by Victor Sanh, Lysandre Debut and Thomas Wolf. .. toctree:: :maxdepth: 2 diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 6512a0cef3..51f7eb520d 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -1,7 +1,7 @@ Installation ================================================ -PyTorch-Transformers is tested on Python 2.7 and 3.5+ (examples are tested only on python 3.5+) and PyTorch 1.1.0 +Transformers is tested on Python 2.7 and 3.5+ (examples are tested only on python 3.5+) and PyTorch 1.1.0 With pip ^^^^^^^^ @@ -10,7 +10,7 @@ PyTorch Transformers can be installed using pip as follows: .. code-block:: bash - pip install pytorch-transformers + pip install transformers From source ^^^^^^^^^^^ @@ -19,15 +19,15 @@ To install from source, clone the repository and install with: .. code-block:: bash - git clone https://github.com/huggingface/pytorch-transformers.git - cd pytorch-transformers + git clone https://github.com/huggingface/transformers.git + cd transformers pip install [--editable] . Tests ^^^^^ -An extensive test suite is included to test the library behavior and several examples. Library tests can be found in the `tests folder `_ and examples tests in the `examples folder `_. +An extensive test suite is included to test the library behavior and several examples. Library tests can be found in the `tests folder `_ and examples tests in the `examples folder `_. Tests can be run using `pytest` (install pytest if needed with `pip install pytest`). @@ -35,7 +35,7 @@ Run all the tests from the root of the cloned repository with the commands: .. code-block:: bash - python -m pytest -sv ./pytorch_transformers/tests/ + python -m pytest -sv ./transformers/tests/ python -m pytest -sv ./examples/ diff --git a/docs/source/main_classes/configuration.rst b/docs/source/main_classes/configuration.rst index 5181874c1a..2131433759 100644 --- a/docs/source/main_classes/configuration.rst +++ b/docs/source/main_classes/configuration.rst @@ -6,5 +6,5 @@ The base class ``PretrainedConfig`` implements the common methods for loading/sa ``PretrainedConfig`` ~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.PretrainedConfig +.. autoclass:: transformers.PretrainedConfig :members: diff --git a/docs/source/main_classes/model.rst b/docs/source/main_classes/model.rst index ba61afadf0..d22467f907 100644 --- a/docs/source/main_classes/model.rst +++ b/docs/source/main_classes/model.rst @@ -11,5 +11,5 @@ The base class ``PreTrainedModel`` implements the common methods for loading/sav ``PreTrainedModel`` ~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.PreTrainedModel +.. autoclass:: transformers.PreTrainedModel :members: diff --git a/docs/source/main_classes/optimizer_schedules.rst b/docs/source/main_classes/optimizer_schedules.rst index 70fefb7c6d..ff0c9e6929 100644 --- a/docs/source/main_classes/optimizer_schedules.rst +++ b/docs/source/main_classes/optimizer_schedules.rst @@ -9,7 +9,7 @@ The ``.optimization`` module provides: ``AdamW`` ~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.AdamW +.. autoclass:: transformers.AdamW :members: Schedules @@ -18,11 +18,11 @@ Schedules Learning Rate Schedules ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. autoclass:: pytorch_transformers.ConstantLRSchedule +.. autoclass:: transformers.ConstantLRSchedule :members: -.. autoclass:: pytorch_transformers.WarmupConstantSchedule +.. autoclass:: transformers.WarmupConstantSchedule :members: .. image:: /imgs/warmup_constant_schedule.png @@ -30,7 +30,7 @@ Learning Rate Schedules :alt: -.. autoclass:: pytorch_transformers.WarmupCosineSchedule +.. autoclass:: transformers.WarmupCosineSchedule :members: .. image:: /imgs/warmup_cosine_schedule.png @@ -38,7 +38,7 @@ Learning Rate Schedules :alt: -.. autoclass:: pytorch_transformers.WarmupCosineWithHardRestartsSchedule +.. autoclass:: transformers.WarmupCosineWithHardRestartsSchedule :members: .. image:: /imgs/warmup_cosine_hard_restarts_schedule.png @@ -47,7 +47,7 @@ Learning Rate Schedules -.. autoclass:: pytorch_transformers.WarmupLinearSchedule +.. autoclass:: transformers.WarmupLinearSchedule :members: .. image:: /imgs/warmup_linear_schedule.png diff --git a/docs/source/main_classes/tokenizer.rst b/docs/source/main_classes/tokenizer.rst index 12ca5522de..c33eb45829 100644 --- a/docs/source/main_classes/tokenizer.rst +++ b/docs/source/main_classes/tokenizer.rst @@ -12,5 +12,5 @@ The base class ``PreTrainedTokenizer`` implements the common methods for loading ``PreTrainedTokenizer`` ~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.PreTrainedTokenizer +.. autoclass:: transformers.PreTrainedTokenizer :members: diff --git a/docs/source/migration.md b/docs/source/migration.md index 9cfcaade13..553a79c82b 100644 --- a/docs/source/migration.md +++ b/docs/source/migration.md @@ -1,17 +1,17 @@ # Migrating from pytorch-pretrained-bert -Here is a quick summary of what you should take care of when migrating from `pytorch-pretrained-bert` to `pytorch-transformers` +Here is a quick summary of what you should take care of when migrating from `pytorch-pretrained-bert` to `transformers` ### Models always output `tuples` -The main breaking change when migrating from `pytorch-pretrained-bert` to `pytorch-transformers` is that the models forward method always outputs a `tuple` with various elements depending on the model and the configuration parameters. +The main breaking change when migrating from `pytorch-pretrained-bert` to `transformers` is that the models forward method always outputs a `tuple` with various elements depending on the model and the configuration parameters. -The exact content of the tuples for each model are detailled in the models' docstrings and the [documentation](https://huggingface.co/pytorch-transformers/). +The exact content of the tuples for each model are detailled in the models' docstrings and the [documentation](https://huggingface.co/transformers/). In pretty much every case, you will be fine by taking the first element of the output as the output you previously used in `pytorch-pretrained-bert`. -Here is a `pytorch-pretrained-bert` to `pytorch-transformers` conversion example for a `BertForSequenceClassification` classification model: +Here is a `pytorch-pretrained-bert` to `transformers` conversion example for a `BertForSequenceClassification` classification model: ```python # Let's load our model @@ -20,11 +20,11 @@ model = BertForSequenceClassification.from_pretrained('bert-base-uncased') # If you used to have this line in pytorch-pretrained-bert: loss = model(input_ids, labels=labels) -# Now just use this line in pytorch-transformers to extract the loss from the output tuple: +# Now just use this line in transformers to extract the loss from the output tuple: outputs = model(input_ids, labels=labels) loss = outputs[0] -# In pytorch-transformers you can also have access to the logits: +# In transformers you can also have access to the logits: loss, logits = outputs[:2] # And even the attention weigths if you configure the model to output them (and other outputs too, see the docstrings and documentation) @@ -96,7 +96,7 @@ for batch in train_data: loss.backward() optimizer.step() -### In PyTorch-Transformers, optimizer and schedules are splitted and instantiated like this: +### In Transformers, optimizer and schedules are splitted and instantiated like this: optimizer = AdamW(model.parameters(), lr=lr, correct_bias=False) # To reproduce BertAdam specific behavior set correct_bias=False scheduler = WarmupLinearSchedule(optimizer, warmup_steps=num_warmup_steps, t_total=num_total_steps) # PyTorch scheduler ### and used like this: diff --git a/docs/source/model_doc/auto.rst b/docs/source/model_doc/auto.rst index 7b56eabafe..4b900d8e55 100644 --- a/docs/source/model_doc/auto.rst +++ b/docs/source/model_doc/auto.rst @@ -11,19 +11,19 @@ Instantiating one of ``AutoModel``, ``AutoConfig`` and ``AutoTokenizer`` will di ``AutoConfig`` ~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.AutoConfig +.. autoclass:: transformers.AutoConfig :members: ``AutoModel`` ~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.AutoModel +.. autoclass:: transformers.AutoModel :members: ``AutoTokenizer`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.AutoTokenizer +.. autoclass:: transformers.AutoTokenizer :members: diff --git a/docs/source/model_doc/bert.rst b/docs/source/model_doc/bert.rst index cbce74e73b..9f2caf3e80 100644 --- a/docs/source/model_doc/bert.rst +++ b/docs/source/model_doc/bert.rst @@ -4,69 +4,69 @@ BERT ``BertConfig`` ~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.BertConfig +.. autoclass:: transformers.BertConfig :members: ``BertTokenizer`` ~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.BertTokenizer +.. autoclass:: transformers.BertTokenizer :members: ``BertModel`` ~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.BertModel +.. autoclass:: transformers.BertModel :members: ``BertForPreTraining`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.BertForPreTraining +.. autoclass:: transformers.BertForPreTraining :members: ``BertForMaskedLM`` ~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.BertForMaskedLM +.. autoclass:: transformers.BertForMaskedLM :members: ``BertForNextSentencePrediction`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.BertForNextSentencePrediction +.. autoclass:: transformers.BertForNextSentencePrediction :members: ``BertForSequenceClassification`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.BertForSequenceClassification +.. autoclass:: transformers.BertForSequenceClassification :members: ``BertForMultipleChoice`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.BertForMultipleChoice +.. autoclass:: transformers.BertForMultipleChoice :members: ``BertForTokenClassification`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.BertForTokenClassification +.. autoclass:: transformers.BertForTokenClassification :members: ``BertForQuestionAnswering`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.BertForQuestionAnswering +.. autoclass:: transformers.BertForQuestionAnswering :members: diff --git a/docs/source/model_doc/distilbert.rst b/docs/source/model_doc/distilbert.rst index 141d3e151f..de1ac73675 100644 --- a/docs/source/model_doc/distilbert.rst +++ b/docs/source/model_doc/distilbert.rst @@ -4,40 +4,40 @@ DistilBERT ``DistilBertConfig`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.DistilBertConfig +.. autoclass:: transformers.DistilBertConfig :members: ``DistilBertTokenizer`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.DistilBertTokenizer +.. autoclass:: transformers.DistilBertTokenizer :members: ``DistilBertModel`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.DistilBertModel +.. autoclass:: transformers.DistilBertModel :members: ``DistilBertForMaskedLM`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.DistilBertForMaskedLM +.. autoclass:: transformers.DistilBertForMaskedLM :members: ``DistilBertForSequenceClassification`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.DistilBertForSequenceClassification +.. autoclass:: transformers.DistilBertForSequenceClassification :members: ``DistilBertForQuestionAnswering`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.DistilBertForQuestionAnswering +.. autoclass:: transformers.DistilBertForQuestionAnswering :members: diff --git a/docs/source/model_doc/gpt.rst b/docs/source/model_doc/gpt.rst index 26762ae011..39995a98fc 100644 --- a/docs/source/model_doc/gpt.rst +++ b/docs/source/model_doc/gpt.rst @@ -4,33 +4,33 @@ OpenAI GPT ``OpenAIGPTConfig`` ~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.OpenAIGPTConfig +.. autoclass:: transformers.OpenAIGPTConfig :members: ``OpenAIGPTTokenizer`` ~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.OpenAIGPTTokenizer +.. autoclass:: transformers.OpenAIGPTTokenizer :members: ``OpenAIGPTModel`` ~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.OpenAIGPTModel +.. autoclass:: transformers.OpenAIGPTModel :members: ``OpenAIGPTLMHeadModel`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.OpenAIGPTLMHeadModel +.. autoclass:: transformers.OpenAIGPTLMHeadModel :members: ``OpenAIGPTDoubleHeadsModel`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.OpenAIGPTDoubleHeadsModel +.. autoclass:: transformers.OpenAIGPTDoubleHeadsModel :members: diff --git a/docs/source/model_doc/gpt2.rst b/docs/source/model_doc/gpt2.rst index a49d1b4258..92decb14de 100644 --- a/docs/source/model_doc/gpt2.rst +++ b/docs/source/model_doc/gpt2.rst @@ -4,33 +4,33 @@ OpenAI GPT2 ``GPT2Config`` ~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.GPT2Config +.. autoclass:: transformers.GPT2Config :members: ``GPT2Tokenizer`` ~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.GPT2Tokenizer +.. autoclass:: transformers.GPT2Tokenizer :members: ``GPT2Model`` ~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.GPT2Model +.. autoclass:: transformers.GPT2Model :members: ``GPT2LMHeadModel`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.GPT2LMHeadModel +.. autoclass:: transformers.GPT2LMHeadModel :members: ``GPT2DoubleHeadsModel`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.GPT2DoubleHeadsModel +.. autoclass:: transformers.GPT2DoubleHeadsModel :members: diff --git a/docs/source/model_doc/roberta.rst b/docs/source/model_doc/roberta.rst index e2de917e35..5351e018cd 100644 --- a/docs/source/model_doc/roberta.rst +++ b/docs/source/model_doc/roberta.rst @@ -4,33 +4,33 @@ RoBERTa ``RobertaConfig`` ~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.RobertaConfig +.. autoclass:: transformers.RobertaConfig :members: ``RobertaTokenizer`` ~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.RobertaTokenizer +.. autoclass:: transformers.RobertaTokenizer :members: ``RobertaModel`` ~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.RobertaModel +.. autoclass:: transformers.RobertaModel :members: ``RobertaForMaskedLM`` ~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.RobertaForMaskedLM +.. autoclass:: transformers.RobertaForMaskedLM :members: ``RobertaForSequenceClassification`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.RobertaForSequenceClassification +.. autoclass:: transformers.RobertaForSequenceClassification :members: diff --git a/docs/source/model_doc/transformerxl.rst b/docs/source/model_doc/transformerxl.rst index 88cca450ee..c8a9cc7d99 100644 --- a/docs/source/model_doc/transformerxl.rst +++ b/docs/source/model_doc/transformerxl.rst @@ -5,26 +5,26 @@ Transformer XL ``TransfoXLConfig`` ~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.TransfoXLConfig +.. autoclass:: transformers.TransfoXLConfig :members: ``TransfoXLTokenizer`` ~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.TransfoXLTokenizer +.. autoclass:: transformers.TransfoXLTokenizer :members: ``TransfoXLModel`` ~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.TransfoXLModel +.. autoclass:: transformers.TransfoXLModel :members: ``TransfoXLLMHeadModel`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.TransfoXLLMHeadModel +.. autoclass:: transformers.TransfoXLLMHeadModel :members: diff --git a/docs/source/model_doc/xlm.rst b/docs/source/model_doc/xlm.rst index 217952ea5e..344371ad52 100644 --- a/docs/source/model_doc/xlm.rst +++ b/docs/source/model_doc/xlm.rst @@ -4,38 +4,38 @@ XLM ``XLMConfig`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.XLMConfig +.. autoclass:: transformers.XLMConfig :members: ``XLMTokenizer`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.XLMTokenizer +.. autoclass:: transformers.XLMTokenizer :members: ``XLMModel`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.XLMModel +.. autoclass:: transformers.XLMModel :members: ``XLMWithLMHeadModel`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.XLMWithLMHeadModel +.. autoclass:: transformers.XLMWithLMHeadModel :members: ``XLMForSequenceClassification`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.XLMForSequenceClassification +.. autoclass:: transformers.XLMForSequenceClassification :members: ``XLMForQuestionAnswering`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.XLMForQuestionAnswering +.. autoclass:: transformers.XLMForQuestionAnswering :members: diff --git a/docs/source/model_doc/xlnet.rst b/docs/source/model_doc/xlnet.rst index e388934c56..a471506278 100644 --- a/docs/source/model_doc/xlnet.rst +++ b/docs/source/model_doc/xlnet.rst @@ -4,40 +4,40 @@ XLNet ``XLNetConfig`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.XLNetConfig +.. autoclass:: transformers.XLNetConfig :members: ``XLNetTokenizer`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.XLNetTokenizer +.. autoclass:: transformers.XLNetTokenizer :members: ``XLNetModel`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.XLNetModel +.. autoclass:: transformers.XLNetModel :members: ``XLNetLMHeadModel`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.XLNetLMHeadModel +.. autoclass:: transformers.XLNetLMHeadModel :members: ``XLNetForSequenceClassification`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.XLNetForSequenceClassification +.. autoclass:: transformers.XLNetForSequenceClassification :members: ``XLNetForQuestionAnswering`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.XLNetForQuestionAnswering +.. autoclass:: transformers.XLNetForQuestionAnswering :members: diff --git a/docs/source/notebooks.rst b/docs/source/notebooks.rst index 7e214fa00a..fe669e8e47 100644 --- a/docs/source/notebooks.rst +++ b/docs/source/notebooks.rst @@ -1,16 +1,16 @@ Notebooks ================================================ -We include `three Jupyter Notebooks `_ that can be used to check that the predictions of the PyTorch model are identical to the predictions of the original TensorFlow model. +We include `three Jupyter Notebooks `_ that can be used to check that the predictions of the PyTorch model are identical to the predictions of the original TensorFlow model. * - The first NoteBook (\ `Comparing-TF-and-PT-models.ipynb `_\ ) extracts the hidden states of a full sequence on each layers of the TensorFlow and the PyTorch models and computes the standard deviation between them. In the given example, we get a standard deviation of 1.5e-7 to 9e-7 on the various hidden state of the models. + The first NoteBook (\ `Comparing-TF-and-PT-models.ipynb `_\ ) extracts the hidden states of a full sequence on each layers of the TensorFlow and the PyTorch models and computes the standard deviation between them. In the given example, we get a standard deviation of 1.5e-7 to 9e-7 on the various hidden state of the models. * - The second NoteBook (\ `Comparing-TF-and-PT-models-SQuAD.ipynb `_\ ) compares the loss computed by the TensorFlow and the PyTorch models for identical initialization of the fine-tuning layer of the ``BertForQuestionAnswering`` and computes the standard deviation between them. In the given example, we get a standard deviation of 2.5e-7 between the models. + The second NoteBook (\ `Comparing-TF-and-PT-models-SQuAD.ipynb `_\ ) compares the loss computed by the TensorFlow and the PyTorch models for identical initialization of the fine-tuning layer of the ``BertForQuestionAnswering`` and computes the standard deviation between them. In the given example, we get a standard deviation of 2.5e-7 between the models. * - The third NoteBook (\ `Comparing-TF-and-PT-models-MLM-NSP.ipynb `_\ ) compares the predictions computed by the TensorFlow and the PyTorch models for masked token language modeling using the pre-trained masked language modeling model. + The third NoteBook (\ `Comparing-TF-and-PT-models-MLM-NSP.ipynb `_\ ) compares the predictions computed by the TensorFlow and the PyTorch models for masked token language modeling using the pre-trained masked language modeling model. Please follow the instructions given in the notebooks to run and modify them. diff --git a/docs/source/pretrained_models.rst b/docs/source/pretrained_models.rst index d6e273797f..0e55767d76 100644 --- a/docs/source/pretrained_models.rst +++ b/docs/source/pretrained_models.rst @@ -44,15 +44,15 @@ Here is the full list of the currently provided pretrained models together with | +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``bert-large-uncased-whole-word-masking-finetuned-squad`` | | 24-layer, 1024-hidden, 16-heads, 340M parameters. | | | | | The ``bert-large-uncased-whole-word-masking`` model fine-tuned on SQuAD | -| | | (see details of fine-tuning in the `example section `__). | +| | | (see details of fine-tuning in the `example section `__). | | +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``bert-large-cased-whole-word-masking-finetuned-squad`` | | 24-layer, 1024-hidden, 16-heads, 340M parameters | | | | | The ``bert-large-cased-whole-word-masking`` model fine-tuned on SQuAD | -| | | (see `details of fine-tuning in the example section `__) | +| | | (see `details of fine-tuning in the example section `__) | | +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``bert-base-cased-finetuned-mrpc`` | | 12-layer, 768-hidden, 12-heads, 110M parameters. | | | | | The ``bert-base-cased`` model fine-tuned on MRPC | -| | | (see `details of fine-tuning in the example section `__) | +| | | (see `details of fine-tuning in the example section `__) | +-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | GPT | ``openai-gpt`` | | 12-layer, 768-hidden, 12-heads, 110M parameters. | | | | | OpenAI GPT English model | @@ -120,4 +120,4 @@ Here is the full list of the currently provided pretrained models together with | | | (see `details `__) | +-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ -.. `__ \ No newline at end of file +.. `__ \ No newline at end of file diff --git a/docs/source/quickstart.md b/docs/source/quickstart.md index f037a95a3a..c7ad5c7930 100644 --- a/docs/source/quickstart.md +++ b/docs/source/quickstart.md @@ -2,7 +2,7 @@ ## Philosophy -PyTorch-Transformers is an opinionated library built for NLP researchers seeking to use/study/extend large-scale transformers models. +Transformers is an opinionated library built for NLP researchers seeking to use/study/extend large-scale transformers models. The library was designed with two strong goals in mind: @@ -39,7 +39,7 @@ The library is build around three type of classes for each models: All these classes can be instantiated from pretrained instances and saved locally using two methods: -- `from_pretrained()` let you instantiate a model/configuration/tokenizer from a pretrained version either provided by the library itself (currently 27 models are provided as listed [here](https://huggingface.co/pytorch-transformers/pretrained_models.html)) or stored locally (or on a server) by the user, +- `from_pretrained()` let you instantiate a model/configuration/tokenizer from a pretrained version either provided by the library itself (currently 27 models are provided as listed [here](https://huggingface.co/transformers/pretrained_models.html)) or stored locally (or on a server) by the user, - `save_pretrained()` let you save a model/configuration/tokenizer locally so that it can be reloaded using `from_pretrained()`. We'll finish this quickstart tour by going through a few simple quick-start examples to see how we can instantiate and use these classes. The rest of the documentation is organized in two parts: @@ -59,7 +59,7 @@ Let's start by preparing a tokenized input (a list of token embeddings indices t ```python import torch -from pytorch_transformers import BertTokenizer, BertModel, BertForMaskedLM +from transformers import BertTokenizer, BertModel, BertForMaskedLM # OPTIONAL: if you want to have more information on what's happening under the hood, activate the logger as follows import logging @@ -106,7 +106,7 @@ model.to('cuda') with torch.no_grad(): # See the models docstrings for the detail of the inputs outputs = model(tokens_tensor, token_type_ids=segments_tensors) - # PyTorch-Transformers models always output tuples. + # Transformers models always output tuples. # See the models docstrings for the detail of all the outputs # In our case, the first element is the hidden state of the last layer of the Bert model encoded_layers = outputs[0] @@ -145,7 +145,7 @@ First let's prepare a tokenized input from our text string using `GPT2Tokenizer` ```python import torch -from pytorch_transformers import GPT2Tokenizer, GPT2LMHeadModel +from transformers import GPT2Tokenizer, GPT2LMHeadModel # OPTIONAL: if you want to have more information on what's happening, activate the logger as follows import logging diff --git a/docs/source/serialization.rst b/docs/source/serialization.rst index 7117d7ffa6..0b0b600ec1 100644 --- a/docs/source/serialization.rst +++ b/docs/source/serialization.rst @@ -45,7 +45,7 @@ where * ``bert_config.json`` or ``openai_gpt_config.json`` a configuration file for the model, and * ``pytorch_model.bin`` a PyTorch dump of a pre-trained instance of ``BertForPreTraining``\ , ``OpenAIGPTModel``\ , ``TransfoXLModel``\ , ``GPT2LMHeadModel`` (saved with the usual ``torch.save()``\ ) - If ``PRE_TRAINED_MODEL_NAME_OR_PATH`` is a shortcut name, the pre-trained weights will be downloaded from AWS S3 (see the links `here `__\ ) and stored in a cache folder to avoid future download (the cache folder can be found at ``~/.pytorch_pretrained_bert/``\ ). + If ``PRE_TRAINED_MODEL_NAME_OR_PATH`` is a shortcut name, the pre-trained weights will be downloaded from AWS S3 (see the links `here `__\ ) and stored in a cache folder to avoid future download (the cache folder can be found at ``~/.pytorch_pretrained_bert/``\ ). * ``cache_dir`` can be an optional path to a specific directory to download and cache the pre-trained model weights. This option is useful in particular when you are using distributed training: to avoid concurrent access to the same weights you can set for example ``cache_dir='./pretrained_model_{}'.format(args.local_rank)`` (see the section on distributed training for more information). @@ -122,7 +122,7 @@ Here is the recommended way of saving the model, configuration and vocabulary to .. code-block:: python - from pytorch_transformers import WEIGHTS_NAME, CONFIG_NAME + from transformers import WEIGHTS_NAME, CONFIG_NAME output_dir = "./models/" diff --git a/docs/source/torchscript.rst b/docs/source/torchscript.rst index 5811572c07..fd1eeb5363 100644 --- a/docs/source/torchscript.rst +++ b/docs/source/torchscript.rst @@ -12,7 +12,7 @@ According to Pytorch's documentation: "TorchScript is a way to create serializab Pytorch's two modules `JIT and TRACE `_ allow the developer to export their model to be re-used in other programs, such as efficiency-oriented C++ programs. -We have provided an interface that allows the export of `pytorch-transformers` models to TorchScript so that they can +We have provided an interface that allows the export of `transformers` models to TorchScript so that they can be reused in a different environment than a Pytorch-based python program. Here we explain how to use our models so that they can be exported, and what to be mindful of when using these models with TorchScript. @@ -74,7 +74,7 @@ according to a ``BertConfig`` class and then saved to disk under the filename `` .. code-block:: python - from pytorch_transformers import BertModel, BertTokenizer, BertConfig + from transformers import BertModel, BertTokenizer, BertConfig import torch enc = BertTokenizer.from_pretrained("bert-base-uncased") diff --git a/examples/README.md b/examples/README.md index 3253e5481c..85c4c568f6 100644 --- a/examples/README.md +++ b/examples/README.md @@ -13,7 +13,7 @@ similar API between the different models. ## Language model fine-tuning -Based on the script [`run_lm_finetuning.py`](https://github.com/huggingface/pytorch-transformers/blob/master/examples/run_lm_finetuning.py). +Based on the script [`run_lm_finetuning.py`](https://github.com/huggingface/transformers/blob/master/examples/run_lm_finetuning.py). Fine-tuning the library models for language modeling on a text dataset for GPT, GPT-2, BERT and RoBERTa (DistilBERT to be added soon). GPT and GPT-2 are fine-tuned using a causal language modeling (CLM) loss while BERT and RoBERTa @@ -75,7 +75,7 @@ python run_lm_finetuning.py \ ## Language generation -Based on the script [`run_generation.py`](https://github.com/huggingface/pytorch-transformers/blob/master/examples/run_generation.py). +Based on the script [`run_generation.py`](https://github.com/huggingface/transformers/blob/master/examples/run_generation.py). Conditional text generation using the auto-regressive models of the library: GPT, GPT-2, Transformer-XL and XLNet. A similar script is used for our official demo [Write With Transfomer](https://transformer.huggingface.co), where you @@ -91,7 +91,7 @@ python run_generation.py \ ## GLUE -Based on the script [`run_glue.py`](https://github.com/huggingface/pytorch-transformers/blob/master/examples/run_glue.py). +Based on the script [`run_glue.py`](https://github.com/huggingface/transformers/blob/master/examples/run_glue.py). Fine-tuning the library models for sequence classification on the GLUE benchmark: [General Language Understanding Evaluation](https://gluebenchmark.com/). This script can fine-tune the following models: BERT, XLM, XLNet and RoBERTa. @@ -319,7 +319,7 @@ eval_loss = 0.44457291918821606 ## SQuAD -Based on the script [`run_squad.py`](https://github.com/huggingface/pytorch-transformers/blob/master/examples/run_squad.py). +Based on the script [`run_squad.py`](https://github.com/huggingface/transformers/blob/master/examples/run_squad.py). #### Fine-tuning on SQuAD diff --git a/examples/contrib/run_openai_gpt.py b/examples/contrib/run_openai_gpt.py index 1c9fba8ee8..661c1c305b 100644 --- a/examples/contrib/run_openai_gpt.py +++ b/examples/contrib/run_openai_gpt.py @@ -39,7 +39,7 @@ import torch from torch.utils.data import (DataLoader, RandomSampler, SequentialSampler, TensorDataset) -from pytorch_transformers import (OpenAIGPTDoubleHeadsModel, OpenAIGPTTokenizer, +from transformers import (OpenAIGPTDoubleHeadsModel, OpenAIGPTTokenizer, AdamW, cached_path, WEIGHTS_NAME, CONFIG_NAME, WarmupLinearSchedule) diff --git a/examples/contrib/run_swag.py b/examples/contrib/run_swag.py index 495f40cec9..58aec25877 100644 --- a/examples/contrib/run_swag.py +++ b/examples/contrib/run_swag.py @@ -35,10 +35,10 @@ from tqdm import tqdm, trange from tensorboardX import SummaryWriter -from pytorch_transformers import (WEIGHTS_NAME, BertConfig, +from transformers import (WEIGHTS_NAME, BertConfig, BertForMultipleChoice, BertTokenizer) -from pytorch_transformers import AdamW, WarmupLinearSchedule +from transformers import AdamW, WarmupLinearSchedule logger = logging.getLogger(__name__) @@ -365,7 +365,7 @@ def train(args, train_dataset, model, tokenizer): # inputs.update({'cls_index': batch[5], # 'p_mask': batch[6]}) outputs = model(**inputs) - loss = outputs[0] # model outputs are always tuple in pytorch-transformers (see doc) + loss = outputs[0] # model outputs are always tuple in transformers (see doc) if args.n_gpu > 1: loss = loss.mean() # mean() to average on multi-gpu parallel (not distributed) training @@ -647,7 +647,7 @@ def main(): if args.eval_all_checkpoints: checkpoints = list(os.path.dirname(c) for c in sorted(glob.glob(args.output_dir + '/**/' + WEIGHTS_NAME, recursive=True))) - logging.getLogger("pytorch_transformers.modeling_utils").setLevel(logging.WARN) # Reduce model loading logs + logging.getLogger("transformers.modeling_utils").setLevel(logging.WARN) # Reduce model loading logs logger.info("Evaluate the following checkpoints: %s", checkpoints) diff --git a/examples/contrib/run_transfo_xl.py b/examples/contrib/run_transfo_xl.py index 4c99777b98..f5375269b8 100644 --- a/examples/contrib/run_transfo_xl.py +++ b/examples/contrib/run_transfo_xl.py @@ -28,7 +28,7 @@ import math import torch -from pytorch_transformers import TransfoXLLMHeadModel, TransfoXLCorpus, TransfoXLTokenizer +from transformers import TransfoXLLMHeadModel, TransfoXLCorpus, TransfoXLTokenizer logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', datefmt = '%m/%d/%Y %H:%M:%S', diff --git a/examples/distillation/README.md b/examples/distillation/README.md index 73e0cc0655..ad4f8989b7 100644 --- a/examples/distillation/README.md +++ b/examples/distillation/README.md @@ -13,11 +13,11 @@ For more information on DistilBERT, please refer to our [detailed blog post](htt This part of the library has only be tested with Python3.6+. There are few specific dependencies to install before launching a distillation, you can install them with the command `pip install -r requirements.txt`. -**Important note:** The training scripts have been updated to support PyTorch v1.2.0 (there are breakings changes compared to v1.1.0). It is important to note that there is a small internal bug in the current version of PyTorch available on pip that causes a memory leak in our training/distillation. It has been recently fixed and will likely be integrated into the next release. For the moment, we recommend to [compile PyTorch from source](https://github.com/pytorch/pytorch#from-source). Please refer to [issue 1179](https://github.com/huggingface/pytorch-transformers/issues/1179) for more details. +**Important note:** The training scripts have been updated to support PyTorch v1.2.0 (there are breakings changes compared to v1.1.0). It is important to note that there is a small internal bug in the current version of PyTorch available on pip that causes a memory leak in our training/distillation. It has been recently fixed and will likely be integrated into the next release. For the moment, we recommend to [compile PyTorch from source](https://github.com/pytorch/pytorch#from-source). Please refer to [issue 1179](https://github.com/huggingface/transformers/issues/1179) for more details. ## How to use DistilBERT -PyTorch-Transformers includes two pre-trained DistilBERT models, currently only provided for English (we are investigating the possibility to train and release a multilingual version of DistilBERT): +Transformers includes two pre-trained DistilBERT models, currently only provided for English (we are investigating the possibility to train and release a multilingual version of DistilBERT): - `distilbert-base-uncased`: DistilBERT English language model pretrained on the same data used to pretrain Bert (concatenation of the Toronto Book Corpus and full English Wikipedia) using distillation with the supervision of the `bert-base-uncased` version of Bert. The model has 6 layers, 768 dimension and 12 heads, totalizing 66M parameters. - `distilbert-base-uncased-distilled-squad`: A finetuned version of `distilbert-base-uncased` finetuned using (a second step of) knwoledge distillation on SQuAD 1.0. This model reaches a F1 score of 86.2 on the dev set (for comparison, Bert `bert-base-uncased` version reaches a 88.5 F1 score). diff --git a/examples/distillation/distiller.py b/examples/distillation/distiller.py index 93135e292c..1bfda325dc 100644 --- a/examples/distillation/distiller.py +++ b/examples/distillation/distiller.py @@ -26,7 +26,7 @@ import torch import torch.nn as nn import torch.nn.functional as F -from pytorch_transformers import AdamW, WarmupLinearSchedule +from transformers import AdamW, WarmupLinearSchedule from utils import logger from dataset import Dataset diff --git a/examples/distillation/scripts/binarized_data.py b/examples/distillation/scripts/binarized_data.py index 51be8fd0be..e98662fdc8 100644 --- a/examples/distillation/scripts/binarized_data.py +++ b/examples/distillation/scripts/binarized_data.py @@ -20,7 +20,7 @@ import pickle import random import time import numpy as np -from pytorch_transformers import BertTokenizer +from transformers import BertTokenizer import logging logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', diff --git a/examples/distillation/scripts/extract_for_distil.py b/examples/distillation/scripts/extract_for_distil.py index f3eee024ec..1b9e20c382 100644 --- a/examples/distillation/scripts/extract_for_distil.py +++ b/examples/distillation/scripts/extract_for_distil.py @@ -15,7 +15,7 @@ """ Preprocessing script before training DistilBERT. """ -from pytorch_transformers import BertForPreTraining +from transformers import BertForPreTraining import torch import argparse diff --git a/examples/distillation/train.py b/examples/distillation/train.py index 712f10b47d..6060cd9eb6 100644 --- a/examples/distillation/train.py +++ b/examples/distillation/train.py @@ -23,8 +23,8 @@ import shutil import numpy as np import torch -from pytorch_transformers import BertTokenizer, BertForMaskedLM -from pytorch_transformers import DistilBertForMaskedLM, DistilBertConfig +from transformers import BertTokenizer, BertForMaskedLM +from transformers import DistilBertForMaskedLM, DistilBertConfig from distiller import Distiller from utils import git_log, logger, init_gpu_params, set_seed diff --git a/examples/run_bertology.py b/examples/run_bertology.py index f11b73b54f..f37358359d 100644 --- a/examples/run_bertology.py +++ b/examples/run_bertology.py @@ -32,7 +32,7 @@ from torch.utils.data import DataLoader, SequentialSampler, TensorDataset, Subse from torch.utils.data.distributed import DistributedSampler from torch.nn import CrossEntropyLoss, MSELoss -from pytorch_transformers import (WEIGHTS_NAME, +from transformers import (WEIGHTS_NAME, BertConfig, BertForSequenceClassification, BertTokenizer, XLMConfig, XLMForSequenceClassification, XLMTokenizer, XLNetConfig, XLNetForSequenceClassification, XLNetTokenizer) diff --git a/examples/run_generation.py b/examples/run_generation.py index a2a8f29103..9e98a9e870 100644 --- a/examples/run_generation.py +++ b/examples/run_generation.py @@ -26,12 +26,12 @@ import torch import torch.nn.functional as F import numpy as np -from pytorch_transformers import GPT2Config, OpenAIGPTConfig, XLNetConfig, TransfoXLConfig +from transformers import GPT2Config, OpenAIGPTConfig, XLNetConfig, TransfoXLConfig -from pytorch_transformers import GPT2LMHeadModel, GPT2Tokenizer -from pytorch_transformers import OpenAIGPTLMHeadModel, OpenAIGPTTokenizer -from pytorch_transformers import XLNetLMHeadModel, XLNetTokenizer -from pytorch_transformers import TransfoXLLMHeadModel, TransfoXLTokenizer +from transformers import GPT2LMHeadModel, GPT2Tokenizer +from transformers import OpenAIGPTLMHeadModel, OpenAIGPTTokenizer +from transformers import XLNetLMHeadModel, XLNetTokenizer +from transformers import TransfoXLLMHeadModel, TransfoXLTokenizer logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', diff --git a/examples/run_glue.py b/examples/run_glue.py index 99821f454d..71dad0edbf 100644 --- a/examples/run_glue.py +++ b/examples/run_glue.py @@ -31,7 +31,7 @@ from torch.utils.data.distributed import DistributedSampler from tensorboardX import SummaryWriter from tqdm import tqdm, trange -from pytorch_transformers import (WEIGHTS_NAME, BertConfig, +from transformers import (WEIGHTS_NAME, BertConfig, BertForSequenceClassification, BertTokenizer, RobertaConfig, RobertaForSequenceClassification, @@ -44,12 +44,12 @@ from pytorch_transformers import (WEIGHTS_NAME, BertConfig, DistilBertForSequenceClassification, DistilBertTokenizer) -from pytorch_transformers import AdamW, WarmupLinearSchedule +from transformers import AdamW, WarmupLinearSchedule -from pytorch_transformers import glue_compute_metrics as compute_metrics -from pytorch_transformers import glue_output_modes as output_modes -from pytorch_transformers import glue_processors as processors -from pytorch_transformers import glue_convert_examples_to_features as convert_examples_to_features +from transformers import glue_compute_metrics as compute_metrics +from transformers import glue_output_modes as output_modes +from transformers import glue_processors as processors +from transformers import glue_convert_examples_to_features as convert_examples_to_features logger = logging.getLogger(__name__) @@ -137,7 +137,7 @@ def train(args, train_dataset, model, tokenizer): 'token_type_ids': batch[2] if args.model_type in ['bert', 'xlnet'] else None, # XLM, DistilBERT and RoBERTa don't use segment_ids 'labels': batch[3]} outputs = model(**inputs) - loss = outputs[0] # model outputs are always tuple in pytorch-transformers (see doc) + loss = outputs[0] # model outputs are always tuple in transformers (see doc) if args.n_gpu > 1: loss = loss.mean() # mean() to average on multi-gpu parallel training @@ -483,7 +483,7 @@ def main(): checkpoints = [args.output_dir] if args.eval_all_checkpoints: checkpoints = list(os.path.dirname(c) for c in sorted(glob.glob(args.output_dir + '/**/' + WEIGHTS_NAME, recursive=True))) - logging.getLogger("pytorch_transformers.modeling_utils").setLevel(logging.WARN) # Reduce logging + logging.getLogger("transformers.modeling_utils").setLevel(logging.WARN) # Reduce logging logger.info("Evaluate the following checkpoints: %s", checkpoints) for checkpoint in checkpoints: global_step = checkpoint.split('-')[-1] if len(checkpoints) > 1 else "" diff --git a/examples/run_lm_finetuning.py b/examples/run_lm_finetuning.py index 556cce6753..7ccf4c3cb7 100644 --- a/examples/run_lm_finetuning.py +++ b/examples/run_lm_finetuning.py @@ -35,7 +35,7 @@ from torch.utils.data.distributed import DistributedSampler from tensorboardX import SummaryWriter from tqdm import tqdm, trange -from pytorch_transformers import (WEIGHTS_NAME, AdamW, WarmupLinearSchedule, +from transformers import (WEIGHTS_NAME, AdamW, WarmupLinearSchedule, BertConfig, BertForMaskedLM, BertTokenizer, GPT2Config, GPT2LMHeadModel, GPT2Tokenizer, OpenAIGPTConfig, OpenAIGPTLMHeadModel, OpenAIGPTTokenizer, @@ -188,7 +188,7 @@ def train(args, train_dataset, model, tokenizer): labels = labels.to(args.device) model.train() outputs = model(inputs, masked_lm_labels=labels) if args.mlm else model(inputs, labels=labels) - loss = outputs[0] # model outputs are always tuple in pytorch-transformers (see doc) + loss = outputs[0] # model outputs are always tuple in transformers (see doc) if args.n_gpu > 1: loss = loss.mean() # mean() to average on multi-gpu parallel training @@ -481,7 +481,7 @@ def main(): checkpoints = [args.output_dir] if args.eval_all_checkpoints: checkpoints = list(os.path.dirname(c) for c in sorted(glob.glob(args.output_dir + '/**/' + WEIGHTS_NAME, recursive=True))) - logging.getLogger("pytorch_transformers.modeling_utils").setLevel(logging.WARN) # Reduce logging + logging.getLogger("transformers.modeling_utils").setLevel(logging.WARN) # Reduce logging logger.info("Evaluate the following checkpoints: %s", checkpoints) for checkpoint in checkpoints: global_step = checkpoint.split('-')[-1] if len(checkpoints) > 1 else "" diff --git a/examples/run_multiple_choice.py b/examples/run_multiple_choice.py index 05f9a48f50..54f3a8a904 100644 --- a/examples/run_multiple_choice.py +++ b/examples/run_multiple_choice.py @@ -32,13 +32,13 @@ from torch.utils.data.distributed import DistributedSampler from tensorboardX import SummaryWriter from tqdm import tqdm, trange -from pytorch_transformers import (WEIGHTS_NAME, BertConfig, +from transformers import (WEIGHTS_NAME, BertConfig, BertForMultipleChoice, BertTokenizer, XLNetConfig, XLNetForMultipleChoice, XLNetTokenizer, RobertaConfig, RobertaForMultipleChoice, RobertaTokenizer) -from pytorch_transformers import AdamW, WarmupLinearSchedule +from transformers import AdamW, WarmupLinearSchedule from utils_multiple_choice import (convert_examples_to_features, processors) @@ -141,7 +141,7 @@ def train(args, train_dataset, model, tokenizer): 'token_type_ids': batch[2] if args.model_type in ['bert', 'xlnet'] else None, # XLM don't use segment_ids 'labels': batch[3]} outputs = model(**inputs) - loss = outputs[0] # model outputs are always tuple in pytorch-transformers (see doc) + loss = outputs[0] # model outputs are always tuple in transformers (see doc) if args.n_gpu > 1: loss = loss.mean() # mean() to average on multi-gpu parallel training @@ -508,7 +508,7 @@ def main(): checkpoints = [args.output_dir] if args.eval_all_checkpoints: checkpoints = list(os.path.dirname(c) for c in sorted(glob.glob(args.output_dir + '/**/' + WEIGHTS_NAME, recursive=True))) - logging.getLogger("pytorch_transformers.modeling_utils").setLevel(logging.WARN) # Reduce logging + logging.getLogger("transformers.modeling_utils").setLevel(logging.WARN) # Reduce logging logger.info("Evaluate the following checkpoints: %s", checkpoints) for checkpoint in checkpoints: global_step = checkpoint.split('-')[-1] if len(checkpoints) > 1 else "" @@ -524,7 +524,7 @@ def main(): checkpoints = [args.output_dir] # if args.eval_all_checkpoints: # can not use this to do test!! # checkpoints = list(os.path.dirname(c) for c in sorted(glob.glob(args.output_dir + '/**/' + WEIGHTS_NAME, recursive=True))) - # logging.getLogger("pytorch_transformers.modeling_utils").setLevel(logging.WARN) # Reduce logging + # logging.getLogger("transformers.modeling_utils").setLevel(logging.WARN) # Reduce logging logger.info("Evaluate the following checkpoints: %s", checkpoints) for checkpoint in checkpoints: global_step = checkpoint.split('-')[-1] if len(checkpoints) > 1 else "" diff --git a/examples/run_squad.py b/examples/run_squad.py index affef90ca9..0c0fbf2963 100644 --- a/examples/run_squad.py +++ b/examples/run_squad.py @@ -32,7 +32,7 @@ from tqdm import tqdm, trange from tensorboardX import SummaryWriter -from pytorch_transformers import (WEIGHTS_NAME, BertConfig, +from transformers import (WEIGHTS_NAME, BertConfig, BertForQuestionAnswering, BertTokenizer, XLMConfig, XLMForQuestionAnswering, XLMTokenizer, XLNetConfig, @@ -40,7 +40,7 @@ from pytorch_transformers import (WEIGHTS_NAME, BertConfig, XLNetTokenizer, DistilBertConfig, DistilBertForQuestionAnswering, DistilBertTokenizer) -from pytorch_transformers import AdamW, WarmupLinearSchedule +from transformers import AdamW, WarmupLinearSchedule from utils_squad import (read_squad_examples, convert_examples_to_features, RawResult, write_predictions, @@ -142,7 +142,7 @@ def train(args, train_dataset, model, tokenizer): inputs.update({'cls_index': batch[5], 'p_mask': batch[6]}) outputs = model(**inputs) - loss = outputs[0] # model outputs are always tuple in pytorch-transformers (see doc) + loss = outputs[0] # model outputs are always tuple in transformers (see doc) if args.n_gpu > 1: loss = loss.mean() # mean() to average on multi-gpu parallel (not distributed) training @@ -510,7 +510,7 @@ def main(): checkpoints = [args.output_dir] if args.eval_all_checkpoints: checkpoints = list(os.path.dirname(c) for c in sorted(glob.glob(args.output_dir + '/**/' + WEIGHTS_NAME, recursive=True))) - logging.getLogger("pytorch_transformers.modeling_utils").setLevel(logging.WARN) # Reduce model loading logs + logging.getLogger("transformers.modeling_utils").setLevel(logging.WARN) # Reduce model loading logs logger.info("Evaluate the following checkpoints: %s", checkpoints) diff --git a/examples/run_tf_glue.py b/examples/run_tf_glue.py index 1301d13e08..3a867f80a8 100644 --- a/examples/run_tf_glue.py +++ b/examples/run_tf_glue.py @@ -1,6 +1,6 @@ import tensorflow as tf import tensorflow_datasets -from pytorch_transformers import * +from transformers import * # Load dataset, tokenizer, model from pretrained model/vocabulary tokenizer = BertTokenizer.from_pretrained('bert-base-cased') diff --git a/examples/utils_squad.py b/examples/utils_squad.py index 34a0c9cc02..b990ecc842 100644 --- a/examples/utils_squad.py +++ b/examples/utils_squad.py @@ -24,7 +24,7 @@ import math import collections from io import open -from pytorch_transformers.tokenization_bert import BasicTokenizer, whitespace_tokenize +from transformers.tokenization_bert import BasicTokenizer, whitespace_tokenize # Required by XLNet evaluation method to compute optimal threshold (see write_predictions_extended() method) from utils_squad_evaluate import find_all_best_thresh_v2, make_qid_to_has_ans, get_raw_scores diff --git a/hubconf.py b/hubconf.py index d9aaa6b53a..3fa354ed5a 100644 --- a/hubconf.py +++ b/hubconf.py @@ -1,7 +1,7 @@ -from pytorch_transformers import ( +from transformers import ( AutoTokenizer, AutoConfig, AutoModel, AutoModelWithLMHead, AutoModelForSequenceClassification, AutoModelForQuestionAnswering ) -from pytorch_transformers.file_utils import add_start_docstrings +from transformers.file_utils import add_start_docstrings dependencies = ['torch', 'tqdm', 'boto3', 'requests', 'regex', 'sentencepiece', 'sacremoses'] @@ -11,12 +11,12 @@ def config(*args, **kwargs): # Using torch.hub ! import torch - config = torch.hub.load('huggingface/pytorch-transformers', 'config', 'bert-base-uncased') # Download configuration from S3 and cache. - config = torch.hub.load('huggingface/pytorch-transformers', 'config', './test/bert_saved_model/') # E.g. config (or model) was saved using `save_pretrained('./test/saved_model/')` - config = torch.hub.load('huggingface/pytorch-transformers', 'config', './test/bert_saved_model/my_configuration.json') - config = torch.hub.load('huggingface/pytorch-transformers', 'config', 'bert-base-uncased', output_attention=True, foo=False) + config = torch.hub.load('huggingface/transformers', 'config', 'bert-base-uncased') # Download configuration from S3 and cache. + config = torch.hub.load('huggingface/transformers', 'config', './test/bert_saved_model/') # E.g. config (or model) was saved using `save_pretrained('./test/saved_model/')` + config = torch.hub.load('huggingface/transformers', 'config', './test/bert_saved_model/my_configuration.json') + config = torch.hub.load('huggingface/transformers', 'config', 'bert-base-uncased', output_attention=True, foo=False) assert config.output_attention == True - config, unused_kwargs = torch.hub.load('huggingface/pytorch-transformers', 'config', 'bert-base-uncased', output_attention=True, foo=False, return_unused_kwargs=True) + config, unused_kwargs = torch.hub.load('huggingface/transformers', 'config', 'bert-base-uncased', output_attention=True, foo=False, return_unused_kwargs=True) assert config.output_attention == True assert unused_kwargs == {'foo': False} @@ -31,8 +31,8 @@ def tokenizer(*args, **kwargs): # Using torch.hub ! import torch - tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'tokenizer', 'bert-base-uncased') # Download vocabulary from S3 and cache. - tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'tokenizer', './test/bert_saved_model/') # E.g. tokenizer was saved using `save_pretrained('./test/saved_model/')` + tokenizer = torch.hub.load('huggingface/transformers', 'tokenizer', 'bert-base-uncased') # Download vocabulary from S3 and cache. + tokenizer = torch.hub.load('huggingface/transformers', 'tokenizer', './test/bert_saved_model/') # E.g. tokenizer was saved using `save_pretrained('./test/saved_model/')` """ @@ -45,13 +45,13 @@ def model(*args, **kwargs): # Using torch.hub ! import torch - model = torch.hub.load('huggingface/pytorch-transformers', 'model', 'bert-base-uncased') # Download model and configuration from S3 and cache. - model = torch.hub.load('huggingface/pytorch-transformers', 'model', './test/bert_model/') # E.g. model was saved using `save_pretrained('./test/saved_model/')` - model = torch.hub.load('huggingface/pytorch-transformers', 'model', 'bert-base-uncased', output_attention=True) # Update configuration during loading + model = torch.hub.load('huggingface/transformers', 'model', 'bert-base-uncased') # Download model and configuration from S3 and cache. + model = torch.hub.load('huggingface/transformers', 'model', './test/bert_model/') # E.g. model was saved using `save_pretrained('./test/saved_model/')` + model = torch.hub.load('huggingface/transformers', 'model', 'bert-base-uncased', output_attention=True) # Update configuration during loading assert model.config.output_attention == True # Loading from a TF checkpoint file instead of a PyTorch model (slower) config = AutoConfig.from_json_file('./tf_model/bert_tf_model_config.json') - model = torch.hub.load('huggingface/pytorch-transformers', 'model', './tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config) + model = torch.hub.load('huggingface/transformers', 'model', './tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config) """ @@ -63,13 +63,13 @@ def modelWithLMHead(*args, **kwargs): # Using torch.hub ! import torch - model = torch.hub.load('huggingface/pytorch-transformers', 'modelWithLMHead', 'bert-base-uncased') # Download model and configuration from S3 and cache. - model = torch.hub.load('huggingface/pytorch-transformers', 'modelWithLMHead', './test/bert_model/') # E.g. model was saved using `save_pretrained('./test/saved_model/')` - model = torch.hub.load('huggingface/pytorch-transformers', 'modelWithLMHead', 'bert-base-uncased', output_attention=True) # Update configuration during loading + model = torch.hub.load('huggingface/transformers', 'modelWithLMHead', 'bert-base-uncased') # Download model and configuration from S3 and cache. + model = torch.hub.load('huggingface/transformers', 'modelWithLMHead', './test/bert_model/') # E.g. model was saved using `save_pretrained('./test/saved_model/')` + model = torch.hub.load('huggingface/transformers', 'modelWithLMHead', 'bert-base-uncased', output_attention=True) # Update configuration during loading assert model.config.output_attention == True # Loading from a TF checkpoint file instead of a PyTorch model (slower) config = AutoConfig.from_json_file('./tf_model/bert_tf_model_config.json') - model = torch.hub.load('huggingface/pytorch-transformers', 'modelWithLMHead', './tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config) + model = torch.hub.load('huggingface/transformers', 'modelWithLMHead', './tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config) """ return AutoModelWithLMHead.from_pretrained(*args, **kwargs) @@ -81,13 +81,13 @@ def modelForSequenceClassification(*args, **kwargs): # Using torch.hub ! import torch - model = torch.hub.load('huggingface/pytorch-transformers', 'modelForSequenceClassification', 'bert-base-uncased') # Download model and configuration from S3 and cache. - model = torch.hub.load('huggingface/pytorch-transformers', 'modelForSequenceClassification', './test/bert_model/') # E.g. model was saved using `save_pretrained('./test/saved_model/')` - model = torch.hub.load('huggingface/pytorch-transformers', 'modelForSequenceClassification', 'bert-base-uncased', output_attention=True) # Update configuration during loading + model = torch.hub.load('huggingface/transformers', 'modelForSequenceClassification', 'bert-base-uncased') # Download model and configuration from S3 and cache. + model = torch.hub.load('huggingface/transformers', 'modelForSequenceClassification', './test/bert_model/') # E.g. model was saved using `save_pretrained('./test/saved_model/')` + model = torch.hub.load('huggingface/transformers', 'modelForSequenceClassification', 'bert-base-uncased', output_attention=True) # Update configuration during loading assert model.config.output_attention == True # Loading from a TF checkpoint file instead of a PyTorch model (slower) config = AutoConfig.from_json_file('./tf_model/bert_tf_model_config.json') - model = torch.hub.load('huggingface/pytorch-transformers', 'modelForSequenceClassification', './tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config) + model = torch.hub.load('huggingface/transformers', 'modelForSequenceClassification', './tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config) """ @@ -100,13 +100,13 @@ def modelForQuestionAnswering(*args, **kwargs): # Using torch.hub ! import torch - model = torch.hub.load('huggingface/pytorch-transformers', 'modelForQuestionAnswering', 'bert-base-uncased') # Download model and configuration from S3 and cache. - model = torch.hub.load('huggingface/pytorch-transformers', 'modelForQuestionAnswering', './test/bert_model/') # E.g. model was saved using `save_pretrained('./test/saved_model/')` - model = torch.hub.load('huggingface/pytorch-transformers', 'modelForQuestionAnswering', 'bert-base-uncased', output_attention=True) # Update configuration during loading + model = torch.hub.load('huggingface/transformers', 'modelForQuestionAnswering', 'bert-base-uncased') # Download model and configuration from S3 and cache. + model = torch.hub.load('huggingface/transformers', 'modelForQuestionAnswering', './test/bert_model/') # E.g. model was saved using `save_pretrained('./test/saved_model/')` + model = torch.hub.load('huggingface/transformers', 'modelForQuestionAnswering', 'bert-base-uncased', output_attention=True) # Update configuration during loading assert model.config.output_attention == True # Loading from a TF checkpoint file instead of a PyTorch model (slower) config = AutoConfig.from_json_file('./tf_model/bert_tf_model_config.json') - model = torch.hub.load('huggingface/pytorch-transformers', 'modelForQuestionAnswering', './tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config) + model = torch.hub.load('huggingface/transformers', 'modelForQuestionAnswering', './tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config) """ return AutoModelForQuestionAnswering.from_pretrained(*args, **kwargs) diff --git a/setup.py b/setup.py index 903f1d8cac..34cb89560e 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ To create the package for pypi. (pypi suggest using twine as other methods upload files via plaintext.) Check that you can install it in a virtualenv by running: - pip install -i https://testpypi.python.org/pypi pytorch-transformers + pip install -i https://testpypi.python.org/pypi transformers 6. Upload the final version to actual pypi: twine upload dist/* -r pypi @@ -37,8 +37,8 @@ from io import open from setuptools import find_packages, setup setup( - name="pytorch_transformers", - version="1.2.0", + name="transformers", + version="2.0.0", author="Thomas Wolf, Lysandre Debut, Victor Sanh, Julien Chaumond, Google AI Language Team Authors, Open AI team Authors", author_email="thomas@huggingface.co", description="Repository of pre-trained NLP Transformer models: BERT & RoBERTa, GPT & GPT-2, Transformer-XL, XLNet and XLM", @@ -46,7 +46,7 @@ setup( long_description_content_type="text/markdown", keywords='NLP deep learning transformer pytorch BERT GPT GPT-2 google openai CMU', license='Apache', - url="https://github.com/huggingface/pytorch-transformers", + url="https://github.com/huggingface/transformers", packages=find_packages(exclude=["*.tests", "*.tests.*", "tests.*", "tests"]), install_requires=['numpy', @@ -58,7 +58,7 @@ setup( 'sacremoses'], entry_points={ 'console_scripts': [ - "pytorch_transformers=pytorch_transformers.__main__:main", + "transformers=transformers.__main__:main", ] }, # python_requires='>=3.5.0', diff --git a/pytorch_transformers/__init__.py b/transformers/__init__.py similarity index 98% rename from pytorch_transformers/__init__.py rename to transformers/__init__.py index e355add4ad..39370cb327 100644 --- a/pytorch_transformers/__init__.py +++ b/transformers/__init__.py @@ -1,4 +1,4 @@ -__version__ = "1.2.0" +__version__ = "2.0.0" # Work around to update TensorFlow's absl.logging threshold which alters the # default Python logging output behavior when present. @@ -17,7 +17,7 @@ import logging logger = logging.getLogger(__name__) # pylint: disable=invalid-name # Files and general utilities -from .file_utils import (PYTORCH_TRANSFORMERS_CACHE, PYTORCH_PRETRAINED_BERT_CACHE, +from .file_utils import (TRANSFORMERS_CACHE, PYTORCH_TRANSFORMERS_CACHE, PYTORCH_PRETRAINED_BERT_CACHE, cached_path, add_start_docstrings, add_end_docstrings, WEIGHTS_NAME, TF2_WEIGHTS_NAME, TF_WEIGHTS_NAME, CONFIG_NAME, is_tf_available, is_torch_available) diff --git a/pytorch_transformers/__main__.py b/transformers/__main__.py similarity index 74% rename from pytorch_transformers/__main__.py rename to transformers/__main__.py index ca4936fedf..31dbd24908 100644 --- a/pytorch_transformers/__main__.py +++ b/transformers/__main__.py @@ -5,25 +5,25 @@ def main(): print( "This command line utility let you convert original (author released) model checkpoint to pytorch.\n" "It should be used as one of: \n" - ">> pytorch_transformers bert TF_CHECKPOINT TF_CONFIG PYTORCH_DUMP_OUTPUT, \n" - ">> pytorch_transformers gpt OPENAI_GPT_CHECKPOINT_FOLDER_PATH PYTORCH_DUMP_OUTPUT [OPENAI_GPT_CONFIG], \n" - ">> pytorch_transformers transfo_xl TF_CHECKPOINT_OR_DATASET PYTORCH_DUMP_OUTPUT [TF_CONFIG] or \n" - ">> pytorch_transformers gpt2 TF_CHECKPOINT PYTORCH_DUMP_OUTPUT [GPT2_CONFIG] or \n" - ">> pytorch_transformers xlnet TF_CHECKPOINT TF_CONFIG PYTORCH_DUMP_OUTPUT [FINETUNING_TASK_NAME] or \n" - ">> pytorch_transformers xlm XLM_CHECKPOINT_PATH PYTORCH_DUMP_OUTPUT") + ">> transformers bert TF_CHECKPOINT TF_CONFIG PYTORCH_DUMP_OUTPUT, \n" + ">> transformers gpt OPENAI_GPT_CHECKPOINT_FOLDER_PATH PYTORCH_DUMP_OUTPUT [OPENAI_GPT_CONFIG], \n" + ">> transformers transfo_xl TF_CHECKPOINT_OR_DATASET PYTORCH_DUMP_OUTPUT [TF_CONFIG] or \n" + ">> transformers gpt2 TF_CHECKPOINT PYTORCH_DUMP_OUTPUT [GPT2_CONFIG] or \n" + ">> transformers xlnet TF_CHECKPOINT TF_CONFIG PYTORCH_DUMP_OUTPUT [FINETUNING_TASK_NAME] or \n" + ">> transformers xlm XLM_CHECKPOINT_PATH PYTORCH_DUMP_OUTPUT") else: if sys.argv[1] == "bert": try: from .convert_bert_original_tf_checkpoint_to_pytorch import convert_tf_checkpoint_to_pytorch except ImportError: - print("pytorch_transformers can only be used from the commandline to convert TensorFlow models in PyTorch, " + print("transformers can only be used from the commandline to convert TensorFlow models in PyTorch, " "In that case, it requires TensorFlow to be installed. Please see " "https://www.tensorflow.org/install/ for installation instructions.") raise if len(sys.argv) != 5: # pylint: disable=line-too-long - print("Should be used as `pytorch_transformers bert TF_CHECKPOINT TF_CONFIG PYTORCH_DUMP_OUTPUT`") + print("Should be used as `transformers bert TF_CHECKPOINT TF_CONFIG PYTORCH_DUMP_OUTPUT`") else: PYTORCH_DUMP_OUTPUT = sys.argv.pop() TF_CONFIG = sys.argv.pop() @@ -33,7 +33,7 @@ def main(): from .convert_openai_original_tf_checkpoint_to_pytorch import convert_openai_checkpoint_to_pytorch if len(sys.argv) < 4 or len(sys.argv) > 5: # pylint: disable=line-too-long - print("Should be used as `pytorch_transformers gpt OPENAI_GPT_CHECKPOINT_FOLDER_PATH PYTORCH_DUMP_OUTPUT [OPENAI_GPT_CONFIG]`") + print("Should be used as `transformers gpt OPENAI_GPT_CHECKPOINT_FOLDER_PATH PYTORCH_DUMP_OUTPUT [OPENAI_GPT_CONFIG]`") else: OPENAI_GPT_CHECKPOINT_FOLDER_PATH = sys.argv[2] PYTORCH_DUMP_OUTPUT = sys.argv[3] @@ -48,13 +48,13 @@ def main(): try: from .convert_transfo_xl_original_tf_checkpoint_to_pytorch import convert_transfo_xl_checkpoint_to_pytorch except ImportError: - print("pytorch_transformers can only be used from the commandline to convert TensorFlow models in PyTorch, " + print("transformers can only be used from the commandline to convert TensorFlow models in PyTorch, " "In that case, it requires TensorFlow to be installed. Please see " "https://www.tensorflow.org/install/ for installation instructions.") raise if len(sys.argv) < 4 or len(sys.argv) > 5: # pylint: disable=line-too-long - print("Should be used as `pytorch_transformers transfo_xl TF_CHECKPOINT/TF_DATASET_FILE PYTORCH_DUMP_OUTPUT [TF_CONFIG]`") + print("Should be used as `transformers transfo_xl TF_CHECKPOINT/TF_DATASET_FILE PYTORCH_DUMP_OUTPUT [TF_CONFIG]`") else: if 'ckpt' in sys.argv[2].lower(): TF_CHECKPOINT = sys.argv[2] @@ -72,14 +72,14 @@ def main(): try: from .convert_gpt2_original_tf_checkpoint_to_pytorch import convert_gpt2_checkpoint_to_pytorch except ImportError: - print("pytorch_transformers can only be used from the commandline to convert TensorFlow models in PyTorch, " + print("transformers can only be used from the commandline to convert TensorFlow models in PyTorch, " "In that case, it requires TensorFlow to be installed. Please see " "https://www.tensorflow.org/install/ for installation instructions.") raise if len(sys.argv) < 4 or len(sys.argv) > 5: # pylint: disable=line-too-long - print("Should be used as `pytorch_transformers gpt2 TF_CHECKPOINT PYTORCH_DUMP_OUTPUT [TF_CONFIG]`") + print("Should be used as `transformers gpt2 TF_CHECKPOINT PYTORCH_DUMP_OUTPUT [TF_CONFIG]`") else: TF_CHECKPOINT = sys.argv[2] PYTORCH_DUMP_OUTPUT = sys.argv[3] @@ -92,14 +92,14 @@ def main(): try: from .convert_xlnet_original_tf_checkpoint_to_pytorch import convert_xlnet_checkpoint_to_pytorch except ImportError: - print("pytorch_transformers can only be used from the commandline to convert TensorFlow models in PyTorch, " + print("transformers can only be used from the commandline to convert TensorFlow models in PyTorch, " "In that case, it requires TensorFlow to be installed. Please see " "https://www.tensorflow.org/install/ for installation instructions.") raise if len(sys.argv) < 5 or len(sys.argv) > 6: # pylint: disable=line-too-long - print("Should be used as `pytorch_transformers xlnet TF_CHECKPOINT TF_CONFIG PYTORCH_DUMP_OUTPUT [FINETUNING_TASK_NAME]`") + print("Should be used as `transformers xlnet TF_CHECKPOINT TF_CONFIG PYTORCH_DUMP_OUTPUT [FINETUNING_TASK_NAME]`") else: TF_CHECKPOINT = sys.argv[2] TF_CONFIG = sys.argv[3] @@ -118,7 +118,7 @@ def main(): if len(sys.argv) != 4: # pylint: disable=line-too-long - print("Should be used as `pytorch_transformers xlm XLM_CHECKPOINT_PATH PYTORCH_DUMP_OUTPUT`") + print("Should be used as `transformers xlm XLM_CHECKPOINT_PATH PYTORCH_DUMP_OUTPUT`") else: XLM_CHECKPOINT_PATH = sys.argv[2] PYTORCH_DUMP_OUTPUT = sys.argv[3] diff --git a/pytorch_transformers/configuration_auto.py b/transformers/configuration_auto.py similarity index 97% rename from pytorch_transformers/configuration_auto.py rename to transformers/configuration_auto.py index 9e35f85dc7..74dda59fcf 100644 --- a/pytorch_transformers/configuration_auto.py +++ b/transformers/configuration_auto.py @@ -31,7 +31,7 @@ logger = logging.getLogger(__name__) class AutoConfig(object): - r""":class:`~pytorch_transformers.AutoConfig` is a generic configuration class + r""":class:`~transformers.AutoConfig` is a generic configuration class that will be instantiated as one of the configuration classes of the library when created with the `AutoConfig.from_pretrained(pretrained_model_name_or_path)` class method. @@ -76,7 +76,7 @@ class AutoConfig(object): pretrained_model_name_or_path: either: - a string with the `shortcut name` of a pre-trained model configuration to load from cache or download, e.g.: ``bert-base-uncased``. - - a path to a `directory` containing a configuration file saved using the :func:`~pytorch_transformers.PretrainedConfig.save_pretrained` method, e.g.: ``./my_model_directory/``. + - a path to a `directory` containing a configuration file saved using the :func:`~transformers.PretrainedConfig.save_pretrained` method, e.g.: ``./my_model_directory/``. - a path or url to a saved configuration JSON `file`, e.g.: ``./my_model_directory/configuration.json``. cache_dir: (`optional`) string: diff --git a/pytorch_transformers/configuration_bert.py b/transformers/configuration_bert.py similarity index 98% rename from pytorch_transformers/configuration_bert.py rename to transformers/configuration_bert.py index 00a22770ac..122a2c9aab 100644 --- a/pytorch_transformers/configuration_bert.py +++ b/transformers/configuration_bert.py @@ -45,7 +45,7 @@ BERT_PRETRAINED_CONFIG_ARCHIVE_MAP = { class BertConfig(PretrainedConfig): r""" - :class:`~pytorch_transformers.BertConfig` is the configuration class to store the configuration of a + :class:`~transformers.BertConfig` is the configuration class to store the configuration of a `BertModel`. diff --git a/pytorch_transformers/configuration_distilbert.py b/transformers/configuration_distilbert.py similarity index 100% rename from pytorch_transformers/configuration_distilbert.py rename to transformers/configuration_distilbert.py diff --git a/pytorch_transformers/configuration_gpt2.py b/transformers/configuration_gpt2.py similarity index 100% rename from pytorch_transformers/configuration_gpt2.py rename to transformers/configuration_gpt2.py diff --git a/pytorch_transformers/configuration_openai.py b/transformers/configuration_openai.py similarity index 100% rename from pytorch_transformers/configuration_openai.py rename to transformers/configuration_openai.py diff --git a/pytorch_transformers/configuration_roberta.py b/transformers/configuration_roberta.py similarity index 100% rename from pytorch_transformers/configuration_roberta.py rename to transformers/configuration_roberta.py diff --git a/pytorch_transformers/configuration_transfo_xl.py b/transformers/configuration_transfo_xl.py similarity index 100% rename from pytorch_transformers/configuration_transfo_xl.py rename to transformers/configuration_transfo_xl.py diff --git a/pytorch_transformers/configuration_utils.py b/transformers/configuration_utils.py similarity index 96% rename from pytorch_transformers/configuration_utils.py rename to transformers/configuration_utils.py index 649a94e28c..8a23be4ff6 100644 --- a/pytorch_transformers/configuration_utils.py +++ b/transformers/configuration_utils.py @@ -59,7 +59,7 @@ class PretrainedConfig(object): def save_pretrained(self, save_directory): """ Save a configuration object to the directory `save_directory`, so that it - can be re-loaded using the :func:`~pytorch_transformers.PretrainedConfig.from_pretrained` class method. + can be re-loaded using the :func:`~transformers.PretrainedConfig.from_pretrained` class method. """ assert os.path.isdir(save_directory), "Saving path should be a directory where the model and configuration can be saved" @@ -71,13 +71,13 @@ class PretrainedConfig(object): @classmethod def from_pretrained(cls, pretrained_model_name_or_path, **kwargs): - r""" Instantiate a :class:`~pytorch_transformers.PretrainedConfig` (or a derived class) from a pre-trained model configuration. + r""" Instantiate a :class:`~transformers.PretrainedConfig` (or a derived class) from a pre-trained model configuration. Parameters: pretrained_model_name_or_path: either: - a string with the `shortcut name` of a pre-trained model configuration to load from cache or download, e.g.: ``bert-base-uncased``. - - a path to a `directory` containing a configuration file saved using the :func:`~pytorch_transformers.PretrainedConfig.save_pretrained` method, e.g.: ``./my_model_directory/``. + - a path to a `directory` containing a configuration file saved using the :func:`~transformers.PretrainedConfig.save_pretrained` method, e.g.: ``./my_model_directory/``. - a path or url to a saved configuration JSON `file`, e.g.: ``./my_model_directory/configuration.json``. cache_dir: (`optional`) string: diff --git a/pytorch_transformers/configuration_xlm.py b/transformers/configuration_xlm.py similarity index 100% rename from pytorch_transformers/configuration_xlm.py rename to transformers/configuration_xlm.py diff --git a/pytorch_transformers/configuration_xlnet.py b/transformers/configuration_xlnet.py similarity index 100% rename from pytorch_transformers/configuration_xlnet.py rename to transformers/configuration_xlnet.py diff --git a/pytorch_transformers/convert_bert_original_tf_checkpoint_to_pytorch.py b/transformers/convert_bert_original_tf_checkpoint_to_pytorch.py similarity index 96% rename from pytorch_transformers/convert_bert_original_tf_checkpoint_to_pytorch.py rename to transformers/convert_bert_original_tf_checkpoint_to_pytorch.py index d382d3588e..75808811ef 100755 --- a/pytorch_transformers/convert_bert_original_tf_checkpoint_to_pytorch.py +++ b/transformers/convert_bert_original_tf_checkpoint_to_pytorch.py @@ -21,7 +21,7 @@ from __future__ import print_function import argparse import torch -from pytorch_transformers import BertConfig, BertForPreTraining, load_tf_weights_in_bert +from transformers import BertConfig, BertForPreTraining, load_tf_weights_in_bert import logging logging.basicConfig(level=logging.INFO) diff --git a/pytorch_transformers/convert_bert_pytorch_checkpoint_to_original_tf.py b/transformers/convert_bert_pytorch_checkpoint_to_original_tf.py similarity index 99% rename from pytorch_transformers/convert_bert_pytorch_checkpoint_to_original_tf.py rename to transformers/convert_bert_pytorch_checkpoint_to_original_tf.py index 15fd6bf5ac..35866caac4 100644 --- a/pytorch_transformers/convert_bert_pytorch_checkpoint_to_original_tf.py +++ b/transformers/convert_bert_pytorch_checkpoint_to_original_tf.py @@ -20,7 +20,7 @@ import argparse import torch import numpy as np import tensorflow as tf -from pytorch_transformers import BertModel +from transformers import BertModel def convert_pytorch_checkpoint_to_tf(model:BertModel, ckpt_dir:str, model_name:str): diff --git a/pytorch_transformers/convert_gpt2_original_tf_checkpoint_to_pytorch.py b/transformers/convert_gpt2_original_tf_checkpoint_to_pytorch.py similarity index 98% rename from pytorch_transformers/convert_gpt2_original_tf_checkpoint_to_pytorch.py rename to transformers/convert_gpt2_original_tf_checkpoint_to_pytorch.py index eb5b3009b4..e2328c08ca 100755 --- a/pytorch_transformers/convert_gpt2_original_tf_checkpoint_to_pytorch.py +++ b/transformers/convert_gpt2_original_tf_checkpoint_to_pytorch.py @@ -21,7 +21,7 @@ from io import open import torch -from pytorch_transformers import (CONFIG_NAME, WEIGHTS_NAME, +from transformers import (CONFIG_NAME, WEIGHTS_NAME, GPT2Config, GPT2Model, load_tf_weights_in_gpt2) diff --git a/pytorch_transformers/convert_openai_original_tf_checkpoint_to_pytorch.py b/transformers/convert_openai_original_tf_checkpoint_to_pytorch.py similarity index 98% rename from pytorch_transformers/convert_openai_original_tf_checkpoint_to_pytorch.py rename to transformers/convert_openai_original_tf_checkpoint_to_pytorch.py index 5eecdd9648..13ebecf2fd 100755 --- a/pytorch_transformers/convert_openai_original_tf_checkpoint_to_pytorch.py +++ b/transformers/convert_openai_original_tf_checkpoint_to_pytorch.py @@ -21,7 +21,7 @@ from io import open import torch -from pytorch_transformers import (CONFIG_NAME, WEIGHTS_NAME, +from transformers import (CONFIG_NAME, WEIGHTS_NAME, OpenAIGPTConfig, OpenAIGPTModel, load_tf_weights_in_openai_gpt) diff --git a/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py b/transformers/convert_pytorch_checkpoint_to_tf2.py similarity index 97% rename from pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py rename to transformers/convert_pytorch_checkpoint_to_tf2.py index 6c0043d6a7..c5f7650b50 100644 --- a/pytorch_transformers/convert_pytorch_checkpoint_to_tf2.py +++ b/transformers/convert_pytorch_checkpoint_to_tf2.py @@ -22,9 +22,9 @@ import os import argparse import tensorflow as tf -from pytorch_transformers import is_torch_available, cached_path +from transformers import is_torch_available, cached_path -from pytorch_transformers import (BertConfig, TFBertForPreTraining, TFBertForQuestionAnswering, TFBertForSequenceClassification, load_bert_pt_weights_in_tf2, BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, +from transformers import (BertConfig, TFBertForPreTraining, TFBertForQuestionAnswering, TFBertForSequenceClassification, load_bert_pt_weights_in_tf2, BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, GPT2Config, TFGPT2LMHeadModel, load_gpt2_pt_weights_in_tf2, GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP, XLNetConfig, TFXLNetLMHeadModel, load_xlnet_pt_weights_in_tf2, XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP, XLMConfig, TFXLMWithLMHeadModel, load_xlm_pt_weights_in_tf2, XLM_PRETRAINED_CONFIG_ARCHIVE_MAP, @@ -36,7 +36,7 @@ from pytorch_transformers import (BertConfig, TFBertForPreTraining, TFBertForQue if is_torch_available(): import torch import numpy as np - from pytorch_transformers import (BertForPreTraining, BertForQuestionAnswering, BertForSequenceClassification, BERT_PRETRAINED_MODEL_ARCHIVE_MAP, + from transformers import (BertForPreTraining, BertForQuestionAnswering, BertForSequenceClassification, BERT_PRETRAINED_MODEL_ARCHIVE_MAP, GPT2LMHeadModel, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP, XLNetLMHeadModel, XLNET_PRETRAINED_MODEL_ARCHIVE_MAP, XLMWithLMHeadModel, XLM_PRETRAINED_MODEL_ARCHIVE_MAP, diff --git a/pytorch_transformers/convert_roberta_original_pytorch_checkpoint_to_pytorch.py b/transformers/convert_roberta_original_pytorch_checkpoint_to_pytorch.py similarity index 98% rename from pytorch_transformers/convert_roberta_original_pytorch_checkpoint_to_pytorch.py rename to transformers/convert_roberta_original_pytorch_checkpoint_to_pytorch.py index 9f74254daa..35f01e9907 100644 --- a/pytorch_transformers/convert_roberta_original_pytorch_checkpoint_to_pytorch.py +++ b/transformers/convert_roberta_original_pytorch_checkpoint_to_pytorch.py @@ -23,12 +23,12 @@ import torch from fairseq.models.roberta import RobertaModel as FairseqRobertaModel from fairseq.modules import TransformerSentenceEncoderLayer -from pytorch_transformers import (BertConfig, BertEncoder, +from transformers import (BertConfig, BertEncoder, BertIntermediate, BertLayer, BertModel, BertOutput, BertSelfAttention, BertSelfOutput) -from pytorch_transformers import (RobertaEmbeddings, +from transformers import (RobertaEmbeddings, RobertaForMaskedLM, RobertaForSequenceClassification, RobertaModel) diff --git a/pytorch_transformers/convert_transfo_xl_original_tf_checkpoint_to_pytorch.py b/transformers/convert_transfo_xl_original_tf_checkpoint_to_pytorch.py similarity index 94% rename from pytorch_transformers/convert_transfo_xl_original_tf_checkpoint_to_pytorch.py rename to transformers/convert_transfo_xl_original_tf_checkpoint_to_pytorch.py index b310b73453..a5ff4ed22c 100755 --- a/pytorch_transformers/convert_transfo_xl_original_tf_checkpoint_to_pytorch.py +++ b/transformers/convert_transfo_xl_original_tf_checkpoint_to_pytorch.py @@ -23,12 +23,12 @@ from io import open import torch -import pytorch_transformers.tokenization_transfo_xl as data_utils +import transformers.tokenization_transfo_xl as data_utils -from pytorch_transformers import CONFIG_NAME, WEIGHTS_NAME -from pytorch_transformers import (TransfoXLConfig, TransfoXLLMHeadModel, +from transformers import CONFIG_NAME, WEIGHTS_NAME +from transformers import (TransfoXLConfig, TransfoXLLMHeadModel, load_tf_weights_in_transfo_xl) -from pytorch_transformers.tokenization_transfo_xl import (CORPUS_NAME, VOCAB_FILES_NAMES) +from transformers.tokenization_transfo_xl import (CORPUS_NAME, VOCAB_FILES_NAMES) if sys.version_info[0] == 2: import cPickle as pickle diff --git a/pytorch_transformers/convert_xlm_original_pytorch_checkpoint_to_pytorch.py b/transformers/convert_xlm_original_pytorch_checkpoint_to_pytorch.py similarity index 96% rename from pytorch_transformers/convert_xlm_original_pytorch_checkpoint_to_pytorch.py rename to transformers/convert_xlm_original_pytorch_checkpoint_to_pytorch.py index aecd936920..91133ef56a 100755 --- a/pytorch_transformers/convert_xlm_original_pytorch_checkpoint_to_pytorch.py +++ b/transformers/convert_xlm_original_pytorch_checkpoint_to_pytorch.py @@ -23,8 +23,8 @@ from io import open import torch import numpy -from pytorch_transformers import CONFIG_NAME, WEIGHTS_NAME -from pytorch_transformers.tokenization_xlm import VOCAB_FILES_NAMES +from transformers import CONFIG_NAME, WEIGHTS_NAME +from transformers.tokenization_xlm import VOCAB_FILES_NAMES import logging logging.basicConfig(level=logging.INFO) diff --git a/pytorch_transformers/convert_xlnet_original_tf_checkpoint_to_pytorch.py b/transformers/convert_xlnet_original_tf_checkpoint_to_pytorch.py similarity index 98% rename from pytorch_transformers/convert_xlnet_original_tf_checkpoint_to_pytorch.py rename to transformers/convert_xlnet_original_tf_checkpoint_to_pytorch.py index a36fa514b5..3669d9944c 100755 --- a/pytorch_transformers/convert_xlnet_original_tf_checkpoint_to_pytorch.py +++ b/transformers/convert_xlnet_original_tf_checkpoint_to_pytorch.py @@ -22,7 +22,7 @@ import os import argparse import torch -from pytorch_transformers import (CONFIG_NAME, WEIGHTS_NAME, +from transformers import (CONFIG_NAME, WEIGHTS_NAME, XLNetConfig, XLNetLMHeadModel, XLNetForQuestionAnswering, XLNetForSequenceClassification, diff --git a/pytorch_transformers/data/__init__.py b/transformers/data/__init__.py similarity index 100% rename from pytorch_transformers/data/__init__.py rename to transformers/data/__init__.py diff --git a/pytorch_transformers/data/metrics/__init__.py b/transformers/data/metrics/__init__.py similarity index 100% rename from pytorch_transformers/data/metrics/__init__.py rename to transformers/data/metrics/__init__.py diff --git a/pytorch_transformers/data/processors/__init__.py b/transformers/data/processors/__init__.py similarity index 100% rename from pytorch_transformers/data/processors/__init__.py rename to transformers/data/processors/__init__.py diff --git a/pytorch_transformers/data/processors/glue.py b/transformers/data/processors/glue.py similarity index 100% rename from pytorch_transformers/data/processors/glue.py rename to transformers/data/processors/glue.py diff --git a/pytorch_transformers/data/processors/utils.py b/transformers/data/processors/utils.py similarity index 100% rename from pytorch_transformers/data/processors/utils.py rename to transformers/data/processors/utils.py diff --git a/pytorch_transformers/file_utils.py b/transformers/file_utils.py similarity index 97% rename from pytorch_transformers/file_utils.py rename to transformers/file_utils.py index 90bdb231f1..47fdb6e8ba 100644 --- a/pytorch_transformers/file_utils.py +++ b/transformers/file_utils.py @@ -48,7 +48,7 @@ except ImportError: torch_cache_home = os.path.expanduser( os.getenv('TORCH_HOME', os.path.join( os.getenv('XDG_CACHE_HOME', '~/.cache'), 'torch'))) -default_cache_path = os.path.join(torch_cache_home, 'pytorch_transformers') +default_cache_path = os.path.join(torch_cache_home, 'transformers') try: from urllib.parse import urlparse @@ -65,6 +65,7 @@ except (AttributeError, ImportError): default_cache_path)) PYTORCH_TRANSFORMERS_CACHE = PYTORCH_PRETRAINED_BERT_CACHE # Kept for backward compatibility +TRANSFORMERS_CACHE = PYTORCH_PRETRAINED_BERT_CACHE # Kept for backward compatibility WEIGHTS_NAME = "pytorch_model.bin" TF2_WEIGHTS_NAME = 'tf_model.h5' @@ -131,7 +132,7 @@ def filename_to_url(filename, cache_dir=None): Raise ``EnvironmentError`` if `filename` or its stored metadata do not exist. """ if cache_dir is None: - cache_dir = PYTORCH_TRANSFORMERS_CACHE + cache_dir = TRANSFORMERS_CACHE if sys.version_info[0] == 3 and isinstance(cache_dir, Path): cache_dir = str(cache_dir) @@ -162,7 +163,7 @@ def cached_path(url_or_filename, cache_dir=None, force_download=False, proxies=N force_download: if True, re-dowload the file even if it's already cached in the cache dir. """ if cache_dir is None: - cache_dir = PYTORCH_TRANSFORMERS_CACHE + cache_dir = TRANSFORMERS_CACHE if sys.version_info[0] == 3 and isinstance(url_or_filename, Path): url_or_filename = str(url_or_filename) if sys.version_info[0] == 3 and isinstance(cache_dir, Path): @@ -251,7 +252,7 @@ def get_from_cache(url, cache_dir=None, force_download=False, proxies=None): If it's not there, download it. Then return the path to the cached file. """ if cache_dir is None: - cache_dir = PYTORCH_TRANSFORMERS_CACHE + cache_dir = TRANSFORMERS_CACHE if sys.version_info[0] == 3 and isinstance(cache_dir, Path): cache_dir = str(cache_dir) if sys.version_info[0] == 2 and not isinstance(cache_dir, str): diff --git a/pytorch_transformers/modeling_auto.py b/transformers/modeling_auto.py similarity index 90% rename from pytorch_transformers/modeling_auto.py rename to transformers/modeling_auto.py index 31c8fafaa9..b76a883b19 100644 --- a/pytorch_transformers/modeling_auto.py +++ b/transformers/modeling_auto.py @@ -36,7 +36,7 @@ logger = logging.getLogger(__name__) class AutoModel(object): r""" - :class:`~pytorch_transformers.AutoModel` is a generic model class + :class:`~transformers.AutoModel` is a generic model class that will be instantiated as one of the base model classes of the library when created with the `AutoModel.from_pretrained(pretrained_model_name_or_path)` class method. @@ -84,23 +84,23 @@ class AutoModel(object): pretrained_model_name_or_path: either: - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. - - a path to a `directory` containing model weights saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. + - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. - a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. model_args: (`optional`) Sequence of positional arguments: All remaning positional arguments will be passed to the underlying model's ``__init__`` method - config: (`optional`) instance of a class derived from :class:`~pytorch_transformers.PretrainedConfig`: + config: (`optional`) instance of a class derived from :class:`~transformers.PretrainedConfig`: Configuration for the model to use instead of an automatically loaded configuation. Configuration can be automatically loaded when: - the model is a model provided by the library (loaded with the ``shortcut-name`` string of a pretrained model), or - - the model was saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained` and is reloaded by suppling the save directory. + - the model was saved using :func:`~transformers.PreTrainedModel.save_pretrained` and is reloaded by suppling the save directory. - the model is loaded by suppling a local directory as ``pretrained_model_name_or_path`` and a configuration JSON file named `config.json` is found in the directory. state_dict: (`optional`) dict: an optional state dictionnary for the model to use instead of a state dictionary loaded from saved weights file. This option can be used if you want to create a model from a pretrained configuration but load your own weights. - In this case though, you should check if using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained` and :func:`~pytorch_transformers.PreTrainedModel.from_pretrained` is not a simpler option. + In this case though, you should check if using :func:`~transformers.PreTrainedModel.save_pretrained` and :func:`~transformers.PreTrainedModel.from_pretrained` is not a simpler option. cache_dir: (`optional`) string: Path to a directory in which a downloaded pre-trained model @@ -120,7 +120,7 @@ class AutoModel(object): Can be used to update the configuration object (after it being loaded) and initiate the model. (e.g. ``output_attention=True``). Behave differently depending on whether a `config` is provided or automatically loaded: - If a configuration is provided with ``config``, ``**kwargs`` will be directly passed to the underlying model's ``__init__`` method (we assume all relevant updates to the configuration have already been done) - - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~pytorch_transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function. + - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function. Examples:: @@ -157,7 +157,7 @@ class AutoModel(object): class AutoModelWithLMHead(object): r""" - :class:`~pytorch_transformers.AutoModelWithLMHead` is a generic model class + :class:`~transformers.AutoModelWithLMHead` is a generic model class that will be instantiated as one of the language modeling model classes of the library when created with the `AutoModelWithLMHead.from_pretrained(pretrained_model_name_or_path)` class method. @@ -208,23 +208,23 @@ class AutoModelWithLMHead(object): pretrained_model_name_or_path: either: - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. - - a path to a `directory` containing model weights saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. + - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. - a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. model_args: (`optional`) Sequence of positional arguments: All remaning positional arguments will be passed to the underlying model's ``__init__`` method - config: (`optional`) instance of a class derived from :class:`~pytorch_transformers.PretrainedConfig`: + config: (`optional`) instance of a class derived from :class:`~transformers.PretrainedConfig`: Configuration for the model to use instead of an automatically loaded configuation. Configuration can be automatically loaded when: - the model is a model provided by the library (loaded with the ``shortcut-name`` string of a pretrained model), or - - the model was saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained` and is reloaded by suppling the save directory. + - the model was saved using :func:`~transformers.PreTrainedModel.save_pretrained` and is reloaded by suppling the save directory. - the model is loaded by suppling a local directory as ``pretrained_model_name_or_path`` and a configuration JSON file named `config.json` is found in the directory. state_dict: (`optional`) dict: an optional state dictionnary for the model to use instead of a state dictionary loaded from saved weights file. This option can be used if you want to create a model from a pretrained configuration but load your own weights. - In this case though, you should check if using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained` and :func:`~pytorch_transformers.PreTrainedModel.from_pretrained` is not a simpler option. + In this case though, you should check if using :func:`~transformers.PreTrainedModel.save_pretrained` and :func:`~transformers.PreTrainedModel.from_pretrained` is not a simpler option. cache_dir: (`optional`) string: Path to a directory in which a downloaded pre-trained model @@ -244,7 +244,7 @@ class AutoModelWithLMHead(object): Can be used to update the configuration object (after it being loaded) and initiate the model. (e.g. ``output_attention=True``). Behave differently depending on whether a `config` is provided or automatically loaded: - If a configuration is provided with ``config``, ``**kwargs`` will be directly passed to the underlying model's ``__init__`` method (we assume all relevant updates to the configuration have already been done) - - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~pytorch_transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function. + - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function. Examples:: @@ -281,7 +281,7 @@ class AutoModelWithLMHead(object): class AutoModelForSequenceClassification(object): r""" - :class:`~pytorch_transformers.AutoModelForSequenceClassification` is a generic model class + :class:`~transformers.AutoModelForSequenceClassification` is a generic model class that will be instantiated as one of the sequence classification model classes of the library when created with the `AutoModelForSequenceClassification.from_pretrained(pretrained_model_name_or_path)` class method. @@ -326,23 +326,23 @@ class AutoModelForSequenceClassification(object): pretrained_model_name_or_path: either: - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. - - a path to a `directory` containing model weights saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. + - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. - a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. model_args: (`optional`) Sequence of positional arguments: All remaning positional arguments will be passed to the underlying model's ``__init__`` method - config: (`optional`) instance of a class derived from :class:`~pytorch_transformers.PretrainedConfig`: + config: (`optional`) instance of a class derived from :class:`~transformers.PretrainedConfig`: Configuration for the model to use instead of an automatically loaded configuation. Configuration can be automatically loaded when: - the model is a model provided by the library (loaded with the ``shortcut-name`` string of a pretrained model), or - - the model was saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained` and is reloaded by suppling the save directory. + - the model was saved using :func:`~transformers.PreTrainedModel.save_pretrained` and is reloaded by suppling the save directory. - the model is loaded by suppling a local directory as ``pretrained_model_name_or_path`` and a configuration JSON file named `config.json` is found in the directory. state_dict: (`optional`) dict: an optional state dictionnary for the model to use instead of a state dictionary loaded from saved weights file. This option can be used if you want to create a model from a pretrained configuration but load your own weights. - In this case though, you should check if using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained` and :func:`~pytorch_transformers.PreTrainedModel.from_pretrained` is not a simpler option. + In this case though, you should check if using :func:`~transformers.PreTrainedModel.save_pretrained` and :func:`~transformers.PreTrainedModel.from_pretrained` is not a simpler option. cache_dir: (`optional`) string: Path to a directory in which a downloaded pre-trained model @@ -362,7 +362,7 @@ class AutoModelForSequenceClassification(object): Can be used to update the configuration object (after it being loaded) and initiate the model. (e.g. ``output_attention=True``). Behave differently depending on whether a `config` is provided or automatically loaded: - If a configuration is provided with ``config``, ``**kwargs`` will be directly passed to the underlying model's ``__init__`` method (we assume all relevant updates to the configuration have already been done) - - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~pytorch_transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function. + - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function. Examples:: @@ -392,7 +392,7 @@ class AutoModelForSequenceClassification(object): class AutoModelForQuestionAnswering(object): r""" - :class:`~pytorch_transformers.AutoModelForQuestionAnswering` is a generic model class + :class:`~transformers.AutoModelForQuestionAnswering` is a generic model class that will be instantiated as one of the question answering model classes of the library when created with the `AutoModelForQuestionAnswering.from_pretrained(pretrained_model_name_or_path)` class method. @@ -435,23 +435,23 @@ class AutoModelForQuestionAnswering(object): pretrained_model_name_or_path: either: - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. - - a path to a `directory` containing model weights saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. + - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. - a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. model_args: (`optional`) Sequence of positional arguments: All remaning positional arguments will be passed to the underlying model's ``__init__`` method - config: (`optional`) instance of a class derived from :class:`~pytorch_transformers.PretrainedConfig`: + config: (`optional`) instance of a class derived from :class:`~transformers.PretrainedConfig`: Configuration for the model to use instead of an automatically loaded configuation. Configuration can be automatically loaded when: - the model is a model provided by the library (loaded with the ``shortcut-name`` string of a pretrained model), or - - the model was saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained` and is reloaded by suppling the save directory. + - the model was saved using :func:`~transformers.PreTrainedModel.save_pretrained` and is reloaded by suppling the save directory. - the model is loaded by suppling a local directory as ``pretrained_model_name_or_path`` and a configuration JSON file named `config.json` is found in the directory. state_dict: (`optional`) dict: an optional state dictionnary for the model to use instead of a state dictionary loaded from saved weights file. This option can be used if you want to create a model from a pretrained configuration but load your own weights. - In this case though, you should check if using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained` and :func:`~pytorch_transformers.PreTrainedModel.from_pretrained` is not a simpler option. + In this case though, you should check if using :func:`~transformers.PreTrainedModel.save_pretrained` and :func:`~transformers.PreTrainedModel.from_pretrained` is not a simpler option. cache_dir: (`optional`) string: Path to a directory in which a downloaded pre-trained model @@ -471,7 +471,7 @@ class AutoModelForQuestionAnswering(object): Can be used to update the configuration object (after it being loaded) and initiate the model. (e.g. ``output_attention=True``). Behave differently depending on whether a `config` is provided or automatically loaded: - If a configuration is provided with ``config``, ``**kwargs`` will be directly passed to the underlying model's ``__init__`` method (we assume all relevant updates to the configuration have already been done) - - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~pytorch_transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function. + - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function. Examples:: diff --git a/pytorch_transformers/modeling_bert.py b/transformers/modeling_bert.py similarity index 99% rename from pytorch_transformers/modeling_bert.py rename to transformers/modeling_bert.py index 4a61a01641..132d0de4ea 100644 --- a/pytorch_transformers/modeling_bert.py +++ b/transformers/modeling_bert.py @@ -486,9 +486,9 @@ BERT_START_DOCSTRING = r""" The BERT model was proposed in https://pytorch.org/docs/stable/nn.html#module Parameters: - config (:class:`~pytorch_transformers.BertConfig`): Model configuration class with all the parameters of the model. + config (:class:`~transformers.BertConfig`): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. - Check out the :meth:`~pytorch_transformers.PreTrainedModel.from_pretrained` method to load the model weights. + Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights. """ BERT_INPUTS_DOCSTRING = r""" @@ -512,9 +512,9 @@ BERT_INPUTS_DOCSTRING = r""" Bert is a model with absolute position embeddings so it's usually advised to pad the inputs on the right rather than the left. - Indices can be obtained using :class:`pytorch_transformers.BertTokenizer`. - See :func:`pytorch_transformers.PreTrainedTokenizer.encode` and - :func:`pytorch_transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + Indices can be obtained using :class:`transformers.BertTokenizer`. + See :func:`transformers.PreTrainedTokenizer.encode` and + :func:`transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``: Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``: diff --git a/pytorch_transformers/modeling_distilbert.py b/transformers/modeling_distilbert.py similarity index 99% rename from pytorch_transformers/modeling_distilbert.py rename to transformers/modeling_distilbert.py index c5cc44be75..2425ab5f47 100644 --- a/pytorch_transformers/modeling_distilbert.py +++ b/transformers/modeling_distilbert.py @@ -372,9 +372,9 @@ DISTILBERT_START_DOCSTRING = r""" https://medium.com/huggingface/distilbert-8cf3380435b5 Parameters: - config (:class:`~pytorch_transformers.DistilBertConfig`): Model configuration class with all the parameters of the model. + config (:class:`~transformers.DistilBertConfig`): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. - Check out the :meth:`~pytorch_transformers.PreTrainedModel.from_pretrained` method to load the model weights. + Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights. """ DISTILBERT_INPUTS_DOCSTRING = r""" diff --git a/pytorch_transformers/modeling_gpt2.py b/transformers/modeling_gpt2.py similarity index 98% rename from pytorch_transformers/modeling_gpt2.py rename to transformers/modeling_gpt2.py index ee246f1731..bc85224022 100644 --- a/pytorch_transformers/modeling_gpt2.py +++ b/transformers/modeling_gpt2.py @@ -280,9 +280,9 @@ GPT2_START_DOCSTRING = r""" OpenAI GPT-2 model was proposed in https://pytorch.org/docs/stable/nn.html#module Parameters: - config (:class:`~pytorch_transformers.GPT2Config`): Model configuration class with all the parameters of the model. + config (:class:`~transformers.GPT2Config`): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. - Check out the :meth:`~pytorch_transformers.PreTrainedModel.from_pretrained` method to load the model weights. + Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights. """ GPT2_INPUTS_DOCSTRING = r""" Inputs: @@ -290,9 +290,9 @@ GPT2_INPUTS_DOCSTRING = r""" Inputs: Indices of input sequence tokens in the vocabulary. GPT-2 is a model with absolute position embeddings so it's usually advised to pad the inputs on the right rather than the left. - Indices can be obtained using :class:`pytorch_transformers.GPT2Tokenizer`. - See :func:`pytorch_transformers.PreTrainedTokenizer.encode` and - :func:`pytorch_transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + Indices can be obtained using :class:`transformers.GPT2Tokenizer`. + See :func:`transformers.PreTrainedTokenizer.encode` and + :func:`transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. **past**: list of ``torch.FloatTensor`` (one for each layer): that contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model @@ -493,7 +493,7 @@ class GPT2LMHeadModel(GPT2PreTrainedModel): Examples:: import torch - from pytorch_transformers import GPT2Tokenizer, GPT2LMHeadModel + from transformers import GPT2Tokenizer, GPT2LMHeadModel tokenizer = GPT2Tokenizer.from_pretrained('gpt2') model = GPT2LMHeadModel.from_pretrained('gpt2') @@ -589,7 +589,7 @@ class GPT2DoubleHeadsModel(GPT2PreTrainedModel): Examples:: import torch - from pytorch_transformers import GPT2Tokenizer, GPT2DoubleHeadsModel + from transformers import GPT2Tokenizer, GPT2DoubleHeadsModel tokenizer = GPT2Tokenizer.from_pretrained('gpt2') model = GPT2DoubleHeadsModel.from_pretrained('gpt2') diff --git a/pytorch_transformers/modeling_openai.py b/transformers/modeling_openai.py similarity index 98% rename from pytorch_transformers/modeling_openai.py rename to transformers/modeling_openai.py index 4b02baf2f4..2827bf11e5 100644 --- a/pytorch_transformers/modeling_openai.py +++ b/transformers/modeling_openai.py @@ -294,9 +294,9 @@ OPENAI_GPT_START_DOCSTRING = r""" OpenAI GPT model was proposed in https://pytorch.org/docs/stable/nn.html#module Parameters: - config (:class:`~pytorch_transformers.OpenAIGPTConfig`): Model configuration class with all the parameters of the model. + config (:class:`~transformers.OpenAIGPTConfig`): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. - Check out the :meth:`~pytorch_transformers.PreTrainedModel.from_pretrained` method to load the model weights. + Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights. """ OPENAI_GPT_INPUTS_DOCSTRING = r""" Inputs: @@ -304,9 +304,9 @@ OPENAI_GPT_INPUTS_DOCSTRING = r""" Inputs: Indices of input sequence tokens in the vocabulary. GPT is a model with absolute position embeddings so it's usually advised to pad the inputs on the right rather than the left. - Indices can be obtained using :class:`pytorch_transformers.BPT2Tokenizer`. - See :func:`pytorch_transformers.PreTrainedTokenizer.encode` and - :func:`pytorch_transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + Indices can be obtained using :class:`transformers.BPT2Tokenizer`. + See :func:`transformers.PreTrainedTokenizer.encode` and + :func:`transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``: Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``: diff --git a/pytorch_transformers/modeling_roberta.py b/transformers/modeling_roberta.py similarity index 97% rename from pytorch_transformers/modeling_roberta.py rename to transformers/modeling_roberta.py index 9b30bcd4be..04ffbecc16 100644 --- a/pytorch_transformers/modeling_roberta.py +++ b/transformers/modeling_roberta.py @@ -77,9 +77,9 @@ ROBERTA_START_DOCSTRING = r""" The RoBERTa model was proposed in https://pytorch.org/docs/stable/nn.html#module Parameters: - config (:class:`~pytorch_transformers.RobertaConfig`): Model configuration class with all the parameters of the + config (:class:`~transformers.RobertaConfig`): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. - Check out the :meth:`~pytorch_transformers.PreTrainedModel.from_pretrained` method to load the model weights. + Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights. """ ROBERTA_INPUTS_DOCSTRING = r""" @@ -102,8 +102,8 @@ ROBERTA_INPUTS_DOCSTRING = r""" RoBERTa is a model with absolute position embeddings so it's usually advised to pad the inputs on the right rather than the left. - See :func:`pytorch_transformers.PreTrainedTokenizer.encode` and - :func:`pytorch_transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + See :func:`transformers.PreTrainedTokenizer.encode` and + :func:`transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``: Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``: @@ -361,9 +361,9 @@ class RobertaForMultipleChoice(BertPreTrainedModel): ``token_type_ids: 0 0 0 0 0 0 0`` - Indices can be obtained using :class:`pytorch_transformers.BertTokenizer`. - See :func:`pytorch_transformers.PreTrainedTokenizer.encode` and - :func:`pytorch_transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + Indices can be obtained using :class:`transformers.BertTokenizer`. + See :func:`transformers.PreTrainedTokenizer.encode` and + :func:`transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. **token_type_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, num_choices, sequence_length)``: Segment token indices to indicate first and second portions of the inputs. The second dimension of the input (`num_choices`) indicates the number of choices to score. diff --git a/pytorch_transformers/modeling_tf_auto.py b/transformers/modeling_tf_auto.py similarity index 89% rename from pytorch_transformers/modeling_tf_auto.py rename to transformers/modeling_tf_auto.py index 3d3cc6306e..a8f1de047f 100644 --- a/pytorch_transformers/modeling_tf_auto.py +++ b/transformers/modeling_tf_auto.py @@ -34,7 +34,7 @@ logger = logging.getLogger(__name__) class TFAutoModel(object): r""" - :class:`~pytorch_transformers.TFAutoModel` is a generic model class + :class:`~transformers.TFAutoModel` is a generic model class that will be instantiated as one of the base model classes of the library when created with the `TFAutoModel.from_pretrained(pretrained_model_name_or_path)` class method. @@ -79,7 +79,7 @@ class TFAutoModel(object): pretrained_model_name_or_path: either: - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. - - a path to a `directory` containing model weights saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. + - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. - a path or url to a `PyTorch, TF 1.X or TF 2.0 checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In the case of a PyTorch checkpoint, ``from_pt`` should be set to True and a configuration object should be provided as ``config`` argument. from_pt: (`Optional`) Boolean @@ -88,17 +88,17 @@ class TFAutoModel(object): model_args: (`optional`) Sequence of positional arguments: All remaning positional arguments will be passed to the underlying model's ``__init__`` method - config: (`optional`) instance of a class derived from :class:`~pytorch_transformers.PretrainedConfig`: + config: (`optional`) instance of a class derived from :class:`~transformers.PretrainedConfig`: Configuration for the model to use instead of an automatically loaded configuation. Configuration can be automatically loaded when: - the model is a model provided by the library (loaded with the ``shortcut-name`` string of a pretrained model), or - - the model was saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained` and is reloaded by suppling the save directory. + - the model was saved using :func:`~transformers.PreTrainedModel.save_pretrained` and is reloaded by suppling the save directory. - the model is loaded by suppling a local directory as ``pretrained_model_name_or_path`` and a configuration JSON file named `config.json` is found in the directory. state_dict: (`optional`) dict: an optional state dictionnary for the model to use instead of a state dictionary loaded from saved weights file. This option can be used if you want to create a model from a pretrained configuration but load your own weights. - In this case though, you should check if using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained` and :func:`~pytorch_transformers.PreTrainedModel.from_pretrained` is not a simpler option. + In this case though, you should check if using :func:`~transformers.PreTrainedModel.save_pretrained` and :func:`~transformers.PreTrainedModel.from_pretrained` is not a simpler option. cache_dir: (`optional`) string: Path to a directory in which a downloaded pre-trained model @@ -118,7 +118,7 @@ class TFAutoModel(object): Can be used to update the configuration object (after it being loaded) and initiate the model. (e.g. ``output_attention=True``). Behave differently depending on whether a `config` is provided or automatically loaded: - If a configuration is provided with ``config``, ``**kwargs`` will be directly passed to the underlying model's ``__init__`` method (we assume all relevant updates to the configuration have already been done) - - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~pytorch_transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function. + - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function. Examples:: @@ -155,7 +155,7 @@ class TFAutoModel(object): class TFAutoModelWithLMHead(object): r""" - :class:`~pytorch_transformers.TFAutoModelWithLMHead` is a generic model class + :class:`~transformers.TFAutoModelWithLMHead` is a generic model class that will be instantiated as one of the language modeling model classes of the library when created with the `TFAutoModelWithLMHead.from_pretrained(pretrained_model_name_or_path)` class method. @@ -203,7 +203,7 @@ class TFAutoModelWithLMHead(object): pretrained_model_name_or_path: either: - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. - - a path to a `directory` containing model weights saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. + - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. - a path or url to a `PyTorch, TF 1.X or TF 2.0 checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In the case of a PyTorch checkpoint, ``from_pt`` should be set to True and a configuration object should be provided as ``config`` argument. from_pt: (`Optional`) Boolean @@ -212,17 +212,17 @@ class TFAutoModelWithLMHead(object): model_args: (`optional`) Sequence of positional arguments: All remaning positional arguments will be passed to the underlying model's ``__init__`` method - config: (`optional`) instance of a class derived from :class:`~pytorch_transformers.PretrainedConfig`: + config: (`optional`) instance of a class derived from :class:`~transformers.PretrainedConfig`: Configuration for the model to use instead of an automatically loaded configuation. Configuration can be automatically loaded when: - the model is a model provided by the library (loaded with the ``shortcut-name`` string of a pretrained model), or - - the model was saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained` and is reloaded by suppling the save directory. + - the model was saved using :func:`~transformers.PreTrainedModel.save_pretrained` and is reloaded by suppling the save directory. - the model is loaded by suppling a local directory as ``pretrained_model_name_or_path`` and a configuration JSON file named `config.json` is found in the directory. state_dict: (`optional`) dict: an optional state dictionnary for the model to use instead of a state dictionary loaded from saved weights file. This option can be used if you want to create a model from a pretrained configuration but load your own weights. - In this case though, you should check if using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained` and :func:`~pytorch_transformers.PreTrainedModel.from_pretrained` is not a simpler option. + In this case though, you should check if using :func:`~transformers.PreTrainedModel.save_pretrained` and :func:`~transformers.PreTrainedModel.from_pretrained` is not a simpler option. cache_dir: (`optional`) string: Path to a directory in which a downloaded pre-trained model @@ -242,7 +242,7 @@ class TFAutoModelWithLMHead(object): Can be used to update the configuration object (after it being loaded) and initiate the model. (e.g. ``output_attention=True``). Behave differently depending on whether a `config` is provided or automatically loaded: - If a configuration is provided with ``config``, ``**kwargs`` will be directly passed to the underlying model's ``__init__`` method (we assume all relevant updates to the configuration have already been done) - - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~pytorch_transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function. + - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function. Examples:: @@ -279,7 +279,7 @@ class TFAutoModelWithLMHead(object): class TFAutoModelForSequenceClassification(object): r""" - :class:`~pytorch_transformers.TFAutoModelForSequenceClassification` is a generic model class + :class:`~transformers.TFAutoModelForSequenceClassification` is a generic model class that will be instantiated as one of the sequence classification model classes of the library when created with the `TFAutoModelForSequenceClassification.from_pretrained(pretrained_model_name_or_path)` class method. @@ -324,7 +324,7 @@ class TFAutoModelForSequenceClassification(object): pretrained_model_name_or_path: either: - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. - - a path to a `directory` containing model weights saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. + - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. - a path or url to a `PyTorch, TF 1.X or TF 2.0 checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In the case of a PyTorch checkpoint, ``from_pt`` should be set to True and a configuration object should be provided as ``config`` argument. from_pt: (`Optional`) Boolean @@ -333,17 +333,17 @@ class TFAutoModelForSequenceClassification(object): model_args: (`optional`) Sequence of positional arguments: All remaning positional arguments will be passed to the underlying model's ``__init__`` method - config: (`optional`) instance of a class derived from :class:`~pytorch_transformers.PretrainedConfig`: + config: (`optional`) instance of a class derived from :class:`~transformers.PretrainedConfig`: Configuration for the model to use instead of an automatically loaded configuation. Configuration can be automatically loaded when: - the model is a model provided by the library (loaded with the ``shortcut-name`` string of a pretrained model), or - - the model was saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained` and is reloaded by suppling the save directory. + - the model was saved using :func:`~transformers.PreTrainedModel.save_pretrained` and is reloaded by suppling the save directory. - the model is loaded by suppling a local directory as ``pretrained_model_name_or_path`` and a configuration JSON file named `config.json` is found in the directory. state_dict: (`optional`) dict: an optional state dictionnary for the model to use instead of a state dictionary loaded from saved weights file. This option can be used if you want to create a model from a pretrained configuration but load your own weights. - In this case though, you should check if using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained` and :func:`~pytorch_transformers.PreTrainedModel.from_pretrained` is not a simpler option. + In this case though, you should check if using :func:`~transformers.PreTrainedModel.save_pretrained` and :func:`~transformers.PreTrainedModel.from_pretrained` is not a simpler option. cache_dir: (`optional`) string: Path to a directory in which a downloaded pre-trained model @@ -363,7 +363,7 @@ class TFAutoModelForSequenceClassification(object): Can be used to update the configuration object (after it being loaded) and initiate the model. (e.g. ``output_attention=True``). Behave differently depending on whether a `config` is provided or automatically loaded: - If a configuration is provided with ``config``, ``**kwargs`` will be directly passed to the underlying model's ``__init__`` method (we assume all relevant updates to the configuration have already been done) - - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~pytorch_transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function. + - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function. Examples:: @@ -393,7 +393,7 @@ class TFAutoModelForSequenceClassification(object): class TFAutoModelForQuestionAnswering(object): r""" - :class:`~pytorch_transformers.TFAutoModelForQuestionAnswering` is a generic model class + :class:`~transformers.TFAutoModelForQuestionAnswering` is a generic model class that will be instantiated as one of the question answering model classes of the library when created with the `TFAutoModelForQuestionAnswering.from_pretrained(pretrained_model_name_or_path)` class method. @@ -436,7 +436,7 @@ class TFAutoModelForQuestionAnswering(object): pretrained_model_name_or_path: either: - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. - - a path to a `directory` containing model weights saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. + - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. - a path or url to a `PyTorch, TF 1.X or TF 2.0 checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In the case of a PyTorch checkpoint, ``from_pt`` should be set to True and a configuration object should be provided as ``config`` argument. from_pt: (`Optional`) Boolean @@ -445,17 +445,17 @@ class TFAutoModelForQuestionAnswering(object): model_args: (`optional`) Sequence of positional arguments: All remaning positional arguments will be passed to the underlying model's ``__init__`` method - config: (`optional`) instance of a class derived from :class:`~pytorch_transformers.PretrainedConfig`: + config: (`optional`) instance of a class derived from :class:`~transformers.PretrainedConfig`: Configuration for the model to use instead of an automatically loaded configuation. Configuration can be automatically loaded when: - the model is a model provided by the library (loaded with the ``shortcut-name`` string of a pretrained model), or - - the model was saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained` and is reloaded by suppling the save directory. + - the model was saved using :func:`~transformers.PreTrainedModel.save_pretrained` and is reloaded by suppling the save directory. - the model is loaded by suppling a local directory as ``pretrained_model_name_or_path`` and a configuration JSON file named `config.json` is found in the directory. state_dict: (`optional`) dict: an optional state dictionnary for the model to use instead of a state dictionary loaded from saved weights file. This option can be used if you want to create a model from a pretrained configuration but load your own weights. - In this case though, you should check if using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained` and :func:`~pytorch_transformers.PreTrainedModel.from_pretrained` is not a simpler option. + In this case though, you should check if using :func:`~transformers.PreTrainedModel.save_pretrained` and :func:`~transformers.PreTrainedModel.from_pretrained` is not a simpler option. cache_dir: (`optional`) string: Path to a directory in which a downloaded pre-trained model @@ -475,7 +475,7 @@ class TFAutoModelForQuestionAnswering(object): Can be used to update the configuration object (after it being loaded) and initiate the model. (e.g. ``output_attention=True``). Behave differently depending on whether a `config` is provided or automatically loaded: - If a configuration is provided with ``config``, ``**kwargs`` will be directly passed to the underlying model's ``__init__`` method (we assume all relevant updates to the configuration have already been done) - - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~pytorch_transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function. + - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function. Examples:: diff --git a/pytorch_transformers/modeling_tf_bert.py b/transformers/modeling_tf_bert.py similarity index 97% rename from pytorch_transformers/modeling_tf_bert.py rename to transformers/modeling_tf_bert.py index 51d5155d4b..d763ca991e 100644 --- a/pytorch_transformers/modeling_tf_bert.py +++ b/transformers/modeling_tf_bert.py @@ -581,9 +581,9 @@ BERT_START_DOCSTRING = r""" The BERT model was proposed in `model({'input_ids': input_ids, 'token_type_ids': token_type_ids})` Parameters: - config (:class:`~pytorch_transformers.BertConfig`): Model configuration class with all the parameters of the model. + config (:class:`~transformers.BertConfig`): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. - Check out the :meth:`~pytorch_transformers.PreTrainedModel.from_pretrained` method to load the model weights. + Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights. """ BERT_INPUTS_DOCSTRING = r""" @@ -607,9 +607,9 @@ BERT_INPUTS_DOCSTRING = r""" Bert is a model with absolute position embeddings so it's usually advised to pad the inputs on the right rather than the left. - Indices can be obtained using :class:`pytorch_transformers.BertTokenizer`. - See :func:`pytorch_transformers.PreTrainedTokenizer.encode` and - :func:`pytorch_transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + Indices can be obtained using :class:`transformers.BertTokenizer`. + See :func:`transformers.PreTrainedTokenizer.encode` and + :func:`transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. **attention_mask**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``: @@ -653,7 +653,7 @@ class TFBertModel(TFBertPreTrainedModel): Examples:: import tensorflow as tf - from pytorch_transformers import BertTokenizer, TFBertModel + from transformers import BertTokenizer, TFBertModel tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = TFBertModel.from_pretrained('bert-base-uncased') @@ -692,7 +692,7 @@ class TFBertForPreTraining(TFBertPreTrainedModel): Examples:: import tensorflow as tf - from pytorch_transformers import BertTokenizer, TFBertForPreTraining + from transformers import BertTokenizer, TFBertForPreTraining tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = TFBertForPreTraining.from_pretrained('bert-base-uncased') @@ -738,7 +738,7 @@ class TFBertForMaskedLM(TFBertPreTrainedModel): Examples:: import tensorflow as tf - from pytorch_transformers import BertTokenizer, TFBertForMaskedLM + from transformers import BertTokenizer, TFBertForMaskedLM tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = TFBertForMaskedLM.from_pretrained('bert-base-uncased') @@ -782,7 +782,7 @@ class TFBertForNextSentencePrediction(TFBertPreTrainedModel): Examples:: import tensorflow as tf - from pytorch_transformers import BertTokenizer, TFBertForNextSentencePrediction + from transformers import BertTokenizer, TFBertForNextSentencePrediction tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = TFBertForNextSentencePrediction.from_pretrained('bert-base-uncased') @@ -827,7 +827,7 @@ class TFBertForSequenceClassification(TFBertPreTrainedModel): Examples:: import tensorflow as tf - from pytorch_transformers import BertTokenizer, TFBertForSequenceClassification + from transformers import BertTokenizer, TFBertForSequenceClassification tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = TFBertForSequenceClassification.from_pretrained('bert-base-uncased') @@ -879,7 +879,7 @@ class TFBertForMultipleChoice(TFBertPreTrainedModel): Examples:: import tensorflow as tf - from pytorch_transformers import BertTokenizer, TFBertForMultipleChoice + from transformers import BertTokenizer, TFBertForMultipleChoice tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = TFBertForMultipleChoice.from_pretrained('bert-base-uncased') @@ -958,7 +958,7 @@ class TFBertForTokenClassification(TFBertPreTrainedModel): Examples:: import tensorflow as tf - from pytorch_transformers import BertTokenizer, TFBertForTokenClassification + from transformers import BertTokenizer, TFBertForTokenClassification tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = TFBertForTokenClassification.from_pretrained('bert-base-uncased') @@ -1011,7 +1011,7 @@ class TFBertForQuestionAnswering(TFBertPreTrainedModel): Examples:: import tensorflow as tf - from pytorch_transformers import BertTokenizer, TFBertForQuestionAnswering + from transformers import BertTokenizer, TFBertForQuestionAnswering tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = TFBertForQuestionAnswering.from_pretrained('bert-base-uncased') diff --git a/pytorch_transformers/modeling_tf_distilbert.py b/transformers/modeling_tf_distilbert.py similarity index 98% rename from pytorch_transformers/modeling_tf_distilbert.py rename to transformers/modeling_tf_distilbert.py index 2ec73fc43d..2a917a30a4 100644 --- a/pytorch_transformers/modeling_tf_distilbert.py +++ b/transformers/modeling_tf_distilbert.py @@ -500,9 +500,9 @@ DISTILBERT_START_DOCSTRING = r""" `model({'input_ids': input_ids, 'token_type_ids': token_type_ids})` Parameters: - config (:class:`~pytorch_transformers.DistilBertConfig`): Model configuration class with all the parameters of the model. + config (:class:`~transformers.DistilBertConfig`): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. - Check out the :meth:`~pytorch_transformers.PreTrainedModel.from_pretrained` method to load the model weights. + Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights. """ DISTILBERT_INPUTS_DOCSTRING = r""" @@ -540,7 +540,7 @@ class TFDistilBertModel(TFDistilBertPreTrainedModel): Examples:: import tensorflow as tf - from pytorch_transformers import DistilBertTokenizer, TFDistilBertModel + from transformers import DistilBertTokenizer, TFDistilBertModel tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased') model = TFDistilBertModel.from_pretrained('distilbert-base-uncased') @@ -598,7 +598,7 @@ class TFDistilBertForMaskedLM(TFDistilBertPreTrainedModel): Examples:: import tensorflow as tf - from pytorch_transformers import DistilBertTokenizer, TFDistilBertForMaskedLM + from transformers import DistilBertTokenizer, TFDistilBertForMaskedLM tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased') model = TFDistilBertForMaskedLM.from_pretrained('distilbert-base-uncased') @@ -653,7 +653,7 @@ class TFDistilBertForSequenceClassification(TFDistilBertPreTrainedModel): Examples:: import tensorflow as tf - from pytorch_transformers import BertTokenizer, TFDistilBertForSequenceClassification + from transformers import BertTokenizer, TFDistilBertForSequenceClassification tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased') model = TFDistilBertForSequenceClassification.from_pretrained('distilbert-base-uncased') @@ -710,7 +710,7 @@ class TFDistilBertForQuestionAnswering(TFDistilBertPreTrainedModel): Examples:: import tensorflow as tf - from pytorch_transformers import BertTokenizer, TFDistilBertForQuestionAnswering + from transformers import BertTokenizer, TFDistilBertForQuestionAnswering tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased') model = TFDistilBertForQuestionAnswering.from_pretrained('distilbert-base-uncased') diff --git a/pytorch_transformers/modeling_tf_gpt2.py b/transformers/modeling_tf_gpt2.py similarity index 97% rename from pytorch_transformers/modeling_tf_gpt2.py rename to transformers/modeling_tf_gpt2.py index 77d78e7a93..e958c2cbf1 100644 --- a/pytorch_transformers/modeling_tf_gpt2.py +++ b/transformers/modeling_tf_gpt2.py @@ -385,9 +385,9 @@ GPT2_START_DOCSTRING = r""" OpenAI GPT-2 model was proposed in `model({'input_ids': input_ids, 'token_type_ids': token_type_ids})` Parameters: - config (:class:`~pytorch_transformers.GPT2Config`): Model configuration class with all the parameters of the model. + config (:class:`~transformers.GPT2Config`): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. - Check out the :meth:`~pytorch_transformers.PreTrainedModel.from_pretrained` method to load the model weights. + Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights. """ GPT2_INPUTS_DOCSTRING = r""" Inputs: @@ -395,9 +395,9 @@ GPT2_INPUTS_DOCSTRING = r""" Inputs: Indices of input sequence tokens in the vocabulary. GPT-2 is a model with absolute position embeddings so it's usually advised to pad the inputs on the right rather than the left. - Indices can be obtained using :class:`pytorch_transformers.BPT2Tokenizer`. - See :func:`pytorch_transformers.PreTrainedTokenizer.encode` and - :func:`pytorch_transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + Indices can be obtained using :class:`transformers.BPT2Tokenizer`. + See :func:`transformers.PreTrainedTokenizer.encode` and + :func:`transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. **past**: list of ``Numpy array`` or ``tf.Tensor`` (one for each layer): that contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model @@ -441,7 +441,7 @@ class TFGPT2Model(TFGPT2PreTrainedModel): Examples:: import tensorflow as tf - from pytorch_transformers import GPT2Tokenizer, TFGPT2Model + from transformers import GPT2Tokenizer, TFGPT2Model tokenizer = GPT2Tokenizer.from_pretrained('gpt2') model = TFGPT2Model.from_pretrained('gpt2') @@ -481,7 +481,7 @@ class TFGPT2LMHeadModel(TFGPT2PreTrainedModel): Examples:: import tensorflow as tf - from pytorch_transformers import GPT2Tokenizer, TFGPT2LMHeadModel + from transformers import GPT2Tokenizer, TFGPT2LMHeadModel tokenizer = GPT2Tokenizer.from_pretrained('gpt2') model = TFGPT2LMHeadModel.from_pretrained('gpt2') @@ -537,7 +537,7 @@ class TFGPT2DoubleHeadsModel(TFGPT2PreTrainedModel): Examples:: import tensorflow as tf - from pytorch_transformers import GPT2Tokenizer, TFGPT2DoubleHeadsModel + from transformers import GPT2Tokenizer, TFGPT2DoubleHeadsModel tokenizer = GPT2Tokenizer.from_pretrained('gpt2') model = TFGPT2DoubleHeadsModel.from_pretrained('gpt2') diff --git a/pytorch_transformers/modeling_tf_openai.py b/transformers/modeling_tf_openai.py similarity index 97% rename from pytorch_transformers/modeling_tf_openai.py rename to transformers/modeling_tf_openai.py index 0704e574a4..7521866c24 100644 --- a/pytorch_transformers/modeling_tf_openai.py +++ b/transformers/modeling_tf_openai.py @@ -371,9 +371,9 @@ OPENAI_GPT_START_DOCSTRING = r""" OpenAI GPT model was proposed in `model({'input_ids': input_ids, 'token_type_ids': token_type_ids})` Parameters: - config (:class:`~pytorch_transformers.OpenAIGPTConfig`): Model configuration class with all the parameters of the model. + config (:class:`~transformers.OpenAIGPTConfig`): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. - Check out the :meth:`~pytorch_transformers.PreTrainedModel.from_pretrained` method to load the model weights. + Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights. """ OPENAI_GPT_INPUTS_DOCSTRING = r""" Inputs: @@ -381,9 +381,9 @@ OPENAI_GPT_INPUTS_DOCSTRING = r""" Inputs: Indices of input sequence tokens in the vocabulary. GPT is a model with absolute position embeddings so it's usually advised to pad the inputs on the right rather than the left. - Indices can be obtained using :class:`pytorch_transformers.BPT2Tokenizer`. - See :func:`pytorch_transformers.PreTrainedTokenizer.encode` and - :func:`pytorch_transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + Indices can be obtained using :class:`transformers.BPT2Tokenizer`. + See :func:`transformers.PreTrainedTokenizer.encode` and + :func:`transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. **attention_mask**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``: @@ -419,7 +419,7 @@ class TFOpenAIGPTModel(TFOpenAIGPTPreTrainedModel): Examples:: import tensorflow as tf - from pytorch_transformers import OpenAIGPTTokenizer, TFOpenAIGPTModel + from transformers import OpenAIGPTTokenizer, TFOpenAIGPTModel tokenizer = OpenAIGPTTokenizer.from_pretrained('openai-gpt') model = TFOpenAIGPTModel.from_pretrained('openai-gpt') @@ -455,7 +455,7 @@ class TFOpenAIGPTLMHeadModel(TFOpenAIGPTPreTrainedModel): Examples:: import tensorflow as tf - from pytorch_transformers import OpenAIGPTTokenizer, TFOpenAIGPTLMHeadModel + from transformers import OpenAIGPTTokenizer, TFOpenAIGPTLMHeadModel tokenizer = OpenAIGPTTokenizer.from_pretrained('openai-gpt') model = TFOpenAIGPTLMHeadModel.from_pretrained('openai-gpt') @@ -506,7 +506,7 @@ class TFOpenAIGPTDoubleHeadsModel(TFOpenAIGPTPreTrainedModel): Examples:: import tensorflow as tf - from pytorch_transformers import OpenAIGPTTokenizer, TFOpenAIGPTDoubleHeadsModel + from transformers import OpenAIGPTTokenizer, TFOpenAIGPTDoubleHeadsModel tokenizer = OpenAIGPTTokenizer.from_pretrained('openai-gpt') model = TFOpenAIGPTDoubleHeadsModel.from_pretrained('openai-gpt') diff --git a/pytorch_transformers/modeling_tf_pytorch_utils.py b/transformers/modeling_tf_pytorch_utils.py similarity index 99% rename from pytorch_transformers/modeling_tf_pytorch_utils.py rename to transformers/modeling_tf_pytorch_utils.py index 25a0cf3bf6..66caa95ec7 100644 --- a/pytorch_transformers/modeling_tf_pytorch_utils.py +++ b/transformers/modeling_tf_pytorch_utils.py @@ -189,14 +189,14 @@ def load_tf2_checkpoint_in_pytorch_model(pt_model, tf_checkpoint_path, tf_inputs "https://pytorch.org/ and https://www.tensorflow.org/install/ for installation instructions.") raise e - import pytorch_transformers + import transformers tf_path = os.path.abspath(tf_checkpoint_path) logger.info("Loading TensorFlow weights from {}".format(tf_checkpoint_path)) # Instantiate and load the associated TF 2.0 model tf_model_class_name = "TF" + pt_model.__class__.__name__ # Add "TF" at the beggining - tf_model_class = getattr(pytorch_transformers, tf_model_class_name) + tf_model_class = getattr(transformers, tf_model_class_name) tf_model = tf_model_class(pt_model.config) if tf_inputs is None: diff --git a/pytorch_transformers/modeling_tf_roberta.py b/transformers/modeling_tf_roberta.py similarity index 96% rename from pytorch_transformers/modeling_tf_roberta.py rename to transformers/modeling_tf_roberta.py index 862540cf10..43747133ff 100644 --- a/pytorch_transformers/modeling_tf_roberta.py +++ b/transformers/modeling_tf_roberta.py @@ -137,9 +137,9 @@ ROBERTA_START_DOCSTRING = r""" The RoBERTa model was proposed in `model({'input_ids': input_ids, 'token_type_ids': token_type_ids})` Parameters: - config (:class:`~pytorch_transformers.RobertaConfig`): Model configuration class with all the parameters of the + config (:class:`~transformers.RobertaConfig`): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. - Check out the :meth:`~pytorch_transformers.PreTrainedModel.from_pretrained` method to load the model weights. + Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights. """ ROBERTA_INPUTS_DOCSTRING = r""" @@ -162,8 +162,8 @@ ROBERTA_INPUTS_DOCSTRING = r""" RoBERTa is a model with absolute position embeddings so it's usually advised to pad the inputs on the right rather than the left. - See :func:`pytorch_transformers.PreTrainedTokenizer.encode` and - :func:`pytorch_transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + See :func:`transformers.PreTrainedTokenizer.encode` and + :func:`transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. **attention_mask**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``: @@ -209,7 +209,7 @@ class TFRobertaModel(TFRobertaPreTrainedModel): Examples:: import tensorflow as tf - from pytorch_transformers import RobertaTokenizer, TFRobertaModel + from transformers import RobertaTokenizer, TFRobertaModel tokenizer = RobertaTokenizer.from_pretrained('roberta-base') model = TFRobertaModel.from_pretrained('roberta-base') @@ -286,7 +286,7 @@ class TFRobertaForMaskedLM(TFRobertaPreTrainedModel): Examples:: import tensorflow as tf - from pytorch_transformers import RobertaTokenizer, TFRobertaForMaskedLM + from transformers import RobertaTokenizer, TFRobertaForMaskedLM tokenizer = RobertaTokenizer.from_pretrained('roberta-base') model = TFRobertaForMaskedLM.from_pretrained('roberta-base') @@ -354,7 +354,7 @@ class TFRobertaForSequenceClassification(TFRobertaPreTrainedModel): Examples:: import tensorflow as tf - from pytorch_transformers import RobertaTokenizer, TFRobertaForSequenceClassification + from transformers import RobertaTokenizer, TFRobertaForSequenceClassification tokenizer = RoertaTokenizer.from_pretrained('roberta-base') model = TFRobertaForSequenceClassification.from_pretrained('roberta-base') diff --git a/pytorch_transformers/modeling_tf_transfo_xl.py b/transformers/modeling_tf_transfo_xl.py similarity index 98% rename from pytorch_transformers/modeling_tf_transfo_xl.py rename to transformers/modeling_tf_transfo_xl.py index 377599d9e5..df8c7e7dc9 100644 --- a/pytorch_transformers/modeling_tf_transfo_xl.py +++ b/transformers/modeling_tf_transfo_xl.py @@ -614,9 +614,9 @@ TRANSFO_XL_START_DOCSTRING = r""" The Transformer-XL model was proposed in `model({'input_ids': input_ids, 'token_type_ids': token_type_ids})` Parameters: - config (:class:`~pytorch_transformers.TransfoXLConfig`): Model configuration class with all the parameters of the model. + config (:class:`~transformers.TransfoXLConfig`): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. - Check out the :meth:`~pytorch_transformers.PreTrainedModel.from_pretrained` method to load the model weights. + Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights. """ TRANSFO_XL_INPUTS_DOCSTRING = r""" @@ -625,9 +625,9 @@ TRANSFO_XL_INPUTS_DOCSTRING = r""" Indices of input sequence tokens in the vocabulary. Transformer-XL is a model with relative position embeddings so you can either pad the inputs on the right or on the left. - Indices can be obtained using :class:`pytorch_transformers.TransfoXLTokenizer`. - See :func:`pytorch_transformers.PreTrainedTokenizer.encode` and - :func:`pytorch_transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + Indices can be obtained using :class:`transformers.TransfoXLTokenizer`. + See :func:`transformers.PreTrainedTokenizer.encode` and + :func:`transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. **mems**: (`optional`) list of ``Numpy array`` or ``tf.Tensor`` (one for each layer): that contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model @@ -660,7 +660,7 @@ class TFTransfoXLModel(TFTransfoXLPreTrainedModel): Examples:: import tensorflow as tf - from pytorch_transformers import TransfoXLTokenizer, TFTransfoXLModel + from transformers import TransfoXLTokenizer, TFTransfoXLModel tokenizer = TransfoXLTokenizer.from_pretrained('transfo-xl-wt103') model = TFTransfoXLModel.from_pretrained('transfo-xl-wt103') @@ -702,7 +702,7 @@ class TFTransfoXLLMHeadModel(TFTransfoXLPreTrainedModel): Examples:: import tensorflow as tf - from pytorch_transformers import TransfoXLTokenizer, TFTransfoXLLMHeadModel + from transformers import TransfoXLTokenizer, TFTransfoXLLMHeadModel tokenizer = TransfoXLTokenizer.from_pretrained('transfo-xl-wt103') model = TFTransfoXLLMHeadModel.from_pretrained('transfo-xl-wt103') diff --git a/pytorch_transformers/modeling_tf_transfo_xl_utilities.py b/transformers/modeling_tf_transfo_xl_utilities.py similarity index 100% rename from pytorch_transformers/modeling_tf_transfo_xl_utilities.py rename to transformers/modeling_tf_transfo_xl_utilities.py diff --git a/pytorch_transformers/modeling_tf_utils.py b/transformers/modeling_tf_utils.py similarity index 95% rename from pytorch_transformers/modeling_tf_utils.py rename to transformers/modeling_tf_utils.py index 67402abbd4..06a333af37 100644 --- a/pytorch_transformers/modeling_tf_utils.py +++ b/transformers/modeling_tf_utils.py @@ -32,16 +32,16 @@ logger = logging.getLogger(__name__) class TFPreTrainedModel(tf.keras.Model): r""" Base class for all TF models. - :class:`~pytorch_transformers.TFPreTrainedModel` takes care of storing the configuration of the models and handles methods for loading/downloading/saving models + :class:`~transformers.TFPreTrainedModel` takes care of storing the configuration of the models and handles methods for loading/downloading/saving models as well as a few methods commons to all models to (i) resize the input embeddings and (ii) prune heads in the self-attention heads. Class attributes (overridden by derived classes): - - ``config_class``: a class derived from :class:`~pytorch_transformers.PretrainedConfig` to use as configuration class for this model architecture. + - ``config_class``: a class derived from :class:`~transformers.PretrainedConfig` to use as configuration class for this model architecture. - ``pretrained_model_archive_map``: a python ``dict`` of with `short-cut-names` (string) as keys and `url` (string) of associated pretrained weights as values. - ``load_tf_weights``: a python ``method`` for loading a TensorFlow checkpoint in a PyTorch model, taking as arguments: - - ``model``: an instance of the relevant subclass of :class:`~pytorch_transformers.PreTrainedModel`, - - ``config``: an instance of the relevant subclass of :class:`~pytorch_transformers.PretrainedConfig`, + - ``model``: an instance of the relevant subclass of :class:`~transformers.PreTrainedModel`, + - ``config``: an instance of the relevant subclass of :class:`~transformers.PretrainedConfig`, - ``path``: a path (string) to the TensorFlow checkpoint. - ``base_model_prefix``: a string indicating the attribute associated to the base model in derived classes of the same architecture adding modules on top of the base model. @@ -123,7 +123,7 @@ class TFPreTrainedModel(tf.keras.Model): def save_pretrained(self, save_directory): """ Save a model and its configuration file to a directory, so that it - can be re-loaded using the `:func:`~pytorch_transformers.PreTrainedModel.from_pretrained`` class method. + can be re-loaded using the `:func:`~transformers.PreTrainedModel.from_pretrained`` class method. """ assert os.path.isdir(save_directory), "Saving path should be a directory where the model and configuration can be saved" @@ -151,17 +151,17 @@ class TFPreTrainedModel(tf.keras.Model): pretrained_model_name_or_path: either: - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. - - a path to a `directory` containing model weights saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. + - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. - a path or url to a `PyTorch state_dict save file` (e.g. `./pt_model/pytorch_model.bin`). In this case, ``from_pt`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the PyTorch checkpoint in a TensorFlow model using the provided conversion scripts and loading the TensorFlow model afterwards. model_args: (`optional`) Sequence of positional arguments: All remaning positional arguments will be passed to the underlying model's ``__init__`` method - config: (`optional`) instance of a class derived from :class:`~pytorch_transformers.PretrainedConfig`: + config: (`optional`) instance of a class derived from :class:`~transformers.PretrainedConfig`: Configuration for the model to use instead of an automatically loaded configuation. Configuration can be automatically loaded when: - the model is a model provided by the library (loaded with the ``shortcut-name`` string of a pretrained model), or - - the model was saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained` and is reloaded by suppling the save directory. + - the model was saved using :func:`~transformers.PreTrainedModel.save_pretrained` and is reloaded by suppling the save directory. - the model is loaded by suppling a local directory as ``pretrained_model_name_or_path`` and a configuration JSON file named `config.json` is found in the directory. from_pt: (`optional`) boolean, default False: @@ -182,7 +182,7 @@ class TFPreTrainedModel(tf.keras.Model): Can be used to update the configuration object (after it being loaded) and initiate the model. (e.g. ``output_attention=True``). Behave differently depending on whether a `config` is provided or automatically loaded: - If a configuration is provided with ``config``, ``**kwargs`` will be directly passed to the underlying model's ``__init__`` method (we assume all relevant updates to the configuration have already been done) - - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~pytorch_transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function. + - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function. Examples:: diff --git a/pytorch_transformers/modeling_tf_xlm.py b/transformers/modeling_tf_xlm.py similarity index 97% rename from pytorch_transformers/modeling_tf_xlm.py rename to transformers/modeling_tf_xlm.py index cc404b6f46..f8f199bbe6 100644 --- a/pytorch_transformers/modeling_tf_xlm.py +++ b/transformers/modeling_tf_xlm.py @@ -484,9 +484,9 @@ XLM_START_DOCSTRING = r""" The XLM model was proposed in `model({'input_ids': input_ids, 'token_type_ids': token_type_ids})` Parameters: - config (:class:`~pytorch_transformers.XLMConfig`): Model configuration class with all the parameters of the model. + config (:class:`~transformers.XLMConfig`): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. - Check out the :meth:`~pytorch_transformers.PreTrainedModel.from_pretrained` method to load the model weights. + Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights. """ XLM_INPUTS_DOCSTRING = r""" @@ -497,9 +497,9 @@ XLM_INPUTS_DOCSTRING = r""" XLM is a model with absolute position embeddings so it's usually advised to pad the inputs on the right rather than the left. - Indices can be obtained using :class:`pytorch_transformers.XLMTokenizer`. - See :func:`pytorch_transformers.PreTrainedTokenizer.encode` and - :func:`pytorch_transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + Indices can be obtained using :class:`transformers.XLMTokenizer`. + See :func:`transformers.PreTrainedTokenizer.encode` and + :func:`transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. **attention_mask**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``: @@ -550,7 +550,7 @@ class TFXLMModel(TFXLMPreTrainedModel): Examples:: import tensorflow as tf - from pytorch_transformers import XLMTokenizer, TFXLMModel + from transformers import XLMTokenizer, TFXLMModel tokenizer = XLMTokenizer.from_pretrained('xlm-mlm-en-2048') model = TFXLMModel.from_pretrained('xlm-mlm-en-2048') @@ -623,7 +623,7 @@ class TFXLMWithLMHeadModel(TFXLMPreTrainedModel): Examples:: import tensorflow as tf - from pytorch_transformers import XLMTokenizer, TFXLMWithLMHeadModel + from transformers import XLMTokenizer, TFXLMWithLMHeadModel tokenizer = XLMTokenizer.from_pretrained('xlm-mlm-en-2048') model = TFXLMWithLMHeadModel.from_pretrained('xlm-mlm-en-2048') @@ -667,7 +667,7 @@ class TFXLMForSequenceClassification(TFXLMPreTrainedModel): Examples:: import tensorflow as tf - from pytorch_transformers import XLMTokenizer, TFXLMForSequenceClassification + from transformers import XLMTokenizer, TFXLMForSequenceClassification tokenizer = XLMTokenizer.from_pretrained('xlm-mlm-en-2048') model = TFXLMForSequenceClassification.from_pretrained('xlm-mlm-en-2048') @@ -715,7 +715,7 @@ class TFXLMForQuestionAnsweringSimple(TFXLMPreTrainedModel): Examples:: import tensorflow as tf - from pytorch_transformers import XLMTokenizer, TFXLMForQuestionAnsweringSimple + from transformers import XLMTokenizer, TFXLMForQuestionAnsweringSimple tokenizer = XLMTokenizer.from_pretrained('xlm-mlm-en-2048') model = TFXLMForQuestionAnsweringSimple.from_pretrained('xlm-mlm-en-2048') diff --git a/pytorch_transformers/modeling_tf_xlnet.py b/transformers/modeling_tf_xlnet.py similarity index 98% rename from pytorch_transformers/modeling_tf_xlnet.py rename to transformers/modeling_tf_xlnet.py index fa9a045fd8..9370bd0915 100644 --- a/pytorch_transformers/modeling_tf_xlnet.py +++ b/transformers/modeling_tf_xlnet.py @@ -716,9 +716,9 @@ XLNET_START_DOCSTRING = r""" The XLNet model was proposed in `model({'input_ids': input_ids, 'token_type_ids': token_type_ids})` Parameters: - config (:class:`~pytorch_transformers.XLNetConfig`): Model configuration class with all the parameters of the model. + config (:class:`~transformers.XLNetConfig`): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. - Check out the :meth:`~pytorch_transformers.PreTrainedModel.from_pretrained` method to load the model weights. + Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights. """ XLNET_INPUTS_DOCSTRING = r""" @@ -727,9 +727,9 @@ XLNET_INPUTS_DOCSTRING = r""" Indices of input sequence tokens in the vocabulary. XLNet is a model with relative position embeddings so you can either pad the inputs on the right or on the left. - Indices can be obtained using :class:`pytorch_transformers.XLNetTokenizer`. - See :func:`pytorch_transformers.PreTrainedTokenizer.encode` and - :func:`pytorch_transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + Indices can be obtained using :class:`transformers.XLNetTokenizer`. + See :func:`transformers.PreTrainedTokenizer.encode` and + :func:`transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. **attention_mask**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``: @@ -793,7 +793,7 @@ class TFXLNetModel(TFXLNetPreTrainedModel): Examples:: import tensorflow as tf - from pytorch_transformers import XLNetTokenizer, TFXLNetModel + from transformers import XLNetTokenizer, TFXLNetModel tokenizer = XLNetTokenizer.from_pretrained('xlnet-large-cased') model = TFXLNetModel.from_pretrained('xlnet-large-cased') @@ -835,7 +835,7 @@ class TFXLNetLMHeadModel(TFXLNetPreTrainedModel): Examples:: import tensorflow as tf - from pytorch_transformers import XLNetTokenizer, TFXLNetLMHeadModel + from transformers import XLNetTokenizer, TFXLNetLMHeadModel tokenizer = XLNetTokenizer.from_pretrained('xlnet-large-cased') model = TFXLNetLMHeadModel.from_pretrained('xlnet-large-cased') @@ -890,7 +890,7 @@ class TFXLNetForSequenceClassification(TFXLNetPreTrainedModel): Examples:: import tensorflow as tf - from pytorch_transformers import XLNetTokenizer, TFXLNetForSequenceClassification + from transformers import XLNetTokenizer, TFXLNetForSequenceClassification tokenizer = XLNetTokenizer.from_pretrained('xlnet-large-cased') model = TFXLNetForSequenceClassification.from_pretrained('xlnet-large-cased') @@ -943,7 +943,7 @@ class TFXLNetForQuestionAnsweringSimple(TFXLNetPreTrainedModel): Examples:: import tensorflow as tf - from pytorch_transformers import XLNetTokenizer, TFXLNetForQuestionAnsweringSimple + from transformers import XLNetTokenizer, TFXLNetForQuestionAnsweringSimple tokenizer = XLNetTokenizer.from_pretrained('xlnet-base-cased') model = TFXLNetForQuestionAnsweringSimple.from_pretrained('xlnet-base-cased') diff --git a/pytorch_transformers/modeling_transfo_xl.py b/transformers/modeling_transfo_xl.py similarity index 98% rename from pytorch_transformers/modeling_transfo_xl.py rename to transformers/modeling_transfo_xl.py index 8925d01c50..6d430e1804 100644 --- a/pytorch_transformers/modeling_transfo_xl.py +++ b/transformers/modeling_transfo_xl.py @@ -531,9 +531,9 @@ TRANSFO_XL_START_DOCSTRING = r""" The Transformer-XL model was proposed in https://pytorch.org/docs/stable/nn.html#module Parameters: - config (:class:`~pytorch_transformers.TransfoXLConfig`): Model configuration class with all the parameters of the model. + config (:class:`~transformers.TransfoXLConfig`): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. - Check out the :meth:`~pytorch_transformers.PreTrainedModel.from_pretrained` method to load the model weights. + Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights. """ TRANSFO_XL_INPUTS_DOCSTRING = r""" @@ -542,9 +542,9 @@ TRANSFO_XL_INPUTS_DOCSTRING = r""" Indices of input sequence tokens in the vocabulary. Transformer-XL is a model with relative position embeddings so you can either pad the inputs on the right or on the left. - Indices can be obtained using :class:`pytorch_transformers.TransfoXLTokenizer`. - See :func:`pytorch_transformers.PreTrainedTokenizer.encode` and - :func:`pytorch_transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + Indices can be obtained using :class:`transformers.TransfoXLTokenizer`. + See :func:`transformers.PreTrainedTokenizer.encode` and + :func:`transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. **mems**: (`optional`) list of ``torch.FloatTensor`` (one for each layer): that contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model diff --git a/pytorch_transformers/modeling_transfo_xl_utilities.py b/transformers/modeling_transfo_xl_utilities.py similarity index 100% rename from pytorch_transformers/modeling_transfo_xl_utilities.py rename to transformers/modeling_transfo_xl_utilities.py diff --git a/pytorch_transformers/modeling_utils.py b/transformers/modeling_utils.py similarity index 96% rename from pytorch_transformers/modeling_utils.py rename to transformers/modeling_utils.py index 541ef7c741..ae2df9514a 100644 --- a/pytorch_transformers/modeling_utils.py +++ b/transformers/modeling_utils.py @@ -52,16 +52,16 @@ except ImportError: class PreTrainedModel(nn.Module): r""" Base class for all models. - :class:`~pytorch_transformers.PreTrainedModel` takes care of storing the configuration of the models and handles methods for loading/downloading/saving models + :class:`~transformers.PreTrainedModel` takes care of storing the configuration of the models and handles methods for loading/downloading/saving models as well as a few methods commons to all models to (i) resize the input embeddings and (ii) prune heads in the self-attention heads. Class attributes (overridden by derived classes): - - ``config_class``: a class derived from :class:`~pytorch_transformers.PretrainedConfig` to use as configuration class for this model architecture. + - ``config_class``: a class derived from :class:`~transformers.PretrainedConfig` to use as configuration class for this model architecture. - ``pretrained_model_archive_map``: a python ``dict`` of with `short-cut-names` (string) as keys and `url` (string) of associated pretrained weights as values. - ``load_tf_weights``: a python ``method`` for loading a TensorFlow checkpoint in a PyTorch model, taking as arguments: - - ``model``: an instance of the relevant subclass of :class:`~pytorch_transformers.PreTrainedModel`, - - ``config``: an instance of the relevant subclass of :class:`~pytorch_transformers.PretrainedConfig`, + - ``model``: an instance of the relevant subclass of :class:`~transformers.PreTrainedModel`, + - ``config``: an instance of the relevant subclass of :class:`~transformers.PretrainedConfig`, - ``path``: a path (string) to the TensorFlow checkpoint. - ``base_model_prefix``: a string indicating the attribute associated to the base model in derived classes of the same architecture adding modules on top of the base model. @@ -189,7 +189,7 @@ class PreTrainedModel(nn.Module): def save_pretrained(self, save_directory): """ Save a model and its configuration file to a directory, so that it - can be re-loaded using the `:func:`~pytorch_transformers.PreTrainedModel.from_pretrained`` class method. + can be re-loaded using the `:func:`~transformers.PreTrainedModel.from_pretrained`` class method. """ assert os.path.isdir(save_directory), "Saving path should be a directory where the model and configuration can be saved" @@ -220,24 +220,24 @@ class PreTrainedModel(nn.Module): pretrained_model_name_or_path: either: - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. - - a path to a `directory` containing model weights saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. + - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. - a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. - None if you are both providing the configuration and state dictionary (resp. with keyword arguments ``config`` and ``state_dict``) model_args: (`optional`) Sequence of positional arguments: All remaning positional arguments will be passed to the underlying model's ``__init__`` method - config: (`optional`) instance of a class derived from :class:`~pytorch_transformers.PretrainedConfig`: + config: (`optional`) instance of a class derived from :class:`~transformers.PretrainedConfig`: Configuration for the model to use instead of an automatically loaded configuation. Configuration can be automatically loaded when: - the model is a model provided by the library (loaded with the ``shortcut-name`` string of a pretrained model), or - - the model was saved using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained` and is reloaded by suppling the save directory. + - the model was saved using :func:`~transformers.PreTrainedModel.save_pretrained` and is reloaded by suppling the save directory. - the model is loaded by suppling a local directory as ``pretrained_model_name_or_path`` and a configuration JSON file named `config.json` is found in the directory. state_dict: (`optional`) dict: an optional state dictionnary for the model to use instead of a state dictionary loaded from saved weights file. This option can be used if you want to create a model from a pretrained configuration but load your own weights. - In this case though, you should check if using :func:`~pytorch_transformers.PreTrainedModel.save_pretrained` and :func:`~pytorch_transformers.PreTrainedModel.from_pretrained` is not a simpler option. + In this case though, you should check if using :func:`~transformers.PreTrainedModel.save_pretrained` and :func:`~transformers.PreTrainedModel.from_pretrained` is not a simpler option. cache_dir: (`optional`) string: Path to a directory in which a downloaded pre-trained model @@ -257,7 +257,7 @@ class PreTrainedModel(nn.Module): Can be used to update the configuration object (after it being loaded) and initiate the model. (e.g. ``output_attention=True``). Behave differently depending on whether a `config` is provided or automatically loaded: - If a configuration is provided with ``config``, ``**kwargs`` will be directly passed to the underlying model's ``__init__`` method (we assume all relevant updates to the configuration have already been done) - - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~pytorch_transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function. + - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function. Examples:: @@ -355,7 +355,7 @@ class PreTrainedModel(nn.Module): else: # Load from our TensorFlow 2.0 checkpoints try: - from pytorch_transformers import load_tf2_checkpoint_in_pytorch_model + from transformers import load_tf2_checkpoint_in_pytorch_model model = load_tf2_checkpoint_in_pytorch_model(model, resolved_archive_file, allow_missing_keys=True) except ImportError as e: logger.error("Loading a TensorFlow model in PyTorch, requires both PyTorch and TensorFlow to be installed. Please see " @@ -554,7 +554,7 @@ class SQuADHead(nn.Module): r""" A SQuAD head inspired by XLNet. Parameters: - config (:class:`~pytorch_transformers.XLNetConfig`): Model configuration class with all the parameters of the model. + config (:class:`~transformers.XLNetConfig`): Model configuration class with all the parameters of the model. Inputs: **hidden_states**: ``torch.FloatTensor`` of shape ``(batch_size, seq_len, hidden_size)`` diff --git a/pytorch_transformers/modeling_xlm.py b/transformers/modeling_xlm.py similarity index 98% rename from pytorch_transformers/modeling_xlm.py rename to transformers/modeling_xlm.py index 782e2265ce..b29e721556 100644 --- a/pytorch_transformers/modeling_xlm.py +++ b/transformers/modeling_xlm.py @@ -63,7 +63,7 @@ def gelu(x): GELU activation https://arxiv.org/abs/1606.08415 https://github.com/huggingface/pytorch-openai-transformer-lm/blob/master/model_pytorch.py#L14 - https://github.com/huggingface/pytorch-transformers/blob/master/modeling.py + https://github.com/huggingface/transformers/blob/master/modeling.py """ # return 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) return 0.5 * x * (1.0 + torch.erf(x / math.sqrt(2.0))) @@ -265,9 +265,9 @@ XLM_START_DOCSTRING = r""" The XLM model was proposed in https://github.com/facebookresearch/XLM Parameters: - config (:class:`~pytorch_transformers.XLMConfig`): Model configuration class with all the parameters of the model. + config (:class:`~transformers.XLMConfig`): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. - Check out the :meth:`~pytorch_transformers.PreTrainedModel.from_pretrained` method to load the model weights. + Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights. """ XLM_INPUTS_DOCSTRING = r""" @@ -278,9 +278,9 @@ XLM_INPUTS_DOCSTRING = r""" XLM is a model with absolute position embeddings so it's usually advised to pad the inputs on the right rather than the left. - Indices can be obtained using :class:`pytorch_transformers.XLMTokenizer`. - See :func:`pytorch_transformers.PreTrainedTokenizer.encode` and - :func:`pytorch_transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + Indices can be obtained using :class:`transformers.XLMTokenizer`. + See :func:`transformers.PreTrainedTokenizer.encode` and + :func:`transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``: Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``: diff --git a/pytorch_transformers/modeling_xlnet.py b/transformers/modeling_xlnet.py similarity index 99% rename from pytorch_transformers/modeling_xlnet.py rename to transformers/modeling_xlnet.py index f9960d4945..d6bb2ebd38 100644 --- a/pytorch_transformers/modeling_xlnet.py +++ b/transformers/modeling_xlnet.py @@ -488,9 +488,9 @@ XLNET_START_DOCSTRING = r""" The XLNet model was proposed in https://pytorch.org/docs/stable/nn.html#module Parameters: - config (:class:`~pytorch_transformers.XLNetConfig`): Model configuration class with all the parameters of the model. + config (:class:`~transformers.XLNetConfig`): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. - Check out the :meth:`~pytorch_transformers.PreTrainedModel.from_pretrained` method to load the model weights. + Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights. """ XLNET_INPUTS_DOCSTRING = r""" @@ -499,9 +499,9 @@ XLNET_INPUTS_DOCSTRING = r""" Indices of input sequence tokens in the vocabulary. XLNet is a model with relative position embeddings so you can either pad the inputs on the right or on the left. - Indices can be obtained using :class:`pytorch_transformers.XLNetTokenizer`. - See :func:`pytorch_transformers.PreTrainedTokenizer.encode` and - :func:`pytorch_transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + Indices can be obtained using :class:`transformers.XLNetTokenizer`. + See :func:`transformers.PreTrainedTokenizer.encode` and + :func:`transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. **token_type_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: A parallel sequence of tokens (can be used to indicate various portions of the inputs). The type indices in XLNet are NOT selected in the vocabulary, they can be arbitrary numbers and diff --git a/pytorch_transformers/optimization.py b/transformers/optimization.py similarity index 100% rename from pytorch_transformers/optimization.py rename to transformers/optimization.py diff --git a/pytorch_transformers/tests/__init__.py b/transformers/tests/__init__.py similarity index 100% rename from pytorch_transformers/tests/__init__.py rename to transformers/tests/__init__.py diff --git a/pytorch_transformers/tests/configuration_common_test.py b/transformers/tests/configuration_common_test.py similarity index 100% rename from pytorch_transformers/tests/configuration_common_test.py rename to transformers/tests/configuration_common_test.py diff --git a/pytorch_transformers/tests/conftest.py b/transformers/tests/conftest.py similarity index 100% rename from pytorch_transformers/tests/conftest.py rename to transformers/tests/conftest.py diff --git a/pytorch_transformers/tests/fixtures/input.txt b/transformers/tests/fixtures/input.txt similarity index 100% rename from pytorch_transformers/tests/fixtures/input.txt rename to transformers/tests/fixtures/input.txt diff --git a/pytorch_transformers/tests/fixtures/sample_text.txt b/transformers/tests/fixtures/sample_text.txt similarity index 100% rename from pytorch_transformers/tests/fixtures/sample_text.txt rename to transformers/tests/fixtures/sample_text.txt diff --git a/pytorch_transformers/tests/fixtures/test_sentencepiece.model b/transformers/tests/fixtures/test_sentencepiece.model similarity index 100% rename from pytorch_transformers/tests/fixtures/test_sentencepiece.model rename to transformers/tests/fixtures/test_sentencepiece.model diff --git a/pytorch_transformers/tests/modeling_auto_test.py b/transformers/tests/modeling_auto_test.py similarity index 95% rename from pytorch_transformers/tests/modeling_auto_test.py rename to transformers/tests/modeling_auto_test.py index 4b00891c38..af1de29cce 100644 --- a/pytorch_transformers/tests/modeling_auto_test.py +++ b/transformers/tests/modeling_auto_test.py @@ -21,15 +21,15 @@ import shutil import pytest import logging -from pytorch_transformers import is_torch_available +from transformers import is_torch_available if is_torch_available(): - from pytorch_transformers import (AutoConfig, BertConfig, + from transformers import (AutoConfig, BertConfig, AutoModel, BertModel, AutoModelWithLMHead, BertForMaskedLM, AutoModelForSequenceClassification, BertForSequenceClassification, AutoModelForQuestionAnswering, BertForQuestionAnswering) - from pytorch_transformers.modeling_bert import BERT_PRETRAINED_MODEL_ARCHIVE_MAP + from transformers.modeling_bert import BERT_PRETRAINED_MODEL_ARCHIVE_MAP from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester diff --git a/pytorch_transformers/tests/modeling_bert_test.py b/transformers/tests/modeling_bert_test.py similarity index 98% rename from pytorch_transformers/tests/modeling_bert_test.py rename to transformers/tests/modeling_bert_test.py index cac1f996e9..633c97e263 100644 --- a/pytorch_transformers/tests/modeling_bert_test.py +++ b/transformers/tests/modeling_bert_test.py @@ -20,17 +20,17 @@ import unittest import shutil import pytest -from pytorch_transformers import is_torch_available +from transformers import is_torch_available from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester if is_torch_available(): - from pytorch_transformers import (BertConfig, BertModel, BertForMaskedLM, + from transformers import (BertConfig, BertModel, BertForMaskedLM, BertForNextSentencePrediction, BertForPreTraining, BertForQuestionAnswering, BertForSequenceClassification, BertForTokenClassification, BertForMultipleChoice) - from pytorch_transformers.modeling_bert import BERT_PRETRAINED_MODEL_ARCHIVE_MAP + from transformers.modeling_bert import BERT_PRETRAINED_MODEL_ARCHIVE_MAP else: pytestmark = pytest.mark.skip("Require Torch") @@ -310,7 +310,7 @@ class BertModelTest(CommonTestCases.CommonModelTester): @pytest.mark.slow def test_model_from_pretrained(self): - cache_dir = "/tmp/pytorch_transformers_test/" + cache_dir = "/tmp/transformers_test/" for model_name in list(BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: model = BertModel.from_pretrained(model_name, cache_dir=cache_dir) shutil.rmtree(cache_dir) diff --git a/pytorch_transformers/tests/modeling_common_test.py b/transformers/tests/modeling_common_test.py similarity index 99% rename from pytorch_transformers/tests/modeling_common_test.py rename to transformers/tests/modeling_common_test.py index 0762a7a634..2b66757c28 100644 --- a/pytorch_transformers/tests/modeling_common_test.py +++ b/transformers/tests/modeling_common_test.py @@ -27,12 +27,12 @@ import unittest import logging import pytest -from pytorch_transformers import is_torch_available +from transformers import is_torch_available if is_torch_available(): import torch - from pytorch_transformers import (PretrainedConfig, PreTrainedModel, + from transformers import (PretrainedConfig, PreTrainedModel, BertModel, BertConfig, BERT_PRETRAINED_MODEL_ARCHIVE_MAP, GPT2LMHeadModel, GPT2Config, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP) else: @@ -621,7 +621,7 @@ class CommonTestCases: [[], []]) def create_and_check_model_from_pretrained(self): - cache_dir = "/tmp/pytorch_transformers_test/" + cache_dir = "/tmp/transformers_test/" for model_name in list(self.base_model_class.pretrained_model_archive_map.keys())[:1]: model = self.base_model_class.from_pretrained(model_name, cache_dir=cache_dir) shutil.rmtree(cache_dir) diff --git a/pytorch_transformers/tests/modeling_distilbert_test.py b/transformers/tests/modeling_distilbert_test.py similarity index 97% rename from pytorch_transformers/tests/modeling_distilbert_test.py rename to transformers/tests/modeling_distilbert_test.py index 8fef9d5833..937d03396d 100644 --- a/pytorch_transformers/tests/modeling_distilbert_test.py +++ b/transformers/tests/modeling_distilbert_test.py @@ -19,10 +19,10 @@ from __future__ import print_function import unittest import pytest -from pytorch_transformers import is_torch_available +from transformers import is_torch_available if is_torch_available(): - from pytorch_transformers import (DistilBertConfig, DistilBertModel, DistilBertForMaskedLM, + from transformers import (DistilBertConfig, DistilBertModel, DistilBertForMaskedLM, DistilBertForQuestionAnswering, DistilBertForSequenceClassification) else: pytestmark = pytest.mark.skip("Require Torch") @@ -211,7 +211,7 @@ class DistilBertModelTest(CommonTestCases.CommonModelTester): # @pytest.mark.slow # def test_model_from_pretrained(self): - # cache_dir = "/tmp/pytorch_transformers_test/" + # cache_dir = "/tmp/transformers_test/" # for model_name in list(DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: # model = DistilBertModel.from_pretrained(model_name, cache_dir=cache_dir) # shutil.rmtree(cache_dir) diff --git a/pytorch_transformers/tests/modeling_gpt2_test.py b/transformers/tests/modeling_gpt2_test.py similarity index 98% rename from pytorch_transformers/tests/modeling_gpt2_test.py rename to transformers/tests/modeling_gpt2_test.py index e5accfa8cf..4263e51bc9 100644 --- a/pytorch_transformers/tests/modeling_gpt2_test.py +++ b/transformers/tests/modeling_gpt2_test.py @@ -20,10 +20,10 @@ import unittest import pytest import shutil -from pytorch_transformers import is_torch_available +from transformers import is_torch_available if is_torch_available(): - from pytorch_transformers import (GPT2Config, GPT2Model, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP, + from transformers import (GPT2Config, GPT2Model, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP, GPT2LMHeadModel, GPT2DoubleHeadsModel) else: pytestmark = pytest.mark.skip("Require Torch") @@ -237,7 +237,7 @@ class GPT2ModelTest(CommonTestCases.CommonModelTester): @pytest.mark.slow def test_model_from_pretrained(self): - cache_dir = "/tmp/pytorch_transformers_test/" + cache_dir = "/tmp/transformers_test/" for model_name in list(GPT2_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: model = GPT2Model.from_pretrained(model_name, cache_dir=cache_dir) shutil.rmtree(cache_dir) diff --git a/pytorch_transformers/tests/modeling_openai_test.py b/transformers/tests/modeling_openai_test.py similarity index 97% rename from pytorch_transformers/tests/modeling_openai_test.py rename to transformers/tests/modeling_openai_test.py index 6df4406d03..33218288a0 100644 --- a/pytorch_transformers/tests/modeling_openai_test.py +++ b/transformers/tests/modeling_openai_test.py @@ -20,10 +20,10 @@ import unittest import pytest import shutil -from pytorch_transformers import is_torch_available +from transformers import is_torch_available if is_torch_available(): - from pytorch_transformers import (OpenAIGPTConfig, OpenAIGPTModel, OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP, + from transformers import (OpenAIGPTConfig, OpenAIGPTModel, OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP, OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel) else: pytestmark = pytest.mark.skip("Require Torch") @@ -205,7 +205,7 @@ class OpenAIGPTModelTest(CommonTestCases.CommonModelTester): @pytest.mark.slow def test_model_from_pretrained(self): - cache_dir = "/tmp/pytorch_transformers_test/" + cache_dir = "/tmp/transformers_test/" for model_name in list(OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: model = OpenAIGPTModel.from_pretrained(model_name, cache_dir=cache_dir) shutil.rmtree(cache_dir) diff --git a/pytorch_transformers/tests/modeling_roberta_test.py b/transformers/tests/modeling_roberta_test.py similarity index 96% rename from pytorch_transformers/tests/modeling_roberta_test.py rename to transformers/tests/modeling_roberta_test.py index 11f2893671..82e10da915 100644 --- a/pytorch_transformers/tests/modeling_roberta_test.py +++ b/transformers/tests/modeling_roberta_test.py @@ -20,12 +20,12 @@ import unittest import shutil import pytest -from pytorch_transformers import is_torch_available +from transformers import is_torch_available if is_torch_available(): import torch - from pytorch_transformers import (RobertaConfig, RobertaModel, RobertaForMaskedLM, RobertaForSequenceClassification) - from pytorch_transformers.modeling_roberta import ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP + from transformers import (RobertaConfig, RobertaModel, RobertaForMaskedLM, RobertaForSequenceClassification) + from transformers.modeling_roberta import ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP else: pytestmark = pytest.mark.skip("Require Torch") @@ -180,7 +180,7 @@ class RobertaModelTest(CommonTestCases.CommonModelTester): @pytest.mark.slow def test_model_from_pretrained(self): - cache_dir = "/tmp/pytorch_transformers_test/" + cache_dir = "/tmp/transformers_test/" for model_name in list(ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: model = RobertaModel.from_pretrained(model_name, cache_dir=cache_dir) shutil.rmtree(cache_dir) diff --git a/pytorch_transformers/tests/modeling_tf_auto_test.py b/transformers/tests/modeling_tf_auto_test.py similarity index 95% rename from pytorch_transformers/tests/modeling_tf_auto_test.py rename to transformers/tests/modeling_tf_auto_test.py index 7b080bafcd..2cda3abc1c 100644 --- a/pytorch_transformers/tests/modeling_tf_auto_test.py +++ b/transformers/tests/modeling_tf_auto_test.py @@ -21,15 +21,15 @@ import shutil import pytest import logging -from pytorch_transformers import is_tf_available +from transformers import is_tf_available if is_tf_available(): - from pytorch_transformers import (AutoConfig, BertConfig, + from transformers import (AutoConfig, BertConfig, TFAutoModel, TFBertModel, TFAutoModelWithLMHead, TFBertForMaskedLM, TFAutoModelForSequenceClassification, TFBertForSequenceClassification, TFAutoModelForQuestionAnswering, TFBertForQuestionAnswering) - from pytorch_transformers.modeling_tf_bert import TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP + from transformers.modeling_tf_bert import TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester diff --git a/pytorch_transformers/tests/modeling_tf_bert_test.py b/transformers/tests/modeling_tf_bert_test.py similarity index 98% rename from pytorch_transformers/tests/modeling_tf_bert_test.py rename to transformers/tests/modeling_tf_bert_test.py index 53499110f2..a1715d2568 100644 --- a/pytorch_transformers/tests/modeling_tf_bert_test.py +++ b/transformers/tests/modeling_tf_bert_test.py @@ -24,11 +24,11 @@ import sys from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester -from pytorch_transformers import BertConfig, is_tf_available +from transformers import BertConfig, is_tf_available if is_tf_available(): import tensorflow as tf - from pytorch_transformers.modeling_tf_bert import (TFBertModel, TFBertForMaskedLM, + from transformers.modeling_tf_bert import (TFBertModel, TFBertForMaskedLM, TFBertForNextSentencePrediction, TFBertForPreTraining, TFBertForSequenceClassification, @@ -315,7 +315,7 @@ class TFBertModelTest(TFCommonTestCases.TFCommonModelTester): @pytest.mark.slow def test_model_from_pretrained(self): - cache_dir = "/tmp/pytorch_transformers_test/" + cache_dir = "/tmp/transformers_test/" # for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: for model_name in ['bert-base-uncased']: model = TFBertModel.from_pretrained(model_name, cache_dir=cache_dir) diff --git a/pytorch_transformers/tests/modeling_tf_common_test.py b/transformers/tests/modeling_tf_common_test.py similarity index 96% rename from pytorch_transformers/tests/modeling_tf_common_test.py rename to transformers/tests/modeling_tf_common_test.py index 5e7d29cb7f..483f031b16 100644 --- a/pytorch_transformers/tests/modeling_tf_common_test.py +++ b/transformers/tests/modeling_tf_common_test.py @@ -26,13 +26,13 @@ import uuid import pytest import sys -from pytorch_transformers import is_tf_available, is_torch_available +from transformers import is_tf_available, is_torch_available if is_tf_available(): import tensorflow as tf import numpy as np - from pytorch_transformers import TFPreTrainedModel - # from pytorch_transformers.modeling_bert import BertModel, BertConfig, BERT_PRETRAINED_MODEL_ARCHIVE_MAP + from transformers import TFPreTrainedModel + # from transformers.modeling_bert import BertModel, BertConfig, BERT_PRETRAINED_MODEL_ARCHIVE_MAP else: pytestmark = pytest.mark.skip("Require TensorFlow") @@ -71,19 +71,19 @@ class TFCommonTestCases: if not is_torch_available(): return - import pytorch_transformers + import transformers config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() for model_class in self.all_model_classes: pt_model_class_name = model_class.__name__[2:] # Skip the "TF" at the beggining - pt_model_class = getattr(pytorch_transformers, pt_model_class_name) + pt_model_class = getattr(transformers, pt_model_class_name) tf_model = model_class(config) pt_model = pt_model_class(config) - tf_model = pytorch_transformers.load_pytorch_model_in_tf2_model(tf_model, pt_model, tf_inputs=inputs_dict) - pt_model = pytorch_transformers.load_tf2_model_in_pytorch_model(pt_model, tf_model) + tf_model = transformers.load_pytorch_model_in_tf2_model(tf_model, pt_model, tf_inputs=inputs_dict) + pt_model = transformers.load_tf2_model_in_pytorch_model(pt_model, tf_model) def test_keyword_and_dict_args(self): diff --git a/pytorch_transformers/tests/modeling_tf_distilbert_test.py b/transformers/tests/modeling_tf_distilbert_test.py similarity index 97% rename from pytorch_transformers/tests/modeling_tf_distilbert_test.py rename to transformers/tests/modeling_tf_distilbert_test.py index 5d7f0f7a31..e6d3795914 100644 --- a/pytorch_transformers/tests/modeling_tf_distilbert_test.py +++ b/transformers/tests/modeling_tf_distilbert_test.py @@ -22,11 +22,11 @@ import pytest from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester -from pytorch_transformers import DistilBertConfig, is_tf_available +from transformers import DistilBertConfig, is_tf_available if is_tf_available(): import tensorflow as tf - from pytorch_transformers.modeling_tf_distilbert import (TFDistilBertModel, + from transformers.modeling_tf_distilbert import (TFDistilBertModel, TFDistilBertForMaskedLM, TFDistilBertForQuestionAnswering, TFDistilBertForSequenceClassification) @@ -212,7 +212,7 @@ class TFDistilBertModelTest(TFCommonTestCases.TFCommonModelTester): # @pytest.mark.slow # def test_model_from_pretrained(self): - # cache_dir = "/tmp/pytorch_transformers_test/" + # cache_dir = "/tmp/transformers_test/" # for model_name in list(DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: # model = DistilBertModel.from_pretrained(model_name, cache_dir=cache_dir) # shutil.rmtree(cache_dir) diff --git a/pytorch_transformers/tests/modeling_tf_gpt2_test.py b/transformers/tests/modeling_tf_gpt2_test.py similarity index 98% rename from pytorch_transformers/tests/modeling_tf_gpt2_test.py rename to transformers/tests/modeling_tf_gpt2_test.py index 490d5c4e32..658456d15b 100644 --- a/pytorch_transformers/tests/modeling_tf_gpt2_test.py +++ b/transformers/tests/modeling_tf_gpt2_test.py @@ -24,11 +24,11 @@ import sys from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester -from pytorch_transformers import GPT2Config, is_tf_available +from transformers import GPT2Config, is_tf_available if is_tf_available(): import tensorflow as tf - from pytorch_transformers.modeling_tf_gpt2 import (TFGPT2Model, TFGPT2LMHeadModel, + from transformers.modeling_tf_gpt2 import (TFGPT2Model, TFGPT2LMHeadModel, TFGPT2DoubleHeadsModel, TF_GPT2_PRETRAINED_MODEL_ARCHIVE_MAP) else: @@ -221,7 +221,7 @@ class TFGPT2ModelTest(TFCommonTestCases.TFCommonModelTester): @pytest.mark.slow def test_model_from_pretrained(self): - cache_dir = "/tmp/pytorch_transformers_test/" + cache_dir = "/tmp/transformers_test/" for model_name in list(TF_gpt2_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: model = TFGPT2Model.from_pretrained(model_name, cache_dir=cache_dir) shutil.rmtree(cache_dir) diff --git a/pytorch_transformers/tests/modeling_tf_openai_gpt_test.py b/transformers/tests/modeling_tf_openai_gpt_test.py similarity index 97% rename from pytorch_transformers/tests/modeling_tf_openai_gpt_test.py rename to transformers/tests/modeling_tf_openai_gpt_test.py index c553209db3..d470c8862d 100644 --- a/pytorch_transformers/tests/modeling_tf_openai_gpt_test.py +++ b/transformers/tests/modeling_tf_openai_gpt_test.py @@ -24,11 +24,11 @@ import sys from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester -from pytorch_transformers import OpenAIGPTConfig, is_tf_available +from transformers import OpenAIGPTConfig, is_tf_available if is_tf_available(): import tensorflow as tf - from pytorch_transformers.modeling_tf_openai import (TFOpenAIGPTModel, TFOpenAIGPTLMHeadModel, + from transformers.modeling_tf_openai import (TFOpenAIGPTModel, TFOpenAIGPTLMHeadModel, TFOpenAIGPTDoubleHeadsModel, TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP) else: @@ -220,7 +220,7 @@ class TFOpenAIGPTModelTest(TFCommonTestCases.TFCommonModelTester): @pytest.mark.slow def test_model_from_pretrained(self): - cache_dir = "/tmp/pytorch_transformers_test/" + cache_dir = "/tmp/transformers_test/" for model_name in list(TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: model = TFOpenAIGPTModel.from_pretrained(model_name, cache_dir=cache_dir) shutil.rmtree(cache_dir) diff --git a/pytorch_transformers/tests/modeling_tf_roberta_test.py b/transformers/tests/modeling_tf_roberta_test.py similarity index 97% rename from pytorch_transformers/tests/modeling_tf_roberta_test.py rename to transformers/tests/modeling_tf_roberta_test.py index 1fbd5d0f5f..735c9aae27 100644 --- a/pytorch_transformers/tests/modeling_tf_roberta_test.py +++ b/transformers/tests/modeling_tf_roberta_test.py @@ -23,12 +23,12 @@ import pytest from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester -from pytorch_transformers import RobertaConfig, is_tf_available +from transformers import RobertaConfig, is_tf_available if is_tf_available(): import tensorflow as tf import numpy - from pytorch_transformers.modeling_tf_roberta import (TFRobertaModel, TFRobertaForMaskedLM, + from transformers.modeling_tf_roberta import (TFRobertaModel, TFRobertaForMaskedLM, TFRobertaForSequenceClassification, TF_ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP) else: @@ -178,7 +178,7 @@ class TFRobertaModelTest(TFCommonTestCases.TFCommonModelTester): @pytest.mark.slow def test_model_from_pretrained(self): - cache_dir = "/tmp/pytorch_transformers_test/" + cache_dir = "/tmp/transformers_test/" for model_name in list(TF_ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: model = TFRobertaModel.from_pretrained(model_name, cache_dir=cache_dir) shutil.rmtree(cache_dir) diff --git a/pytorch_transformers/tests/modeling_tf_transfo_xl_test.py b/transformers/tests/modeling_tf_transfo_xl_test.py similarity index 97% rename from pytorch_transformers/tests/modeling_tf_transfo_xl_test.py rename to transformers/tests/modeling_tf_transfo_xl_test.py index 58a4343dfd..534fe39646 100644 --- a/pytorch_transformers/tests/modeling_tf_transfo_xl_test.py +++ b/transformers/tests/modeling_tf_transfo_xl_test.py @@ -24,11 +24,11 @@ import pytest from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester -from pytorch_transformers import TransfoXLConfig, is_tf_available +from transformers import TransfoXLConfig, is_tf_available if is_tf_available(): import tensorflow as tf - from pytorch_transformers.modeling_tf_transfo_xl import (TFTransfoXLModel, + from transformers.modeling_tf_transfo_xl import (TFTransfoXLModel, TFTransfoXLLMHeadModel, TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP) else: @@ -206,7 +206,7 @@ class TFTransfoXLModelTest(TFCommonTestCases.TFCommonModelTester): @pytest.mark.slow def test_model_from_pretrained(self): - cache_dir = "/tmp/pytorch_transformers_test/" + cache_dir = "/tmp/transformers_test/" for model_name in list(TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: model = TFTransfoXLModel.from_pretrained(model_name, cache_dir=cache_dir) shutil.rmtree(cache_dir) diff --git a/pytorch_transformers/tests/modeling_tf_xlm_test.py b/transformers/tests/modeling_tf_xlm_test.py similarity index 98% rename from pytorch_transformers/tests/modeling_tf_xlm_test.py rename to transformers/tests/modeling_tf_xlm_test.py index 26329bebb6..1bd661bebf 100644 --- a/pytorch_transformers/tests/modeling_tf_xlm_test.py +++ b/transformers/tests/modeling_tf_xlm_test.py @@ -20,11 +20,11 @@ import unittest import shutil import pytest -from pytorch_transformers import is_tf_available +from transformers import is_tf_available if is_tf_available(): import tensorflow as tf - from pytorch_transformers import (XLMConfig, TFXLMModel, + from transformers import (XLMConfig, TFXLMModel, TFXLMWithLMHeadModel, TFXLMForSequenceClassification, TFXLMForQuestionAnsweringSimple, @@ -253,7 +253,7 @@ class TFXLMModelTest(TFCommonTestCases.TFCommonModelTester): @pytest.mark.slow def test_model_from_pretrained(self): - cache_dir = "/tmp/pytorch_transformers_test/" + cache_dir = "/tmp/transformers_test/" for model_name in list(TF_XLM_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: model = XLMModel.from_pretrained(model_name, cache_dir=cache_dir) shutil.rmtree(cache_dir) diff --git a/pytorch_transformers/tests/modeling_tf_xlnet_test.py b/transformers/tests/modeling_tf_xlnet_test.py similarity index 98% rename from pytorch_transformers/tests/modeling_tf_xlnet_test.py rename to transformers/tests/modeling_tf_xlnet_test.py index 01c4494664..6a0434938f 100644 --- a/pytorch_transformers/tests/modeling_tf_xlnet_test.py +++ b/transformers/tests/modeling_tf_xlnet_test.py @@ -23,12 +23,12 @@ import random import shutil import pytest -from pytorch_transformers import XLNetConfig, is_tf_available +from transformers import XLNetConfig, is_tf_available if is_tf_available(): import tensorflow as tf - from pytorch_transformers.modeling_tf_xlnet import (TFXLNetModel, TFXLNetLMHeadModel, + from transformers.modeling_tf_xlnet import (TFXLNetModel, TFXLNetLMHeadModel, TFXLNetForSequenceClassification, TFXLNetForQuestionAnsweringSimple, TF_XLNET_PRETRAINED_MODEL_ARCHIVE_MAP) @@ -291,7 +291,7 @@ class TFXLNetModelTest(TFCommonTestCases.TFCommonModelTester): @pytest.mark.slow def test_model_from_pretrained(self): - cache_dir = "/tmp/pytorch_transformers_test/" + cache_dir = "/tmp/transformers_test/" for model_name in list(TF_XLNET_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: model = TFXLNetModel.from_pretrained(model_name, cache_dir=cache_dir) shutil.rmtree(cache_dir) diff --git a/pytorch_transformers/tests/modeling_transfo_xl_test.py b/transformers/tests/modeling_transfo_xl_test.py similarity index 96% rename from pytorch_transformers/tests/modeling_transfo_xl_test.py rename to transformers/tests/modeling_transfo_xl_test.py index 035f20991f..f7b913da5b 100644 --- a/pytorch_transformers/tests/modeling_transfo_xl_test.py +++ b/transformers/tests/modeling_transfo_xl_test.py @@ -21,12 +21,12 @@ import random import shutil import pytest -from pytorch_transformers import is_torch_available +from transformers import is_torch_available if is_torch_available(): import torch - from pytorch_transformers import (TransfoXLConfig, TransfoXLModel, TransfoXLLMHeadModel) - from pytorch_transformers.modeling_transfo_xl import TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP + from transformers import (TransfoXLConfig, TransfoXLModel, TransfoXLLMHeadModel) + from transformers.modeling_transfo_xl import TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP else: pytestmark = pytest.mark.skip("Require Torch") @@ -206,7 +206,7 @@ class TransfoXLModelTest(CommonTestCases.CommonModelTester): @pytest.mark.slow def test_model_from_pretrained(self): - cache_dir = "/tmp/pytorch_transformers_test/" + cache_dir = "/tmp/transformers_test/" for model_name in list(TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: model = TransfoXLModel.from_pretrained(model_name, cache_dir=cache_dir) shutil.rmtree(cache_dir) diff --git a/pytorch_transformers/tests/modeling_xlm_test.py b/transformers/tests/modeling_xlm_test.py similarity index 97% rename from pytorch_transformers/tests/modeling_xlm_test.py rename to transformers/tests/modeling_xlm_test.py index c3f75e2623..0133febb58 100644 --- a/pytorch_transformers/tests/modeling_xlm_test.py +++ b/transformers/tests/modeling_xlm_test.py @@ -20,12 +20,12 @@ import unittest import shutil import pytest -from pytorch_transformers import is_torch_available +from transformers import is_torch_available if is_torch_available(): - from pytorch_transformers import (XLMConfig, XLMModel, XLMWithLMHeadModel, XLMForQuestionAnswering, + from transformers import (XLMConfig, XLMModel, XLMWithLMHeadModel, XLMForQuestionAnswering, XLMForSequenceClassification, XLMForQuestionAnsweringSimple) - from pytorch_transformers.modeling_xlm import XLM_PRETRAINED_MODEL_ARCHIVE_MAP + from transformers.modeling_xlm import XLM_PRETRAINED_MODEL_ARCHIVE_MAP else: pytestmark = pytest.mark.skip("Require Torch") @@ -314,7 +314,7 @@ class XLMModelTest(CommonTestCases.CommonModelTester): @pytest.mark.slow def test_model_from_pretrained(self): - cache_dir = "/tmp/pytorch_transformers_test/" + cache_dir = "/tmp/transformers_test/" for model_name in list(XLM_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: model = XLMModel.from_pretrained(model_name, cache_dir=cache_dir) shutil.rmtree(cache_dir) diff --git a/pytorch_transformers/tests/modeling_xlnet_test.py b/transformers/tests/modeling_xlnet_test.py similarity index 97% rename from pytorch_transformers/tests/modeling_xlnet_test.py rename to transformers/tests/modeling_xlnet_test.py index 8c8b3f964f..10cbdaf37b 100644 --- a/pytorch_transformers/tests/modeling_xlnet_test.py +++ b/transformers/tests/modeling_xlnet_test.py @@ -23,13 +23,13 @@ import random import shutil import pytest -from pytorch_transformers import is_torch_available +from transformers import is_torch_available if is_torch_available(): import torch - from pytorch_transformers import (XLNetConfig, XLNetModel, XLNetLMHeadModel, XLNetForSequenceClassification, XLNetForQuestionAnswering) - from pytorch_transformers.modeling_xlnet import XLNET_PRETRAINED_MODEL_ARCHIVE_MAP + from transformers import (XLNetConfig, XLNetModel, XLNetLMHeadModel, XLNetForSequenceClassification, XLNetForQuestionAnswering) + from transformers.modeling_xlnet import XLNET_PRETRAINED_MODEL_ARCHIVE_MAP else: pytestmark = pytest.mark.skip("Require Torch") @@ -317,7 +317,7 @@ class XLNetModelTest(CommonTestCases.CommonModelTester): @pytest.mark.slow def test_model_from_pretrained(self): - cache_dir = "/tmp/pytorch_transformers_test/" + cache_dir = "/tmp/transformers_test/" for model_name in list(XLNET_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: model = XLNetModel.from_pretrained(model_name, cache_dir=cache_dir) shutil.rmtree(cache_dir) diff --git a/pytorch_transformers/tests/optimization_test.py b/transformers/tests/optimization_test.py similarity index 97% rename from pytorch_transformers/tests/optimization_test.py rename to transformers/tests/optimization_test.py index c1c6270f32..84dbaca52a 100644 --- a/pytorch_transformers/tests/optimization_test.py +++ b/transformers/tests/optimization_test.py @@ -20,12 +20,12 @@ import unittest import os import pytest -from pytorch_transformers import is_torch_available +from transformers import is_torch_available if is_torch_available(): import torch - from pytorch_transformers import (AdamW, ConstantLRSchedule, WarmupConstantSchedule, + from transformers import (AdamW, ConstantLRSchedule, WarmupConstantSchedule, WarmupCosineSchedule, WarmupCosineWithHardRestartsSchedule, WarmupLinearSchedule) else: pytestmark = pytest.mark.skip("Require Torch") diff --git a/pytorch_transformers/tests/tokenization_auto_test.py b/transformers/tests/tokenization_auto_test.py similarity index 88% rename from pytorch_transformers/tests/tokenization_auto_test.py rename to transformers/tests/tokenization_auto_test.py index 7cee7ebc28..0f49ec75fb 100644 --- a/pytorch_transformers/tests/tokenization_auto_test.py +++ b/transformers/tests/tokenization_auto_test.py @@ -21,8 +21,8 @@ import shutil import pytest import logging -from pytorch_transformers import AutoTokenizer, BertTokenizer, AutoTokenizer, GPT2Tokenizer -from pytorch_transformers import BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP +from transformers import AutoTokenizer, BertTokenizer, AutoTokenizer, GPT2Tokenizer +from transformers import BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP class AutoTokenizerTest(unittest.TestCase): diff --git a/pytorch_transformers/tests/tokenization_bert_test.py b/transformers/tests/tokenization_bert_test.py similarity index 98% rename from pytorch_transformers/tests/tokenization_bert_test.py rename to transformers/tests/tokenization_bert_test.py index 4cfdfed136..b70941f884 100644 --- a/pytorch_transformers/tests/tokenization_bert_test.py +++ b/transformers/tests/tokenization_bert_test.py @@ -18,7 +18,7 @@ import os import unittest from io import open -from pytorch_transformers.tokenization_bert import (BasicTokenizer, +from transformers.tokenization_bert import (BasicTokenizer, BertTokenizer, WordpieceTokenizer, _is_control, _is_punctuation, diff --git a/pytorch_transformers/tests/tokenization_distilbert_test.py b/transformers/tests/tokenization_distilbert_test.py similarity index 95% rename from pytorch_transformers/tests/tokenization_distilbert_test.py rename to transformers/tests/tokenization_distilbert_test.py index 674c78d104..64a88df99f 100644 --- a/pytorch_transformers/tests/tokenization_distilbert_test.py +++ b/transformers/tests/tokenization_distilbert_test.py @@ -18,7 +18,7 @@ import os import unittest from io import open -from pytorch_transformers.tokenization_distilbert import (DistilBertTokenizer) +from transformers.tokenization_distilbert import (DistilBertTokenizer) from .tokenization_tests_commons import CommonTestCases from .tokenization_bert_test import BertTokenizationTest diff --git a/pytorch_transformers/tests/tokenization_gpt2_test.py b/transformers/tests/tokenization_gpt2_test.py similarity index 97% rename from pytorch_transformers/tests/tokenization_gpt2_test.py rename to transformers/tests/tokenization_gpt2_test.py index ecc8c7da67..a77cc75ec2 100644 --- a/pytorch_transformers/tests/tokenization_gpt2_test.py +++ b/transformers/tests/tokenization_gpt2_test.py @@ -19,7 +19,7 @@ import unittest import json from io import open -from pytorch_transformers.tokenization_gpt2 import GPT2Tokenizer, VOCAB_FILES_NAMES +from transformers.tokenization_gpt2 import GPT2Tokenizer, VOCAB_FILES_NAMES from .tokenization_tests_commons import CommonTestCases diff --git a/pytorch_transformers/tests/tokenization_openai_test.py b/transformers/tests/tokenization_openai_test.py similarity index 96% rename from pytorch_transformers/tests/tokenization_openai_test.py rename to transformers/tests/tokenization_openai_test.py index 6b86416d2d..56aa219ddc 100644 --- a/pytorch_transformers/tests/tokenization_openai_test.py +++ b/transformers/tests/tokenization_openai_test.py @@ -18,7 +18,7 @@ import os import unittest import json -from pytorch_transformers.tokenization_openai import OpenAIGPTTokenizer, VOCAB_FILES_NAMES +from transformers.tokenization_openai import OpenAIGPTTokenizer, VOCAB_FILES_NAMES from .tokenization_tests_commons import CommonTestCases diff --git a/pytorch_transformers/tests/tokenization_roberta_test.py b/transformers/tests/tokenization_roberta_test.py similarity index 97% rename from pytorch_transformers/tests/tokenization_roberta_test.py rename to transformers/tests/tokenization_roberta_test.py index f7807d0c80..f14b26a2e4 100644 --- a/pytorch_transformers/tests/tokenization_roberta_test.py +++ b/transformers/tests/tokenization_roberta_test.py @@ -19,7 +19,7 @@ import json import unittest from io import open -from pytorch_transformers.tokenization_roberta import RobertaTokenizer, VOCAB_FILES_NAMES +from transformers.tokenization_roberta import RobertaTokenizer, VOCAB_FILES_NAMES from .tokenization_tests_commons import CommonTestCases diff --git a/pytorch_transformers/tests/tokenization_tests_commons.py b/transformers/tests/tokenization_tests_commons.py similarity index 100% rename from pytorch_transformers/tests/tokenization_tests_commons.py rename to transformers/tests/tokenization_tests_commons.py diff --git a/pytorch_transformers/tests/tokenization_transfo_xl_test.py b/transformers/tests/tokenization_transfo_xl_test.py similarity index 94% rename from pytorch_transformers/tests/tokenization_transfo_xl_test.py rename to transformers/tests/tokenization_transfo_xl_test.py index 563871f690..4e99484b0c 100644 --- a/pytorch_transformers/tests/tokenization_transfo_xl_test.py +++ b/transformers/tests/tokenization_transfo_xl_test.py @@ -19,11 +19,11 @@ import unittest import pytest from io import open -from pytorch_transformers import is_torch_available +from transformers import is_torch_available if is_torch_available(): import torch - from pytorch_transformers.tokenization_transfo_xl import TransfoXLTokenizer, VOCAB_FILES_NAMES + from transformers.tokenization_transfo_xl import TransfoXLTokenizer, VOCAB_FILES_NAMES else: pytestmark = pytest.mark.skip("Require Torch") # TODO: untangle Transfo-XL tokenizer from torch.load and torch.save diff --git a/pytorch_transformers/tests/tokenization_utils_test.py b/transformers/tests/tokenization_utils_test.py similarity index 93% rename from pytorch_transformers/tests/tokenization_utils_test.py rename to transformers/tests/tokenization_utils_test.py index 26ec2d7a39..cf55982c8f 100644 --- a/pytorch_transformers/tests/tokenization_utils_test.py +++ b/transformers/tests/tokenization_utils_test.py @@ -19,8 +19,8 @@ from __future__ import print_function import unittest import six -from pytorch_transformers import PreTrainedTokenizer -from pytorch_transformers.tokenization_gpt2 import GPT2Tokenizer +from transformers import PreTrainedTokenizer +from transformers.tokenization_gpt2 import GPT2Tokenizer class TokenizerUtilsTest(unittest.TestCase): def check_tokenizer_from_pretrained(self, tokenizer_class): diff --git a/pytorch_transformers/tests/tokenization_xlm_test.py b/transformers/tests/tokenization_xlm_test.py similarity index 97% rename from pytorch_transformers/tests/tokenization_xlm_test.py rename to transformers/tests/tokenization_xlm_test.py index 13fdb4a8bb..b1a71ede59 100644 --- a/pytorch_transformers/tests/tokenization_xlm_test.py +++ b/transformers/tests/tokenization_xlm_test.py @@ -18,7 +18,7 @@ import os import unittest import json -from pytorch_transformers.tokenization_xlm import XLMTokenizer, VOCAB_FILES_NAMES +from transformers.tokenization_xlm import XLMTokenizer, VOCAB_FILES_NAMES from .tokenization_tests_commons import CommonTestCases diff --git a/pytorch_transformers/tests/tokenization_xlnet_test.py b/transformers/tests/tokenization_xlnet_test.py similarity index 98% rename from pytorch_transformers/tests/tokenization_xlnet_test.py rename to transformers/tests/tokenization_xlnet_test.py index a6e9f23fe7..f4418c7fe5 100644 --- a/pytorch_transformers/tests/tokenization_xlnet_test.py +++ b/transformers/tests/tokenization_xlnet_test.py @@ -17,7 +17,7 @@ from __future__ import absolute_import, division, print_function, unicode_litera import os import unittest -from pytorch_transformers.tokenization_xlnet import (XLNetTokenizer, SPIECE_UNDERLINE) +from transformers.tokenization_xlnet import (XLNetTokenizer, SPIECE_UNDERLINE) from .tokenization_tests_commons import CommonTestCases diff --git a/pytorch_transformers/tokenization_auto.py b/transformers/tokenization_auto.py similarity index 96% rename from pytorch_transformers/tokenization_auto.py rename to transformers/tokenization_auto.py index 889774b36c..504727dcc8 100644 --- a/pytorch_transformers/tokenization_auto.py +++ b/transformers/tokenization_auto.py @@ -30,7 +30,7 @@ from .tokenization_distilbert import DistilBertTokenizer logger = logging.getLogger(__name__) class AutoTokenizer(object): - r""":class:`~pytorch_transformers.AutoTokenizer` is a generic tokenizer class + r""":class:`~transformers.AutoTokenizer` is a generic tokenizer class that will be instantiated as one of the tokenizer classes of the library when created with the `AutoTokenizer.from_pretrained(pretrained_model_name_or_path)` class method. @@ -75,7 +75,7 @@ class AutoTokenizer(object): pretrained_model_name_or_path: either: - a string with the `shortcut name` of a predefined tokenizer to load from cache or download, e.g.: ``bert-base-uncased``. - - a path to a `directory` containing vocabulary files required by the tokenizer, for instance saved using the :func:`~pytorch_transformers.PreTrainedTokenizer.save_pretrained` method, e.g.: ``./my_model_directory/``. + - a path to a `directory` containing vocabulary files required by the tokenizer, for instance saved using the :func:`~transformers.PreTrainedTokenizer.save_pretrained` method, e.g.: ``./my_model_directory/``. - (not applicable to all derived classes) a path or url to a single saved vocabulary file if and only if the tokenizer only requires a single vocabulary file (e.g. Bert, XLNet), e.g.: ``./my_model_directory/vocab.txt``. cache_dir: (`optional`) string: @@ -90,7 +90,7 @@ class AutoTokenizer(object): inputs: (`optional`) positional arguments: will be passed to the Tokenizer ``__init__`` method. - kwargs: (`optional`) keyword arguments: will be passed to the Tokenizer ``__init__`` method. Can be used to set special tokens like ``bos_token``, ``eos_token``, ``unk_token``, ``sep_token``, ``pad_token``, ``cls_token``, ``mask_token``, ``additional_special_tokens``. See parameters in the doc string of :class:`~pytorch_transformers.PreTrainedTokenizer` for details. + kwargs: (`optional`) keyword arguments: will be passed to the Tokenizer ``__init__`` method. Can be used to set special tokens like ``bos_token``, ``eos_token``, ``unk_token``, ``sep_token``, ``pad_token``, ``cls_token``, ``mask_token``, ``additional_special_tokens``. See parameters in the doc string of :class:`~transformers.PreTrainedTokenizer` for details. Examples:: diff --git a/pytorch_transformers/tokenization_bert.py b/transformers/tokenization_bert.py similarity index 99% rename from pytorch_transformers/tokenization_bert.py rename to transformers/tokenization_bert.py index 225152e065..42163cb8ec 100644 --- a/pytorch_transformers/tokenization_bert.py +++ b/transformers/tokenization_bert.py @@ -103,7 +103,7 @@ def whitespace_tokenize(text): class BertTokenizer(PreTrainedTokenizer): r""" Constructs a BertTokenizer. - :class:`~pytorch_transformers.BertTokenizer` runs end-to-end tokenization: punctuation splitting + wordpiece + :class:`~transformers.BertTokenizer` runs end-to-end tokenization: punctuation splitting + wordpiece Args: vocab_file: Path to a one-wordpiece-per-line vocabulary file diff --git a/pytorch_transformers/tokenization_distilbert.py b/transformers/tokenization_distilbert.py similarity index 93% rename from pytorch_transformers/tokenization_distilbert.py rename to transformers/tokenization_distilbert.py index 5a6d02f98d..dfa02926d8 100644 --- a/pytorch_transformers/tokenization_distilbert.py +++ b/transformers/tokenization_distilbert.py @@ -45,7 +45,7 @@ PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { class DistilBertTokenizer(BertTokenizer): r""" Constructs a DistilBertTokenizer. - :class:`~pytorch_transformers.DistilBertTokenizer` is identical to BertTokenizer and runs end-to-end tokenization: punctuation splitting + wordpiece + :class:`~transformers.DistilBertTokenizer` is identical to BertTokenizer and runs end-to-end tokenization: punctuation splitting + wordpiece Args: vocab_file: Path to a one-wordpiece-per-line vocabulary file diff --git a/pytorch_transformers/tokenization_gpt2.py b/transformers/tokenization_gpt2.py similarity index 100% rename from pytorch_transformers/tokenization_gpt2.py rename to transformers/tokenization_gpt2.py diff --git a/pytorch_transformers/tokenization_openai.py b/transformers/tokenization_openai.py similarity index 100% rename from pytorch_transformers/tokenization_openai.py rename to transformers/tokenization_openai.py diff --git a/pytorch_transformers/tokenization_roberta.py b/transformers/tokenization_roberta.py similarity index 100% rename from pytorch_transformers/tokenization_roberta.py rename to transformers/tokenization_roberta.py diff --git a/pytorch_transformers/tokenization_transfo_xl.py b/transformers/tokenization_transfo_xl.py similarity index 100% rename from pytorch_transformers/tokenization_transfo_xl.py rename to transformers/tokenization_transfo_xl.py diff --git a/pytorch_transformers/tokenization_utils.py b/transformers/tokenization_utils.py similarity index 98% rename from pytorch_transformers/tokenization_utils.py rename to transformers/tokenization_utils.py index ec5de3b772..e8ffff3cb9 100644 --- a/pytorch_transformers/tokenization_utils.py +++ b/transformers/tokenization_utils.py @@ -236,13 +236,13 @@ class PreTrainedTokenizer(object): @classmethod def from_pretrained(cls, *inputs, **kwargs): r""" - Instantiate a :class:`~pytorch_transformers.PreTrainedTokenizer` (or a derived class) from a predefined tokenizer. + Instantiate a :class:`~transformers.PreTrainedTokenizer` (or a derived class) from a predefined tokenizer. Args: pretrained_model_name_or_path: either: - a string with the `shortcut name` of a predefined tokenizer to load from cache or download, e.g.: ``bert-base-uncased``. - - a path to a `directory` containing vocabulary files required by the tokenizer, for instance saved using the :func:`~pytorch_transformers.PreTrainedTokenizer.save_pretrained` method, e.g.: ``./my_model_directory/``. + - a path to a `directory` containing vocabulary files required by the tokenizer, for instance saved using the :func:`~transformers.PreTrainedTokenizer.save_pretrained` method, e.g.: ``./my_model_directory/``. - (not applicable to all derived classes) a path or url to a single saved vocabulary file if and only if the tokenizer only requires a single vocabulary file (e.g. Bert, XLNet), e.g.: ``./my_model_directory/vocab.txt``. cache_dir: (`optional`) string: @@ -257,7 +257,7 @@ class PreTrainedTokenizer(object): inputs: (`optional`) positional arguments: will be passed to the Tokenizer ``__init__`` method. - kwargs: (`optional`) keyword arguments: will be passed to the Tokenizer ``__init__`` method. Can be used to set special tokens like ``bos_token``, ``eos_token``, ``unk_token``, ``sep_token``, ``pad_token``, ``cls_token``, ``mask_token``, ``additional_special_tokens``. See parameters in the doc string of :class:`~pytorch_transformers.PreTrainedTokenizer` for details. + kwargs: (`optional`) keyword arguments: will be passed to the Tokenizer ``__init__`` method. Can be used to set special tokens like ``bos_token``, ``eos_token``, ``unk_token``, ``sep_token``, ``pad_token``, ``cls_token``, ``mask_token``, ``additional_special_tokens``. See parameters in the doc string of :class:`~transformers.PreTrainedTokenizer` for details. Examples:: @@ -432,7 +432,7 @@ class PreTrainedTokenizer(object): This won't save modifications other than (added tokens and special token mapping) you may have applied to the tokenizer after the instantion (e.g. modifying tokenizer.do_lower_case after creation). - This method make sure the full tokenizer can then be re-loaded using the :func:`~pytorch_transformers.PreTrainedTokenizer.from_pretrained` class method. + This method make sure the full tokenizer can then be re-loaded using the :func:`~transformers.PreTrainedTokenizer.from_pretrained` class method. """ if not os.path.isdir(save_directory): logger.error("Saving directory ({}) should be a directory".format(save_directory)) @@ -469,7 +469,7 @@ class PreTrainedTokenizer(object): """ Save the tokenizer vocabulary to a directory. This method does *NOT* save added tokens and special token mappings. - Please use :func:`~pytorch_transformers.PreTrainedTokenizer.save_pretrained` `()` to save the full Tokenizer state if you want to reload it using the :func:`~pytorch_transformers.PreTrainedTokenizer.from_pretrained` class method. + Please use :func:`~transformers.PreTrainedTokenizer.save_pretrained` `()` to save the full Tokenizer state if you want to reload it using the :func:`~transformers.PreTrainedTokenizer.from_pretrained` class method. """ raise NotImplementedError @@ -916,7 +916,7 @@ class PreTrainedTokenizer(object): # To avoid mixing byte-level and unicode for byte-level BPT # we need to build string separatly for added tokens and byte-level tokens - # cf. https://github.com/huggingface/pytorch-transformers/issues/1133 + # cf. https://github.com/huggingface/transformers/issues/1133 sub_texts = [] current_sub_text = [] for token in filtered_tokens: diff --git a/pytorch_transformers/tokenization_xlm.py b/transformers/tokenization_xlm.py similarity index 100% rename from pytorch_transformers/tokenization_xlm.py rename to transformers/tokenization_xlm.py diff --git a/pytorch_transformers/tokenization_xlnet.py b/transformers/tokenization_xlnet.py similarity index 100% rename from pytorch_transformers/tokenization_xlnet.py rename to transformers/tokenization_xlnet.py From 9fabc0b6a953ff2a2a5e1ac9e150531e05cbe89a Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 26 Sep 2019 11:21:34 +0200 Subject: [PATCH 148/219] wip readme --- README.md | 82 ++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 63 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 8acdc8eb09..8f4fae5314 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,62 @@ # 🤗 Transformers -[![CircleCI](https://circleci.com/gh/huggingface/transformers.svg?style=svg)](https://circleci.com/gh/huggingface/transformers) +

+
+ +
+

+

+ + Build + + + GitHub + + + Documentation + + + GitHub release + +

-Transformers (formerly known as `pytorch-pretrained-bert`) is a library of state-of-the-art pre-trained models for Natural Language Processing (NLP). +🤗 Transformers (formerly known as `pytorch-transformers` and `pytorch-pretrained-bert`) is a state-of-the-art Natural Language Processing (NLP) library for TensorFlow 2.0 and PyTorch. -The library currently contains PyTorch implementations, pre-trained model weights, usage scripts and conversion utilities for the following models: +🤗 Transformers provides general-purpose architectures (BERT, GPT, GPT-2, RoBERTa, XLM, DistilBert, XLNet...) for Natural Language Understanding (NLU) and Natural Language Generation (NLG) with more than 32+ pretrained checkpoints, some of them available in 100+ languages. + +The best of both worlds +- As easy to use as pytorch-transformers +- As powerful and concise as Keras +- High performance on NLU and NLG tasks +- Low barrier to entry for educators and practitioners + +State-of-the-art NLP for everyone +- Deep learning researchers +- Hands-on practitioners +- AI/ML/NLP teachers and educators + +Lower compute costs, smaller carbon footprint +- Researchers can share trained models instead of always retraining +- Practitioners can reduce compute time and production costs +- 8 architectures with over 30 pretrained models, some in more than 100 languages + +Choose the right framework for every part of a model's lifetime +- Train state-of-the-art models in 3 lines of code +- Move a single model between frameworks at will +- Seamlessly pick the right framework for training, evaluation, production + + +| Section | Description | +|-|-| +| [Model architectures](#model-architectures) | Architectures (with pretrained weights) | +| [Installation](#installation) | How to install the package | +| [Online demo](#online-demo) | Experimenting with this repo’s text generation capabilities | +| [Quick tour: Usage](#quick-tour) | Tokenizers & models usage: Bert and GPT-2 | +| [Quick tour: Fine-tuning/usage scripts](#quick-tour-of-the-fine-tuningusage-scripts) | Using provided scripts: GLUE, SQuAD and Text generation | +| [Migrating from pytorch-pretrained-bert to transformers](#Migrating-from-pytorch-pretrained-bert-to-transformers) | Migrating your code from pytorch-pretrained-bert to transformers | +| [Documentation](https://huggingface.co/transformers/) | Full API documentation and more | + +## Model architectures 1. **[BERT](https://github.com/google-research/bert)** (from Google) released with the paper [BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding](https://arxiv.org/abs/1810.04805) by Jacob Devlin, Ming-Wei Chang, Kenton Lee and Kristina Toutanova. 2. **[GPT](https://github.com/openai/finetune-transformer-lm)** (from OpenAI) released with the paper [Improving Language Understanding by Generative Pre-Training](https://blog.openai.com/language-unsupervised/) by Alec Radford, Karthik Narasimhan, Tim Salimans and Ilya Sutskever. @@ -18,15 +70,6 @@ The library currently contains PyTorch implementations, pre-trained model weight These implementations have been tested on several datasets (see the example scripts) and should match the performances of the original implementations (e.g. ~93 F1 on SQuAD for BERT Whole-Word-Masking, ~88 F1 on RocStories for OpenAI GPT, ~18.3 perplexity on WikiText 103 for Transformer-XL, ~0.916 Peason R coefficient on STS-B for XLNet). You can find more details on the performances in the Examples section of the [documentation](https://huggingface.co/transformers/examples.html). -| Section | Description | -|-|-| -| [Installation](#installation) | How to install the package | -| [Online demo](#online-demo) | Experimenting with this repo’s text generation capabilities | -| [Quick tour: Usage](#quick-tour) | Tokenizers & models usage: Bert and GPT-2 | -| [Quick tour: Fine-tuning/usage scripts](#quick-tour-of-the-fine-tuningusage-scripts) | Using provided scripts: GLUE, SQuAD and Text generation | -| [Migrating from pytorch-pretrained-bert to transformers](#Migrating-from-pytorch-pretrained-bert-to-transformers) | Migrating your code from pytorch-pretrained-bert to transformers | -| [Documentation](https://huggingface.co/transformers/) | Full API documentation and more | - ## Installation This repo is tested on Python 2.7 and 3.5+ (examples are tested only on python 3.5+) and PyTorch 1.0.0+ @@ -89,13 +132,14 @@ from transformers import * # Transformers has a unified API # for 7 transformer architectures and 30 pretrained weights. # Model | Tokenizer | Pretrained weights shortcut -MODELS = [(BertModel, BertTokenizer, 'bert-base-uncased'), - (OpenAIGPTModel, OpenAIGPTTokenizer, 'openai-gpt'), - (GPT2Model, GPT2Tokenizer, 'gpt2'), - (TransfoXLModel, TransfoXLTokenizer, 'transfo-xl-wt103'), - (XLNetModel, XLNetTokenizer, 'xlnet-base-cased'), - (XLMModel, XLMTokenizer, 'xlm-mlm-enfr-1024'), - (RobertaModel, RobertaTokenizer, 'roberta-base')] +MODELS = [(BertModel, BertTokenizer, 'bert-base-uncased'), + (OpenAIGPTModel, OpenAIGPTTokenizer, 'openai-gpt'), + (GPT2Model, GPT2Tokenizer, 'gpt2'), + (TransfoXLModel, TransfoXLTokenizer, 'transfo-xl-wt103'), + (XLNetModel, XLNetTokenizer, 'xlnet-base-cased'), + (XLMModel, XLMTokenizer, 'xlm-mlm-enfr-1024'), + (DistilBertModel, DistilBertTokenizer, 'distilbert-base-uncased'), + (RobertaModel, RobertaTokenizer, 'roberta-base')] # Let's encode some text in a sequence of hidden-states using each model: for model_class, tokenizer_class, pretrained_weights in MODELS: From f47f7f4611cfb7f61df663741b421686580f5a46 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 26 Sep 2019 11:28:44 +0200 Subject: [PATCH 149/219] add logo --- README.md | 2 +- docs/source/imgs/transformers_logo_name.png | Bin 0 -> 8868 bytes 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 docs/source/imgs/transformers_logo_name.png diff --git a/README.md b/README.md index 8f4fae5314..ed59a56f0b 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@


- +

diff --git a/docs/source/imgs/transformers_logo_name.png b/docs/source/imgs/transformers_logo_name.png new file mode 100644 index 0000000000000000000000000000000000000000..5e4c2dcf575b7f7cf7e64640dee771fc311b7068 GIT binary patch literal 8868 zcmYj%cQjnzAMNNf>L}48qmAC86O1uhf)FJ{XY>*xqBCUl-n(cKy+nyFI?+WKMDIio zqQB((Tkoy+&pmgo`#EQy&)%PP?p>dK!!VxiD=$5~_puI29-0f*M0U$~_7LikF`e21V@WzCPBT zo5Q2C!fvBk3md!lDAvmJH={rAU*BB{NH$0ealvuD*3lX(GNo#Peq7<|Kc$)WYD}^ zlncc~*qp%!5fp6s?5XJN`zs>Zl7UASj`u=`_XPRv;RM4zGmH=4ga6GL??eVUuoTs? zVHcVYCT`IzKGk4S8|b~Wp_LX~r6eYaKCV=q)cIeiIkJnDT%s#&8Nu?^euRUp!&g)q zn;b7LyuVMz;yCL-o635a^??;V(|>!fOIJH$jZRcM9{_~^Sr+YE3>XP;df%?T_@nIJ z@l5!z*}+53mWPlj@6QcS^Yg7%hmaWA|FW>9Og{?i5z($EhmemW(&EDI_@bOnEBXBH z@L0^T!4cABckb7umOLKAD!nu&6L#FCi|#*JsUFaC{|`4z?Z`4(Ta8o@FEi|e+~K`b z$iv$x630{YwRNAj_fM+rMVUTl|5f8Jw;dyZ_ahws#lq8!tcHaD2nWA{K|Q~6fy2A5 zG7sB!9a!@L(F*>knofTY+rB8u)1>)mn`0|`R)-IXv;Qbro)h}GJv+& zc~fxB8DuE0z|7~&lrE8>#r5QzBHq0IHm!&YF<{n8CM;Kbq&e&%&kfF|zS}B%$OaZ8 zOkaLSVUq@Y>VD2HhiD-7v}eS$9EYP}Q}G1YhQV&$khN_ie0Jb~Ax)eXHgiap2K&vv z{AOqe2N>vts_6Ofii3h?(UFY;0>1WeCyNK5Qxx&?y+cGFYd;y03WQa^#}ul4Y$PBD zVVkqYf%rN7q(O|Yvu-ukQqT8)N|-P-ps_0}xc%>1n;{w#aV+5Sz~L?hE`QS^=-=G| z)0Zh_G09b&3|KvJw3xXzzK>;(L?{n~Phz6pWKv)Un&A(pt|VO)zYwMXGN9C4uy%-4 zJzsDCVgs|Z%wvs2dZSP897;Ke>^Ok9Zc!4A{RZ|**ycEKAhNZ+s<(8oi}jWv4VKUK z-t?-JVTeZch{9|LILci3W$CcoBsdiooDmPR6|kr{ZmQ^Mx(dim{mjhgP$DTg&OWIsp2Hs zwBD_)_X|5J>(6U(7_NxXe_Bz5NMQXZcOrV&S{ih)Q3(rog zf?AIGNp=wu+l2Qqcz_2M_{sKfLE&Prv_mP|@5PA)e%vtZj{XFf-omLe>>fz1(76*k zIm;D-b%BO2sZgxn$)hH3#CP={TljeSnzO6pF?Lzy!+6vZUA%V9!zR7 zyg;y~k~*t}Pn)vaF8=L?7U%H__vvA_*?H+cOGiu(xaskI%fJU+>zyZ9glg!tSf!@H z*P)jZT8G*zPLKqBUEy8v3F}Mft$zH~5pUJ=gcB;!vF9f()@$FWycW+xJtgx$Cr*22 z)1(>FOf^O;>DpN)IR(WgGk?6(&wZBR>Ud4O(5N%7`1>|7iKYrYfa0b3+}UWz-?Nlp*A+GvNJFRy(gW*s7oe|yCDGSEFro*PHXlpOo>TZxQrY-I^n z1PI&tu+`0{)f?k z&?Q&jWn978W>Tj$M`g!Y^>5}SV<$C4+ljhtrn=f^|-m*)3m#?|86o0>CueVjIU+;X@W%4 zO`E0h^AZUWjdE@3g@vnnK2~5!PN`n^x=a8^Xf591^`CK{^cE%dU%rK7(}?SRhQ7d) z9aTTJ?%K*)KT~~`3R3JALe|)+wCvbIJB;E16J)h?=VyYo&w8uJ3m1$ijcX1>j2Y_1 zC>Ncj%w+B10@(m{fGshpoPQ?C?S-Bb)gXutbZ$x8gek*Kvz z_j0pek-nMbY7i1s(Y1nWY0t@;FWz}5nbLVvlm0==ZE1s^vR;gzt4Byrn4D_+YSwcl z9qu#E%@RK=yuwtv$qxEr5J*m#Qr`0I_f~a7BL7I&_AAcDob_62r=$ z{&zkI1^1xLx1V;_j$6Z-prM6PFXyM8MSSf6rFafKxUUc2gqFvO6+W{SO)0z}t*+h* z16@iwK81#`;Idj8iU|h=G4<+qC}o5vrN!$1QA59X;Y;?g?~>U~3r0yMJ=zKY15IqL#L<+o#cn+niEy+a;}+No*Kowud6K!bd`^2eU~W^?h3(TTSbTwR$lT)p}f_;|M5 zEITcnxkb^WDyUQcHAdw3W4JfUY~cp2NFf;HJi)En!bNP7H(h}Mg0xxk<{w7%QqLGz z7Zovna}lEbo>cSA!|cyb9d!nqSZ$HU=Zx{-eeppL;adECF{+7BWzh}c!4dPlJG+9A z9P^3F*Im)mI&q{n#p&emCYJhBi4W531Q)Rx(iTS3X>Hze_T`aHw!8I0NI|93kXs!~ zZw4B#=rqCNGKj%P;;ZK2@EC}x1!Yn!#`d9b<5J;@8`-IUI4c$FIWj0l5_k}fc7B{{ zfnEyhY+GA>0Su8{gho#Ub&l3|by%_>*LF2Uj9_ZyGw)c&^`wmzl}Ye)YU?&nJRCub zj`&D8$M#UGoRWd}+8B4FTz*Zq_=EQ~vVC}|;qSW80~?f;GR@5yag!y)1Bh@@de0Qe zo$N4LL)TN^i8p?Kb9<*JYL^oVe)Rm+f)!5k=jYrMo^Y;xB|cSCo&q`BL-691^KIHtg*|= z3gUa5>=joANUx^hG>P2#p2s)OZ)z(%s@>}FuV=#gju$t)_u9O-KX=U{h3p-Pxvg|Rx$}SHw_ZIMSw9iOVG&n885?+N+M!9{sUo9 zpLy{8C?Z*QRa4qE$ZF5ouy%WFkJ`y8f#2xS zb*A^N{`~vjZ@UOxHvQzr2)axa{;>W$zgWc3FuakDL4;kSQ~-~boDJQ}q9D*ud%@L? zCCRoljurb45jw&q>q?gvZZU~Ol*z6f78sXdwrFMxdcE0?)+`RIdZT7f+R{5^GV`_@ zbHp;zSYxj8GAs(LN_%@I@VaEX*Q~l~^m<{!6Q)3d%2$_Ds^i>rIP(YggMdo!!fyf# zW}yKd?B}wOM{>7^^5+E~_SHU+Pcgq~VvG-8r<|*=mh5{)NiV*iZ{XA|OynD%uK6x4 zlGhJjr}q_UZmOb1W?m?6&$&Dqb74=Ua$+!a)xzY#uX4)Vf0nez@JW(OYB%)FH!o)e z;*gJ`)kVX{J?s2AYL0Tbu|Ydvp!ZMK$j}FsI9OZSI zTpd6|R_w01M<({3JY&*keFvr2r0yGNY&(pJkf?ato?XE?o1di7n3;e#;O zG>Ze)`DQlC6Nejp~GU#HB2u8LeZ zanIOuno(ok3)ES#7b+-c!UN&i>_1dEPpB~0bMXlnV$xcL=L$e zG)r|!NIxD=J5OW%+k=>%p1!&NrRSIdf#pqi)G-3J1Ixr|_m;XaIr4jGGt=uy_w%b* z53A|P=W<@GlHuTW`CjG4K6B-Owd)nj7{v^S(RW)0?7F+?1>I@dHA|tP8|XOkL$iZg z?#V1>P1AlHWp;u~Xjz(;q@vPv$g-)75Z-CZ!x8>#@t5@BXaI!?pYgfJaVf0txE(I7P@cn(+Jz&M4@b$%ZAV=!-Z4D&tw|r$)QtXo?l}~P z`nxj%`B4|EWJ8tAwkGkSHwxtfOWRELLC>;iVG3A*ybR8(qA@Bi`;14TbWA3H`jVRi zKkiOn+RR^LYAWQlI0&vV0x5?T9n^)`GjfY}QMcVs|6o=&;v?PRHKdHsdXnvCaPB-D zW1Y_#^X%eD9KU-GIY-+Uwp~lIl!O(EkIb8+o4yFATUCrP+CRMuc|oE0y+#3%TdQe4 zYsVm6Q#Eu$fuCGyyc(L8kXab0R$fuD!7#-Jj(bMFk0%;nTGdft+Vow8)7v}42W2+1 z)KbF#mHDXSe*g5$HVw-aGu+q(3qHogQy5Y}qJ3wTe~PI1X*RNy6f#ix_?anE*8i1?jtg!;W-F zz;mwa*Y|#47g#U*z-uJmF*(UyxRZjUDur;gQAm35;JKyeio34G%CA910){ZxbhY1{ z2yCKnO`84Xyd{eutXy}@m+woxa;@_L3k8g#X^jb8=O`lQo&?X!re|j}<4VN{`0~VJ z&eca8e*!+dFm&ULlz)q(9yD16CGrrnH=}DL1xwU+2#dvj-}ybOnEaQJg@j$q$wP7S zaAc1~%op>R`+87E=39a#`+dZ#A9*{6Ln4Jzgh`LMfX(=sMelFu*G}6L8q|FrME9sd zqiP}Gfa3&??Mw`r^R69LBb=4|V=qQ}`_aR!4r=OM!9`rUl{s9D=951K%58^Ghx(42 zZuVP24KM$LdNZv#~%eg^1GjRPV%xUChvY0+bmMbj(A zTpL(&77k=NK7W}(30gjokuk^AE7V#pQQ7i?V|=P8zT<^NB|t($R`WVMW1a499rP-` z`y4?TJXdgHRg7$Jj+!laxt=^nq{Ddgzg4Uz(i##X4e@F`FM>6@eF~kD`l>~*MJKkC z9%GE(QTD)vhU>*K5*Q?L#nIA3>+G^ip+OK9OVV4?7J);aq+7H7x&=-j_Pp?W) zM}Fdp9rcL)<g%xeusAyi$xsI_uyctp%;E)2)il{KpJJ3q!`|{At)Gt@mY0;*`yMng)Ve zxi-Y68LSv=yCilKBIWqm9B+51i^j>z#vNFij+-j#{@53fAwT)2KT^{wUzDg{r&{ikOS z2ax&8@n0f+x8&>GW2?H!s?_=n>Plti8DY<8(=+ie2MvE6mDU;9 z5~*Dod(x%@AEMpfvk%jl=Ve`P3TIC6rt2otV7;ZYKlIAYYEEluJ>^i zbXrrezRV@Zxq8HJwkM+m0|zy{iWNMj4EQRc)g%_dhZ(dzIZoPbHoOB{lJA?s##jsP zK6=V_@@?{|n0J~vrX75H?uhb=xzaFPoLL9faaWDWykOm%=xBEhcbY1uC@%u>@p0j@yYkuB$ps$_bu$iT z4_rbmpr6kku4mP z9DGdpw2zl@u!H#$?k)r!5|SM#qu>%egKdiq3)cQ*$7@FgHSSDC7Lbq$xe*;wmbc-D zv1pf;h{+=r)C=n2eq2$WK0<@xL7;Fd1H^`zp5oBm0WMz>WDJ^t25F>J=kRkQiXkJ% zLFTtxZ;>pGT%+e!0;D}C6hk4mQSc^~w9Bt)W=D54cE@{{J{xeK1a{m$7!sAY|3~F% z9KlNL;ogY}1^~ZpeS`bnX79i!y}uderBi{X7ouc>NR5#ltNDr-E*xJK-2>Fm)+IWB z86_LpJ&16$>?bNd!8=D11=|N(C4LkMH~TVy_~^H&rF5)5M?#Jo>2nb*5js~RIQv54 zHR=Wz+C?=6ixEf9?{yZg43>+3+?i3e}yVy_uoGSjShxyymlolawD4*0jP8-OS10BCeHu?pThbBDCBghaL zY?Y~~#ZE!>GGAD*|FfJuw>!XmI=WMDq$Qy;+NaU5cnR(%@!5S^t(Decc3<}QW@Uf(?q8b@O2kwGdy z$8n1%y=(?HFo~~y8~%3h+qZwQy0E<$YobqFWjx5kI$;Vg7>*c3>;C8*RjmFa!75J< zC}PmGm@Qcy;(IlaKzP$_B)#@#ODt7Li@A`4Dv|WMweN6d@STuFbk@-JqUFdg*-j%9 zAdh*K-H`}_2%eS)YJ>5Bnm?+wZj@-D@Zuo^-~xN)t0<=GzJgwcvWu~#Q~ktlTZI_# zxcc7X_mXn@k=ozW$Cmda^RqEjVi9rWt8-(K!)$9bmWZI zCZ)QG88wOjjCy4ud^=W><-vCBT6sP9w8$JC&CU#s$XeKVdRpRiq==vH>P7lIz@55q zE0=XYN`;Err^5vLfv5}#KleuE_KY%hen4MCak<{zNaSzNQ$g_rTYH?_G_e2JNWIj2 z;MuopV-5#I17pa~iN)^u+BqIo0a?gD>H;)1vR%3SLxJh`W;_6^cv12o+(9Q4B)j;1 z`6ZA5$D!A&bg-Q!)c(F_JA-AlJE)*P}5-z;g`Z7bif8S5f4;qq;=L-%a+TI$ z=L=Tpm7x+o+>m|#4Dm{)uV9yNvoZW<3Fqn0;=3kMvb}pB2M_&aW}x;5r~?NOTO}A= z+{MZo!-;`;=#c9&Vc8NQSNMpbnYVrT5T4I`Ap`q0>?}K}_Le78V@S*4H~gsug{B9Z z!(pvI5BE6}!V82kE1kd<8UEjsZ|6f569<3+>C#C)%)%TSkY*)Q2z7Gwh4cu z-!aPHg;k^4AI1|fO5cSxspFZHnJ|_-o&IhsmsdLx7yO0Tmc)o0lot<(hovw7AXj9A z`5zf4tH4;vV?=9Ix?}&DP75d#{wd!vY-2Napt7gv@6%;%DQ3bvnd&i`ZGHvwHG|Zu z&;m<%^C&rpOC>piSbQ}+~r6|r|9 z6Y%A>Q_=J~mzBgl@ESEw;j9~`ko2;jgD^epv!x1lU3YtSGy;w$4?S1%`~sZJvsX-^ zKN+X8{ePLZy^q;{sE{g18gpiQE27R{aI-*E@);&~+g8qGtg-MaB5>4IS z)T0nEqP(IPu~RPVc0K=Pu?GOrHU<@YN#>teqIABh|79DyCEXnP%IQ5lmv986I3p}s zdNv6}PY0v5C~bPO-jG;N0in255Lx{XG8xGN9P%_P)JnVGd|IcK7e;!CQJ@1I7)B%V z4UV|z>NEdN&fzJ3O%)D~7y2QlEn~$=Ebz};Yz|XPn!XRsKD87OqaghCnHf?$2}9Vi zr@R(R0R2#Zu+;72*A@+ziNLkOTA?762I=x(KaWuepv;$wYL)yCoPB#_)<^m7=s{5&HE{y$yrbwU4I3*E=@2jy$2~Dud)W)3W zB-@R<@Dz!>+*z#lU*;e&qV)T0yIdABEb`|R|5do`<4%xbJY|Hh#S3P%c#wLa&TnTz z?B^v)2@o>4rh>}e6J<0_k8;Swr#upr7<<-%G$)M;8IUZ-LFc_j9C}RF^4CFzX^QlR zs{r)^?`bEEK04{6zm~TxTMT9VKerYBG7+S~+3F-JJ@|GOhq;iP{q%N*9>>TE3k*We zB^h(OBd|UUDTDruwg{jTQ>+Y>Y#}g65K4zTuGupxM(iKuE~asyaGfacfFgaJLM7l^ zTW{8OvO6O@5~QUo8P54#ll3onm!C2;N;Mcl{gV2& z0BBUkCAJJ9{}AwTmVb$KH4Oh%gvdtFx#b0Uhf#C`nA;@@D>jG3HV@MZ+GEpPx zaYv&Xo^$eLKb?e+zRS+#S5;7qjf;s^9Vg{C%@vwa+i$B9!T&7^$4BOe3krzwAtIfK w8B4h8!T*-~2l@XEZ>8=`QCI-Ryd--^%ntPLZzlj%MGZtH+$`w-0Q<>!ssI20 literal 0 HcmV?d00001 From 4ddc31ff40b399ded2b8613a32697d4c2f6505c4 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 26 Sep 2019 12:00:38 +0200 Subject: [PATCH 150/219] update readme with migration change --- README.md | 115 ++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 90 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index ed59a56f0b..2a88aaeaf7 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,3 @@ -# 🤗 Transformers -


@@ -20,11 +18,11 @@

-🤗 Transformers (formerly known as `pytorch-transformers` and `pytorch-pretrained-bert`) is a state-of-the-art Natural Language Processing (NLP) library for TensorFlow 2.0 and PyTorch. +State-of-the-art Natural Language Processing (NLP) for TensorFlow 2.0 and PyTorch. -🤗 Transformers provides general-purpose architectures (BERT, GPT, GPT-2, RoBERTa, XLM, DistilBert, XLNet...) for Natural Language Understanding (NLU) and Natural Language Generation (NLG) with more than 32+ pretrained checkpoints, some of them available in 100+ languages. +🤗 Transformers (formerly known as `pytorch-transformers` and `pytorch-pretrained-bert`) provides general-purpose architectures (BERT, GPT, GPT-2, RoBERTa, XLM, DistilBert, XLNet...) for Natural Language Understanding (NLU) and Natural Language Generation (NLG) with more than 32+ pretrained checkpoints in 100+ languages. -The best of both worlds +Features - As easy to use as pytorch-transformers - As powerful and concise as Keras - High performance on NLU and NLG tasks @@ -42,34 +40,23 @@ Lower compute costs, smaller carbon footprint Choose the right framework for every part of a model's lifetime - Train state-of-the-art models in 3 lines of code -- Move a single model between frameworks at will +- Deep interoperability between TensorFlow 2.0 and PyTorch models +- Move a single model between TF2.0/PyTorch frameworks at will - Seamlessly pick the right framework for training, evaluation, production | Section | Description | |-|-| -| [Model architectures](#model-architectures) | Architectures (with pretrained weights) | | [Installation](#installation) | How to install the package | +| [Model architectures](#model-architectures) | Architectures (with pretrained weights) | | [Online demo](#online-demo) | Experimenting with this repo’s text generation capabilities | | [Quick tour: Usage](#quick-tour) | Tokenizers & models usage: Bert and GPT-2 | +| [Quick tour: TF 2.0 and PyTorch ](#Quick-tour-TF-2.0-training-and-PyTorch-interoperability) | Train a TF 2.0 model in 10 lines of code, load it in PyTorch | | [Quick tour: Fine-tuning/usage scripts](#quick-tour-of-the-fine-tuningusage-scripts) | Using provided scripts: GLUE, SQuAD and Text generation | -| [Migrating from pytorch-pretrained-bert to transformers](#Migrating-from-pytorch-pretrained-bert-to-transformers) | Migrating your code from pytorch-pretrained-bert to transformers | +| [Migrating from pytorch-transformers to transformers](#Migrating-from-pytorch-pretrained-bert-to-transformers) | Migrating your code from pytorch-pretrained-bert to transformers | +| [Migrating from pytorch-pretrained-bert to pytorch-transformers](#Migrating-from-pytorch-pretrained-bert-to-transformers) | Migrating your code from pytorch-pretrained-bert to transformers | | [Documentation](https://huggingface.co/transformers/) | Full API documentation and more | -## Model architectures - -1. **[BERT](https://github.com/google-research/bert)** (from Google) released with the paper [BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding](https://arxiv.org/abs/1810.04805) by Jacob Devlin, Ming-Wei Chang, Kenton Lee and Kristina Toutanova. -2. **[GPT](https://github.com/openai/finetune-transformer-lm)** (from OpenAI) released with the paper [Improving Language Understanding by Generative Pre-Training](https://blog.openai.com/language-unsupervised/) by Alec Radford, Karthik Narasimhan, Tim Salimans and Ilya Sutskever. -3. **[GPT-2](https://blog.openai.com/better-language-models/)** (from OpenAI) released with the paper [Language Models are Unsupervised Multitask Learners](https://blog.openai.com/better-language-models/) by Alec Radford*, Jeffrey Wu*, Rewon Child, David Luan, Dario Amodei** and Ilya Sutskever**. -4. **[Transformer-XL](https://github.com/kimiyoung/transformer-xl)** (from Google/CMU) released with the paper [Transformer-XL: Attentive Language Models Beyond a Fixed-Length Context](https://arxiv.org/abs/1901.02860) by Zihang Dai*, Zhilin Yang*, Yiming Yang, Jaime Carbonell, Quoc V. Le, Ruslan Salakhutdinov. -5. **[XLNet](https://github.com/zihangdai/xlnet/)** (from Google/CMU) released with the paper [​XLNet: Generalized Autoregressive Pretraining for Language Understanding](https://arxiv.org/abs/1906.08237) by Zhilin Yang*, Zihang Dai*, Yiming Yang, Jaime Carbonell, Ruslan Salakhutdinov, Quoc V. Le. -6. **[XLM](https://github.com/facebookresearch/XLM/)** (from Facebook) released together with the paper [Cross-lingual Language Model Pretraining](https://arxiv.org/abs/1901.07291) by Guillaume Lample and Alexis Conneau. -7. **[RoBERTa](https://github.com/pytorch/fairseq/tree/master/examples/roberta)** (from Facebook), released together with the paper a [Robustly Optimized BERT Pretraining Approach](https://arxiv.org/abs/1907.11692) by Yinhan Liu, Myle Ott, Naman Goyal, Jingfei Du, Mandar Joshi, Danqi Chen, Omer Levy, Mike Lewis, Luke Zettlemoyer, Veselin Stoyanov. -8. **[DistilBERT](https://github.com/huggingface/transformers/tree/master/examples/distillation)** (from HuggingFace), released together with the blogpost [Smaller, faster, cheaper, lighter: Introducing DistilBERT, a distilled version of BERT](https://medium.com/huggingface/distilbert-8cf3380435b5 -) by Victor Sanh, Lysandre Debut and Thomas Wolf. - -These implementations have been tested on several datasets (see the example scripts) and should match the performances of the original implementations (e.g. ~93 F1 on SQuAD for BERT Whole-Word-Masking, ~88 F1 on RocStories for OpenAI GPT, ~18.3 perplexity on WikiText 103 for Transformer-XL, ~0.916 Peason R coefficient on STS-B for XLNet). You can find more details on the performances in the Examples section of the [documentation](https://huggingface.co/transformers/examples.html). - ## Installation This repo is tested on Python 2.7 and 3.5+ (examples are tested only on python 3.5+) and PyTorch 1.0.0+ @@ -112,6 +99,22 @@ It contains an example of a conversion script from a Pytorch trained Transformer At some point in the future, you'll be able to seamlessly move from pre-training or fine-tuning models in PyTorch to productizing them in CoreML, or prototype a model or an app in CoreML then research its hyperparameters or architecture from PyTorch. Super exciting! +## Model architectures + +🤗 Transformers currently provides 8 NLU/NLG architectures: + +1. **[BERT](https://github.com/google-research/bert)** (from Google) released with the paper [BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding](https://arxiv.org/abs/1810.04805) by Jacob Devlin, Ming-Wei Chang, Kenton Lee and Kristina Toutanova. +2. **[GPT](https://github.com/openai/finetune-transformer-lm)** (from OpenAI) released with the paper [Improving Language Understanding by Generative Pre-Training](https://blog.openai.com/language-unsupervised/) by Alec Radford, Karthik Narasimhan, Tim Salimans and Ilya Sutskever. +3. **[GPT-2](https://blog.openai.com/better-language-models/)** (from OpenAI) released with the paper [Language Models are Unsupervised Multitask Learners](https://blog.openai.com/better-language-models/) by Alec Radford*, Jeffrey Wu*, Rewon Child, David Luan, Dario Amodei** and Ilya Sutskever**. +4. **[Transformer-XL](https://github.com/kimiyoung/transformer-xl)** (from Google/CMU) released with the paper [Transformer-XL: Attentive Language Models Beyond a Fixed-Length Context](https://arxiv.org/abs/1901.02860) by Zihang Dai*, Zhilin Yang*, Yiming Yang, Jaime Carbonell, Quoc V. Le, Ruslan Salakhutdinov. +5. **[XLNet](https://github.com/zihangdai/xlnet/)** (from Google/CMU) released with the paper [​XLNet: Generalized Autoregressive Pretraining for Language Understanding](https://arxiv.org/abs/1906.08237) by Zhilin Yang*, Zihang Dai*, Yiming Yang, Jaime Carbonell, Ruslan Salakhutdinov, Quoc V. Le. +6. **[XLM](https://github.com/facebookresearch/XLM/)** (from Facebook) released together with the paper [Cross-lingual Language Model Pretraining](https://arxiv.org/abs/1901.07291) by Guillaume Lample and Alexis Conneau. +7. **[RoBERTa](https://github.com/pytorch/fairseq/tree/master/examples/roberta)** (from Facebook), released together with the paper a [Robustly Optimized BERT Pretraining Approach](https://arxiv.org/abs/1907.11692) by Yinhan Liu, Myle Ott, Naman Goyal, Jingfei Du, Mandar Joshi, Danqi Chen, Omer Levy, Mike Lewis, Luke Zettlemoyer, Veselin Stoyanov. +8. **[DistilBERT](https://github.com/huggingface/transformers/tree/master/examples/distillation)** (from HuggingFace), released together with the blogpost [Smaller, faster, cheaper, lighter: Introducing DistilBERT, a distilled version of BERT](https://medium.com/huggingface/distilbert-8cf3380435b5 +) by Victor Sanh, Lysandre Debut and Thomas Wolf. + +These implementations have been tested on several datasets (see the example scripts) and should match the performances of the original implementations (e.g. ~93 F1 on SQuAD for BERT Whole-Word-Masking, ~88 F1 on RocStories for OpenAI GPT, ~18.3 perplexity on WikiText 103 for Transformer-XL, ~0.916 Peason R coefficient on STS-B for XLNet). You can find more details on the performances in the Examples section of the [documentation](https://huggingface.co/transformers/examples.html). + ## Online demo **[Write With Transformer](https://transformer.huggingface.co)**, built by the Hugging Face team at transformer.huggingface.co, is the official demo of this repo’s text generation capabilities. @@ -123,14 +126,14 @@ You can use it to experiment with completions generated by `GPT2Model`, `Transfo ## Quick tour -Let's do a very quick overview of Transformers. Detailed examples for each model architecture (Bert, GPT, GPT-2, Transformer-XL, XLNet and XLM) can be found in the [full documentation](https://huggingface.co/transformers/). +Let's do a very quick overview of the model architectures in 🤗 Transformers. Detailed examples for each model architecture (Bert, GPT, GPT-2, Transformer-XL, XLNet and XLM) can be found in the [full documentation](https://huggingface.co/transformers/). ```python import torch from transformers import * # Transformers has a unified API -# for 7 transformer architectures and 30 pretrained weights. +# for 8 transformer architectures and 30 pretrained weights. # Model | Tokenizer | Pretrained weights shortcut MODELS = [(BertModel, BertTokenizer, 'bert-base-uncased'), (OpenAIGPTModel, OpenAIGPTTokenizer, 'openai-gpt'), @@ -141,6 +144,8 @@ MODELS = [(BertModel, BertTokenizer, 'bert-base-uncased'), (DistilBertModel, DistilBertTokenizer, 'distilbert-base-uncased'), (RobertaModel, RobertaTokenizer, 'roberta-base')] +# To use TensorFlow 2.0 versions of the models, simply prefix the class names with 'TF', e.g. `TFRobertaModel` is the TF 2.0 counterpart of the PyTorch model `RobertaModel` + # Let's encode some text in a sequence of hidden-states using each model: for model_class, tokenizer_class, pretrained_weights in MODELS: # Load pretrained model/tokenizer @@ -185,6 +190,53 @@ tokenizer = tokenizer_class.from_pretrained('./directory/to/save/') # re-load # SOTA examples for GLUE, SQUAD, text generation... ``` +## Quick tour TF 2.0 training and PyTorch interoperability + +Let's do a quick example of how a TensorFlow 2.0 model can be trained in 12 lines of code with 🤗 Transformers and then loaded in PyTorch for fast inspection/tests. + +```python +import tensorflow as tf +import tensorflow_datasets +from pytorch_transformers import * + +# Load dataset, tokenizer, model from pretrained model/vocabulary +tokenizer = BertTokenizer.from_pretrained('bert-base-cased') +model = TFBertForSequenceClassification.from_pretrained('bert-base-cased') +data = tensorflow_datasets.load('glue/mrpc') + +# Prepare dataset for GLUE as a tf.data.Dataset instance +train_dataset = glue_convert_examples_to_features(data['train'], tokenizer, 128, 'mrpc') +valid_dataset = glue_convert_examples_to_features(data['validation'], tokenizer, 128, 'mrpc') +train_dataset = train_dataset.shuffle(100).batch(32).repeat(2) +valid_dataset = valid_dataset.batch(64) + +# Prepare training: Compile tf.keras model with optimizer, loss and learning rate schedule +optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5, epsilon=1e-08, clipnorm=1.0) +loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) +metric = tf.keras.metrics.SparseCategoricalAccuracy('accuracy') +model.compile(optimizer=optimizer, loss=loss, metrics=[metric]) + +# Train and evaluate using tf.keras.Model.fit() +history = model.fit(train_dataset, epochs=2, steps_per_epoch=115, + validation_data=valid_dataset, validation_steps=7) + +# Load the TensorFlow model in PyTorch for inspection +model.save_pretrained('./save/') +pytorch_model = BertForSequenceClassification.from_pretrained('./save/', from_tf=True) + +# Quickly test a few predictions - MRPC is a paraphrasing task, let's see if our model learned the task +sentence_0 = "This research was consistent with his findings." +sentence_1 = "His findings were compatible with this research." +sentence_2 = "His findings were not compatible with this research." +inputs_1 = tokenizer.encode_plus(sentence_0, sentence_1, add_special_tokens=True, return_tensors='pt') +inputs_2 = tokenizer.encode_plus(sentence_0, sentence_2, add_special_tokens=True, return_tensors='pt') + +pred_1 = pytorch_model(**inputs_1)[0].argmax().item() +pred_2 = pytorch_model(**inputs_2)[0].argmax().item() +print("sentence_1 is", "a paraphrase" if pred_1 else "not a paraphrase", "of sentence_0") +print("sentence_2 is", "a paraphrase" if pred_2 else "not a paraphrase", "of sentence_0") +``` + ## Quick tour of the fine-tuning/usage scripts The library comprises several example scripts with SOTA performances for NLU and NLG tasks: @@ -343,9 +395,22 @@ python ./examples/run_generation.py \ --model_name_or_path=gpt2 \ ``` +## Migrating from pytorch-transformers to transformers + +Here is a quick summary of what you should take care of when migrating from `pytorch-transformers` to `transformers`. + +### Positional order of some models' keywords inputs (`attention_mask`, `token_type_ids`...) changed + +To be able to use Torchscript (see #1010, #1204 and #1195) the specific order of some models **keywords inputs** (`attention_mask`, `token_type_ids`...) has been changed. + +If you used to call the models with keyword names for keyword arguments, e.g. `model(inputs_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)`, this should not cause any change. + +If you used to call the models with positional inputs for keyword arguments, e.g. `model(inputs_ids, attention_mask, token_type_ids)`, you may have to double check the exact order of input arguments. + + ## Migrating from pytorch-pretrained-bert to transformers -Here is a quick summary of what you should take care of when migrating from `pytorch-pretrained-bert` to `transformers` +Here is a quick summary of what you should take care of when migrating from `pytorch-pretrained-bert` to `transformers`. ### Models always output `tuples` From 4dde31cb764fdac176b4c06c327d0bcf8227b094 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 26 Sep 2019 12:18:26 +0200 Subject: [PATCH 151/219] update readme --- README.md | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 2a88aaeaf7..44684bde99 100644 --- a/README.md +++ b/README.md @@ -5,24 +5,27 @@

- Build + Build - GitHub + GitHub - Documentation + Documentation - GitHub release + GitHub release

-State-of-the-art Natural Language Processing (NLP) for TensorFlow 2.0 and PyTorch. +

+

State-of-the-art Natural Language Processing for TensorFlow 2.0 and PyTorch +

-🤗 Transformers (formerly known as `pytorch-transformers` and `pytorch-pretrained-bert`) provides general-purpose architectures (BERT, GPT, GPT-2, RoBERTa, XLM, DistilBert, XLNet...) for Natural Language Understanding (NLU) and Natural Language Generation (NLG) with more than 32+ pretrained checkpoints in 100+ languages. +🤗 Transformers (formerly known as `pytorch-transformers` and `pytorch-pretrained-bert`) provides general-purpose architectures (BERT, GPT-2, RoBERTa, XLM, DistilBert, XLNet...) for Natural Language Understanding (NLU) and Natural Language Generation (NLG) with over 32+ pretrained models in 100+ languages and deep interoperability between TensorFlow 2.0 and PyTorch. + +### Features -Features - As easy to use as pytorch-transformers - As powerful and concise as Keras - High performance on NLU and NLG tasks From cf5c5c9e1cfd79b2654a74bc0f3803e7b78b720a Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Thu, 26 Sep 2019 07:43:13 -0400 Subject: [PATCH 152/219] Documentation --- docs/source/index.rst | 34 +++++++++++++++++++++++++++++-- docs/source/pretrained_models.rst | 6 +++--- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/docs/source/index.rst b/docs/source/index.rst index a205b0b314..3a30b61b2c 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,9 +1,38 @@ Transformers ================================================================================================================================================ -Transformers is a library of state-of-the-art pre-trained models for Natural Language Processing (NLP). +🤗 Transformers (formerly known as `pytorch-transformers` and `pytorch-pretrained-bert`) provides general-purpose architectures +(BERT, GPT-2, RoBERTa, XLM, DistilBert, XLNet...) for Natural Language Understanding (NLU) and Natural Language Generation +(NLG) with over 32+ pretrained models in 100+ languages and deep interoperability between TensorFlow 2.0 and PyTorch. -The library currently contains PyTorch implementations, pre-trained model weights, usage scripts and conversion utilities for the following models: +Features +--------------------------------------------------- + +- As easy to use as pytorch-transformers +- As powerful and concise as Keras +- High performance on NLU and NLG tasks +- Low barrier to entry for educators and practitioners + +State-of-the-art NLP for everyone +- Deep learning researchers +- Hands-on practitioners +- AI/ML/NLP teachers and educators + +Lower compute costs, smaller carbon footprint +- Researchers can share trained models instead of always retraining +- Practitioners can reduce compute time and production costs +- 8 architectures with over 30 pretrained models, some in more than 100 languages + +Choose the right framework for every part of a model's lifetime +- Train state-of-the-art models in 3 lines of code +- Deep interoperability between TensorFlow 2.0 and PyTorch models +- Move a single model between TF2.0/PyTorch frameworks at will +- Seamlessly pick the right framework for training, evaluation, production + +Contents +--------------------------------- + +The library currently contains PyTorch and Tensorflow implementations, pre-trained model weights, usage scripts and conversion utilities for the following models: 1. `BERT `_ (from Google) released with the paper `BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding `_ by Jacob Devlin, Ming-Wei Chang, Kenton Lee and Kristina Toutanova. 2. `GPT `_ (from OpenAI) released with the paper `Improving Language Understanding by Generative Pre-Training `_ by Alec Radford, Karthik Narasimhan, Tim Salimans and Ilya Sutskever. @@ -14,6 +43,7 @@ The library currently contains PyTorch implementations, pre-trained model weight 7. `RoBERTa `_ (from Facebook), released together with the paper a `Robustly Optimized BERT Pretraining Approach `_ by Yinhan Liu, Myle Ott, Naman Goyal, Jingfei Du, Mandar Joshi, Danqi Chen, Omer Levy, Mike Lewis, Luke Zettlemoyer, Veselin Stoyanov. 8. `DistilBERT `_ (from HuggingFace) released together with the blog post `Smaller, faster, cheaper, lighter: Introducing DistilBERT, a distilled version of BERT `_ by Victor Sanh, Lysandre Debut and Thomas Wolf. + .. toctree:: :maxdepth: 2 :caption: Notes diff --git a/docs/source/pretrained_models.rst b/docs/source/pretrained_models.rst index 0e55767d76..4c17b35c84 100644 --- a/docs/source/pretrained_models.rst +++ b/docs/source/pretrained_models.rst @@ -44,15 +44,15 @@ Here is the full list of the currently provided pretrained models together with | +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``bert-large-uncased-whole-word-masking-finetuned-squad`` | | 24-layer, 1024-hidden, 16-heads, 340M parameters. | | | | | The ``bert-large-uncased-whole-word-masking`` model fine-tuned on SQuAD | -| | | (see details of fine-tuning in the `example section `__). | +| | | (see details of fine-tuning in the `example section `__). | | +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``bert-large-cased-whole-word-masking-finetuned-squad`` | | 24-layer, 1024-hidden, 16-heads, 340M parameters | | | | | The ``bert-large-cased-whole-word-masking`` model fine-tuned on SQuAD | -| | | (see `details of fine-tuning in the example section `__) | +| | | (see `details of fine-tuning in the example section `__) | | +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``bert-base-cased-finetuned-mrpc`` | | 12-layer, 768-hidden, 12-heads, 110M parameters. | | | | | The ``bert-base-cased`` model fine-tuned on MRPC | -| | | (see `details of fine-tuning in the example section `__) | +| | | (see `details of fine-tuning in the example section `__) | +-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | GPT | ``openai-gpt`` | | 12-layer, 768-hidden, 12-heads, 110M parameters. | | | | | OpenAI GPT English model | From 4acd87ff4e6a363b47d13b8a960b4b8340a3d615 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Wed, 25 Sep 2019 06:31:05 -0400 Subject: [PATCH 153/219] TF models added to documentation --- docs/source/main_classes/model.rst | 6 +++ docs/source/model_doc/bert.rst | 56 +++++++++++++++++++++++++ docs/source/model_doc/distilbert.rst | 27 ++++++++++++ docs/source/model_doc/gpt.rst | 21 ++++++++++ docs/source/model_doc/gpt2.rst | 21 ++++++++++ docs/source/model_doc/roberta.rst | 21 ++++++++++ docs/source/model_doc/transformerxl.rst | 14 +++++++ docs/source/model_doc/xlm.rst | 28 +++++++++++++ docs/source/model_doc/xlnet.rst | 28 +++++++++++++ 9 files changed, 222 insertions(+) diff --git a/docs/source/main_classes/model.rst b/docs/source/main_classes/model.rst index d22467f907..179ec53d0e 100644 --- a/docs/source/main_classes/model.rst +++ b/docs/source/main_classes/model.rst @@ -13,3 +13,9 @@ The base class ``PreTrainedModel`` implements the common methods for loading/sav .. autoclass:: transformers.PreTrainedModel :members: + +``TFPreTrainedModel`` +~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: pytorch_transformers.TFPreTrainedModel + :members: diff --git a/docs/source/model_doc/bert.rst b/docs/source/model_doc/bert.rst index 9f2caf3e80..cfc221d381 100644 --- a/docs/source/model_doc/bert.rst +++ b/docs/source/model_doc/bert.rst @@ -70,3 +70,59 @@ BERT .. autoclass:: transformers.BertForQuestionAnswering :members: + +``TFBertModel`` +~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: pytorch_transformers.TFBertModel + :members: + + +``TFBertForPreTraining`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: pytorch_transformers.TFBertForPreTraining + :members: + + +``TFBertForMaskedLM`` +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: pytorch_transformers.TFBertForMaskedLM + :members: + + +``TFBertForNextSentencePrediction`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: pytorch_transformers.TFBertForNextSentencePrediction + :members: + + +``TFBertForSequenceClassification`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: pytorch_transformers.TFBertForSequenceClassification + :members: + + +``TFBertForMultipleChoice`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: pytorch_transformers.TFBertForMultipleChoice + :members: + + +``TFBertForTokenClassification`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: pytorch_transformers.TFBertForTokenClassification + :members: + + +``TFBertForQuestionAnswering`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: pytorch_transformers.TFBertForQuestionAnswering + :members: + diff --git a/docs/source/model_doc/distilbert.rst b/docs/source/model_doc/distilbert.rst index de1ac73675..6d0fdcd4f3 100644 --- a/docs/source/model_doc/distilbert.rst +++ b/docs/source/model_doc/distilbert.rst @@ -41,3 +41,30 @@ DistilBERT .. autoclass:: transformers.DistilBertForQuestionAnswering :members: + +``TFDistilBertModel`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: pytorch_transformers.TFDistilBertModel + :members: + + +``TFDistilBertForMaskedLM`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: pytorch_transformers.TFDistilBertForMaskedLM + :members: + + +``TFDistilBertForSequenceClassification`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: pytorch_transformers.TFDistilBertForSequenceClassification + :members: + + +``TFDistilBertForQuestionAnswering`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: pytorch_transformers.TFDistilBertForQuestionAnswering + :members: diff --git a/docs/source/model_doc/gpt.rst b/docs/source/model_doc/gpt.rst index 39995a98fc..dd6e314b8f 100644 --- a/docs/source/model_doc/gpt.rst +++ b/docs/source/model_doc/gpt.rst @@ -34,3 +34,24 @@ OpenAI GPT .. autoclass:: transformers.OpenAIGPTDoubleHeadsModel :members: + + +``TFOpenAIGPTModel`` +~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: pytorch_transformers.TFOpenAIGPTModel + :members: + + +``TFOpenAIGPTLMHeadModel`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: pytorch_transformers.TFOpenAIGPTLMHeadModel + :members: + + +``TFOpenAIGPTDoubleHeadsModel`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: pytorch_transformers.TFOpenAIGPTDoubleHeadsModel + :members: diff --git a/docs/source/model_doc/gpt2.rst b/docs/source/model_doc/gpt2.rst index 92decb14de..19b435fc03 100644 --- a/docs/source/model_doc/gpt2.rst +++ b/docs/source/model_doc/gpt2.rst @@ -34,3 +34,24 @@ OpenAI GPT2 .. autoclass:: transformers.GPT2DoubleHeadsModel :members: + + +``TFGPT2Model`` +~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: pytorch_transformers.TFGPT2Model + :members: + + +``TFGPT2LMHeadModel`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: pytorch_transformers.TFGPT2LMHeadModel + :members: + + +``TFGPT2DoubleHeadsModel`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: pytorch_transformers.TFGPT2DoubleHeadsModel + :members: diff --git a/docs/source/model_doc/roberta.rst b/docs/source/model_doc/roberta.rst index 5351e018cd..ed839456af 100644 --- a/docs/source/model_doc/roberta.rst +++ b/docs/source/model_doc/roberta.rst @@ -34,3 +34,24 @@ RoBERTa .. autoclass:: transformers.RobertaForSequenceClassification :members: + + +``TFRobertaModel`` +~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: pytorch_transformers.TFRobertaModel + :members: + + +``TFRobertaForMaskedLM`` +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: pytorch_transformers.TFRobertaForMaskedLM + :members: + + +``TFRobertaForSequenceClassification`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: pytorch_transformers.TFRobertaForSequenceClassification + :members: diff --git a/docs/source/model_doc/transformerxl.rst b/docs/source/model_doc/transformerxl.rst index c8a9cc7d99..243726aa95 100644 --- a/docs/source/model_doc/transformerxl.rst +++ b/docs/source/model_doc/transformerxl.rst @@ -28,3 +28,17 @@ Transformer XL .. autoclass:: transformers.TransfoXLLMHeadModel :members: + + +``TFTransfoXLModel`` +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: pytorch_transformers.TFTransfoXLModel + :members: + + +``TFTransfoXLLMHeadModel`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: pytorch_transformers.TFTransfoXLLMHeadModel + :members: diff --git a/docs/source/model_doc/xlm.rst b/docs/source/model_doc/xlm.rst index 344371ad52..367938fab5 100644 --- a/docs/source/model_doc/xlm.rst +++ b/docs/source/model_doc/xlm.rst @@ -39,3 +39,31 @@ XLM .. autoclass:: transformers.XLMForQuestionAnswering :members: + + +``TFXLMModel`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: pytorch_transformers.TFXLMModel + :members: + + +``TFXLMWithLMHeadModel`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: pytorch_transformers.TFXLMWithLMHeadModel + :members: + + +``TFXLMForSequenceClassification`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: pytorch_transformers.TFXLMForSequenceClassification + :members: + + +``TFXLMForQuestionAnswering`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: pytorch_transformers.TFXLMForQuestionAnswering + :members: diff --git a/docs/source/model_doc/xlnet.rst b/docs/source/model_doc/xlnet.rst index a471506278..000f163ec0 100644 --- a/docs/source/model_doc/xlnet.rst +++ b/docs/source/model_doc/xlnet.rst @@ -41,3 +41,31 @@ XLNet .. autoclass:: transformers.XLNetForQuestionAnswering :members: + + +``TFXLNetModel`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: pytorch_transformers.TFXLNetModel + :members: + + +``TFXLNetLMHeadModel`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: pytorch_transformers.TFXLNetLMHeadModel + :members: + + +``TFXLNetForSequenceClassification`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: pytorch_transformers.TFXLNetForSequenceClassification + :members: + + +``TFXLNetForQuestionAnswering`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: pytorch_transformers.TFXLNetForQuestionAnswering + :members: From c4ac7a76db7aa40bd636a59dc71ef1dd27b2c2c4 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Wed, 25 Sep 2019 07:09:30 -0400 Subject: [PATCH 154/219] GLUE processors --- docs/source/index.rst | 1 + docs/source/main_classes/processors.rst | 45 +++++++++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 docs/source/main_classes/processors.rst diff --git a/docs/source/index.rst b/docs/source/index.rst index 3a30b61b2c..0d5bd01352 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -67,6 +67,7 @@ The library currently contains PyTorch and Tensorflow implementations, pre-train main_classes/model main_classes/tokenizer main_classes/optimizer_schedules + main_classes/processors .. toctree:: :maxdepth: 2 diff --git a/docs/source/main_classes/processors.rst b/docs/source/main_classes/processors.rst new file mode 100644 index 0000000000..12e5339ddb --- /dev/null +++ b/docs/source/main_classes/processors.rst @@ -0,0 +1,45 @@ +Processors +---------------------------------------------------- + +This library includes processors for several traditional tasks. These processors can be used to process a dataset into +examples that can be fed to a model. + +``GLUE`` +~~~~~~~~~~~~~~~~~~~~~ + +`General Language Understanding Evaluation (GLUE)`__ is a benchmark that evaluates +the performance of models across a diverse set of existing NLU tasks. It was released together with the paper +`GLUE: A multi-task benchmark and analysis platform for natural language understanding`__ + +This library hosts a total of 10 processors for the following tasks: MRPC, MNLI, MNLI (mismatched), +CoLA, SST2, STSB, QQP, QNLI, RTE and WNLI. + +.. autoclass:: pytorch_transformers.data.processors.glue.MrpcProcessor + :members: + +.. autoclass:: pytorch_transformers.data.processors.glue.MnliProcessor + :members: + +.. autoclass:: pytorch_transformers.data.processors.glue.MnliMismatchedProcessor + :members: + +.. autoclass:: pytorch_transformers.data.processors.glue.ColaProcessor + :members: + +.. autoclass:: pytorch_transformers.data.processors.glue.Sst2Processor + :members: + +.. autoclass:: pytorch_transformers.data.processors.glue.StsbProcessor + :members: + +.. autoclass:: pytorch_transformers.data.processors.glue.QqpProcessor + :members: + +.. autoclass:: pytorch_transformers.data.processors.glue.QnliProcessor + :members: + +.. autoclass:: pytorch_transformers.data.processors.glue.RteProcessor + :members: + +.. autoclass:: pytorch_transformers.data.processors.glue.WnliProcessor + :members: From ad4a393e2e59951a0edbec0b9b3be852dd086cc7 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Wed, 25 Sep 2019 08:30:07 -0400 Subject: [PATCH 155/219] Changed processor documentation architecture. Added documentation for GLUE --- docs/source/main_classes/processors.rst | 62 +++++++++++++------------ transformers/data/processors/glue.py | 23 ++++++++- 2 files changed, 55 insertions(+), 30 deletions(-) diff --git a/docs/source/main_classes/processors.rst b/docs/source/main_classes/processors.rst index 12e5339ddb..d65f48af83 100644 --- a/docs/source/main_classes/processors.rst +++ b/docs/source/main_classes/processors.rst @@ -4,42 +4,46 @@ Processors This library includes processors for several traditional tasks. These processors can be used to process a dataset into examples that can be fed to a model. -``GLUE`` +Processors ~~~~~~~~~~~~~~~~~~~~~ -`General Language Understanding Evaluation (GLUE)`__ is a benchmark that evaluates +All processors follow the same architecture which is that of the +:class:`~pytorch_transformers.data.processors.utils.DataProcessor`. The processor returns a list +of :class:`~pytorch_transformers.data.processors.utils.InputExample`. + +.. autoclass:: pytorch_transformers.data.processors.utils.DataProcessor + :members: + + +.. autoclass:: pytorch_transformers.data.processors.utils.InputExample + :members: + + +GLUE +~~~~~~~~~~~~~~~~~~~~~ + +`General Language Understanding Evaluation (GLUE) `__ is a benchmark that evaluates the performance of models across a diverse set of existing NLU tasks. It was released together with the paper -`GLUE: A multi-task benchmark and analysis platform for natural language understanding`__ +`GLUE: A multi-task benchmark and analysis platform for natural language understanding `__ This library hosts a total of 10 processors for the following tasks: MRPC, MNLI, MNLI (mismatched), CoLA, SST2, STSB, QQP, QNLI, RTE and WNLI. -.. autoclass:: pytorch_transformers.data.processors.glue.MrpcProcessor - :members: +Those processors are: + - :class:`~pytorch_transformers.data.processors.utils.MrpcProcessor` + - :class:`~pytorch_transformers.data.processors.utils.MnliProcessor` + - :class:`~pytorch_transformers.data.processors.utils.MnliMismatchedProcessor` + - :class:`~pytorch_transformers.data.processors.utils.Sst2Processor` + - :class:`~pytorch_transformers.data.processors.utils.StsbProcessor` + - :class:`~pytorch_transformers.data.processors.utils.QqpProcessor` + - :class:`~pytorch_transformers.data.processors.utils.QnliProcessor` + - :class:`~pytorch_transformers.data.processors.utils.RteProcessor` + - :class:`~pytorch_transformers.data.processors.utils.WnliProcessor` -.. autoclass:: pytorch_transformers.data.processors.glue.MnliProcessor - :members: +Additionally, the following method can be used to load values from a data file and convert them to a list of +:class:`~pytorch_transformers.data.processors.utils.InputExample`. -.. autoclass:: pytorch_transformers.data.processors.glue.MnliMismatchedProcessor - :members: +.. automethod:: pytorch_transformers.data.processors.glue.glue_convert_examples_to_features -.. autoclass:: pytorch_transformers.data.processors.glue.ColaProcessor - :members: - -.. autoclass:: pytorch_transformers.data.processors.glue.Sst2Processor - :members: - -.. autoclass:: pytorch_transformers.data.processors.glue.StsbProcessor - :members: - -.. autoclass:: pytorch_transformers.data.processors.glue.QqpProcessor - :members: - -.. autoclass:: pytorch_transformers.data.processors.glue.QnliProcessor - :members: - -.. autoclass:: pytorch_transformers.data.processors.glue.RteProcessor - :members: - -.. autoclass:: pytorch_transformers.data.processors.glue.WnliProcessor - :members: +Example usage +^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/transformers/data/processors/glue.py b/transformers/data/processors/glue.py index 3010ce9840..2322f58604 100644 --- a/transformers/data/processors/glue.py +++ b/transformers/data/processors/glue.py @@ -26,6 +26,7 @@ if is_tf_available(): logger = logging.getLogger(__name__) + def glue_convert_examples_to_features(examples, tokenizer, max_length=512, task=None, @@ -36,7 +37,27 @@ def glue_convert_examples_to_features(examples, tokenizer, pad_token_segment_id=0, mask_padding_with_zero=True): """ - Loads a data file into a list of `InputBatch`s + Loads a data file into a list of ``InputFeatures`` + + Args: + examples: List of ``InputExamples`` or ``tf.data.Dataset`` containing the examples. + tokenizer: Instance of a tokenizer that will tokenize the examples + max_length: Maximum example length + task: GLUE task + label_list: List of labels. Can be obtained from the processor using the ``processor.get_labels()`` method + output_mode: String indicating the output mode. Either ``regression`` or ``classification`` + pad_on_left: If set to ``True``, the examples will be padded on the left rather than on the right (default) + pad_token: Padding token + pad_token_segment_id: The segment ID for the padding token (It is usually 0, but can vary such as for XLNet where it is 4) + mask_padding_with_zero: If set to ``True``, the attention mask will be filled by ``1`` for actual values + and by ``0`` for padded values. If set to ``False``, inverts it (``1`` for padded values, ``0`` for + actual values) + + Returns: + If the ``examples`` input is a ``tf.data.Dataset``, will return a ``tf.data.Dataset`` + containing the task-specific features. If the input is a list of ``InputExamples``, will return + a list of task-specific ``InputFeatures`` which can be fed to the model. + """ is_tf_dataset = False if is_tf_available() and isinstance(examples, tf.data.Dataset): From 36f592cc828c77cb651dc1c17a2c5d6ad41451aa Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Wed, 25 Sep 2019 08:39:33 -0400 Subject: [PATCH 156/219] Updated doc for `InputExample` and `InputFeatures` --- docs/source/main_classes/processors.rst | 8 +++++- transformers/data/processors/utils.py | 36 ++++++++++++++++--------- 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/docs/source/main_classes/processors.rst b/docs/source/main_classes/processors.rst index d65f48af83..ab4b91143b 100644 --- a/docs/source/main_classes/processors.rst +++ b/docs/source/main_classes/processors.rst @@ -9,7 +9,9 @@ Processors All processors follow the same architecture which is that of the :class:`~pytorch_transformers.data.processors.utils.DataProcessor`. The processor returns a list -of :class:`~pytorch_transformers.data.processors.utils.InputExample`. +of :class:`~pytorch_transformers.data.processors.utils.InputExample`. These +:class:`~pytorch_transformers.data.processors.utils.InputExample` can be converted to +:class:`~pytorch_transformers.data.processors.utils.InputFeatures` in order to be fed to the model. .. autoclass:: pytorch_transformers.data.processors.utils.DataProcessor :members: @@ -19,6 +21,10 @@ of :class:`~pytorch_transformers.data.processors.utils.InputExample`. :members: +.. autoclass:: pytorch_transformers.data.processors.utils.InputFeatures + :members: + + GLUE ~~~~~~~~~~~~~~~~~~~~~ diff --git a/transformers/data/processors/utils.py b/transformers/data/processors/utils.py index a616372054..d16ea786a0 100644 --- a/transformers/data/processors/utils.py +++ b/transformers/data/processors/utils.py @@ -20,19 +20,19 @@ import copy import json class InputExample(object): - """A single training/test example for simple sequence classification.""" - def __init__(self, guid, text_a, text_b=None, label=None): - """Constructs a InputExample. + """ + A single training/test example for simple sequence classification. - Args: - guid: Unique id for the example. - text_a: string. The untokenized text of the first sequence. For single - sequence tasks, only this sequence must be specified. - text_b: (Optional) string. The untokenized text of the second sequence. - Only must be specified for sequence pair tasks. - label: (Optional) string. The label of the example. This should be - specified for train and dev examples, but not for test examples. - """ + Args: + guid: Unique id for the example. + text_a: string. The untokenized text of the first sequence. For single + sequence tasks, only this sequence must be specified. + text_b: (Optional) string. The untokenized text of the second sequence. + Only must be specified for sequence pair tasks. + label: (Optional) string. The label of the example. This should be + specified for train and dev examples, but not for test examples. + """ + def __init__(self, guid, text_a, text_b=None, label=None): self.guid = guid self.text_a = text_a self.text_b = text_b @@ -52,7 +52,17 @@ class InputExample(object): class InputFeatures(object): - """A single set of features of data.""" + """ + A single set of features of data. + + Args: + input_ids: Indices of input sequence tokens in the vocabulary. + attention_mask: Mask to avoid performing attention on padding token indices. + Mask values selected in ``[0, 1]``: + Usually ``1`` for tokens that are NOT MASKED, ``0`` for MASKED (padded) tokens. + token_type_ids: Segment token indices to indicate first and second portions of the inputs. + label: Label corresponding to the input + """ def __init__(self, input_ids, attention_mask, token_type_ids, label): self.input_ids = input_ids From fb056494e50be6ff729166734eff96f9f26d9e96 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Wed, 25 Sep 2019 08:48:53 -0400 Subject: [PATCH 157/219] Example usage --- docs/source/main_classes/processors.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/source/main_classes/processors.rst b/docs/source/main_classes/processors.rst index ab4b91143b..f24cc8a8cf 100644 --- a/docs/source/main_classes/processors.rst +++ b/docs/source/main_classes/processors.rst @@ -53,3 +53,6 @@ Additionally, the following method can be used to load values from a data file Example usage ^^^^^^^^^^^^^^^^^^^^^^^^^ + +An example using these processors is given in the +`run_glue.py ` script. \ No newline at end of file From 8349d75773ec24a641b0bf5610b15917b650a210 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Wed, 25 Sep 2019 09:05:43 -0400 Subject: [PATCH 158/219] Various small doc fixes --- docs/source/main_classes/processors.rst | 2 +- docs/source/model_doc/xlm.rst | 4 ++-- docs/source/model_doc/xlnet.rst | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/source/main_classes/processors.rst b/docs/source/main_classes/processors.rst index f24cc8a8cf..79bd8ddaac 100644 --- a/docs/source/main_classes/processors.rst +++ b/docs/source/main_classes/processors.rst @@ -55,4 +55,4 @@ Example usage ^^^^^^^^^^^^^^^^^^^^^^^^^ An example using these processors is given in the -`run_glue.py ` script. \ No newline at end of file +`run_glue.py `__ script. \ No newline at end of file diff --git a/docs/source/model_doc/xlm.rst b/docs/source/model_doc/xlm.rst index 367938fab5..157f8d71e4 100644 --- a/docs/source/model_doc/xlm.rst +++ b/docs/source/model_doc/xlm.rst @@ -62,8 +62,8 @@ XLM :members: -``TFXLMForQuestionAnswering`` +``TFXLMForQuestionAnsweringSimple`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.TFXLMForQuestionAnswering +.. autoclass:: pytorch_transformers.TFXLMForQuestionAnsweringSimple :members: diff --git a/docs/source/model_doc/xlnet.rst b/docs/source/model_doc/xlnet.rst index 000f163ec0..197710f2f9 100644 --- a/docs/source/model_doc/xlnet.rst +++ b/docs/source/model_doc/xlnet.rst @@ -64,8 +64,8 @@ XLNet :members: -``TFXLNetForQuestionAnswering`` +``TFXLNetForQuestionAnsweringSimple`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.TFXLNetForQuestionAnswering +.. autoclass:: pytorch_transformers.TFXLNetForQuestionAnsweringSimple :members: From 9676d1a2a86138544995cbf48c5da44d7937a239 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 26 Sep 2019 13:47:58 +0200 Subject: [PATCH 159/219] update readme and setup.py --- README.md | 4 ++-- setup.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 44684bde99..b484f1781d 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@

State-of-the-art Natural Language Processing for TensorFlow 2.0 and PyTorch -🤗 Transformers (formerly known as `pytorch-transformers` and `pytorch-pretrained-bert`) provides general-purpose architectures (BERT, GPT-2, RoBERTa, XLM, DistilBert, XLNet...) for Natural Language Understanding (NLU) and Natural Language Generation (NLG) with over 32+ pretrained models in 100+ languages and deep interoperability between TensorFlow 2.0 and PyTorch. +🤗 Transformers (formerly known as `pytorch-transformers` and `pytorch-pretrained-bert`) provides state-of-the-art general-purpose architectures (BERT, GPT-2, RoBERTa, XLM, DistilBert, XLNet...) for Natural Language Understanding (NLU) and Natural Language Generation (NLG) with over 32+ pretrained models in 100+ languages and deep interoperability between TensorFlow 2.0 and PyTorch. ### Features @@ -56,7 +56,7 @@ Choose the right framework for every part of a model's lifetime | [Quick tour: Usage](#quick-tour) | Tokenizers & models usage: Bert and GPT-2 | | [Quick tour: TF 2.0 and PyTorch ](#Quick-tour-TF-2.0-training-and-PyTorch-interoperability) | Train a TF 2.0 model in 10 lines of code, load it in PyTorch | | [Quick tour: Fine-tuning/usage scripts](#quick-tour-of-the-fine-tuningusage-scripts) | Using provided scripts: GLUE, SQuAD and Text generation | -| [Migrating from pytorch-transformers to transformers](#Migrating-from-pytorch-pretrained-bert-to-transformers) | Migrating your code from pytorch-pretrained-bert to transformers | +| [Migrating from pytorch-transformers to transformers](#Migrating-from-pytorch-transformers-to-transformers) | Migrating your code from pytorch-pretrained-bert to transformers | | [Migrating from pytorch-pretrained-bert to pytorch-transformers](#Migrating-from-pytorch-pretrained-bert-to-transformers) | Migrating your code from pytorch-pretrained-bert to transformers | | [Documentation](https://huggingface.co/transformers/) | Full API documentation and more | diff --git a/setup.py b/setup.py index 34cb89560e..273eec888b 100644 --- a/setup.py +++ b/setup.py @@ -13,11 +13,11 @@ To create the package for pypi. 4. Build both the sources and the wheel. Do not change anything in setup.py between creating the wheel and the source distribution (obviously). - For the wheel, run: "python setup.py bdist_wheel" in the top level allennlp directory. + For the wheel, run: "python setup.py bdist_wheel" in the top level directory. (this will build a wheel for the python version you use to build it - make sure you use python 3.x). For the sources, run: "python setup.py sdist" - You should now have a /dist directory with both .whl and .tar.gz source versions of allennlp. + You should now have a /dist directory with both .whl and .tar.gz source versions. 5. Check that everything looks correct by uploading the package to the pypi test server: From e4e35296fb4a6af49c18dc814629a6acfdbe96a2 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 26 Sep 2019 13:52:24 +0200 Subject: [PATCH 160/219] update setup.py metadata --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 273eec888b..8f27bc24ab 100644 --- a/setup.py +++ b/setup.py @@ -39,12 +39,12 @@ from setuptools import find_packages, setup setup( name="transformers", version="2.0.0", - author="Thomas Wolf, Lysandre Debut, Victor Sanh, Julien Chaumond, Google AI Language Team Authors, Open AI team Authors", + author="Thomas Wolf, Lysandre Debut, Victor Sanh, Julien Chaumond, Google AI Language Team Authors, Open AI team Authors, Facebook AI Authors, Carnegie Mellon University Authors", author_email="thomas@huggingface.co", - description="Repository of pre-trained NLP Transformer models: BERT & RoBERTa, GPT & GPT-2, Transformer-XL, XLNet and XLM", + description="State-of-the-art Natural Language Processing for TensorFlow 2.0 and PyTorch", long_description=open("README.md", "r", encoding='utf-8').read(), long_description_content_type="text/markdown", - keywords='NLP deep learning transformer pytorch BERT GPT GPT-2 google openai CMU', + keywords='NLP deep learning transformer pytorch tensorflow BERT GPT GPT-2 google openai CMU', license='Apache', url="https://github.com/huggingface/transformers", packages=find_packages(exclude=["*.tests", "*.tests.*", From de5e4864cb009ca999627437349f0f454c6e7306 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Thu, 26 Sep 2019 08:04:54 -0400 Subject: [PATCH 161/219] Documentation --- docs/source/index.rst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/source/index.rst b/docs/source/index.rst index 0d5bd01352..5866c37a1e 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -13,17 +13,20 @@ Features - High performance on NLU and NLG tasks - Low barrier to entry for educators and practitioners -State-of-the-art NLP for everyone +State-of-the-art NLP for everyone: + - Deep learning researchers - Hands-on practitioners - AI/ML/NLP teachers and educators -Lower compute costs, smaller carbon footprint +Lower compute costs, smaller carbon footprint: + - Researchers can share trained models instead of always retraining - Practitioners can reduce compute time and production costs - 8 architectures with over 30 pretrained models, some in more than 100 languages -Choose the right framework for every part of a model's lifetime +Choose the right framework for every part of a model's lifetime: + - Train state-of-the-art models in 3 lines of code - Deep interoperability between TensorFlow 2.0 and PyTorch models - Move a single model between TF2.0/PyTorch frameworks at will From 294edfd83d591f3ae841c993662fac6ec7924515 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Thu, 26 Sep 2019 08:16:12 -0400 Subject: [PATCH 162/219] Release version in documentation --- docs/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index ae9cc67e2c..b8fb2ceace 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -26,7 +26,7 @@ author = u'huggingface' # The short X.Y version version = u'' # The full version, including alpha/beta/rc tags -release = u'1.2.0' +release = u'2.0.0' # -- General configuration --------------------------------------------------- From 927904bc911225c1ea6087c5e24aa77d265bfb9e Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Thu, 26 Sep 2019 08:47:15 -0400 Subject: [PATCH 163/219] [doc] pytorch_transformers -> transformers --- docs/source/main_classes/model.rst | 2 +- docs/source/main_classes/processors.rst | 36 ++++++++++++------------- docs/source/model_doc/bert.rst | 16 +++++------ docs/source/model_doc/distilbert.rst | 8 +++--- docs/source/model_doc/gpt.rst | 6 ++--- docs/source/model_doc/gpt2.rst | 6 ++--- docs/source/model_doc/roberta.rst | 6 ++--- docs/source/model_doc/transformerxl.rst | 4 +-- docs/source/model_doc/xlnet.rst | 8 +++--- 9 files changed, 46 insertions(+), 46 deletions(-) diff --git a/docs/source/main_classes/model.rst b/docs/source/main_classes/model.rst index 179ec53d0e..6e3da45bc2 100644 --- a/docs/source/main_classes/model.rst +++ b/docs/source/main_classes/model.rst @@ -17,5 +17,5 @@ The base class ``PreTrainedModel`` implements the common methods for loading/sav ``TFPreTrainedModel`` ~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.TFPreTrainedModel +.. autoclass:: transformers.TFPreTrainedModel :members: diff --git a/docs/source/main_classes/processors.rst b/docs/source/main_classes/processors.rst index 79bd8ddaac..a85c126956 100644 --- a/docs/source/main_classes/processors.rst +++ b/docs/source/main_classes/processors.rst @@ -8,20 +8,20 @@ Processors ~~~~~~~~~~~~~~~~~~~~~ All processors follow the same architecture which is that of the -:class:`~pytorch_transformers.data.processors.utils.DataProcessor`. The processor returns a list -of :class:`~pytorch_transformers.data.processors.utils.InputExample`. These -:class:`~pytorch_transformers.data.processors.utils.InputExample` can be converted to -:class:`~pytorch_transformers.data.processors.utils.InputFeatures` in order to be fed to the model. +:class:`~transformers.data.processors.utils.DataProcessor`. The processor returns a list +of :class:`~transformers.data.processors.utils.InputExample`. These +:class:`~transformers.data.processors.utils.InputExample` can be converted to +:class:`~transformers.data.processors.utils.InputFeatures` in order to be fed to the model. -.. autoclass:: pytorch_transformers.data.processors.utils.DataProcessor +.. autoclass:: transformers.data.processors.utils.DataProcessor :members: -.. autoclass:: pytorch_transformers.data.processors.utils.InputExample +.. autoclass:: transformers.data.processors.utils.InputExample :members: -.. autoclass:: pytorch_transformers.data.processors.utils.InputFeatures +.. autoclass:: transformers.data.processors.utils.InputFeatures :members: @@ -36,20 +36,20 @@ This library hosts a total of 10 processors for the following tasks: MRPC, MNLI, CoLA, SST2, STSB, QQP, QNLI, RTE and WNLI. Those processors are: - - :class:`~pytorch_transformers.data.processors.utils.MrpcProcessor` - - :class:`~pytorch_transformers.data.processors.utils.MnliProcessor` - - :class:`~pytorch_transformers.data.processors.utils.MnliMismatchedProcessor` - - :class:`~pytorch_transformers.data.processors.utils.Sst2Processor` - - :class:`~pytorch_transformers.data.processors.utils.StsbProcessor` - - :class:`~pytorch_transformers.data.processors.utils.QqpProcessor` - - :class:`~pytorch_transformers.data.processors.utils.QnliProcessor` - - :class:`~pytorch_transformers.data.processors.utils.RteProcessor` - - :class:`~pytorch_transformers.data.processors.utils.WnliProcessor` + - :class:`~transformers.data.processors.utils.MrpcProcessor` + - :class:`~transformers.data.processors.utils.MnliProcessor` + - :class:`~transformers.data.processors.utils.MnliMismatchedProcessor` + - :class:`~transformers.data.processors.utils.Sst2Processor` + - :class:`~transformers.data.processors.utils.StsbProcessor` + - :class:`~transformers.data.processors.utils.QqpProcessor` + - :class:`~transformers.data.processors.utils.QnliProcessor` + - :class:`~transformers.data.processors.utils.RteProcessor` + - :class:`~transformers.data.processors.utils.WnliProcessor` Additionally, the following method can be used to load values from a data file and convert them to a list of -:class:`~pytorch_transformers.data.processors.utils.InputExample`. +:class:`~transformers.data.processors.utils.InputExample`. -.. automethod:: pytorch_transformers.data.processors.glue.glue_convert_examples_to_features +.. automethod:: transformers.data.processors.glue.glue_convert_examples_to_features Example usage ^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/source/model_doc/bert.rst b/docs/source/model_doc/bert.rst index cfc221d381..98b377d0f2 100644 --- a/docs/source/model_doc/bert.rst +++ b/docs/source/model_doc/bert.rst @@ -74,55 +74,55 @@ BERT ``TFBertModel`` ~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.TFBertModel +.. autoclass:: transformers.TFBertModel :members: ``TFBertForPreTraining`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.TFBertForPreTraining +.. autoclass:: transformers.TFBertForPreTraining :members: ``TFBertForMaskedLM`` ~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.TFBertForMaskedLM +.. autoclass:: transformers.TFBertForMaskedLM :members: ``TFBertForNextSentencePrediction`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.TFBertForNextSentencePrediction +.. autoclass:: transformers.TFBertForNextSentencePrediction :members: ``TFBertForSequenceClassification`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.TFBertForSequenceClassification +.. autoclass:: transformers.TFBertForSequenceClassification :members: ``TFBertForMultipleChoice`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.TFBertForMultipleChoice +.. autoclass:: transformers.TFBertForMultipleChoice :members: ``TFBertForTokenClassification`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.TFBertForTokenClassification +.. autoclass:: transformers.TFBertForTokenClassification :members: ``TFBertForQuestionAnswering`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.TFBertForQuestionAnswering +.. autoclass:: transformers.TFBertForQuestionAnswering :members: diff --git a/docs/source/model_doc/distilbert.rst b/docs/source/model_doc/distilbert.rst index 6d0fdcd4f3..0eabdabc68 100644 --- a/docs/source/model_doc/distilbert.rst +++ b/docs/source/model_doc/distilbert.rst @@ -45,26 +45,26 @@ DistilBERT ``TFDistilBertModel`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.TFDistilBertModel +.. autoclass:: transformers.TFDistilBertModel :members: ``TFDistilBertForMaskedLM`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.TFDistilBertForMaskedLM +.. autoclass:: transformers.TFDistilBertForMaskedLM :members: ``TFDistilBertForSequenceClassification`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.TFDistilBertForSequenceClassification +.. autoclass:: transformers.TFDistilBertForSequenceClassification :members: ``TFDistilBertForQuestionAnswering`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.TFDistilBertForQuestionAnswering +.. autoclass:: transformers.TFDistilBertForQuestionAnswering :members: diff --git a/docs/source/model_doc/gpt.rst b/docs/source/model_doc/gpt.rst index dd6e314b8f..d8281e8aff 100644 --- a/docs/source/model_doc/gpt.rst +++ b/docs/source/model_doc/gpt.rst @@ -39,19 +39,19 @@ OpenAI GPT ``TFOpenAIGPTModel`` ~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.TFOpenAIGPTModel +.. autoclass:: transformers.TFOpenAIGPTModel :members: ``TFOpenAIGPTLMHeadModel`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.TFOpenAIGPTLMHeadModel +.. autoclass:: transformers.TFOpenAIGPTLMHeadModel :members: ``TFOpenAIGPTDoubleHeadsModel`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.TFOpenAIGPTDoubleHeadsModel +.. autoclass:: transformers.TFOpenAIGPTDoubleHeadsModel :members: diff --git a/docs/source/model_doc/gpt2.rst b/docs/source/model_doc/gpt2.rst index 19b435fc03..bd76700a9d 100644 --- a/docs/source/model_doc/gpt2.rst +++ b/docs/source/model_doc/gpt2.rst @@ -39,19 +39,19 @@ OpenAI GPT2 ``TFGPT2Model`` ~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.TFGPT2Model +.. autoclass:: transformers.TFGPT2Model :members: ``TFGPT2LMHeadModel`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.TFGPT2LMHeadModel +.. autoclass:: transformers.TFGPT2LMHeadModel :members: ``TFGPT2DoubleHeadsModel`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.TFGPT2DoubleHeadsModel +.. autoclass:: transformers.TFGPT2DoubleHeadsModel :members: diff --git a/docs/source/model_doc/roberta.rst b/docs/source/model_doc/roberta.rst index ed839456af..105cb85a8d 100644 --- a/docs/source/model_doc/roberta.rst +++ b/docs/source/model_doc/roberta.rst @@ -39,19 +39,19 @@ RoBERTa ``TFRobertaModel`` ~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.TFRobertaModel +.. autoclass:: transformers.TFRobertaModel :members: ``TFRobertaForMaskedLM`` ~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.TFRobertaForMaskedLM +.. autoclass:: transformers.TFRobertaForMaskedLM :members: ``TFRobertaForSequenceClassification`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.TFRobertaForSequenceClassification +.. autoclass:: transformers.TFRobertaForSequenceClassification :members: diff --git a/docs/source/model_doc/transformerxl.rst b/docs/source/model_doc/transformerxl.rst index 243726aa95..4d018c0696 100644 --- a/docs/source/model_doc/transformerxl.rst +++ b/docs/source/model_doc/transformerxl.rst @@ -33,12 +33,12 @@ Transformer XL ``TFTransfoXLModel`` ~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.TFTransfoXLModel +.. autoclass:: transformers.TFTransfoXLModel :members: ``TFTransfoXLLMHeadModel`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.TFTransfoXLLMHeadModel +.. autoclass:: transformers.TFTransfoXLLMHeadModel :members: diff --git a/docs/source/model_doc/xlnet.rst b/docs/source/model_doc/xlnet.rst index 197710f2f9..4005ce3a0a 100644 --- a/docs/source/model_doc/xlnet.rst +++ b/docs/source/model_doc/xlnet.rst @@ -46,26 +46,26 @@ XLNet ``TFXLNetModel`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.TFXLNetModel +.. autoclass:: transformers.TFXLNetModel :members: ``TFXLNetLMHeadModel`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.TFXLNetLMHeadModel +.. autoclass:: transformers.TFXLNetLMHeadModel :members: ``TFXLNetForSequenceClassification`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.TFXLNetForSequenceClassification +.. autoclass:: transformers.TFXLNetForSequenceClassification :members: ``TFXLNetForQuestionAnsweringSimple`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.TFXLNetForQuestionAnsweringSimple +.. autoclass:: transformers.TFXLNetForQuestionAnsweringSimple :members: From 7d8b395afa418476735c5503fa343697b4d0eaed Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Thu, 26 Sep 2019 08:49:31 -0400 Subject: [PATCH 164/219] Doc building requirements --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 0c2a31c09a..1a682b219a 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -26,4 +26,4 @@ sphinxcontrib-jsmath==1.0.1 sphinxcontrib-qthelp==1.0.2 sphinxcontrib-serializinghtml==1.1.3 urllib3==1.25.3 -sphinx-markdown-tables==0.0.9 \ No newline at end of file +numpy==1.17.2 \ No newline at end of file From 4094958df28bd5f6dc3b07323dec9bcc7303be94 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Thu, 26 Sep 2019 08:50:55 -0400 Subject: [PATCH 165/219] Doc building requirements --- docs/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/requirements.txt b/docs/requirements.txt index 1a682b219a..d4c4100669 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -26,4 +26,5 @@ sphinxcontrib-jsmath==1.0.1 sphinxcontrib-qthelp==1.0.2 sphinxcontrib-serializinghtml==1.1.3 urllib3==1.25.3 +sphinx-markdown-tables==0.0.9 numpy==1.17.2 \ No newline at end of file From 0f92f76ca3bb4fc2acdfd1cb89b9b0058ffbfef0 Mon Sep 17 00:00:00 2001 From: Lysandre Debut Date: Thu, 26 Sep 2019 08:59:52 -0400 Subject: [PATCH 166/219] CircleCI reference in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b484f1781d..4e47737a9e 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@

- + Build From a3e0dbba9512866064c20e9bc99c62725f6c36fb Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Thu, 26 Sep 2019 09:51:14 -0400 Subject: [PATCH 167/219] Doc building requirements [TF] --- docs/requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index d4c4100669..96cd4f95da 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -27,4 +27,5 @@ sphinxcontrib-qthelp==1.0.2 sphinxcontrib-serializinghtml==1.1.3 urllib3==1.25.3 sphinx-markdown-tables==0.0.9 -numpy==1.17.2 \ No newline at end of file +numpy==1.17.2 +tensorflow==1.14.0 \ No newline at end of file From f71a4577b8a54d4a19fb1b1ca8ba15cfc6b5bb6e Mon Sep 17 00:00:00 2001 From: mgrankin Date: Thu, 26 Sep 2019 16:53:13 +0300 Subject: [PATCH 168/219] faster dataset building --- examples/run_lm_finetuning.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/examples/run_lm_finetuning.py b/examples/run_lm_finetuning.py index 7ccf4c3cb7..8d440ebcc6 100644 --- a/examples/run_lm_finetuning.py +++ b/examples/run_lm_finetuning.py @@ -74,9 +74,8 @@ class TextDataset(Dataset): tokenized_text = tokenizer.convert_tokens_to_ids(tokenizer.tokenize(text)) - while len(tokenized_text) >= block_size: # Truncate in block of block_size - self.examples.append(tokenizer.add_special_tokens_single_sequence(tokenized_text[:block_size])) - tokenized_text = tokenized_text[block_size:] + for i in range(0, len(tokenized_text)-block_size+1, block_size): # Truncate in block of block_size + self.examples.append(tokenizer.add_special_tokens_single_sentence(tokenized_text[i:i+block_size])) # Note that we are loosing the last truncated example here for the sake of simplicity (no padding) # If your dataset is small, first you should loook for a bigger one :-) and second you # can change this behavior by adding (model specific) padding. From 302a4813a5c17a2f595a912f399fe108df7ab98f Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Thu, 26 Sep 2019 09:57:30 -0400 Subject: [PATCH 169/219] Doc building requirements [TF2] --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 96cd4f95da..66a76122d7 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -28,4 +28,4 @@ sphinxcontrib-serializinghtml==1.1.3 urllib3==1.25.3 sphinx-markdown-tables==0.0.9 numpy==1.17.2 -tensorflow==1.14.0 \ No newline at end of file +tensorflow==2.0.0rc2 \ No newline at end of file From 7e957237e4247ca222096f6120eea2359b3012d1 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Thu, 26 Sep 2019 10:08:56 -0400 Subject: [PATCH 170/219] [Doc] XLM + Torch in documentation --- docs/requirements.txt | 3 ++- docs/source/model_doc/xlm.rst | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 66a76122d7..25fcdb0f00 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -28,4 +28,5 @@ sphinxcontrib-serializinghtml==1.1.3 urllib3==1.25.3 sphinx-markdown-tables==0.0.9 numpy==1.17.2 -tensorflow==2.0.0rc2 \ No newline at end of file +tensorflow==2.0.0rc2 +torch==1.1.0 \ No newline at end of file diff --git a/docs/source/model_doc/xlm.rst b/docs/source/model_doc/xlm.rst index 157f8d71e4..786f2fb7d9 100644 --- a/docs/source/model_doc/xlm.rst +++ b/docs/source/model_doc/xlm.rst @@ -51,19 +51,19 @@ XLM ``TFXLMWithLMHeadModel`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.TFXLMWithLMHeadModel +.. autoclass:: transformers.TFXLMWithLMHeadModel :members: ``TFXLMForSequenceClassification`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.TFXLMForSequenceClassification +.. autoclass:: transformers.TFXLMForSequenceClassification :members: ``TFXLMForQuestionAnsweringSimple`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.TFXLMForQuestionAnsweringSimple +.. autoclass:: transformers.TFXLMForQuestionAnsweringSimple :members: From 4e63c907205d642ed04cba4eb95731b24e1db2c1 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 26 Sep 2019 16:14:21 +0200 Subject: [PATCH 171/219] update installation instructions in readme --- README.md | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 4e47737a9e..8f564f8944 100644 --- a/README.md +++ b/README.md @@ -62,11 +62,14 @@ Choose the right framework for every part of a model's lifetime ## Installation -This repo is tested on Python 2.7 and 3.5+ (examples are tested only on python 3.5+) and PyTorch 1.0.0+ +This repo is tested on Python 2.7 and 3.5+ (examples are tested only on python 3.5+), PyTorch 1.0.0+ and TensorFlow 2.0.0-rc1 ### With pip -Transformers can be installed by pip as follows: +First you need to install one of, or both, TensorFlow 2.0 and PyTorch. +Please refere to [TensorFlow installation page](https://www.tensorflow.org/install/pip#tensorflow-2.0-rc-is-available) and/or [PyTorch installation page](https://pytorch.org/get-started/locally/#start-locally) regarding the specific install command for your platform. + +When TensorFlow 2.0 and/or PyTorch has been installed, 🤗 Transformers can be installed using pip as follows: ```bash pip install transformers @@ -74,7 +77,10 @@ pip install transformers ### From source -Clone the repository and run: +Here also, you first need to install one of, or both, TensorFlow 2.0 and PyTorch. +Please refere to [TensorFlow installation page](https://www.tensorflow.org/install/pip#tensorflow-2.0-rc-is-available) and/or [PyTorch installation page](https://pytorch.org/get-started/locally/#start-locally) regarding the specific install command for your platform. + +When TensorFlow 2.0 and/or PyTorch has been installed, you can install from source by cloning the repository and runing: ```bash pip install [--editable] . @@ -86,6 +92,8 @@ A series of tests is included for the library and the example scripts. Library t These tests can be run using `pytest` (install pytest if needed with `pip install pytest`). +Depending on which framework is installed (TensorFlow 2.0 and/or PyTorch), the irrelevant tests will be skipped. Ensure that both frameworks are installed if you want to execute all tests. + You can run the tests from the root of the cloned repository with the commands: ```bash @@ -99,8 +107,7 @@ You should check out our [`swift-coreml-transformers`](https://github.com/huggin It contains an example of a conversion script from a Pytorch trained Transformer model (here, `GPT-2`) to a CoreML model that runs on iOS devices. -At some point in the future, you'll be able to seamlessly move from pre-training or fine-tuning models in PyTorch to productizing them in CoreML, -or prototype a model or an app in CoreML then research its hyperparameters or architecture from PyTorch. Super exciting! +At some point in the future, you'll be able to seamlessly move from pre-training or fine-tuning models to productizing them in CoreML, or prototype a model or an app in CoreML then research its hyperparameters or architecture from TensorFlow 2.0 and/or PyTorch. Super exciting! ## Model architectures From 6c3b131516d6230d4fe92a735cb81df6a0c81611 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 26 Sep 2019 16:23:28 +0200 Subject: [PATCH 172/219] typo in readme/doc --- README.md | 6 +++--- docs/source/model_doc/xlm.rst | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 8f564f8944..a111b5f943 100644 --- a/README.md +++ b/README.md @@ -207,7 +207,7 @@ Let's do a quick example of how a TensorFlow 2.0 model can be trained in 12 line ```python import tensorflow as tf import tensorflow_datasets -from pytorch_transformers import * +from transformers import * # Load dataset, tokenizer, model from pretrained model/vocabulary tokenizer = BertTokenizer.from_pretrained('bert-base-cased') @@ -215,8 +215,8 @@ model = TFBertForSequenceClassification.from_pretrained('bert-base-cased') data = tensorflow_datasets.load('glue/mrpc') # Prepare dataset for GLUE as a tf.data.Dataset instance -train_dataset = glue_convert_examples_to_features(data['train'], tokenizer, 128, 'mrpc') -valid_dataset = glue_convert_examples_to_features(data['validation'], tokenizer, 128, 'mrpc') +train_dataset = glue_convert_examples_to_features(data['train'], tokenizer, max_length=128, task='mrpc') +valid_dataset = glue_convert_examples_to_features(data['validation'], tokenizer, max_length=128, task='mrpc') train_dataset = train_dataset.shuffle(100).batch(32).repeat(2) valid_dataset = valid_dataset.batch(64) diff --git a/docs/source/model_doc/xlm.rst b/docs/source/model_doc/xlm.rst index 786f2fb7d9..f7034bb9d8 100644 --- a/docs/source/model_doc/xlm.rst +++ b/docs/source/model_doc/xlm.rst @@ -44,7 +44,7 @@ XLM ``TFXLMModel`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autoclass:: pytorch_transformers.TFXLMModel +.. autoclass:: transformers.TFXLMModel :members: From 93f0c5fc72a596894072ebd03b26be47c4f48edc Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Thu, 26 Sep 2019 11:45:00 -0400 Subject: [PATCH 173/219] Repository link in the documentation --- docs/source/index.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/index.rst b/docs/source/index.rst index 5866c37a1e..8c76b89185 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -5,6 +5,8 @@ Transformers (BERT, GPT-2, RoBERTa, XLM, DistilBert, XLNet...) for Natural Language Understanding (NLU) and Natural Language Generation (NLG) with over 32+ pretrained models in 100+ languages and deep interoperability between TensorFlow 2.0 and PyTorch. +This is the documentation of our repository `transformers `__. + Features --------------------------------------------------- From ecfddc6034a12c3ef6c4ef3e6f56f7d034ec3075 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Thu, 26 Sep 2019 16:49:03 -0400 Subject: [PATCH 174/219] Update RoBERTa and GPT-2 Tokenizer documentation (fix #1343) --- transformers/tokenization_gpt2.py | 7 ++++--- transformers/tokenization_roberta.py | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/transformers/tokenization_gpt2.py b/transformers/tokenization_gpt2.py index 3d5d3029dd..3e931dfcf8 100644 --- a/transformers/tokenization_gpt2.py +++ b/transformers/tokenization_gpt2.py @@ -101,9 +101,10 @@ class GPT2Tokenizer(PreTrainedTokenizer): """ GPT-2 BPE tokenizer. Peculiarities: - Byte-level Byte-Pair-Encoding - - Requires a space to start the input string => will add a space is there isn't. - As a consequence, this tokenizer `encode` and `decode` method will not conserve - the absence of a space at the beginning of a string: `tokenizer.decode(tokenizer.encode("Hello")) = " Hello" + - Requires a space to start the input string => the encoding methods should be called with the + ``add_prefix_space`` flag set to ``True``. + Otherwise, this tokenizer ``encode`` and ``decode`` method will not conserve + the absence of a space at the beginning of a string: `tokenizer.decode(tokenizer.encode("Hello")) = " Hello"` """ vocab_files_names = VOCAB_FILES_NAMES pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP diff --git a/transformers/tokenization_roberta.py b/transformers/tokenization_roberta.py index ee8e97d6bf..32e92be211 100644 --- a/transformers/tokenization_roberta.py +++ b/transformers/tokenization_roberta.py @@ -66,9 +66,10 @@ class RobertaTokenizer(GPT2Tokenizer): """ RoBERTa BPE tokenizer, derived from the GPT-2 tokenizer. Peculiarities: - Byte-level Byte-Pair-Encoding - - Requires a space to start the input string => will add a space is there isn't. - As a consequence, this tokenizer `encode` and `decode` method will not conserve - the absence of a space at the beginning of a string: `tokenizer.decode(tokenizer.encode("Hello")) = " Hello" + - Requires a space to start the input string => the encoding methods should be called with the + ``add_prefix_space`` flag set to ``True``. + Otherwise, this tokenizer ``encode`` and ``decode`` method will not conserve + the absence of a space at the beginning of a string: `tokenizer.decode(tokenizer.encode("Hello")) = " Hello"` """ vocab_files_names = VOCAB_FILES_NAMES pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP From fc9faa8a473ace9ef45754fbb478cca26202a2bb Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Thu, 26 Sep 2019 18:19:51 -0400 Subject: [PATCH 175/219] [docs] Doc tweaks Co-Authored-By: Lysandre Debut --- docs/README.md | 4 ++-- docs/requirements.txt | 2 +- docs/source/_static/js/custom.js | 41 +++++++++++++++++++++++++------- docs/source/examples.md | 1 + 4 files changed, 37 insertions(+), 11 deletions(-) create mode 120000 docs/source/examples.md diff --git a/docs/README.md b/docs/README.md index 6804a22e69..87fa5b90a0 100644 --- a/docs/README.md +++ b/docs/README.md @@ -34,11 +34,11 @@ pip install recommonmark ## Building the documentation -Make sure that there is a symlink from the `example` file (in /examples) inside the source folder. Run the followig +Make sure that there is a symlink from the `example` file (in /examples) inside the source folder. Run the following command to generate it: ```bash -ln -s ../../examples/README.md source/examples.md +ln -s ../../examples/README.md examples.md ``` Once you have setup `sphinx`, you can build the documentation by running the following command in the `/docs` folder: diff --git a/docs/requirements.txt b/docs/requirements.txt index 25fcdb0f00..63480293f5 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -29,4 +29,4 @@ urllib3==1.25.3 sphinx-markdown-tables==0.0.9 numpy==1.17.2 tensorflow==2.0.0rc2 -torch==1.1.0 \ No newline at end of file +torch==1.2.0 \ No newline at end of file diff --git a/docs/source/_static/js/custom.js b/docs/source/_static/js/custom.js index 1d5827c0fa..2c7836fd20 100644 --- a/docs/source/_static/js/custom.js +++ b/docs/source/_static/js/custom.js @@ -1,5 +1,5 @@ function addIcon() { - const huggingFaceLogo = "http://lysand.re/huggingface_logo.svg"; + const huggingFaceLogo = "https://huggingface.co/assets/transformers-docs/huggingface_logo.svg"; const image = document.createElement("img"); image.setAttribute("src", huggingFaceLogo); @@ -9,7 +9,7 @@ function addIcon() { div.style.paddingTop = '30px'; div.style.backgroundColor = '#6670FF'; - const scrollDiv = document.getElementsByClassName("wy-side-scroll")[0]; + const scrollDiv = document.querySelector(".wy-side-scroll"); scrollDiv.prepend(div); } @@ -24,10 +24,10 @@ function addCustomFooter() { social.classList.add("footer__Social"); const imageDetails = [ - { link: "https://huggingface.co", imageLink: "http://lysand.re/icons/website.svg" }, - { link: "https://twitter.com/huggingface", imageLink: "http://lysand.re/icons/twitter.svg" }, - { link: "https://github.com/huggingface", imageLink: "http://lysand.re/icons/github.svg" }, - { link: "https://www.linkedin.com/company/huggingface/", imageLink: "http://lysand.re/icons/linkedin.svg" } + { link: "https://huggingface.co", imageLink: "https://huggingface.co/assets/transformers-docs/website.svg" }, + { link: "https://twitter.com/huggingface", imageLink: "https://huggingface.co/assets/transformers-docs/twitter.svg" }, + { link: "https://github.com/huggingface", imageLink: "https://huggingface.co/assets/transformers-docs/github.svg" }, + { link: "https://www.linkedin.com/company/huggingface/", imageLink: "https://huggingface.co/assets/transformers-docs/linkedin.svg" } ]; imageDetails.forEach(imageLinks => { @@ -42,13 +42,38 @@ function addCustomFooter() { }); customFooter.appendChild(social); - document.getElementsByTagName("footer")[0].appendChild(customFooter); + document.querySelector("footer").appendChild(customFooter); } +function addGithubButton() { + const div = ` +

+ + Star + +
+ `; + document.querySelector(".wy-side-nav-search .icon-home").insertAdjacentHTML('afterend', div); +} + +/*! + * github-buttons v2.2.10 + * (c) 2019 なつき + * @license BSD-2-Clause + */ +/** + * modified to run programmatically + */ +function parseGithubButtons (){"use strict";var e=window.document,t=e.location,o=window.encodeURIComponent,r=window.decodeURIComponent,n=window.Math,a=window.HTMLElement,i=window.XMLHttpRequest,l="https://unpkg.com/github-buttons@2.2.10/dist/buttons.html",c=i&&i.prototype&&"withCredentials"in i.prototype,d=c&&a&&a.prototype.attachShadow&&!a.prototype.attachShadow.prototype,s=function(e,t,o){e.addEventListener?e.addEventListener(t,o):e.attachEvent("on"+t,o)},u=function(e,t,o){e.removeEventListener?e.removeEventListener(t,o):e.detachEvent("on"+t,o)},h=function(e,t,o){var r=function(n){return u(e,t,r),o(n)};s(e,t,r)},f=function(e,t,o){var r=function(n){if(t.test(e.readyState))return u(e,"readystatechange",r),o(n)};s(e,"readystatechange",r)},p=function(e){return function(t,o,r){var n=e.createElement(t);if(o)for(var a in o){var i=o[a];null!=i&&(null!=n[a]?n[a]=i:n.setAttribute(a,i))}if(r)for(var l=0,c=r.length;l'},eye:{width:16,height:16,path:''},star:{width:14,height:16,path:''},"repo-forked":{width:10,height:16,path:''},"issue-opened":{width:14,height:16,path:''},"cloud-download":{width:16,height:16,path:''}},w={},x=function(e,t,o){var r=p(e.ownerDocument),n=e.appendChild(r("style",{type:"text/css"}));n.styleSheet?n.styleSheet.cssText=m:n.appendChild(e.ownerDocument.createTextNode(m));var a,l,d=r("a",{className:"btn",href:t.href,target:"_blank",innerHTML:(a=t["data-icon"],l=/^large$/i.test(t["data-size"])?16:14,a=(""+a).toLowerCase().replace(/^octicon-/,""),{}.hasOwnProperty.call(v,a)||(a="mark-github"),'"),"aria-label":t["aria-label"]||void 0},[" ",r("span",{},[t["data-text"]||""])]);/\.github\.com$/.test("."+d.hostname)?/^https?:\/\/((gist\.)?github\.com\/[^\/?#]+\/[^\/?#]+\/archive\/|github\.com\/[^\/?#]+\/[^\/?#]+\/releases\/download\/|codeload\.github\.com\/)/.test(d.href)&&(d.target="_top"):(d.href="#",d.target="_self");var u,h,g,x,y=e.appendChild(r("div",{className:"widget"+(/^large$/i.test(t["data-size"])?" lg":"")},[d]));/^(true|1)$/i.test(t["data-show-count"])&&"github.com"===d.hostname&&(u=d.pathname.replace(/^(?!\/)/,"/").match(/^\/([^\/?#]+)(?:\/([^\/?#]+)(?:\/(?:(subscription)|(fork)|(issues)|([^\/?#]+)))?)?(?:[\/?#]|$)/))&&!u[6]?(u[2]?(h="/repos/"+u[1]+"/"+u[2],u[3]?(x="subscribers_count",g="watchers"):u[4]?(x="forks_count",g="network"):u[5]?(x="open_issues_count",g="issues"):(x="stargazers_count",g="stargazers")):(h="/users/"+u[1],g=x="followers"),function(e,t){var o=w[e]||(w[e]=[]);if(!(o.push(t)>1)){var r=b(function(){for(delete w[e];t=o.shift();)t.apply(null,arguments)});if(c){var n=new i;s(n,"abort",r),s(n,"error",r),s(n,"load",function(){var e;try{e=JSON.parse(n.responseText)}catch(e){return void r(e)}r(200!==n.status,e)}),n.open("GET",e),n.send()}else{var a=this||window;a._=function(e){a._=null,r(200!==e.meta.status,e.data)};var l=p(a.document)("script",{async:!0,src:e+(/\?/.test(e)?"&":"?")+"callback=_"}),d=function(){a._&&a._({meta:{}})};s(l,"load",d),s(l,"error",d),l.readyState&&f(l,/de|m/,d),a.document.getElementsByTagName("head")[0].appendChild(l)}}}.call(this,"https://api.github.com"+h,function(e,t){if(!e){var n=t[x];y.appendChild(r("a",{className:"social-count",href:t.html_url+"/"+g,target:"_blank","aria-label":n+" "+x.replace(/_count$/,"").replace("_"," ").slice(0,n<2?-1:void 0)+" on GitHub"},[r("b"),r("i"),r("span",{},[(""+n).replace(/\B(?=(\d{3})+(?!\d))/g,",")])]))}o&&o(y)})):o&&o(y)},y=window.devicePixelRatio||1,C=function(e){return(y>1?n.ceil(n.round(e*y)/y*2)/2:n.ceil(e))||0},F=function(e,t){e.style.width=t[0]+"px",e.style.height=t[1]+"px"},k=function(t,r){if(null!=t&&null!=r)if(t.getAttribute&&(t=function(e){for(var t={href:e.href,title:e.title,"aria-label":e.getAttribute("aria-label")},o=["icon","text","size","show-count"],r=0,n=o.length;r Date: Thu, 26 Sep 2019 18:22:45 -0400 Subject: [PATCH 176/219] [docs] Fix doc auto-deploy Co-Authored-By: Lysandre Debut --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1bb3fd0877..35ca1454a5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -81,7 +81,6 @@ jobs: - checkout - run: sudo pip install --progress-bar off -r docs/requirements.txt - run: sudo pip install --progress-bar off -r requirements.txt - - run: cd docs/source && ln -s ../../examples/README.md examples.md && cd - - run: cd docs && make clean && make html && scp -r -oStrictHostKeyChecking=no _build/html/* $doc:$dir workflow_filters: &workflow_filters filters: From 702f589848baba97ea4897aa3f0bb937e1ec3bcf Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Fri, 27 Sep 2019 00:20:14 -0400 Subject: [PATCH 177/219] fix input in run_glue for distilbert --- examples/run_glue.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/run_glue.py b/examples/run_glue.py index 71dad0edbf..fc3b617da0 100644 --- a/examples/run_glue.py +++ b/examples/run_glue.py @@ -134,8 +134,9 @@ def train(args, train_dataset, model, tokenizer): batch = tuple(t.to(args.device) for t in batch) inputs = {'input_ids': batch[0], 'attention_mask': batch[1], - 'token_type_ids': batch[2] if args.model_type in ['bert', 'xlnet'] else None, # XLM, DistilBERT and RoBERTa don't use segment_ids 'labels': batch[3]} + if args.model_type != 'distilbert': + inputs['token_type_ids'] = batch[2] if args.model_type in ['bert', 'xlnet'] else None # XLM, DistilBERT and RoBERTa don't use segment_ids outputs = model(**inputs) loss = outputs[0] # model outputs are always tuple in transformers (see doc) @@ -224,8 +225,9 @@ def evaluate(args, model, tokenizer, prefix=""): with torch.no_grad(): inputs = {'input_ids': batch[0], 'attention_mask': batch[1], - 'token_type_ids': batch[2] if args.model_type in ['bert', 'xlnet'] else None, # XLM, DistilBERT and RoBERTa don't use segment_ids 'labels': batch[3]} + if args.model_type != 'distilbert': + inputs['token_type_ids'] = batch[2] if args.model_type in ['bert', 'xlnet'] else None # XLM, DistilBERT and RoBERTa don't use segment_ids outputs = model(**inputs) tmp_eval_loss, logits = outputs[:2] From 528c288fa94b0a258c610559c52fbc3a03e46805 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Fri, 27 Sep 2019 09:40:29 +0200 Subject: [PATCH 178/219] clean up run_tf_glue --- examples/run_tf_glue.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/examples/run_tf_glue.py b/examples/run_tf_glue.py index 3a867f80a8..9235612cb0 100644 --- a/examples/run_tf_glue.py +++ b/examples/run_tf_glue.py @@ -23,12 +23,6 @@ model.compile(optimizer=optimizer, loss=loss, metrics=[metric]) history = model.fit(train_dataset, epochs=2, steps_per_epoch=115, validation_data=valid_dataset, validation_steps=7) ->>> Train for 115 steps, validate for 7 steps ->>> Epoch 1/2 ->>> 115/115 [==============================] - 53s 459ms/step - loss: 0.6033 - accuracy: 0.6712 - val_loss: 0.4964 - val_accuracy: 0.7647 ->>> Epoch 2/2 ->>> 115/115 [==============================] - 33s 289ms/step - loss: 0.4141 - accuracy: 0.8160 - val_loss: 0.3914 - val_accuracy: 0.8382 - # Load the TensorFlow model in PyTorch for inspection model.save_pretrained('./save/') pytorch_model = BertForSequenceClassification.from_pretrained('./save/', from_tf=True) @@ -44,5 +38,3 @@ pred_1 = pytorch_model(**inputs_1)[0].argmax().item() pred_2 = pytorch_model(**inputs_2)[0].argmax().item() print("sentence_1 is", "a paraphrase" if pred_1 else "not a paraphrase", "of sentence_0") print("sentence_2 is", "a paraphrase" if pred_2 else "not a paraphrase", "of sentence_0") ->>> sentence_1 is a paraphrase of sentence_0 ->>> sentence_2 is not a paraphrase of sentence_0 \ No newline at end of file From da2e47ad15e552b84815da20daf3282b517103f7 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Fri, 27 Sep 2019 09:41:15 +0200 Subject: [PATCH 179/219] clean up a little run_tf_glue --- examples/run_tf_glue.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/run_tf_glue.py b/examples/run_tf_glue.py index 9235612cb0..f2e94ae39e 100644 --- a/examples/run_tf_glue.py +++ b/examples/run_tf_glue.py @@ -1,6 +1,6 @@ import tensorflow as tf import tensorflow_datasets -from transformers import * +from transformers import BertTokenizer, TFBertForSequenceClassification, glue_convert_examples_to_features, BertForSequenceClassification # Load dataset, tokenizer, model from pretrained model/vocabulary tokenizer = BertTokenizer.from_pretrained('bert-base-cased') From 15749bfc10b97b26f8c0dbfa5c67c30ed046e92b Mon Sep 17 00:00:00 2001 From: BramVanroy Date: Fri, 27 Sep 2019 10:01:36 +0200 Subject: [PATCH 180/219] Add small note about the output of hidden states --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index a111b5f943..6accc600ff 100644 --- a/README.md +++ b/README.md @@ -452,6 +452,10 @@ outputs = model(input_ids, labels=labels) loss, logits, attentions = outputs ``` +### Using hidden states + +By enabling the configuration option `output_hidden_states`, it was possible to retrieve the last hidden states of the encoder. In `pytorch-transformers` as well as `transformers` the return value has changed slightly: `all_hidden_states` now also includes the hidden state of the embeddings in addition to those of the encoding layers. This allows users to easily access the embeddings final state. + ### Serialization Breaking change in the `from_pretrained()`method: From d2de5b9d8c4150281bd204e50ded32019f543ad8 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Fri, 27 Sep 2019 07:08:36 -0300 Subject: [PATCH 181/219] Just some typos --- docs/source/quickstart.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/source/quickstart.md b/docs/source/quickstart.md index c7ad5c7930..c73d3a5429 100644 --- a/docs/source/quickstart.md +++ b/docs/source/quickstart.md @@ -93,8 +93,8 @@ Let's see how we can use `BertModel` to encode our inputs in hidden-states: # Load pre-trained model (weights) model = BertModel.from_pretrained('bert-base-uncased') -# Set the model in evaluation mode to desactivate the DropOut modules -# This is IMPORTANT to have reproductible results during evaluation! +# Set the model in evaluation mode to deactivate the DropOut modules +# This is IMPORTANT to have reproducible results during evaluation! model.eval() # If you have a GPU, put everything on cuda @@ -168,8 +168,8 @@ Let's see how to use `GPT2LMHeadModel` to generate the next token following our # Load pre-trained model (weights) model = GPT2LMHeadModel.from_pretrained('gpt2') -# Set the model in evaluation mode to desactivate the DropOut modules -# This is IMPORTANT to have reproductible results during evaluation! +# Set the model in evaluation mode to deactivate the DropOut modules +# This is IMPORTANT to have reproducible results during evaluation! model.eval() # If you have a GPU, put everything on cuda From 4f2b6579bf59b033e5eda27a51d4192e9c2d2623 Mon Sep 17 00:00:00 2001 From: pj Date: Fri, 27 Sep 2019 22:55:43 +0800 Subject: [PATCH 182/219] Fix some typos --- docs/source/quickstart.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/quickstart.md b/docs/source/quickstart.md index c73d3a5429..4eb04c17d2 100644 --- a/docs/source/quickstart.md +++ b/docs/source/quickstart.md @@ -19,12 +19,12 @@ The library was designed with two strong goals in mind: A few other goals: -- expose the models internals as consistently as possible: +- expose the models' internals as consistently as possible: - we give access, using a single API to the full hidden-states and attention weights, - tokenizer and base model's API are standardized to easily switch between models. -- incorporate a subjective selection of promising tools for fine-tuning/investiguating these models: +- incorporate a subjective selection of promising tools for fine-tuning/investigating these models: - a simple/consistent way to add new tokens to the vocabulary and embeddings for fine-tuning, - simple ways to mask and prune transformer heads. @@ -51,7 +51,7 @@ We'll finish this quickstart tour by going through a few simple quick-start exam Here are two examples showcasing a few `Bert` and `GPT2` classes and pre-trained models. -See full API reference for examples for each model classe. +See full API reference for examples for each model class. ### BERT example From e31a4728013db1914b7dd0f50cc18c5702db5b79 Mon Sep 17 00:00:00 2001 From: Agrin Hilmkil Date: Fri, 27 Sep 2019 16:51:17 +0200 Subject: [PATCH 183/219] Fix tensorflow_dataset glue support `glue_convert_examples_to_features` assumed that tensorflow_dataset examples contains the features `'sentence1'` and `'sentence2'`. This commit encapsulates the choice of features in the glue processor and uses that to parse examples. --- transformers/data/processors/glue.py | 59 ++++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 4 deletions(-) diff --git a/transformers/data/processors/glue.py b/transformers/data/processors/glue.py index 2322f58604..8bf209a08b 100644 --- a/transformers/data/processors/glue.py +++ b/transformers/data/processors/glue.py @@ -79,10 +79,7 @@ def glue_convert_examples_to_features(examples, tokenizer, if ex_index % 10000 == 0: logger.info("Writing example %d" % (ex_index)) if is_tf_dataset: - example = InputExample(example['idx'].numpy(), - example['sentence1'].numpy().decode('utf-8'), - example['sentence2'].numpy().decode('utf-8'), - str(example['label'].numpy())) + example = processor.get_example_from_tensor_dict(example) inputs = tokenizer.encode_plus( example.text_a, @@ -157,6 +154,12 @@ def glue_convert_examples_to_features(examples, tokenizer, class MrpcProcessor(DataProcessor): """Processor for the MRPC data set (GLUE version).""" + def get_example_from_tensor_dict(self, tensor_dict): + return InputExample(tensor_dict['idx'].numpy(), + tensor_dict['sentence1'].numpy().decode('utf-8'), + tensor_dict['sentence2'].numpy().decode('utf-8'), + str(tensor_dict['label'].numpy())) + def get_train_examples(self, data_dir): """See base class.""" logger.info("LOOKING AT {}".format(os.path.join(data_dir, "train.tsv"))) @@ -190,6 +193,12 @@ class MrpcProcessor(DataProcessor): class MnliProcessor(DataProcessor): """Processor for the MultiNLI data set (GLUE version).""" + def get_example_from_tensor_dict(self, tensor_dict): + return InputExample(tensor_dict['idx'].numpy(), + tensor_dict['premise'].numpy().decode('utf-8'), + tensor_dict['hypothesis'].numpy().decode('utf-8'), + str(tensor_dict['label'].numpy())) + def get_train_examples(self, data_dir): """See base class.""" return self._create_examples( @@ -233,6 +242,12 @@ class MnliMismatchedProcessor(MnliProcessor): class ColaProcessor(DataProcessor): """Processor for the CoLA data set (GLUE version).""" + def get_example_from_tensor_dict(self, tensor_dict): + return InputExample(tensor_dict['idx'].numpy(), + tensor_dict['sentence'].numpy().decode('utf-8'), + None, + str(tensor_dict['label'].numpy())) + def get_train_examples(self, data_dir): """See base class.""" return self._create_examples( @@ -262,6 +277,12 @@ class ColaProcessor(DataProcessor): class Sst2Processor(DataProcessor): """Processor for the SST-2 data set (GLUE version).""" + def get_example_from_tensor_dict(self, tensor_dict): + return InputExample(tensor_dict['idx'].numpy(), + tensor_dict['sentence'].numpy().decode('utf-8'), + None, + str(tensor_dict['label'].numpy())) + def get_train_examples(self, data_dir): """See base class.""" return self._create_examples( @@ -293,6 +314,12 @@ class Sst2Processor(DataProcessor): class StsbProcessor(DataProcessor): """Processor for the STS-B data set (GLUE version).""" + def get_example_from_tensor_dict(self, tensor_dict): + return InputExample(tensor_dict['idx'].numpy(), + tensor_dict['sentence1'].numpy().decode('utf-8'), + tensor_dict['sentence2'].numpy().decode('utf-8'), + str(tensor_dict['label'].numpy())) + def get_train_examples(self, data_dir): """See base class.""" return self._create_examples( @@ -325,6 +352,12 @@ class StsbProcessor(DataProcessor): class QqpProcessor(DataProcessor): """Processor for the QQP data set (GLUE version).""" + def get_example_from_tensor_dict(self, tensor_dict): + return InputExample(tensor_dict['idx'].numpy(), + tensor_dict['question1'].numpy().decode('utf-8'), + tensor_dict['question2'].numpy().decode('utf-8'), + str(tensor_dict['label'].numpy())) + def get_train_examples(self, data_dir): """See base class.""" return self._create_examples( @@ -360,6 +393,12 @@ class QqpProcessor(DataProcessor): class QnliProcessor(DataProcessor): """Processor for the QNLI data set (GLUE version).""" + def get_example_from_tensor_dict(self, tensor_dict): + return InputExample(tensor_dict['idx'].numpy(), + tensor_dict['question'].numpy().decode('utf-8'), + tensor_dict['sentence'].numpy().decode('utf-8'), + str(tensor_dict['label'].numpy())) + def get_train_examples(self, data_dir): """See base class.""" return self._create_examples( @@ -393,6 +432,12 @@ class QnliProcessor(DataProcessor): class RteProcessor(DataProcessor): """Processor for the RTE data set (GLUE version).""" + def get_example_from_tensor_dict(self, tensor_dict): + return InputExample(tensor_dict['idx'].numpy(), + tensor_dict['sentence1'].numpy().decode('utf-8'), + tensor_dict['sentence2'].numpy().decode('utf-8'), + str(tensor_dict['label'].numpy())) + def get_train_examples(self, data_dir): """See base class.""" return self._create_examples( @@ -425,6 +470,12 @@ class RteProcessor(DataProcessor): class WnliProcessor(DataProcessor): """Processor for the WNLI data set (GLUE version).""" + def get_example_from_tensor_dict(self, tensor_dict): + return InputExample(tensor_dict['idx'].numpy(), + tensor_dict['sentence1'].numpy().decode('utf-8'), + tensor_dict['sentence2'].numpy().decode('utf-8'), + str(tensor_dict['label'].numpy())) + def get_train_examples(self, data_dir): """See base class.""" return self._create_examples( From 795b3e76ffbeb224ee334252abc1b3c359b26067 Mon Sep 17 00:00:00 2001 From: Agrin Hilmkil Date: Fri, 27 Sep 2019 17:32:28 +0200 Subject: [PATCH 184/219] Add docstring for processor method --- transformers/data/processors/glue.py | 9 +++++++++ transformers/data/processors/utils.py | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/transformers/data/processors/glue.py b/transformers/data/processors/glue.py index 8bf209a08b..61bca8c11b 100644 --- a/transformers/data/processors/glue.py +++ b/transformers/data/processors/glue.py @@ -155,6 +155,7 @@ class MrpcProcessor(DataProcessor): """Processor for the MRPC data set (GLUE version).""" def get_example_from_tensor_dict(self, tensor_dict): + """See base class.""" return InputExample(tensor_dict['idx'].numpy(), tensor_dict['sentence1'].numpy().decode('utf-8'), tensor_dict['sentence2'].numpy().decode('utf-8'), @@ -194,6 +195,7 @@ class MnliProcessor(DataProcessor): """Processor for the MultiNLI data set (GLUE version).""" def get_example_from_tensor_dict(self, tensor_dict): + """See base class.""" return InputExample(tensor_dict['idx'].numpy(), tensor_dict['premise'].numpy().decode('utf-8'), tensor_dict['hypothesis'].numpy().decode('utf-8'), @@ -243,6 +245,7 @@ class ColaProcessor(DataProcessor): """Processor for the CoLA data set (GLUE version).""" def get_example_from_tensor_dict(self, tensor_dict): + """See base class.""" return InputExample(tensor_dict['idx'].numpy(), tensor_dict['sentence'].numpy().decode('utf-8'), None, @@ -278,6 +281,7 @@ class Sst2Processor(DataProcessor): """Processor for the SST-2 data set (GLUE version).""" def get_example_from_tensor_dict(self, tensor_dict): + """See base class.""" return InputExample(tensor_dict['idx'].numpy(), tensor_dict['sentence'].numpy().decode('utf-8'), None, @@ -315,6 +319,7 @@ class StsbProcessor(DataProcessor): """Processor for the STS-B data set (GLUE version).""" def get_example_from_tensor_dict(self, tensor_dict): + """See base class.""" return InputExample(tensor_dict['idx'].numpy(), tensor_dict['sentence1'].numpy().decode('utf-8'), tensor_dict['sentence2'].numpy().decode('utf-8'), @@ -353,6 +358,7 @@ class QqpProcessor(DataProcessor): """Processor for the QQP data set (GLUE version).""" def get_example_from_tensor_dict(self, tensor_dict): + """See base class.""" return InputExample(tensor_dict['idx'].numpy(), tensor_dict['question1'].numpy().decode('utf-8'), tensor_dict['question2'].numpy().decode('utf-8'), @@ -394,6 +400,7 @@ class QnliProcessor(DataProcessor): """Processor for the QNLI data set (GLUE version).""" def get_example_from_tensor_dict(self, tensor_dict): + """See base class.""" return InputExample(tensor_dict['idx'].numpy(), tensor_dict['question'].numpy().decode('utf-8'), tensor_dict['sentence'].numpy().decode('utf-8'), @@ -433,6 +440,7 @@ class RteProcessor(DataProcessor): """Processor for the RTE data set (GLUE version).""" def get_example_from_tensor_dict(self, tensor_dict): + """See base class.""" return InputExample(tensor_dict['idx'].numpy(), tensor_dict['sentence1'].numpy().decode('utf-8'), tensor_dict['sentence2'].numpy().decode('utf-8'), @@ -471,6 +479,7 @@ class WnliProcessor(DataProcessor): """Processor for the WNLI data set (GLUE version).""" def get_example_from_tensor_dict(self, tensor_dict): + """See base class.""" return InputExample(tensor_dict['idx'].numpy(), tensor_dict['sentence1'].numpy().decode('utf-8'), tensor_dict['sentence2'].numpy().decode('utf-8'), diff --git a/transformers/data/processors/utils.py b/transformers/data/processors/utils.py index d16ea786a0..27138f9959 100644 --- a/transformers/data/processors/utils.py +++ b/transformers/data/processors/utils.py @@ -86,6 +86,15 @@ class InputFeatures(object): class DataProcessor(object): """Base class for data converters for sequence classification data sets.""" + def get_example_from_tensor_dict(self, tensor_dict): + """Gets an example from a dict with tensorflow tensors + + Args: + tensor_dict: Keys and values should match the corresponding Glue + tensorflow_dataset examples. + """ + raise NotImplementedError() + def get_train_examples(self, data_dir): """Gets a collection of `InputExample`s for the train set.""" raise NotImplementedError() From 94785906309fb1154f63da16b053ee9416e04c7b Mon Sep 17 00:00:00 2001 From: Denny Date: Fri, 27 Sep 2019 15:18:42 -0300 Subject: [PATCH 185/219] Update run_lm_finetuning.py The previous method, just as phrased, did not exist in the class. --- examples/run_lm_finetuning.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/run_lm_finetuning.py b/examples/run_lm_finetuning.py index 8d440ebcc6..6e1a150313 100644 --- a/examples/run_lm_finetuning.py +++ b/examples/run_lm_finetuning.py @@ -75,7 +75,7 @@ class TextDataset(Dataset): tokenized_text = tokenizer.convert_tokens_to_ids(tokenizer.tokenize(text)) for i in range(0, len(tokenized_text)-block_size+1, block_size): # Truncate in block of block_size - self.examples.append(tokenizer.add_special_tokens_single_sentence(tokenized_text[i:i+block_size])) + self.examples.append(tokenizer.add_special_tokens_single_sequence(tokenized_text[i:i+block_size])) # Note that we are loosing the last truncated example here for the sake of simplicity (no padding) # If your dataset is small, first you should loook for a bigger one :-) and second you # can change this behavior by adding (model specific) padding. From c6acbdd50af6d1b915916001c01c6d2760fe316d Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Fri, 27 Sep 2019 17:02:29 -0400 Subject: [PATCH 186/219] Close #1304 --- transformers/tokenization_roberta.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/transformers/tokenization_roberta.py b/transformers/tokenization_roberta.py index 32e92be211..7adeea689e 100644 --- a/transformers/tokenization_roberta.py +++ b/transformers/tokenization_roberta.py @@ -81,6 +81,8 @@ class RobertaTokenizer(GPT2Tokenizer): bos_token=bos_token, eos_token=eos_token, unk_token=unk_token, sep_token=sep_token, cls_token=cls_token, pad_token=pad_token, mask_token=mask_token, **kwargs) + self.max_len_single_sentence = self.max_len - 2 # take into account special tokens + self.max_len_sentences_pair = self.max_len - 4 # take into account special tokens def add_special_tokens_single_sequence(self, token_ids): """ From d8b641c8393f1f7ce0ec0d0a1f7c1cefcd966971 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Fri, 27 Sep 2019 17:22:01 -0400 Subject: [PATCH 187/219] 6 -> 8 models --- docs/source/quickstart.md | 2 +- transformers/tokenization_utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/quickstart.md b/docs/source/quickstart.md index 4eb04c17d2..ccba75e7c0 100644 --- a/docs/source/quickstart.md +++ b/docs/source/quickstart.md @@ -33,7 +33,7 @@ A few other goals: The library is build around three type of classes for each models: -- **model classes** which are PyTorch models (`torch.nn.Modules`) of the 6 models architectures currently provided in the library, e.g. `BertModel` +- **model classes** which are PyTorch models (`torch.nn.Modules`) of the 8 models architectures currently provided in the library, e.g. `BertModel` - **configuration classes** which store all the parameters required to build a model, e.g. `BertConfig`. You don't always need to instantiate these your-self, in particular if you are using a pretrained model without any modification, creating the model will automatically take care of instantiating the configuration (which is part of the model) - **tokenizer classes** which store the vocabulary for each model and provide methods for encoding/decoding strings in list of token embeddings indices to be fed to a model, e.g. `BertTokenizer` diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py index e8ffff3cb9..1e20588f83 100644 --- a/transformers/tokenization_utils.py +++ b/transformers/tokenization_utils.py @@ -430,7 +430,7 @@ class PreTrainedTokenizer(object): - tokenizer instantiation positional and keywords inputs (e.g. do_lower_case for Bert). This won't save modifications other than (added tokens and special token mapping) you may have - applied to the tokenizer after the instantion (e.g. modifying tokenizer.do_lower_case after creation). + applied to the tokenizer after the instantiation (e.g. modifying tokenizer.do_lower_case after creation). This method make sure the full tokenizer can then be re-loaded using the :func:`~transformers.PreTrainedTokenizer.from_pretrained` class method. """ From a6a6d9e6382961dc92a1a08d1bab05a52dc815f9 Mon Sep 17 00:00:00 2001 From: Ikuya Yamada Date: Thu, 12 Sep 2019 12:13:37 -1000 Subject: [PATCH 188/219] fix padding_idx of RoBERTa model --- transformers/modeling_roberta.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/transformers/modeling_roberta.py b/transformers/modeling_roberta.py index 04ffbecc16..7e130a8c52 100644 --- a/transformers/modeling_roberta.py +++ b/transformers/modeling_roberta.py @@ -43,6 +43,9 @@ class RobertaEmbeddings(BertEmbeddings): def __init__(self, config): super(RobertaEmbeddings, self).__init__(config) self.padding_idx = 1 + self.word_embeddings = nn.Embedding(config.vocab_size, config.hidden_size, padding_idx=self.padding_idx) + self.position_embeddings = nn.Embedding(config.max_position_embeddings, config.hidden_size, + padding_idx=self.padding_idx) def forward(self, input_ids, token_type_ids=None, position_ids=None): seq_length = input_ids.size(1) From 60f791631bd2e7154552d071fe50640287343890 Mon Sep 17 00:00:00 2001 From: wangfei <1140554608@qq.com> Date: Sat, 28 Sep 2019 16:20:17 +0800 Subject: [PATCH 189/219] Fix link in readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6accc600ff..8dd2c2fb66 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ Choose the right framework for every part of a model's lifetime | [Model architectures](#model-architectures) | Architectures (with pretrained weights) | | [Online demo](#online-demo) | Experimenting with this repo’s text generation capabilities | | [Quick tour: Usage](#quick-tour) | Tokenizers & models usage: Bert and GPT-2 | -| [Quick tour: TF 2.0 and PyTorch ](#Quick-tour-TF-2.0-training-and-PyTorch-interoperability) | Train a TF 2.0 model in 10 lines of code, load it in PyTorch | +| [Quick tour: TF 2.0 and PyTorch ](#Quick-tour-TF-20-training-and-PyTorch-interoperability) | Train a TF 2.0 model in 10 lines of code, load it in PyTorch | | [Quick tour: Fine-tuning/usage scripts](#quick-tour-of-the-fine-tuningusage-scripts) | Using provided scripts: GLUE, SQuAD and Text generation | | [Migrating from pytorch-transformers to transformers](#Migrating-from-pytorch-transformers-to-transformers) | Migrating your code from pytorch-pretrained-bert to transformers | | [Migrating from pytorch-pretrained-bert to pytorch-transformers](#Migrating-from-pytorch-pretrained-bert-to-transformers) | Migrating your code from pytorch-pretrained-bert to transformers | From 2dc8cb87341223e86220516951bb4ad84f880b4a Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Sun, 29 Sep 2019 19:51:01 -0400 Subject: [PATCH 190/219] fix unknown imports (*ForMultipleChoice) in run_multiple_choice --- transformers/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/transformers/__init__.py b/transformers/__init__.py index 39370cb327..5248bc9f1b 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -74,14 +74,15 @@ if is_torch_available(): GPT2LMHeadModel, GPT2DoubleHeadsModel, load_tf_weights_in_gpt2, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_xlnet import (XLNetPreTrainedModel, XLNetModel, XLNetLMHeadModel, - XLNetForSequenceClassification, XLNetForQuestionAnsweringSimple, - XLNetForQuestionAnswering, + XLNetForSequenceClassification, XLNetForMultipleChoice, + XLNetForQuestionAnsweringSimple, XLNetForQuestionAnswering, load_tf_weights_in_xlnet, XLNET_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_xlm import (XLMPreTrainedModel , XLMModel, XLMWithLMHeadModel, XLMForSequenceClassification, XLMForQuestionAnswering, XLMForQuestionAnsweringSimple, XLM_PRETRAINED_MODEL_ARCHIVE_MAP) - from .modeling_roberta import (RobertaForMaskedLM, RobertaModel, RobertaForSequenceClassification, + from .modeling_roberta import (RobertaForMaskedLM, RobertaModel, + RobertaForSequenceClassification, RobertaForMultipleChoice, ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_distilbert import (DistilBertForMaskedLM, DistilBertModel, DistilBertForSequenceClassification, DistilBertForQuestionAnswering, From 5c3b32d44d0164aaa9b91405f48e53cf53a82b35 Mon Sep 17 00:00:00 2001 From: Santosh Gupta Date: Sat, 28 Sep 2019 16:35:06 -0700 Subject: [PATCH 191/219] Update README.md Lines 183 - 200, fixed indentation. Line 198, replaced `tokenizer_class` with `BertTokenizer`, since `tokenizer_class` is not defined in the loop it belongs to. --- README.md | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 8dd2c2fb66..b2c5df77e9 100644 --- a/README.md +++ b/README.md @@ -180,24 +180,24 @@ for model_class in BERT_MODEL_CLASSES: # Load pretrained model/tokenizer model = model_class.from_pretrained('bert-base-uncased') -# Models can return full list of hidden-states & attentions weights at each layer -model = model_class.from_pretrained(pretrained_weights, - output_hidden_states=True, - output_attentions=True) -input_ids = torch.tensor([tokenizer.encode("Let's see all hidden-states and attentions on this text")]) -all_hidden_states, all_attentions = model(input_ids)[-2:] + # Models can return full list of hidden-states & attentions weights at each layer + model = model_class.from_pretrained(pretrained_weights, + output_hidden_states=True, + output_attentions=True) + input_ids = torch.tensor([tokenizer.encode("Let's see all hidden-states and attentions on this text")]) + all_hidden_states, all_attentions = model(input_ids)[-2:] -# Models are compatible with Torchscript -model = model_class.from_pretrained(pretrained_weights, torchscript=True) -traced_model = torch.jit.trace(model, (input_ids,)) + # Models are compatible with Torchscript + model = model_class.from_pretrained(pretrained_weights, torchscript=True) + traced_model = torch.jit.trace(model, (input_ids,)) -# Simple serialization for models and tokenizers -model.save_pretrained('./directory/to/save/') # save -model = model_class.from_pretrained('./directory/to/save/') # re-load -tokenizer.save_pretrained('./directory/to/save/') # save -tokenizer = tokenizer_class.from_pretrained('./directory/to/save/') # re-load + # Simple serialization for models and tokenizers + model.save_pretrained('./directory/to/save/') # save + model = model_class.from_pretrained('./directory/to/save/') # re-load + tokenizer.save_pretrained('./directory/to/save/') # save + tokenizer = BertTokenizer.from_pretrained('./directory/to/save/') # re-load -# SOTA examples for GLUE, SQUAD, text generation... + # SOTA examples for GLUE, SQUAD, text generation... ``` ## Quick tour TF 2.0 training and PyTorch interoperability From 6971556ab83a5a3edd2f99d322b0954499393d2b Mon Sep 17 00:00:00 2001 From: DenysNahurnyi Date: Tue, 1 Oct 2019 21:57:18 +0300 Subject: [PATCH 192/219] Fix syntax typo in README.md --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index b2c5df77e9..b5b7245bd9 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ pip install transformers Here also, you first need to install one of, or both, TensorFlow 2.0 and PyTorch. Please refere to [TensorFlow installation page](https://www.tensorflow.org/install/pip#tensorflow-2.0-rc-is-available) and/or [PyTorch installation page](https://pytorch.org/get-started/locally/#start-locally) regarding the specific install command for your platform. -When TensorFlow 2.0 and/or PyTorch has been installed, you can install from source by cloning the repository and runing: +When TensorFlow 2.0 and/or PyTorch has been installed, you can install from source by cloning the repository and running: ```bash pip install [--editable] . @@ -88,7 +88,7 @@ pip install [--editable] . ### Tests -A series of tests is included for the library and the example scripts. Library tests can be found in the [tests folder](https://github.com/huggingface/transformers/tree/master/transformers/tests) and examples tests in the [examples folder](https://github.com/huggingface/transformers/tree/master/examples). +A series of tests are included for the library and the example scripts. Library tests can be found in the [tests folder](https://github.com/huggingface/transformers/tree/master/transformers/tests) and examples tests in the [examples folder](https://github.com/huggingface/transformers/tree/master/examples). These tests can be run using `pytest` (install pytest if needed with `pip install pytest`). @@ -394,7 +394,7 @@ This is the model provided as `bert-large-uncased-whole-word-masking-finetuned-s ### `run_generation.py`: Text generation with GPT, GPT-2, Transformer-XL and XLNet A conditional generation script is also included to generate text from a prompt. -The generation script includes the [tricks](https://github.com/rusiaaman/XLNet-gen#methodology) proposed by Aman Rusia to get high quality generation with memory models like Transformer-XL and XLNet (include a predefined text to make short inputs longer). +The generation script includes the [tricks](https://github.com/rusiaaman/XLNet-gen#methodology) proposed by Aman Rusia to get high-quality generation with memory models like Transformer-XL and XLNet (include a predefined text to make short inputs longer). Here is how to run the script with the small version of OpenAI GPT-2 model: @@ -426,7 +426,7 @@ Here is a quick summary of what you should take care of when migrating from `pyt The main breaking change when migrating from `pytorch-pretrained-bert` to `transformers` is that the models forward method always outputs a `tuple` with various elements depending on the model and the configuration parameters. -The exact content of the tuples for each model are detailed in the models' docstrings and the [documentation](https://huggingface.co/transformers/). +The exact content of the tuples for each model is detailed in the models' docstrings and the [documentation](https://huggingface.co/transformers/). In pretty much every case, you will be fine by taking the first element of the output as the output you previously used in `pytorch-pretrained-bert`. @@ -458,7 +458,7 @@ By enabling the configuration option `output_hidden_states`, it was possible to ### Serialization -Breaking change in the `from_pretrained()`method: +Breaking change in the `from_pretrained()` method: 1. Models are now set in evaluation mode by default when instantiated with the `from_pretrained()` method. To train them don't forget to set them back in training mode (`model.train()`) to activate the dropout modules. @@ -534,4 +534,4 @@ for batch in train_data: ## Citation -At the moment, there is no paper associated to Transformers but we are working on preparing one. In the meantime, please include a mention of the library and a link to the present repository if you use this work in a published or open-source project. +At the moment, there is no paper associated with Transformers but we are working on preparing one. In the meantime, please include a mention of the library and a link to the present repository if you use this work in a published or open-source project. From f7978f70ecadd3ac6e3072f72e9dcc8c01e187c8 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 1 Oct 2019 18:45:38 -0400 Subject: [PATCH 193/219] use format instead of f-strings --- examples/run_lm_finetuning.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/run_lm_finetuning.py b/examples/run_lm_finetuning.py index dad9fab83f..0115dcb98b 100644 --- a/examples/run_lm_finetuning.py +++ b/examples/run_lm_finetuning.py @@ -57,7 +57,7 @@ class TextDataset(Dataset): def __init__(self, tokenizer, file_path='train', block_size=512): assert os.path.isfile(file_path) directory, filename = os.path.split(file_path) - cached_features_file = os.path.join(directory, 'cached_lm_{block_size}_{filename}') + cached_features_file = os.path.join(directory, 'cached_lm_{}_{}'.format(block_size, filename)) if os.path.exists(cached_features_file): logger.info("Loading features from cached file %s", cached_features_file) From 391db836ab7ed2ca61c51a7cf1b135b6ab92be58 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 1 Oct 2019 19:09:13 -0400 Subject: [PATCH 194/219] fix #1260 - remove special logic for decoding pairs of sequence --- transformers/tokenization_utils.py | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py index 1e20588f83..db9e9cd72e 100644 --- a/transformers/tokenization_utils.py +++ b/transformers/tokenization_utils.py @@ -933,20 +933,11 @@ class PreTrainedTokenizer(object): sub_texts.append(self.convert_tokens_to_string(current_sub_text)) text = ''.join(sub_texts) - if self._sep_token is not None and self._sep_token in text: - text = text.replace(self._cls_token, self._sep_token) - split_text = list(filter(lambda sentence: len(sentence) > 0, text.split(self._sep_token))) - if clean_up_tokenization_spaces: - clean_text = [self.clean_up_tokenization(text) for text in split_text] - return clean_text - else: - return split_text + if clean_up_tokenization_spaces: + clean_text = self.clean_up_tokenization(text) + return clean_text else: - if clean_up_tokenization_spaces: - clean_text = self.clean_up_tokenization(text) - return clean_text - else: - return text + return text @property def special_tokens_map(self): From a95158518d65fe640ecb35813280609e27ba3ab7 Mon Sep 17 00:00:00 2001 From: danai-antoniou Date: Wed, 2 Oct 2019 07:44:15 +0100 Subject: [PATCH 195/219] Moved duplicate token check --- transformers/tokenization_utils.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py index d8b3c0c74b..de3f48f4c3 100644 --- a/transformers/tokenization_utils.py +++ b/transformers/tokenization_utils.py @@ -508,14 +508,12 @@ class PreTrainedTokenizer(object): if not new_tokens: return 0 - if len(new_tokens) != len(set(new_tokens)): - raise ValueError("The provided list of tokens contains duplicates.") - to_add_tokens = [] for token in new_tokens: assert isinstance(token, str) or (six.PY2 and isinstance(token, unicode)) if token != self.unk_token and \ - self.convert_tokens_to_ids(token) == self.convert_tokens_to_ids(self.unk_token): + self.convert_tokens_to_ids(token) == self.convert_tokens_to_ids(self.unk_token) and \ + token not in to_add_tokens: to_add_tokens.append(token) logger.info("Adding %s to the vocabulary", token) From 63ed224b7c550ead5f9599187e665ded57ce80d4 Mon Sep 17 00:00:00 2001 From: Santiago Castro Date: Wed, 2 Oct 2019 11:02:08 -0400 Subject: [PATCH 196/219] initialy -> initially --- transformers/modeling_bert.py | 2 +- transformers/modeling_tf_bert.py | 2 +- transformers/modeling_tf_distilbert.py | 2 +- transformers/modeling_tf_xlm.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py index 51e407d0a6..fc448fa366 100644 --- a/transformers/modeling_bert.py +++ b/transformers/modeling_bert.py @@ -118,7 +118,7 @@ def load_tf_weights_in_bert(model, config, tf_checkpoint_path): def gelu(x): - """ Original Implementation of the gelu activation function in Google Bert repo when initialy created. + """ Original Implementation of the gelu activation function in Google Bert repo when initially created. For information: OpenAI GPT's gelu is slightly different (and gives slightly different results): 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) Also see https://arxiv.org/abs/1606.08415 diff --git a/transformers/modeling_tf_bert.py b/transformers/modeling_tf_bert.py index d763ca991e..4de94751f8 100644 --- a/transformers/modeling_tf_bert.py +++ b/transformers/modeling_tf_bert.py @@ -62,7 +62,7 @@ def load_bert_pt_weights_in_tf2(tf_model, pytorch_checkpoint_path): def gelu(x): """ Gaussian Error Linear Unit. - Original Implementation of the gelu activation function in Google Bert repo when initialy created. + Original Implementation of the gelu activation function in Google Bert repo when initially created. For information: OpenAI GPT's gelu is slightly different (and gives slightly different results): 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) Also see https://arxiv.org/abs/1606.08415 diff --git a/transformers/modeling_tf_distilbert.py b/transformers/modeling_tf_distilbert.py index 2a917a30a4..5ce1616bcc 100644 --- a/transformers/modeling_tf_distilbert.py +++ b/transformers/modeling_tf_distilbert.py @@ -45,7 +45,7 @@ TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP = { ### UTILS AND BUILDING BLOCKS OF THE ARCHITECTURE ### def gelu(x): """ Gaussian Error Linear Unit. - Original Implementation of the gelu activation function in Google Bert repo when initialy created. + Original Implementation of the gelu activation function in Google Bert repo when initially created. For information: OpenAI GPT's gelu is slightly different (and gives slightly different results): 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) Also see https://arxiv.org/abs/1606.08415 diff --git a/transformers/modeling_tf_xlm.py b/transformers/modeling_tf_xlm.py index f8f199bbe6..83cc37c6a7 100644 --- a/transformers/modeling_tf_xlm.py +++ b/transformers/modeling_tf_xlm.py @@ -69,7 +69,7 @@ def create_sinusoidal_embeddings(n_pos, dim, out): def gelu(x): """ Gaussian Error Linear Unit. - Original Implementation of the gelu activation function in Google Bert repo when initialy created. + Original Implementation of the gelu activation function in Google Bert repo when initially created. For information: OpenAI GPT's gelu is slightly different (and gives slightly different results): 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) Also see https://arxiv.org/abs/1606.08415 From ebb32261b19eaa258f998d2725116fe7a08224a6 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Wed, 2 Oct 2019 17:52:56 -0400 Subject: [PATCH 197/219] fix #1401 --- docs/source/pretrained_models.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/source/pretrained_models.rst b/docs/source/pretrained_models.rst index 4c17b35c84..c12a9bc52f 100644 --- a/docs/source/pretrained_models.rst +++ b/docs/source/pretrained_models.rst @@ -98,6 +98,12 @@ Here is the full list of the currently provided pretrained models together with | +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``xlm-clm-ende-1024`` | | 6-layer, 1024-hidden, 8-heads | | | | | XLM English-German model trained with CLM (Causal Language Modeling) on the concatenation of English and German wikipedia | +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``xlm-mlm-17-1280`` | | 16-layer, 1280-hidden, 16-heads | +| | | | XLM model trained with MLM (Masked Language Modeling) on 17 languages. | +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``xlm-mlm-100-1280`` | | 16-layer, 1280-hidden, 16-heads | +| | | | XLM model trained with MLM (Masked Language Modeling) on 100 languages. | +-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | RoBERTa | ``roberta-base`` | | 12-layer, 768-hidden, 12-heads, 125M parameters | | | | | RoBERTa using the BERT-base architecture | From 38084507c45c784dd5041058b8aa1676a633a18c Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Wed, 2 Oct 2019 11:00:46 -0400 Subject: [PATCH 198/219] add distillation_configs --- .../training_configs/distilbert-base-uncased.json | 15 +++++++++++++++ .../distillation/training_configs/distilgpt2.json | 10 ++++++++++ 2 files changed, 25 insertions(+) create mode 100644 examples/distillation/training_configs/distilbert-base-uncased.json create mode 100644 examples/distillation/training_configs/distilgpt2.json diff --git a/examples/distillation/training_configs/distilbert-base-uncased.json b/examples/distillation/training_configs/distilbert-base-uncased.json new file mode 100644 index 0000000000..15d1e7fe00 --- /dev/null +++ b/examples/distillation/training_configs/distilbert-base-uncased.json @@ -0,0 +1,15 @@ +{ + "activation": "gelu", + "attention_dropout": 0.1, + "dim": 768, + "dropout": 0.1, + "hidden_dim": 3072, + "initializer_range": 0.02, + "max_position_embeddings": 512, + "n_heads": 12, + "n_layers": 6, + "sinusoidal_pos_embds": true, + "tie_weights_": true, + "vocab_size": 30522 + } + \ No newline at end of file diff --git a/examples/distillation/training_configs/distilgpt2.json b/examples/distillation/training_configs/distilgpt2.json new file mode 100644 index 0000000000..8616e8e60f --- /dev/null +++ b/examples/distillation/training_configs/distilgpt2.json @@ -0,0 +1,10 @@ +{ + "initializer_range": 0.02, + "layer_norm_epsilon": 0.00001, + "n_ctx": 1024, + "n_embd": 768, + "n_head": 12, + "n_layer": 6, + "n_positions": 1024, + "vocab_size": 50257 +} \ No newline at end of file From 594202a9348d7c2f27f7deaf1a7308e3751b3fbc Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Wed, 2 Oct 2019 11:00:57 -0400 Subject: [PATCH 199/219] lm_seqs_dataset --- .../{dataset.py => lm_seqs_dataset.py} | 124 ++++++------------ 1 file changed, 37 insertions(+), 87 deletions(-) rename examples/distillation/{dataset.py => lm_seqs_dataset.py} (54%) diff --git a/examples/distillation/dataset.py b/examples/distillation/lm_seqs_dataset.py similarity index 54% rename from examples/distillation/dataset.py rename to examples/distillation/lm_seqs_dataset.py index 4babf73ea4..54e9742ce8 100644 --- a/examples/distillation/dataset.py +++ b/examples/distillation/lm_seqs_dataset.py @@ -12,30 +12,33 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" Dataloaders to train DistilBERT +""" Dataset to distilled models adapted in part from Facebook, Inc XLM model (https://github.com/facebookresearch/XLM) """ -from typing import List -import math -from itertools import chain -from collections import Counter -import numpy as np import torch +from torch.utils.data import Dataset +import numpy as np from utils import logger -class Dataset: +class LmSeqsDataset(Dataset): + """Custom Dataset wrapping language modeling sequences. + + Each sample will be retrieved by indexing the list of token_ids and their corresponding lengths. + + Input: + ------ + params: `NameSpace` parameters + data: `List[np.array[int]] + """ + def __init__(self, params, data): self.params = params - self.tokens_per_batch = params.tokens_per_batch - self.batch_size = params.batch_size - self.shuffle = params.shuffle - self.group_by_size = params.group_by_size self.token_ids = np.array(data) - self.lengths = np.uint16([len(t) for t in data]) + self.lengths = np.array([len(t) for t in data]) self.check() self.remove_long_sequences() @@ -43,6 +46,9 @@ class Dataset: self.check() self.print_statistics() + def __getitem__(self, index): + return (self.token_ids[index], self.lengths[index]) + def __len__(self): return len(self.lengths) @@ -51,12 +57,14 @@ class Dataset: Some sanity checks """ assert len(self.token_ids) == len(self.lengths) + assert all(self.lengths[i] == len(self.token_ids[i]) for i in range(len(self.lengths))) def remove_long_sequences(self): """ - Sequences that are too long are splitted by chunk of max_position_embeddings. + Sequences that are too long are splitted by chunk of max_model_input_size. """ - indices = self.lengths >= self.params.max_position_embeddings + max_len = self.params.max_model_input_size + indices = self.lengths > max_len logger.info(f'Splitting {sum(indices)} too long sequences.') def divide_chunks(l, n): @@ -64,10 +72,13 @@ class Dataset: new_tok_ids = [] new_lengths = [] - cls_id, sep_id = self.params.special_tok_ids['cls_token'], self.params.special_tok_ids['sep_token'] - max_len = self.params.max_position_embeddings + if self.params.mlm: + cls_id, sep_id = self.params.special_tok_ids['cls_token'], self.params.special_tok_ids['sep_token'] + else: + cls_id, sep_id = self.params.special_tok_ids['bos_token'], self.params.special_tok_ids['eos_token'] for seq_, len_ in zip(self.token_ids, self.lengths): + assert (seq_[0] == cls_id) and (seq_[-1] == sep_id), seq_ if len_ <= max_len: new_tok_ids.append(seq_) new_lengths.append(len_) @@ -79,6 +90,7 @@ class Dataset: if sub_s[-1] != sep_id: sub_s = np.insert(sub_s, len(sub_s), sep_id) assert len(sub_s) <= max_len + assert (sub_s[0] == cls_id) and (sub_s[-1] == sep_id), sub_s sub_seqs.append(sub_s) new_tok_ids.extend(sub_seqs) @@ -113,89 +125,27 @@ class Dataset: # nb_unkown = sum([(t==unk_idx).sum() for t in self.token_ids]) # logger.info(f'{nb_unkown} unknown tokens (covering {100*nb_unkown/data_len:.2f}% of the data)') - def select_data(self, a: int, b: int): - """ - Select a subportion of the data. - """ - n_sequences = len(self) - assert 0 <= a < b <= n_sequences, ValueError(f'`0 <= a < b <= n_sequences` is not met with a={a} and b={b}') - - logger.info(f'Selecting sequences from {a} to {b} (excluded).') - self.token_ids = self.token_ids[a:b] - self.lengths = self.lengths[a:b] - - self.check() - - def split(self): - """ - Distributed training: split the data accross the processes. - """ - assert self.params.n_gpu > 1 - logger.info('Splitting the data accross the processuses.') - n_seq = len(self) - n_seq_per_procesus = n_seq // self.params.world_size - a = n_seq_per_procesus * self.params.global_rank - b = a + n_seq_per_procesus - self.select_data(a=a, b=b) - def batch_sequences(self, - token_ids: List[List[int]], - lengths: List[int]): + batch): """ Do the padding and transform into torch.tensor. """ + token_ids = [t[0] for t in batch] + lengths = [t[1] for t in batch] assert len(token_ids) == len(lengths) # Max for paddings max_seq_len_ = max(lengths) # Pad token ids - pad_idx = self.params.special_tok_ids['pad_token'] + if self.params.mlm: + pad_idx = self.params.special_tok_ids['pad_token'] + else: + pad_idx = self.params.special_tok_ids['unk_token'] tk_ = [list(t.astype(int)) + [pad_idx]*(max_seq_len_-len(t)) for t in token_ids] assert len(tk_) == len(token_ids) assert all(len(t) == max_seq_len_ for t in tk_) - tk_t = torch.tensor(tk_) # (bs, max_seq_len_) - lg_t = torch.tensor(lengths.astype(int)) # (bs) + tk_t = torch.tensor(tk_) # (bs, max_seq_len_) + lg_t = torch.tensor(lengths) # (bs) return tk_t, lg_t - - def get_batches_iterator(self, - batches): - """ - Return an iterator over batches. - """ - for sequences_ids in batches: - token_ids, lengths = self.batch_sequences(self.token_ids[sequences_ids], - self.lengths[sequences_ids]) - yield (token_ids, lengths) - - def get_iterator(self, - seed: int = None): - """ - Return a data iterator. - """ - rng = np.random.RandomState(seed) - - n_sequences = len(self) - indices = np.arange(n_sequences) - - if self.group_by_size: - indices = indices[np.argsort(self.lengths[indices], kind='mergesort')] - - if self.tokens_per_batch == -1: - batches = np.array_split(indices, math.ceil(len(indices) * 1. / self.batch_size)) - else: - assert self.tokens_per_batch > 0 - batch_ids = np.cumsum(self.lengths[indices]) // self.tokens_per_batch - _, bounds = np.unique(batch_ids, return_index=True) - batches = [indices[bounds[i]:bounds[i + 1]] for i in range(len(bounds) - 1)] - if bounds[-1] < len(indices): - batches.append(indices[bounds[-1]:]) - - if self.shuffle: - rng.shuffle(batches) - - assert n_sequences == sum([len(x) for x in batches]) - assert self.lengths[indices].sum() == sum([self.lengths[x].sum() for x in batches]) - - return self.get_batches_iterator(batches=batches) From 19e4ebbe3fcded8a345fed05d9c3644b78312839 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Wed, 2 Oct 2019 11:01:07 -0400 Subject: [PATCH 200/219] grouped_batch_sampler --- .../distillation/grouped_batch_sampler.py | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 examples/distillation/grouped_batch_sampler.py diff --git a/examples/distillation/grouped_batch_sampler.py b/examples/distillation/grouped_batch_sampler.py new file mode 100644 index 0000000000..46d943a3d4 --- /dev/null +++ b/examples/distillation/grouped_batch_sampler.py @@ -0,0 +1,105 @@ +# coding=utf-8 +# Copyright 2019-present, the HuggingFace Inc. team and Facebook, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Adapted from PyTorch Vision (https://github.com/pytorch/vision/blob/master/references/detection/group_by_aspect_ratio.py) +""" +import bisect +import copy +from collections import defaultdict +import numpy as np + +from torch.utils.data.sampler import BatchSampler, Sampler + +from utils import logger + +def _quantize(x, bins): + bins = copy.deepcopy(bins) + bins = sorted(bins) + quantized = list(map(lambda y: bisect.bisect_right(bins, y), x)) + return quantized + +def create_lengths_groups(lengths, k=0): + bins = np.arange(start=3, stop=k, step=4).tolist() if k > 0 else [10] + groups = _quantize(lengths, bins) + # count number of elements per group + counts = np.unique(groups, return_counts=True)[1] + fbins = [0] + bins + [np.inf] + logger.info("Using {} as bins for aspect lengths quantization".format(fbins)) + logger.info("Count of instances per bin: {}".format(counts)) + return groups + +class GroupedBatchSampler(BatchSampler): + """ + Wraps another sampler to yield a mini-batch of indices. + It enforces that the batch only contain elements from the same group. + It also tries to provide mini-batches which follows an ordering which is + as close as possible to the ordering from the original sampler. + Arguments: + sampler (Sampler): Base sampler. + group_ids (list[int]): If the sampler produces indices in range [0, N), + `group_ids` must be a list of `N` ints which contains the group id of each sample. + The group ids must be a continuous set of integers starting from + 0, i.e. they must be in the range [0, num_groups). + batch_size (int): Size of mini-batch. + """ + def __init__(self, sampler, group_ids, batch_size): + if not isinstance(sampler, Sampler): + raise ValueError( + "sampler should be an instance of " + "torch.utils.data.Sampler, but got sampler={}".format(sampler) + ) + self.sampler = sampler + self.group_ids = group_ids + self.batch_size = batch_size + + def __iter__(self): + buffer_per_group = defaultdict(list) + samples_per_group = defaultdict(list) + + num_batches = 0 + for idx in self.sampler: + group_id = self.group_ids[idx] + buffer_per_group[group_id].append(idx) + samples_per_group[group_id].append(idx) + if len(buffer_per_group[group_id]) == self.batch_size: + yield buffer_per_group[group_id] #TODO + num_batches += 1 + del buffer_per_group[group_id] + assert len(buffer_per_group[group_id]) < self.batch_size + + # now we have run out of elements that satisfy + # the group criteria, let's return the remaining + # elements so that the size of the sampler is + # deterministic + expected_num_batches = len(self) + num_remaining = expected_num_batches - num_batches + if num_remaining > 0: + # for the remaining batches, group the batches by similar lengths + batch_idx = [] + for group_id, idxs in sorted(buffer_per_group.items(), key=lambda x: x[0]): + batch_idx.extend(idxs) + if len(batch_idx) >= self.batch_size: + yield batch_idx[:self.batch_size] + batch_idx = batch_idx[self.batch_size:] + num_remaining -= 1 + if len(batch_idx) > 0: + yield batch_idx + num_remaining -= 1 + assert num_remaining == 0 + + def __len__(self): + """ + Return the number of mini-batches rather than the number of samples. + """ + return (len(self.sampler) + self.batch_size - 1) // self.batch_size From cbfcfce205d754f2019b6a795d6c7939ddbf58ba Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Wed, 2 Oct 2019 11:01:20 -0400 Subject: [PATCH 201/219] update token_counts --- examples/distillation/scripts/token_counts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/distillation/scripts/token_counts.py b/examples/distillation/scripts/token_counts.py index a484a6f51b..d9de17da4e 100644 --- a/examples/distillation/scripts/token_counts.py +++ b/examples/distillation/scripts/token_counts.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """ -Preprocessing script before training DistilBERT. +Preprocessing script before training the distilled model. """ from collections import Counter import argparse From 23edebc0797008f0525fd1eef7f1299b513457ad Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Wed, 2 Oct 2019 11:01:33 -0400 Subject: [PATCH 202/219] update extract_distilbert --- ...ct_for_distil.py => extract_distilbert.py} | 28 +++++++------------ 1 file changed, 10 insertions(+), 18 deletions(-) rename examples/distillation/scripts/{extract_for_distil.py => extract_distilbert.py} (76%) diff --git a/examples/distillation/scripts/extract_for_distil.py b/examples/distillation/scripts/extract_distilbert.py similarity index 76% rename from examples/distillation/scripts/extract_for_distil.py rename to examples/distillation/scripts/extract_distilbert.py index 2e7e5c73d8..fdb0662ca7 100644 --- a/examples/distillation/scripts/extract_for_distil.py +++ b/examples/distillation/scripts/extract_distilbert.py @@ -14,6 +14,7 @@ # limitations under the License. """ Preprocessing script before training DistilBERT. +Specific to BERT -> DistilBERT. """ from transformers import BertForMaskedLM, RobertaForMaskedLM import torch @@ -21,7 +22,7 @@ import argparse if __name__ == '__main__': parser = argparse.ArgumentParser(description="Extraction some layers of the full BertForMaskedLM or RObertaForMaskedLM for Transfer Learned Distillation") - parser.add_argument("--model_type", default="bert", choices=["bert", "roberta"]) + parser.add_argument("--model_type", default="bert", choices=["bert"]) parser.add_argument("--model_name", default='bert-base-uncased', type=str) parser.add_argument("--dump_checkpoint", default='serialization_dir/tf_bert-base-uncased_0247911.pth', type=str) parser.add_argument("--vocab_transform", action='store_true') @@ -31,9 +32,8 @@ if __name__ == '__main__': if args.model_type == 'bert': model = BertForMaskedLM.from_pretrained(args.model_name) prefix = 'bert' - elif args.model_type == 'roberta': - model = RobertaForMaskedLM.from_pretrained(args.model_name) - prefix = 'roberta' + else: + raise ValueError(f'args.model_type should be "bert".') state_dict = model.state_dict() compressed_sd = {} @@ -68,20 +68,12 @@ if __name__ == '__main__': state_dict[f'{prefix}.encoder.layer.{teacher_idx}.output.LayerNorm.{w}'] std_idx += 1 - if args.model_type == 'bert': - compressed_sd[f'vocab_projector.weight'] = state_dict[f'cls.predictions.decoder.weight'] - compressed_sd[f'vocab_projector.bias'] = state_dict[f'cls.predictions.bias'] - if args.vocab_transform: - for w in ['weight', 'bias']: - compressed_sd[f'vocab_transform.{w}'] = state_dict[f'cls.predictions.transform.dense.{w}'] - compressed_sd[f'vocab_layer_norm.{w}'] = state_dict[f'cls.predictions.transform.LayerNorm.{w}'] - elif args.model_type == 'roberta': - compressed_sd[f'vocab_projector.weight'] = state_dict[f'lm_head.decoder.weight'] - compressed_sd[f'vocab_projector.bias'] = state_dict[f'lm_head.bias'] - if args.vocab_transform: - for w in ['weight', 'bias']: - compressed_sd[f'vocab_transform.{w}'] = state_dict[f'lm_head.dense.{w}'] - compressed_sd[f'vocab_layer_norm.{w}'] = state_dict[f'lm_head.layer_norm.{w}'] + compressed_sd[f'vocab_projector.weight'] = state_dict[f'cls.predictions.decoder.weight'] + compressed_sd[f'vocab_projector.bias'] = state_dict[f'cls.predictions.bias'] + if args.vocab_transform: + for w in ['weight', 'bias']: + compressed_sd[f'vocab_transform.{w}'] = state_dict[f'cls.predictions.transform.dense.{w}'] + compressed_sd[f'vocab_layer_norm.{w}'] = state_dict[f'cls.predictions.transform.LayerNorm.{w}'] print(f'N layers selected for distillation: {std_idx}') print(f'Number of params transfered for distillation: {len(compressed_sd.keys())}') From 4d6dfbd3762ac57e44d97b1c6d0243cfffd1880b Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Wed, 2 Oct 2019 11:01:41 -0400 Subject: [PATCH 203/219] update extract --- examples/distillation/scripts/extract.py | 89 ++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 examples/distillation/scripts/extract.py diff --git a/examples/distillation/scripts/extract.py b/examples/distillation/scripts/extract.py new file mode 100644 index 0000000000..5ae1607f3f --- /dev/null +++ b/examples/distillation/scripts/extract.py @@ -0,0 +1,89 @@ +# coding=utf-8 +# Copyright 2019-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Preprocessing script before training the distilled model. +Specific to RoBERTa -> DistilRoBERTa and GPT2 -> DistilGPT2. +""" +from transformers import BertForMaskedLM, RobertaForMaskedLM, GPT2LMHeadModel +import torch +import argparse + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description="Extraction some layers of the full RobertaForMaskedLM or GPT2LMHeadModel for Transfer Learned Distillation") + parser.add_argument("--model_type", default="roberta", choices=["roberta", "gpt2"]) + parser.add_argument("--model_name", default='roberta-large', type=str) + parser.add_argument("--dump_checkpoint", default='serialization_dir/tf_roberta_048131723.pth', type=str) + parser.add_argument("--vocab_transform", action='store_true') + args = parser.parse_args() + + + if args.model_type == 'roberta': + model = RobertaForMaskedLM.from_pretrained(args.model_name) + prefix = 'roberta' + elif args.model_type == 'gpt2': + model = GPT2LMHeadModel.from_pretrained(args.model_name) + prefix = 'transformer' + + state_dict = model.state_dict() + compressed_sd = {} + + ### Embeddings ### + if args.model_type == 'gpt2': + for param_name in ['wte.weight', 'wpe.weight']: + compressed_sd[f'{prefix}.{param_name}'] = state_dict[f'{prefix}.{param_name}'] + else: + for w in ['word_embeddings', 'position_embeddings', 'token_type_embeddings']: + param_name = f'{prefix}.embeddings.{w}.weight' + compressed_sd[param_name] = state_dict[param_name] + for w in ['weight', 'bias']: + param_name = f'{prefix}.embeddings.LayerNorm.{w}' + compressed_sd[param_name] = state_dict[param_name] + + ### Transformer Blocks ### + std_idx = 0 + for teacher_idx in [0, 2, 4, 7, 9, 11]: + if args.model_type == 'gpt2': + for layer in ['ln_1', 'attn.c_attn', 'attn.c_proj', 'ln_2', 'mlp.c_fc', 'mlp.c_proj']: + for w in ['weight', 'bias']: + compressed_sd[f'{prefix}.h.{std_idx}.{layer}.{w}'] = \ + state_dict[f'{prefix}.h.{teacher_idx}.{layer}.{w}'] + compressed_sd[f'{prefix}.h.{std_idx}.attn.bias'] = state_dict[f'{prefix}.h.{teacher_idx}.attn.bias'] + else: + for layer in ['attention.self.query', 'attention.self.key', 'attention.self.value', + 'attention.output.dense', 'attention.output.LayerNorm', + 'intermediate.dense', 'output.dense', 'output.LayerNorm']: + for w in ['weight', 'bias']: + compressed_sd[f'{prefix}.encoder.layer.{std_idx}.{layer}.{w}'] = \ + state_dict[f'{prefix}.encoder.layer.{teacher_idx}.{layer}.{w}'] + std_idx += 1 + + ### Language Modeling Head ###s + if args.model_type == 'roberta': + for layer in ['lm_head.decoder.weight', 'lm_head.bias']: + compressed_sd[f'{layer}'] = state_dict[f'{layer}'] + if args.vocab_transform: + for w in ['weight', 'bias']: + compressed_sd[f'lm_head.dense.{w}'] = state_dict[f'lm_head.dense.{w}'] + compressed_sd[f'lm_head.layer_norm.{w}'] = state_dict[f'lm_head.layer_norm.{w}'] + elif args.model_type == 'gpt2': + for w in ['weight', 'bias']: + compressed_sd[f'{prefix}.ln_f.{w}'] = state_dict[f'{prefix}.ln_f.{w}'] + compressed_sd[f'lm_head.weight'] = state_dict[f'lm_head.weight'] + + print(f'N layers selected for distillation: {std_idx}') + print(f'Number of params transfered for distillation: {len(compressed_sd.keys())}') + + print(f'Save transfered checkpoint to {args.dump_checkpoint}.') + torch.save(compressed_sd, args.dump_checkpoint) From a12ab0a8dba640730b5353d07ca8893c2d64688f Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Wed, 2 Oct 2019 11:01:55 -0400 Subject: [PATCH 204/219] update binarized_data --- examples/distillation/scripts/binarized_data.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/examples/distillation/scripts/binarized_data.py b/examples/distillation/scripts/binarized_data.py index eb4af08b0f..43824e9964 100644 --- a/examples/distillation/scripts/binarized_data.py +++ b/examples/distillation/scripts/binarized_data.py @@ -13,14 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. """ -Preprocessing script before training DistilBERT. +Preprocessing script before distillation. """ import argparse import pickle import random import time import numpy as np -from transformers import BertTokenizer, RobertaTokenizer +from transformers import BertTokenizer, RobertaTokenizer, GPT2Tokenizer import logging logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', @@ -32,7 +32,7 @@ def main(): parser = argparse.ArgumentParser(description="Preprocess the data to avoid re-doing it several times by (tokenization + token_to_ids).") parser.add_argument('--file_path', type=str, default='data/dump.txt', help='The path to the data.') - parser.add_argument('--tokenizer_type', type=str, default='bert', choices=['bert', 'roberta']) + parser.add_argument('--tokenizer_type', type=str, default='bert', choices=['bert', 'roberta', 'gpt2']) parser.add_argument('--tokenizer_name', type=str, default='bert-base-uncased', help="The tokenizer to use.") parser.add_argument('--dump_file', type=str, default='data/dump', @@ -43,10 +43,16 @@ def main(): logger.info(f'Loading Tokenizer ({args.tokenizer_name})') if args.tokenizer_type == 'bert': tokenizer = BertTokenizer.from_pretrained(args.tokenizer_name) + bos = tokenizer.special_tokens_map['cls_token'] # `[CLS]` + sep = tokenizer.special_tokens_map['sep_token'] # `[SEP]` elif args.tokenizer_type == 'roberta': tokenizer = RobertaTokenizer.from_pretrained(args.tokenizer_name) - bos = tokenizer.special_tokens_map['bos_token'] # `[CLS]` for bert, `` for roberta - sep = tokenizer.special_tokens_map['sep_token'] # `[SEP]` for bert, `` for roberta + bos = tokenizer.special_tokens_map['cls_token'] # `` + sep = tokenizer.special_tokens_map['sep_token'] # `` + elif args.tokenizer_type == 'gpt2': + tokenizer = GPT2Tokenizer.from_pretrained(args.tokenizer_name) + bos = tokenizer.special_tokens_map['bos_token'] # `<|endoftext|>` + sep = tokenizer.special_tokens_map['eos_token'] # `<|endoftext|>` logger.info(f'Loading text from {args.file_path}') with open(args.file_path, 'r', encoding='utf8') as fp: From bb9c5ead5444d7510e4734f540bf87fa9c5669fb Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Wed, 2 Oct 2019 11:02:30 -0400 Subject: [PATCH 205/219] update distiller --- examples/distillation/distiller.py | 187 ++++++++++++++++++----------- 1 file changed, 117 insertions(+), 70 deletions(-) diff --git a/examples/distillation/distiller.py b/examples/distillation/distiller.py index 79755b81e0..f736936449 100644 --- a/examples/distillation/distiller.py +++ b/examples/distillation/distiller.py @@ -12,8 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" The distiller to distil DistilBERT - adapted in part from Facebook, Inc XLM model (https://github.com/facebookresearch/XLM) +""" The distiller to distil the student. + Adapted in part from Facebook, Inc XLM model (https://github.com/facebookresearch/XLM) """ import os import math @@ -28,16 +28,19 @@ import torch import torch.nn as nn import torch.nn.functional as F from torch.optim import AdamW +from torch.utils.data.distributed import DistributedSampler +from torch.utils.data import RandomSampler, BatchSampler, DataLoader from transformers import WarmupLinearSchedule from utils import logger -from dataset import Dataset +from lm_seqs_dataset import LmSeqsDataset +from grouped_batch_sampler import GroupedBatchSampler, create_lengths_groups class Distiller: def __init__(self, params: dict, - dataloader: Dataset, + dataset: LmSeqsDataset, token_probs: torch.tensor, student: nn.Module, teacher: nn.Module): @@ -50,33 +53,47 @@ class Distiller: self.student = student self.teacher = teacher - self.dataloader = dataloader - if self.params.n_gpu > 1: - self.dataloader.split() - self.get_iterator(seed=params.seed) + self.student_config = student.config + self.vocab_size = student.config.vocab_size + + if params.n_gpu <= 1: + sampler = RandomSampler(dataset) + else: + sampler = DistributedSampler(dataset) + + if params.group_by_size: + groups = create_lengths_groups(lengths=dataset.lengths, k=params.max_model_input_size) + sampler = GroupedBatchSampler(sampler=sampler, group_ids=groups, batch_size=params.batch_size) + else: + sampler = BatchSampler(sampler=sampler, batch_size=params.batch_size, drop_last=False) + + self.dataloader = DataLoader(dataset=dataset, + batch_sampler=sampler, + collate_fn=dataset.batch_sequences) self.temperature = params.temperature assert self.temperature > 0. self.alpha_ce = params.alpha_ce self.alpha_mlm = params.alpha_mlm + self.alpha_clm = params.alpha_clm self.alpha_mse = params.alpha_mse self.alpha_cos = params.alpha_cos - assert self.alpha_ce >= 0. - assert self.alpha_mlm >= 0. - assert self.alpha_mse >= 0. - assert self.alpha_cos >= 0. - assert self.alpha_ce + self.alpha_mlm + self.alpha_mse + self.alpha_cos > 0. - self.mlm_mask_prop = params.mlm_mask_prop - assert 0.0 <= self.mlm_mask_prop <= 1.0 - assert params.word_mask + params.word_keep + params.word_rand == 1.0 - self.pred_probs = torch.FloatTensor([params.word_mask, params.word_keep, params.word_rand]) - self.pred_probs = self.pred_probs.to(f'cuda:{params.local_rank}') if params.n_gpu > 0 else self.pred_probs - self.token_probs = token_probs.to(f'cuda:{params.local_rank}') if params.n_gpu > 0 else token_probs - if self.fp16: - self.pred_probs = self.pred_probs.half() - self.token_probs = self.token_probs.half() + self.mlm = params.mlm + if self.mlm: + logger.info(f'Using MLM loss for LM step.') + self.mlm_mask_prop = params.mlm_mask_prop + assert 0.0 <= self.mlm_mask_prop <= 1.0 + assert params.word_mask + params.word_keep + params.word_rand == 1.0 + self.pred_probs = torch.FloatTensor([params.word_mask, params.word_keep, params.word_rand]) + self.pred_probs = self.pred_probs.to(f'cuda:{params.local_rank}') if params.n_gpu > 0 else self.pred_probs + self.token_probs = token_probs.to(f'cuda:{params.local_rank}') if params.n_gpu > 0 else token_probs + if self.fp16: + self.pred_probs = self.pred_probs.half() + self.token_probs = self.token_probs.half() + else: + logger.info(f'Using CLM loss for LM step.') self.epoch = 0 self.n_iter = 0 @@ -86,12 +103,13 @@ class Distiller: self.last_loss = 0 self.last_loss_ce = 0 self.last_loss_mlm = 0 + self.last_loss_clm = 0 if self.alpha_mse > 0.: self.last_loss_mse = 0 if self.alpha_cos > 0.: self.last_loss_cos = 0 self.last_log = 0 self.ce_loss_fct = nn.KLDivLoss(reduction='batchmean') - self.mlm_loss_fct = nn.CrossEntropyLoss(ignore_index=-1) + self.lm_loss_fct = nn.CrossEntropyLoss(ignore_index=-1) if self.alpha_mse > 0.: self.mse_loss_fct = nn.MSELoss(reduction='sum') if self.alpha_cos > 0.: @@ -99,7 +117,7 @@ class Distiller: logger.info('--- Initializing model optimizer') assert params.gradient_accumulation_steps >= 1 - self.num_steps_epoch = int(len(self.dataloader) / params.batch_size) + 1 + self.num_steps_epoch = len(self.dataloader) num_train_optimization_steps = int(self.num_steps_epoch / params.gradient_accumulation_steps * params.n_epoch) + 1 no_decay = ['bias', 'LayerNorm.weight'] @@ -140,43 +158,18 @@ class Distiller: logger.info("Using nn.parallel.DistributedDataParallel for distributed training.") self.student = DistributedDataParallel(self.student, device_ids=[params.local_rank], - output_device=params.local_rank) + output_device=params.local_rank, + find_unused_parameters=True) self.is_master = params.is_master if self.is_master: logger.info('--- Initializing Tensorboard') self.tensorboard = SummaryWriter(log_dir=os.path.join(self.dump_path, 'log', 'train')) - self.tensorboard.add_text(tag='config', text_string=str(self.params), global_step=0) + self.tensorboard.add_text(tag='config/training', text_string=str(self.params), global_step=0) + self.tensorboard.add_text(tag='config/student', text_string=str(self.student_config), global_step=0) - def get_iterator(self, - seed: int = None): - """ - Initialize the data iterator. - Each process has its own data iterator (iterating on his own random portion of the dataset). - - Input: - ------ - seed: `int` - The random seed. - """ - logger.info('--- Initializing Data Iterator') - self.data_iterator = self.dataloader.get_iterator(seed=seed) - - def get_batch(self): - """ - Call the data iterator to output a new batch. - If the data iterator went through the whole dataset, create a new iterator. - """ - assert hasattr(self, 'data_iterator') - try: - x = next(self.data_iterator) - except StopIteration: - logger.warning('--- Went through the whole dataset. Creating new data iterator.') - self.data_iterator = self.dataloader.get_iterator() - x = next(self.data_iterator) - return x - - def prepare_batch(self, - batch): + def prepare_batch_mlm(self, + batch): """ Prepare the batch: from the token_ids and the lenghts, compute the attention mask and the masked label for MLM. @@ -222,7 +215,7 @@ class Distiller: assert pred_mask.sum().item() % 8 == 0, pred_mask.sum().item() _token_ids_real = token_ids[pred_mask] - _token_ids_rand = _token_ids_real.clone().random_(self.params.vocab_size) + _token_ids_rand = _token_ids_real.clone().random_(self.vocab_size) _token_ids_mask = _token_ids_real.clone().fill_(self.params.special_tok_ids['mask_token']) probs = torch.multinomial(self.pred_probs, len(_token_ids_real), replacement=True) _token_ids = _token_ids_mask * (probs == 0).long() + _token_ids_real * (probs == 1).long() + _token_ids_rand * (probs == 2).long() @@ -230,8 +223,41 @@ class Distiller: mlm_labels[~pred_mask] = -1 # previously `mlm_labels[1-pred_mask] = -1`, cf pytorch 1.2.0 compatibility + # sanity checks + assert 0 <= token_ids.min() <= token_ids.max() < self.vocab_size + return token_ids, attn_mask, mlm_labels + def prepare_batch_clm(self, + batch): + """ + Prepare the batch: from the token_ids and the lenghts, compute the attention mask and the labels for CLM. + + Input: + ------ + batch: `Tuple` + token_ids: `torch.tensor(bs, seq_length)` - The token ids for each of the sequence. It is padded. + lengths: `torch.tensor(bs)` - The lengths of each of the sequences in the batch. + + Output: + ------- + token_ids: `torch.tensor(bs, seq_length)` - The token ids after the modifications for MLM. + attn_mask: `torch.tensor(bs, seq_length)` - The attention mask for the self-attention. + clm_labels: `torch.tensor(bs, seq_length)` - The causal languge modeling labels. There is a -1 where there is nothing to predict. + """ + token_ids, lengths = batch + token_ids, lengths = self.round_batch(x=token_ids, lengths=lengths) + assert token_ids.size(0) == lengths.size(0) + + attn_mask = (torch.arange(token_ids.size(1), dtype=torch.long, device=lengths.device) < lengths[:, None]) + clm_labels = token_ids.new(token_ids.size()).copy_(token_ids) + clm_labels[~attn_mask] = -1 # previously `clm_labels[1-attn_mask] = -1`, cf pytorch 1.2.0 compatibility + + # sanity checks + assert 0 <= token_ids.min() <= token_ids.max() < self.vocab_size + + return token_ids, attn_mask, clm_labels + def round_batch(self, x: torch.tensor, lengths: torch.tensor): @@ -269,7 +295,10 @@ class Distiller: if ml1 % 8 != 0: pad = 8 - (ml1 % 8) ml2 = ml1 + pad - pad_id = self.params.special_tok_ids['pad_token'] + if self.mlm: + pad_id = self.params.special_tok_ids['pad_token'] + else: + pad_id = self.params.special_tok_ids['unk_token'] padding_tensor = torch.zeros(bs2, pad, dtype=torch.long, device=x.device).fill_(pad_id) x = torch.cat([x, padding_tensor], 1) assert x.size() == (bs2, ml2) @@ -292,14 +321,16 @@ class Distiller: if self.multi_gpu: torch.distributed.barrier() - iter_bar = trange(self.num_steps_epoch, desc="-Iter", disable=self.params.local_rank not in [-1, 0]) - for __ in range(self.num_steps_epoch): - batch = self.get_batch() + iter_bar = tqdm(self.dataloader, desc="-Iter", disable=self.params.local_rank not in [-1, 0]) + for batch in iter_bar: if self.params.n_gpu > 0: batch = tuple(t.to(f'cuda:{self.params.local_rank}') for t in batch) - token_ids, attn_mask, mlm_labels = self.prepare_batch(batch=batch) - self.step(input_ids=token_ids, attention_mask=attn_mask, mlm_labels=mlm_labels) + if self.mlm: + token_ids, attn_mask, lm_labels = self.prepare_batch_mlm(batch=batch) + else: + token_ids, attn_mask, lm_labels = self.prepare_batch_clm(batch=batch) + self.step(input_ids=token_ids, attention_mask=attn_mask, lm_labels=lm_labels) iter_bar.update() iter_bar.set_postfix({'Last_loss': f'{self.last_loss:.2f}', @@ -317,7 +348,7 @@ class Distiller: def step(self, input_ids: torch.tensor, attention_mask: torch.tensor, - mlm_labels: torch.tensor): + lm_labels: torch.tensor): """ One optimization step: forward of student AND teacher, backward on the loss (for gradient accumulation), and possibly a parameter update (depending on the gradient accumulation). @@ -326,17 +357,22 @@ class Distiller: ------ input_ids: `torch.tensor(bs, seq_length)` - The token ids. attention_mask: `torch.tensor(bs, seq_length)` - The attention mask for self attention. - mlm_labels: `torch.tensor(bs, seq_length)` - The masked language modeling labels. + lm_labels: `torch.tensor(bs, seq_length)` - The language modeling labels (mlm labels for MLM and clm labels for CLM). """ - s_logits, s_hidden_states = self.student(input_ids=input_ids, attention_mask=attention_mask) # (bs, seq_length, voc_size) - with torch.no_grad(): - t_logits, t_hidden_states = self.teacher(input_ids=input_ids, attention_mask=attention_mask) # (bs, seq_length, voc_size) + if self.mlm: + s_logits, s_hidden_states = self.student(input_ids=input_ids, attention_mask=attention_mask) # (bs, seq_length, voc_size) + with torch.no_grad(): + t_logits, t_hidden_states = self.teacher(input_ids=input_ids, attention_mask=attention_mask) # (bs, seq_length, voc_size) + else: + s_logits, _, s_hidden_states = self.student(input_ids=input_ids, attention_mask=None) # (bs, seq_length, voc_size) + with torch.no_grad(): + t_logits, _, t_hidden_states = self.teacher(input_ids=input_ids, attention_mask=None) # (bs, seq_length, voc_size) assert s_logits.size() == t_logits.size() #https://github.com/peterliht/knowledge-distillation-pytorch/blob/master/model/net.py#L100 #https://github.com/peterliht/knowledge-distillation-pytorch/issues/2 if self.params.restrict_ce_to_mask: - mask = (mlm_labels>-1).unsqueeze(-1).expand_as(s_logits) # (bs, seq_lenth, voc_size) + mask = (lm_labels>-1).unsqueeze(-1).expand_as(s_logits) # (bs, seq_lenth, voc_size) else: mask = attention_mask.unsqueeze(-1).expand_as(s_logits) # (bs, seq_lenth, voc_size) s_logits_slct = torch.masked_select(s_logits, mask) # (bs * seq_length * voc_size) modulo the 1s in mask @@ -348,13 +384,20 @@ class Distiller: loss_ce = self.ce_loss_fct(F.log_softmax(s_logits_slct/self.temperature, dim=-1), F.softmax(t_logits_slct/self.temperature, dim=-1)) * (self.temperature)**2 loss = self.alpha_ce*loss_ce + if self.alpha_mlm > 0.: - loss_mlm = self.mlm_loss_fct(s_logits.view(-1, s_logits.size(-1)), mlm_labels.view(-1)) + loss_mlm = self.lm_loss_fct(s_logits.view(-1, s_logits.size(-1)), lm_labels.view(-1)) loss += self.alpha_mlm * loss_mlm + if self.alpha_clm > 0.: + shift_logits = s_logits[..., :-1, :].contiguous() + shift_labels = lm_labels[..., 1:].contiguous() + loss_clm = self.lm_loss_fct(shift_logits.view(-1, shift_logits.size(-1)), + shift_labels.view(-1)) + loss += self.alpha_clm * loss_clm + if self.alpha_mse > 0.: loss_mse = self.mse_loss_fct(s_logits_slct, t_logits_slct)/s_logits_slct.size(0) # Reproducing batchmean reduction loss += self.alpha_mse * loss_mse - if self.alpha_cos > 0.: s_hidden_states = s_hidden_states[-1] # (bs, seq_length, dim) t_hidden_states = t_hidden_states[-1] # (bs, seq_length, dim) @@ -376,6 +419,8 @@ class Distiller: self.last_loss_ce = loss_ce.item() if self.alpha_mlm > 0.: self.last_loss_mlm = loss_mlm.item() + if self.alpha_clm > 0.: + self.last_loss_clm = loss_clm.item() if self.alpha_mse > 0.: self.last_loss_mse = loss_mse.item() if self.alpha_cos > 0.: @@ -452,6 +497,8 @@ class Distiller: self.tensorboard.add_scalar(tag="losses/loss_ce", scalar_value=self.last_loss_ce, global_step=self.n_total_iter) if self.alpha_mlm > 0.: self.tensorboard.add_scalar(tag="losses/loss_mlm", scalar_value=self.last_loss_mlm, global_step=self.n_total_iter) + if self.alpha_clm > 0.: + self.tensorboard.add_scalar(tag="losses/loss_clm", scalar_value=self.last_loss_clm, global_step=self.n_total_iter) if self.alpha_mse > 0.: self.tensorboard.add_scalar(tag="losses/loss_mse", scalar_value=self.last_loss_mse, global_step=self.n_total_iter) if self.alpha_cos > 0.: From a76c3f9cb0fafc87dbed6f5a6b2b0bf3e3a00c03 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Wed, 2 Oct 2019 11:02:43 -0400 Subject: [PATCH 206/219] update requirements --- examples/distillation/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/distillation/requirements.txt b/examples/distillation/requirements.txt index 2cf6ee2d81..d76273b34a 100644 --- a/examples/distillation/requirements.txt +++ b/examples/distillation/requirements.txt @@ -3,4 +3,4 @@ tensorboard>=1.14.0 tensorboardX==1.8 psutil==5.6.3 scipy==1.3.1 -pytorch_transformers==1.2.0 +transformers==2.0.0 From c51e533a5febe3fae2bb33b060f6b1f36a92e003 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Wed, 2 Oct 2019 11:02:53 -0400 Subject: [PATCH 207/219] update train.py --- examples/distillation/train.py | 193 ++++++++++++++++++++------------- 1 file changed, 118 insertions(+), 75 deletions(-) diff --git a/examples/distillation/train.py b/examples/distillation/train.py index f0255d08fe..311f0580ff 100644 --- a/examples/distillation/train.py +++ b/examples/distillation/train.py @@ -13,7 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. """ -Training DistilBERT. +Training the distilled model. +Supported architectures include: BERT -> DistilBERT, RoBERTa -> DistilRoBERTa, GPT2 -> DistilGPT2. """ import os import argparse @@ -23,68 +24,96 @@ import shutil import numpy as np import torch -from transformers import BertTokenizer, BertForMaskedLM, RobertaTokenizer, RobertaForMaskedLM -from transformers import DistilBertForMaskedLM, DistilBertConfig +from transformers import BertConfig, BertForMaskedLM, BertTokenizer +from transformers import RobertaConfig, RobertaForMaskedLM, RobertaTokenizer +from transformers import DistilBertConfig, DistilBertForMaskedLM, DistilBertTokenizer +from transformers import GPT2Config, GPT2LMHeadModel, GPT2Tokenizer from distiller import Distiller from utils import git_log, logger, init_gpu_params, set_seed -from dataset import Dataset +from lm_seqs_dataset import LmSeqsDataset +MODEL_CLASSES = { + 'distilbert': (DistilBertConfig, DistilBertForMaskedLM, DistilBertTokenizer), + 'roberta': (RobertaConfig, RobertaForMaskedLM, RobertaTokenizer), + 'bert': (BertConfig, BertForMaskedLM, BertTokenizer), + 'gpt2': (GPT2Config, GPT2LMHeadModel, GPT2Tokenizer) +} + +def sanity_checks(args): + """ + A bunch of args sanity checks to perform even starting... + """ + assert (args.mlm and args.alpha_mlm > 0.) or (not args.mlm and args.alpha_mlm == 0.) + assert (args.alpha_mlm > 0. and args.alpha_clm == 0.) or (args.alpha_mlm == 0. and args.alpha_clm > 0.) + if args.mlm: + assert os.path.isfile(args.token_counts) + assert (args.student_type in ['roberta', 'distilbert']) and (args.teacher_type in ['roberta', 'bert']) + else: + assert (args.student_type in ['gpt2']) and (args.teacher_type in ['gpt2']) + + assert args.teacher_type == args.student_type or (args.student_type=='distilbert' and args.teacher_type=='bert') + assert os.path.isfile(args.student_config) + if args.student_pretrained_weights is not None: + assert os.path.isfile(args.student_pretrained_weights) + + if args.freeze_token_type_embds: assert args.student_type in ['roberta'] + + assert args.alpha_ce >= 0. + assert args.alpha_mlm >= 0. + assert args.alpha_clm >= 0. + assert args.alpha_mse >= 0. + assert args.alpha_cos >= 0. + assert args.alpha_ce + args.alpha_mlm + args.alpha_clm + args.alpha_mse + args.alpha_cos > 0. + +def freeze_pos_embeddings(student, args): + if args.student_type == 'roberta': + student.roberta.embeddings.position_embeddings.weight.requires_grad = False + elif args.student_type == 'gpt2': + student.transformer.wpe.weight.requires_grad = False + +def freeze_token_type_embeddings(student, args): + if args.student_type == 'roberta': + student.roberta.embeddings.token_type_embeddings.weight.requires_grad = False + def main(): parser = argparse.ArgumentParser(description="Training") + parser.add_argument("--force", action='store_true', + help="Overwrite dump_path if it already exists.") parser.add_argument("--dump_path", type=str, required=True, help="The output directory (log, checkpoints, parameters, etc.)") parser.add_argument("--data_file", type=str, required=True, help="The binarized file (tokenized + tokens_to_ids) and grouped by sequence.") - parser.add_argument("--token_counts", type=str, required=True, - help="The token counts in the data_file for MLM.") - parser.add_argument("--force", action='store_true', - help="Overwrite dump_path if it already exists.") - parser.add_argument("--vocab_size", default=30522, type=int, - help="The vocabulary size.") - parser.add_argument("--max_position_embeddings", default=512, type=int, - help="Maximum sequence length we can model (including [CLS] and [SEP]).") - parser.add_argument("--sinusoidal_pos_embds", action='store_false', - help="If true, the position embeddings are simply fixed with sinusoidal embeddings.") - parser.add_argument("--n_layers", default=6, type=int, - help="Number of Transformer blocks.") - parser.add_argument("--n_heads", default=12, type=int, - help="Number of heads in the self-attention module.") - parser.add_argument("--dim", default=768, type=int, - help="Dimension through the network. Must be divisible by n_heads") - parser.add_argument("--hidden_dim", default=3072, type=int, - help="Intermediate dimension in the FFN.") - parser.add_argument("--dropout", default=0.1, type=float, - help="Dropout.") - parser.add_argument("--attention_dropout", default=0.1, type=float, - help="Dropout in self-attention.") - parser.add_argument("--activation", default='gelu', type=str, - help="Activation to use in self-attention") - parser.add_argument("--tie_weights_", action='store_false', - help="If true, we tie the embeddings matrix with the projection over the vocabulary matrix. Default is true.") - - parser.add_argument("--from_pretrained_weights", default=None, type=str, + parser.add_argument("--student_type", type=str, choices=["distilbert", "roberta", "gpt2"], required=True, + help="The student type (DistilBERT, RoBERTa).") + parser.add_argument("--student_config", type=str, required=True, + help="Path to the student configuration.") + parser.add_argument("--student_pretrained_weights", default=None, type=str, help="Load student initialization checkpoint.") - parser.add_argument("--from_pretrained_config", default=None, type=str, - help="Load student initialization architecture config.") - parser.add_argument("--teacher_type", default="bert", choices=["bert", "roberta"], + + parser.add_argument("--teacher_type", choices=["bert", "roberta", "gpt2"], required=True, help="Teacher type (BERT, RoBERTa).") - parser.add_argument("--teacher_name", default="bert-base-uncased", type=str, + parser.add_argument("--teacher_name", type=str, required=True, help="The teacher model.") parser.add_argument("--temperature", default=2., type=float, help="Temperature for the softmax temperature.") parser.add_argument("--alpha_ce", default=0.5, type=float, help="Linear weight for the distillation loss. Must be >=0.") - parser.add_argument("--alpha_mlm", default=0.5, type=float, - help="Linear weight for the MLM loss. Must be >=0.") + parser.add_argument("--alpha_mlm", default=0.0, type=float, + help="Linear weight for the MLM loss. Must be >=0. Should be used in coonjunction with `mlm` flag.") + parser.add_argument("--alpha_clm", default=0.5, type=float, + help="Linear weight for the CLM loss. Must be >=0.") parser.add_argument("--alpha_mse", default=0.0, type=float, help="Linear weight of the MSE loss. Must be >=0.") parser.add_argument("--alpha_cos", default=0.0, type=float, help="Linear weight of the cosine embedding loss. Must be >=0.") + + parser.add_argument("--mlm", action="store_true", + help="The LM step: MLM or CLM. If `mlm` is True, the MLM is used over CLM.") parser.add_argument("--mlm_mask_prop", default=0.15, type=float, help="Proportion of tokens for which we need to make a prediction.") parser.add_argument("--word_mask", default=0.8, type=float, @@ -95,17 +124,20 @@ def main(): help="Proportion of tokens to randomly replace.") parser.add_argument("--mlm_smoothing", default=0.7, type=float, help="Smoothing parameter to emphasize more rare tokens (see XLM, similar to word2vec).") + parser.add_argument("--token_counts", type=str, + help="The token counts in the data_file for MLM.") + parser.add_argument("--restrict_ce_to_mask", action='store_true', help="If true, compute the distilation loss only the [MLM] prediction distribution.") + parser.add_argument("--freeze_pos_embs", action="store_true", + help="Freeze positional embeddings during distillation. For student_type in ['roberta', 'gpt2'] only.") + parser.add_argument("--freeze_token_type_embds", action="store_true", + help="Freeze token type embeddings during distillation if existent. For student_type in ['roberta'] only.") parser.add_argument("--n_epoch", type=int, default=3, help="Number of pass on the whole dataset.") parser.add_argument("--batch_size", type=int, default=5, help="Batch size (for each process).") - parser.add_argument("--tokens_per_batch", type=int, default=-1, - help="If specified, modify the batches so that they have approximately this number of tokens.") - parser.add_argument("--shuffle", action='store_false', - help="If true, shuffle the sequence order. Default is true.") parser.add_argument("--group_by_size", action='store_false', help="If true, group sequences that have similar length into the same batch. Default is true.") @@ -141,6 +173,7 @@ def main(): parser.add_argument("--checkpoint_interval", type=int, default=4000, help="Checkpoint interval.") args = parser.parse_args() + sanity_checks(args) ## ARGS ## @@ -164,21 +197,19 @@ def main(): with open(os.path.join(args.dump_path, 'parameters.json'), 'w') as f: json.dump(vars(args), f, indent=4) git_log(args.dump_path) - assert (args.from_pretrained_weights is None and args.from_pretrained_config is None) or \ - (args.from_pretrained_weights is not None and args.from_pretrained_config is not None) + student_config_class, student_model_class, _ = MODEL_CLASSES[args.student_type] + teacher_config_class, teacher_model_class, teacher_tokenizer_class = MODEL_CLASSES[args.teacher_type] ### TOKENIZER ### - if args.teacher_type == 'bert': - tokenizer = BertTokenizer.from_pretrained(args.teacher_name) - elif args.teacher_type == 'roberta': - tokenizer = RobertaTokenizer.from_pretrained(args.teacher_name) + tokenizer = teacher_tokenizer_class.from_pretrained(args.teacher_name) special_tok_ids = {} for tok_name, tok_symbol in tokenizer.special_tokens_map.items(): idx = tokenizer.all_special_tokens.index(tok_symbol) special_tok_ids[tok_name] = tokenizer.all_special_ids[idx] logger.info(f'Special tokens {special_tok_ids}') args.special_tok_ids = special_tok_ids + args.max_model_input_size = tokenizer.max_model_input_sizes[args.teacher_name] ## DATA LOADER ## @@ -187,35 +218,34 @@ def main(): data = pickle.load(fp) - assert os.path.isfile(args.token_counts) - logger.info(f'Loading token counts from {args.token_counts} (already pre-computed)') - with open(args.token_counts, 'rb') as fp: - counts = pickle.load(fp) - assert len(counts) == args.vocab_size - token_probs = np.maximum(counts, 1) ** -args.mlm_smoothing - for idx in special_tok_ids.values(): - token_probs[idx] = 0. # do not predict special tokens - token_probs = torch.from_numpy(token_probs) + if args.mlm: + logger.info(f'Loading token counts from {args.token_counts} (already pre-computed)') + with open(args.token_counts, 'rb') as fp: + counts = pickle.load(fp) + + token_probs = np.maximum(counts, 1) ** -args.mlm_smoothing + for idx in special_tok_ids.values(): + token_probs[idx] = 0. # do not predict special tokens + token_probs = torch.from_numpy(token_probs) + else: + token_probs = None - train_dataloader = Dataset(params=args, data=data) + train_lm_seq_dataset = LmSeqsDataset(params=args, data=data) logger.info(f'Data loader created.') ## STUDENT ## - if args.from_pretrained_weights is not None: - assert os.path.isfile(args.from_pretrained_weights) - assert os.path.isfile(args.from_pretrained_config) - logger.info(f'Loading pretrained weights from {args.from_pretrained_weights}') - logger.info(f'Loading pretrained config from {args.from_pretrained_config}') - stu_architecture_config = DistilBertConfig.from_json_file(args.from_pretrained_config) - stu_architecture_config.output_hidden_states = True - student = DistilBertForMaskedLM.from_pretrained(args.from_pretrained_weights, - config=stu_architecture_config) + logger.info(f'Loading student config from {args.student_config}') + stu_architecture_config = student_config_class.from_pretrained(args.student_config) + stu_architecture_config.output_hidden_states = True + + if args.student_pretrained_weights is not None: + logger.info(f'Loading pretrained weights from {args.student_pretrained_weights}') + student = student_model_class.from_pretrained(args.student_pretrained_weights, + config=stu_architecture_config) else: - args.vocab_size_or_config_json_file = args.vocab_size - stu_architecture_config = DistilBertConfig(**vars(args), output_hidden_states=True) - student = DistilBertForMaskedLM(stu_architecture_config) + student = student_model_class(stu_architecture_config) if args.n_gpu > 0: @@ -224,18 +254,31 @@ def main(): ## TEACHER ## - if args.teacher_type == 'bert': - teacher = BertForMaskedLM.from_pretrained(args.teacher_name, output_hidden_states=True) - elif args.teacher_type == 'roberta': - teacher = RobertaForMaskedLM.from_pretrained(args.teacher_name, output_hidden_states=True) + teacher = teacher_model_class.from_pretrained(args.teacher_name, output_hidden_states=True) if args.n_gpu > 0: teacher.to(f'cuda:{args.local_rank}') logger.info(f'Teacher loaded from {args.teacher_name}.') + + ## FREEZING ## + if args.freeze_pos_embs: + freeze_pos_embeddings(student, args) + if args.freeze_token_type_embds: + freeze_token_type_embeddings(student, args) + + + ## SANITY CHECKS ## + assert student.config.vocab_size == teacher.config.vocab_size + assert student.config.hidden_size == teacher.config.hidden_size + assert student.config.max_position_embeddings == teacher.config.max_position_embeddings + if args.mlm: + assert token_probs.size(0) == stu_architecture_config.vocab_size + + ## DISTILLER ## torch.cuda.empty_cache() distiller = Distiller(params=args, - dataloader=train_dataloader, + dataset=train_lm_seq_dataset, token_probs=token_probs, student=student, teacher=teacher) From 2a91f6071ff7ada3fe9fc35fcdfe456c323b7788 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Wed, 2 Oct 2019 15:02:42 -0400 Subject: [PATCH 208/219] upddate README - TODO updadte link to paper --- examples/distillation/README.md | 57 ++++++++++++++++++++++----------- 1 file changed, 39 insertions(+), 18 deletions(-) diff --git a/examples/distillation/README.md b/examples/distillation/README.md index 4cddbd3a2e..ad439cf5f8 100644 --- a/examples/distillation/README.md +++ b/examples/distillation/README.md @@ -1,22 +1,25 @@ -# DistilBERT +# Distil* -This folder contains the original code used to train DistilBERT as well as examples showcasing how to use DistilBERT. +This folder contains the original code used to train Distil* as well as examples showcasing how to use DistilBERT and DistilGPT2. + +**2019, October 3rd - Update** We release our [NeurIPS workshop paper](TODO LINK) explaining our approach on DistilBERT. It includes updated results and further experiments. We applied the same method to GPT2 and release the weights of DistilGPT2. DistilGPT2 is two times faster and 33% smaller than GPT2. **2019, September 19th - Update:** We fixed bugs in the code and released an upadted version of the weights trained with a modification of the distillation loss. DistilBERT now reaches 97% of `BERT-base`'s performance on GLUE, and 86.9 F1 score on SQuAD v1.1 dev set (compared to 88.5 for `BERT-base`). We will publish a formal write-up of our approach in the near future! -## What is DistilBERT +## What is Distil* -DistilBERT stands for Distillated-BERT. DistilBERT is a small, fast, cheap and light Transformer model based on Bert architecture. It has 40% less parameters than `bert-base-uncased`, runs 60% faster while preserving 97% of BERT's performances as measured on the GLUE language understanding benchmark. DistilBERT is trained using knowledge distillation, a technique to compress a large model called the teacher into a smaller model called the student. By distillating Bert, we obtain a smaller Transformer model that bears a lot of similarities with the original BERT model while being lighter, smaller and faster to run. DistilBERT is thus an interesting option to put large-scaled trained Transformer model into production. +Distil* is a class of compressed models that started with DistilBERT. DistilBERT stands for Distillated-BERT. DistilBERT is a small, fast, cheap and light Transformer model based on Bert architecture. It has 40% less parameters than `bert-base-uncased`, runs 60% faster while preserving 97% of BERT's performances as measured on the GLUE language understanding benchmark. DistilBERT is trained using knowledge distillation, a technique to compress a large model called the teacher into a smaller model called the student. By distillating Bert, we obtain a smaller Transformer model that bears a lot of similarities with the original BERT model while being lighter, smaller and faster to run. DistilBERT is thus an interesting option to put large-scaled trained Transformer model into production. -For more information on DistilBERT, please refer to our [detailed blog post](https://medium.com/huggingface/smaller-faster-cheaper-lighter-introducing-distilbert-a-distilled-version-of-bert-8cf3380435b5 -). *Please note that we will publish a formal write-up with updated and more complete results in the near future (September 19th).* +We have applied the same method to GPT2 and release the weights of the compressed model. On the [WikiText-103](https://blog.einstein.ai/the-wikitext-long-term-dependency-language-modeling-dataset/) benchmark, GPT2 reaches a perplexity on the test of 15.84 compared to 19.91 for DistilGPT2 (after fine-tuning on the train set). -Here's the updated results on the dev sets of GLUE: +For more information on DistilBERT, please refer to our [NeurIPS workshop paper](TODO LINK). The paper superseeds our [previous blogpost](https://medium.com/huggingface/distilbert-8cf3380435b5) with a different distillation loss and better performances. -| Model | Macro-score | CoLA | MNLI | MRPC | QNLI | QQP | RTE | SST-2 | STS-B | WNLI | +Here are the results on the dev sets of GLUE: + +| Model | Macro-score | CoLA | MNLI | MRPC | QNLI | QQP | RTE | SST-2| STS-B| WNLI | | :---: | :---: | :---:| :---:| :---:| :---:| :---:| :---:| :---:| :---:| :---:| | BERT-base | **77.6** | 48.9 | 84.3 | 88.6 | 89.3 | 89.5 | 71.3 | 91.7 | 91.2 | 43.7 | -| DistilBERT | **75.2** | 49.1 | 81.8 | 90.2 | 87.0 | 89.2 | 62.9 | 92.7 | 90.7 | 44.4 | +| DistilBERT | **76.8** | 49.1 | 81.8 | 90.2 | 90.2 | 89.2 | 62.9 | 92.7 | 90.7 | 44.4 | ## Setup @@ -26,10 +29,12 @@ This part of the library has only be tested with Python3.6+. There are few speci ## How to use DistilBERT -Transformers includes two pre-trained DistilBERT models, currently only provided for English (we are investigating the possibility to train and release a multilingual version of DistilBERT): +Transformers includes two pre-trained Distil* models, currently only provided for English (we are investigating the possibility to train and release a multilingual version of DistilBERT): - `distilbert-base-uncased`: DistilBERT English language model pretrained on the same data used to pretrain Bert (concatenation of the Toronto Book Corpus and full English Wikipedia) using distillation with the supervision of the `bert-base-uncased` version of Bert. The model has 6 layers, 768 dimension and 12 heads, totalizing 66M parameters. - `distilbert-base-uncased-distilled-squad`: A finetuned version of `distilbert-base-uncased` finetuned using (a second step of) knwoledge distillation on SQuAD 1.0. This model reaches a F1 score of 86.9 on the dev set (for comparison, Bert `bert-base-uncased` version reaches a 88.5 F1 score). +- `distilgpt2`: DistilGPT2 English language model pretrained with the supervision of `gpt2` (the smallest version of GPT2) on [OpenWebTextCorpus](https://skylion007.github.io/OpenWebTextCorpus/), a reproduction of OpenAI's WebText dataset and . The model has 6 layers, 768 dimension and 12 heads, totalizing 82M (compared to 124M parameters for GPT2). On average, DistilGPT2 is two times faster than GPT2. +- and more to come! 🤗🤗🤗 Using DistilBERT is very similar to using BERT. DistilBERT share the same tokenizer as BERT's `bert-base-uncased` even though we provide a link to this tokenizer under the `DistilBertTokenizer` name to have a consistent naming between the library models. @@ -42,9 +47,11 @@ outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple ``` -## How to train DistilBERT +Similarly, using DistilGPT2 simply consists in calling the GPT2 classes from a different pretrained checkpoint: `model = GPT2Model.from_pretrained('distilgpt2')`. -In the following, we will explain how you can train your own compressed model. +## How to train Distil* + +In the following, we will explain how you can train DistilBERT. ### A. Preparing the data @@ -57,7 +64,8 @@ First, we will binarize the data, i.e. tokenize the data and convert each token ```bash python scripts/binarized_data.py \ --file_path data/dump.txt \ - --bert_tokenizer bert-base-uncased \ + --tokenizer_type bert \ + --tokenizer_name bert-base-uncased \ --dump_file data/binarized_text ``` @@ -66,7 +74,8 @@ Our implementation of masked language modeling loss follows [XLM](https://github ```bash python scripts/token_counts.py \ --data_file data/binarized_text.bert-base-uncased.pickle \ - --token_counts_dump data/token_counts.bert-base-uncased.pickle + --token_counts_dump data/token_counts.bert-base-uncased.pickle \ + --vocab_size 30522 ``` ### B. Training @@ -75,6 +84,12 @@ Training with distillation is really simple once you have pre-processed the data ```bash python train.py \ + --student_type distilbert \ + --student_config training_configs/distilbert-base-uncased.json \ + --teacher_type bert \ + --teacher_name bert-base-uncased \ + --alpha_ce 0.33 --alpha_mlm 0.33 --alpha_cos 0.33 --mlm \ + --freeze_pos_embs \ --dump_path serialization_dir/my_first_training \ --data_file data/binarized_text.bert-base-uncased.pickle \ --token_counts data/token_counts.bert-base-uncased.pickle \ @@ -83,7 +98,7 @@ python train.py \ By default, this will launch a training on a single GPU (even if more are available on the cluster). Other parameters are available in the command line, please look in `train.py` or run `python train.py --help` to list them. -We highly encourage you to use distributed training for training DistilBert as the training corpus is quite large. Here's an example that runs a distributed training on a single node having 4 GPUs: +We highly encourage you to use distributed training for training DistilBERT as the training corpus is quite large. Here's an example that runs a distributed training on a single node having 4 GPUs: ```bash export NODE_RANK=0 @@ -105,11 +120,17 @@ python -m torch.distributed.launch \ train.py \ --force \ --n_gpu $WORLD_SIZE \ + --student_type distilbert \ + --student_config training_configs/distilbert-base-uncased.json \ + --teacher_type bert \ + --teacher_name bert-base-uncased \ + --alpha_ce 0.33 --alpha_mlm 0.33 --alpha_cos 0.33 --mlm \ + --freeze_pos_embs \ + --dump_path serialization_dir/my_first_training \ --data_file data/binarized_text.bert-base-uncased.pickle \ - --token_counts data/token_counts.bert-base-uncased.pickle \ - --dump_path serialization_dir/my_first_distillation + --token_counts data/token_counts.bert-base-uncased.pickle ``` -**Tips:** Starting distillated training with good initialization of the model weights is crucial to reach decent performance. In our experiments, we initialized our model from a few layers of the teacher (Bert) itself! Please refer to `scripts/extract_for_distil.py` to create a valid initialization checkpoint and use `--from_pretrained_weights` and `--from_pretrained_config` arguments to use this initialization for the distilled training! +**Tips:** Starting distillated training with good initialization of the model weights is crucial to reach decent performance. In our experiments, we initialized our model from a few layers of the teacher (Bert) itself! Please refer to `scripts/extract.py` and `scripts/extract_distilbert.py` to create a valid initialization checkpoint and use `--student_pretrained_weights` argument to use this initialization for the distilled training! Happy distillation! From f1f23ad1710953e75b53a85953b018b8caceb427 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Wed, 2 Oct 2019 19:03:32 -0400 Subject: [PATCH 209/219] fix buf in convert_pt_chkpt_to_tf2 --- transformers/convert_pytorch_checkpoint_to_tf2.py | 1 + 1 file changed, 1 insertion(+) diff --git a/transformers/convert_pytorch_checkpoint_to_tf2.py b/transformers/convert_pytorch_checkpoint_to_tf2.py index c5f7650b50..d8a48e9dcd 100644 --- a/transformers/convert_pytorch_checkpoint_to_tf2.py +++ b/transformers/convert_pytorch_checkpoint_to_tf2.py @@ -228,6 +228,7 @@ if __name__ == "__main__": convert_all_pt_checkpoints_to_tf(args.model_type.lower() if args.model_type is not None else None, args.tf_dump_path, model_shortcut_names_or_path=[args.pytorch_checkpoint_path] if args.pytorch_checkpoint_path is not None else None, + config_shortcut_names_or_path=[args.config_file] if args.config_file is not None else None, compare_with_pt_model=args.compare_with_pt_model, use_cached_models=args.use_cached_models, only_convert_finetuned_models=args.only_convert_finetuned_models) From 35071007cb1600acf7e8197e42f16e3698dc5f35 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Wed, 2 Oct 2019 23:01:36 -0400 Subject: [PATCH 210/219] =?UTF-8?q?incoming=20release=20=F0=9F=94=A5=20upd?= =?UTF-8?q?ate=20links=20to=20arxiv=20preprint?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 3 +-- examples/distillation/README.md | 6 +++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index b5b7245bd9..4b4f6d5def 100644 --- a/README.md +++ b/README.md @@ -120,8 +120,7 @@ At some point in the future, you'll be able to seamlessly move from pre-training 5. **[XLNet](https://github.com/zihangdai/xlnet/)** (from Google/CMU) released with the paper [​XLNet: Generalized Autoregressive Pretraining for Language Understanding](https://arxiv.org/abs/1906.08237) by Zhilin Yang*, Zihang Dai*, Yiming Yang, Jaime Carbonell, Ruslan Salakhutdinov, Quoc V. Le. 6. **[XLM](https://github.com/facebookresearch/XLM/)** (from Facebook) released together with the paper [Cross-lingual Language Model Pretraining](https://arxiv.org/abs/1901.07291) by Guillaume Lample and Alexis Conneau. 7. **[RoBERTa](https://github.com/pytorch/fairseq/tree/master/examples/roberta)** (from Facebook), released together with the paper a [Robustly Optimized BERT Pretraining Approach](https://arxiv.org/abs/1907.11692) by Yinhan Liu, Myle Ott, Naman Goyal, Jingfei Du, Mandar Joshi, Danqi Chen, Omer Levy, Mike Lewis, Luke Zettlemoyer, Veselin Stoyanov. -8. **[DistilBERT](https://github.com/huggingface/transformers/tree/master/examples/distillation)** (from HuggingFace), released together with the blogpost [Smaller, faster, cheaper, lighter: Introducing DistilBERT, a distilled version of BERT](https://medium.com/huggingface/distilbert-8cf3380435b5 -) by Victor Sanh, Lysandre Debut and Thomas Wolf. +8. **[DistilBERT](https://github.com/huggingface/transformers/tree/master/examples/distillation)** (from HuggingFace), released together with the paper [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter](https://arxiv.org/abs/1910.01108) by Victor Sanh, Lysandre Debut and Thomas Wolf. The same method has been applied to compress GPT2 into [DistilGPT2](https://github.com/huggingface/transformers/tree/master/examples/distillation). These implementations have been tested on several datasets (see the example scripts) and should match the performances of the original implementations (e.g. ~93 F1 on SQuAD for BERT Whole-Word-Masking, ~88 F1 on RocStories for OpenAI GPT, ~18.3 perplexity on WikiText 103 for Transformer-XL, ~0.916 Peason R coefficient on STS-B for XLNet). You can find more details on the performances in the Examples section of the [documentation](https://huggingface.co/transformers/examples.html). diff --git a/examples/distillation/README.md b/examples/distillation/README.md index ad439cf5f8..8436ab95bf 100644 --- a/examples/distillation/README.md +++ b/examples/distillation/README.md @@ -2,7 +2,7 @@ This folder contains the original code used to train Distil* as well as examples showcasing how to use DistilBERT and DistilGPT2. -**2019, October 3rd - Update** We release our [NeurIPS workshop paper](TODO LINK) explaining our approach on DistilBERT. It includes updated results and further experiments. We applied the same method to GPT2 and release the weights of DistilGPT2. DistilGPT2 is two times faster and 33% smaller than GPT2. +**2019, October 3rd - Update** We release our [NeurIPS workshop paper](https://arxiv.org/abs/1910.01108) explaining our approach on **DistilBERT**. It includes updated results and further experiments. We applied the same method to GPT2 and release the weights of **DistilGPT2**. DistilGPT2 is two times faster and 33% smaller than GPT2. **2019, September 19th - Update:** We fixed bugs in the code and released an upadted version of the weights trained with a modification of the distillation loss. DistilBERT now reaches 97% of `BERT-base`'s performance on GLUE, and 86.9 F1 score on SQuAD v1.1 dev set (compared to 88.5 for `BERT-base`). We will publish a formal write-up of our approach in the near future! @@ -12,7 +12,7 @@ Distil* is a class of compressed models that started with DistilBERT. DistilBERT We have applied the same method to GPT2 and release the weights of the compressed model. On the [WikiText-103](https://blog.einstein.ai/the-wikitext-long-term-dependency-language-modeling-dataset/) benchmark, GPT2 reaches a perplexity on the test of 15.84 compared to 19.91 for DistilGPT2 (after fine-tuning on the train set). -For more information on DistilBERT, please refer to our [NeurIPS workshop paper](TODO LINK). The paper superseeds our [previous blogpost](https://medium.com/huggingface/distilbert-8cf3380435b5) with a different distillation loss and better performances. +For more information on DistilBERT, please refer to our [NeurIPS workshop paper](https://arxiv.org/abs/1910.01108). The paper superseeds our [previous blogpost](https://medium.com/huggingface/distilbert-8cf3380435b5) with a different distillation loss and better performances. Here are the results on the dev sets of GLUE: @@ -88,7 +88,7 @@ python train.py \ --student_config training_configs/distilbert-base-uncased.json \ --teacher_type bert \ --teacher_name bert-base-uncased \ - --alpha_ce 0.33 --alpha_mlm 0.33 --alpha_cos 0.33 --mlm \ + --alpha_ce 5.0 --alpha_mlm 2.0 --alpha_cos 1.0 --mlm \ --freeze_pos_embs \ --dump_path serialization_dir/my_first_training \ --data_file data/binarized_text.bert-base-uncased.pickle \ From 5f07d8f11a66ea58be31b93a0dc21f428b7e1714 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Thu, 3 Oct 2019 09:59:32 -0400 Subject: [PATCH 211/219] prepare release --- examples/distillation/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/distillation/README.md b/examples/distillation/README.md index 8436ab95bf..5b6fbd2e9a 100644 --- a/examples/distillation/README.md +++ b/examples/distillation/README.md @@ -10,7 +10,7 @@ This folder contains the original code used to train Distil* as well as examples Distil* is a class of compressed models that started with DistilBERT. DistilBERT stands for Distillated-BERT. DistilBERT is a small, fast, cheap and light Transformer model based on Bert architecture. It has 40% less parameters than `bert-base-uncased`, runs 60% faster while preserving 97% of BERT's performances as measured on the GLUE language understanding benchmark. DistilBERT is trained using knowledge distillation, a technique to compress a large model called the teacher into a smaller model called the student. By distillating Bert, we obtain a smaller Transformer model that bears a lot of similarities with the original BERT model while being lighter, smaller and faster to run. DistilBERT is thus an interesting option to put large-scaled trained Transformer model into production. -We have applied the same method to GPT2 and release the weights of the compressed model. On the [WikiText-103](https://blog.einstein.ai/the-wikitext-long-term-dependency-language-modeling-dataset/) benchmark, GPT2 reaches a perplexity on the test of 15.84 compared to 19.91 for DistilGPT2 (after fine-tuning on the train set). +We have applied the same method to GPT2 and release the weights of the compressed model. On the [WikiText-103](https://blog.einstein.ai/the-wikitext-long-term-dependency-language-modeling-dataset/) benchmark, GPT2 reaches a perplexity on the test of 15.8 compared to 19.3 for DistilGPT2 (after fine-tuning on the train set). For more information on DistilBERT, please refer to our [NeurIPS workshop paper](https://arxiv.org/abs/1910.01108). The paper superseeds our [previous blogpost](https://medium.com/huggingface/distilbert-8cf3380435b5) with a different distillation loss and better performances. From 6be46a6e6422d7ab34984d6fbcaadad0323e5349 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Thu, 3 Oct 2019 10:07:18 -0400 Subject: [PATCH 212/219] update links to new weights --- transformers/configuration_gpt2.py | 3 ++- transformers/modeling_gpt2.py | 3 ++- transformers/modeling_tf_gpt2.py | 3 ++- transformers/tokenization_gpt2.py | 3 +++ 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/transformers/configuration_gpt2.py b/transformers/configuration_gpt2.py index c83d9e82ce..e7d853f317 100644 --- a/transformers/configuration_gpt2.py +++ b/transformers/configuration_gpt2.py @@ -28,7 +28,8 @@ logger = logging.getLogger(__name__) GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP = {"gpt2": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-config.json", "gpt2-medium": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-medium-config.json", - "gpt2-large": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-large-config.json"} + "gpt2-large": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-large-config.json", + "distilgpt2": "https://s3.amazonaws.com/models.huggingface.co/bert/distilgpt2-config.json",} class GPT2Config(PretrainedConfig): """Configuration class to store the configuration of a `GPT2Model`. diff --git a/transformers/modeling_gpt2.py b/transformers/modeling_gpt2.py index bc85224022..891dfc5677 100644 --- a/transformers/modeling_gpt2.py +++ b/transformers/modeling_gpt2.py @@ -38,7 +38,8 @@ logger = logging.getLogger(__name__) GPT2_PRETRAINED_MODEL_ARCHIVE_MAP = {"gpt2": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-pytorch_model.bin", "gpt2-medium": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-medium-pytorch_model.bin", - "gpt2-large": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-large-pytorch_model.bin"} + "gpt2-large": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-large-pytorch_model.bin", + "distilgpt2": "https://s3.amazonaws.com/models.huggingface.co/bert/distilgpt2-pytorch_model.bin",} def load_tf_weights_in_gpt2(model, config, gpt2_checkpoint_path): """ Load tf checkpoints in a pytorch model diff --git a/transformers/modeling_tf_gpt2.py b/transformers/modeling_tf_gpt2.py index e958c2cbf1..883340cac9 100644 --- a/transformers/modeling_tf_gpt2.py +++ b/transformers/modeling_tf_gpt2.py @@ -38,7 +38,8 @@ logger = logging.getLogger(__name__) TF_GPT2_PRETRAINED_MODEL_ARCHIVE_MAP = {"gpt2": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-tf_model.h5", "gpt2-medium": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-medium-tf_model.h5", - "gpt2-large": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-large-tf_model.h5"} + "gpt2-large": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-large-tf_model.h5", + "distilgpt2": "https://s3.amazonaws.com/models.huggingface.co/bert/distilgpt2-tf_model.h5",} def load_gpt2_pt_weights_in_tf2(tf_model, pytorch_checkpoint_path): diff --git a/transformers/tokenization_gpt2.py b/transformers/tokenization_gpt2.py index 3e931dfcf8..6a7f75acb2 100644 --- a/transformers/tokenization_gpt2.py +++ b/transformers/tokenization_gpt2.py @@ -46,12 +46,14 @@ PRETRAINED_VOCAB_FILES_MAP = { 'gpt2': "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-vocab.json", 'gpt2-medium': "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-medium-vocab.json", 'gpt2-large': "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-large-vocab.json", + 'distilgpt2': "https://s3.amazonaws.com/models.huggingface.co/bert/distilgpt2-vocab.json", }, 'merges_file': { 'gpt2': "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-merges.txt", 'gpt2-medium': "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-medium-merges.txt", 'gpt2-large': "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-large-merges.txt", + 'distilgpt2': "https://s3.amazonaws.com/models.huggingface.co/bert/distilgpt2-merges.txt", }, } @@ -59,6 +61,7 @@ PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { 'gpt2': 1024, 'gpt2-medium': 1024, 'gpt2-large': 1024, + 'distilgpt2': 1024, } @lru_cache() From 4a790c40b1817fd457043f9933266b4d5e20b3b7 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Thu, 3 Oct 2019 10:54:02 -0400 Subject: [PATCH 213/219] update doc for distil* --- docs/source/pretrained_models.rst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/source/pretrained_models.rst b/docs/source/pretrained_models.rst index c12a9bc52f..7606082c7e 100644 --- a/docs/source/pretrained_models.rst +++ b/docs/source/pretrained_models.rst @@ -119,11 +119,14 @@ Here is the full list of the currently provided pretrained models together with +-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | DistilBERT | ``distilbert-base-uncased`` | | 6-layer, 768-hidden, 12-heads, 66M parameters | | | | | The DistilBERT model distilled from the BERT model `bert-base-uncased` checkpoint | -| | | (see `details `__) | +| | | (see `details `__) | | +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``distilbert-base-uncased-distilled-squad`` | | 6-layer, 768-hidden, 12-heads, 66M parameters | | | | | The DistilBERT model distilled from the BERT model `bert-base-uncased` checkpoint, with an additional linear layer. | -| | | (see `details `__) | +| | | (see `details `__) | +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``distilbert-base-uncased-distilled-squad`` | | 6-layer, 768-hidden, 12-heads, 82M parameters | +| | | | The DistilGPT2 model distilled from the GPT2 model `gpt2` checkpoint. | +| | | (see `details `__) | +-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ - .. `__ \ No newline at end of file From c1689ac30164d190f366d95d1f5153af53e66355 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Thu, 3 Oct 2019 10:56:39 -0400 Subject: [PATCH 214/219] fix name --- docs/source/pretrained_models.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/pretrained_models.rst b/docs/source/pretrained_models.rst index 7606082c7e..2622f3cd80 100644 --- a/docs/source/pretrained_models.rst +++ b/docs/source/pretrained_models.rst @@ -125,7 +125,7 @@ Here is the full list of the currently provided pretrained models together with | | | | The DistilBERT model distilled from the BERT model `bert-base-uncased` checkpoint, with an additional linear layer. | | | | (see `details `__) | | +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ -| | ``distilbert-base-uncased-distilled-squad`` | | 6-layer, 768-hidden, 12-heads, 82M parameters | +| | ``distilgpt2`` | | 6-layer, 768-hidden, 12-heads, 82M parameters | | | | | The DistilGPT2 model distilled from the GPT2 model `gpt2` checkpoint. | | | | (see `details `__) | +-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ From 7af0777910f1450965819111c6cd5a637630d086 Mon Sep 17 00:00:00 2001 From: Brian Ma Date: Thu, 3 Oct 2019 16:29:43 +0800 Subject: [PATCH 215/219] Update run_glue.py add DistilBert model shortcut into ALL_MODELS --- examples/run_glue.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/run_glue.py b/examples/run_glue.py index fc3b617da0..e02e9b4294 100644 --- a/examples/run_glue.py +++ b/examples/run_glue.py @@ -53,7 +53,8 @@ from transformers import glue_convert_examples_to_features as convert_examples_t logger = logging.getLogger(__name__) -ALL_MODELS = sum((tuple(conf.pretrained_config_archive_map.keys()) for conf in (BertConfig, XLNetConfig, XLMConfig, RobertaConfig)), ()) +ALL_MODELS = sum((tuple(conf.pretrained_config_archive_map.keys()) for conf in (BertConfig, XLNetConfig, XLMConfig, + RobertaConfig, DistilBertConfig)), ()) MODEL_CLASSES = { 'bert': (BertConfig, BertForSequenceClassification, BertTokenizer), From e2ae9c0b73e87a0d8053046ba1b33c3632750028 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Thu, 3 Oct 2019 11:42:21 -0400 Subject: [PATCH 216/219] fix links in doc index --- docs/source/index.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/source/index.rst b/docs/source/index.rst index 8c76b89185..3b4fe4d1e8 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -46,8 +46,7 @@ The library currently contains PyTorch and Tensorflow implementations, pre-train 5. `XLNet `_ (from Google/CMU) released with the paper `​XLNet: Generalized Autoregressive Pretraining for Language Understanding `_ by Zhilin Yang*, Zihang Dai*, Yiming Yang, Jaime Carbonell, Ruslan Salakhutdinov, Quoc V. Le. 6. `XLM `_ (from Facebook) released together with the paper `Cross-lingual Language Model Pretraining `_ by Guillaume Lample and Alexis Conneau. 7. `RoBERTa `_ (from Facebook), released together with the paper a `Robustly Optimized BERT Pretraining Approach `_ by Yinhan Liu, Myle Ott, Naman Goyal, Jingfei Du, Mandar Joshi, Danqi Chen, Omer Levy, Mike Lewis, Luke Zettlemoyer, Veselin Stoyanov. -8. `DistilBERT `_ (from HuggingFace) released together with the blog post `Smaller, faster, cheaper, lighter: Introducing DistilBERT, a distilled version of BERT `_ by Victor Sanh, Lysandre Debut and Thomas Wolf. - +8. `DistilBERT `_ (from HuggingFace) released together with the paper `DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter `_ by Victor Sanh, Lysandre Debut and Thomas Wolf. The same method has been applied to compress GPT2 into `DistilGPT2 `_. .. toctree:: :maxdepth: 2 From e1b2949ae6cb34cc39e3934ca87423474f8c8d02 Mon Sep 17 00:00:00 2001 From: drc10723 Date: Thu, 3 Oct 2019 21:22:36 +0530 Subject: [PATCH 217/219] DistillBert Documentation Code Example fixes --- transformers/modeling_distilbert.py | 2 +- transformers/modeling_tf_distilbert.py | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/transformers/modeling_distilbert.py b/transformers/modeling_distilbert.py index 2425ab5f47..ebb89f0f95 100644 --- a/transformers/modeling_distilbert.py +++ b/transformers/modeling_distilbert.py @@ -649,7 +649,7 @@ class DistilBertForQuestionAnswering(DistilBertPreTrainedModel): start_positions = torch.tensor([1]) end_positions = torch.tensor([3]) outputs = model(input_ids, start_positions=start_positions, end_positions=end_positions) - loss, start_scores, end_scores = outputs[:2] + loss, start_scores, end_scores = outputs[:3] """ def __init__(self, config): diff --git a/transformers/modeling_tf_distilbert.py b/transformers/modeling_tf_distilbert.py index 5ce1616bcc..6ed2844567 100644 --- a/transformers/modeling_tf_distilbert.py +++ b/transformers/modeling_tf_distilbert.py @@ -603,7 +603,7 @@ class TFDistilBertForMaskedLM(TFDistilBertPreTrainedModel): tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased') model = TFDistilBertForMaskedLM.from_pretrained('distilbert-base-uncased') input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 - outputs = model(input_ids, masked_lm_labels=input_ids) + outputs = model(input_ids) prediction_scores = outputs[0] """ @@ -715,9 +715,7 @@ class TFDistilBertForQuestionAnswering(TFDistilBertPreTrainedModel): tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased') model = TFDistilBertForQuestionAnswering.from_pretrained('distilbert-base-uncased') input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 - start_positions = tf.constant([1]) - end_positions = tf.constant([3]) - outputs = model(input_ids, start_positions=start_positions, end_positions=end_positions) + outputs = model(input_ids) start_scores, end_scores = outputs[:2] """ From c2c2ca0fdba6f16f2e66d0a21152aa4ae493ca78 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Thu, 3 Oct 2019 17:18:48 -0400 Subject: [PATCH 218/219] Added XLM to run_generation, with prompt language selection. --- examples/run_generation.py | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/examples/run_generation.py b/examples/run_generation.py index 9e98a9e870..a70a0e7842 100644 --- a/examples/run_generation.py +++ b/examples/run_generation.py @@ -26,12 +26,13 @@ import torch import torch.nn.functional as F import numpy as np -from transformers import GPT2Config, OpenAIGPTConfig, XLNetConfig, TransfoXLConfig +from transformers import GPT2Config, OpenAIGPTConfig, XLNetConfig, TransfoXLConfig, XLMConfig from transformers import GPT2LMHeadModel, GPT2Tokenizer from transformers import OpenAIGPTLMHeadModel, OpenAIGPTTokenizer from transformers import XLNetLMHeadModel, XLNetTokenizer from transformers import TransfoXLLMHeadModel, TransfoXLTokenizer +from transformers import XLMWithLMHeadModel, XLMTokenizer logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', @@ -41,13 +42,14 @@ logger = logging.getLogger(__name__) MAX_LENGTH = int(10000) # Hardcoded max length to avoid infinite loop -ALL_MODELS = sum((tuple(conf.pretrained_config_archive_map.keys()) for conf in (GPT2Config, OpenAIGPTConfig, XLNetConfig, TransfoXLConfig)), ()) +ALL_MODELS = sum((tuple(conf.pretrained_config_archive_map.keys()) for conf in (GPT2Config, OpenAIGPTConfig, XLNetConfig, TransfoXLConfig, XLMConfig)), ()) MODEL_CLASSES = { 'gpt2': (GPT2LMHeadModel, GPT2Tokenizer), 'openai-gpt': (OpenAIGPTLMHeadModel, OpenAIGPTTokenizer), 'xlnet': (XLNetLMHeadModel, XLNetTokenizer), 'transfo-xl': (TransfoXLLMHeadModel, TransfoXLTokenizer), + 'xlm': (XLMWithLMHeadModel, XLMTokenizer), } # Padding text to help Transformer-XL and XLNet with short prompts as proposed by Aman Rusia @@ -103,7 +105,8 @@ def top_k_top_p_filtering(logits, top_k=0, top_p=0.0, filter_value=-float('Inf') return logits -def sample_sequence(model, length, context, num_samples=1, temperature=1, top_k=0, top_p=0.0, is_xlnet=False, device='cpu'): +def sample_sequence(model, length, context, num_samples=1, temperature=1, top_k=0, top_p=0.0, is_xlnet=False, + xlm_lang=None, device='cpu'): context = torch.tensor(context, dtype=torch.long, device=device) context = context.unsqueeze(0).repeat(num_samples, 1) generated = context @@ -121,6 +124,9 @@ def sample_sequence(model, length, context, num_samples=1, temperature=1, top_k= target_mapping[0, 0, -1] = 1.0 # predict last token inputs = {'input_ids': input_ids, 'perm_mask': perm_mask, 'target_mapping': target_mapping} + if xlm_lang is not None: + inputs["langs"] = torch.tensor([xlm_lang] * inputs["input_ids"].shape[1]).view(1, -1) + outputs = model(**inputs) # Note: we could also use 'past' with GPT-2/Transfo-XL/XLNet (cached hidden-states) next_token_logits = outputs[0][0, -1, :] / temperature filtered_logits = top_k_top_p_filtering(next_token_logits, top_k=top_k, top_p=top_p) @@ -137,6 +143,7 @@ def main(): help="Path to pre-trained model or shortcut name selected in the list: " + ", ".join(ALL_MODELS)) parser.add_argument("--prompt", type=str, default="") parser.add_argument("--padding_text", type=str, default="") + parser.add_argument("--xlm_lang", type=str, default="", help="Optional language when used with the XLM model.") parser.add_argument("--length", type=int, default=20) parser.add_argument("--temperature", type=float, default=1.0) parser.add_argument("--top_k", type=int, default=0) @@ -168,6 +175,17 @@ def main(): print(args) while True: + xlm_lang = None + # XLM Language usage detailed in the issues #1414 + if args.model_type in ["xlm"] and hasattr(tokenizer, 'lang2id'): + if args.xlm_lang: + language = args.xlm_lang + else: + language = None + while language not in tokenizer.lang2id.keys(): + language = input("Using XLM. Select language in " + str(list(tokenizer.lang2id.keys())) + " >>> ") + xlm_lang = tokenizer.lang2id[language] + raw_text = args.prompt if args.prompt else input("Model prompt >>> ") if args.model_type in ["transfo-xl", "xlnet"]: # Models with memory likes to have a long prompt for short inputs. @@ -180,11 +198,12 @@ def main(): temperature=args.temperature, top_k=args.top_k, top_p=args.top_p, - device=args.device, is_xlnet=bool(args.model_type == "xlnet"), + xlm_lang=xlm_lang, + device=args.device, ) out = out[0, len(context_tokens):].tolist() - text = tokenizer.decode(out, clean_up_tokenization_spaces=True) + text = tokenizer.decode(out, clean_up_tokenization_spaces=True, skip_special_tokens=True) print(text) if args.prompt: break From ecc4f1bdfae5b3a9679fae499c5c9b375d927547 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Thu, 3 Oct 2019 17:42:16 -0400 Subject: [PATCH 219/219] XLM use_lang_embedding flag in run_generation --- examples/run_generation.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/run_generation.py b/examples/run_generation.py index a70a0e7842..83926f42b7 100644 --- a/examples/run_generation.py +++ b/examples/run_generation.py @@ -177,7 +177,8 @@ def main(): while True: xlm_lang = None # XLM Language usage detailed in the issues #1414 - if args.model_type in ["xlm"] and hasattr(tokenizer, 'lang2id'): + if args.model_type in ["xlm"] and hasattr(tokenizer, 'lang2id') and hasattr(model.config, 'use_lang_emb') \ + and model.config.use_lang_emb: if args.xlm_lang: language = args.xlm_lang else: