update doc for XLM and XLNet
This commit is contained in:
thomwolf
@@ -30,7 +30,7 @@ from torch import nn
|
||||
from torch.nn import functional as F
|
||||
from torch.nn import CrossEntropyLoss, MSELoss
|
||||
|
||||
from .modeling_utils import (PretrainedConfig, PreTrainedModel,
|
||||
from .modeling_utils import (PretrainedConfig, PreTrainedModel, add_start_docstrings,
|
||||
prune_linear_layer, SequenceSummary, SQuADHead)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -392,28 +392,94 @@ class XLMPreTrainedModel(PreTrainedModel):
|
||||
module.weight.data.fill_(1.0)
|
||||
|
||||
|
||||
XLM_START_DOCSTRING = r""" The XLM model was proposed in
|
||||
`Cross-lingual Language Model Pretraining`_
|
||||
by Guillaume Lample*, Alexis Conneau*. It's a transformer pre-trained using one of the following objectives:
|
||||
|
||||
- a causal language modeling (CLM) objective (next token prediction),
|
||||
- a masked language modeling (MLM) objective (Bert-like), or
|
||||
- a Translation Language Modeling (TLM) object (extension of Bert's MLM to multiple language inputs)
|
||||
|
||||
Original code can be found `here`_.
|
||||
|
||||
This model is a PyTorch `torch.nn.Module`_ sub-class. Use it as a regular PyTorch Module and
|
||||
refer to the PyTorch documentation for all matter related to general usage and behavior.
|
||||
|
||||
.. _`Cross-lingual Language Model Pretraining`:
|
||||
https://arxiv.org/abs/1901.07291
|
||||
|
||||
.. _`torch.nn.Module`:
|
||||
https://pytorch.org/docs/stable/nn.html#module
|
||||
|
||||
.. _`here`:
|
||||
https://github.com/facebookresearch/XLM
|
||||
|
||||
Parameters:
|
||||
config (:class:`~pytorch_transformers.XLMConfig`): Model configuration class with all the parameters of the model.
|
||||
"""
|
||||
|
||||
XLM_INPUTS_DOCSTRING = r"""
|
||||
Inputs:
|
||||
**input_ids**: ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``:
|
||||
Indices of input sequence tokens in the vocabulary.
|
||||
Indices can be obtained using :class:`pytorch_transformers.XLMTokenizer`.
|
||||
See :func:`pytorch_transformers.PreTrainedTokenizer.encode` and
|
||||
:func:`pytorch_transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details.
|
||||
**position_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``:
|
||||
Indices of positions of each input sequence tokens in the position embeddings.
|
||||
Selected in the range ``[0, config.max_position_embeddings - 1[``.
|
||||
**token_type_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``:
|
||||
A parallel sequence of tokens (can be used to indicate various portions of the inputs).
|
||||
The embeddings from these tokens will be summed with the respective token embeddings.
|
||||
Indices are selected in the vocabulary (unlike BERT which has a specific vocabulary for segment indices).
|
||||
**langs**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``:
|
||||
A parallel sequence of tokens to be used to indicate the language of each token in the input.
|
||||
Indices are selected in the pre-trained language vocabulary,
|
||||
i.e. in the range ``[0, config.n_langs - 1[``.
|
||||
**attention_mask**: (`optional`) ``torch.Tensor`` of shape ``(batch_size, sequence_length)``:
|
||||
Mask to avoid performing attention on padding token indices.
|
||||
Mask values selected in ``[0, 1]``:
|
||||
``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens.
|
||||
**lengths**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``:
|
||||
Length of each sentence that can be used to avoid performing attention on padding token indices.
|
||||
You can also use `attention_mask` for the same result (see above), kept here for compatbility.
|
||||
Indices selected in ``[0, ..., input_ids.size(-1)]``:
|
||||
**cache**:
|
||||
dictionary with ``torch.FloatTensor`` that contains pre-computed
|
||||
hidden-states (key and values in the attention blocks) as computed by the model
|
||||
(see `cache` output below). Can be used to speed up sequential decoding.
|
||||
The dictionary object will be modified in-place during the forward pass to add newly computed hidden-states.
|
||||
**head_mask**: (`optional`) ``torch.Tensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``:
|
||||
Mask to nullify selected heads of the self-attention modules.
|
||||
Mask values selected in ``[0, 1]``:
|
||||
``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**.
|
||||
"""
|
||||
|
||||
@add_start_docstrings("The bare XLM Model transformer outputing raw hidden-states without any specific head on top.",
|
||||
XLM_START_DOCSTRING, XLM_INPUTS_DOCSTRING)
|
||||
class XLMModel(XLMPreTrainedModel):
|
||||
r"""
|
||||
Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:
|
||||
**last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)``
|
||||
Sequence of hidden-states at the last layer of the model.
|
||||
**attentions**: (`optional`, returned when ``config.output_attentions=True``)
|
||||
list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:
|
||||
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads.
|
||||
**hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)
|
||||
list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings)
|
||||
of shape ``(batch_size, sequence_length, hidden_size)``:
|
||||
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
|
||||
|
||||
Examples::
|
||||
|
||||
>>> config = XLMConfig.from_pretrained('xlm-mlm-en-2048')
|
||||
>>> tokenizer = XLMTokenizer.from_pretrained('xlm-mlm-en-2048')
|
||||
>>> model = XLMModel(config)
|
||||
>>> input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1
|
||||
>>> outputs = model(input_ids)
|
||||
>>> last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple
|
||||
|
||||
"""
|
||||
XLM model from: "Cross-lingual Language Model Pretraining" by Guillaume Lample, Alexis Conneau
|
||||
|
||||
Paper: https://arxiv.org/abs/1901.07291
|
||||
|
||||
Original code: https://github.com/facebookresearch/XLM
|
||||
|
||||
Args:
|
||||
`config`: a XLMConfig class instance with the configuration to build a new model
|
||||
`output_attentions`: If True, also output attentions weights computed by the model at each layer. Default: False
|
||||
`keep_multihead_output`: If True, saves output of the multi-head attention module with its gradient.
|
||||
This can be used to compute head importance metrics. Default: False
|
||||
|
||||
Example::
|
||||
|
||||
config = modeling.XLMConfig(vocab_size_or_config_json_file=32000, hidden_size=768,
|
||||
num_hidden_layers=12, num_attention_heads=12, intermediate_size=3072)
|
||||
|
||||
model = modeling.XLMModel(config=config)
|
||||
"""
|
||||
|
||||
ATTRIBUTES = ['encoder', 'eos_index', 'pad_index', # 'with_output',
|
||||
'n_langs', 'n_words', 'dim', 'n_layers', 'n_heads',
|
||||
'hidden_dim', 'dropout', 'attention_dropout', 'asm',
|
||||
@@ -493,57 +559,8 @@ class XLMModel(XLMPreTrainedModel):
|
||||
for layer, heads in heads_to_prune.items():
|
||||
self.attentions[layer].prune_heads(heads)
|
||||
|
||||
def forward(self, input_ids, lengths=None, positions=None, langs=None,
|
||||
def forward(self, input_ids, lengths=None, position_ids=None, langs=None,
|
||||
token_type_ids=None, attention_mask=None, cache=None, head_mask=None): # src_enc=None, src_len=None,
|
||||
"""
|
||||
Performs a model forward pass. **Can be called by calling the class directly, once it has been instantiated.**
|
||||
|
||||
Parameters:
|
||||
`input_ids`: a ``torch.LongTensor`` of shape [batch_size, sequence_length]
|
||||
with the word token indices in the vocabulary(see the tokens preprocessing logic in the scripts
|
||||
`run_bert_extract_features.py`, `run_bert_classifier.py` and `run_bert_squad.py`)
|
||||
`lengths`: ``torch.LongTensor`` of size ``bs``, containing the length of each sentence
|
||||
`positions`: ``torch.LongTensor`` of size ``(bs, slen)``, containing word positions
|
||||
`langs`: ``torch.LongTensor`` of size ``(bs, slen)``, containing language IDs
|
||||
`token_type_ids`: an optional ``torch.LongTensor`` of shape [batch_size, sequence_length] with the token
|
||||
types indices selected in [0, 1]. Type 0 corresponds to a `sentence A` and type 1 corresponds to
|
||||
a `sentence B` token (see XLM paper for more details).
|
||||
`attention_mask`: an optional ``torch.LongTensor`` of shape [batch_size, sequence_length] with indices
|
||||
selected in [0, 1]. It's a mask to be used if the input sequence length is smaller than the max
|
||||
input sequence length in the current batch. It's the mask that we typically use for attention when
|
||||
a batch has varying length sentences.
|
||||
`cache`: TODO
|
||||
`head_mask`: an optional ``torch.Tensor`` of shape [num_heads] or [num_layers, num_heads] with indices between 0 and 1.
|
||||
It's a mask to be used to nullify some heads of the transformer. 1.0 => head is fully masked, 0.0 => head is not masked.
|
||||
|
||||
|
||||
Returns:
|
||||
A ``tuple(encoded_layers, pooled_output)``, with
|
||||
|
||||
``encoded_layers``: controlled by ``output_all_encoded_layers`` argument:
|
||||
|
||||
- ``output_all_encoded_layers=True``: outputs a list of the full sequences of encoded-hidden-states at the end \
|
||||
of each attention block (i.e. 12 full sequences for XLM-base, 24 for XLM-large), each \
|
||||
encoded-hidden-state is a ``torch.FloatTensor`` of size [batch_size, sequence_length, hidden_size],
|
||||
|
||||
- ``output_all_encoded_layers=False``: outputs only the full sequence of hidden-states corresponding \
|
||||
to the last attention block of shape [batch_size, sequence_length, hidden_size],
|
||||
|
||||
``pooled_output``: a ``torch.FloatTensor`` of size [batch_size, hidden_size] which is the output of a
|
||||
classifier pre-trained on top of the hidden state associated to the first character of the
|
||||
input (`CLS`) to train on the Next-Sentence task (see XLM's paper).
|
||||
|
||||
Example::
|
||||
|
||||
# Already been converted into WordPiece token ids
|
||||
input_ids = torch.LongTensor([[31, 51, 99], [15, 5, 0]])
|
||||
input_mask = torch.LongTensor([[1, 1, 1], [1, 1, 0]])
|
||||
token_type_ids = torch.LongTensor([[0, 0, 1], [0, 1, 0]])
|
||||
|
||||
all_encoder_layers, pooled_output = model(input_ids, token_type_ids, input_mask)
|
||||
# or
|
||||
all_encoder_layers, pooled_output = model.forward(input_ids, token_type_ids, input_mask)
|
||||
"""
|
||||
if lengths is None:
|
||||
lengths = (input_ids != self.pad_index).sum(dim=1).long()
|
||||
# mask = input_ids != self.pad_index
|
||||
@@ -563,18 +580,15 @@ class XLMModel(XLMPreTrainedModel):
|
||||
# if self.is_decoder and src_enc is not None:
|
||||
# src_mask = torch.arange(src_len.max(), dtype=torch.long, device=lengths.device) < src_len[:, None]
|
||||
|
||||
# positions
|
||||
if positions is None:
|
||||
positions = input_ids.new((slen,)).long()
|
||||
positions = torch.arange(slen, out=positions).unsqueeze(0)
|
||||
# position_ids
|
||||
if position_ids is None:
|
||||
position_ids = input_ids.new((slen,)).long()
|
||||
position_ids = torch.arange(slen, out=position_ids).unsqueeze(0)
|
||||
else:
|
||||
assert positions.size() == (bs, slen) # (slen, bs)
|
||||
# positions = positions.transpose(0, 1)
|
||||
assert position_ids.size() == (bs, slen) # (slen, bs)
|
||||
# position_ids = position_ids.transpose(0, 1)
|
||||
|
||||
# langs
|
||||
assert langs is None or token_type_ids is None, "You can only use one among langs and token_type_ids"
|
||||
if token_type_ids is not None:
|
||||
langs = token_type_ids
|
||||
if langs is not None:
|
||||
assert langs.size() == (bs, slen) # (slen, bs)
|
||||
# langs = langs.transpose(0, 1)
|
||||
@@ -598,7 +612,7 @@ class XLMModel(XLMPreTrainedModel):
|
||||
if cache is not None:
|
||||
_slen = slen - cache['slen']
|
||||
input_ids = input_ids[:, -_slen:]
|
||||
positions = positions[:, -_slen:]
|
||||
position_ids = position_ids[:, -_slen:]
|
||||
if langs is not None:
|
||||
langs = langs[:, -_slen:]
|
||||
mask = mask[:, -_slen:]
|
||||
@@ -606,9 +620,11 @@ class XLMModel(XLMPreTrainedModel):
|
||||
|
||||
# embeddings
|
||||
tensor = self.embeddings(input_ids)
|
||||
tensor = tensor + self.position_embeddings(positions).expand_as(tensor)
|
||||
tensor = tensor + self.position_embeddings(position_ids).expand_as(tensor)
|
||||
if langs is not None:
|
||||
tensor = tensor + self.lang_embeddings(langs)
|
||||
if token_type_ids is not None:
|
||||
tensor = tensor + self.embeddings(token_type_ids)
|
||||
tensor = self.layer_norm_emb(tensor)
|
||||
tensor = F.dropout(tensor, p=self.dropout, training=self.training)
|
||||
tensor *= mask.unsqueeze(-1).to(tensor.dtype)
|
||||
@@ -702,25 +718,40 @@ class XLMPredLayer(nn.Module):
|
||||
return outputs
|
||||
|
||||
|
||||
@add_start_docstrings("""The XLM Model transformer with a language modeling head on top
|
||||
(linear layer with weights tied to the input embeddings). """,
|
||||
XLM_START_DOCSTRING, XLM_INPUTS_DOCSTRING)
|
||||
class XLMWithLMHeadModel(XLMPreTrainedModel):
|
||||
""" XLM model from: "Cross-lingual Language Model Pretraining" by Guillaume Lample, Alexis Conneau
|
||||
r"""
|
||||
**labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``:
|
||||
Labels for language modeling.
|
||||
Note that the labels **are shifted** inside the model, i.e. you can set ``lm_labels = input_ids``
|
||||
Indices are selected in ``[-1, 0, ..., config.vocab_size]``
|
||||
All labels set to ``-1`` are ignored (masked), the loss is only
|
||||
computed for labels in ``[0, ..., config.vocab_size]``
|
||||
|
||||
Paper: https://arxiv.org/abs/1901.07291
|
||||
Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:
|
||||
**loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``:
|
||||
Language modeling loss.
|
||||
**prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)``
|
||||
Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
|
||||
**attentions**: (`optional`, returned when ``config.output_attentions=True``)
|
||||
list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:
|
||||
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads.
|
||||
**hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)
|
||||
list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings)
|
||||
of shape ``(batch_size, sequence_length, hidden_size)``:
|
||||
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
|
||||
|
||||
Original code: https://github.com/facebookresearch/XLM
|
||||
Examples::
|
||||
|
||||
Args:
|
||||
`config`: a XLMConfig class instance with the configuration to build a new model
|
||||
`output_attentions`: If True, also output attentions weights computed by the model at each layer. Default: False
|
||||
`keep_multihead_output`: If True, saves output of the multi-head attention module with its gradient.
|
||||
This can be used to compute head importance metrics. Default: False
|
||||
>>> config = XLMConfig.from_pretrained('xlm-mlm-en-2048')
|
||||
>>> tokenizer = XLMTokenizer.from_pretrained('xlm-mlm-en-2048')
|
||||
>>> model = XLMWithLMHeadModel(config)
|
||||
>>> input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1
|
||||
>>> outputs = model(input_ids)
|
||||
>>> last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple
|
||||
|
||||
Example::
|
||||
|
||||
config = modeling.XLMConfig(vocab_size_or_config_json_file=32000, hidden_size=768,
|
||||
num_hidden_layers=12, num_attention_heads=12, intermediate_size=3072)
|
||||
|
||||
model = modeling.XLMModel(config=config)
|
||||
"""
|
||||
def __init__(self, config):
|
||||
super(XLMWithLMHeadModel, self).__init__(config)
|
||||
@@ -735,57 +766,9 @@ class XLMWithLMHeadModel(XLMPreTrainedModel):
|
||||
"""
|
||||
self._tie_or_clone_weights(self.pred_layer.proj, self.transformer.embeddings)
|
||||
|
||||
def forward(self, input_ids, lengths=None, positions=None, langs=None, token_type_ids=None,
|
||||
def forward(self, input_ids, lengths=None, position_ids=None, langs=None, token_type_ids=None,
|
||||
attention_mask=None, cache=None, labels=None, head_mask=None):
|
||||
"""
|
||||
Args:
|
||||
`input_ids`: a ``torch.LongTensor`` of shape [batch_size, sequence_length]
|
||||
with the word token indices in the vocabulary(see the tokens preprocessing logic in the scripts
|
||||
`run_bert_extract_features.py`, `run_bert_classifier.py` and `run_bert_squad.py`)
|
||||
`lengths`: TODO
|
||||
`positions`: TODO
|
||||
`langs`: TODO
|
||||
`token_type_ids`: an optional ``torch.LongTensor`` of shape [batch_size, sequence_length] with the token
|
||||
types indices selected in [0, 1]. Type 0 corresponds to a `sentence A` and type 1 corresponds to
|
||||
a `sentence B` token (see XLM paper for more details).
|
||||
`attention_mask`: an optional ``torch.LongTensor`` of shape [batch_size, sequence_length] with indices
|
||||
selected in [0, 1]. It's a mask to be used if the input sequence length is smaller than the max
|
||||
input sequence length in the current batch. It's the mask that we typically use for attention when
|
||||
a batch has varying length sentences.
|
||||
`cache`: TODO
|
||||
`labels`: TODO
|
||||
`head_mask`: an optional ``torch.Tensor`` of shape [num_heads] or [num_layers, num_heads] with indices between 0 and 1.
|
||||
It's a mask to be used to nullify some heads of the transformer. 1.0 => head is fully masked, 0.0 => head is not masked.
|
||||
|
||||
|
||||
Returns:
|
||||
A ``tuple(encoded_layers, pooled_output)``, with
|
||||
|
||||
``encoded_layers``: controlled by ``output_all_encoded_layers`` argument:
|
||||
|
||||
If ``output_all_encoded_layers=True``: outputs a list of the full sequences of encoded-hidden-states \
|
||||
at the end of each attention block (i.e. 12 full sequences for XLM-base, 24 for XLM-large), each \
|
||||
encoded-hidden-state is a ``torch.FloatTensor`` of size [batch_size, sequence_length, hidden_size],
|
||||
|
||||
If ``output_all_encoded_layers=False``: outputs only the full sequence of hidden-states corresponding \
|
||||
to the last attention block of shape [batch_size, sequence_length, hidden_size],
|
||||
|
||||
``pooled_output``: a ``torch.FloatTensor`` of size [batch_size, hidden_size] which is the output of a \
|
||||
classifier pre-trained on top of the hidden state associated to the first character of the \
|
||||
input (`CLS`) to train on the Next-Sentence task (see XLM's paper).
|
||||
|
||||
Example::
|
||||
|
||||
# Already been converted into WordPiece token ids
|
||||
input_ids = torch.LongTensor([[31, 51, 99], [15, 5, 0]])
|
||||
input_mask = torch.LongTensor([[1, 1, 1], [1, 1, 0]])
|
||||
token_type_ids = torch.LongTensor([[0, 0, 1], [0, 1, 0]])
|
||||
|
||||
all_encoder_layers, pooled_output = model(input_ids, token_type_ids, input_mask)
|
||||
# or
|
||||
all_encoder_layers, pooled_output = model.forward(input_ids, token_type_ids, input_mask)
|
||||
"""
|
||||
transformer_outputs = self.transformer(input_ids, lengths=lengths, positions=positions, token_type_ids=token_type_ids,
|
||||
transformer_outputs = self.transformer(input_ids, lengths=lengths, position_ids=position_ids, token_type_ids=token_type_ids,
|
||||
langs=langs, attention_mask=attention_mask, cache=cache, head_mask=head_mask)
|
||||
|
||||
output = transformer_outputs[0]
|
||||
@@ -795,25 +778,40 @@ class XLMWithLMHeadModel(XLMPreTrainedModel):
|
||||
return outputs
|
||||
|
||||
|
||||
@add_start_docstrings("""XLM Model with a sequence classification/regression head on top (a linear layer on top of
|
||||
the pooled output) e.g. for GLUE tasks. """,
|
||||
XLM_START_DOCSTRING, XLM_INPUTS_DOCSTRING)
|
||||
class XLMForSequenceClassification(XLMPreTrainedModel):
|
||||
"""XLM model ("XLM: Generalized Autoregressive Pretraining for Language Understanding").
|
||||
r"""
|
||||
**labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``:
|
||||
Labels for computing the sequence classification/regression loss.
|
||||
Indices should be in ``[0, ..., config.num_labels]``.
|
||||
If ``config.num_labels == 1`` a regression loss is computed (Mean-Square loss),
|
||||
If ``config.num_labels > 1`` a classification loss is computed (Cross-Entropy).
|
||||
|
||||
Args:
|
||||
`config`: a XLMConfig class instance with the configuration to build a new model
|
||||
`output_attentions`: If True, also output attentions weights computed by the model at each layer. Default: False
|
||||
`keep_multihead_output`: If True, saves output of the multi-head attention module with its gradient.
|
||||
This can be used to compute head importance metrics. Default: False
|
||||
`summary_type`: str, "last", "first", "mean", or "attn". The method
|
||||
to pool the input to get a vector representation. Default: last
|
||||
Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:
|
||||
**loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``:
|
||||
Classification (or regression if config.num_labels==1) loss.
|
||||
**logits**: ``torch.FloatTensor`` of shape ``(batch_size, config.num_labels)``
|
||||
Classification (or regression if config.num_labels==1) scores (before SoftMax).
|
||||
**attentions**: (`optional`, returned when ``config.output_attentions=True``)
|
||||
list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:
|
||||
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads.
|
||||
**hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)
|
||||
list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings)
|
||||
of shape ``(batch_size, sequence_length, hidden_size)``:
|
||||
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
|
||||
|
||||
Examples::
|
||||
|
||||
|
||||
Example::
|
||||
|
||||
config = modeling.XLMConfig(vocab_size_or_config_json_file=32000, d_model=768,
|
||||
n_layer=12, num_attention_heads=12, intermediate_size=3072)
|
||||
|
||||
model = modeling.XLMModel(config=config)
|
||||
>>> config = XLMConfig.from_pretrained('xlm-mlm-en-2048')
|
||||
>>> tokenizer = XLMTokenizer.from_pretrained('xlm-mlm-en-2048')
|
||||
>>>
|
||||
>>> model = XLMForSequenceClassification(config)
|
||||
>>> input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1
|
||||
>>> labels = torch.tensor([1]).unsqueeze(0) # Batch size 1
|
||||
>>> outputs = model(input_ids, labels=labels)
|
||||
>>> loss, logits = outputs[:2]
|
||||
|
||||
"""
|
||||
def __init__(self, config):
|
||||
@@ -825,42 +823,9 @@ class XLMForSequenceClassification(XLMPreTrainedModel):
|
||||
|
||||
self.apply(self.init_weights)
|
||||
|
||||
def forward(self, input_ids, lengths=None, positions=None, langs=None, token_type_ids=None,
|
||||
def forward(self, input_ids, lengths=None, position_ids=None, langs=None, token_type_ids=None,
|
||||
attention_mask=None, cache=None, labels=None, head_mask=None):
|
||||
"""
|
||||
Args:
|
||||
input_ids: TODO
|
||||
lengths: TODO
|
||||
positions: TODO
|
||||
langs: TODO
|
||||
token_type_ids: int32 Tensor in shape [bsz, len], the input segment IDs.
|
||||
attention_mask: [optional] float32 Tensor, SAME FUNCTION as `input_mask`
|
||||
but with 1 for real tokens and 0 for padding.
|
||||
Added for easy compatibility with the XLM model (which uses this negative masking).
|
||||
You can only uses one among `input_mask` and `attention_mask`
|
||||
cache: TODO
|
||||
labels: TODO
|
||||
head_mask: TODO
|
||||
|
||||
|
||||
Returns:
|
||||
A ``tuple(logits_or_loss, new_mems)``. If ``labels`` is ``None``, return token logits with shape
|
||||
[batch_size, sequence_length]. If it isn't ``None``, return the ``CrossEntropy`` loss with the targets.
|
||||
|
||||
``new_mems`` is a list (num layers) of updated mem states at the entry of each layer \
|
||||
each mem state is a ``torch.FloatTensor`` of size [self.config.mem_len, batch_size, self.config.d_model] \
|
||||
Note that the first two dimensions are transposed in ``mems`` with regards to ``input_ids`` and ``labels``
|
||||
|
||||
Example::
|
||||
|
||||
# Already been converted into WordPiece token ids
|
||||
input_ids = torch.LongTensor([[31, 51, 99], [15, 5, 0]])
|
||||
input_mask = torch.LongTensor([[1, 1, 1], [1, 1, 0]])
|
||||
token_type_ids = torch.LongTensor([[0, 0, 1], [0, 1, 0]])
|
||||
|
||||
all_encoder_layers, pooled_output = model(input_ids, token_type_ids, input_mask)
|
||||
"""
|
||||
transformer_outputs = self.transformer(input_ids, lengths=lengths, positions=positions, token_type_ids=token_type_ids,
|
||||
transformer_outputs = self.transformer(input_ids, lengths=lengths, position_ids=position_ids, token_type_ids=token_type_ids,
|
||||
langs=langs, attention_mask=attention_mask, cache=cache, head_mask=head_mask)
|
||||
|
||||
output = transformer_outputs[0]
|
||||
@@ -881,26 +846,53 @@ class XLMForSequenceClassification(XLMPreTrainedModel):
|
||||
return outputs
|
||||
|
||||
|
||||
@add_start_docstrings("""XLM Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layers on top of
|
||||
the hidden-states output to compute `span start logits` and `span end logits`). """,
|
||||
XLM_START_DOCSTRING, XLM_INPUTS_DOCSTRING)
|
||||
class XLMForQuestionAnswering(XLMPreTrainedModel):
|
||||
"""
|
||||
XLM model for Question Answering (span extraction).
|
||||
This module is composed of the XLM model with a linear layer on top of
|
||||
the sequence output that computes start_logits and end_logits
|
||||
r"""
|
||||
**start_positions**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``:
|
||||
Labels for position (index) of the start of the labelled span for computing the token classification loss.
|
||||
Positions are clamped to the length of the sequence (`sequence_length`).
|
||||
Position outside of the sequence are not taken into account for computing the loss.
|
||||
**end_positions**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``:
|
||||
Labels for position (index) of the end of the labelled span for computing the token classification loss.
|
||||
Positions are clamped to the length of the sequence (`sequence_length`).
|
||||
Position outside of the sequence are not taken into account for computing the loss.
|
||||
**is_impossible**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``:
|
||||
Labels whether a question has an answer or no answer (SQuAD 2.0)
|
||||
**cls_index**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``:
|
||||
Labels for position (index) of the classification token to use as input for computing plausibility of the answer.
|
||||
**p_mask**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``:
|
||||
Optional mask of tokens which can't be in answers (e.g. [CLS], [PAD], ...)
|
||||
|
||||
Args:
|
||||
`config`: a XLMConfig class instance with the configuration to build a new model
|
||||
`output_attentions`: If True, also output attentions weights computed by the model at each layer. Default: False
|
||||
`keep_multihead_output`: If True, saves output of the multi-head attention module with its gradient.
|
||||
This can be used to compute head importance metrics. Default: False
|
||||
Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:
|
||||
**loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``:
|
||||
Total span extraction loss is the sum of a Cross-Entropy for the start and end positions.
|
||||
**start_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length,)``
|
||||
Span-start scores (before SoftMax).
|
||||
**end_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length,)``
|
||||
Span-end scores (before SoftMax).
|
||||
**attentions**: (`optional`, returned when ``config.output_attentions=True``)
|
||||
list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:
|
||||
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads.
|
||||
**hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)
|
||||
list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings)
|
||||
of shape ``(batch_size, sequence_length, hidden_size)``:
|
||||
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
|
||||
|
||||
Examples::
|
||||
|
||||
>>> config = XLMConfig.from_pretrained('xlm-mlm-en-2048')
|
||||
>>> tokenizer = XLMTokenizer.from_pretrained('xlm-mlm-en-2048')
|
||||
>>>
|
||||
>>> model = XLMForQuestionAnswering(config)
|
||||
>>> input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1
|
||||
>>> start_positions = torch.tensor([1])
|
||||
>>> end_positions = torch.tensor([3])
|
||||
>>> outputs = model(input_ids, start_positions=start_positions, end_positions=end_positions)
|
||||
>>> loss, start_scores, end_scores = outputs[:2]
|
||||
|
||||
Example::
|
||||
|
||||
config = XLMConfig(vocab_size_or_config_json_file=32000, hidden_size=768,
|
||||
num_hidden_layers=12, num_attention_heads=12, intermediate_size=3072)
|
||||
|
||||
model = XLMForQuestionAnswering(config)
|
||||
"""
|
||||
def __init__(self, config):
|
||||
super(XLMForQuestionAnswering, self).__init__(config)
|
||||
@@ -910,63 +902,10 @@ class XLMForQuestionAnswering(XLMPreTrainedModel):
|
||||
|
||||
self.apply(self.init_weights)
|
||||
|
||||
def forward(self, input_ids, lengths=None, positions=None, langs=None, token_type_ids=None,
|
||||
def forward(self, input_ids, lengths=None, position_ids=None, langs=None, token_type_ids=None,
|
||||
attention_mask=None, cache=None, start_positions=None, end_positions=None,
|
||||
cls_index=None, is_impossible=None, p_mask=None, head_mask=None):
|
||||
|
||||
"""
|
||||
Performs a model forward pass. **Can be called by calling the class directly, once it has been instantiated.**
|
||||
|
||||
Args:
|
||||
input_ids: a ``torch.LongTensor`` of shape [batch_size, sequence_length]
|
||||
with the word token indices in the vocabulary(see the tokens preprocessing logic in the scripts
|
||||
`run_bert_extract_features.py`, `run_bert_classifier.py` and `run_bert_squad.py`)
|
||||
lengths: TODO
|
||||
positions: TODO
|
||||
langs: TODO
|
||||
token_type_ids: an optional ``torch.LongTensor`` of shape [batch_size, sequence_length] with the token
|
||||
types indices selected in [0, 1]. Type 0 corresponds to a `sentence A` and type 1 corresponds to
|
||||
a `sentence B` token (see XLM paper for more details).
|
||||
attention_mask: [optional] float32 Tensor, SAME FUNCTION as `input_mask`
|
||||
but with 1 for real tokens and 0 for padding.
|
||||
Added for easy compatibility with the XLM model (which uses this negative masking).
|
||||
You can only uses one among `input_mask` and `attention_mask`
|
||||
cache: TODO
|
||||
start_positions: position of the first token for the labeled span: ``torch.LongTensor`` of shape [batch_size].
|
||||
Positions are clamped to the length of the sequence and position outside of the sequence are not taken
|
||||
into account for computing the loss.
|
||||
end_positions: position of the last token for the labeled span: ``torch.LongTensor`` of shape [batch_size].
|
||||
Positions are clamped to the length of the sequence and position outside of the sequence are not taken
|
||||
into account for computing the loss.
|
||||
cls_index: TODO
|
||||
is_impossible: TODO
|
||||
p_mask: TODO
|
||||
head_mask: an optional ``torch.Tensor`` of shape [num_heads] or [num_layers, num_heads] with indices between 0 and 1.
|
||||
It's a mask to be used to nullify some heads of the transformer. 1.0 => head is fully masked, 0.0 => head is not masked.
|
||||
|
||||
Returns:
|
||||
Either the ``total_loss`` or a ``tuple(start_logits, end_logits)``
|
||||
|
||||
if ``start_positions`` and ``end_positions`` are not ``None``, \
|
||||
outputs the total_loss which is the sum of the CrossEntropy loss for the start and end token positions.
|
||||
|
||||
if ``start_positions`` or ``end_positions`` is ``None``:
|
||||
Outputs a ``tuple(start_logits, end_logits)`` which are the logits respectively for the start and end
|
||||
position tokens of shape [batch_size, sequence_length].
|
||||
|
||||
Example::
|
||||
|
||||
# Already been converted into WordPiece token ids
|
||||
input_ids = torch.LongTensor([[31, 51, 99], [15, 5, 0]])
|
||||
input_mask = torch.LongTensor([[1, 1, 1], [1, 1, 0]])
|
||||
token_type_ids = torch.LongTensor([[0, 0, 1], [0, 1, 0]])
|
||||
|
||||
start_logits, end_logits = model(input_ids, token_type_ids, input_mask)
|
||||
# or
|
||||
start_logits, end_logits = model.forward(input_ids, token_type_ids, input_mask)
|
||||
"""
|
||||
|
||||
transformer_outputs = self.transformer(input_ids, lengths=lengths, positions=positions, token_type_ids=token_type_ids,
|
||||
transformer_outputs = self.transformer(input_ids, lengths=lengths, position_ids=position_ids, token_type_ids=token_type_ids,
|
||||
langs=langs, attention_mask=attention_mask, cache=cache, head_mask=head_mask)
|
||||
|
||||
output = transformer_outputs[0]
|
||||
|
||||
Reference in New Issue
Block a user