From 003c4771290b00e6d14b871210c3a369edccaeed Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Tue, 26 May 2020 19:43:58 +0200 Subject: [PATCH] [GPT2, CTRL] Allow input of input_ids and past of variable length (#4581) * revert convenience method * clean docs a bit --- src/transformers/modeling_ctrl.py | 22 +++++----------------- src/transformers/modeling_gpt2.py | 21 +++++---------------- src/transformers/modeling_tf_ctrl.py | 14 +++++++------- src/transformers/modeling_tf_gpt2.py | 25 ++++++++----------------- 4 files changed, 25 insertions(+), 57 deletions(-) diff --git a/src/transformers/modeling_ctrl.py b/src/transformers/modeling_ctrl.py index 6c6a264cb8..bcdc9d77c6 100644 --- a/src/transformers/modeling_ctrl.py +++ b/src/transformers/modeling_ctrl.py @@ -208,9 +208,11 @@ CTRL_START_DOCSTRING = r""" CTRL_INPUTS_DOCSTRING = r""" Args: - input_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`): + input_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, input_ids_length)`): + :obj:`input_ids_length` = ``sequence_length`` if ``past`` is ``None`` else ``past[0].shape[-2]`` (``sequence_length`` of input past key value states). Indices of input sequence tokens in the vocabulary. - If `past` is used, optionally only the last `input_ids` have to be input (see `past`). + + If `past` is used, only input_ids that do not have their past calculated should be passed as input_ids. Indices can be obtained using :class:`transformers.CTRLTokenizer`. See :func:`transformers.PreTrainedTokenizer.encode` and @@ -220,9 +222,7 @@ CTRL_INPUTS_DOCSTRING = r""" past (:obj:`List[torch.FloatTensor]` of length :obj:`config.n_layers`): Contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model (see `past` output below). Can be used to speed up sequential decoding. - If `past` is used, the user can optionally input only the last `input_ids` - (those that don't have their past given to this model) of shape :obj:`(batch_size, 1)` - instead of all `input_ids` of shape :obj:`(batch_size, sequence_length)`. + The input_ids which have their past given to this model should not be passed as input ids as they have already been computed. attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`, defaults to :obj:`None`): Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``: @@ -233,7 +233,6 @@ CTRL_INPUTS_DOCSTRING = r""" Segment token indices to indicate first and second portions of the inputs. Indices are selected in ``[0, 1]``: ``0`` corresponds to a `sentence A` token, ``1`` corresponds to a `sentence B` token - If `past` is used, optionally only the last `token_type_ids` have to be input (see `past`). `What are token type IDs? <../glossary.html#token-type-ids>`_ position_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`, defaults to :obj:`None`): @@ -246,7 +245,6 @@ CTRL_INPUTS_DOCSTRING = r""" Mask values selected in ``[0, 1]``: :obj:`1` indicates the head is **not masked**, :obj:`0` indicates the head is **masked**. input_embeds (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`, defaults to :obj:`None`): - Optionally, instead of passing :obj:`input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. If `past` is used, optionally only the last `input_embeds` have to be input (see `past`). @@ -344,16 +342,6 @@ class CTRLModel(CTRLPreTrainedModel): """ - # If using past key value states, only the last tokens - # should be given as an input - if past is not None: - if input_ids is not None: - input_ids = input_ids[:, -1:] - if inputs_embeds is not None: - inputs_embeds = inputs_embeds[:, -1:] - if token_type_ids is not None: - token_type_ids = token_type_ids[:, -1:] - if input_ids is not None and inputs_embeds is not None: raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") elif input_ids is not None: diff --git a/src/transformers/modeling_gpt2.py b/src/transformers/modeling_gpt2.py index 756f30c001..9ef251f651 100644 --- a/src/transformers/modeling_gpt2.py +++ b/src/transformers/modeling_gpt2.py @@ -286,9 +286,11 @@ GPT2_START_DOCSTRING = r""" GPT2_INPUTS_DOCSTRING = r""" Args: - input_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`): + input_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, input_ids_length)`): + :obj:`input_ids_length` = ``sequence_length`` if ``past`` is ``None`` else ``past[0].shape[-2]`` (``sequence_length`` of input past key value states). Indices of input sequence tokens in the vocabulary. - If `past` is used, optionally only the last `input_ids` have to be input (see `past`). + + If `past` is used, only `input_ids` that do not have their past calculated should be passed as `input_ids`. Indices can be obtained using :class:`transformers.GPT2Tokenizer`. See :func:`transformers.PreTrainedTokenizer.encode` and @@ -299,7 +301,7 @@ GPT2_INPUTS_DOCSTRING = r""" past (:obj:`List[torch.FloatTensor]` of length :obj:`config.n_layers`): Contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model (see `past` output below). Can be used to speed up sequential decoding. - If `past` is used, the user can optionally input only the last `input_ids` (those that don't have their past given to this model) of shape :obj:`(batch_size, 1)` instead of all `input_ids` of shape :obj:`(batch_size, sequence_length)`. + The `input_ids` which have their past given to this model should not be passed as `input_ids` as they have already been computed. attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`, defaults to :obj:`None`): Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``: @@ -311,8 +313,6 @@ GPT2_INPUTS_DOCSTRING = r""" Segment token indices to indicate first and second portions of the inputs. Indices are selected in ``[0, 1]``: ``0`` corresponds to a `sentence A` token, ``1`` corresponds to a `sentence B` token - If `past` is used, optionally only the last `token_type_ids` have to be input (see `past`). - `What are token type IDs? <../glossary.html#token-type-ids>`_ position_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`, defaults to :obj:`None`): Indices of positions of each input sequence tokens in the position embeddings. @@ -324,7 +324,6 @@ GPT2_INPUTS_DOCSTRING = r""" Mask values selected in ``[0, 1]``: :obj:`1` indicates the head is **not masked**, :obj:`0` indicates the head is **masked**. input_embeds (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`, defaults to :obj:`None`): - Optionally, instead of passing :obj:`input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. If `past` is used, optionally only the last `input_embeds` have to be input (see `past`). @@ -410,16 +409,6 @@ class GPT2Model(GPT2PreTrainedModel): """ - # If using past key value states, only the last tokens - # should be given as an input - if past is not None: - if input_ids is not None: - input_ids = input_ids[:, -1:] - if inputs_embeds is not None: - inputs_embeds = inputs_embeds[:, -1:] - if token_type_ids is not None: - token_type_ids = token_type_ids[:, -1:] - if input_ids is not None and inputs_embeds is not None: raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") elif input_ids is not None: diff --git a/src/transformers/modeling_tf_ctrl.py b/src/transformers/modeling_tf_ctrl.py index 13484e870f..6c16705b3e 100644 --- a/src/transformers/modeling_tf_ctrl.py +++ b/src/transformers/modeling_tf_ctrl.py @@ -411,9 +411,12 @@ CTRL_START_DOCSTRING = r""" CTRL_INPUTS_DOCSTRING = r""" Args: - input_ids (:obj:`Numpy array` or :obj:`tf.Tensor` of shape :obj:`(batch_size, sequence_length)`): + input_ids (:obj:`Numpy array` or :obj:`tf.Tensor` of shape :obj:`(batch_size, input_ids_length)`): + :obj:`input_ids_length` = ``sequence_length`` if ``past`` is ``None`` else ``past[0].shape[-2]`` (``sequence_length`` of input past key value states). + Indices of input sequence tokens in the vocabulary. - If `past` is used, optionally only the last `input_ids` have to be input (see `past`). + + If `past` is used, only input_ids that do not have their past calculated should be passed as input_ids (see `past`). Indices can be obtained using :class:`transformers.CTRLTokenizer`. See :func:`transformers.PreTrainedTokenizer.encode` and @@ -423,9 +426,8 @@ CTRL_INPUTS_DOCSTRING = r""" past (:obj:`List[tf.Tensor]` of length :obj:`config.n_layers`): Contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model (see `past` output below). Can be used to speed up sequential decoding. - If `past` is used, the user can optionally input only the last `input_ids` - (those that don't have their past given to this model) of shape :obj:`(batch_size, 1)` - instead of all `input_ids` of shape :obj:`(batch_size, sequence_length)`. + The token ids which have their past given to this model + should not be passed as input ids as they have already been computed. attention_mask (:obj:`tf.Tensor` or :obj:`Numpy array` of shape :obj:`(batch_size, sequence_length)`, `optional`, defaults to :obj:`None`): Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``: @@ -436,7 +438,6 @@ CTRL_INPUTS_DOCSTRING = r""" Segment token indices to indicate first and second portions of the inputs. Indices are selected in ``[0, 1]``: ``0`` corresponds to a `sentence A` token, ``1`` corresponds to a `sentence B` token - If `past` is used, optionally only the last `token_type_ids` have to be input (see `past`). `What are token type IDs? <../glossary.html#token-type-ids>`_ position_ids (:obj:`tf.Tensor` or :obj:`Numpy array` of shape :obj:`(batch_size, sequence_length)`, `optional`, defaults to :obj:`None`): @@ -452,7 +453,6 @@ CTRL_INPUTS_DOCSTRING = r""" Optionally, instead of passing :obj:`input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. - If `past` is used, optionally only the last `input_embeds` have to be input (see `past`). use_cache (:obj:`bool`): If `use_cache` is True, `past` key value states are returned and can be used to speed up decoding (see `past`). Defaults to `True`. diff --git a/src/transformers/modeling_tf_gpt2.py b/src/transformers/modeling_tf_gpt2.py index 8dffb131a6..887609ac9c 100644 --- a/src/transformers/modeling_tf_gpt2.py +++ b/src/transformers/modeling_tf_gpt2.py @@ -284,16 +284,6 @@ class TFGPT2MainLayer(tf.keras.layers.Layer): else: input_ids = inputs - # If using past key value states, only the last tokens - # should be given as an input - if past is not None: - if input_ids is not None: - input_ids = input_ids[:, -1:] - if inputs_embeds is not None: - inputs_embeds = inputs_embeds[:, -1:] - if token_type_ids is not None: - token_type_ids = token_type_ids[:, -1:] - if input_ids is not None and inputs_embeds is not None: raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") elif input_ids is not None: @@ -431,9 +421,11 @@ GPT2_START_DOCSTRING = r""" GPT2_INPUTS_DOCSTRING = r""" Args: - input_ids (:obj:`Numpy array` or :obj:`tf.Tensor` of shape :obj:`(batch_size, sequence_length)`): + input_ids (:obj:`Numpy array` or :obj:`tf.Tensor` of shape :obj:`(batch_size, input_ids_length)`): + :obj:`input_ids_length` = ``sequence_length`` if ``past`` is ``None`` else ``past[0].shape[-2]`` (``sequence_length`` of input past key value states). Indices of input sequence tokens in the vocabulary. - If `past` is used, optionally only the last `input_ids` have to be input (see `past`). + + If `past` is used, only `input_ids` that do not have their past calculated should be passed as `input_ids`. Indices can be obtained using :class:`transformers.GPT2Tokenizer`. See :func:`transformers.PreTrainedTokenizer.encode` and @@ -442,8 +434,9 @@ GPT2_INPUTS_DOCSTRING = r""" `What are input IDs? <../glossary.html#input-ids>`__ past (:obj:`List[tf.Tensor]` of length :obj:`config.n_layers`): Contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model - (see `past` output below). Can be used to speed up sequential decoding. The token ids which have their past given to this model - should not be passed as input ids as they have already been computed. + (see `past` output below). Can be used to speed up sequential decoding. + The token ids which have their past given to this model + should not be passed as `input_ids` as they have already been computed. attention_mask (:obj:`tf.Tensor` or :obj:`Numpy array` of shape :obj:`(batch_size, sequence_length)`, `optional`, defaults to :obj:`None`): Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``: @@ -454,7 +447,6 @@ GPT2_INPUTS_DOCSTRING = r""" Segment token indices to indicate first and second portions of the inputs. Indices are selected in ``[0, 1]``: ``0`` corresponds to a `sentence A` token, ``1`` corresponds to a `sentence B` token - If `past` is used, optionally only the last `token_type_ids` have to be input (see `past`). `What are token type IDs? <../glossary.html#token-type-ids>`_ position_ids (:obj:`tf.Tensor` or :obj:`Numpy array` of shape :obj:`(batch_size, sequence_length)`, `optional`, defaults to :obj:`None`): @@ -470,7 +462,6 @@ GPT2_INPUTS_DOCSTRING = r""" Optionally, instead of passing :obj:`input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. - If `past` is used, optionally only the last `input_embeds` have to be input (see `past`). training (:obj:`boolean`, `optional`, defaults to :obj:`False`): Whether to activate dropout modules (if set to :obj:`True`) during training or to de-activate them (if set to :obj:`False`) for evaluation. @@ -639,7 +630,7 @@ class TFGPT2DoubleHeadsModel(TFGPT2PreTrainedModel): past (:obj:`List[tf.Tensor]` of length :obj:`config.n_layers` with each tensor of shape :obj:`(2, batch_size, num_heads, sequence_length, embed_size_per_head)`): Contains pre-computed hidden-states (key and values in the attention blocks). Can be used (see `past` input) to speed up sequential decoding. The token ids which have their past given to this model - should not be passed as input ids as they have already been computed. + should not be passed as `input_ids` as they have already been computed. hidden_states (:obj:`tuple(tf.Tensor)`, `optional`, returned when ``config.output_hidden_states=True``): Tuple of :obj:`tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape :obj:`(batch_size, sequence_length, hidden_size)`.