From c30139a013f8d65dc691efaac107691bb798419e Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 30 Apr 2019 10:45:26 +0200 Subject: [PATCH 01/32] add special tokens to gpt-2 --- pytorch_pretrained_bert/modeling_gpt2.py | 60 +++++++++++++++++++--- pytorch_pretrained_bert/modeling_openai.py | 6 +-- tests/modeling_gpt2_test.py | 10 ++-- 3 files changed, 62 insertions(+), 14 deletions(-) diff --git a/pytorch_pretrained_bert/modeling_gpt2.py b/pytorch_pretrained_bert/modeling_gpt2.py index 063c525d98..05a748d43c 100644 --- a/pytorch_pretrained_bert/modeling_gpt2.py +++ b/pytorch_pretrained_bert/modeling_gpt2.py @@ -107,6 +107,7 @@ class GPT2Config(object): def __init__( self, vocab_size_or_config_json_file=50257, + n_special=0, n_positions=1024, n_ctx=1024, n_embd=768, @@ -119,6 +120,7 @@ class GPT2Config(object): Args: vocab_size_or_config_json_file: Vocabulary size of `inputs_ids` in `GPT2Model` or a configuration json file. + n_special: The number of special tokens to learn during fine-tuning ('[SEP]', '[CLF]', ...) n_positions: Number of positional embeddings. n_ctx: Size of the causal mask (usually same as n_positions). n_embd: Dimensionality of the embeddings and hidden states. @@ -137,6 +139,7 @@ class GPT2Config(object): self.__dict__[key] = value elif isinstance(vocab_size_or_config_json_file, int): self.vocab_size = vocab_size_or_config_json_file + self.n_special = n_special self.n_ctx = n_ctx self.n_positions = n_positions self.n_embd = n_embd @@ -150,6 +153,10 @@ class GPT2Config(object): "or the path to a pretrained model config file (str)" ) + @property + def total_tokens_embeddings(self): + return self.vocab_size + self.n_special + @classmethod def from_dict(cls, json_object): """Constructs a `GPT2Config` from a Python dictionary of parameters.""" @@ -290,11 +297,12 @@ class GPT2LMHead(nn.Module): def __init__(self, model_embeddings_weights, config): super(GPT2LMHead, self).__init__() self.n_embd = config.n_embd + embed_shape = model_embeddings_weights.shape + self.decoder = nn.Linear(embed_shape[1], embed_shape[0], bias=False) self.set_embeddings_weights(model_embeddings_weights) def set_embeddings_weights(self, model_embeddings_weights): embed_shape = model_embeddings_weights.shape - self.decoder = nn.Linear(embed_shape[1], embed_shape[0], bias=False) self.decoder.weight = model_embeddings_weights # Tied weights def forward(self, hidden_state): @@ -345,7 +353,7 @@ class GPT2PreTrainedModel(nn.Module): ) self.config = config - def set_tied(self): + def set_num_special_tokens(self, num_special_tokens): pass def init_weights(self, module): @@ -475,14 +483,32 @@ class GPT2PreTrainedModel(nn.Module): "Error(s) in loading state_dict for {}:\n\t{}".format(model.__class__.__name__, "\n\t".join(error_msgs)) ) - # Make sure we are still sharing the output and input embeddings after loading weights - model.set_tied() + # Add additional embeddings for special tokens if needed + # This step also make sure we are still sharing the output and input embeddings after loading weights + model.set_num_special_tokens(num_special_tokens if num_special_tokens is not None else config.n_special) return model class GPT2Model(GPT2PreTrainedModel): """OpenAI GPT-2 model ("Language Models are Unsupervised Multitask Learners"). + GPT-2 use a single embedding matrix to store the word and special embeddings. + Special tokens embeddings are additional tokens that are not pre-trained: [SEP], [CLS]... + Special tokens need to be trained during the fine-tuning if you use them. + The number of special embeddings can be controled using the `set_num_special_tokens(num_special_tokens)` function. + + The embeddings are ordered as follow in the token embeddings matrice: + [0, ---------------------- + ... -> word embeddings + config.vocab_size - 1, ______________________ + config.vocab_size, + ... -> special embeddings + config.vocab_size + config.n_special - 1] ______________________ + + where total_tokens_embeddings can be obtained as config.total_tokens_embeddings and is: + total_tokens_embeddings = config.vocab_size + config.n_special + You should use the associate indices to index the embeddings. + Params: config: a GPT2Config class instance with the configuration to build a new model @@ -529,6 +555,20 @@ class GPT2Model(GPT2PreTrainedModel): self.apply(self.init_weights) + def set_num_special_tokens(self, num_special_tokens): + " Update input embeddings with new embedding matrice if needed " + if self.config.n_special == num_special_tokens: + return + # Update config + self.config.n_special = num_special_tokens + # Build new embeddings and initialize all new embeddings (in particular the special tokens) + old_embed = self.wte + self.wte = nn.Embedding(self.config.total_tokens_embeddings, self.config.n_embd) + self.wte.to(old_embed.weight.device) + self.init_weights(self.wte) + # Copy word embeddings from the previous weights + self.wte.weight.data[:self.config.vocab_size, :] = old_embed.weight.data[:self.config.vocab_size, :] + def forward(self, input_ids, position_ids=None, token_type_ids=None, past=None): if past is None: past_length = 0 @@ -610,9 +650,11 @@ class GPT2LMHeadModel(GPT2PreTrainedModel): self.lm_head = GPT2LMHead(self.transformer.wte.weight, config) self.apply(self.init_weights) - def set_tied(self): - """ Make sure we are sharing the embeddings + def set_num_special_tokens(self, num_special_tokens): + """ Update input and output embeddings with new embedding matrice + Make sure we are sharing the embeddings """ + self.transformer.set_num_special_tokens(num_special_tokens) self.lm_head.set_embeddings_weights(self.transformer.wte.weight) def forward(self, input_ids, position_ids=None, token_type_ids=None, lm_labels=None, past=None): @@ -687,9 +729,11 @@ class GPT2DoubleHeadsModel(GPT2PreTrainedModel): self.multiple_choice_head = GPT2MultipleChoiceHead(config) self.apply(self.init_weights) - def set_tied(self): - """ Make sure we are sharing the embeddings + def set_num_special_tokens(self, num_special_tokens): + """ Update input and output embeddings with new embedding matrice + Make sure we are sharing the embeddings """ + self.transformer.set_num_special_tokens(num_special_tokens) self.lm_head.set_embeddings_weights(self.transformer.wte.weight) def forward(self, input_ids, mc_token_ids, lm_labels=None, mc_labels=None, token_type_ids=None, position_ids=None, past=None): diff --git a/pytorch_pretrained_bert/modeling_openai.py b/pytorch_pretrained_bert/modeling_openai.py index f956462ddb..7ac3782b42 100644 --- a/pytorch_pretrained_bert/modeling_openai.py +++ b/pytorch_pretrained_bert/modeling_openai.py @@ -344,11 +344,12 @@ class OpenAIGPTLMHead(nn.Module): def __init__(self, model_embeddings_weights, config): super(OpenAIGPTLMHead, self).__init__() self.n_embd = config.n_embd + embed_shape = model_embeddings_weights.shape + self.decoder = nn.Linear(embed_shape[1], embed_shape[0], bias=False) self.set_embeddings_weights(model_embeddings_weights) def set_embeddings_weights(self, model_embeddings_weights): embed_shape = model_embeddings_weights.shape - self.decoder = nn.Linear(embed_shape[1], embed_shape[0], bias=False) self.decoder.weight = model_embeddings_weights # Tied weights def forward(self, hidden_state): @@ -592,8 +593,7 @@ class OpenAIGPTModel(OpenAIGPTPreTrainedModel): def __init__(self, config): super(OpenAIGPTModel, self).__init__(config) - num_tokens = config.vocab_size + config.n_special - self.tokens_embed = nn.Embedding(num_tokens, config.n_embd) + self.tokens_embed = nn.Embedding(config.total_tokens_embeddings, config.n_embd) self.positions_embed = nn.Embedding(config.n_positions, config.n_embd) self.drop = nn.Dropout(config.embd_pdrop) block = Block(config.n_ctx, config, scale=True) diff --git a/tests/modeling_gpt2_test.py b/tests/modeling_gpt2_test.py index 8f4581b37f..6804b794c5 100644 --- a/tests/modeling_gpt2_test.py +++ b/tests/modeling_gpt2_test.py @@ -41,6 +41,7 @@ class GPT2ModelTest(unittest.TestCase): use_token_type_ids=True, use_labels=True, vocab_size=99, + n_special=1, n_positions=33, n_embd=32, n_layer=5, @@ -58,6 +59,7 @@ class GPT2ModelTest(unittest.TestCase): self.use_token_type_ids = use_token_type_ids self.use_labels = use_labels self.vocab_size = vocab_size + self.n_special = n_special self.n_positions = n_positions self.n_embd = n_embd self.n_layer = n_layer @@ -69,7 +71,8 @@ class GPT2ModelTest(unittest.TestCase): self.scope = scope def prepare_config_and_inputs(self): - input_ids = GPT2ModelTest.ids_tensor([self.batch_size, self.n_choices, self.seq_length], self.vocab_size) + total_num_tokens = self.vocab_size + self.n_special + input_ids = GPT2ModelTest.ids_tensor([self.batch_size, self.n_choices, self.seq_length], total_num_tokens) position_ids = None if self.use_position_ids: @@ -90,6 +93,7 @@ class GPT2ModelTest(unittest.TestCase): config = GPT2Config( vocab_size_or_config_json_file=self.vocab_size, + n_special=self.n_special, n_positions=self.n_positions, n_embd=self.n_embd, n_layer=self.n_layer, @@ -130,7 +134,7 @@ class GPT2ModelTest(unittest.TestCase): return outputs def check_gpt2_lm_head_output(self, result): - total_voc = self.vocab_size + total_voc = self.n_special + self.vocab_size self.parent.assertListEqual( list(result["lm_logits"].size()), [self.batch_size, self.n_choices, self.seq_length, total_voc]) @@ -157,7 +161,7 @@ class GPT2ModelTest(unittest.TestCase): return outputs def check_gpt2_double_heads_output(self, result): - total_voc = self.vocab_size + total_voc = self.n_special + self.vocab_size self.parent.assertListEqual( list(result["lm_logits"].size()), [self.batch_size, self.n_choices, self.seq_length, total_voc]) From 1f5fc95b6831a2b41c45f51b2eaf9aa92d100ab7 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 30 Apr 2019 11:05:26 +0200 Subject: [PATCH 02/32] add code coverage --- .circleci/config.yml | 9 ++++++--- .coveragerc | 8 ++++++++ 2 files changed, 14 insertions(+), 3 deletions(-) create mode 100644 .coveragerc diff --git a/.circleci/config.yml b/.circleci/config.yml index 7296e07ca3..6f4434228a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,9 +7,11 @@ jobs: steps: - checkout - run: sudo pip install --progress-bar off . - - run: sudo pip install pytest ftfy spacy + - run: sudo pip install pytest codecov pytest-cov + - run: sudo pip install spacy ftfy==4.4.3 - run: sudo python -m spacy download en - run: python -m pytest -sv tests/ --runslow + - run: codecov build_py2: working_directory: ~/pytorch-pretrained-BERT docker: @@ -17,10 +19,11 @@ jobs: steps: - checkout - run: sudo pip install --progress-bar off . - - run: sudo pip install pytest spacy - - run: sudo pip install ftfy==4.4.3 + - run: sudo pip install pytest codecov pytest-cov + - run: sudo pip install spacy ftfy==4.4.3 - run: sudo python -m spacy download en - run: python -m pytest -sv tests/ --runslow + - run: codecov workflows: version: 2 build_and_test: diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000000..fe05dda9a8 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,8 @@ +[run] +source=pytorch_pretrained_bert +[report] +exclude_lines = + pragma: no cover + raise + except + register_parameter \ No newline at end of file From e79ceb15331ddadbe0f0ccb857218b1ba2cca368 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 30 Apr 2019 11:05:54 +0200 Subject: [PATCH 03/32] gpt-2 special tokens --- pytorch_pretrained_bert/modeling_gpt2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_pretrained_bert/modeling_gpt2.py b/pytorch_pretrained_bert/modeling_gpt2.py index 05a748d43c..5537f93f66 100644 --- a/pytorch_pretrained_bert/modeling_gpt2.py +++ b/pytorch_pretrained_bert/modeling_gpt2.py @@ -547,7 +547,7 @@ class GPT2Model(GPT2PreTrainedModel): def __init__(self, config): super(GPT2Model, self).__init__(config) - self.wte = nn.Embedding(config.vocab_size, config.n_embd) + self.wte = nn.Embedding(config.total_tokens_embeddings, config.n_embd) self.wpe = nn.Embedding(config.n_positions, config.n_embd) block = Block(config.n_ctx, config, scale=True) self.h = nn.ModuleList([copy.deepcopy(block) for _ in range(config.n_layer)]) From 80f53f7380c7072263ea3af460a0464017a81e03 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 30 Apr 2019 11:10:22 +0200 Subject: [PATCH 04/32] gpt-2 from_pretrained can use special tokens --- pytorch_pretrained_bert/modeling_gpt2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_pretrained_bert/modeling_gpt2.py b/pytorch_pretrained_bert/modeling_gpt2.py index 5537f93f66..37c5a2d9fb 100644 --- a/pytorch_pretrained_bert/modeling_gpt2.py +++ b/pytorch_pretrained_bert/modeling_gpt2.py @@ -371,7 +371,7 @@ class GPT2PreTrainedModel(nn.Module): @classmethod def from_pretrained( - cls, pretrained_model_name_or_path, state_dict=None, cache_dir=None, from_tf=False, *inputs, **kwargs + cls, pretrained_model_name_or_path, num_special_tokens=None, state_dict=None, cache_dir=None, from_tf=False, *inputs, **kwargs ): """ Instantiate a GPT2PreTrainedModel from a pre-trained model file or a pytorch state dict. From cd110835a051fe859214d067211f525f705e4354 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 30 Apr 2019 11:35:40 +0200 Subject: [PATCH 05/32] coverage in circle-ci --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6f4434228a..04adf715e0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,7 +10,7 @@ jobs: - run: sudo pip install pytest codecov pytest-cov - run: sudo pip install spacy ftfy==4.4.3 - run: sudo python -m spacy download en - - run: python -m pytest -sv tests/ --runslow + - run: python -m pytest -sv tests/ --runslow --cov - run: codecov build_py2: working_directory: ~/pytorch-pretrained-BERT @@ -22,7 +22,7 @@ jobs: - run: sudo pip install pytest codecov pytest-cov - run: sudo pip install spacy ftfy==4.4.3 - run: sudo python -m spacy download en - - run: python -m pytest -sv tests/ --runslow + - run: python -m pytest -sv tests/ --runslow --cov - run: codecov workflows: version: 2 From db98a4a48b79bd11e24acd4b9ea2fdf17075f9e5 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Wed, 1 May 2019 11:40:48 +0200 Subject: [PATCH 06/32] gpt-2 tokenizer --- pytorch_pretrained_bert/tokenization_gpt2.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pytorch_pretrained_bert/tokenization_gpt2.py b/pytorch_pretrained_bert/tokenization_gpt2.py index 07777292a3..8ffd7a68e2 100644 --- a/pytorch_pretrained_bert/tokenization_gpt2.py +++ b/pytorch_pretrained_bert/tokenization_gpt2.py @@ -221,7 +221,10 @@ class GPT2Tokenizer(object): """ Tokenize a string. """ bpe_tokens = [] for token in re.findall(self.pat, text): - token = ''.join(self.byte_encoder[ord(b)] for b in token) + if sys.version_info[0] == 2: + token = ''.join(self.byte_encoder[ord(b)] for b in token) + else: + token = ''.join(self.byte_encoder[b] for b in token.encode('utf-8')) bpe_tokens.extend(bpe_token for bpe_token in self.bpe(token).split(' ')) return bpe_tokens From e211785ada54a761bd1b0f10e3878539d9d1d72d Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 2 May 2019 18:31:26 +0200 Subject: [PATCH 07/32] extract attention weights from GPT --- pytorch_pretrained_bert/modeling_openai.py | 48 +++++++++++++++++----- 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/pytorch_pretrained_bert/modeling_openai.py b/pytorch_pretrained_bert/modeling_openai.py index 7ac3782b42..77e9cda349 100644 --- a/pytorch_pretrained_bert/modeling_openai.py +++ b/pytorch_pretrained_bert/modeling_openai.py @@ -253,7 +253,7 @@ class Conv1D(nn.Module): class Attention(nn.Module): - def __init__(self, nx, n_ctx, config, scale=False): + def __init__(self, nx, n_ctx, config, scale=False, output_attentions=False): super(Attention, self).__init__() n_state = nx # in Attention: n_state=768 (nx=n_embd) # [switch nx => n_state from Block to Attention to keep identical to TF implem] @@ -262,6 +262,7 @@ class Attention(nn.Module): self.n_head = config.n_head self.split_size = n_state self.scale = scale + self.output_attentions = output_attentions self.c_attn = Conv1D(n_state * 3, 1, nx) self.c_proj = Conv1D(n_state, 1, nx) self.attn_dropout = nn.Dropout(config.attn_pdrop) @@ -278,6 +279,8 @@ class Attention(nn.Module): w = nn.Softmax(dim=-1)(w) w = self.attn_dropout(w) + if self.output_attentions: + return w, torch.matmul(w, v) return torch.matmul(w, v) def merge_heads(self, x): @@ -300,9 +303,13 @@ class Attention(nn.Module): key = self.split_heads(key, k=True) value = self.split_heads(value) a = self._attn(query, key, value) + if self.output_attentions: + attentions, a = a a = self.merge_heads(a) a = self.c_proj(a) a = self.resid_dropout(a) + if self.output_attentions: + return attentions, a return a @@ -322,19 +329,24 @@ class MLP(nn.Module): class Block(nn.Module): - def __init__(self, n_ctx, config, scale=False): + def __init__(self, n_ctx, config, scale=False, output_attentions=False): super(Block, self).__init__() nx = config.n_embd - self.attn = Attention(nx, n_ctx, config, scale) + self.output_attentions = output_attentions + self.attn = Attention(nx, n_ctx, config, scale, output_attentions) self.ln_1 = LayerNorm(nx, eps=config.layer_norm_epsilon) self.mlp = MLP(4 * nx, config) self.ln_2 = LayerNorm(nx, eps=config.layer_norm_epsilon) def forward(self, x): a = self.attn(x) + if self.output_attentions: + attentions, a = a n = self.ln_1(x + a) m = self.mlp(n) h = self.ln_2(n + m) + if self.output_attentions: + return attentions, h return h @@ -591,12 +603,13 @@ class OpenAIGPTModel(OpenAIGPTPreTrainedModel): ``` """ - def __init__(self, config): + def __init__(self, config, output_attentions=False): super(OpenAIGPTModel, self).__init__(config) + self.output_attentions = output_attentions self.tokens_embed = nn.Embedding(config.total_tokens_embeddings, config.n_embd) self.positions_embed = nn.Embedding(config.n_positions, config.n_embd) self.drop = nn.Dropout(config.embd_pdrop) - block = Block(config.n_ctx, config, scale=True) + block = Block(config.n_ctx, config, scale=True, output_attentions=output_attentions) self.h = nn.ModuleList([copy.deepcopy(block) for _ in range(config.n_layer)]) self.apply(self.init_weights) @@ -639,9 +652,16 @@ class OpenAIGPTModel(OpenAIGPTPreTrainedModel): # Add the position information to the input embeddings # h = e.sum(dim=2) hidden_states = inputs_embeds + position_embeds + token_type_embeds + all_attentions = [] for block in self.h: - hidden_states = block(hidden_states) + if self.output_attentions: + attentions, hidden_states = block(hidden_states) + all_attentions.append(attentions) + else: + hidden_states = block(hidden_states) output_shape = input_shape + (hidden_states.size(-1),) + if self.output_attentions: + return all_attentions, hidden_states.view(*output_shape) return hidden_states.view(*output_shape) @@ -701,9 +721,9 @@ class OpenAIGPTLMHeadModel(OpenAIGPTPreTrainedModel): ``` """ - def __init__(self, config): + def __init__(self, config, output_attentions=False): super(OpenAIGPTLMHeadModel, self).__init__(config) - self.transformer = OpenAIGPTModel(config) + self.transformer = OpenAIGPTModel(config, output_attentions=output_attentions) self.lm_head = OpenAIGPTLMHead(self.transformer.tokens_embed.weight, config) self.apply(self.init_weights) @@ -716,6 +736,8 @@ class OpenAIGPTLMHeadModel(OpenAIGPTPreTrainedModel): def forward(self, input_ids, position_ids=None, token_type_ids=None, lm_labels=None): hidden_states = self.transformer(input_ids, position_ids, token_type_ids) + if self.transformer.output_attentions: + all_attentions, hidden_states = hidden_states lm_logits = self.lm_head(hidden_states) if lm_labels is not None: # Shift so that tokens < n predict n @@ -726,6 +748,8 @@ class OpenAIGPTLMHeadModel(OpenAIGPTPreTrainedModel): loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1)) return loss + if self.transformer.output_attentions: + return all_attentions, lm_logits return lm_logits @@ -790,9 +814,9 @@ class OpenAIGPTDoubleHeadsModel(OpenAIGPTPreTrainedModel): ``` """ - def __init__(self, config): + def __init__(self, config, output_attentions=False): super(OpenAIGPTDoubleHeadsModel, self).__init__(config) - self.transformer = OpenAIGPTModel(config) + self.transformer = OpenAIGPTModel(config, output_attentions=output_attentions) self.lm_head = OpenAIGPTLMHead(self.transformer.tokens_embed.weight, config) self.multiple_choice_head = OpenAIGPTMultipleChoiceHead(config) self.apply(self.init_weights) @@ -806,6 +830,8 @@ class OpenAIGPTDoubleHeadsModel(OpenAIGPTPreTrainedModel): def forward(self, input_ids, mc_token_ids, lm_labels=None, mc_labels=None, token_type_ids=None, position_ids=None): hidden_states = self.transformer(input_ids, position_ids, token_type_ids) + if self.transformer.output_attentions: + all_attentions, hidden_states = hidden_states lm_logits = self.lm_head(hidden_states) mc_logits = self.multiple_choice_head(hidden_states, mc_token_ids) losses = [] @@ -819,4 +845,6 @@ class OpenAIGPTDoubleHeadsModel(OpenAIGPTPreTrainedModel): losses.append(loss_fct(mc_logits.view(-1, mc_logits.size(-1)), mc_labels.view(-1))) if losses: return losses + if self.transformer.output_attentions: + return all_attentions, lm_logits, mc_logits return lm_logits, mc_logits From d1b6979aa57b3a214a329fd693a67a1369d65fc9 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 7 May 2019 16:25:53 +0200 Subject: [PATCH 08/32] GPT-2 option to avoid predicting special tokens --- pytorch_pretrained_bert/modeling_gpt2.py | 26 +++++++++++--------- pytorch_pretrained_bert/tokenization_gpt2.py | 4 +-- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/pytorch_pretrained_bert/modeling_gpt2.py b/pytorch_pretrained_bert/modeling_gpt2.py index 37c5a2d9fb..1c579de83c 100644 --- a/pytorch_pretrained_bert/modeling_gpt2.py +++ b/pytorch_pretrained_bert/modeling_gpt2.py @@ -115,6 +115,7 @@ class GPT2Config(object): n_head=12, layer_norm_epsilon=1e-5, initializer_range=0.02, + predict_special_tokens=True ): """Constructs GPT2Config. @@ -130,6 +131,7 @@ class GPT2Config(object): layer_norm_epsilon: epsilon to use in the layer norm layers initializer_range: The sttdev of the truncated_normal_initializer for initializing all weight matrices. + predict_special_tokens: should we predict special tokens (when the model has a LM head) """ if isinstance(vocab_size_or_config_json_file, str) or (sys.version_info[0] == 2 and isinstance(vocab_size_or_config_json_file, unicode)): @@ -147,6 +149,7 @@ class GPT2Config(object): self.n_head = n_head self.layer_norm_epsilon = layer_norm_epsilon self.initializer_range = initializer_range + self.predict_special_tokens = predict_special_tokens else: raise ValueError( "First argument must be either a vocabulary size (int)" @@ -297,18 +300,20 @@ class GPT2LMHead(nn.Module): def __init__(self, model_embeddings_weights, config): super(GPT2LMHead, self).__init__() self.n_embd = config.n_embd + self.vocab_size = config.vocab_size + self.predict_special_tokens = config.predict_special_tokens embed_shape = model_embeddings_weights.shape self.decoder = nn.Linear(embed_shape[1], embed_shape[0], bias=False) self.set_embeddings_weights(model_embeddings_weights) - def set_embeddings_weights(self, model_embeddings_weights): - embed_shape = model_embeddings_weights.shape + def set_embeddings_weights(self, model_embeddings_weights, predict_special_tokens=True): + self.predict_special_tokens = predict_special_tokens self.decoder.weight = model_embeddings_weights # Tied weights def forward(self, hidden_state): - # Truncated Language modeling logits (we remove the last token) - # h_trunc = h[:, :-1].contiguous().view(-1, self.n_embd) lm_logits = self.decoder(hidden_state) + if not self.predict_special_tokens: + lm_logits = lm_logits[..., :self.vocab_size] return lm_logits @@ -353,9 +358,6 @@ class GPT2PreTrainedModel(nn.Module): ) self.config = config - def set_num_special_tokens(self, num_special_tokens): - pass - def init_weights(self, module): """ Initialize the weights. """ @@ -650,12 +652,13 @@ class GPT2LMHeadModel(GPT2PreTrainedModel): self.lm_head = GPT2LMHead(self.transformer.wte.weight, config) self.apply(self.init_weights) - def set_num_special_tokens(self, num_special_tokens): + def set_num_special_tokens(self, num_special_tokens, predict_special_tokens=True): """ Update input and output embeddings with new embedding matrice Make sure we are sharing the embeddings """ + self.config.predict_special_tokens = self.transformer.config.predict_special_tokens = predict_special_tokens self.transformer.set_num_special_tokens(num_special_tokens) - self.lm_head.set_embeddings_weights(self.transformer.wte.weight) + self.lm_head.set_embeddings_weights(self.transformer.wte.weight, predict_special_tokens=predict_special_tokens) def forward(self, input_ids, position_ids=None, token_type_ids=None, lm_labels=None, past=None): hidden_states, presents = self.transformer(input_ids, position_ids, token_type_ids, past) @@ -729,12 +732,13 @@ class GPT2DoubleHeadsModel(GPT2PreTrainedModel): self.multiple_choice_head = GPT2MultipleChoiceHead(config) self.apply(self.init_weights) - def set_num_special_tokens(self, num_special_tokens): + def set_num_special_tokens(self, num_special_tokens, predict_special_tokens=True): """ Update input and output embeddings with new embedding matrice Make sure we are sharing the embeddings """ + self.config.predict_special_tokens = self.transformer.config.predict_special_tokens = predict_special_tokens self.transformer.set_num_special_tokens(num_special_tokens) - self.lm_head.set_embeddings_weights(self.transformer.wte.weight) + self.lm_head.set_embeddings_weights(self.transformer.wte.weight, predict_special_tokens=predict_special_tokens) def forward(self, input_ids, mc_token_ids, lm_labels=None, mc_labels=None, token_type_ids=None, position_ids=None, past=None): hidden_states, presents = self.transformer(input_ids, position_ids, token_type_ids, past) diff --git a/pytorch_pretrained_bert/tokenization_gpt2.py b/pytorch_pretrained_bert/tokenization_gpt2.py index 8ffd7a68e2..c18589b7b0 100644 --- a/pytorch_pretrained_bert/tokenization_gpt2.py +++ b/pytorch_pretrained_bert/tokenization_gpt2.py @@ -263,8 +263,8 @@ class GPT2Tokenizer(object): def encode(self, text): return self.convert_tokens_to_ids(self.tokenize(text)) - def decode(self, tokens): - text = ''.join([self.decoder[token] for token in tokens]) + def decode(self, tokens, skip_special_tokens=False): + text = ''.join(self.convert_ids_to_tokens(tokens, skip_special_tokens=skip_special_tokens)) text = bytearray([self.byte_decoder[c] for c in text]).decode('utf-8', errors=self.errors) return text From ce863365459d5c7b96fd1b5917bc9fb00f509d18 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 7 May 2019 16:47:22 +0200 Subject: [PATCH 09/32] add predict_special_tokens option to GPT also --- pytorch_pretrained_bert/modeling_openai.py | 26 +++++++++++++--------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/pytorch_pretrained_bert/modeling_openai.py b/pytorch_pretrained_bert/modeling_openai.py index 77e9cda349..be33eda1c6 100644 --- a/pytorch_pretrained_bert/modeling_openai.py +++ b/pytorch_pretrained_bert/modeling_openai.py @@ -143,6 +143,7 @@ class OpenAIGPTConfig(object): attn_pdrop=0.1, layer_norm_epsilon=1e-5, initializer_range=0.02, + predict_special_tokens=True ): """Constructs OpenAIGPTConfig. @@ -165,6 +166,7 @@ class OpenAIGPTConfig(object): layer_norm_epsilon: epsilon to use in the layer norm layers initializer_range: The sttdev of the truncated_normal_initializer for initializing all weight matrices. + predict_special_tokens: should we predict special tokens (when the model has a LM head) """ if isinstance(vocab_size_or_config_json_file, str) or (sys.version_info[0] == 2 and isinstance(vocab_size_or_config_json_file, unicode)): @@ -186,6 +188,7 @@ class OpenAIGPTConfig(object): self.attn_pdrop = attn_pdrop self.layer_norm_epsilon = layer_norm_epsilon self.initializer_range = initializer_range + self.predict_special_tokens = predict_special_tokens else: raise ValueError( "First argument must be either a vocabulary size (int)" @@ -356,18 +359,21 @@ class OpenAIGPTLMHead(nn.Module): def __init__(self, model_embeddings_weights, config): super(OpenAIGPTLMHead, self).__init__() self.n_embd = config.n_embd + self.vocab_size = config.vocab_size + self.predict_special_tokens = config.predict_special_tokens embed_shape = model_embeddings_weights.shape self.decoder = nn.Linear(embed_shape[1], embed_shape[0], bias=False) self.set_embeddings_weights(model_embeddings_weights) - def set_embeddings_weights(self, model_embeddings_weights): + def set_embeddings_weights(self, model_embeddings_weights, predict_special_tokens=True): + self.predict_special_tokens = predict_special_tokens embed_shape = model_embeddings_weights.shape self.decoder.weight = model_embeddings_weights # Tied weights def forward(self, hidden_state): - # Truncated Language modeling logits (we remove the last token) - # h_trunc = h[:, :-1].contiguous().view(-1, self.n_embd) lm_logits = self.decoder(hidden_state) + if not self.predict_special_tokens: + lm_logits = lm_logits[..., :self.vocab_size] return lm_logits @@ -428,9 +434,6 @@ class OpenAIGPTPreTrainedModel(nn.Module): if isinstance(module, nn.Linear) and module.bias is not None: module.bias.data.zero_() - def set_num_special_tokens(self, num_special_tokens): - pass - @classmethod def from_pretrained( cls, pretrained_model_name_or_path, num_special_tokens=None, state_dict=None, cache_dir=None, from_tf=False, *inputs, **kwargs @@ -613,7 +616,6 @@ class OpenAIGPTModel(OpenAIGPTPreTrainedModel): self.h = nn.ModuleList([copy.deepcopy(block) for _ in range(config.n_layer)]) self.apply(self.init_weights) - # nn.init.normal_(self.embed.weight, std=0.02) def set_num_special_tokens(self, num_special_tokens): " Update input embeddings with new embedding matrice if needed " @@ -727,12 +729,13 @@ class OpenAIGPTLMHeadModel(OpenAIGPTPreTrainedModel): self.lm_head = OpenAIGPTLMHead(self.transformer.tokens_embed.weight, config) self.apply(self.init_weights) - def set_num_special_tokens(self, num_special_tokens): + def set_num_special_tokens(self, num_special_tokens, predict_special_tokens=True): """ Update input and output embeddings with new embedding matrice Make sure we are sharing the embeddings """ + self.config.predict_special_tokens = self.transformer.config.predict_special_tokens = predict_special_tokens self.transformer.set_num_special_tokens(num_special_tokens) - self.lm_head.set_embeddings_weights(self.transformer.tokens_embed.weight) + self.lm_head.set_embeddings_weights(self.transformer.tokens_embed.weight, predict_special_tokens=predict_special_tokens) def forward(self, input_ids, position_ids=None, token_type_ids=None, lm_labels=None): hidden_states = self.transformer(input_ids, position_ids, token_type_ids) @@ -821,12 +824,13 @@ class OpenAIGPTDoubleHeadsModel(OpenAIGPTPreTrainedModel): self.multiple_choice_head = OpenAIGPTMultipleChoiceHead(config) self.apply(self.init_weights) - def set_num_special_tokens(self, num_special_tokens): + def set_num_special_tokens(self, num_special_tokens, predict_special_tokens=True): """ Update input and output embeddings with new embedding matrice Make sure we are sharing the embeddings """ + self.config.predict_special_tokens = self.transformer.config.predict_special_tokens = predict_special_tokens self.transformer.set_num_special_tokens(num_special_tokens) - self.lm_head.set_embeddings_weights(self.transformer.tokens_embed.weight) + self.lm_head.set_embeddings_weights(self.transformer.tokens_embed.weight, predict_special_tokens=predict_special_tokens) def forward(self, input_ids, mc_token_ids, lm_labels=None, mc_labels=None, token_type_ids=None, position_ids=None): hidden_states = self.transformer(input_ids, position_ids, token_type_ids) From ea9dbea9d5b65ca6333e378ea0a8a288399640c2 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 7 May 2019 23:27:18 +0200 Subject: [PATCH 10/32] update GPT2 loss computation for more flexbility --- pytorch_pretrained_bert/modeling_gpt2.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/pytorch_pretrained_bert/modeling_gpt2.py b/pytorch_pretrained_bert/modeling_gpt2.py index 1c579de83c..ca5a38524a 100644 --- a/pytorch_pretrained_bert/modeling_gpt2.py +++ b/pytorch_pretrained_bert/modeling_gpt2.py @@ -336,6 +336,7 @@ class GPT2MultipleChoiceHead(nn.Module): # (bsz, num_choices, 1, hidden_size) multiple_choice_h = hidden_states.gather(2, mc_token_ids).squeeze(2) # (bsz, num_choices, hidden_size) + multiple_choice_h = self.dropout(multiple_choice_h.transpose(1, 2)).transpose(1, 2) multiple_choice_logits = self.linear(multiple_choice_h).squeeze(-1) # (bsz, num_choices) return multiple_choice_logits @@ -665,9 +666,8 @@ class GPT2LMHeadModel(GPT2PreTrainedModel): lm_logits = self.lm_head(hidden_states) if lm_labels is not None: # Shift so that tokens < n predict n - shift_logits = lm_logits[:, :-1].contiguous() - shift_labels = lm_labels[:, 1:].contiguous() - + shift_logits = lm_logits[..., :-1, :].contiguous() + shift_labels = lm_labels[..., 1:].contiguous() # Flatten the tokens loss_fct = CrossEntropyLoss(ignore_index=-1) loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), @@ -746,11 +746,10 @@ class GPT2DoubleHeadsModel(GPT2PreTrainedModel): mc_logits = self.multiple_choice_head(hidden_states, mc_token_ids) losses = [] if lm_labels is not None: - shift_logits = lm_logits[:, :-1].contiguous() - shift_labels = lm_labels[:, 1:].contiguous() + shift_logits = lm_logits[..., :-1, :].contiguous() + shift_labels = lm_labels[..., 1:].contiguous() loss_fct = CrossEntropyLoss(ignore_index=-1) - losses.append(loss_fct(shift_logits.view(-1, - shift_logits.size(-1)), shift_labels.view(-1))) + losses.append(loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1))) if mc_labels is not None: loss_fct = CrossEntropyLoss() losses.append(loss_fct(mc_logits.view(-1, mc_logits.size(-1)), mc_labels.view(-1))) From 0efc4ab632ddf6f726bdf3ce1b9350f8ec183a2f Mon Sep 17 00:00:00 2001 From: thomwolf Date: Wed, 8 May 2019 10:41:35 +0200 Subject: [PATCH 11/32] adding dropout to GPT-2 and embedding dropout to GPT --- pytorch_pretrained_bert/modeling_gpt2.py | 22 +++++++++++++++++++++- pytorch_pretrained_bert/modeling_openai.py | 5 ++--- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/pytorch_pretrained_bert/modeling_gpt2.py b/pytorch_pretrained_bert/modeling_gpt2.py index ca5a38524a..7623e4ddad 100644 --- a/pytorch_pretrained_bert/modeling_gpt2.py +++ b/pytorch_pretrained_bert/modeling_gpt2.py @@ -113,6 +113,9 @@ class GPT2Config(object): n_embd=768, n_layer=12, n_head=12, + resid_pdrop=0.1, + embd_pdrop=0.1, + attn_pdrop=0.1, layer_norm_epsilon=1e-5, initializer_range=0.02, predict_special_tokens=True @@ -129,6 +132,11 @@ class GPT2Config(object): n_head: Number of attention heads for each attention layer in the Transformer encoder. layer_norm_epsilon: epsilon to use in the layer norm layers + resid_pdrop: The dropout probabilitiy for all fully connected + layers in the embeddings, encoder, and pooler. + attn_pdrop: The dropout ratio for the attention + probabilities. + embd_pdrop: The dropout ratio for the embeddings. initializer_range: The sttdev of the truncated_normal_initializer for initializing all weight matrices. predict_special_tokens: should we predict special tokens (when the model has a LM head) @@ -147,6 +155,9 @@ class GPT2Config(object): self.n_embd = n_embd self.n_layer = n_layer self.n_head = n_head + self.resid_pdrop = resid_pdrop + self.embd_pdrop = embd_pdrop + self.attn_pdrop = attn_pdrop self.layer_norm_epsilon = layer_norm_epsilon self.initializer_range = initializer_range self.predict_special_tokens = predict_special_tokens @@ -221,6 +232,8 @@ class Attention(nn.Module): self.scale = scale self.c_attn = Conv1D(n_state * 3, nx) self.c_proj = Conv1D(n_state, nx) + self.attn_dropout = nn.Dropout(config.attn_pdrop) + self.resid_dropout = nn.Dropout(config.resid_pdrop) def _attn(self, q, k, v): w = torch.matmul(q, k) @@ -231,6 +244,7 @@ class Attention(nn.Module): w = w * b - 1e4 * (1 - b) w = nn.Softmax(dim=-1)(w) + w = self.attn_dropout(w) return torch.matmul(w, v) def merge_heads(self, x): @@ -260,6 +274,7 @@ class Attention(nn.Module): a = self._attn(query, key, value) a = self.merge_heads(a) a = self.c_proj(a) + a = self.resid_dropout(a) return a, present @@ -270,11 +285,12 @@ class MLP(nn.Module): self.c_fc = Conv1D(n_state, nx) self.c_proj = Conv1D(nx, n_state) self.act = gelu + self.dropout = nn.Dropout(config.resid_pdrop) def forward(self, x): h = self.act(self.c_fc(x)) h2 = self.c_proj(h) - return h2 + return self.dropout(h2) class Block(nn.Module): @@ -323,6 +339,7 @@ class GPT2MultipleChoiceHead(nn.Module): def __init__(self, config): super(GPT2MultipleChoiceHead, self).__init__() self.n_embd = config.n_embd + self.dropout = nn.Dropout2d(config.resid_pdrop) # To reproduce the noise_shape parameter of TF implementation self.linear = nn.Linear(config.n_embd, 1) nn.init.normal_(self.linear.weight, std=0.02) @@ -552,6 +569,7 @@ class GPT2Model(GPT2PreTrainedModel): super(GPT2Model, self).__init__(config) self.wte = nn.Embedding(config.total_tokens_embeddings, config.n_embd) self.wpe = nn.Embedding(config.n_positions, config.n_embd) + self.drop = nn.Dropout(config.embd_pdrop) block = Block(config.n_ctx, config, scale=True) self.h = nn.ModuleList([copy.deepcopy(block) for _ in range(config.n_layer)]) self.ln_f = LayerNorm(config.n_embd, eps=config.layer_norm_epsilon) @@ -594,6 +612,8 @@ class GPT2Model(GPT2PreTrainedModel): else: token_type_embeds = 0 hidden_states = inputs_embeds + position_embeds + token_type_embeds + hidden_states = self.drop(hidden_states) + presents = [] for block, layer_past in zip(self.h, past): hidden_states, present = block(hidden_states, layer_past) diff --git a/pytorch_pretrained_bert/modeling_openai.py b/pytorch_pretrained_bert/modeling_openai.py index be33eda1c6..769a6b3288 100644 --- a/pytorch_pretrained_bert/modeling_openai.py +++ b/pytorch_pretrained_bert/modeling_openai.py @@ -383,7 +383,6 @@ class OpenAIGPTMultipleChoiceHead(nn.Module): def __init__(self, config): super(OpenAIGPTMultipleChoiceHead, self).__init__() self.n_embd = config.n_embd - # self.multiple_choice_token = multiple_choice_token self.dropout = nn.Dropout2d(config.resid_pdrop) # To reproduce the noise_shape parameter of TF implementation self.linear = nn.Linear(config.n_embd, 1) @@ -651,9 +650,9 @@ class OpenAIGPTModel(OpenAIGPTPreTrainedModel): token_type_embeds = self.tokens_embed(token_type_ids) else: token_type_embeds = 0 - # Add the position information to the input embeddings - # h = e.sum(dim=2) hidden_states = inputs_embeds + position_embeds + token_type_embeds + hidden_states = self.drop(hidden_states) + all_attentions = [] for block in self.h: if self.output_attentions: From 366a3b02857a1fdae447358cc76bf8abf1bf11eb Mon Sep 17 00:00:00 2001 From: thomwolf Date: Wed, 8 May 2019 21:43:51 +0200 Subject: [PATCH 12/32] clean up in tokenization --- pytorch_pretrained_bert/modeling_gpt2.py | 6 ++++-- pytorch_pretrained_bert/tokenization_gpt2.py | 9 ++++++++- pytorch_pretrained_bert/tokenization_openai.py | 2 +- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/pytorch_pretrained_bert/modeling_gpt2.py b/pytorch_pretrained_bert/modeling_gpt2.py index 7623e4ddad..0554442b7f 100644 --- a/pytorch_pretrained_bert/modeling_gpt2.py +++ b/pytorch_pretrained_bert/modeling_gpt2.py @@ -39,8 +39,10 @@ from .modeling import BertLayerNorm as LayerNorm logger = logging.getLogger(__name__) -PRETRAINED_MODEL_ARCHIVE_MAP = {"gpt2": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-pytorch_model.bin"} -PRETRAINED_CONFIG_ARCHIVE_MAP = {"gpt2": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-config.json"} +PRETRAINED_MODEL_ARCHIVE_MAP = {"gpt2": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-pytorch_model.bin", + "gpt2-medium": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-medium-pytorch_model.bin"} +PRETRAINED_CONFIG_ARCHIVE_MAP = {"gpt2": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-config.json", + "gpt2-medium": "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-medium-config.json"} def load_tf_weights_in_gpt2(model, gpt2_checkpoint_path): """ Load tf checkpoints in a pytorch model diff --git a/pytorch_pretrained_bert/tokenization_gpt2.py b/pytorch_pretrained_bert/tokenization_gpt2.py index c18589b7b0..c66af3ff13 100644 --- a/pytorch_pretrained_bert/tokenization_gpt2.py +++ b/pytorch_pretrained_bert/tokenization_gpt2.py @@ -37,9 +37,11 @@ logger = logging.getLogger(__name__) PRETRAINED_VOCAB_ARCHIVE_MAP = { 'gpt2': "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-vocab.json", + 'gpt2-medium': "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-medium-vocab.json", } PRETRAINED_MERGES_ARCHIVE_MAP = { 'gpt2': "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-merges.txt", + 'gpt2-medium': "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-medium-merges.txt", } PRETRAINED_VOCAB_POSITIONAL_EMBEDDINGS_SIZE_MAP = { 'gpt2': 1024, @@ -263,9 +265,14 @@ class GPT2Tokenizer(object): def encode(self, text): return self.convert_tokens_to_ids(self.tokenize(text)) - def decode(self, tokens, skip_special_tokens=False): + def decode(self, tokens, skip_special_tokens=False, clean_up_tokenization_spaces=True): text = ''.join(self.convert_ids_to_tokens(tokens, skip_special_tokens=skip_special_tokens)) text = bytearray([self.byte_decoder[c] for c in text]).decode('utf-8', errors=self.errors) + if clean_up_tokenization_spaces: + text = text.replace('', '') + text = text.replace(' .', '.').replace(' ?', '?').replace(' !', '!').replace(' ,', ',' + ).replace(" ' ", "'").replace(" n't", "n't").replace(" 'm", "'m").replace(" do not", " don't" + ).replace(" 's", "'s").replace(" 've", "'ve").replace(" 're", "'re") return text def save_vocabulary(self, vocab_path): diff --git a/pytorch_pretrained_bert/tokenization_openai.py b/pytorch_pretrained_bert/tokenization_openai.py index 214a476ce9..c68e247e1e 100644 --- a/pytorch_pretrained_bert/tokenization_openai.py +++ b/pytorch_pretrained_bert/tokenization_openai.py @@ -272,7 +272,7 @@ class OpenAIGPTTokenizer(object): out_string = ''.join(tokens).replace('', ' ').strip() if clean_up_tokenization_spaces: out_string = out_string.replace('', '') - out_string = out_string.replace(' .', '.').replace(' ?', '?').replace(' !', '!').replace(' ,', ',').replace(' ,', ',' + out_string = out_string.replace(' .', '.').replace(' ?', '?').replace(' !', '!').replace(' ,', ',' ).replace(" ' ", "'").replace(" n't", "n't").replace(" 'm", "'m").replace(" do not", " don't" ).replace(" 's", "'s").replace(" 've", "'ve").replace(" 're", "'re") return out_string From 275179a0033c9e065f97ea3bd3f0a8d7e0c4a17a Mon Sep 17 00:00:00 2001 From: thomwolf Date: Wed, 8 May 2019 22:24:42 +0200 Subject: [PATCH 13/32] output attentions in GPT-2 --- pytorch_pretrained_bert/modeling_gpt2.py | 60 +++++++++++++++++++----- 1 file changed, 47 insertions(+), 13 deletions(-) diff --git a/pytorch_pretrained_bert/modeling_gpt2.py b/pytorch_pretrained_bert/modeling_gpt2.py index 0554442b7f..d462fe04ef 100644 --- a/pytorch_pretrained_bert/modeling_gpt2.py +++ b/pytorch_pretrained_bert/modeling_gpt2.py @@ -223,7 +223,7 @@ class Conv1D(nn.Module): class Attention(nn.Module): - def __init__(self, nx, n_ctx, config, scale=False): + def __init__(self, nx, n_ctx, config, scale=False, output_attentions=False): super(Attention, self).__init__() n_state = nx # in Attention: n_state=768 (nx=n_embd) # [switch nx => n_state from Block to Attention to keep identical to TF implem] @@ -232,6 +232,7 @@ class Attention(nn.Module): self.n_head = config.n_head self.split_size = n_state self.scale = scale + self.output_attentions = output_attentions self.c_attn = Conv1D(n_state * 3, nx) self.c_proj = Conv1D(n_state, nx) self.attn_dropout = nn.Dropout(config.attn_pdrop) @@ -247,6 +248,8 @@ class Attention(nn.Module): w = nn.Softmax(dim=-1)(w) w = self.attn_dropout(w) + if self.output_attentions: + return w, torch.matmul(w, v) return torch.matmul(w, v) def merge_heads(self, x): @@ -274,9 +277,13 @@ class Attention(nn.Module): value = torch.cat((past_value, value), dim=-2) present = torch.stack((key.transpose(-2, -1), value)) # transpose to have same shapes for stacking a = self._attn(query, key, value) + if self.output_attentions: + attentions, a = a a = self.merge_heads(a) a = self.c_proj(a) a = self.resid_dropout(a) + if self.output_attentions: + return attentions, a, present return a, present @@ -296,19 +303,26 @@ class MLP(nn.Module): class Block(nn.Module): - def __init__(self, n_ctx, config, scale=False): + def __init__(self, n_ctx, config, scale=False, output_attentions=False): super(Block, self).__init__() nx = config.n_embd + self.output_attentions = output_attentions self.ln_1 = LayerNorm(nx, eps=config.layer_norm_epsilon) - self.attn = Attention(nx, n_ctx, config, scale) + self.attn = Attention(nx, n_ctx, config, scale, output_attentions) self.ln_2 = LayerNorm(nx, eps=config.layer_norm_epsilon) self.mlp = MLP(4 * nx, config) def forward(self, x, layer_past=None): - a, present = self.attn(self.ln_1(x), layer_past=layer_past) + output_attn = self.attn(self.ln_1(x), layer_past=layer_past) + if self.output_attentions: + attentions, a, present = output_attn + else: + a, present = output_attn x = x + a m = self.mlp(self.ln_2(x)) x = x + m + if self.output_attentions: + return attentions, x, present return x, present @@ -567,12 +581,13 @@ class GPT2Model(GPT2PreTrainedModel): ``` """ - def __init__(self, config): + def __init__(self, config, output_attentions=False): super(GPT2Model, self).__init__(config) + self.output_attentions = output_attentions self.wte = nn.Embedding(config.total_tokens_embeddings, config.n_embd) self.wpe = nn.Embedding(config.n_positions, config.n_embd) self.drop = nn.Dropout(config.embd_pdrop) - block = Block(config.n_ctx, config, scale=True) + block = Block(config.n_ctx, config, scale=True, output_attentions=output_attentions) self.h = nn.ModuleList([copy.deepcopy(block) for _ in range(config.n_layer)]) self.ln_f = LayerNorm(config.n_embd, eps=config.layer_norm_epsilon) @@ -617,11 +632,18 @@ class GPT2Model(GPT2PreTrainedModel): hidden_states = self.drop(hidden_states) presents = [] + all_attentions = [] for block, layer_past in zip(self.h, past): - hidden_states, present = block(hidden_states, layer_past) + if self.output_attentions: + attentions, hidden_states, present = block(hidden_states, layer_past) + all_attentions.append(attentions) + else: + hidden_states, present = block(hidden_states, layer_past) presents.append(present) hidden_states = self.ln_f(hidden_states) output_shape = input_shape + (hidden_states.size(-1),) + if self.output_attentions: + return all_attentions, hidden_states.view(*output_shape), presents return hidden_states.view(*output_shape), presents @@ -669,9 +691,9 @@ class GPT2LMHeadModel(GPT2PreTrainedModel): ``` """ - def __init__(self, config): + def __init__(self, config, output_attentions=False): super(GPT2LMHeadModel, self).__init__(config) - self.transformer = GPT2Model(config) + self.transformer = GPT2Model(config, output_attentions=output_attentions) self.lm_head = GPT2LMHead(self.transformer.wte.weight, config) self.apply(self.init_weights) @@ -684,7 +706,11 @@ class GPT2LMHeadModel(GPT2PreTrainedModel): self.lm_head.set_embeddings_weights(self.transformer.wte.weight, predict_special_tokens=predict_special_tokens) def forward(self, input_ids, position_ids=None, token_type_ids=None, lm_labels=None, past=None): - hidden_states, presents = self.transformer(input_ids, position_ids, token_type_ids, past) + transformer_output = self.transformer(input_ids, position_ids, token_type_ids, past) + if self.transformer.output_attentions: + all_attentions, hidden_states, presents = transformer_output + else: + hidden_states, presents = transformer_output lm_logits = self.lm_head(hidden_states) if lm_labels is not None: # Shift so that tokens < n predict n @@ -695,6 +721,8 @@ class GPT2LMHeadModel(GPT2PreTrainedModel): loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1)) return loss + if self.transformer.output_attentions: + return all_attentions, lm_logits, presents return lm_logits, presents @@ -747,9 +775,9 @@ class GPT2DoubleHeadsModel(GPT2PreTrainedModel): ``` """ - def __init__(self, config): + def __init__(self, config, output_attentions=False): super(GPT2DoubleHeadsModel, self).__init__(config) - self.transformer = GPT2Model(config) + self.transformer = GPT2Model(config, output_attentions=output_attentions) self.lm_head = GPT2LMHead(self.transformer.wte.weight, config) self.multiple_choice_head = GPT2MultipleChoiceHead(config) self.apply(self.init_weights) @@ -763,7 +791,11 @@ class GPT2DoubleHeadsModel(GPT2PreTrainedModel): self.lm_head.set_embeddings_weights(self.transformer.wte.weight, predict_special_tokens=predict_special_tokens) def forward(self, input_ids, mc_token_ids, lm_labels=None, mc_labels=None, token_type_ids=None, position_ids=None, past=None): - hidden_states, presents = self.transformer(input_ids, position_ids, token_type_ids, past) + transformer_output = self.transformer(input_ids, position_ids, token_type_ids, past) + if self.transformer.output_attentions: + all_attentions, hidden_states, presents = transformer_output + else: + hidden_states, presents = transformer_output lm_logits = self.lm_head(hidden_states) mc_logits = self.multiple_choice_head(hidden_states, mc_token_ids) losses = [] @@ -777,4 +809,6 @@ class GPT2DoubleHeadsModel(GPT2PreTrainedModel): losses.append(loss_fct(mc_logits.view(-1, mc_logits.size(-1)), mc_labels.view(-1))) if losses: return losses + if self.transformer.output_attentions: + return all_attentions, lm_logits, mc_logits, presents return lm_logits, mc_logits, presents From 3bf3f9596fe58b244ceed9e4a8d454e9717b8ea2 Mon Sep 17 00:00:00 2001 From: "samuel.broscheit" Date: Sun, 12 May 2019 00:13:45 +0200 Subject: [PATCH 14/32] Fixing the issues reported in https://github.com/huggingface/pytorch-pretrained-BERT/issues/556 Reason for issue was that optimzation steps where computed from example size, which is different from actual size of dataloader when an example is chunked into multiple instances. Solution in this pull request is to compute num_optimization_steps directly from len(data_loader). --- examples/run_classifier.py | 39 +++++++++++------------ examples/run_openai_gpt.py | 2 +- examples/run_squad.py | 63 +++++++++++++++++++------------------- examples/run_swag.py | 28 ++++++++--------- 4 files changed, 66 insertions(+), 66 deletions(-) diff --git a/examples/run_classifier.py b/examples/run_classifier.py index 1ebdf9fd51..eff48ca97d 100644 --- a/examples/run_classifier.py +++ b/examples/run_classifier.py @@ -25,6 +25,7 @@ import random import sys import numpy as np +import math import torch from torch.utils.data import (DataLoader, RandomSampler, SequentialSampler, TensorDataset) @@ -739,8 +740,25 @@ def main(): num_train_optimization_steps = None if args.do_train: train_examples = processor.get_train_examples(args.data_dir) - num_train_optimization_steps = int( - len(train_examples) / args.train_batch_size / args.gradient_accumulation_steps) * args.num_train_epochs + train_features = convert_examples_to_features( + train_examples, label_list, args.max_seq_length, tokenizer, output_mode) + all_input_ids = torch.tensor([f.input_ids for f in train_features], dtype=torch.long) + all_input_mask = torch.tensor([f.input_mask for f in train_features], dtype=torch.long) + all_segment_ids = torch.tensor([f.segment_ids for f in train_features], dtype=torch.long) + + if output_mode == "classification": + all_label_ids = torch.tensor([f.label_id for f in train_features], dtype=torch.long) + elif output_mode == "regression": + all_label_ids = torch.tensor([f.label_id for f in train_features], dtype=torch.float) + + train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids) + if args.local_rank == -1: + train_sampler = RandomSampler(train_data) + else: + train_sampler = DistributedSampler(train_data) + train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=args.train_batch_size) + + num_train_optimization_steps = len(train_dataloader) / args.gradient_accumulation_steps * args.num_train_epochs if args.local_rank != -1: num_train_optimization_steps = num_train_optimization_steps // torch.distributed.get_world_size() @@ -798,27 +816,10 @@ def main(): nb_tr_steps = 0 tr_loss = 0 if args.do_train: - train_features = convert_examples_to_features( - train_examples, label_list, args.max_seq_length, tokenizer, output_mode) logger.info("***** Running training *****") logger.info(" Num examples = %d", len(train_examples)) logger.info(" Batch size = %d", args.train_batch_size) logger.info(" Num steps = %d", num_train_optimization_steps) - all_input_ids = torch.tensor([f.input_ids for f in train_features], dtype=torch.long) - all_input_mask = torch.tensor([f.input_mask for f in train_features], dtype=torch.long) - all_segment_ids = torch.tensor([f.segment_ids for f in train_features], dtype=torch.long) - - if output_mode == "classification": - all_label_ids = torch.tensor([f.label_id for f in train_features], dtype=torch.long) - elif output_mode == "regression": - all_label_ids = torch.tensor([f.label_id for f in train_features], dtype=torch.float) - - train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids) - if args.local_rank == -1: - train_sampler = RandomSampler(train_data) - else: - train_sampler = DistributedSampler(train_data) - train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=args.train_batch_size) model.train() for _ in trange(int(args.num_train_epochs), desc="Epoch"): diff --git a/examples/run_openai_gpt.py b/examples/run_openai_gpt.py index f0a14f7e87..ac5c474491 100644 --- a/examples/run_openai_gpt.py +++ b/examples/run_openai_gpt.py @@ -190,7 +190,7 @@ def main(): {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01}, {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0} ] - num_train_optimization_steps = len(train_data) * args.num_train_epochs // args.train_batch_size + num_train_optimization_steps = len(train_dataloader) * args.num_train_epochs optimizer = OpenAIAdam(optimizer_grouped_parameters, lr=args.learning_rate, warmup=args.warmup_proportion, diff --git a/examples/run_squad.py b/examples/run_squad.py index 249aff7f8a..ae163afc91 100644 --- a/examples/run_squad.py +++ b/examples/run_squad.py @@ -899,8 +899,37 @@ def main(): if args.do_train: train_examples = read_squad_examples( input_file=args.train_file, is_training=True, version_2_with_negative=args.version_2_with_negative) - num_train_optimization_steps = int( - len(train_examples) / args.train_batch_size / args.gradient_accumulation_steps) * args.num_train_epochs + cached_train_features_file = args.train_file+'_{0}_{1}_{2}_{3}'.format( + list(filter(None, args.bert_model.split('/'))).pop(), str(args.max_seq_length), str(args.doc_stride), str(args.max_query_length)) + train_features = None + try: + with open(cached_train_features_file, "rb") as reader: + train_features = pickle.load(reader) + except: + train_features = convert_examples_to_features( + examples=train_examples, + tokenizer=tokenizer, + max_seq_length=args.max_seq_length, + doc_stride=args.doc_stride, + max_query_length=args.max_query_length, + is_training=True) + if args.local_rank == -1 or torch.distributed.get_rank() == 0: + logger.info(" Saving train features into cached file %s", cached_train_features_file) + with open(cached_train_features_file, "wb") as writer: + pickle.dump(train_features, writer) + all_input_ids = torch.tensor([f.input_ids for f in train_features], dtype=torch.long) + all_input_mask = torch.tensor([f.input_mask for f in train_features], dtype=torch.long) + all_segment_ids = torch.tensor([f.segment_ids for f in train_features], dtype=torch.long) + all_start_positions = torch.tensor([f.start_position for f in train_features], dtype=torch.long) + all_end_positions = torch.tensor([f.end_position for f in train_features], dtype=torch.long) + train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, + all_start_positions, all_end_positions) + if args.local_rank == -1: + train_sampler = RandomSampler(train_data) + else: + train_sampler = DistributedSampler(train_data) + train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=args.train_batch_size) + num_train_optimization_steps = len(train_dataloader) / args.gradient_accumulation_steps * args.num_train_epochs if args.local_rank != -1: num_train_optimization_steps = num_train_optimization_steps // torch.distributed.get_world_size() @@ -960,41 +989,11 @@ def main(): global_step = 0 if args.do_train: - cached_train_features_file = args.train_file+'_{0}_{1}_{2}_{3}'.format( - list(filter(None, args.bert_model.split('/'))).pop(), str(args.max_seq_length), str(args.doc_stride), str(args.max_query_length)) - train_features = None - try: - with open(cached_train_features_file, "rb") as reader: - train_features = pickle.load(reader) - except: - train_features = convert_examples_to_features( - examples=train_examples, - tokenizer=tokenizer, - max_seq_length=args.max_seq_length, - doc_stride=args.doc_stride, - max_query_length=args.max_query_length, - is_training=True) - if args.local_rank == -1 or torch.distributed.get_rank() == 0: - logger.info(" Saving train features into cached file %s", cached_train_features_file) - with open(cached_train_features_file, "wb") as writer: - pickle.dump(train_features, writer) logger.info("***** Running training *****") logger.info(" Num orig examples = %d", len(train_examples)) logger.info(" Num split examples = %d", len(train_features)) logger.info(" Batch size = %d", args.train_batch_size) logger.info(" Num steps = %d", num_train_optimization_steps) - all_input_ids = torch.tensor([f.input_ids for f in train_features], dtype=torch.long) - all_input_mask = torch.tensor([f.input_mask for f in train_features], dtype=torch.long) - all_segment_ids = torch.tensor([f.segment_ids for f in train_features], dtype=torch.long) - all_start_positions = torch.tensor([f.start_position for f in train_features], dtype=torch.long) - all_end_positions = torch.tensor([f.end_position for f in train_features], dtype=torch.long) - train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, - all_start_positions, all_end_positions) - if args.local_rank == -1: - train_sampler = RandomSampler(train_data) - else: - train_sampler = DistributedSampler(train_data) - train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=args.train_batch_size) model.train() for _ in trange(int(args.num_train_epochs), desc="Epoch"): diff --git a/examples/run_swag.py b/examples/run_swag.py index 5e7ac85c63..bdc256cf14 100644 --- a/examples/run_swag.py +++ b/examples/run_swag.py @@ -362,8 +362,20 @@ def main(): num_train_optimization_steps = None if args.do_train: train_examples = read_swag_examples(os.path.join(args.data_dir, 'train.csv'), is_training = True) - num_train_optimization_steps = int( - len(train_examples) / args.train_batch_size / args.gradient_accumulation_steps) * args.num_train_epochs + train_features = convert_examples_to_features( + train_examples, tokenizer, args.max_seq_length, True) + all_input_ids = torch.tensor(select_field(train_features, 'input_ids'), dtype=torch.long) + all_input_mask = torch.tensor(select_field(train_features, 'input_mask'), dtype=torch.long) + all_segment_ids = torch.tensor(select_field(train_features, 'segment_ids'), dtype=torch.long) + all_label = torch.tensor([f.label for f in train_features], dtype=torch.long) + train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label) + if args.local_rank == -1: + train_sampler = RandomSampler(train_data) + else: + train_sampler = DistributedSampler(train_data) + train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=args.train_batch_size) + + num_train_optimization_steps = len(train_dataloader) / args.gradient_accumulation_steps * args.num_train_epochs if args.local_rank != -1: num_train_optimization_steps = num_train_optimization_steps // torch.distributed.get_world_size() @@ -422,22 +434,10 @@ def main(): global_step = 0 if args.do_train: - train_features = convert_examples_to_features( - train_examples, tokenizer, args.max_seq_length, True) logger.info("***** Running training *****") logger.info(" Num examples = %d", len(train_examples)) logger.info(" Batch size = %d", args.train_batch_size) logger.info(" Num steps = %d", num_train_optimization_steps) - all_input_ids = torch.tensor(select_field(train_features, 'input_ids'), dtype=torch.long) - all_input_mask = torch.tensor(select_field(train_features, 'input_mask'), dtype=torch.long) - all_segment_ids = torch.tensor(select_field(train_features, 'segment_ids'), dtype=torch.long) - all_label = torch.tensor([f.label for f in train_features], dtype=torch.long) - train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label) - if args.local_rank == -1: - train_sampler = RandomSampler(train_data) - else: - train_sampler = DistributedSampler(train_data) - train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=args.train_batch_size) model.train() for _ in trange(int(args.num_train_epochs), desc="Epoch"): From 49a77ac16ff31b5ea938d8796bf0f4b5428774e6 Mon Sep 17 00:00:00 2001 From: "samuel.broscheit" Date: Sun, 12 May 2019 00:31:10 +0200 Subject: [PATCH 15/32] Clean up a little bit --- examples/run_classifier.py | 51 +++++++++++++++++++------------------- examples/run_squad.py | 46 +++++++++++++++++----------------- examples/run_swag.py | 45 +++++++++++++++++---------------- 3 files changed, 72 insertions(+), 70 deletions(-) diff --git a/examples/run_classifier.py b/examples/run_classifier.py index eff48ca97d..908559d577 100644 --- a/examples/run_classifier.py +++ b/examples/run_classifier.py @@ -736,9 +736,28 @@ def main(): tokenizer = BertTokenizer.from_pretrained(args.bert_model, do_lower_case=args.do_lower_case) - train_examples = None - num_train_optimization_steps = None + # Prepare model + cache_dir = args.cache_dir if args.cache_dir else os.path.join(str(PYTORCH_PRETRAINED_BERT_CACHE), 'distributed_{}'.format(args.local_rank)) + model = BertForSequenceClassification.from_pretrained(args.bert_model, + cache_dir=cache_dir, + num_labels=num_labels) + if args.fp16: + model.half() + model.to(device) + if args.local_rank != -1: + try: + from apex.parallel import DistributedDataParallel as DDP + except ImportError: + raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training.") + + model = DDP(model) + elif n_gpu > 1: + model = torch.nn.DataParallel(model) + if args.do_train: + + # Prepare data loader + train_examples = processor.get_train_examples(args.data_dir) train_features = convert_examples_to_features( train_examples, label_list, args.max_seq_length, tokenizer, output_mode) @@ -762,26 +781,8 @@ def main(): if args.local_rank != -1: num_train_optimization_steps = num_train_optimization_steps // torch.distributed.get_world_size() - # Prepare model - cache_dir = args.cache_dir if args.cache_dir else os.path.join(str(PYTORCH_PRETRAINED_BERT_CACHE), 'distributed_{}'.format(args.local_rank)) - model = BertForSequenceClassification.from_pretrained(args.bert_model, - cache_dir=cache_dir, - num_labels=num_labels) - if args.fp16: - model.half() - model.to(device) - if args.local_rank != -1: - try: - from apex.parallel import DistributedDataParallel as DDP - except ImportError: - raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training.") + # Prepare optimizer - model = DDP(model) - elif n_gpu > 1: - model = torch.nn.DataParallel(model) - - # Prepare optimizer - if args.do_train: param_optimizer = list(model.named_parameters()) no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight'] optimizer_grouped_parameters = [ @@ -812,10 +813,10 @@ def main(): warmup=args.warmup_proportion, t_total=num_train_optimization_steps) - global_step = 0 - nb_tr_steps = 0 - tr_loss = 0 - if args.do_train: + global_step = 0 + nb_tr_steps = 0 + tr_loss = 0 + logger.info("***** Running training *****") logger.info(" Num examples = %d", len(train_examples)) logger.info(" Batch size = %d", args.train_batch_size) diff --git a/examples/run_squad.py b/examples/run_squad.py index ae163afc91..8ce8b60294 100644 --- a/examples/run_squad.py +++ b/examples/run_squad.py @@ -894,14 +894,31 @@ def main(): tokenizer = BertTokenizer.from_pretrained(args.bert_model, do_lower_case=args.do_lower_case) - train_examples = None - num_train_optimization_steps = None + # Prepare model + model = BertForQuestionAnswering.from_pretrained(args.bert_model, + cache_dir=os.path.join(str(PYTORCH_PRETRAINED_BERT_CACHE), 'distributed_{}'.format(args.local_rank))) + + if args.fp16: + model.half() + model.to(device) + if args.local_rank != -1: + try: + from apex.parallel import DistributedDataParallel as DDP + except ImportError: + raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training.") + + model = DDP(model) + elif n_gpu > 1: + model = torch.nn.DataParallel(model) + if args.do_train: + + # Prepare data loader + train_examples = read_squad_examples( input_file=args.train_file, is_training=True, version_2_with_negative=args.version_2_with_negative) cached_train_features_file = args.train_file+'_{0}_{1}_{2}_{3}'.format( list(filter(None, args.bert_model.split('/'))).pop(), str(args.max_seq_length), str(args.doc_stride), str(args.max_query_length)) - train_features = None try: with open(cached_train_features_file, "rb") as reader: train_features = pickle.load(reader) @@ -933,25 +950,8 @@ def main(): if args.local_rank != -1: num_train_optimization_steps = num_train_optimization_steps // torch.distributed.get_world_size() - # Prepare model - model = BertForQuestionAnswering.from_pretrained(args.bert_model, - cache_dir=os.path.join(str(PYTORCH_PRETRAINED_BERT_CACHE), 'distributed_{}'.format(args.local_rank))) + # Prepare optimizer - if args.fp16: - model.half() - model.to(device) - if args.local_rank != -1: - try: - from apex.parallel import DistributedDataParallel as DDP - except ImportError: - raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training.") - - model = DDP(model) - elif n_gpu > 1: - model = torch.nn.DataParallel(model) - - # Prepare optimizer - if args.do_train: param_optimizer = list(model.named_parameters()) # hack to remove pooler, which is not used @@ -987,8 +987,8 @@ def main(): warmup=args.warmup_proportion, t_total=num_train_optimization_steps) - global_step = 0 - if args.do_train: + global_step = 0 + logger.info("***** Running training *****") logger.info(" Num orig examples = %d", len(train_examples)) logger.info(" Num split examples = %d", len(train_features)) diff --git a/examples/run_swag.py b/examples/run_swag.py index bdc256cf14..daae3971f7 100644 --- a/examples/run_swag.py +++ b/examples/run_swag.py @@ -358,9 +358,27 @@ def main(): tokenizer = BertTokenizer.from_pretrained(args.bert_model, do_lower_case=args.do_lower_case) - train_examples = None - num_train_optimization_steps = None + # Prepare model + model = BertForMultipleChoice.from_pretrained(args.bert_model, + cache_dir=os.path.join(str(PYTORCH_PRETRAINED_BERT_CACHE), 'distributed_{}'.format(args.local_rank)), + num_choices=4) + if args.fp16: + model.half() + model.to(device) + if args.local_rank != -1: + try: + from apex.parallel import DistributedDataParallel as DDP + except ImportError: + raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training.") + + model = DDP(model) + elif n_gpu > 1: + model = torch.nn.DataParallel(model) + if args.do_train: + + # Prepare data loader + train_examples = read_swag_examples(os.path.join(args.data_dir, 'train.csv'), is_training = True) train_features = convert_examples_to_features( train_examples, tokenizer, args.max_seq_length, True) @@ -379,25 +397,8 @@ def main(): if args.local_rank != -1: num_train_optimization_steps = num_train_optimization_steps // torch.distributed.get_world_size() - # Prepare model - model = BertForMultipleChoice.from_pretrained(args.bert_model, - cache_dir=os.path.join(str(PYTORCH_PRETRAINED_BERT_CACHE), 'distributed_{}'.format(args.local_rank)), - num_choices=4) - if args.fp16: - model.half() - model.to(device) - if args.local_rank != -1: - try: - from apex.parallel import DistributedDataParallel as DDP - except ImportError: - raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training.") + # Prepare optimizer - model = DDP(model) - elif n_gpu > 1: - model = torch.nn.DataParallel(model) - - # Prepare optimizer - if args.do_train: param_optimizer = list(model.named_parameters()) # hack to remove pooler, which is not used @@ -432,8 +433,8 @@ def main(): warmup=args.warmup_proportion, t_total=num_train_optimization_steps) - global_step = 0 - if args.do_train: + global_step = 0 + logger.info("***** Running training *****") logger.info(" Num examples = %d", len(train_examples)) logger.info(" Batch size = %d", args.train_batch_size) From 94247ad6cb8404307be31e33cc38ca98a274d21e Mon Sep 17 00:00:00 2001 From: samuelbroscheit Date: Mon, 13 May 2019 12:38:22 +0200 Subject: [PATCH 16/32] Make num_train_optimization_steps int --- examples/run_classifier.py | 2 +- examples/run_squad.py | 2 +- examples/run_swag.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/run_classifier.py b/examples/run_classifier.py index 908559d577..94099204de 100644 --- a/examples/run_classifier.py +++ b/examples/run_classifier.py @@ -777,7 +777,7 @@ def main(): train_sampler = DistributedSampler(train_data) train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=args.train_batch_size) - num_train_optimization_steps = len(train_dataloader) / args.gradient_accumulation_steps * args.num_train_epochs + num_train_optimization_steps = len(train_dataloader) // args.gradient_accumulation_steps * args.num_train_epochs if args.local_rank != -1: num_train_optimization_steps = num_train_optimization_steps // torch.distributed.get_world_size() diff --git a/examples/run_squad.py b/examples/run_squad.py index 8ce8b60294..b145303fb0 100644 --- a/examples/run_squad.py +++ b/examples/run_squad.py @@ -946,7 +946,7 @@ def main(): else: train_sampler = DistributedSampler(train_data) train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=args.train_batch_size) - num_train_optimization_steps = len(train_dataloader) / args.gradient_accumulation_steps * args.num_train_epochs + num_train_optimization_steps = len(train_dataloader) // args.gradient_accumulation_steps * args.num_train_epochs if args.local_rank != -1: num_train_optimization_steps = num_train_optimization_steps // torch.distributed.get_world_size() diff --git a/examples/run_swag.py b/examples/run_swag.py index daae3971f7..73cab42830 100644 --- a/examples/run_swag.py +++ b/examples/run_swag.py @@ -393,7 +393,7 @@ def main(): train_sampler = DistributedSampler(train_data) train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=args.train_batch_size) - num_train_optimization_steps = len(train_dataloader) / args.gradient_accumulation_steps * args.num_train_epochs + num_train_optimization_steps = len(train_dataloader) // args.gradient_accumulation_steps * args.num_train_epochs if args.local_rank != -1: num_train_optimization_steps = num_train_optimization_steps // torch.distributed.get_world_size() From 9e7bc51b95118528bb6c394d02c341cb5401e25d Mon Sep 17 00:00:00 2001 From: tguens <50817608+tguens@users.noreply.github.com> Date: Wed, 22 May 2019 17:27:59 +0800 Subject: [PATCH 17/32] Update run_squad.py Indentation change so that the output "nbest_predictions.json" is not empty. --- examples/run_squad.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/run_squad.py b/examples/run_squad.py index 249aff7f8a..9f76fb8fbb 100644 --- a/examples/run_squad.py +++ b/examples/run_squad.py @@ -617,7 +617,7 @@ def write_predictions(all_examples, all_features, all_results, n_best_size, all_predictions[example.qas_id] = "" else: all_predictions[example.qas_id] = best_non_null_entry.text - all_nbest_json[example.qas_id] = nbest_json + all_nbest_json[example.qas_id] = nbest_json with open(output_prediction_file, "w") as writer: writer.write(json.dumps(all_predictions, indent=4) + "\n") From c4fe56dcc0ca30d777650ba95167ec72582fdfea Mon Sep 17 00:00:00 2001 From: Ahmad Barqawi Date: Mon, 27 May 2019 11:27:41 +0200 Subject: [PATCH 18/32] support latest multi language bert fine tune fix issue of bert-base-multilingual and add support for uncased multilingual --- examples/lm_finetuning/pregenerate_training_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/lm_finetuning/pregenerate_training_data.py b/examples/lm_finetuning/pregenerate_training_data.py index e6c3598a9f..8a59e0d6a7 100644 --- a/examples/lm_finetuning/pregenerate_training_data.py +++ b/examples/lm_finetuning/pregenerate_training_data.py @@ -235,7 +235,7 @@ def main(): parser.add_argument("--output_dir", type=Path, required=True) parser.add_argument("--bert_model", type=str, required=True, choices=["bert-base-uncased", "bert-large-uncased", "bert-base-cased", - "bert-base-multilingual", "bert-base-chinese"]) + "bert-base-multilingual-uncased", "bert-base-chinese", "bert-base-multilingual-cased"]) parser.add_argument("--do_lower_case", action="store_true") parser.add_argument("--reduce_memory", action="store_true", From 1eba8b9d96ed90f6bfa1f836a94d19cae753f309 Mon Sep 17 00:00:00 2001 From: Colanim <43774355+Colanim@users.noreply.github.com> Date: Thu, 30 May 2019 14:01:46 +0900 Subject: [PATCH 19/32] Fix link in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e8b4f3c6b6..b1cb84619d 100644 --- a/README.md +++ b/README.md @@ -1033,7 +1033,7 @@ An overview of the implemented schedules: |-|-| | [Training large models: introduction, tools and examples](#Training-large-models-introduction,-tools-and-examples) | How to use gradient-accumulation, multi-gpu training, distributed training, optimize on CPU and 16-bits training to train Bert models | | [Fine-tuning with BERT: running the examples](#Fine-tuning-with-BERT-running-the-examples) | Running the examples in [`./examples`](./examples/): `extract_classif.py`, `run_classifier.py`, `run_squad.py` and `run_lm_finetuning.py` | -| [Fine-tuning with OpenAI GPT, Transformer-XL and GPT-2](#Fine-tuning-with-OpenAI-GPT-Transformer-XL-and-GPT-2) | Running the examples in [`./examples`](./examples/): `run_openai_gpt.py`, `run_transfo_xl.py` and `run_gpt2.py` | +| [Fine-tuning with OpenAI GPT, Transformer-XL and GPT-2](#openai-gpt-transformer-xl-and-gpt-2-running-the-examples) | Running the examples in [`./examples`](./examples/): `run_openai_gpt.py`, `run_transfo_xl.py` and `run_gpt2.py` | | [Fine-tuning BERT-large on GPUs](#Fine-tuning-BERT-large-on-GPUs) | How to fine tune `BERT large`| ### Training large models: introduction, tools and examples From de5e5682a12463465a9eda4d2b13efad9c50d0dd Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Mon, 3 Jun 2019 17:05:24 -0400 Subject: [PATCH 20/32] add output_attentions for BertModel --- pytorch_pretrained_bert/modeling.py | 34 +++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/pytorch_pretrained_bert/modeling.py b/pytorch_pretrained_bert/modeling.py index b9b6837193..72d8ff5195 100644 --- a/pytorch_pretrained_bert/modeling.py +++ b/pytorch_pretrained_bert/modeling.py @@ -275,7 +275,7 @@ class BertEmbeddings(nn.Module): class BertSelfAttention(nn.Module): - def __init__(self, config): + def __init__(self, config, output_attentions=False): super(BertSelfAttention, self).__init__() if config.hidden_size % config.num_attention_heads != 0: raise ValueError( @@ -291,6 +291,8 @@ class BertSelfAttention(nn.Module): self.dropout = nn.Dropout(config.attention_probs_dropout_prob) + self.output_attentions = output_attentions + def transpose_for_scores(self, x): new_x_shape = x.size()[:-1] + (self.num_attention_heads, self.attention_head_size) x = x.view(*new_x_shape) @@ -322,7 +324,10 @@ class BertSelfAttention(nn.Module): context_layer = context_layer.permute(0, 2, 1, 3).contiguous() new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,) context_layer = context_layer.view(*new_context_layer_shape) - return context_layer + if self.output_attentions: + return attention_probs, context_layer + else: + return context_layer class BertSelfOutput(nn.Module): @@ -381,33 +386,43 @@ class BertOutput(nn.Module): class BertLayer(nn.Module): - def __init__(self, config): + def __init__(self, config, output_attentions=False): super(BertLayer, self).__init__() self.attention = BertAttention(config) self.intermediate = BertIntermediate(config) self.output = BertOutput(config) + self.output_attentions = output_attentions def forward(self, hidden_states, attention_mask): attention_output = self.attention(hidden_states, attention_mask) intermediate_output = self.intermediate(attention_output) layer_output = self.output(intermediate_output, attention_output) + if self.output_attentions: + return attention_output, layer_output return layer_output class BertEncoder(nn.Module): - def __init__(self, config): + def __init__(self, config, output_attentions=False): super(BertEncoder, self).__init__() - layer = BertLayer(config) + layer = BertLayer(config, output_attentions=output_attentions) self.layer = nn.ModuleList([copy.deepcopy(layer) for _ in range(config.num_hidden_layers)]) + self.output_attentions = output_attentions def forward(self, hidden_states, attention_mask, output_all_encoded_layers=True): all_encoder_layers = [] + all_attentions = [] for layer_module in self.layer: hidden_states = layer_module(hidden_states, attention_mask) + if self.output_attentions: + attentions, hidden_states = hidden_states + all_attentions.append(attentions) if output_all_encoded_layers: all_encoder_layers.append(hidden_states) if not output_all_encoded_layers: all_encoder_layers.append(hidden_states) + if self.output_attentions: + return all_attentions, all_encoder_layers return all_encoder_layers @@ -699,12 +714,13 @@ class BertModel(BertPreTrainedModel): all_encoder_layers, pooled_output = model(input_ids, token_type_ids, input_mask) ``` """ - def __init__(self, config): + def __init__(self, config, output_attentions=False): super(BertModel, self).__init__(config) self.embeddings = BertEmbeddings(config) - self.encoder = BertEncoder(config) + self.encoder = BertEncoder(config, output_attentions=output_attentions) self.pooler = BertPooler(config) self.apply(self.init_bert_weights) + self.output_attentions = output_attentions def forward(self, input_ids, token_type_ids=None, attention_mask=None, output_all_encoded_layers=True): if attention_mask is None: @@ -731,10 +747,14 @@ class BertModel(BertPreTrainedModel): encoded_layers = self.encoder(embedding_output, extended_attention_mask, output_all_encoded_layers=output_all_encoded_layers) + if self.output_attentions: + all_attentions, encoded_layers = encoded_layers sequence_output = encoded_layers[-1] pooled_output = self.pooler(sequence_output) if not output_all_encoded_layers: encoded_layers = encoded_layers[-1] + if self.output_attentions: + return all_attentions, encoded_layers, pooled_output return encoded_layers, pooled_output From 826496580beae08289452da0eda914bdc40a95bb Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Mon, 3 Jun 2019 17:10:25 -0400 Subject: [PATCH 21/32] Revert "add output_attentions for BertModel" This reverts commit de5e5682a12463465a9eda4d2b13efad9c50d0dd. --- pytorch_pretrained_bert/modeling.py | 34 ++++++----------------------- 1 file changed, 7 insertions(+), 27 deletions(-) diff --git a/pytorch_pretrained_bert/modeling.py b/pytorch_pretrained_bert/modeling.py index 72d8ff5195..b9b6837193 100644 --- a/pytorch_pretrained_bert/modeling.py +++ b/pytorch_pretrained_bert/modeling.py @@ -275,7 +275,7 @@ class BertEmbeddings(nn.Module): class BertSelfAttention(nn.Module): - def __init__(self, config, output_attentions=False): + def __init__(self, config): super(BertSelfAttention, self).__init__() if config.hidden_size % config.num_attention_heads != 0: raise ValueError( @@ -291,8 +291,6 @@ class BertSelfAttention(nn.Module): self.dropout = nn.Dropout(config.attention_probs_dropout_prob) - self.output_attentions = output_attentions - def transpose_for_scores(self, x): new_x_shape = x.size()[:-1] + (self.num_attention_heads, self.attention_head_size) x = x.view(*new_x_shape) @@ -324,10 +322,7 @@ class BertSelfAttention(nn.Module): context_layer = context_layer.permute(0, 2, 1, 3).contiguous() new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,) context_layer = context_layer.view(*new_context_layer_shape) - if self.output_attentions: - return attention_probs, context_layer - else: - return context_layer + return context_layer class BertSelfOutput(nn.Module): @@ -386,43 +381,33 @@ class BertOutput(nn.Module): class BertLayer(nn.Module): - def __init__(self, config, output_attentions=False): + def __init__(self, config): super(BertLayer, self).__init__() self.attention = BertAttention(config) self.intermediate = BertIntermediate(config) self.output = BertOutput(config) - self.output_attentions = output_attentions def forward(self, hidden_states, attention_mask): attention_output = self.attention(hidden_states, attention_mask) intermediate_output = self.intermediate(attention_output) layer_output = self.output(intermediate_output, attention_output) - if self.output_attentions: - return attention_output, layer_output return layer_output class BertEncoder(nn.Module): - def __init__(self, config, output_attentions=False): + def __init__(self, config): super(BertEncoder, self).__init__() - layer = BertLayer(config, output_attentions=output_attentions) + layer = BertLayer(config) self.layer = nn.ModuleList([copy.deepcopy(layer) for _ in range(config.num_hidden_layers)]) - self.output_attentions = output_attentions def forward(self, hidden_states, attention_mask, output_all_encoded_layers=True): all_encoder_layers = [] - all_attentions = [] for layer_module in self.layer: hidden_states = layer_module(hidden_states, attention_mask) - if self.output_attentions: - attentions, hidden_states = hidden_states - all_attentions.append(attentions) if output_all_encoded_layers: all_encoder_layers.append(hidden_states) if not output_all_encoded_layers: all_encoder_layers.append(hidden_states) - if self.output_attentions: - return all_attentions, all_encoder_layers return all_encoder_layers @@ -714,13 +699,12 @@ class BertModel(BertPreTrainedModel): all_encoder_layers, pooled_output = model(input_ids, token_type_ids, input_mask) ``` """ - def __init__(self, config, output_attentions=False): + def __init__(self, config): super(BertModel, self).__init__(config) self.embeddings = BertEmbeddings(config) - self.encoder = BertEncoder(config, output_attentions=output_attentions) + self.encoder = BertEncoder(config) self.pooler = BertPooler(config) self.apply(self.init_bert_weights) - self.output_attentions = output_attentions def forward(self, input_ids, token_type_ids=None, attention_mask=None, output_all_encoded_layers=True): if attention_mask is None: @@ -747,14 +731,10 @@ class BertModel(BertPreTrainedModel): encoded_layers = self.encoder(embedding_output, extended_attention_mask, output_all_encoded_layers=output_all_encoded_layers) - if self.output_attentions: - all_attentions, encoded_layers = encoded_layers sequence_output = encoded_layers[-1] pooled_output = self.pooler(sequence_output) if not output_all_encoded_layers: encoded_layers = encoded_layers[-1] - if self.output_attentions: - return all_attentions, encoded_layers, pooled_output return encoded_layers, pooled_output From a3274ac40b14025ee857897ecfaff4fb07bcb61d Mon Sep 17 00:00:00 2001 From: thomwolf Date: Mon, 3 Jun 2019 16:11:45 -0500 Subject: [PATCH 22/32] adding attention outputs in bert --- pytorch_pretrained_bert/modeling.py | 43 +++++++++++++++++++++++------ tests/modeling_gpt2_test.py | 34 +++++++++++++++++++++++ 2 files changed, 68 insertions(+), 9 deletions(-) diff --git a/pytorch_pretrained_bert/modeling.py b/pytorch_pretrained_bert/modeling.py index b9b6837193..27682eb369 100644 --- a/pytorch_pretrained_bert/modeling.py +++ b/pytorch_pretrained_bert/modeling.py @@ -275,12 +275,13 @@ class BertEmbeddings(nn.Module): class BertSelfAttention(nn.Module): - def __init__(self, config): + def __init__(self, config, output_attentions=False): super(BertSelfAttention, self).__init__() if config.hidden_size % config.num_attention_heads != 0: raise ValueError( "The hidden size (%d) is not a multiple of the number of attention " "heads (%d)" % (config.hidden_size, config.num_attention_heads)) + self.output_attentions = output_attentions self.num_attention_heads = config.num_attention_heads self.attention_head_size = int(config.hidden_size / config.num_attention_heads) self.all_head_size = self.num_attention_heads * self.attention_head_size @@ -322,6 +323,8 @@ class BertSelfAttention(nn.Module): context_layer = context_layer.permute(0, 2, 1, 3).contiguous() new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,) context_layer = context_layer.view(*new_context_layer_shape) + if self.output_attentions: + return attention_probs, context_layer return context_layer @@ -340,14 +343,19 @@ class BertSelfOutput(nn.Module): class BertAttention(nn.Module): - def __init__(self, config): + def __init__(self, config, output_attentions=False): super(BertAttention, self).__init__() - self.self = BertSelfAttention(config) + self.output_attentions = output_attentions + self.self = BertSelfAttention(config, output_attentions=output_attentions) self.output = BertSelfOutput(config) def forward(self, input_tensor, attention_mask): self_output = self.self(input_tensor, attention_mask) + if self.output_attentions: + attentions, self_output = self_output attention_output = self.output(self_output, input_tensor) + if self.output_attentions: + return attentions, attention_output return attention_output @@ -381,33 +389,45 @@ class BertOutput(nn.Module): class BertLayer(nn.Module): - def __init__(self, config): + def __init__(self, config, output_attentions=False): super(BertLayer, self).__init__() - self.attention = BertAttention(config) + self.output_attentions = output_attentions + self.attention = BertAttention(config, output_attentions=output_attentions) self.intermediate = BertIntermediate(config) self.output = BertOutput(config) def forward(self, hidden_states, attention_mask): attention_output = self.attention(hidden_states, attention_mask) + if self.output_attentions: + attentions, attention_output = attention_output intermediate_output = self.intermediate(attention_output) layer_output = self.output(intermediate_output, attention_output) + if self.output_attentions: + return attentions, layer_output return layer_output class BertEncoder(nn.Module): - def __init__(self, config): + def __init__(self, config, output_attentions=False): super(BertEncoder, self).__init__() - layer = BertLayer(config) + self.output_attentions = output_attentions + layer = BertLayer(config, output_attentions=output_attentions) self.layer = nn.ModuleList([copy.deepcopy(layer) for _ in range(config.num_hidden_layers)]) def forward(self, hidden_states, attention_mask, output_all_encoded_layers=True): all_encoder_layers = [] + all_attentions = [] for layer_module in self.layer: hidden_states = layer_module(hidden_states, attention_mask) + if self.output_attentions: + attentions, hidden_states = hidden_states + all_attentions.append(attentions) if output_all_encoded_layers: all_encoder_layers.append(hidden_states) if not output_all_encoded_layers: all_encoder_layers.append(hidden_states) + if self.output_attentions: + return all_attentions, all_encoder_layers return all_encoder_layers @@ -699,10 +719,11 @@ class BertModel(BertPreTrainedModel): all_encoder_layers, pooled_output = model(input_ids, token_type_ids, input_mask) ``` """ - def __init__(self, config): + def __init__(self, config, output_attentions=False): super(BertModel, self).__init__(config) + self.output_attentions = output_attentions self.embeddings = BertEmbeddings(config) - self.encoder = BertEncoder(config) + self.encoder = BertEncoder(config, output_attentions=output_attentions) self.pooler = BertPooler(config) self.apply(self.init_bert_weights) @@ -731,10 +752,14 @@ class BertModel(BertPreTrainedModel): encoded_layers = self.encoder(embedding_output, extended_attention_mask, output_all_encoded_layers=output_all_encoded_layers) + if self.output_attentions: + all_attentions, encoded_layers = encoded_layers sequence_output = encoded_layers[-1] pooled_output = self.pooler(sequence_output) if not output_all_encoded_layers: encoded_layers = encoded_layers[-1] + if self.output_attentions: + return all_attentions, encoded_layers, pooled_output return encoded_layers, pooled_output diff --git a/tests/modeling_gpt2_test.py b/tests/modeling_gpt2_test.py index 6804b794c5..41cc9b8fd3 100644 --- a/tests/modeling_gpt2_test.py +++ b/tests/modeling_gpt2_test.py @@ -133,11 +133,28 @@ class GPT2ModelTest(unittest.TestCase): } return outputs + def create_gpt2_lm_head_with_output_attention(self, config, input_ids, token_type_ids, position_ids, + mc_labels, lm_labels, mc_token_ids): + model = GPT2LMHeadModel(config, output_attentions=True) + model.eval() + loss = model(input_ids, position_ids, token_type_ids, lm_labels) + attentions, lm_logits, presents = model(input_ids, position_ids, token_type_ids) + outputs = { + "loss": loss, + "lm_logits": lm_logits, + "presents": presents, + "attentions": attentions, + } + return outputs + def check_gpt2_lm_head_output(self, result): total_voc = self.n_special + self.vocab_size self.parent.assertListEqual( list(result["lm_logits"].size()), [self.batch_size, self.n_choices, self.seq_length, total_voc]) + self.parent.assertListEqual( + list(result["presents"].size()), + [self.batch_size, self.n_choices, self.seq_length, total_voc]) def check_gpt2_lm_head_loss_output(self, result): self.parent.assertListEqual( @@ -160,6 +177,23 @@ class GPT2ModelTest(unittest.TestCase): } return outputs + def create_gpt2_double_heads_with_output_attention(self, config, input_ids, token_type_ids, position_ids, + mc_labels, lm_labels, mc_token_ids): + model = GPT2DoubleHeadsModel(config, output_attentions=True) + model.eval() + loss = model(input_ids, mc_token_ids, + lm_labels=lm_labels, mc_labels=mc_labels, + token_type_ids=token_type_ids, position_ids=position_ids) + attentions, lm_logits, mc_logits, presents = model(input_ids, mc_token_ids, position_ids=position_ids, token_type_ids=token_type_ids) + outputs = { + "loss": loss, + "lm_logits": lm_logits, + "mc_logits": mc_logits, + "presents": presents, + "attentions": attentions, + } + return outputs + def check_gpt2_double_heads_output(self, result): total_voc = self.n_special + self.vocab_size self.parent.assertListEqual( From cf44d9839202d4d67cdc66fbb46162904587409f Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Thu, 6 Jun 2019 16:36:02 +0200 Subject: [PATCH 23/32] Add more examples to BERT models for torchhub --- hubconfs/bert_hubconf.py | 108 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 100 insertions(+), 8 deletions(-) diff --git a/hubconfs/bert_hubconf.py b/hubconfs/bert_hubconf.py index 67397aeec8..385c284b65 100644 --- a/hubconfs/bert_hubconf.py +++ b/hubconfs/bert_hubconf.py @@ -82,7 +82,7 @@ def bertTokenizer(*args, **kwargs): Example: >>> sentence = 'Hello, World!' - >>> tokenizer = torch.hub.load('huggingface/pytorch-pretrained-BERT', 'bertTokenizer', 'bert-base-cased', do_basic_tokenize=False, force_reload=False) + >>> tokenizer = torch.hub.load('huggingface/pytorch-pretrained-BERT', 'bertTokenizer', 'bert-base-cased', do_basic_tokenize=False) >>> toks = tokenizer.tokenize(sentence) ['Hello', '##,', 'World', '##!'] >>> ids = tokenizer.convert_tokens_to_ids(toks) @@ -101,7 +101,7 @@ def bertModel(*args, **kwargs): Example: # Load the tokenizer - >>> tokenizer = torch.hub.load('huggingface/pytorch-pretrained-BERT', 'bertTokenizer', 'bert-base-cased', do_basic_tokenize=False, force_reload=False) + >>> tokenizer = torch.hub.load('huggingface/pytorch-pretrained-BERT', 'bertTokenizer', 'bert-base-cased', do_basic_tokenize=False) # Prepare tokenized input >>> text = "[CLS] Who was Jim Henson ? [SEP] Jim Henson was a puppeteer [SEP]" >>> tokenized_text = tokenizer.tokenize(text) @@ -113,7 +113,7 @@ def bertModel(*args, **kwargs): >>> segments_tensors = torch.tensor([segments_ids]) tensor([[0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1]]) # Load bertModel - >>> model = torch.hub.load('huggingface/pytorch-pretrained-BERT', 'bertModel', 'bert-base-cased', force_reload=False) + >>> model = torch.hub.load('huggingface/pytorch-pretrained-BERT', 'bertModel', 'bert-base-cased') >>> model.eval() # Predict hidden states features for each layer >>> with torch.no_grad(): @@ -129,6 +129,23 @@ def bertForNextSentencePrediction(*args, **kwargs): BERT model with next sentence prediction head. This module comprises the BERT model followed by the next sentence classification head. + + Example: + # Load the tokenizer + >>> tokenizer = torch.hub.load('huggingface/pytorch-pretrained-BERT', 'bertTokenizer', 'bert-base-cased', do_basic_tokenize=False) + # Prepare tokenized input + >>> text = "[CLS] Who was Jim Henson ? [SEP] Jim Henson was a puppeteer [SEP]" + >>> tokenized_text = tokenizer.tokenize(text) + >>> indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text) + >>> segments_ids = [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1] + >>> tokens_tensor = torch.tensor([indexed_tokens]) + >>> segments_tensors = torch.tensor([segments_ids]) + # Load bertForNextSentencePrediction + >>> model = torch.hub.load('huggingface/pytorch-pretrained-BERT', 'bertForNextSentencePrediction', 'bert-base-cased') + >>> model.eval() + # Predict the next sentence classification logits + >>> with torch.no_grad(): + next_sent_classif_logits = model(tokens_tensor, segments_tensors) """ model = BertForNextSentencePrediction.from_pretrained(*args, **kwargs) return model @@ -154,7 +171,7 @@ def bertForMaskedLM(*args, **kwargs): Example: # Load the tokenizer - >>> tokenizer = torch.hub.load('huggingface/pytorch-pretrained-BERT', 'bertTokenizer', 'bert-base-cased', do_basic_tokenize=False, force_reload=False) + >>> tokenizer = torch.hub.load('huggingface/pytorch-pretrained-BERT', 'bertTokenizer', 'bert-base-cased', do_basic_tokenize=False) # Prepare tokenized input >>> text = "[CLS] Who was Jim Henson ? [SEP] Jim Henson was a puppeteer [SEP]" >>> tokenized_text = tokenizer.tokenize(text) @@ -166,7 +183,7 @@ def bertForMaskedLM(*args, **kwargs): >>> tokens_tensor = torch.tensor([indexed_tokens]) >>> segments_tensors = torch.tensor([segments_ids]) # Load bertForMaskedLM - >>> model = torch.hub.load('huggingface/pytorch-pretrained-BERT', 'bertForMaskedLM', 'bert-base-cased', force_reload=False) + >>> model = torch.hub.load('huggingface/pytorch-pretrained-BERT', 'bertForMaskedLM', 'bert-base-cased') >>> model.eval() # Predict all tokens >>> with torch.no_grad(): @@ -194,7 +211,25 @@ def bertForSequenceClassification(*args, **kwargs): num_labels: the number (>=2) of classes for the classifier. Example: - >>> torch.hub.load('huggingface/pytorch-pretrained-BERT', 'bertForSequenceClassification', 'bert-base-cased', num_labels=2, force_reload=True) + # Load the tokenizer + >>> tokenizer = torch.hub.load('huggingface/pytorch-pretrained-BERT', 'bertTokenizer', 'bert-base-cased', do_basic_tokenize=False) + # Prepare tokenized input + >>> text = "[CLS] Who was Jim Henson ? [SEP] Jim Henson was a puppeteer [SEP]" + >>> tokenized_text = tokenizer.tokenize(text) + >>> indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text) + >>> segments_ids = [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1] + >>> tokens_tensor = torch.tensor([indexed_tokens]) + >>> segments_tensors = torch.tensor([segments_ids]) + # Load bertForSequenceClassification + >>> model = torch.hub.load('huggingface/pytorch-pretrained-BERT', 'bertForSequenceClassification', 'bert-base-cased', num_labels=2) + >>> model.eval() + # Predict the sequence classification logits + >>> with torch.no_grad(): + seq_classif_logits = model(tokens_tensor, segments_tensors) + # Or get the sequence classification loss + >>> labels = torch.tensor([1]) + >>> with torch.no_grad(): + seq_classif_loss = model(tokens_tensor, segments_tensors, labels=labels) """ model = BertForSequenceClassification.from_pretrained(*args, **kwargs) return model @@ -210,7 +245,25 @@ def bertForMultipleChoice(*args, **kwargs): num_choices: the number (>=2) of classes for the classifier. Example: - >>> torch.hub.load('huggingface/pytorch-pretrained-BERT', 'bertForMultipleChoice', 'bert-base-cased', num_choices=2, force_reload=True) + # Load the tokenizer + >>> tokenizer = torch.hub.load('huggingface/pytorch-pretrained-BERT', 'bertTokenizer', 'bert-base-cased', do_basic_tokenize=False) + # Prepare tokenized input + >>> text = "[CLS] Who was Jim Henson ? [SEP] Jim Henson was a puppeteer [SEP]" + >>> tokenized_text = tokenizer.tokenize(text) + >>> indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text) + >>> segments_ids = [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1] + >>> tokens_tensor = torch.tensor([indexed_tokens, indexed_tokens]).unsqueeze(0) + >>> segments_tensors = torch.tensor([segments_ids, segments_ids]).unsqueeze(0) + # Load bertForMultipleChoice + >>> model = torch.hub.load('huggingface/pytorch-pretrained-BERT', 'bertForMultipleChoice', 'bert-base-cased', num_choices=2) + >>> model.eval() + # Predict the multiple choice logits + >>> with torch.no_grad(): + multiple_choice_logits = model(tokens_tensor, segments_tensors) + # Or get the multiple choice loss + >>> labels = torch.tensor([1]) + >>> with torch.no_grad(): + multiple_choice_loss = model(tokens_tensor, segments_tensors, labels=labels) """ model = BertForMultipleChoice.from_pretrained(*args, **kwargs) return model @@ -222,6 +275,27 @@ def bertForQuestionAnswering(*args, **kwargs): BertForQuestionAnswering is a fine-tuning model that includes BertModel with a token-level classifiers on top of the full sequence of last hidden states. + + Example: + # Load the tokenizer + >>> tokenizer = torch.hub.load('huggingface/pytorch-pretrained-BERT', 'bertTokenizer', 'bert-base-cased', do_basic_tokenize=False) + # Prepare tokenized input + >>> text = "[CLS] Who was Jim Henson ? [SEP] Jim Henson was a puppeteer [SEP]" + >>> tokenized_text = tokenizer.tokenize(text) + >>> indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text) + >>> segments_ids = [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1] + >>> tokens_tensor = torch.tensor([indexed_tokens]) + >>> segments_tensors = torch.tensor([segments_ids]) + # Load bertForQuestionAnswering + >>> model = torch.hub.load('huggingface/pytorch-pretrained-BERT', 'bertForQuestionAnswering', 'bert-base-cased') + >>> model.eval() + # Predict the start and end positions logits + >>> with torch.no_grad(): + start_logits, end_logits = model(tokens_tensor, segments_tensors) + # Or get the total loss which is the sum of the CrossEntropy loss for the start and end token positions + >>> start_positions, end_positions = torch.tensor([12]), torch.tensor([14]) + >>> with torch.no_grad(): + multiple_choice_loss = model(tokens_tensor, segments_tensors, start_positions=start_positions, end_positions=end_positions) """ model = BertForQuestionAnswering.from_pretrained(*args, **kwargs) return model @@ -240,7 +314,25 @@ def bertForTokenClassification(*args, **kwargs): num_labels: the number (>=2) of classes for the classifier. Example: - >>> torch.hub.load('huggingface/pytorch-pretrained-BERT', 'bertForTokenClassification', 'bert-base-cased', num_labels=2, force_reload=True) + # Load the tokenizer + >>> tokenizer = torch.hub.load('huggingface/pytorch-pretrained-BERT', 'bertTokenizer', 'bert-base-cased', do_basic_tokenize=False) + # Prepare tokenized input + >>> text = "[CLS] Who was Jim Henson ? [SEP] Jim Henson was a puppeteer [SEP]" + >>> tokenized_text = tokenizer.tokenize(text) + >>> indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text) + >>> segments_ids = [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1] + >>> tokens_tensor = torch.tensor([indexed_tokens]) + >>> segments_tensors = torch.tensor([segments_ids]) + # Load bertForTokenClassification + >>> model = torch.hub.load('huggingface/pytorch-pretrained-BERT', 'bertForTokenClassification', 'bert-base-cased', num_labels=2) + >>> model.eval() + # Predict the token classification logits + >>> with torch.no_grad(): + classif_logits = model(tokens_tensor, segments_tensors) + # Or get the token classification loss + >>> labels = torch.tensor([[0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 1, 0, 0, 1, 0]]) + >>> with torch.no_grad(): + classif_loss = model(tokens_tensor, segments_tensors, labels=labels) """ model = BertForTokenClassification.from_pretrained(*args, **kwargs) return model From 2647ac3294d14ffe270db1b235f37f4990430cd6 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Thu, 6 Jun 2019 16:57:40 +0200 Subject: [PATCH 24/32] forgot bertForPreTraining --- hubconfs/bert_hubconf.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/hubconfs/bert_hubconf.py b/hubconfs/bert_hubconf.py index 385c284b65..14e5a17239 100644 --- a/hubconfs/bert_hubconf.py +++ b/hubconfs/bert_hubconf.py @@ -158,6 +158,19 @@ def bertForPreTraining(*args, **kwargs): This module comprises the BERT model followed by the two pre-training heads - the masked language modeling head, and - the next sentence classification head. + + Example: + # Load the tokenizer + >>> tokenizer = torch.hub.load('huggingface/pytorch-pretrained-BERT', 'bertTokenizer', 'bert-base-cased', do_basic_tokenize=False) + # Prepare tokenized input + >>> text = "[CLS] Who was Jim Henson ? [SEP] Jim Henson was a puppeteer [SEP]" + >>> tokenized_text = tokenizer.tokenize(text) + >>> segments_ids = [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1] + >>> tokens_tensor = torch.tensor([indexed_tokens]) + >>> segments_tensors = torch.tensor([segments_ids]) + # Load bertForPreTraining + >>> model = torch.hub.load('huggingface/pytorch-pretrained-BERT', 'bertForPreTraining', 'bert-base-cased') + >>> masked_lm_logits_scores, seq_relationship_logits = model(tokens_tensor, segments_tensors) """ model = BertForPreTraining.from_pretrained(*args, **kwargs) return model From 122d5c52acb70c368aa09328e12281760e01ce75 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Thu, 6 Jun 2019 17:02:51 +0200 Subject: [PATCH 25/32] distinguish was is not trained --- hubconfs/bert_hubconf.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/hubconfs/bert_hubconf.py b/hubconfs/bert_hubconf.py index 14e5a17239..7cd2a123c0 100644 --- a/hubconfs/bert_hubconf.py +++ b/hubconfs/bert_hubconf.py @@ -214,7 +214,8 @@ def bertForSequenceClassification(*args, **kwargs): """ BertForSequenceClassification is a fine-tuning model that includes BertModel and a sequence-level (sequence or pair of sequences) classifier - on top of the BertModel. + on top of the BertModel. Note that the classification head is only initialized + and has to be trained. The sequence-level classifier is a linear layer that takes as input the last hidden state of the first character in the input sequence @@ -252,7 +253,8 @@ def bertForSequenceClassification(*args, **kwargs): def bertForMultipleChoice(*args, **kwargs): """ BertForMultipleChoice is a fine-tuning model that includes BertModel and a - linear layer on top of the BertModel. + linear layer on top of the BertModel. Note that the multiple choice head is + only initialized and has to be trained. Args: num_choices: the number (>=2) of classes for the classifier. @@ -287,7 +289,8 @@ def bertForQuestionAnswering(*args, **kwargs): """ BertForQuestionAnswering is a fine-tuning model that includes BertModel with a token-level classifiers on top of the full sequence of last hidden - states. + states. Note that the classification head is only initialized + and has to be trained. Example: # Load the tokenizer @@ -318,7 +321,8 @@ def bertForQuestionAnswering(*args, **kwargs): def bertForTokenClassification(*args, **kwargs): """ BertForTokenClassification is a fine-tuning model that includes BertModel - and a token-level classifier on top of the BertModel. + and a token-level classifier on top of the BertModel. Note that the classification + head is only initialized and has to be trained. The token-level classifier is a linear layer that takes as input the last hidden state of the sequence. From 6b8d227092302eff4ff6a294034c4c16b81569ba Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Thu, 6 Jun 2019 17:07:03 +0200 Subject: [PATCH 26/32] some cleaning --- hubconfs/bert_hubconf.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/hubconfs/bert_hubconf.py b/hubconfs/bert_hubconf.py index 7cd2a123c0..c7bcfbffb6 100644 --- a/hubconfs/bert_hubconf.py +++ b/hubconfs/bert_hubconf.py @@ -105,13 +105,10 @@ def bertModel(*args, **kwargs): # Prepare tokenized input >>> text = "[CLS] Who was Jim Henson ? [SEP] Jim Henson was a puppeteer [SEP]" >>> tokenized_text = tokenizer.tokenize(text) - ['[CLS]', 'Who', 'was', 'Jim', 'He', '##nson', '?', '[SEP]', 'Jim', 'He', '##nson', 'was', 'a', 'puppet', '##eer', '[SEP]'] >>> indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text) >>> segments_ids = [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1] >>> tokens_tensor = torch.tensor([indexed_tokens]) - tensor([[101, 2627, 1108, 3104, 1124, 15703, 136, 102, 3104, 1124, 15703, 1108, 170, 16797, 8284, 102]]) >>> segments_tensors = torch.tensor([segments_ids]) - tensor([[0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1]]) # Load bertModel >>> model = torch.hub.load('huggingface/pytorch-pretrained-BERT', 'bertModel', 'bert-base-cased') >>> model.eval() @@ -190,7 +187,6 @@ def bertForMaskedLM(*args, **kwargs): >>> tokenized_text = tokenizer.tokenize(text) >>> masked_index = 8 >>> tokenized_text[masked_index] = '[MASK]' - ['[CLS]', 'who', 'was', 'jim', 'henson', '?', '[SEP]', 'jim', '[MASK]', 'was', 'a', 'puppet', '##eer', '[SEP]'] >>> indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text) >>> segments_ids = [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1] >>> tokens_tensor = torch.tensor([indexed_tokens]) From 2d07f945adfd41389b5dd45d85af37d404a09599 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Thu, 6 Jun 2019 17:10:24 +0200 Subject: [PATCH 27/32] fix error with torch.no_grad and loss computation --- hubconfs/bert_hubconf.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/hubconfs/bert_hubconf.py b/hubconfs/bert_hubconf.py index c7bcfbffb6..a547a33c22 100644 --- a/hubconfs/bert_hubconf.py +++ b/hubconfs/bert_hubconf.py @@ -238,8 +238,7 @@ def bertForSequenceClassification(*args, **kwargs): seq_classif_logits = model(tokens_tensor, segments_tensors) # Or get the sequence classification loss >>> labels = torch.tensor([1]) - >>> with torch.no_grad(): - seq_classif_loss = model(tokens_tensor, segments_tensors, labels=labels) + >>> seq_classif_loss = model(tokens_tensor, segments_tensors, labels=labels) """ model = BertForSequenceClassification.from_pretrained(*args, **kwargs) return model @@ -273,8 +272,7 @@ def bertForMultipleChoice(*args, **kwargs): multiple_choice_logits = model(tokens_tensor, segments_tensors) # Or get the multiple choice loss >>> labels = torch.tensor([1]) - >>> with torch.no_grad(): - multiple_choice_loss = model(tokens_tensor, segments_tensors, labels=labels) + >>> multiple_choice_loss = model(tokens_tensor, segments_tensors, labels=labels) """ model = BertForMultipleChoice.from_pretrained(*args, **kwargs) return model @@ -306,8 +304,7 @@ def bertForQuestionAnswering(*args, **kwargs): start_logits, end_logits = model(tokens_tensor, segments_tensors) # Or get the total loss which is the sum of the CrossEntropy loss for the start and end token positions >>> start_positions, end_positions = torch.tensor([12]), torch.tensor([14]) - >>> with torch.no_grad(): - multiple_choice_loss = model(tokens_tensor, segments_tensors, start_positions=start_positions, end_positions=end_positions) + >>> multiple_choice_loss = model(tokens_tensor, segments_tensors, start_positions=start_positions, end_positions=end_positions) """ model = BertForQuestionAnswering.from_pretrained(*args, **kwargs) return model @@ -344,8 +341,7 @@ def bertForTokenClassification(*args, **kwargs): classif_logits = model(tokens_tensor, segments_tensors) # Or get the token classification loss >>> labels = torch.tensor([[0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 1, 0, 0, 1, 0]]) - >>> with torch.no_grad(): - classif_loss = model(tokens_tensor, segments_tensors, labels=labels) + >>> classif_loss = model(tokens_tensor, segments_tensors, labels=labels) """ model = BertForTokenClassification.from_pretrained(*args, **kwargs) return model From ee0308f79ded65dac82c53dfb03e9ff7f06aeee4 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Thu, 6 Jun 2019 17:30:49 +0200 Subject: [PATCH 28/32] fix typo --- hubconfs/bert_hubconf.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/hubconfs/bert_hubconf.py b/hubconfs/bert_hubconf.py index a547a33c22..0595bdeccb 100644 --- a/hubconfs/bert_hubconf.py +++ b/hubconfs/bert_hubconf.py @@ -238,7 +238,7 @@ def bertForSequenceClassification(*args, **kwargs): seq_classif_logits = model(tokens_tensor, segments_tensors) # Or get the sequence classification loss >>> labels = torch.tensor([1]) - >>> seq_classif_loss = model(tokens_tensor, segments_tensors, labels=labels) + >>> seq_classif_loss = model(tokens_tensor, segments_tensors, labels=labels) # set model.train() before if training this loss """ model = BertForSequenceClassification.from_pretrained(*args, **kwargs) return model @@ -272,7 +272,7 @@ def bertForMultipleChoice(*args, **kwargs): multiple_choice_logits = model(tokens_tensor, segments_tensors) # Or get the multiple choice loss >>> labels = torch.tensor([1]) - >>> multiple_choice_loss = model(tokens_tensor, segments_tensors, labels=labels) + >>> multiple_choice_loss = model(tokens_tensor, segments_tensors, labels=labels) # set model.train() before if training this loss """ model = BertForMultipleChoice.from_pretrained(*args, **kwargs) return model @@ -304,6 +304,7 @@ def bertForQuestionAnswering(*args, **kwargs): start_logits, end_logits = model(tokens_tensor, segments_tensors) # Or get the total loss which is the sum of the CrossEntropy loss for the start and end token positions >>> start_positions, end_positions = torch.tensor([12]), torch.tensor([14]) + # set model.train() before if training this loss >>> multiple_choice_loss = model(tokens_tensor, segments_tensors, start_positions=start_positions, end_positions=end_positions) """ model = BertForQuestionAnswering.from_pretrained(*args, **kwargs) @@ -341,7 +342,7 @@ def bertForTokenClassification(*args, **kwargs): classif_logits = model(tokens_tensor, segments_tensors) # Or get the token classification loss >>> labels = torch.tensor([[0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 1, 0, 0, 1, 0]]) - >>> classif_loss = model(tokens_tensor, segments_tensors, labels=labels) + >>> classif_loss = model(tokens_tensor, segments_tensors, labels=labels) # set model.train() before if training this loss """ model = BertForTokenClassification.from_pretrained(*args, **kwargs) return model From a3a604cefbd96d6f23366b6c9c87c3e98889461c Mon Sep 17 00:00:00 2001 From: jeonsworld <37530102+jeonsworld@users.noreply.github.com> Date: Mon, 10 Jun 2019 12:17:23 +0900 Subject: [PATCH 29/32] Update pregenerate_training_data.py apply Whole Word Masking technique. referred to [create_pretraining_data.py](https://github.com/google-research/bert/blob/master/create_pretraining_data.py) --- .../pregenerate_training_data.py | 82 +++++++++++++------ 1 file changed, 59 insertions(+), 23 deletions(-) diff --git a/examples/lm_finetuning/pregenerate_training_data.py b/examples/lm_finetuning/pregenerate_training_data.py index e6c3598a9f..6cb8954465 100644 --- a/examples/lm_finetuning/pregenerate_training_data.py +++ b/examples/lm_finetuning/pregenerate_training_data.py @@ -4,11 +4,11 @@ from tqdm import tqdm, trange from tempfile import TemporaryDirectory import shelve -from random import random, randrange, randint, shuffle, choice, sample +from random import random, randrange, randint, shuffle, choice from pytorch_pretrained_bert.tokenization import BertTokenizer import numpy as np import json - +import collections class DocumentDatabase: def __init__(self, reduce_memory=False): @@ -98,42 +98,77 @@ def truncate_seq_pair(tokens_a, tokens_b, max_num_tokens): else: trunc_tokens.pop() +MaskedLmInstance = collections.namedtuple("MaskedLmInstance", + ["index", "label"]) -def create_masked_lm_predictions(tokens, masked_lm_prob, max_predictions_per_seq, vocab_list): +def create_masked_lm_predictions(tokens, masked_lm_prob, max_predictions_per_seq, whole_word_mask, vocab_list): """Creates the predictions for the masked LM objective. This is mostly copied from the Google BERT repo, but with several refactors to clean it up and remove a lot of unnecessary variables.""" cand_indices = [] for (i, token) in enumerate(tokens): if token == "[CLS]" or token == "[SEP]": continue - cand_indices.append(i) + # Whole Word Masking means that if we mask all of the wordpieces + # corresponding to an original word. When a word has been split into + # WordPieces, the first token does not have any marker and any subsequence + # tokens are prefixed with ##. So whenever we see the ## token, we + # append it to the previous set of word indexes. + # + # Note that Whole Word Masking does *not* change the training code + # at all -- we still predict each WordPiece independently, softmaxed + # over the entire vocabulary. + if (whole_word_mask and len(cand_indices) >= 1 and token.startswith("##")): + cand_indices[-1].append(i) + else: + cand_indices.append([i]) num_to_mask = min(max_predictions_per_seq, max(1, int(round(len(tokens) * masked_lm_prob)))) shuffle(cand_indices) - mask_indices = sorted(sample(cand_indices, num_to_mask)) - masked_token_labels = [] - for index in mask_indices: - # 80% of the time, replace with [MASK] - if random() < 0.8: - masked_token = "[MASK]" - else: - # 10% of the time, keep original - if random() < 0.5: - masked_token = tokens[index] - # 10% of the time, replace with random word + masked_lms = [] + covered_indexes = set() + for index_set in cand_indices: + if len(masked_lms) >= num_to_mask: + break + # If adding a whole-word mask would exceed the maximum number of + # predictions, then just skip this candidate. + if len(masked_lms) + len(index_set) > num_to_mask: + continue + is_any_index_covered = False + for index in index_set: + if index in covered_indexes: + is_any_index_covered = True + break + if is_any_index_covered: + continue + for index in index_set: + covered_indexes.add(index) + + masked_token = None + # 80% of the time, replace with [MASK] + if random() < 0.8: + masked_token = "[MASK]" else: - masked_token = choice(vocab_list) - masked_token_labels.append(tokens[index]) - # Once we've saved the true label for that token, we can overwrite it with the masked version - tokens[index] = masked_token + # 10% of the time, keep original + if random() < 0.5: + masked_token = tokens[index] + # 10% of the time, replace with random word + else: + masked_token = choice(vocab_list) + masked_lms.append(MaskedLmInstance(index=index, label=tokens[index])) + tokens[index] = masked_token + + assert len(masked_lms) <= num_to_mask + masked_lms = sorted(masked_lms, key=lambda x: x.index) + mask_indices = [p.index for p in masked_lms] + masked_token_labels = [p.label for p in masked_lms] return tokens, mask_indices, masked_token_labels def create_instances_from_document( doc_database, doc_idx, max_seq_length, short_seq_prob, - masked_lm_prob, max_predictions_per_seq, vocab_list): + masked_lm_prob, max_predictions_per_seq, whole_word_mask, vocab_list): """This code is mostly a duplicate of the equivalent function from Google BERT's repo. However, we make some changes and improvements. Sampling is improved and no longer requires a loop in this function. Also, documents are sampled proportionally to the number of sentences they contain, which means each sentence @@ -213,7 +248,7 @@ def create_instances_from_document( segment_ids = [0 for _ in range(len(tokens_a) + 2)] + [1 for _ in range(len(tokens_b) + 1)] tokens, masked_lm_positions, masked_lm_labels = create_masked_lm_predictions( - tokens, masked_lm_prob, max_predictions_per_seq, vocab_list) + tokens, masked_lm_prob, max_predictions_per_seq, whole_word_mask, vocab_list) instance = { "tokens": tokens, @@ -237,7 +272,8 @@ def main(): choices=["bert-base-uncased", "bert-large-uncased", "bert-base-cased", "bert-base-multilingual", "bert-base-chinese"]) parser.add_argument("--do_lower_case", action="store_true") - + parser.add_argument("--do_whole_word_mask", action="store_true", + help="Whether to use whole word masking rather than per-WordPiece masking.") parser.add_argument("--reduce_memory", action="store_true", help="Reduce memory usage for large datasets by keeping data on disc rather than in memory") @@ -284,7 +320,7 @@ def main(): doc_instances = create_instances_from_document( docs, doc_idx, max_seq_length=args.max_seq_len, short_seq_prob=args.short_seq_prob, masked_lm_prob=args.masked_lm_prob, max_predictions_per_seq=args.max_predictions_per_seq, - vocab_list=vocab_list) + whole_word_mask=args.do_whole_word_mask, vocab_list=vocab_list) doc_instances = [json.dumps(instance) for instance in doc_instances] for instance in doc_instances: epoch_file.write(instance + '\n') From e02ce4dc7935582bf614eef0a480d3cb609ca062 Mon Sep 17 00:00:00 2001 From: Meet Pragnesh Shah Date: Tue, 11 Jun 2019 15:13:53 -0700 Subject: [PATCH 30/32] [hotfix] Fix frozen pooler parameters in SWAG example. --- examples/run_swag.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/run_swag.py b/examples/run_swag.py index 5e7ac85c63..59bb9866c3 100644 --- a/examples/run_swag.py +++ b/examples/run_swag.py @@ -390,7 +390,7 @@ def main(): # hack to remove pooler, which is not used # thus it produce None grad that break apex - param_optimizer = [n for n in param_optimizer if 'pooler' not in n[0]] + param_optimizer = [n for n in param_optimizer] no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight'] optimizer_grouped_parameters = [ From bcc9e93e6f585eec96444218b61b517f3f2f6314 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Fri, 14 Jun 2019 15:38:20 +0200 Subject: [PATCH 31/32] fix test --- tests/modeling_gpt2_test.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/modeling_gpt2_test.py b/tests/modeling_gpt2_test.py index 41cc9b8fd3..7817b98875 100644 --- a/tests/modeling_gpt2_test.py +++ b/tests/modeling_gpt2_test.py @@ -152,9 +152,10 @@ class GPT2ModelTest(unittest.TestCase): self.parent.assertListEqual( list(result["lm_logits"].size()), [self.batch_size, self.n_choices, self.seq_length, total_voc]) + self.parent.assertEqual(self.n_layer, len(result["presents"])) self.parent.assertListEqual( - list(result["presents"].size()), - [self.batch_size, self.n_choices, self.seq_length, total_voc]) + list(result["presents"][0].size()), + [2, self.batch_size * self.n_choices, self.n_head, self.seq_length, self.n_embd // self.n_head]) def check_gpt2_lm_head_loss_output(self, result): self.parent.assertListEqual( From 5e1207b8ad00fd649c0f35b9697cd67ce9897505 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Fri, 14 Jun 2019 16:28:25 +0200 Subject: [PATCH 32/32] add attention to all bert models and add test --- pytorch_pretrained_bert/modeling.py | 116 +++++++++++++++++++--------- tests/modeling_test.py | 71 ++++++++++++++--- 2 files changed, 140 insertions(+), 47 deletions(-) diff --git a/pytorch_pretrained_bert/modeling.py b/pytorch_pretrained_bert/modeling.py index 27682eb369..bfcbcc9edf 100644 --- a/pytorch_pretrained_bert/modeling.py +++ b/pytorch_pretrained_bert/modeling.py @@ -813,15 +813,20 @@ class BertForPreTraining(BertPreTrainedModel): masked_lm_logits_scores, seq_relationship_logits = model(input_ids, token_type_ids, input_mask) ``` """ - def __init__(self, config): + def __init__(self, config, output_attentions=False): super(BertForPreTraining, self).__init__(config) - self.bert = BertModel(config) + self.output_attentions = output_attentions + self.bert = BertModel(config, output_attentions=output_attentions) self.cls = BertPreTrainingHeads(config, self.bert.embeddings.word_embeddings.weight) self.apply(self.init_bert_weights) def forward(self, input_ids, token_type_ids=None, attention_mask=None, masked_lm_labels=None, next_sentence_label=None): - sequence_output, pooled_output = self.bert(input_ids, token_type_ids, attention_mask, + outputs = self.bert(input_ids, token_type_ids, attention_mask, output_all_encoded_layers=False) + if self.output_attentions: + all_attentions, sequence_output, pooled_output = outputs + else: + sequence_output, pooled_output = outputs prediction_scores, seq_relationship_score = self.cls(sequence_output, pooled_output) if masked_lm_labels is not None and next_sentence_label is not None: @@ -830,8 +835,9 @@ class BertForPreTraining(BertPreTrainedModel): next_sentence_loss = loss_fct(seq_relationship_score.view(-1, 2), next_sentence_label.view(-1)) total_loss = masked_lm_loss + next_sentence_loss return total_loss - else: - return prediction_scores, seq_relationship_score + elif self.output_attentions: + return all_attentions, prediction_scores, seq_relationship_score + return prediction_scores, seq_relationship_score class BertForMaskedLM(BertPreTrainedModel): @@ -876,23 +882,29 @@ class BertForMaskedLM(BertPreTrainedModel): masked_lm_logits_scores = model(input_ids, token_type_ids, input_mask) ``` """ - def __init__(self, config): + def __init__(self, config, output_attentions=False): super(BertForMaskedLM, self).__init__(config) - self.bert = BertModel(config) + self.output_attentions = output_attentions + self.bert = BertModel(config, output_attentions=output_attentions) self.cls = BertOnlyMLMHead(config, self.bert.embeddings.word_embeddings.weight) self.apply(self.init_bert_weights) def forward(self, input_ids, token_type_ids=None, attention_mask=None, masked_lm_labels=None): - sequence_output, _ = self.bert(input_ids, token_type_ids, attention_mask, + outputs = self.bert(input_ids, token_type_ids, attention_mask, output_all_encoded_layers=False) + if self.output_attentions: + all_attentions, sequence_output, _ = outputs + else: + sequence_output, _ = outputs prediction_scores = self.cls(sequence_output) if masked_lm_labels is not None: loss_fct = CrossEntropyLoss(ignore_index=-1) masked_lm_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), masked_lm_labels.view(-1)) return masked_lm_loss - else: - return prediction_scores + elif self.output_attentions: + return all_attentions, prediction_scores + return prediction_scores class BertForNextSentencePrediction(BertPreTrainedModel): @@ -938,23 +950,29 @@ class BertForNextSentencePrediction(BertPreTrainedModel): seq_relationship_logits = model(input_ids, token_type_ids, input_mask) ``` """ - def __init__(self, config): + def __init__(self, config, output_attentions=False): super(BertForNextSentencePrediction, self).__init__(config) - self.bert = BertModel(config) + self.output_attentions = output_attentions + self.bert = BertModel(config, output_attentions=output_attentions) self.cls = BertOnlyNSPHead(config) self.apply(self.init_bert_weights) def forward(self, input_ids, token_type_ids=None, attention_mask=None, next_sentence_label=None): - _, pooled_output = self.bert(input_ids, token_type_ids, attention_mask, + outputs = self.bert(input_ids, token_type_ids, attention_mask, output_all_encoded_layers=False) - seq_relationship_score = self.cls( pooled_output) + if self.output_attentions: + all_attentions, _, pooled_output = outputs + else: + _, pooled_output = outputs + seq_relationship_score = self.cls(pooled_output) if next_sentence_label is not None: loss_fct = CrossEntropyLoss(ignore_index=-1) next_sentence_loss = loss_fct(seq_relationship_score.view(-1, 2), next_sentence_label.view(-1)) return next_sentence_loss - else: - return seq_relationship_score + elif self.output_attentions: + return all_attentions, seq_relationship_score + return seq_relationship_score class BertForSequenceClassification(BertPreTrainedModel): @@ -1002,16 +1020,21 @@ class BertForSequenceClassification(BertPreTrainedModel): logits = model(input_ids, token_type_ids, input_mask) ``` """ - def __init__(self, config, num_labels): + def __init__(self, config, num_labels, output_attentions=False): super(BertForSequenceClassification, self).__init__(config) + self.output_attentions = output_attentions self.num_labels = num_labels - self.bert = BertModel(config) + self.bert = BertModel(config, output_attentions=output_attentions) self.dropout = nn.Dropout(config.hidden_dropout_prob) self.classifier = nn.Linear(config.hidden_size, num_labels) self.apply(self.init_bert_weights) def forward(self, input_ids, token_type_ids=None, attention_mask=None, labels=None): - _, pooled_output = self.bert(input_ids, token_type_ids, attention_mask, output_all_encoded_layers=False) + outputs = self.bert(input_ids, token_type_ids, attention_mask, output_all_encoded_layers=False) + if self.output_attentions: + all_attentions, _, pooled_output = outputs + else: + _, pooled_output = outputs pooled_output = self.dropout(pooled_output) logits = self.classifier(pooled_output) @@ -1019,8 +1042,9 @@ class BertForSequenceClassification(BertPreTrainedModel): loss_fct = CrossEntropyLoss() loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1)) return loss - else: - return logits + elif self.output_attentions: + return all_attentions, logits + return logits class BertForMultipleChoice(BertPreTrainedModel): @@ -1067,10 +1091,11 @@ class BertForMultipleChoice(BertPreTrainedModel): logits = model(input_ids, token_type_ids, input_mask) ``` """ - def __init__(self, config, num_choices): + def __init__(self, config, num_choices, output_attentions=False): super(BertForMultipleChoice, self).__init__(config) + self.output_attentions = output_attentions self.num_choices = num_choices - self.bert = BertModel(config) + self.bert = BertModel(config, output_attentions=output_attentions) self.dropout = nn.Dropout(config.hidden_dropout_prob) self.classifier = nn.Linear(config.hidden_size, 1) self.apply(self.init_bert_weights) @@ -1079,7 +1104,11 @@ class BertForMultipleChoice(BertPreTrainedModel): flat_input_ids = input_ids.view(-1, input_ids.size(-1)) flat_token_type_ids = token_type_ids.view(-1, token_type_ids.size(-1)) if token_type_ids is not None else None flat_attention_mask = attention_mask.view(-1, attention_mask.size(-1)) if attention_mask is not None else None - _, pooled_output = self.bert(flat_input_ids, flat_token_type_ids, flat_attention_mask, output_all_encoded_layers=False) + outputs = self.bert(flat_input_ids, flat_token_type_ids, flat_attention_mask, output_all_encoded_layers=False) + if self.output_attentions: + all_attentions, _, pooled_output = outputs + else: + _, pooled_output = outputs pooled_output = self.dropout(pooled_output) logits = self.classifier(pooled_output) reshaped_logits = logits.view(-1, self.num_choices) @@ -1088,8 +1117,9 @@ class BertForMultipleChoice(BertPreTrainedModel): loss_fct = CrossEntropyLoss() loss = loss_fct(reshaped_logits, labels) return loss - else: - return reshaped_logits + elif self.output_attentions: + return all_attentions, reshaped_logits + return reshaped_logits class BertForTokenClassification(BertPreTrainedModel): @@ -1137,16 +1167,21 @@ class BertForTokenClassification(BertPreTrainedModel): logits = model(input_ids, token_type_ids, input_mask) ``` """ - def __init__(self, config, num_labels): + def __init__(self, config, num_labels, output_attentions=False): super(BertForTokenClassification, self).__init__(config) + self.output_attentions = output_attentions self.num_labels = num_labels - self.bert = BertModel(config) + self.bert = BertModel(config, output_attentions=output_attentions) self.dropout = nn.Dropout(config.hidden_dropout_prob) self.classifier = nn.Linear(config.hidden_size, num_labels) self.apply(self.init_bert_weights) def forward(self, input_ids, token_type_ids=None, attention_mask=None, labels=None): - sequence_output, _ = self.bert(input_ids, token_type_ids, attention_mask, output_all_encoded_layers=False) + outputs = self.bert(input_ids, token_type_ids, attention_mask, output_all_encoded_layers=False) + if self.output_attentions: + all_attentions, sequence_output, _ = outputs + else: + sequence_output, _ = outputs sequence_output = self.dropout(sequence_output) logits = self.classifier(sequence_output) @@ -1161,8 +1196,9 @@ class BertForTokenClassification(BertPreTrainedModel): else: loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1)) return loss - else: - return logits + elif self.output_attentions: + return all_attentions, logits + return logits class BertForQuestionAnswering(BertPreTrainedModel): @@ -1212,16 +1248,19 @@ class BertForQuestionAnswering(BertPreTrainedModel): start_logits, end_logits = model(input_ids, token_type_ids, input_mask) ``` """ - def __init__(self, config): + def __init__(self, config, output_attentions=False): super(BertForQuestionAnswering, self).__init__(config) - self.bert = BertModel(config) - # TODO check with Google if it's normal there is no dropout on the token classifier of SQuAD in the TF version - # self.dropout = nn.Dropout(config.hidden_dropout_prob) + self.output_attentions = output_attentions + self.bert = BertModel(config, output_attentions=output_attentions) self.qa_outputs = nn.Linear(config.hidden_size, 2) self.apply(self.init_bert_weights) def forward(self, input_ids, token_type_ids=None, attention_mask=None, start_positions=None, end_positions=None): - sequence_output, _ = self.bert(input_ids, token_type_ids, attention_mask, output_all_encoded_layers=False) + outputs = self.bert(input_ids, token_type_ids, attention_mask, output_all_encoded_layers=False) + if self.output_attentions: + all_attentions, sequence_output, _ = outputs + else: + sequence_output, _ = outputs logits = self.qa_outputs(sequence_output) start_logits, end_logits = logits.split(1, dim=-1) start_logits = start_logits.squeeze(-1) @@ -1243,5 +1282,6 @@ class BertForQuestionAnswering(BertPreTrainedModel): end_loss = loss_fct(end_logits, end_positions) total_loss = (start_loss + end_loss) / 2 return total_loss - else: - return start_logits, end_logits + elif self.output_attentions: + return all_attentions, start_logits, end_logits + return start_logits, end_logits diff --git a/tests/modeling_test.py b/tests/modeling_test.py index 5cde383fdf..79993ed840 100644 --- a/tests/modeling_test.py +++ b/tests/modeling_test.py @@ -28,7 +28,7 @@ import torch from pytorch_pretrained_bert import (BertConfig, BertModel, BertForMaskedLM, BertForNextSentencePrediction, BertForPreTraining, BertForQuestionAnswering, BertForSequenceClassification, - BertForTokenClassification) + BertForTokenClassification, BertForMultipleChoice) from pytorch_pretrained_bert.modeling import PRETRAINED_MODEL_ARCHIVE_MAP @@ -56,6 +56,7 @@ class BertModelTest(unittest.TestCase): type_sequence_label_size=2, initializer_range=0.02, num_labels=3, + num_choices=4, scope=None): self.parent = parent self.batch_size = batch_size @@ -77,6 +78,7 @@ class BertModelTest(unittest.TestCase): self.type_sequence_label_size = type_sequence_label_size self.initializer_range = initializer_range self.num_labels = num_labels + self.num_choices = num_choices self.scope = scope def prepare_config_and_inputs(self): @@ -92,9 +94,11 @@ class BertModelTest(unittest.TestCase): sequence_labels = None token_labels = None + choice_labels = None if self.use_labels: sequence_labels = BertModelTest.ids_tensor([self.batch_size], self.type_sequence_label_size) token_labels = BertModelTest.ids_tensor([self.batch_size, self.seq_length], self.num_labels) + choice_labels = BertModelTest.ids_tensor([self.batch_size], self.num_choices) config = BertConfig( vocab_size_or_config_json_file=self.vocab_size, @@ -109,14 +113,14 @@ class BertModelTest(unittest.TestCase): type_vocab_size=self.type_vocab_size, initializer_range=self.initializer_range) - return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels + return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels def check_loss_output(self, result): self.parent.assertListEqual( list(result["loss"].size()), []) - def create_bert_model(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels): + def create_bert_model(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): model = BertModel(config=config) model.eval() all_encoder_layers, pooled_output = model(input_ids, token_type_ids, input_mask) @@ -137,7 +141,7 @@ class BertModelTest(unittest.TestCase): self.parent.assertListEqual(list(result["pooled_output"].size()), [self.batch_size, self.hidden_size]) - def create_bert_for_masked_lm(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels): + def create_bert_for_masked_lm(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): model = BertForMaskedLM(config=config) model.eval() loss = model(input_ids, token_type_ids, input_mask, token_labels) @@ -153,7 +157,7 @@ class BertModelTest(unittest.TestCase): list(result["prediction_scores"].size()), [self.batch_size, self.seq_length, self.vocab_size]) - def create_bert_for_next_sequence_prediction(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels): + def create_bert_for_next_sequence_prediction(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): model = BertForNextSentencePrediction(config=config) model.eval() loss = model(input_ids, token_type_ids, input_mask, sequence_labels) @@ -170,7 +174,7 @@ class BertModelTest(unittest.TestCase): [self.batch_size, 2]) - def create_bert_for_pretraining(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels): + def create_bert_for_pretraining(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): model = BertForPreTraining(config=config) model.eval() loss = model(input_ids, token_type_ids, input_mask, token_labels, sequence_labels) @@ -191,7 +195,7 @@ class BertModelTest(unittest.TestCase): [self.batch_size, 2]) - def create_bert_for_question_answering(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels): + def create_bert_for_question_answering(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): model = BertForQuestionAnswering(config=config) model.eval() loss = model(input_ids, token_type_ids, input_mask, sequence_labels, sequence_labels) @@ -212,7 +216,7 @@ class BertModelTest(unittest.TestCase): [self.batch_size, self.seq_length]) - def create_bert_for_sequence_classification(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels): + def create_bert_for_sequence_classification(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): model = BertForSequenceClassification(config=config, num_labels=self.num_labels) model.eval() loss = model(input_ids, token_type_ids, input_mask, sequence_labels) @@ -229,7 +233,7 @@ class BertModelTest(unittest.TestCase): [self.batch_size, self.num_labels]) - def create_bert_for_token_classification(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels): + def create_bert_for_token_classification(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): model = BertForTokenClassification(config=config, num_labels=self.num_labels) model.eval() loss = model(input_ids, token_type_ids, input_mask, token_labels) @@ -246,6 +250,49 @@ class BertModelTest(unittest.TestCase): [self.batch_size, self.seq_length, self.num_labels]) + def create_bert_for_multiple_choice(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + model = BertForMultipleChoice(config=config, num_choices=self.num_choices) + model.eval() + multiple_choice_inputs_ids = input_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous() + multiple_choice_token_type_ids = token_type_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous() + multiple_choice_input_mask = input_mask.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous() + loss = model(multiple_choice_inputs_ids, + multiple_choice_token_type_ids, + multiple_choice_input_mask, + choice_labels) + logits = model(multiple_choice_inputs_ids, + multiple_choice_token_type_ids, + multiple_choice_input_mask) + outputs = { + "loss": loss, + "logits": logits, + } + return outputs + + def check_bert_for_multiple_choice(self, result): + self.parent.assertListEqual( + list(result["logits"].size()), + [self.batch_size, self.num_choices]) + + + def create_and_check_bert_for_attentions(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + for model_class in (BertModel, BertForMaskedLM, BertForNextSentencePrediction, + BertForPreTraining, BertForQuestionAnswering, BertForSequenceClassification, + BertForTokenClassification): + if model_class in [BertForSequenceClassification, + BertForTokenClassification]: + model = model_class(config=config, num_labels=self.num_labels, output_attentions=True) + else: + model = model_class(config=config, output_attentions=True) + model.eval() + output = model(input_ids, token_type_ids, input_mask) + attentions = output[0] + self.parent.assertEqual(len(attentions), self.num_hidden_layers) + self.parent.assertListEqual( + list(attentions[0].size()), + [self.batch_size, self.num_attention_heads, self.seq_length, self.seq_length]) + + def test_default(self): self.run_tester(BertModelTest.BertModelTester(self)) @@ -300,6 +347,12 @@ class BertModelTest(unittest.TestCase): tester.check_bert_for_token_classification_output(output_result) tester.check_loss_output(output_result) + output_result = tester.create_bert_for_multiple_choice(*config_and_inputs) + tester.check_bert_for_multiple_choice(output_result) + tester.check_loss_output(output_result) + + tester.create_and_check_bert_for_attentions(*config_and_inputs) + @classmethod def ids_tensor(cls, shape, vocab_size, rng=None, name=None): """Creates a random int32 tensor of the shape within the vocab size."""