From e0f867a9ba454008ecb830832bcb76c5b821e4fc Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Sat, 31 Aug 2019 00:50:59 -0400 Subject: [PATCH 1/2] XLNet bias fix on resize embeddings (cf #1124) --- pytorch_transformers/modeling_utils.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pytorch_transformers/modeling_utils.py b/pytorch_transformers/modeling_utils.py index 0d4fce67f0..4550c75620 100644 --- a/pytorch_transformers/modeling_utils.py +++ b/pytorch_transformers/modeling_utils.py @@ -327,6 +327,14 @@ class PreTrainedModel(nn.Module): else: first_module.weight = second_module.weight + if hasattr(first_module, 'bias'): + first_module.bias.data = torch.nn.functional.pad( + first_module.bias.data, + (0, first_module.weight.shape[0] - first_module.bias.shape[0]), + 'constant', + 0 + ) + def resize_token_embeddings(self, new_num_tokens=None): """ Resize input token embeddings matrix of the model if new_num_tokens != config.vocab_size. Take care of tying weights embeddings afterwards if the model class has a `tie_weights()` method. From ea86bef545f3790faa48f4e002482695641ce490 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Sat, 31 Aug 2019 00:56:22 -0400 Subject: [PATCH 2/2] Check for None --- pytorch_transformers/modeling_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_transformers/modeling_utils.py b/pytorch_transformers/modeling_utils.py index 4550c75620..e4b67d8095 100644 --- a/pytorch_transformers/modeling_utils.py +++ b/pytorch_transformers/modeling_utils.py @@ -327,7 +327,7 @@ class PreTrainedModel(nn.Module): else: first_module.weight = second_module.weight - if hasattr(first_module, 'bias'): + if hasattr(first_module, 'bias') and first_module.bias is not None: first_module.bias.data = torch.nn.functional.pad( first_module.bias.data, (0, first_module.weight.shape[0] - first_module.bias.shape[0]),