SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Reorganize ALBERT conversion script

Browse Source
This commit is contained in:
Lysandre
2019-12-02 15:01:52 -05:00
parent b0ee7c7df3
commit b3d834ae11
1 changed files with 26 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
transformers/modeling_albert.py
View File

@@ -68,14 +68,36 @@ def load_tf_weights_in_albert(model, config, tf_checkpoint_path):
for name, array in zip(names, arrays): for name, array in zip(names, arrays):
original_name = name original_name = name
# If saved from the TF HUB module
name = name.replace("module/", "")
# Renaming and simplifying
name = name.replace("ffn_1", "ffn") name = name.replace("ffn_1", "ffn")
name = name.replace("/bert/", "/albert/") name = name.replace("bert/", "albert/")
name = name.replace("ffn/intermediate/output", "ffn_output")
name = name.replace("attention_1", "attention") name = name.replace("attention_1", "attention")
name = name.replace("cls/predictions", "predictions")
name = name.replace("transform/", "") name = name.replace("transform/", "")
name = name.replace("LayerNorm_1", "full_layer_layer_norm") name = name.replace("LayerNorm_1", "full_layer_layer_norm")
name = name.replace("LayerNorm", "attention/LayerNorm") name = name.replace("LayerNorm", "attention/LayerNorm")
name = name.replace("transformer/", "")
# The feed forward layer had an 'intermediate' step which has been abstracted away
name = name.replace("intermediate/dense/", "")
name = name.replace("ffn/intermediate/output/dense/", "ffn_output/")
# ALBERT attention was split between self and output which have been abstracted away
name = name.replace("/output/", "/")
name = name.replace("/self/", "/")
# The pooler is a linear layer
name = name.replace("pooler/dense", "pooler")
# The classifier was simplified to predictions from cls/predictions
name = name.replace("cls/predictions", "predictions")
name = name.replace("predictions/attention", "predictions")
# Naming was changed to be more explicit
name = name.replace("embeddings/attention", "embeddings")
name = name.replace("inner_group_", "albert_layers/") name = name.replace("inner_group_", "albert_layers/")
name = name.replace("group_", "albert_layer_groups/") name = name.replace("group_", "albert_layer_groups/")
name = name.split('/') name = name.split('/')