SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

minor spring cleaning - missing configs + processing

Browse Source
This commit is contained in:
VictorSanh
2020-01-10 19:14:58 -05:00
parent b1e1a9f9b2
commit ebba9e929d
3 changed files with 46 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
examples/distillation/training_configs/distilbert-base-multilingual-cased.json Normal file
View File

@@ -0,0 +1,15 @@
{
"activation": "gelu",
"attention_dropout": 0.1,
"dim": 768,
"dropout": 0.1,
"hidden_dim": 3072,
"initializer_range": 0.02,
"max_position_embeddings": 512,
"n_heads": 12,
"n_layers": 6,
"sinusoidal_pos_embds": true,
"tie_weights_": true,
"vocab_size": 119547
}

14
examples/distillation/training_configs/distilroberta-base.json Normal file
View File

@@ -0,0 +1,14 @@
{
"vocab_size": 50265,
"hidden_size": 768,
"num_hidden_layers": 6,
"num_attention_heads": 12,
"intermediate_size": 3072,
"hidden_act": "gelu",
"hidden_dropout_prob": 0.1,
"attention_probs_dropout_prob": 0.1,
"max_position_embeddings": 514,
"type_vocab_size": 1,
"initializer_range": 0.02,
"layer_norm_eps": 0.00001
}