SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

add distillation_configs

Browse Source
This commit is contained in:
VictorSanh
2019-10-02 11:00:46 -04:00
committed by Victor SANH
parent ebb32261b1
commit 38084507c4
2 changed files with 25 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
examples/distillation/training_configs/distilbert-base-uncased.json Normal file
View File

@@ -0,0 +1,15 @@
{
"activation": "gelu",
"attention_dropout": 0.1,
"dim": 768,
"dropout": 0.1,
"hidden_dim": 3072,
"initializer_range": 0.02,
"max_position_embeddings": 512,
"n_heads": 12,
"n_layers": 6,
"sinusoidal_pos_embds": true,
"tie_weights_": true,
"vocab_size": 30522
}

10
examples/distillation/training_configs/distilgpt2.json Normal file
View File

@@ -0,0 +1,10 @@
{
"initializer_range": 0.02,
"layer_norm_epsilon": 0.00001,
"n_ctx": 1024,
"n_embd": 768,
"n_head": 12,
"n_layer": 6,
"n_positions": 1024,
"vocab_size": 50257
}