From 6b2136a8a9f70f99f719a10e793e6a7f4ef26565 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 20 Nov 2018 10:12:44 +0100 Subject: [PATCH] fixing weights decay in run_squad example --- examples/run_squad.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/run_squad.py b/examples/run_squad.py index b166757656..7f58ddc35a 100644 --- a/examples/run_squad.py +++ b/examples/run_squad.py @@ -842,8 +842,8 @@ def main(): param_optimizer = list(model.named_parameters()) no_decay = ['bias', 'gamma', 'beta'] optimizer_grouped_parameters = [ - {'params': [p for n, p in param_optimizer if n not in no_decay], 'weight_decay_rate': 0.01}, - {'params': [p for n, p in param_optimizer if n in no_decay], 'weight_decay_rate': 0.0} + {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay_rate': 0.01}, + {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay_rate': 0.0} ] optimizer = BertAdam(optimizer_grouped_parameters, lr=args.learning_rate,