How can I set different learning rates for different parameters in the model? I have rewritten the optimizers and separately set the learning rate for the act_fn
in the model, but during training, I found that it doesn’t work:
if optimizer_grouped_parameters is None:
# Default parameter groups
decay_parameters = Trainer.get_decay_parameter_names(None, model)
optimizer_grouped_parameters = [
{
'params': [p for n, p in model.named_parameters() if (n in decay_parameters and p.requires_grad and
'act_fn' not in n)],
'weight_decay': args.weight_decay,
},
{
'params': [p for n, p in model.named_parameters() if (n not in decay_parameters and p.requires_grad)],
'weight_decay': 0.0,
},
{
"params": [
p for n, p in model.named_parameters() if (n in decay_parameters and p.requires_grad and
'act_fn' in n)
],
"weight_decay": 0.0,
'lr': 0.5
},
]
optimizer_cls, optimizer_kwargs = Trainer.get_optimizer_cls_and_kwargs(args)