here is how i create it:
from torch import nn
from transformers import Trainer
class custom_optimizer(Trainer):
def create_optimizer(self):
optimizer = AdamW(model.parameters(),
lr = 2e-7
)
return self.optimizer
from torch.optim.lr_scheduler import MultiStepLR
class custom_scheduler(Trainer):
def create_scheduler(self, num_training_steps):
scheduler = MultiStepLR(optimizer, milestones=[1,5], gamma=0.5)
return self.scheduler
and my trainer looks like this:
from transformers import Trainer, AdamW
trainer = Trainer(
model=model,
args=training_args,
train_dataset=small_train_dataset,
eval_dataset=small_eval_dataset,
compute_metrics=compute_metrics,
callbacks=[EarlyStoppingCallback(early_stopping_patience=3)],
optimizers = (custom_optimizer(model).create_optimizer(), custom_scheduler(model).create_scheduler(total_steps))
)
trainer.train()
But this is giving me an error:
NameError: name 'optimizer' is not defined
Can you please help me to resolve this issue?