I’m having issues useing Ray Tune. I’m trying to use ‘f1_metric’
my code is
def f1_metric(predictions):
class_probs = predictions[0]
true_labels = predictions[1]
f1s = []
for i in range(len(class_probs)):
cp = class_probs[i]
tl = true_labels[i]
pred_labels = [np.argmax(x) for x in class_probs[0]]
f1s.append(f1_score(tl,pred_labels))
return {'f1_metric':np.mean(f1s)}
trainer = CustomTrainer(
model_init=model_init,
args=training_args,
train_dataset=train_dataset,
eval_dataset=test_dataset,
compute_metrics=f1_metric,
tokenizer=tokenizer,
data_collator=data_collator)
def hp_space_fn(*args, **kwargs):
config = {
"warmup_steps": tune.choice([50, 100, 500]),
"learning_rate": tune.choice([2e-5, 3e-5, 1e-5]),
"num_train_epochs": tune.quniform(0.01, .02, 0.01),
'label_smoothing_factor': tune.choice([0.01,0.03,.07,.10]),
'weight_picks_0' : tune.choice([0.01]),
'weight_picks_1' : tune.choice([0.05,.07,.10,.2,.5]),
}
wandb_config = {
"wandb": {
"project": os.environ.get(
"WANDB_PROJECT",
"wandb_project"),
"api_key": os.environ.get("API_KEY"),
"log_config": True
}
}
config.update(wandb_config)
return config
best_run = trainer.hyperparameter_search(
direction="maximize",
backend="ray",
scheduler=PopulationBasedTraining(
time_attr="time_total_s",
mode="max",
metric='f1_metric',
perturbation_interval=600.0
),
hp_space=hp_space_fn,
loggers=DEFAULT_LOGGERS + (WandbLogger, ),
time_budget_s=4000
)
No matter what I name the f1_metric I can’t get Ray Tune to recognize it.
When I remove the metric from best_run
best_run = trainer.hyperparameter_search(
direction="maximize",
backend="ray",
scheduler=PopulationBasedTraining(
time_attr="time_total_s",
mode="max",
#### metric='f1_metric',
perturbation_interval=600.0
),
hp_space=hp_space_fn,
loggers=DEFAULT_LOGGERS + (WandbLogger, ),
time_budget_s=4000
)
RuntimeError: Cannot find metric _metric in trial result {‘objective’: 0.0018931623078665865, ‘eval_loss’: 0.5891237854957581, ‘eval_f1_metric’: 0.0018931623078665865, ‘eval_runtime’: 168.5186, ‘eval_samples_per_second’: 5.204, ‘eval_steps_per_second’: 1.305, ‘epoch’: 0.0, ‘time_this_iter_s’: 219.17230415344238, ‘done’: False, ‘timesteps_total’: None, ‘episodes_total’: None, ‘training_iteration’: 1, ‘trial_id’: ‘da2b7_00000’, ‘experiment_id’: ‘f36eae0544124c1cb87fdbbe9cda499c’, ‘date’: ‘2022-01-10_15-52-59’, ‘timestamp’: 1641829979, ‘time_total_s’: 219.17230415344238, ‘pid’: 2537, ‘hostname’: ‘9996f5f6eb70’, ‘node_ip’: ‘172.28.0.2’, ‘time_since_restore’: 219.17230415344238, ‘timesteps_since_restore’: 0, ‘iterations_since_restore’: 1, ‘config/warmup_steps’: 500, ‘config/learning_rate’: 2e-05, ‘config/num_train_epochs’: 0.02, ‘config/label_smoothing_factor’: 0.03, ‘config/weight_picks_0’: 0.01, ‘config/weight_picks_1’: 0.07}. Make sure that this attribute is returned in the results of your Trainable.If this error is expected, you can change this to a warning message by setting PBT(require_attrs=False)