The model did not return a loss from the inputs, only the following keys: logits. For reference, the inputs it received are input_values


Fine-Tuning  of Self-Supervised Models for Audio based Bird Detection.
I am trying to add linear classifier on top of wav2vec2. I want to fine tune the wav2vec2 mdel for multi-class classification as below.

I followed the tutorial from HuggingFace, but I am still getting the following error 

ValueError: The model did not return a loss from the inputs, only the following keys: logits. For reference, the inputs it received are input_values.
 

"""
import torch
import numpy as np
import pandas as pd 
import librosa 
from sklearn.metrics import confusion_matrix
from sklearn.metrics import confusion_matrix, recall_score, accuracy_score


from transformers import pipeline, AutoFeatureExtractor, Wav2Vec2FeatureExtractor, Wav2Vec2Processor, AutoModelForAudioClassification, TrainingArguments, Trainer
from datasets import load_dataset, Dataset, load_metric, Audio, concatenate_datasets
"""

> FineTuning Wav2Vec2 for bird classification

class FineTuneSSLModels():
    def __init__(self):
        self.categories = {'Pigeon': 0, 'Sparrow': 1, 'Crow': 2, 'Eagle': 3, 'Hawk': 4, 'Parrot': 5, 'Dove': 6, 'Peacock': 7}
        self.freeze_encoder = False
        self.transformer = False
        self.pre_trained_model = "facebook/wav2vec2-base-960h"  #w2v2 base 960 hours librispeech

        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
        self.f1_metric = load_metric("f1")
        self.recall_metric = load_metric("recall")
        self.target_sr = 16000
        #Read CSV Files
        self.path_to_train='train.csv'
        self.path_to_val='val.csv'
        self.path_to_test='test.csv'

        self.df_train = pd.read_csv(self.path_to_train, encoding='utf-8')
        self.df_val = pd.read_csv(self.path_to_val, encoding='utf-8')
        self.df_test = pd.read_csv(self.path_to_test, encoding='utf-8')
        print("self train", self.df_train)
        
        self.labels=sorted(self.df_train.Label.unique())
        self.label_dict = {self.labels[i]: [j for j in range(len(self.labels))][i] for i in range(len(self.labels))}
        print(self.label_dict)
        self.df_train = self.df_train.replace({"Label": self.label_dict})
        self.df_val = self.df_val.replace({"Label": self.label_dict})
        self.df_test = self.df_test.replace({"Label": self.label_dict})
        
        #Get Full Path 
        self.df_train['FilePath']='full_path/train/' + self.df_train['FileName']
        self.df_val['FilePath']=''full_path/eval/' + self.df_val['FileName'] 
        self.df_test['FilePath']='full_pathtest/' + self.df_test['FileName']

        #Create Datasets
        self.train_dataset = Dataset.from_pandas(self.df_train)
        self.val_dataset = Dataset.from_pandas(self.df_val)
        self.test_dataset = Dataset.from_pandas(self.df_test)
        
        self.train_dataset = self.train_dataset.map(self.preprocess_data)
        self.val_dataset = self.val_dataset.map(self.preprocess_data)
        self.test_dataset = self.test_dataset.map(self.preprocess_data)
        self.feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(self.pre_trained_model, cache_dir="SSLFineTune/")
        self.processor = Wav2Vec2Processor.from_pretrained(self.pre_trained_model, cache_dir="SSLFineTune/")

        self.train_dataset = self.train_dataset.map(self.prepare_datasets, remove_columns = ['FilePath', 
'SpeakerLabel', 'FileName'], batched=True, batch_size=1)
self.val_dataset = self.val_dataset.map(self.prepare_datasets, remove_columns = ['FilePath', 'SpeakerLabel', 'FileName'], batched=True, batch_size=1)
self.test_dataset = self.test_dataset.map(self.prepare_datasets, remove_columns = ['FilePath', 'SpeakerLabel', 'FileName'], batched=True, batch_size=1)

        self.model = AutoModelForAudioClassification.from_pretrained(
                                                                self.pre_trained_model,
                                                                trust_remote_code=True,
                                                                cache_dir="SSLFineTune/",
                                                                num_labels = len(self.df_train['Label'].unique()),                                                               
                                                                )
        


       #Classification head for 8-classes
        self.model.classifier=torch.nn.Linear(in_features=256, out_features=8, bias=True)

        self.freeze_feature_extractor = False
        self.freeze_transformer = False
        if self.freeze_feature_extractor:
            self.model.freeze_feature_extractor()
        if self.freeze_transformer:
            self.model.freeze_transformer()


        self.args = TrainingArguments(
                "SSLFineTune/",
                overwrite_output_dir=True,
                evaluation_strategy = "epoch",
                save_strategy = "epoch",
                learning_rate=3e-5,
                per_device_train_batch_size=8,
                gradient_accumulation_steps=1,
                per_device_eval_batch_size=1,
                num_train_epochs=10,
                warmup_ratio=0.1,
                logging_steps=10,
                load_best_model_at_end=True,
                metric_for_best_model="uar",
                push_to_hub=False,
                gradient_checkpointing=True,
                save_total_limit=5
                )
        
        self.trainer = Trainer(
                self.model,
                self.args,
                train_dataset=self.train_dataset,
                eval_dataset=self.val_dataset,
                tokenizer=self.feature_extractor,
                compute_metrics=self.compute_metrics)

        self.trainer.train()

        self.predictions = self.trainer.predict(self.val_dataset)
        print(self.compute_metrics(self.predictions))



    def compute_metrics(self, eval_pred):
        """Computes accuracy on a batch of predictions"""
        print(evl_pred)
        predictions = np.argmax(eval_pred.predictions, axis=1)
        recall = self.recall_metric.compute(predictions=predictions, references=eval_pred.label_ids,average="macro")
        # f1 = f1_metric.compute(predictions=predictions, references=eval_pred.label_ids, average="macro")
        #return {"f1": f1, "spearmanr": spearmanr}
        return recall


    def preprocess_data(self, audio_example):
        """Adds Audio and Sampling Rate to Panda Frame in another columns"""
        audio_example['audio'], audio_example['sampling_rate'] = librosa.load(audio_example["FilePath"], sr=16000)
        #print("Single Audio", np.array(audio_example['audio']).shape, audio_example)
        #exit(0)
        return audio_example

    def prepare_datasets(self, audio_example):
        audio_data = audio_example['audio']     

        enc_embeds = self.feature_extractor(audio_data, sampling_rate = self.feature_extractor.sampling_rate)
        return enc_embeds

    




if __name__=="__main__":
    initiate = FineTuneSSLModels()

2 Likes

Edit: fixed for me by changing BertModel to BertForSequenceClassification.

I’m getting the same error, also following the tutorials in the course, chapter 3: Fine-tuning a model with the Trainer API - Hugging Face Course.

My code:

import evaluate
import numpy as np
from datasets import load_dataset
from transformers import (
    BertModel,
    BertTokenizerFast,
    TrainingArguments,
    Trainer,
    EvalPrediction,
)

checkpoint = "bert-base-uncased"
model = BertModel.from_pretrained(checkpoint)
tokenizer = BertTokenizerFast.from_pretrained(checkpoint)
raw_ds = load_dataset("glue", "mrpc")
metric = evaluate.load("glue", "mrpc")

dataset = raw_ds.map(
    lambda x: tokenizer(x["sentence1"], x["sentence2"], truncation=True),
    batched=True,
)
dataset = dataset.remove_columns(["sentence1", "sentence2", "idx"])
dataset = dataset.rename_column("label", "labels")
dataset = dataset.with_format("torch")

trainer_args = TrainingArguments("test-trainer", evaluation_strategy="epoch")


def compute_metrics(eval_preds: EvalPrediction):
    x, y = eval_preds
    preds = np.argmax(x, -1)
    return metric.compute(predictions=preds, references=y)


trainer = Trainer(
    model=model,
    args=trainer_args,
    train_dataset=dataset["train"],
    eval_dataset=dataset["validation"],
    tokenizer=tokenizer,
    compute_metrics=compute_metrics,
)

trainer.train()

I get a similar error when using

# Set DistilBERT tokenizer
tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased")

# Define DistilBERT as our base model:
model = AutoModelForSequenceClassification.from_pretrained("distilbert-base-uncased", num_labels=9)

# Use data_collector to convert our samples to PyTorch tensors and concatenate them with the correct amount of padding
data_collator = DataCollatorWithPadding(tokenizer=tokenizer)

Specifically, when using

# Define a new Trainer with all the objects we constructed so far
repo_name = "sentiment-model-amazon-reviews-distilbert"

training_args = TrainingArguments(
    output_dir=repo_name,
    learning_rate=2e-5,
    per_device_train_batch_size=16,
    per_device_eval_batch_size=16,
    num_train_epochs=1,
    weight_decay=0.01,
    save_strategy="epoch", 
    push_to_hub=True
)

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=tokenized_train,
    eval_dataset=tokenized_cal,
    tokenizer=tokenizer,
    data_collator=data_collator,
    compute_metrics=compute_metrics
)

# Train and push to hub
trainer.train()

returns

---------------------------------------------------------------------------

ValueError                                Traceback (most recent call last)

<ipython-input-50-f0acdc25090a> in <module>
     24 
     25 # Train and push to hub
---> 26 trainer.train()

3 frames

/usr/local/lib/python3.8/dist-packages/transformers/trainer.py in train(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)
   1541             self._inner_training_loop, self._train_batch_size, args.auto_find_batch_size
   1542         )
-> 1543         return inner_training_loop(
   1544             args=args,
   1545             resume_from_checkpoint=resume_from_checkpoint,

/usr/local/lib/python3.8/dist-packages/transformers/trainer.py in _inner_training_loop(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval)
   1789                         tr_loss_step = self.training_step(model, inputs)
   1790                 else:
-> 1791                     tr_loss_step = self.training_step(model, inputs)
   1792 
   1793                 if (

/usr/local/lib/python3.8/dist-packages/transformers/trainer.py in training_step(self, model, inputs)
   2537 
   2538         with self.compute_loss_context_manager():
-> 2539             loss = self.compute_loss(model, inputs)
   2540 
   2541         if self.args.n_gpu > 1:

/usr/local/lib/python3.8/dist-packages/transformers/trainer.py in compute_loss(self, model, inputs, return_outputs)
   2582         else:
   2583             if isinstance(outputs, dict) and "loss" not in outputs:
-> 2584                 raise ValueError(
   2585                     "The model did not return a loss from the inputs, only the following keys: "
   2586                     f"{','.join(outputs.keys())}. For reference, the inputs it received are {','.join(inputs.keys())}."

ValueError: The model did not return a loss from the inputs, only the following keys: logits. For reference, the inputs it received are input_ids,attention_mask.
1 Like

Rename columns to text and labels for text classification using distilbert-base-uncased model. It needs to be checked for other domains and models too.

6 Likes

I am trying to fine-tune mT5 for reading comprehension task and I get the same error. Here is my code:

tokenizer = MT5Tokenizer.from_pretrained(model_dir)

def model_init1():
    return MT5ForQuestionAnswering.from_pretrained(model_dir)

trainer = Seq2SeqTrainer(
    model_init=model_init1,
    args=args,
    train_dataset=training_set,
    eval_dataset=validation_set,
    data_collator=data_collator,
    tokenizer=tokenizer,
)

# Training the model
trainer.train()

the error:

ValueError: The model did not return a loss from the inputs, only the following keys: start_logits,end_logits,past_key_values,encoder_last_hidden_state. For reference, the inputs it received are input_ids,attention_mask.

could anybody solve the issue?

How do you know which columns to use? I am using a different model but getting same error but with different model. This is my error statement and code

ValueError: The model did not return a loss from the inputs, only the following keys: logits,past_key_values. For reference, the inputs it received are input_ids,attention_mask.

CODE

model_checkpoint = "EleutherAI/pythia-70m-deduped"
tokenizer = tr.AutoTokenizer.from_pretrained(
model_checkpoint,
revision="step3000",
cache_dir=DA.paths.datasets,
)

model = tr.AutoModelForCausalLM.from_pretrained (
model_checkpoint,cache_dir=DA.paths.datasets)
data_collator = tr.DataCollatorWithPadding(tokenizer=tokenizer)
trainer = tr.Trainer(
    model,
    training_args,
    train_dataset=split_dataset["train"],
    eval_dataset=split_dataset["test"],
    tokenizer=tokenizer,
    data_collator=data_collator,
)

trainer.train()

Has anyone definitively solved this error?

I am running into the same problem:

from transformers import BertForQuestionAnswering, BertForTokenClassification, BertForSequenceClassification, pipeline, BertTokenizer, Trainer, TrainingArguments, DistilBertForSequenceClassification, DistilBertTokenizerFast,DataCollatorWithPadding, pipeline
from datasets import load_metric, load_dataset, DatasetDict, Dataset
from sklearn.model_selection import train_test_split
import torch
import pandas as pd
import numpy as np

from datasets import load_metric, load_dataset, Dataset

# load distillbert tokenizer
tokenizer = DistilBertTokenizerFast.from_pretrained("distilbert-base-uncased")

# Explanation:
# ...<keyword>... represents a body of text which includes the keyword which is
# associated with the values in that text; these values represent the label we
# want to 'predict'
data = {
    'key': ['1', '2', '3', '4', '5'],
    'text': ['...fraction... 30 35', '...fraction... 25 30', '...fraction... 20 30', '...fraction... 30 40', '...fraction... 40 50'],
    'val1': ['30', '25', '20', '30', '40'],
    'val2': ['35', '30', '30', '40', '50']
}

pdf = pd.DataFrame(data)
features = pdf.drop(columns=["val1", "val2"])
labels = pdf[["val1", "val2"]]
train_df, test_df, train_labels, test_labels = train_test_split(features, labels, test_size=0.2, random_state=42)

features_dataset = Dataset.from_pandas(features)
train_dataset = Dataset.from_pandas(train_pdf)
test_dataset = Dataset.from_pandas(test_pdf)
train_test = DatasetDict({"train": train_dataset,"test": test_dataset})

from transformers import AutoModelForSequenceClassification
model = AutoModelForSequenceClassification.from_pretrained("bert-base-cased")

from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")

def tokenize_function(dataset):
  return tokenizer(dataset["text"], padding="max_length", truncation=True)

tokenized_datasets = features_dataset.map(tokenize_function, batched=True)
from transformers import AutoModelForSequenceClassification
model = AutoModelForSequenceClassification.from_pretrained("bert-base-cased")

from transformers import TrainingArguments
output_path = "/test_trainer"
training_args = TrainingArguments(output_dir=output_path)

import numpy as np
import evaluate
metric = evaluate.load('accuracy')

def compute_metrics(eval_pred):
  logits, labels = eval_pred
  predictions = np.argmax(logits, axis=-1)
  return metric.compute(predictions, references=labels)

from transformers import TrainingArguments, Trainer
training_args = TrainingArguments(output_dir=output_path, evaluation_strategy="epoch")

trainer = Trainer(
  model=model,
  args=training_args,
  train_dataset=train_dataset,
  eval_dataset=test_dataset,
  compute_metrics=compute_metrics
)

trainer.evaluate()

The following columns in the evaluation set don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: __index_level_0__. If __index_level_0__ are not expected by `BertForSequenceClassification.forward`,  you can safely ignore this message.
***** Running Evaluation *****
  Num examples = 1
  Batch size = 8

Out[27]: {'eval_runtime': 0.6222,
 'eval_samples_per_second': 1.607,
 'eval_steps_per_second': 1.607}
trainer.train()

The following columns in the training set don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: __index_level_0__. If __index_level_0__ are not expected by `BertForSequenceClassification.forward`,  you can safely ignore this message.
/databricks/python/lib/python3.9/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning
  warnings.warn(
***** Running training *****
  Num examples = 4
  Num Epochs = 3
  Instantaneous batch size per device = 8
  Total train batch size (w. parallel, distributed & accumulation) = 8
  Gradient Accumulation steps = 1
  Total optimization steps = 3

**ValueError: The model did not return a loss from the inputs, only the following keys: logits. For reference, the inputs it received are input_ids,attention_mask.**