Same checkpoint produces different output

🤗Transformers
1

I am trying to load a model trained using the trainer class. Here is the script that I used to load the saved model. However, I get different outputs each time I reload the model. Here is the code


import torch
from transformers import BertForSequenceClassification, BertTokenizer

class ModelPredictor:
    def __init__(self, model_path, model_name):
        self.model = BertForSequenceClassification.from_pretrained(model_path, return_dict=False)
        self.tokenizer = BertTokenizer.from_pretrained(model_name, 
                                                       cache_dir='/data/hfacecache')
        self.model.eval()  # Enters evaluation mode
    def predict(self, name):
        test_name_prompt = 'Tell me this persons race: ' + name
        inputs = self.tokenizer(test_name_prompt, padding=True, truncation=True, max_length=512, return_tensors="pt")
        # with torch.no_grad():
        with torch.no_grad():
            outputs = self.model(**inputs)
            predictions = torch.nn.functional.softmax(outputs[0], dim=-1)

        predicted_class_index = torch.argmax(predictions[0]).item()
        predicted_class_label = self.model.config.id2label[predicted_class_index]
        return predictions, predicted_class_label

Evaluation

#%%
model_path = "/output/checkpoint-30"
model_name = "bert-base-uncased"
predictor = ModelPredictor(model_path, model_name)

name_testing = "Joe Doe"
predictions, predicted_class_label = predictor.predict(name_testing)
print(predicted_class_label)
print(predictions)


This is the output
Asian
tensor([[0.1894, 0.1887, 0.3228, 0.2991]])


#%%
model_path = "/output/checkpoint-30"
model_name = "bert-base-uncased"
predictor = ModelPredictor(model_path, model_name)
name_testing = "Joe Doe"
predictions, predicted_class_label = predictor.predict(name_testing)
print(predicted_class_label)
print(predictions)

This is the output
White
tensor([[0.2871, 0.2415, 0.2611, 0.2102]])

I am not sure why this is happening despite loading the pre-trained model and setting the model in evaluation mode.