I pretrained a RoBERTa model using the tutorial given in this blog post. Now, I am trying to use this pretrained model for custom NER task. I have prepared the dataset in the format as shown in this image.
Further, I run the below script on it to convert it to required format and start the training.
label_list = ['junk', 'road', 'landmark', 'sub_region', 'sub_dist', 'dist', 'state', 'pincode', 'hno']
label_encoding_dict = { 'junk':0, 'road':1, 'landmark':2, 'sub_region':3, 'sub_dist':4, 'dist':5, 'state':6, 'pincode':7, 'hno':8}
task = "ner" # Should be one of "ner", "pos" or "chunk"
model_checkpoint = "./RoBERTa_addressModel"
batch_size = 64
tokenizer = RobertaTokenizerFast.from_pretrained("./NER_tokenizer", max_len=150,add_prefix_space=True)
def tokenize_and_align_labels(examples):
label_all_tokens = True
tokenized_inputs = tokenizer(list(examples["tokens"]), truncation=True, is_split_into_words=True)
labels = []
for i, label in enumerate(examples[f"{task}_tags"]):
word_ids = tokenized_inputs.word_ids(batch_index=i)
previous_word_idx = None
label_ids = []
for word_idx in word_ids:
# Special tokens have a word id that is None. We set the label to -100 so they are automatically ignored in the loss function.
if word_idx is None:
label_ids.append(-100)
elif label[word_idx] == '0':
label_ids.append(0)
# We set the label for the first token of each word.
elif word_idx != previous_word_idx:
label_ids.append(label_encoding_dict[label[word_idx]])
# For the other tokens in a word, we set the label to either the current label or -100, depending on the label_all_tokens flag.
else:
label_ids.append(label_encoding_dict[label[word_idx]] if label_all_tokens else -100)
previous_word_idx = word_idx
labels.append(label_ids)
tokenized_inputs["labels"] = labels
return tokenized_inputs
train_dataset, test_dataset = get_un_token_dataset('./conell/tagged-training', './conell/tagged-test')
train_tokenized_datasets = train_dataset.map(tokenize_and_align_labels, batched=True)
test_tokenized_datasets = test_dataset.map(tokenize_and_align_labels, batched=True)
model_path = "RoBERTa_addressModel"
model = AutoModelForTokenClassification.from_pretrained(model_path, num_labels=len(label_list))
args = TrainingArguments(
f"test-{task}",
evaluation_strategy = "epoch",
learning_rate=2e-5,
per_device_train_batch_size=batch_size,
per_device_eval_batch_size=batch_size,
num_train_epochs=3,
weight_decay=0.01,
save_total_limit=5,
)
data_collator = DataCollatorForTokenClassification(tokenizer)
metric = load_metric("seqeval")
def compute_metrics(p):
predictions, labels = p
predictions = np.argmax(predictions, axis=2)
# Remove ignored index (special tokens)
true_predictions = [
[label_list[p] for (p, l) in zip(prediction, label) if l != -100]
for prediction, label in zip(predictions, labels)
]
true_labels = [
[label_list[l] for (p, l) in zip(prediction, label) if l != -100]
for prediction, label in zip(predictions, labels)
]
results = metric.compute(predictions=true_predictions, references=true_labels)
return {
"precision": results["overall_precision"],
"recall": results["overall_recall"],
"f1": results["overall_f1"],
"accuracy": results["overall_accuracy"],
}
trainer = Trainer(
model,
args,
train_dataset=train_tokenized_datasets,
eval_dataset=test_tokenized_datasets,
data_collator=data_collator,
tokenizer=tokenizer,
compute_metrics=compute_metrics
)
trainer.train()
trainer.evaluate()
trainer.save_model('./address-ner.model')
The model is trained well. However, when I make predictions using the script below,
label_list = ['junk', 'road', 'landmark', 'sub_region', 'sub_dist', 'dist', 'state', 'pincode', 'hno']
label_encoding_dict = { 'junk':0, 'road':1, 'landmark':2, 'sub_region':3, 'sub_dist':4, 'dist':5, 'state':6, 'pincode':7, 'hno':8}
model = AutoModelForTokenClassification.from_pretrained('./ner_model', num_labels=len(label_list))
paragraph = basicPreprocess(paragraph)
print(paragraph)
tokens = tokenizer(paragraph)
torch.tensor(tokens['input_ids']).unsqueeze(0).size()
predictions = model.forward(input_ids=torch.tensor(tokens['input_ids']).unsqueeze(0), attention_mask=torch.tensor(tokens['attention_mask']).unsqueeze(0))
predictions = torch.argmax(predictions.logits.squeeze(), axis=1)
predictions = [label_list[i] for i in predictions]
words = tokenizer.batch_decode(tokens['input_ids'])
pd.DataFrame({'ner': predictions, 'words': words})
Output is as follows :
Note the ‘start of the sentence’ tag (line no. 0 ) and ‘the end tag’ (line number 14) have been predicted to have label ‘sub_dist’ which is absurd. In other similar examples, random labels have been predicted for start and end tag. My question is, is this behaviour normal ? what should ideally be the corresponding labels for the ‘start’ and ‘end’ ? There must be some pre-defined labels for these tags. What should those labels be ?
My another question is if you see the words like ‘gornament’ has been broken down into ‘gor’ and ‘nament’, I suppose this behaviour is due to ‘Byte level tokenizer’. But, while getting the output, how can I ensure that the final output does not show words in parts but instead as a whole ? that is one label for one word ?