I am trying to fine tune mt5-base model to get summary in bengali from english sentence. but while decoding model always gives result in english not in bengali. how can i control or can i give instruction to target language detection.
here is some of my codes:
model_checkpoint1 = “google/mt5-small”
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint1, use_fast=False)
mt5_config = AutoConfig.from_pretrained(model_checkpoint1)
mt5_config.decoder_start_token_id = 250042
model = AutoModelForSeq2SeqLM.from_pretrained(model_checkpoint1,config=mt5_config)
custom_tokenizer_config = {
“additional_special_tokens”: [“▁<extra_id_69>”, “▁<extra_id_57>”, “▁<extra_id_58>”, “▁<extra_id_0>”],
}
tokenizer.add_special_tokens(custom_tokenizer_config)
data preprocess
max_input_length = 1024
max_target_length = 128
prefix = “en to bn:”
prefix1 = “▁<extra_id_57>”
def preprocess_function(examples):
inputs = [prefix + doc for doc in examples[“text”]]
# Ensure the prefix is added to the "summary" field
summaries = [prefix1 + summary for summary in examples["summary"]]
print(summaries)
model_inputs = tokenizer(inputs, max_length=max_input_length, truncation=True)
# Setup the tokenizer for targets
with tokenizer.as_target_tokenizer():
labels = tokenizer(summaries, max_length=max_target_length, truncation=True)
print(labels)
model_inputs["labels"] = labels["input_ids"]
return model_inputs
#compute metrics
import nltk
import numpy as np
nltk.download(‘punkt’)
def compute_metrics(eval_pred):
predictions, labels = eval_pred
decoded_preds = tokenizer.batch_decode(predictions, skip_special_tokens=True)
# Replace -100 in the labels as we can’t decode them.
labels = np.where(labels != -100, labels, tokenizer.pad_token_id)
decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)
# Rouge expects a newline after each sentence
decoded_preds = ["\n".join(nltk.sent_tokenize(pred.strip())) for pred in decoded_preds]
decoded_labels = ["\n".join(nltk.sent_tokenize(label.strip())) for label in decoded_labels]
print(decoded_preds)
print(decoded_labels)
result = metric.compute(predictions=decoded_preds, references=decoded_labels, use_stemmer=True)
# Extract a few results
result = {key: value.mid.fmeasure * 100 for key, value in result.items()}
# Add mean generated length
prediction_lens = [np.count_nonzero(pred != tokenizer.pad_token_id) for pred in predictions]
result["gen_len"] = np.mean(prediction_lens)
return {k: round(v, 4) for k, v in result.items()}
