How to handle <s> and </s> tags for custom NER using RoBERTa?

I pretrained a RoBERTa model using the tutorial given in this blog post. Now, I am trying to use this pretrained model for custom NER task. I have prepared the dataset in the format as shown in this image.

Further, I run the below script on it to convert it to required format and start the training.

label_list = ['junk', 'road', 'landmark', 'sub_region', 'sub_dist', 'dist', 'state', 'pincode', 'hno']

label_encoding_dict = { 'junk':0, 'road':1, 'landmark':2, 'sub_region':3, 'sub_dist':4, 'dist':5, 'state':6, 'pincode':7, 'hno':8}

task = "ner" # Should be one of "ner", "pos" or "chunk"
model_checkpoint = "./RoBERTa_addressModel" 
batch_size = 64 

tokenizer = RobertaTokenizerFast.from_pretrained("./NER_tokenizer", max_len=150,add_prefix_space=True)

def tokenize_and_align_labels(examples):
    label_all_tokens = True
    tokenized_inputs = tokenizer(list(examples["tokens"]), truncation=True, is_split_into_words=True)

    labels = []
    for i, label in enumerate(examples[f"{task}_tags"]):
        word_ids = tokenized_inputs.word_ids(batch_index=i)
        previous_word_idx = None
        label_ids = []
        for word_idx in word_ids:
            # Special tokens have a word id that is None. We set the label to -100 so they are automatically ignored in the loss function.
            if word_idx is None:
                label_ids.append(-100)
            elif label[word_idx] == '0':
                label_ids.append(0)
            # We set the label for the first token of each word.
            elif word_idx != previous_word_idx:
                label_ids.append(label_encoding_dict[label[word_idx]])
            # For the other tokens in a word, we set the label to either the current label or -100, depending on the label_all_tokens flag.
            else:
                label_ids.append(label_encoding_dict[label[word_idx]] if label_all_tokens else -100)
            previous_word_idx = word_idx

        labels.append(label_ids)

    tokenized_inputs["labels"] = labels
    return tokenized_inputs


train_dataset, test_dataset = get_un_token_dataset('./conell/tagged-training', './conell/tagged-test')
train_tokenized_datasets = train_dataset.map(tokenize_and_align_labels, batched=True)
test_tokenized_datasets = test_dataset.map(tokenize_and_align_labels, batched=True)

model_path = "RoBERTa_addressModel"
model = AutoModelForTokenClassification.from_pretrained(model_path, num_labels=len(label_list))

args = TrainingArguments(
    f"test-{task}",
    evaluation_strategy = "epoch",
    learning_rate=2e-5,
    per_device_train_batch_size=batch_size,
    per_device_eval_batch_size=batch_size,
    num_train_epochs=3,
    weight_decay=0.01,
    save_total_limit=5,
)

data_collator = DataCollatorForTokenClassification(tokenizer)
metric = load_metric("seqeval")


def compute_metrics(p):
    predictions, labels = p
    predictions = np.argmax(predictions, axis=2)

    # Remove ignored index (special tokens)
    true_predictions = [
        [label_list[p] for (p, l) in zip(prediction, label) if l != -100]
        for prediction, label in zip(predictions, labels)
    ]
    true_labels = [
        [label_list[l] for (p, l) in zip(prediction, label) if l != -100]
        for prediction, label in zip(predictions, labels)
    ]

    results = metric.compute(predictions=true_predictions, references=true_labels)
    return {
        "precision": results["overall_precision"],
        "recall": results["overall_recall"],
        "f1": results["overall_f1"],
        "accuracy": results["overall_accuracy"],
    }

trainer = Trainer(
    model,
    args,
    train_dataset=train_tokenized_datasets,
    eval_dataset=test_tokenized_datasets,
    data_collator=data_collator,
    tokenizer=tokenizer,
    compute_metrics=compute_metrics
)

trainer.train()
trainer.evaluate()
trainer.save_model('./address-ner.model')

The model is trained well. However, when I make predictions using the script below,

label_list = ['junk', 'road', 'landmark', 'sub_region', 'sub_dist', 'dist', 'state', 'pincode', 'hno']
label_encoding_dict = { 'junk':0, 'road':1, 'landmark':2, 'sub_region':3, 'sub_dist':4, 'dist':5, 'state':6, 'pincode':7, 'hno':8}
model = AutoModelForTokenClassification.from_pretrained('./ner_model', num_labels=len(label_list))

paragraph = basicPreprocess(paragraph)
print(paragraph)
tokens = tokenizer(paragraph)
torch.tensor(tokens['input_ids']).unsqueeze(0).size()

predictions = model.forward(input_ids=torch.tensor(tokens['input_ids']).unsqueeze(0), attention_mask=torch.tensor(tokens['attention_mask']).unsqueeze(0))
predictions = torch.argmax(predictions.logits.squeeze(), axis=1)
predictions = [label_list[i] for i in predictions]

words = tokenizer.batch_decode(tokens['input_ids'])
pd.DataFrame({'ner': predictions, 'words': words})

Output is as follows :


Note the ‘start of the sentence’ tag (line no. 0 ) and ‘the end tag’ (line number 14) have been predicted to have label ‘sub_dist’ which is absurd. In other similar examples, random labels have been predicted for start and end tag. My question is, is this behaviour normal ? what should ideally be the corresponding labels for the ‘start’ and ‘end’ ? There must be some pre-defined labels for these tags. What should those labels be ?

My another question is if you see the words like ‘gornament’ has been broken down into ‘gor’ and ‘nament’, I suppose this behaviour is due to ‘Byte level tokenizer’. But, while getting the output, how can I ensure that the final output does not show words in parts but instead as a whole ? that is one label for one word ?