Hello.
Working with pretrained BERT model for regression task. I have idea that dataset has usefull not text feature, which can help to improve result. Is it possible to modify my model so that the text field is processed by bert taking into account this useful feature?
class MyModel(torch.nn.Module):
def __init__(self):
super().__init__()
config = AutoConfig.from_pretrained(model_name)
config.update({"output_hidden_states":True,
"hidden_dropout_prob": 0.0,
"layer_norm_eps": 1e-7})
self.roberta = AutoModel.from_pretrained(model_name, config=config)
self.attention = torch.nn.Sequential(
torch.nn.Linear(768, 512),
torch.nn.Tanh(),
torch.nn.Linear(512, 1),
torch.nn.Softmax(dim=1)
)
self.regressor = torch.nn.Sequential(
torch.nn.Linear(768, 1)
)
def forward(self, input_ids, attention_mask):
roberta_output = self.roberta(input_ids=input_ids,
attention_mask=attention_mask)
last_layer_hidden_states = roberta_output.hidden_states[-1]
weights = self.attention(last_layer_hidden_states)
context_vector = torch.sum(weights * last_layer_hidden_states, dim=1)
return self.regressor(context_vector)
df - some dataset with ‘text’ feature processed by Bert and usefull feature ‘stat’(float). I what to use ‘stat’ feature to improve my predictions.