Hello, I try transformers.BertModel to deal with a simple text classification, but the result makes me puzzled.
the code is simple,I implement the model with pytorch.
they are…
# a Dataset class for BertModel
class BertDataset(Dataset):
def __init__(self, train_file, tokenizer):
super(BertDataset, self).__init__()
self.train_file = train_file
self.data = []
self.label2id = {}
self.id2label = {}
self.tokenizer = tokenizer
self.init()
def init(self):
with open(self.train_file, 'r', encoding='utf-8') as f:
for line in f:
blocks = line.strip().split('\t')
if blocks[1] not in self.label2id:
self.label2id[blocks[1]] = len(self.label2id)
self.id2label[len(self.id2label)] = blocks[1]
self.data.append({'token': self.tokenizer(blocks[0], add_special_tokens=True, max_length=100,
padding='max_length', return_tensors='pt',
truncation=True),
'label': self.label2id[blocks[1]]})
def __getitem__(self, item):
return self.data[item]
def __len__(self):
return len(self.data)
# a collate function for torch.utils.data.DataLoader
def bert_collate_fn(batch_data):
input_ids, token_type_ids, attention_mask, labels = [], [], [], []
for instance in copy.deepcopy(batch_data):
input_ids.append(instance['token']['input_ids'][0].squeeze(0))
token_type_ids.append(instance['token']['token_type_ids'][0].squeeze(0))
attention_mask.append(instance['token']['attention_mask'][0].squeeze(0))
labels.append(instance['label'])
return torch.stack(input_ids), torch.stack(token_type_ids), \
torch.stack(attention_mask), torch.tensor(labels)
# Model
class PTModel(nn.Module):
def __init__(self, model, n_class):
super(PTModel, self).__init__()
self.n_class = n_class
self.model = model
self.linear = nn.Linear(768, self.n_class)
self.softmax = nn.Softmax(dim=-1)
def forward(self, input_ids, token_type_ids=None, attention_mask=None):
cls_emb = self.model(input_ids=input_ids, token_type_ids=token_type_ids, attention_mask=attention_mask)
cls_emb = cls_emb[0][:, 0, :].squeeze(1)
logits = self.linear(cls_emb)
# logits = self.softmax(logits)
return logits
# train code
def train1():
# data
batch_size = 16
tokenizer = BertTokenizer.from_pretrained(pretrained_path)
dataset = BertDataset('../data/dataset/data.txt', tokenizer)
train_len = int(len(dataset)*0.8)
train_dataset, dev_dataset = random_split(dataset=dataset, lengths=[train_len, len(dataset)-train_len])
train_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, collate_fn=bert_collate_fn)
dev_dataloader = DataLoader(dev_dataset, batch_size=batch_size, shuffle=True, collate_fn=bert_collate_fn)
# model
device = torch.device('cuda:{}'.format(args.cuda))
bert_model = BertModel.from_pretrained(pretrained_path)
model = PTModel(model=bert_model, n_class=len(dataset.label2id)).to(device)
optimizer = torch.optim.Adam(params=model.parameters(), lr=args.lr)
scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer=optimizer, milestones=[30, 40], gamma=0.1)
loss_func = torch.nn.CrossEntropyLoss()
# train
for i in range(args.epoch):
model.train()
train_loss, dev_loss, f1_train, f1_dev = [], [], [], []
dev_pred_list, dev_gold_list = [], []
for input_ids, token_type_ids, attention_mask, label in tqdm(train_dataloader):
input_ids, token_type_ids, attention_mask, label = input_ids.to(device), token_type_ids.to(device), \
attention_mask.to(device), label.to(device),
outputs = model(input_ids=input_ids, token_type_ids=token_type_ids, attention_mask=attention_mask)
array_outputs = np.array(outputs.cuda().data.cpu())
optimizer.zero_grad()
loss = loss_func(outputs, label)
results = outputs.cuda().data.cpu().argmax(dim=1)
score = f1_score(label.cuda().data.cpu(), results, average='micro')
train_loss.append(loss.item())
f1_train.append(score)
# optim
loss.backward()
optimizer.step()
scheduler.step()
print('epoch {}'.format(i))
print('train_loss:{}'.format(np.mean(train_loss)))
print('train_f1:{}'.format(np.mean(f1_train)))
The train log is following(only 10 epoches). And the result was already clear: The model could not learn anything!!!
PS: the learning rate was 1e-3.
100%|█████████████████████████████████████████| 250/250 [00:43<00:00, 5.72it/s]
epoch 0
train_loss:4.217772917747498
train_f1:0.081
100%|███████████████████████████████████████████| 63/63 [00:03<00:00, 19.52it/s]
dev_f1:0.08928571428571429
dev_loss:4.111690880760314
100%|█████████████████████████████████████████| 250/250 [00:43<00:00, 5.71it/s]
epoch 1
train_loss:4.094675525665283
train_f1:0.084
100%|███████████████████████████████████████████| 63/63 [00:03<00:00, 19.16it/s]
dev_f1:0.0882936507936508
dev_loss:4.1316274839734275
100%|█████████████████████████████████████████| 250/250 [00:43<00:00, 5.71it/s]
epoch 2
train_loss:4.084259546279907
train_f1:0.08525
100%|███████████████████████████████████████████| 63/63 [00:03<00:00, 19.37it/s]
dev_f1:0.08928571428571429
dev_loss:4.108004717599778
100%|█████████████████████████████████████████| 250/250 [00:44<00:00, 5.62it/s]
epoch 3
train_loss:4.0770455904006955
train_f1:0.09425
100%|███████████████████████████████████████████| 63/63 [00:03<00:00, 19.07it/s]
dev_f1:0.08928571428571429
dev_loss:4.1077501395392035
100%|█████████████████████████████████████████| 250/250 [00:45<00:00, 5.54it/s]
epoch 4
train_loss:4.070150758743286
train_f1:0.086
100%|███████████████████████████████████████████| 63/63 [00:03<00:00, 19.41it/s]
dev_f1:0.09027777777777778
dev_loss:4.103204295748756
100%|█████████████████████████████████████████| 250/250 [00:45<00:00, 5.52it/s]
epoch 5
train_loss:4.064209712982178
train_f1:0.0895
100%|███████████████████████████████████████████| 63/63 [00:03<00:00, 19.31it/s]
dev_f1:0.08928571428571429
dev_loss:4.117827377622089
100%|█████████████████████████████████████████| 250/250 [00:43<00:00, 5.70it/s]
epoch 6
train_loss:4.065111406326294
train_f1:0.08425
100%|███████████████████████████████████████████| 63/63 [00:03<00:00, 19.34it/s]
dev_f1:0.0882936507936508
dev_loss:4.099656305615864
100%|█████████████████████████████████████████| 250/250 [00:44<00:00, 5.58it/s]
epoch 7
train_loss:4.0547873935699466
train_f1:0.09175
100%|███████████████████████████████████████████| 63/63 [00:03<00:00, 19.30it/s]
dev_f1:0.08928571428571429
dev_loss:4.105985126798115
100%|█████████████████████████████████████████| 250/250 [00:43<00:00, 5.76it/s]
epoch 8
train_loss:4.0595885887145995
train_f1:0.08875
100%|███████████████████████████████████████████| 63/63 [00:03<00:00, 19.26it/s]
dev_f1:0.09027777777777778
dev_loss:4.121003010916332
100%|█████████████████████████████████████████| 250/250 [00:45<00:00, 5.46it/s]
epoch 9
train_loss:4.054850312232971
train_f1:0.08825
100%|███████████████████████████████████████████| 63/63 [00:03<00:00, 18.86it/s]
dev_f1:0.08928571428571429
dev_loss:4.12501887669639
100%|█████████████████████████████████████████| 250/250 [00:45<00:00, 5.46it/s]
epoch 10
train_loss:4.0566882238388065
train_f1:0.08525
100%|███████████████████████████████████████████| 63/63 [00:03<00:00, 18.85it/s]
dev_f1:0.09126984126984126
dev_loss:4.103033436669244
Before this BertModel, I have tried LSTM, and the LSTM worked well. the dev f1 reached 0.96.
# LSTM
class SimpleModel(nn.Module):
def __init__(self, **kwargs):
super(SimpleModel, self).__init__()
self.embedding = nn.Embedding.from_pretrained(kwargs['pretrained_embedding'], freeze=False)
self.lstm = nn.LSTM(kwargs['pretrained_embedding'].shape[1],
kwargs['hidden_size'],
batch_first=True,
bidirectional=True)
self.linear = nn.Linear(kwargs['hidden_size']*2, kwargs['n_class'])
def forward(self, inputs, lens):
inputs = self.embedding(inputs)
_, (h, _) = self.lstm(pack_padded_sequence(inputs, lens, batch_first=True, enforce_sorted=False))
h = h.permute(1, 0, 2).contiguous().view(h.shape[1], -1)
logits = self.linear(h)
logits = logits.softmax(dim=-1)
return logits
Could any good man tell me why this code can’t work.
Is there something wrong with my writing?
I have been confused for days…
Thank you very much!