I ran this code and it looks like I’m getting a recursion error:
def sd_data_collator(dataset_samples_list):
tokenizer = GPT2Tokenizer.from_pretrained('gpt2', padding_side='right')
tokenizer.pad_token = tokenizer.eos_token
encoded_results = tokenizer(dataset_samples_list, padding=True, truncation=True, return_tensors='pt', return_attention_mask=True)
batch = {}
batch['input_ids'] = torch.stack([result for result in encoded_results['input_ids']])
batch['past'] = None
batch['attention_mask'] = torch.stack([result for result in encoded_results['attention_mask']])
batch['position_ids'] = None
batch['head_mask'] = None
batch['inputs_embeds'] = None
batch['labels'] = None
batch['use_cache'] = True
return batch
sd_dataset = SDAbstractsDataset('/path/to/sd_samples_64.csv')
training_args = TrainingArguments(
output_dir='/path/to/finetuned_gpt2',
do_train=True,
per_device_train_batch_size=4,
learning_rate=1e-3,
num_train_epochs=1
)
model = GPT2FinetunedWithNgrams.from_pretrained('gpt2')
trainer = Trainer(
model=model,
args=training_args,
train_dataset=sd_dataset,
data_collator = sd_data_collator
)
trainer.train()
class GPT2FinetunedWithNgrams(GPT2LMHeadModel):
def __init__(self, config, model_tokenizer=None):
super().__init__(config)
self.tokenizer = GPT2Tokenizer.from_pretrained('gpt2', padding_side='right')
self.tokenizer.pad_token = self.tokenizer.eos_token
def load_ngrams_model(self, ngrams_model_path):
self.ngrams_model = NGrams(ngrams_model_path)
def forward(
self,
input_ids=None,
past=None,
attention_mask=None,
token_type_ids=None,
position_ids=None,
head_mask=None,
inputs_embeds=None,
labels=None,
use_cache=True,
):
output = self.generate(input_ids=input_ids, max_length=470)
Here’s the whole error (it’s really lengthy):
Some weights of GPT2FinetunedWithNgrams were not initialized from the model checkpoint at gpt2 and are newly initialized: ['h.0.attn.masked_bias', 'h.1.attn.masked_bias', 'h.2.attn.masked_bias', 'h.3.attn.masked_bias', 'h.4.attn.masked_bias', 'h.5.attn.masked_bias', 'h.6.attn.masked_bias', 'h.7.attn.masked_bias', 'h.8.attn.masked_bias', 'h.9.attn.masked_bias', 'h.10.attn.masked_bias', 'h.11.attn.masked_bias', 'lm_head.weight']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
Epoch: 0%| | 0/1 [00:00<?, ?it/s]
Iteration: 0%| | 0/16 [00:00<?, ?it/s]Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence
Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence
Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence
Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence
.
.
.
File "/path/to/anaconda3/lib/python3.7/site-packages/torch/autograd/grad_mode.py", line 15, in decorate_context
return func(*args, **kwargs)
File "/path/to/anaconda3/lib/python3.7/site-packages/transformers/generation_utils.py", line 480, in generate
model_specific_kwargs=model_specific_kwargs,
File "/path/to/anaconda3/lib/python3.7/site-packages/transformers/generation_utils.py", line 520, in _generate_no_beam_search
outputs = self(**model_inputs)
File "/path/to/anaconda3/lib/python3.7/site-packages/torch/nn/modules/module.py", line 550, in __call__
result = self.forward(*input, **kwargs)
File "/path/to/ric-2020/text_gen_w_transformers/finetune_gpt2.py", line 33, in forward
File "/path/to/anaconda3/lib/python3.7/site-packages/torch/autograd/grad_mode.py", line 15, in decorate_context
return func(*args, **kwargs)
File "/path/to/anaconda3/lib/python3.7/site-packages/transformers/generation_utils.py", line 350, in generate
"Setting `pad_token_id` to {} (first `eos_token_id`) to generate sequence".format(eos_token_id)
.
.
.
File "/path/to/anaconda3/lib/python3.7/logging/__init__.py", line 1390, in warning
self._log(WARNING, msg, args, **kwargs)
File "/path/to/anaconda3/lib/python3.7/logging/__init__.py", line 1514, in _log
self.handle(record)
File "/path/to/anaconda3/lib/python3.7/logging/__init__.py", line 1524, in handle
self.callHandlers(record)
File "/path/to/anaconda3/lib/python3.7/logging/__init__.py", line 1594, in callHandlers
lastResort.handle(record)
File "/path/to/anaconda3/lib/python3.7/logging/__init__.py", line 894, in handle
self.emit(record)
File "/path/to/anaconda3/lib/python3.7/logging/__init__.py", line 1025, in emit
msg = self.format(record)
File "/path/to/anaconda3/lib/python3.7/logging/__init__.py", line 869, in format
return fmt.format(record)
File "/path/to/anaconda3/lib/python3.7/logging/__init__.py", line 608, in format
record.message = record.getMessage()
File "/path/to/anaconda3/lib/python3.7/logging/__init__.py", line 360, in getMessage
def getMessage(self):
RecursionError: maximum recursion depth exceeded while calling a Python object
My guess is the self.generate() being called within the model produces the recursion problem.
Is it possible to use the functionality within the generate method (like beam search, top-k, etc.) without causing this recursion error during finetuning?