I am trying to train transformers language model with marathi language database (Wikipedia text). While training I get following error:
~/anaconda3/lib/python3.8/site-packages/torch/nn/modules/module.py in getattr(self, name)
776 if name in modules:
777 return modules[name]
–> 778 raise ModuleAttributeError("’{}’ object has no attribute ‘{}’".format(
779 type(self).name, name))
780
ModuleAttributeError: ‘TransfoXLModel’ object has no attribute 'reset’
Can anyone tell me what I am doing wrong? Please note that I have been able to train with AWD_LSTM model from fastai in similar setup.
Fastai version : 2.1.5, Pytorch version: 1.7.0, Transformer version: 4.0.0
Here is my code snippet:
from fastai import *
from fastai.text.all import *
import pathlib
import sentencepiece as spm
import io
import fastai, torch
fastai.version , torch.version
import transformers
from transformers import TransfoXLConfig, TransfoXLModel, TransfoXLTokenizer
from transformers import GPT2LMHeadModel, GPT2TokenizerFast
path = pathlib.Path(’/home/rajendra/ML/NLP/marathi_nlp/’)
sample_path = ‘/home/rajendra/ML/NLP/marathi_nlp/data/’
get_texts = partial(get_text_files, folders=[‘txt’, ‘wikipedia_txt’])
dls_lm = DataBlock(
blocks=TextBlock.from_folder(sample_path, is_lm=True),
get_items=get_texts, splitter=RandomSplitter(0.1)
).dataloaders(sample_path, path=sample_path, bs=16, seq_len=80)
dls_lm.show_batch(max_n=3) #This shows correct batches of text
pretrained_weights = ‘gpt2’
tokenizer = GPT2TokenizerFast.from_pretrained(pretrained_weights)
model = GPT2LMHeadModel.from_pretrained(pretrained_weights)
loss_func = CrossEntropyLossFlat()
learn = LMLearner(dls_lm, model, loss_func = loss_func, metrics=[accuracy, Perplexity()])
model.train() #This should set model to train mode. Prints model info
learn.fit_one_cycle(1, 2e-2)
Call to fit_one_cycle fails with above mentioned error. Please help!
The error trace starts with:
epoch train_loss valid_loss accuracy perplexity time
0 nan 00:00
ModuleAttributeError Traceback (most recent call last)
~/anaconda3/lib/python3.8/site-packages/fastai/learner.py in with_events(self, f, event_type, ex, final)
153 def with_events(self, f, event_type, ex, final=noop):
–> 154 try: self(f’before{event_type}’) ;f()
155 except ex: self(f’after_cancel{event_type}’)
~/anaconda3/lib/python3.8/site-packages/fastai/learner.py in _do_fit(self)
195 self.epoch=epoch
–> 196 self._with_events(self._do_epoch, ‘epoch’, CancelEpochException)
197