I’m passing in input embeddings to a T5 Model like this:
config= T5Config.from_pretrained('google-t5/t5-base')
config.d_model = 1024
model=T5ForConditionalGeneration(config)
tokenizer = T5Tokenizer.from_pretrained("google-t5/t5-base", is_fast=True)
tokenized_inputs = tokenizer(["yooo yoooo"], return_tensors='pt', truncation=True, padding=True)
inputs_embeds = torch.load('07068.pt', weights_only=True).unsqueeze(0) # loads tensor of shape (1,x, 1024)
attention_mask = torch.zeros(inputs_embeds.shape[0],inputs_embeds.shape[1])
attention_mask[attention_mask==0] = 1
loss = model(inputs_embeds=inputs_embeds, labels=tokenized_inputs).loss
I’m trying to get loss to train this model. But I get this error:
Traceback (most recent call last):
File "/home/workstation/anaconda3/envs/T5/lib/python3.12/site-packages/transformers/tokenization_utils_base.py", line 270, in __getattr__
return self.data[item]
~~~~~~~~~^^^^^^
KeyError: 'new_zeros'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/workstation/Desktop/T5/test.py", line 43, in <module>
loss = model(inputs_embeds=inputs_embeds, labels=tokenized_inputs).loss
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/workstation/anaconda3/envs/T5/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/workstation/anaconda3/envs/T5/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/workstation/anaconda3/envs/T5/lib/python3.12/site-packages/transformers/models/t5/modeling_t5.py", line 1725, in forward
decoder_input_ids = self._shift_right(labels)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/workstation/anaconda3/envs/T5/lib/python3.12/site-packages/transformers/models/t5/modeling_t5.py", line 887, in _shift_right
shifted_input_ids = input_ids.new_zeros(input_ids.shape)
^^^^^^^^^^^^^^^^^^^
File "/home/workstation/anaconda3/envs/T5/lib/python3.12/site-packages/transformers/tokenization_utils_base.py", line 272, in __getattr__
raise AttributeError
AttributeError
How do I get loss so I can train T5?