I noticed that the pytorch tokenizers output tensors with int64 precision, and the model (I’m experimenting with different transformer models) only accepts int64 tensors, otherwise it throws an error:
RuntimeError Traceback (most recent call last)
in <cell line: 1>()
----> 1 model(input_ids = input_ids, attention_mask=attention_mask, labels=labels)
6 frames
/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py in _wrapped_call_impl(self, *args, **kwargs)
1516 return self._compiled_call_impl(*args, **kwargs) # type: ignore[misc]
1517 else:
→ 1518 return self._call_impl(*args, **kwargs)
1519
1520 def _call_impl(self, *args, **kwargs):
/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py in _call_impl(self, *args, **kwargs)
1525 or _global_backward_pre_hooks or _global_backward_hooks
1526 or _global_forward_hooks or _global_forward_pre_hooks):
→ 1527 return forward_call(*args, **kwargs)
1528
1529 try:
/usr/local/lib/python3.10/dist-packages/transformers/models/t5/modeling_t5.py in forward(self, input_ids, attention_mask, decoder_input_ids, decoder_attention_mask, head_mask, decoder_head_mask, cross_attn_head_mask, encoder_outputs, past_key_values, inputs_embeds, decoder_inputs_embeds, labels, use_cache, output_attentions, output_hidden_states, return_dict)
1776 # move labels to correct device to enable PP
1777 labels = labels.to(lm_logits.device)
→ 1778 loss = loss_fct(lm_logits.view(-1, lm_logits.size(-1)), labels.view(-1))
1779 # TODO(thom): Add z_loss mesh/mesh_tensorflow/layers.py at fa19d69eafc9a482aff0b59ddd96b025c0cb207d · tensorflow/mesh · GitHub
1780
/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py in _wrapped_call_impl(self, *args, **kwargs)
1516 return self._compiled_call_impl(*args, **kwargs) # type: ignore[misc]
1517 else:
→ 1518 return self._call_impl(*args, **kwargs)
1519
1520 def _call_impl(self, *args, **kwargs):
/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py in _call_impl(self, *args, **kwargs)
1525 or _global_backward_pre_hooks or _global_backward_hooks
1526 or _global_forward_hooks or _global_forward_pre_hooks):
→ 1527 return forward_call(*args, **kwargs)
1528
1529 try:
/usr/local/lib/python3.10/dist-packages/torch/nn/modules/loss.py in forward(self, input, target)
1177
1178 def forward(self, input: Tensor, target: Tensor) → Tensor:
→ 1179 return F.cross_entropy(input, target, weight=self.weight,
1180 ignore_index=self.ignore_index, reduction=self.reduction,
1181 label_smoothing=self.label_smoothing)
/usr/local/lib/python3.10/dist-packages/torch/nn/functional.py in cross_entropy(input, target, weight, size_average, ignore_index, reduce, reduction, label_smoothing)
3051 if size_average is not None or reduce is not None:
3052 reduction = _Reduction.legacy_get_string(size_average, reduce)
→ 3053 return torch._C._nn.cross_entropy_loss(input, target, weight, _Reduction.get_enum(reduction), ignore_index, label_smoothing)
3054
3055
RuntimeError: expected scalar type Long but found Int
Is there a way that allows the model to accept tesnors with preciosn int 32 ? (I’m aware that I can change the input type to int64, but due to memory constraints I would like my input to be int32)