I was finetuning paligemma with image and label data. I converted dataframe to dictionary with records which contains list of dictionaries containing two keys of image and label(evolution).
I have gotten error stating that can’t add eos_token with the labels(suffix). labels were in str.
my collate function is as below:
/opt/conda/lib/python3.10/site-packages/transformers/optimization.py:591: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning
warnings.warn(
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
Cell In[40], line 1
----> 1 trainer.train()
File /opt/conda/lib/python3.10/site-packages/transformers/trainer.py:2012, in Trainer.train(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)
2009 try:
2010 # Disable progress bars when uploading models during checkpoints to avoid polluting stdout
2011 hf_hub_utils.disable_progress_bars()
-> 2012 return inner_training_loop(
2013 args=args,
2014 resume_from_checkpoint=resume_from_checkpoint,
2015 trial=trial,
2016 ignore_keys_for_eval=ignore_keys_for_eval,
2017 )
2018 finally:
2019 hf_hub_utils.enable_progress_bars()
File /opt/conda/lib/python3.10/site-packages/transformers/trainer.py:2314, in Trainer._inner_training_loop(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval)
2311 rng_to_sync = True
2313 step = -1
-> 2314 for step, inputs in enumerate(epoch_iterator):
2315 total_batched_samples += 1
2317 if self.args.include_num_input_tokens_seen:
File /opt/conda/lib/python3.10/site-packages/accelerate/data_loader.py:550, in DataLoaderShard.__iter__(self)
548 # We iterate one batch ahead to check when we are at the end
549 try:
--> 550 current_batch = next(dataloader_iter)
551 except StopIteration:
552 yield
File /opt/conda/lib/python3.10/site-packages/torch/utils/data/dataloader.py:630, in _BaseDataLoaderIter.__next__(self)
627 if self._sampler_iter is None:
628 # TODO(https://github.com/pytorch/pytorch/issues/76750)
629 self._reset() # type: ignore[call-arg]
--> 630 data = self._next_data()
631 self._num_yielded += 1
632 if self._dataset_kind == _DatasetKind.Iterable and \
633 self._IterableDataset_len_called is not None and \
634 self._num_yielded > self._IterableDataset_len_called:
File /opt/conda/lib/python3.10/site-packages/torch/utils/data/dataloader.py:673, in _SingleProcessDataLoaderIter._next_data(self)
671 def _next_data(self):
672 index = self._next_index() # may raise StopIteration
--> 673 data = self._dataset_fetcher.fetch(index) # may raise StopIteration
674 if self._pin_memory:
675 data = _utils.pin_memory.pin_memory(data, self._pin_memory_device)
File /opt/conda/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py:55, in _MapDatasetFetcher.fetch(self, possibly_batched_index)
53 else:
54 data = self.dataset[possibly_batched_index]
---> 55 return self.collate_fn(data)
Cell In[38], line 11, in collate_fn(examples)
9 evolution = [example['Evolution'] for example in examples]
10 #names = [example['Name'] for example in examples]
---> 11 tokens = processor(text=texts, images=images, suffix=evolution,
12 return_tensors="pt",padding="longest",
13 tokenize_newline_separately=False)
15 tokens = tokens.to(torch.bfloat16).to(device)
16 return tokens
File /opt/conda/lib/python3.10/site-packages/transformers/models/paligemma/processing_paligemma.py:242, in PaliGemmaProcessor.__call__(self, text, images, tokenize_newline_separately, padding, truncation, max_length, return_tensors, do_resize, do_normalize, image_mean, image_std, data_format, input_data_format, resample, do_convert_rgb, do_thumbnail, do_align_long_axis, do_rescale, suffix)
240 suffix = [suffix]
241 if suffix is not None:
--> 242 suffix = [sfx + self.tokenizer.eos_token for sfx in suffix]
244 input_strings = [
245 build_string_from_input(
246 prompt=prompt,
(...)
251 for prompt in text
252 ]
254 pixel_values = self.image_processor(
255 images,
256 do_resize=do_resize,
(...)
264 do_convert_rgb=do_convert_rgb,
265 )["pixel_values"]
File /opt/conda/lib/python3.10/site-packages/transformers/models/paligemma/processing_paligemma.py:242, in <listcomp>(.0)
240 suffix = [suffix]
241 if suffix is not None:
--> 242 **suffix = [sfx + self.tokenizer.eos_token for sfx in suffix]**
244 input_strings = [
245 build_string_from_input(
246 prompt=prompt,
(...)
251 for prompt in text
252 ]
254 pixel_values = self.image_processor(
255 images,
256 do_resize=do_resize,
(...)
264 do_convert_rgb=do_convert_rgb,
265 )["pixel_values"]
**TypeError: unsupported operand type(s) for +: 'float' and 'str'**