Paligemma finetuning

I was finetuning paligemma with image and label data. I converted dataframe to dictionary with records which contains list of dictionaries containing two keys of image and label(evolution).

I have gotten error stating that can’t add eos_token with the labels(suffix). labels were in str.

my collate function is as below:

/opt/conda/lib/python3.10/site-packages/transformers/optimization.py:591: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning
  warnings.warn(
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
Cell In[40], line 1
----> 1 trainer.train()

File /opt/conda/lib/python3.10/site-packages/transformers/trainer.py:2012, in Trainer.train(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)
   2009 try:
   2010     # Disable progress bars when uploading models during checkpoints to avoid polluting stdout
   2011     hf_hub_utils.disable_progress_bars()
-> 2012     return inner_training_loop(
   2013         args=args,
   2014         resume_from_checkpoint=resume_from_checkpoint,
   2015         trial=trial,
   2016         ignore_keys_for_eval=ignore_keys_for_eval,
   2017     )
   2018 finally:
   2019     hf_hub_utils.enable_progress_bars()

File /opt/conda/lib/python3.10/site-packages/transformers/trainer.py:2314, in Trainer._inner_training_loop(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval)
   2311     rng_to_sync = True
   2313 step = -1
-> 2314 for step, inputs in enumerate(epoch_iterator):
   2315     total_batched_samples += 1
   2317     if self.args.include_num_input_tokens_seen:

File /opt/conda/lib/python3.10/site-packages/accelerate/data_loader.py:550, in DataLoaderShard.__iter__(self)
    548 # We iterate one batch ahead to check when we are at the end
    549 try:
--> 550     current_batch = next(dataloader_iter)
    551 except StopIteration:
    552     yield

File /opt/conda/lib/python3.10/site-packages/torch/utils/data/dataloader.py:630, in _BaseDataLoaderIter.__next__(self)
    627 if self._sampler_iter is None:
    628     # TODO(https://github.com/pytorch/pytorch/issues/76750)
    629     self._reset()  # type: ignore[call-arg]
--> 630 data = self._next_data()
    631 self._num_yielded += 1
    632 if self._dataset_kind == _DatasetKind.Iterable and \
    633         self._IterableDataset_len_called is not None and \
    634         self._num_yielded > self._IterableDataset_len_called:

File /opt/conda/lib/python3.10/site-packages/torch/utils/data/dataloader.py:673, in _SingleProcessDataLoaderIter._next_data(self)
    671 def _next_data(self):
    672     index = self._next_index()  # may raise StopIteration
--> 673     data = self._dataset_fetcher.fetch(index)  # may raise StopIteration
    674     if self._pin_memory:
    675         data = _utils.pin_memory.pin_memory(data, self._pin_memory_device)

File /opt/conda/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py:55, in _MapDatasetFetcher.fetch(self, possibly_batched_index)
     53 else:
     54     data = self.dataset[possibly_batched_index]
---> 55 return self.collate_fn(data)

Cell In[38], line 11, in collate_fn(examples)
      9 evolution = [example['Evolution'] for example in examples]
     10 #names = [example['Name'] for example in examples]
---> 11 tokens = processor(text=texts, images=images, suffix=evolution,
     12                   return_tensors="pt",padding="longest",
     13                   tokenize_newline_separately=False)
     15 tokens = tokens.to(torch.bfloat16).to(device)
     16 return tokens

File /opt/conda/lib/python3.10/site-packages/transformers/models/paligemma/processing_paligemma.py:242, in PaliGemmaProcessor.__call__(self, text, images, tokenize_newline_separately, padding, truncation, max_length, return_tensors, do_resize, do_normalize, image_mean, image_std, data_format, input_data_format, resample, do_convert_rgb, do_thumbnail, do_align_long_axis, do_rescale, suffix)
    240     suffix = [suffix]
    241 if suffix is not None:
--> 242     suffix = [sfx + self.tokenizer.eos_token for sfx in suffix]
    244 input_strings = [
    245     build_string_from_input(
    246         prompt=prompt,
   (...)
    251     for prompt in text
    252 ]
    254 pixel_values = self.image_processor(
    255     images,
    256     do_resize=do_resize,
   (...)
    264     do_convert_rgb=do_convert_rgb,
    265 )["pixel_values"]

File /opt/conda/lib/python3.10/site-packages/transformers/models/paligemma/processing_paligemma.py:242, in <listcomp>(.0)
    240     suffix = [suffix]
    241 if suffix is not None:
--> 242     **suffix = [sfx + self.tokenizer.eos_token for sfx in suffix]**
    244 input_strings = [
    245     build_string_from_input(
    246         prompt=prompt,
   (...)
    251     for prompt in text
    252 ]
    254 pixel_values = self.image_processor(
    255     images,
    256     do_resize=do_resize,
   (...)
    264     do_convert_rgb=do_convert_rgb,
    265 )["pixel_values"]

**TypeError: unsupported operand type(s) for +: 'float' and 'str'**

hi @harshilgohil
Either evolution or tokenizer.eos_token is float. To be sure, can you please run the following:

all(type(suffix) == str for suffix in evolution)

Thanks! That’s what I was confused about, but yesterday night I tried on other label dataset with numbers and it turned out that labels were float. I converted it to string and it started training. Thanks again!

1 Like