I am trying to continue the fine-tuning for a fine-tuned model using unsloth[SFTrainer]
However when I am running the trainer , I am getting below error :
expected mat1 and mat2 to have the same dtype, but got: c10::Half != float
Has anyone encountered the same? If yes, please help me to fix it, below is the code snippet for reference :
base_with_adapter_model, tokenizer = FastLanguageModel.from_pretrained(
model_name = adapter_id,
max_seq_length = max_seq_length,
device_map = 'auto'
)
#merge adapter parameters with base model
model = base_with_adapter_model.merge_and_unload() # now we just have a regular AutoModelForCausalLM Transformers model
# Do model patching and add fast LoRA weights
base_model_with_new_adapter = FastLanguageModel.get_peft_model(
model,
r = 16,
target_modules = ["q_proj", "k_proj", "v_proj", "o_proj",
"gate_proj", "up_proj", "down_proj"],
lora_alpha = 16,
lora_dropout = 0, # Supports any, but = 0 is optimized
bias = "none", # Supports any, but = "none" is optimized
# [NEW] "unsloth" uses 30% less VRAM, fits 2x larger batch sizes!
use_gradient_checkpointing = "unsloth", # True or "unsloth" for very long context
random_state = 3407,
max_seq_length = max_seq_length,
use_rslora = False, # We support rank stabilized LoRA
loftq_config = None, # And LoftQ
)
base_model_with_new_adapter.print_trainable_parameters()
# Check if bfloat16 is supported
use_fp16 = not is_bfloat16_supported()
use_bf16 = is_bfloat16_supported()
continued_trainer = SFTTrainer(
model = base_model_with_new_adapter,
train_dataset = train_dataset,
dataset_text_field = "text",
max_seq_length = max_seq_length,
tokenizer = tokenizer,
dataset_num_proc = 2,
packing = False, # Can make training 5x faster for short sequences.
args = TrainingArguments(
per_device_train_batch_size = 2,
gradient_accumulation_steps = 4,
warmup_steps = 5,
num_train_epochs= 3,
learning_rate = 5e-5,
fp16 =use_fp16,
bf16 = use_bf16,
logging_steps = 1,
weight_decay = 0.01,
optim = "adamw_8bit",
lr_scheduler_type = "linear",
seed = 3407,
output_dir = "outputs",
save_steps = 20,
save_total_limit = 2,
),
)
Error trace is as below ::
RuntimeError Traceback (most recent call last)
Cell In[7], line 2
1 #train the model
----> 2 trainer_stats = continued_trainer.train()
File :126, in train(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)
File :367, in _fast_inner_training_loop(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval)
File /opt/conda/lib/python3.10/site-packages/transformers/trainer.py:3324, in Trainer.training_step(failed resolving arguments)
3322 scaled_loss.backward()
3323 else:
→ 3324 self.accelerator.backward(loss, **kwargs)
3326 return loss.detach() / self.args.gradient_accumulation_steps
File /opt/conda/lib/python3.10/site-packages/accelerate/accelerator.py:2130, in Accelerator.backward(self, loss, **kwargs)
2128 return
2129 elif self.scaler is not None:
→ 2130 self.scaler.scale(loss).backward(**kwargs)
2131 elif learning_rate is not None and self.has_lomo_optimizer:
2132 self.lomo_backward(loss, learning_rate)