ImportError in bitsandbytes with Accelerate

Beginners
1

I am fune-tuning “TinyLlama/TinyLlama-1.1B-Chat-v1.0”, but I get this error:

ImportError                               Traceback (most recent call last)
<ipython-input-38-eb7c9c5f446e> in <cell line: 1>()
----> 1 model, tokenizer = get_model_and_tokenizer(model_id)

3 frames
/usr/local/lib/python3.10/dist-packages/transformers/quantizers/quantizer_bnb_4bit.py in validate_environment(self, *args, **kwargs)
     60     def validate_environment(self, *args, **kwargs):
     61         if not (is_accelerate_available() and is_bitsandbytes_available()):
---> 62             raise ImportError(
     63                 "Using `bitsandbytes` 8-bit quantization requires Accelerate: `pip install accelerate` "
     64                 "and the latest version of bitsandbytes: `pip install -i https://pypi.org/simple/ bitsandbytes`"

ImportError: Using `bitsandbytes` 8-bit quantization requires Accelerate: `pip install accelerate` and the latest version of bitsandbytes: `pip install -i https://pypi.org/simple/ bitsandbytes`

Libraries

!pip install accelerate peft bitsandbytes transformers trl

Code:

import torch
from datasets import load_dataset, Dataset
from peft import LoraConfig, AutoPeftModelForCausalLM
from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig, TrainingArguments
from trl import SFTTrainer
import os

dataset="burkelibbey/colors"
model_id="TinyLlama/TinyLlama-1.1B-Chat-v1.0"
output_model="tinyllama-colorist-v1"

def formatted_train(input,response)->str:
    return f"<|im_start|>user\n{input}<|im_end|>\n<|im_start|>assistant\n{response}<|im_end|>\n"

def prepare_train_data(data_id):
  data = load_dataset(data_id, split="train")
  data_df = data.to_pandas()
  data_df['text'] = data_df[["description", "color"]].apply(
      lambda x: "<|im_start|>user\n" + x["description"] + " <|im_end|>\n<|im_start|>assistant\n" + x["color"] + "<|im_end|>\n",
      axis=1
  )
  data = Dataset.from_pandas(data_df)
  return data

data = prepare_train_data(dataset)

def get_model_and_tokenizer(model_id):
    tokenizer = AutoTokenizer.from_pretrained(model_id)
    tokenizer.pad_token = tokenizer.eos_token
    bnb_config = BitsAndBytesConfig(
        load_in_4bit=True, bnb_4bit_quant_type="nf4", bnb_4bit_compute_dtype="float16", bnb_4bit_use_double_quant=True
    )
    model = AutoModelForCausalLM.from_pretrained(
        model_id, quantization_config=bnb_config, device_map="auto"
    )
    model.config.user_cache = False
    model.config.pretraining_tp=1
    return model, tokenizer

model, tokenizer = get_model_and_tokenizer(model_id)

I am using google colab.

1 Like
2

You need to restart the runtime after the install to reload all the libraries

2 Likes
3

Thank you :hugs:.

1 Like
4

This topic was automatically closed 12 hours after the last reply. New replies are no longer allowed.