import streamlit as st
import transformers
import torch
from accelerate import init_empty_weights, load_checkpoint_and_dispatch
Initialize the model
model_id = “meta-llama/Meta-Llama-3.1-8B-Instruct”
Use disk offload to manage memory
with init_empty_weights():
model = transformers.AutoModelForCausalLM.from_pretrained(model_id)
Specify the folder where the model will be offloaded
offload_folder = “./offload_folder”
Load the model with offloading
model = load_checkpoint_and_dispatch(
model, model_id, device_map=“auto”, offload_folder=offload_folder
)
Initialize the pipeline with the offloaded model
pipeline = transformers.pipeline(
“text-generation”,
model=model,
tokenizer=model_id,
device_map=“auto”
)
def getLLMResponse(query, tasktype_option, template_option, customization):
# Prefix and suffix to wrap the user input and context for the model
prefix = f"You are a curious and intelligent adult tasked with {tasktype_option}:\n\n"
suffix = “\n\nResponse:”
# Combine the customization inputs into a single string
combined_text = "\n\n".join([f"{key.capitalize()}: {value}" for key, value in customization.items()])
prompt = f"{prefix}{query}\n\n{combined_text}{suffix}"
# Generate the response
outputs = pipeline(prompt, max_new_tokens=256)
return outputs[0]['generated_text']
Pre-built templates for different content types
prebuilt_templates = {
“Blog”: {
“title”: “Your Blog Title Here”,
“introduction”: “Your Introduction Here”,
“content”: “Your Content Here”,
“conclusion”: “Your Conclusion Here”
},
“Ad”: {
“headline”: “Your Ad Headline Here”,
“main_text”: “Your Main Text Here”,
“call_to_action”: “Your Call to Action Here”
},
“Social Media Post”: {
“caption”: “Your Caption Here”,
“tags”: “Your Tags Here”
}
}
UI Starts here
st.set_page_config(page_title=“AdAlly”,
page_icon=‘
’,
layout=‘centered’,
initial_sidebar_state=‘collapsed’)
st.header(“Hey, How can I help you today?”)
form_input = st.text_area(‘Enter text’, height=275)
tasktype_option = st.selectbox(
‘Please select the action to be performed?’,
(‘Write a Blog’, ‘Write an Ad’, ‘Write a Social Media Post’), key=1)
template_option = st.selectbox(
‘Select a pre-built template’,
(‘Blog’, ‘Ad’, ‘Social Media Post’), key=2)
customization = {}
for key, value in prebuilt_templates[template_option].items():
customization[key] = st.text_area(f’Customize your {key}:', height=100, value=value)
submit = st.button(“Generate”)
if submit:
# Combine the customization inputs into a single string
customized_text = “\n\n”.join([f"{key.capitalize()}: {value}" for key, value in customization.items()])
response = getLLMResponse(form_input, tasktype_option, template_option, customization)
st.write(response)
, this code gets me error as ValueError: checkpoint should be the path to a file containing a whole state dict, or the index of a sharded checkpoint, or a folder containing a sharded checkpoint or the whole state dict, but got meta-llama/Meta-Llama-3.1-8B-Instruct.