Issues regarding using model google t-5 large

🤗Datasets
1

I was encountering an issue of using this code. When i click to use ‘flan-t5’, it shows the image as follows . Does anyone have any idea? Here’s my code: (p.s. i commented on the url as it doesn’t allow me to add more than 2 links in a post
import streamlit as st
from langchain_community.llms import HuggingFaceEndpoint
from langchain.chains import ConversationChain

from langchain.memory import ConversationBufferMemory

from langchain_core.prompts import PromptTemplate
from langchain.memory import ConversationSummaryMemory
from langchain.prompts import PromptTemplate
import os

st.set_page_config(page_title=“Chatbot for ForexForest​:evergreen_tree::seedling:”)

session state variables for chat messages and history

if ‘chat_message’ not in st.session_state:
st.session_state[‘chat_message’] =

if ‘chat_history’ not in st.session_state:
st.session_state[‘chat_history’] =

#allow user to select model
selected_model = st.radio(‘Select a model’, [‘mistral7b’, ‘bert’, ‘distilbert’], key=‘selected_model’)

Define model descriptions

model_descriptions = {
‘mistral7b’: ‘A Text Generation Chatbot.’,
‘bert’: ‘A Text-2-Text Generation Chatbot.’,
‘distilbert’: ‘A Q&A Chatbot.’
}

#display model description
st.write(f" {model_descriptions[selected_model]}")
if selected_model == ‘flan-t5’:
#llm =‘https://api-inference.huggingface.co/models/google/flan-t5-large’

#Hugging Face API token
huggingfacehub_api_token = “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”

#get url based on model name
def get_endpoint_url(llm):
return llm

endpoint_url = get_endpoint_url(llm)

#Hugging Face Endpoint with parameters
hf_endpoint = HuggingFaceEndpoint(
endpoint_url=endpoint_url,
huggingfacehub_api_token=huggingfacehub_api_token,
task=“text2text-generation”,
max_length=200, # Change this value to 200 or lower
temperature=st.sidebar.slider(‘temperature’, min_value=0.01, max_value=1.00, value=0.1, step=0.01)
)

Define the prompt template for summarization

template = “”“You are a helpful assistant that answers question as simple as possible, Please respond concisely .
Current conversation : {history}
{input}
“””

prompt = PromptTemplate(template=template, input_variables=[“input”])

Create a memory object to store conversation history

memory = ConversationSummaryMemory(llm=hf_endpoint)

#load chat history
if ‘chat_history’ not in st.session_state:
st.session_state.chat_history =
else:
for message in st.session_state.chat_history:
memory.save_context({‘input’:message[‘human’]},{‘output’:message[‘assistant’]})

Create the ConversationChain

conversation = ConversationChain(
prompt = prompt,
llm=hf_endpoint,
memory=memory,
verbose=True,
)

start chat sessions

if ‘chat_sessions’ not in st.session_state:
st.session_state[‘chat_sessions’] = {
‘Chat 01’: [{“role”: “assistant”, “content”: “How may I assist you today?”}]}
if ‘current_chat’ not in st.session_state:
st.session_state[‘current_chat’] = ‘Chat 01’

#new chat session function
def create_new_chat():
chat_count = len(st.session_state[‘chat_sessions’])
new_chat_id = f’Chat{chat_count + 1:02d}’
st.session_state[‘chat_sessions’][new_chat_id] = [{“role”: “assistant”, “content”: “How may I assist you today?”}]
st.session_state[‘current_chat’] = new_chat_id
st.rerun()

#function to clear chat history
def clear_chat_history():
st.session_state[‘chat_sessions’][st.session_state[‘current_chat’]] = [{“role”: “assistant”, “content”: “How may I assist you today?”}]

#sidebar for chat controls
with st.sidebar:
st.title(“Chatbot for ForexForest​:evergreen_tree::seedling:”)
if st.button(“New Chat”):
create_new_chat()
st.write(“Chat Sessions:”)
for chat_id in st.session_state[‘chat_sessions’].keys():
if st.button(chat_id):
st.session_state[‘current_chat’] = chat_id
st.rerun()

Ask for the input of question

st.subheader(“Please ask your question”)

Display chat messages

for message in st.session_state[‘chat_sessions’][st.session_state[‘current_chat’]]:
with st.chat_message(message[“role”]):
st.write(message[“content”])

#add sidebar button for clearing chat history
st.sidebar.button(‘Clear Chat History’, on_click=clear_chat_history)

User provided prompt

if user_input := st.chat_input(“Say Something”):
st.session_state[‘chat_sessions’][st.session_state[‘current_chat’]].append({“role”: “user”, “content”: user_input})
with st.chat_message(“user”):
st.write(user_input)

#Initialize messages if not present
if “messages” not in st.session_state.keys():
st.session_state.messages = [{“role”: “assistant”, “content”: “How may I assist you today?”}]

Generate a new response if last message is not from assistant

if st.session_state[‘chat_sessions’][st.session_state[‘current_chat’]] and st.session_state[‘chat_sessions’][st.session_state[‘current_chat’]][-1][“role”] != “assistant”:
with st.chat_message(“assistant”):
with st.spinner(“Thinking…”):
response = conversation.run(user_input)
placeholder = st.empty()
full_response = ‘’
for item in response:
full_response += item
placeholder.markdown(full_response)
message = {“role”: “assistant”, “content”: full_response}
st.session_state[‘chat_sessions’][st.session_state[‘current_chat’]].append(message)

Update chat history in memory

if ‘chat_history’ not in st.session_state:
st.session_state.chat_history =
else:
for message in st.session_state[‘chat_sessions’][st.session_state[‘current_chat’]]:
if message[‘role’] == ‘user’:
memory.save_context({‘input’: message[‘content’]}, {‘output’: ‘’})
elif message[‘role’] == ‘assistant’:
memory.save_context({‘input’: ‘’}, {‘output’: message[‘content’]})

#display chat history
if len(st.session_state.chat_message) == len(st.session_state.chat_history) and len(st.session_state.chat_message) > 1:
for i in reversed(range(len(st.session_state.chat_message) - 1)):
with st.container():
with st.chat_message(“user”):
st.markdown(st.session_state.chat_message[i])
with st.chat_message(“assistant”):
st.markdown(st.session_state.chat_history[i])

image
image594×933 65 KB

2

Hi @Justinlkl!

I see you’re running into some issues using flan-t5 in your Streamlit app. The code looks really well-structured, and it seems like you’ve got the core functionality down.

The issue might be with how you’re referencing the model. Instead of using a variable (llm) to hold the URL, you could try directly passing the URL string to the HuggingFaceEndpoint class.

Here’s an adjusted version of the code snippet:

endpoint_url = "https://api-inference.huggingface.co/models/google/flan-t5-large"
hf_endpoint = HuggingFaceEndpoint(
    endpoint_url=endpoint_url,
    huggingfacehub_api_token=huggingfacehub_api_token,
    task="text2text-generation",
    max_length=200,
    temperature=st.sidebar.slider('temperature', min_value=0.01, max_value=1.00, value=0.1, step=0.01)
)

Let me know if this helps! If not, feel free to share any error messages you’re encountering, and we can troubleshoot further.