I have the following code
from langchain_community.document_loaders.csv_loader import CSVLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter
# from langchain_community.embeddings import HuggingFaceEmbeddings
from langchain_huggingface import HuggingFaceEmbeddings
from langchain_community.vectorstores import FAISS
from langchain_community.llms import CTransformers
from langchain.memory import ConversationBufferMemory
from langchain.chains import ConversationalRetrievalChain
import sys
# CTransformers for GGML models
# This script works relatively well with the small example
# but we have not tried it with the big one
DB_FAISS_PATH = "vectorstore/db_faiss"
# # datafile = "../expdata/output_2024-09-06_17-44-59.txt"
# datafile = "../expdata/example.txt"
datafile = "../expdata/output_2024-09-06_17-44-59.csv"
# datafile = "../expdata/example.csv"
loader = CSVLoader(file_path=datafile, encoding="utf-8", csv_args={'delimiter': ','})
data = loader.load()
# Split the text into Chunks
text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=20) #Put 100 later
text_chunks = text_splitter.split_documents(data)
print(f" Text chunks len {len(text_chunks)}")
print("----------------")
print(text_chunks[0])
print("-----------------")
# Download Sentence Transformers Embedding From Hugging Face
embeddings = HuggingFaceEmbeddings(model_name = 'sentence-transformers/all-MiniLM-L6-v2')
# what we called vectordb in the other script
docsearch = FAISS.from_documents(text_chunks, embeddings)
docsearch.save_local(DB_FAISS_PATH)
MAX_TOKENS= 1024
llm = CTransformers(model="../models/llama-2-7b-chat.ggmlv3.q4_0.bin",
model_type="llama",
max_new_tokens=MAX_TOKENS,
temperature=0.1)
qa = ConversationalRetrievalChain.from_llm(llm, retriever=docsearch.as_retriever())
while True:
chat_history = []
#query = "What is the value of GDP per capita of Finland provided in the data?"
query = input(f"Input Prompt: ")
if query == 'exit':
print('Exiting')
sys.exit()
if query == '':
continue
result = qa({"question":query, "chat_history":chat_history})
print(f"History {chat_history}")
print("RESPONSE: ", result['answer'])
When I run this code with a small csv file (10 rows, one sentence) (example.csv) the script works reasonably well. (and this was when MAX_TOKENS was 512)
However when I run the script with a bigger csv file (output_2024-09-06_17-44-59.csv), I got the following:
Number of tokens (525) exceeded maximum context length (512)
his was when MAX_TOKENS was 512 but also now that it is 1024 ( and there is no 512 value anywhere)
Why is this happening and how can I solve this?