basic_rag_sparse_retrival.py

import os


os.environ["COHERE_API_KEY"] = "w7f2pzS4Ku5JSkVidvDeyDtZJWLlxZHwI5YmGZZP"


from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain_cohere import CohereEmbeddings
from langchain_community.document_loaders import WebBaseLoader
from langchain_community.vectorstores import Chroma
from langchain.chains import RetrievalQA

from langchain_community.retrievers import BM25Retriever
from langchain_core.documents import Document

from langchain_cohere import ChatCohere

llm = ChatCohere(model="command-r", temperature=0)


urls = [
    "https://lilianweng.github.io/posts/2023-06-23-agent/",
    "https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/",
    "https://lilianweng.github.io/posts/2023-10-25-adv-attack-llm/",
    # bez tego llm nie radzi sobie z odpowiedzenim na ptyanie o autorow
    "https://www.sciencedirect.com/science/article/pii/S0378437119303528",
    "https://milvus.io/ai-quick-reference/what-is-the-difference-between-sparse-and-dense-retrieval?utm_source=chatgpt.com",
]

# tutaj uzywamy dokumentow zaladowanych z neta dlatego uzywamy WebBaseLoader
docs = [WebBaseLoader(url).load() for url in urls]

# Tutaj tworzy sie jedna plaska lista, potencjalne miejsce gdzie mozna by strukturyzowac dane do grafu
docs_list = [item for sublist in docs for item in sublist]

text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(
    chunk_size=512,
    chunk_overlap=0,
    # chunk_size=50, chunk_overlap=0
)

chunks = text_splitter.split_documents(docs_list)

bm25_retriever = BM25Retriever.from_documents(chunks)
# bm25_retriever.k = 2
bm25_retriever.k = 4


# tutaj warto pokazac ze 25 chunk_size * 4 chunki nie daja poprawnych rezultatów, natomiast, 2 chunki po 50 chunk_size dziala poprawnie, a teoretycznie w oby przypadkach została zwrocona taka sama ilosc znakow
# query = "What are authors of: 'A review of cellular automata models for crowd evacuation' paper?"
# query = "What are writers of A review of cellular automata evacuation paper?"
# query = "What are authors of A review of cellular automata evacuation paper?"
# query = "How do simulation methods replicate panic-driven group behavior in confined areas?"
# query = "Which computational framework divides the environment into discrete cells to simulate evacuation patterns?"
# query = "What kind of spatial logic is used to simulate individual flow in emergency exits?"
# query = "When was the CA review published?"
# query = "Who wrote the evacuation review?"
# query = "What types of lattice are used in simulation?"
query = "Which retrieval method handles synonyms like 'repair bicycle' and 'fix a bike' more effectively?"

retrieved_docs = bm25_retriever.get_relevant_documents(query)

qa_chain = RetrievalQA.from_chain_type(
    llm=llm, retriever=bm25_retriever, chain_type="stuff"
)

response = qa_chain.invoke(query)


print("Answer: ", response)