rag_ingest_pc.py

from langchain_huggingface import HuggingFaceEmbeddings
from langchain_text_splitters import RecursiveCharacterTextSplitter
from langchain_pinecone import PineconeVectorStore
from pinecone import Pinecone, ServerlessSpec
from langchain_community.document_loaders import PyPDFLoader
from dotenv import load_dotenv
import os

load_dotenv()
api_key = os.environ['PINECONE_API_KEY']

# Load PDF
loaders = [
    PyPDFLoader("data/bindt-guidance.pdf")
]
docs = []
for loader in loaders:
    docs.extend(loader.load())

text_splitter = RecursiveCharacterTextSplitter(chunk_size=800, chunk_overlap=250)
all_split = text_splitter.split_documents(docs)

embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
pc = Pinecone(api_key=api_key)

print(pc.list_indexes())

index_name = "bindt-384"
existing_indexes = [index_info["name"] for index_info in pc.list_indexes()]
if index_name not in existing_indexes:
    pc.create_index(
        name=index_name,
        dimension=384,
        metric="cosine",
        spec=ServerlessSpec(
            cloud="aws",
            region="us-east-1"
        ),
    )
index = pc.Index(index_name)

vector_store = PineconeVectorStore(index=index, embedding=embeddings)
result = vector_store.add_documents(all_split)
print(result)