semantic.py

from pathlib import Path
from langchain_core.documents import Document
from langchain_community.vectorstores import FAISS
from langchain_huggingface import HuggingFaceEmbeddings

def build_semantic_search(documents: list[Document], save_path: str = "../data/processed/faiss_index") -> tuple:
    """
    Builds a dense retrieval system using Transformer embeddings and FAISS.
    """
    print("Initializing Transformer Embeddings...")
    # This uses the Transformer architecture discussed in Lecture 5 & 6
    embeddings = HuggingFaceEmbeddings(model_name="all-MiniLM-L6-v2")
    
    print("Building FAISS Vector Store...")
    vector_store = FAISS.from_documents(documents, embeddings)

    # Persist the vector store
    Path(save_path).mkdir(parents=True, exist_ok=True)
    vector_store.save_local(save_path)

    print(f"Semantic FAISS index successfully saved to {save_path}")
    return vector_store, embeddings

def load_semantic(path: str) -> tuple:
    """Load a saved FAISS vector store from disk."""
    embeddings = HuggingFaceEmbeddings(model_name="all-MiniLM-L6-v2")
    vector_store = FAISS.load_local(path, embeddings, allow_dangerous_deserialization=True)
    return vector_store, embeddings


def execute_semantic_search(vector_store: FAISS, query: str, k: int = 20):
    """
    Executes a semantic search and returns documents with their L2 distance scores.
    """
    # Returns a list of (Document, score) tuples. 
    # Note: Lower score indicates higher similarity (closer distance in vector space).
    docs_and_scores = vector_store.similarity_search_with_score(query, k=k)
    
    return docs_and_scores