Skip to content

Commit c6623cb

Browse files
authored
feat: Add RAG endpoint generate_referrals_rag (#111)
1 parent 37f3f4e commit c6623cb

13 files changed

Lines changed: 3444 additions & 1313 deletions

File tree

app/.dockleconfig

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
# https://github.com/goodwithtech/dockle#accept-suspicious-environment-variables--files--file-extensions
55

66
# Hayhooks has an out of the box example and it uses a settings file, we don't need or use it but need to ignore it
7-
DOCKLE_ACCEPT_FILES=app/.venv/lib/python3.12/site-packages/hayhooks/settings.py,app/.venv/lib/python3.12/site-packages/phoenix/otel/settings.py,app/.venv/lib/python3.12/site-packages/phoenix/settings.py,app/.venv/lib/python3.12/site-packages/scipy/_lib/cobyqa/settings.py
7+
DOCKLE_ACCEPT_FILES=app/.venv/lib/python3.12/site-packages/hayhooks/settings.py,app/.venv/lib/python3.12/site-packages/phoenix/otel/settings.py,app/.venv/lib/python3.12/site-packages/phoenix/settings.py,app/.venv/lib/python3.12/site-packages/scipy/_lib/cobyqa/settings.py,app/.venv/lib/python3.12/site-packages/docx/oxml/settings.py,app/.venv/lib/python3.12/site-packages/htmldate/settings.py,app/.venv/lib/python3.12/site-packages/docx/parts/settings.py,app/.venv/lib/python3.12/site-packages/courlan/settings.py,app/.venv/lib/python3.12/site-packages/docx/settings.py,app/.venv/lib/python3.12/site-packages/trafilatura/settings.py,app/.venv/lib/python3.12/site-packages/dateparser_data/settings.py
88

99
# python:3.12-slim doesn't clear package caches after apt-get install; we do so in our own Dockerfile
1010
DOCKLE_IGNORES=DKL-DI-0005

app/Dockerfile

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ FROM python:3.12-slim AS base
99

1010
# Install poetry, the package manager.
1111
# https://python-poetry.org
12-
RUN pip install --no-cache-dir --upgrade pip==25.3 poetry==1.5
12+
RUN pip install --no-cache-dir --upgrade pip==25.3 poetry==2.1
1313

1414
RUN apt-get update \
1515
# Install security updates
@@ -118,7 +118,10 @@ ENV PATH="/app/.venv/bin:$PATH"
118118
# to the Docker container that it's running in.
119119
ENV HOST=0.0.0.0
120120

121+
# Disable posthog telemetry for ChromaDB (used for RAG) https://docs.trychroma.com/docs/overview/telemetry
122+
ENV ANONYMIZED_TELEMETRY=False
123+
121124
USER ${RUN_USER}
122125

123126
# Run the application.
124-
CMD ["poetry", "run", "gunicorn", "-k", "uvicorn.workers.UvicornWorker", "src.app:hayhooks_app", "-b 0.0.0.0:8000"]
127+
CMD ["poetry", "run", "gunicorn", "-k", "uvicorn.workers.UvicornWorker", "src.app:hayhooks_app", "-b", "0.0.0.0:8000"]

app/gunicorn.conf.py

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,8 @@
1111

1212
import os
1313

14-
from src.app_config import AppConfig
15-
16-
app_config = AppConfig()
14+
from src.app_config import config as app_config
15+
from src.ingestion import rag_utils
1716

1817
bind = app_config.host + ':' + str(app_config.port)
1918
# Calculates the number of usable cores and doubles it. Recommended number of workers per core is two.
@@ -23,3 +22,10 @@
2322
# os.cpu_count(): Return the number of CPUs in the system.
2423
workers = len(os.sched_getaffinity(0)) * 2
2524
threads = 4
25+
26+
27+
# This function is called once regardless of the number of workers.
28+
# https://stackoverflow.com/questions/24101724/gunicorn-with-multiple-workers-is-there-an-easy-way-to-execute-certain-code-onl
29+
def when_ready(server: object) -> None:
30+
print("when_ready()", server)
31+
rag_utils.populate_vector_db()

app/poetry.lock

Lines changed: 3156 additions & 1288 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

app/pyproject.toml

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,11 +32,30 @@ certifi = "^2025.8.3"
3232
presidio-analyzer = "^2.2.359"
3333
presidio-anonymizer = "^2.2.359"
3434
spacy = "^3.8.7"
35-
en-core-web-lg = {url = "https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.8.0/en_core_web_lg-3.8.0-py3-none-any.whl"}
35+
# Since Presidio is disabled, exclude the large English model to reduce disk usage in GitHub Actions
36+
# en-core-web-lg = {url = "https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.8.0/en_core_web_lg-3.8.0-py3-none-any.whl"}
3637
pypdf = "^6.1.3"
3738
arize-phoenix = "^11.37.0"
3839
pip = ">=25.3"
3940
markdown = "^3.10"
41+
chroma-haystack = "^3.4.1"
42+
# chroma and posthog compatibility workaround: https://github.com/vanna-ai/vanna/issues/917#issuecomment-3036668545
43+
posthog = ">=2.4.0,<6.0.0"
44+
sentence-transformers = "^5.1.2"
45+
markdown-it-py = "^4.0.0"
46+
python-docx = "^1.2.0"
47+
mdit-plain = "^1.0.1"
48+
trafilatura = "^2.0.0"
49+
python-pptx = "^1.0.2"
50+
jq = "^1.10.0"
51+
openpyxl = "^3.1.5"
52+
tabulate = "^0.9.0"
53+
pandas = "^2.3.3"
54+
# sentence-transformers requires torch, but we want the smaller CPU-only version in order to deploy in GitHub Action
55+
# Need to specify exact torch version due to "Local version label can only be used with `==` or `!=` operators"
56+
# Side-effect on MacOS: Causes error when running `poetry update`: Unable to find installation candidates for torch (2.9.1+cpu)
57+
# Workaround: Run `poetry update ...` within the docker `app` container (linux) instead of on MacOS host
58+
torch = {version = "2.9.1+cpu", source = "pytorch_cpu"}
4059

4160
[tool.poetry.group.dev.dependencies]
4261
certifi = "^2025.8.3"
@@ -59,6 +78,12 @@ ruff = "^0.4.9"
5978
types-requests = "^2.32.4.20250913"
6079
types-markdown = "^3.9.0.20250906"
6180

81+
82+
[[tool.poetry.source]]
83+
name = "pytorch_cpu"
84+
url = "https://download.pytorch.org/whl/cpu"
85+
priority = "explicit"
86+
6287
[build-system]
6388
requires = ["poetry-core>=1.0.0"]
6489
build-backend = "poetry.core.masonry.api"
@@ -74,6 +99,7 @@ manage-crawl-job= "src.db.manage_crawl_job:main"
7499
process-crawl-jobs= "src.ingestion.process_crawl_jobs:main"
75100
delete-support= "src.db.delete_support:main"
76101
delete-support-listing= "src.db.delete_support_listing:main"
102+
populate-vector-db= "src.ingestion.rag_utils:populate_vector_db"
77103

78104
[tool.black]
79105
line-length = 100

app/src/app.py

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,17 +9,13 @@
99
logging.basicConfig(format="%(levelname)s - %(name)s - %(message)s", level=logging.INFO)
1010

1111
logging.info("Configuring Phoenix...")
12-
1312
phoenix_utils.configure_phoenix()
14-
1513
logging.info("Phoenix configured.")
1614

1715
# Boot the standard Hayhooks app
1816

1917
logging.info("Starting Hayhooks app...")
20-
2118
hayhooks_app = create_app()
22-
2319
logging.info("Hayhooks app started.")
2420

2521

app/src/app_config.py

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
1+
import os
12
from functools import cached_property
23

4+
import chromadb
5+
from chromadb.api import ClientAPI
6+
from haystack_integrations.document_stores.chroma import ChromaDocumentStore
7+
38
from src.adapters import db
49
from src.util.env_config import PydanticBaseEnvConfig
510

@@ -38,5 +43,35 @@ def db_session(self) -> db.Session:
3843
"crawl_indeed": "UHJvbXB0VmVyc2lvbjozNA==",
3944
}
4045

46+
# For RAG vector DB
47+
rag_db_host: str = "52.4.126.145"
48+
rag_db_port: int = 8000
49+
collection_name_prefix: str = "referral_resources"
50+
51+
# The embedding model is downloaded by SentenceTransformersTextEmbedder when it first runs
52+
# multi-qa-mpnet-base-cos-v1 was used for pilot 1 but is large (400M)
53+
# all-MiniLM-L6-v2 is a smaller (100M), more efficient model
54+
# When this is changed, the vector DB should be re-populated with embeddings from the new model (populate-vector-db)
55+
rag_embedding_model: str = "multi-qa-mpnet-base-cos-v1"
56+
57+
# The parameters can be adjusted based on the desired chunk size
58+
rag_chunk_split_length: int = 500
59+
rag_chunk_split_overlap: int = 50
60+
retrieval_top_k: int = 10
61+
62+
def chroma_client(self) -> ClientAPI:
63+
return chromadb.HttpClient(host=self.rag_db_host, port=self.rag_db_port)
64+
65+
def chroma_document_store(self) -> ChromaDocumentStore:
66+
return ChromaDocumentStore(
67+
collection_name=f"{self.collection_name_prefix}_{self.environment}",
68+
host=self.rag_db_host,
69+
port=self.rag_db_port,
70+
)
71+
72+
73+
# SENTENCE_TRANSFORMERS_HOME is used by SentenceTransformersTextEmbedder Haystack component
74+
if "SENTENCE_TRANSFORMERS_HOME" not in os.environ:
75+
os.environ["SENTENCE_TRANSFORMERS_HOME"] = os.curdir + "/sentence_transformers"
4176

4277
config = AppConfig()

app/src/common/phoenix_utils.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -88,10 +88,9 @@ def get_prompt_template(prompt_name: str, prompt_version_id: str = "") -> Prompt
8888
client = _create_client()
8989
prompt = client.prompts.get(**prompt_params)
9090
logger.info(
91-
"Retrieved prompt with %r: id='%s'\n%s",
91+
"Retrieved prompt with %r: id='%s'",
9292
prompt_params,
9393
prompt.id,
94-
pformat(prompt._dumps(), width=160),
9594
)
9695
return prompt
9796

app/src/ingestion/rag_utils.py

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
import logging
2+
import os
3+
from pathlib import Path
4+
5+
from botocore.exceptions import NoCredentialsError
6+
from haystack import Pipeline
7+
from haystack.components.converters import MultiFileConverter
8+
from haystack.components.embedders import SentenceTransformersDocumentEmbedder
9+
from haystack.components.preprocessors import DocumentPreprocessor
10+
from haystack.components.writers import DocumentWriter
11+
from haystack_integrations.document_stores.chroma import ChromaDocumentStore
12+
13+
from src.app_config import config
14+
from src.util import file_util
15+
16+
logger = logging.getLogger(__name__)
17+
18+
19+
def populate_vector_db() -> None:
20+
logging.basicConfig(format="%(levelname)s - %(name)s - %(message)s", level=logging.INFO)
21+
22+
chroma_client = config.chroma_client()
23+
logger.info("ChromaDB collections: %s", chroma_client.list_collections())
24+
doc_store = config.chroma_document_store()
25+
collection_name = doc_store._collection_name
26+
27+
# Clear existing collection if any
28+
if doc_store.count_documents() > 0:
29+
logger.info("Deleting existing vector DB collection=%r", collection_name)
30+
chroma_client.delete_collection(collection_name)
31+
# Re-create the document store after deletion
32+
doc_store = config.chroma_document_store()
33+
34+
# Download files from S3
35+
local_folder = download_s3_folder_to_local()
36+
files_to_ingest = [str(p) for p in Path(local_folder).rglob("*") if p.is_file()]
37+
logger.info("Files to ingest: %s", files_to_ingest)
38+
39+
# Ingest documents into ChromaDB
40+
logger.info("Ingesting documents into collection=%r", collection_name)
41+
# Run the pipeline to index documents
42+
pipeline = _create_ingest_pipeline(doc_store)
43+
pipeline.run({"converter": {"sources": files_to_ingest}})
44+
logger.info("Ingested documents doc_count=%d", doc_store.count_documents())
45+
46+
logger.info("ChromaDB collections: %s", chroma_client.list_collections())
47+
48+
49+
def download_s3_folder_to_local(s3_folder: str = "files_to_ingest_into_vector_db") -> str:
50+
"""Download the contents of a folder directory from S3 to a local folder."""
51+
bucket = os.environ.get("BUCKET_NAME", f"labs-referral-pilot-app-{config.environment}")
52+
try:
53+
local_folder = s3_folder
54+
os.makedirs(local_folder, exist_ok=True)
55+
except PermissionError as e:
56+
logger.error("Error creating directories for %s: %s", s3_folder, e)
57+
local_folder = f"/tmp/{s3_folder}" # nosec B108
58+
logger.info("Downloading s3://%s/%s to local folder %s", bucket, s3_folder, local_folder)
59+
60+
if config.environment == "local":
61+
assert os.path.exists(
62+
local_folder
63+
), f"Local folder {local_folder} should exist with manually downloaded files from S3"
64+
return local_folder
65+
66+
s3 = file_util.get_s3_client()
67+
paginator = s3.get_paginator("list_objects_v2")
68+
try:
69+
for result in paginator.paginate(Bucket=bucket, Prefix=s3_folder):
70+
for obj in result.get("Contents", []):
71+
s3_key = obj["Key"]
72+
if s3_key.endswith("/"):
73+
continue # Skip folders
74+
local_file_path = os.path.join(local_folder, os.path.relpath(s3_key, s3_folder))
75+
os.makedirs(os.path.dirname(local_file_path), exist_ok=True)
76+
s3.download_file(bucket, s3_key, local_file_path)
77+
logger.info("Downloaded %s to %s", s3_key, local_file_path)
78+
return local_folder
79+
except NoCredentialsError:
80+
logger.error("AWS credentials not found. Please configure your AWS credentials.")
81+
raise
82+
83+
84+
def _create_ingest_pipeline(doc_store: ChromaDocumentStore) -> Pipeline:
85+
pipeline = Pipeline()
86+
pipeline.add_component("converter", MultiFileConverter())
87+
pipeline.add_component(
88+
"preprocessor",
89+
DocumentPreprocessor(
90+
split_length=config.rag_chunk_split_length,
91+
split_overlap=config.rag_chunk_split_overlap,
92+
remove_empty_lines=False,
93+
remove_extra_whitespaces=False,
94+
),
95+
)
96+
pipeline.add_component(
97+
"embedder", SentenceTransformersDocumentEmbedder(model=config.rag_embedding_model)
98+
)
99+
pipeline.add_component("writer", DocumentWriter(document_store=doc_store))
100+
101+
# Connect the components
102+
pipeline.connect("converter.documents", "preprocessor.documents")
103+
pipeline.connect("preprocessor.documents", "embedder.documents")
104+
pipeline.connect("embedder.documents", "writer.documents")
105+
return pipeline

app/src/pipelines/generate_referrals/pipeline_wrapper.py

Lines changed: 17 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
from haystack import Pipeline
1111
from haystack.components.builders import ChatPromptBuilder
1212
from haystack.core.errors import PipelineRuntimeError
13+
from haystack.dataclasses.chat_message import ChatMessage
1314
from openinference.instrumentation import _tracers, using_attributes, using_metadata
1415
from opentelemetry.trace.status import Status, StatusCode
1516
from pydantic import BaseModel
@@ -129,17 +130,7 @@ def _run(self, query: str, user_email: str, prompt_version_id: str = "") -> dict
129130

130131
try:
131132
response = self.pipeline.run(
132-
{
133-
"logger": {
134-
"messages_list": [{"query": query, "user_email": user_email}],
135-
},
136-
"prompt_builder": {
137-
"template": prompt_template,
138-
"query": query,
139-
"response_json": response_schema,
140-
},
141-
"llm": {"model": "gpt-5-mini", "reasoning_effort": "low"},
142-
},
133+
self._run_arg_data(query, user_email, prompt_template),
143134
include_outputs_from={"llm", "save_result"},
144135
)
145136
logger.debug("Results: %s", pformat(response, width=160))
@@ -150,3 +141,18 @@ def _run(self, query: str, user_email: str, prompt_version_id: str = "") -> dict
150141
except Exception as e:
151142
logger.error("Error %s: %s", type(e), e, exc_info=True)
152143
raise HTTPException(status_code=500, detail=f"Internal error: {str(e)}") from e
144+
145+
def _run_arg_data(
146+
self, query: str, user_email: str, prompt_template: list[ChatMessage]
147+
) -> dict:
148+
return {
149+
"logger": {
150+
"messages_list": [{"query": query, "user_email": user_email}],
151+
},
152+
"prompt_builder": {
153+
"template": prompt_template,
154+
"query": query,
155+
"response_json": response_schema,
156+
},
157+
"llm": {"model": "gpt-5-mini", "reasoning_effort": "low"},
158+
}

0 commit comments

Comments
 (0)