Merge pull request #297 from ansible/TamiTakamiya/AAP-67022/remove-llama-index-dependencies

Copy required files from Lightspeed Core to minimize external dependencies

Author: TamiTakamiya

Containerfile-aap

@@ -1,14 +1,28 @@
-FROM quay.io/lightspeed-core/rag-content-cpu as builder
+FROM registry.access.redhat.com/ubi9/python-312 as builder
 ARG EMBEDDING_MODEL=sentence-transformers/all-mpnet-base-v2
 ARG AAP_VERSION=2.6
 
 WORKDIR /rag-content
+USER 0
 
 COPY aap-product-docs-plaintext ./aap-product-docs-plaintext
 COPY additional_docs ./additional_docs
 
 COPY scripts/custom_processor-aap.py .
-RUN uv run python custom_processor-aap.py  \
+COPY requirements.txt ./requirements.txt
+COPY embeddings_model ./embeddings_model
+COPY src/aap_rag_content ./aap_rag_content
+
+RUN python3.12 -m venv .venv
+RUN pip3.12 install --upgrade pip
+RUN pip3.12 install --no-cache-dir --extra-index-url https://download.pytorch.org/whl/cpu -r requirements.txt
+
+RUN cd embeddings_model; if [ ! -f model.safetensors ]; then  \
+        curl -L -O https://huggingface.co/sentence-transformers/all-mpnet-base-v2/resolve/9a3225965996d404b775526de6dbfe85d3368642/model.safetensors; \
+    fi
+
+ENV PYTHONPATH=/rag-content
+RUN python custom_processor-aap.py  \
   -o ./llama_stack_vector_db \
   -f /rag-content \
   -md embeddings_model/ \

Makefile

@@ -53,6 +53,9 @@ verify: ## Verify the code using various linters
 	black --check scripts
 	ruff check scripts --per-file-ignores=scripts/*:S101
 
+unit-test: ## Run unit tests
+	PYTHONPATH=src:$$PYTHONPATH .venv/bin/pytest tests/ -v -c pyproject.toml
+
 update-docs: ## Update the plaintext OCP docs in ocp-product-docs-plaintext/
 	@set -e && for OCP_VERSION in $$(ls -1 ocp-product-docs-plaintext); do \
 		scripts/get_ocp_plaintext_docs.sh $$OCP_VERSION; \

README.md

@@ -13,11 +13,10 @@ Knowledge Portal (OKP), which provides off-line access to
 Red Hat product documentation.
 Copies of markdown files are stored in this repository.
 
-For generating vector DB, we are using scripts provided by
-the [lightspeed-core/rag-content](https://github.com/lightspeed-core/rag-content/)
-repository with our own [custom_processor-aap.py script](./scripts/custom_processor-aap.py).
-See [the README.md file of lightspeed-core/rag-content](https://github.com/lightspeed-core/rag-content/blob/main/README.md)
-for technical details.
+For generating vector DB, we use the `aap_rag_content` Python package
+located in the `src/` directory, which provides document processing and
+vector database generation capabilities. The [custom_processor-aap.py script](./scripts/custom_processor-aap.py)
+uses this package to process AAP documentation and build the vector database.
 
 
 ## Input files for Vector DBs
@@ -36,6 +35,21 @@ extension.
 used as the input for vector DB.  Its `.metadata` subdirectory contains
 metadata JSON files.
 
+## Running Unit Tests
+
+To run the unit tests for the `aap_rag_content` package:
+
+```commandline
+make unit-test
+```
+
+This will execute all tests in the `tests/` directory using pytest with verbose output.
+The test suite includes tests for:
+- Metadata processing
+- Document processing
+- Vector database generation
+- Utility functions
+
 ## Build a container image
 
 Following command builds a container image, which includes the generated vector DB.

pyproject.toml

@@ -15,6 +15,7 @@ line-length = 100
 [tool.mypy]
 disable_error_code = ["union-attr", "return-value", "arg-type", "import-untyped"]
 ignore_missing_imports = true
+mypy_path = ["src"]
 
 # https://docs.astral.sh/uv/guides/integration/pytorch/
 [tool.uv.sources]
@@ -41,8 +42,8 @@ dependencies = [
     "cryptography==46.0.5", # Transient dep pinned to handle CVE
     "faiss-cpu==1.12",
     "filelock==3.20.3",
-    "llama-stack-client==0.2.22",
-    "llama-stack==0.2.22",
+    "llama-stack-client==0.4.3",
+    "llama-stack==0.4.3",
     "pillow==12.1.1", # Transient dep pinned to handle CVE
     "pyasn1==0.6.3",
     "python-multipart==0.0.22", # Transient dep pinned to handle CVE

requirements.txt

@@ -169,17 +169,13 @@ cryptography==46.0.5 \
     --hash=sha256:f145bba11b878005c496e93e257c1e88f154d278d2638e6450d17e0f31e558d2
     # via
     #   aap-rag-content
-    #   python-jose
+    #   pyjwt
 distro==1.9.0 \
     --hash=sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed \
     --hash=sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2
     # via
     #   llama-stack-client
     #   openai
-ecdsa==0.19.1 \
-    --hash=sha256:30638e27cf77b7e15c4c4cc1973720149e1033827cfd00661ca5c8cc0cdb24c3 \
-    --hash=sha256:478cba7b62555866fcb3bb3fe985e06decbdb68ef55713c4e5ab98c57d508e61
-    # via python-jose
 faiss-cpu==1.12.0 \
     --hash=sha256:016e391f49933875b8d60d47f282f2e93d8ea9f9ffbda82467aa771b11a237db \
     --hash=sha256:2f87cbcd603f3ed464ebceb857971fdebc318de938566c9ae2b82beda8e953c0 \
@@ -194,7 +190,9 @@ faiss-cpu==1.12.0 \
 fastapi==0.128.0 \
     --hash=sha256:1cc179e1cef10a6be60ffe429f79b829dce99d8de32d7acb7e6c8dfdf7f2645a \
     --hash=sha256:aebd93f9716ee3b4f4fcfe13ffb7cf308d99c9f3ab5622d8877441072561582d
-    # via llama-stack
+    # via
+    #   llama-stack
+    #   llama-stack-api
 filelock==3.20.3 \
     --hash=sha256:18c57ee915c7ec61cff0ecf7f0f869936c7c30191bb0cf406f1341778d0834e1 \
     --hash=sha256:4b0dda527ee31078689fc205ec4f1c1bf7d56cf88b6dc9426c4f230e46c2dce1
@@ -284,7 +282,6 @@ huggingface-hub==0.36.0 \
     --hash=sha256:47b3f0e2539c39bf5cde015d63b72ec49baff67b6931c3d97f3f84532e2b8d25 \
     --hash=sha256:7bcc9ad17d5b3f07b57c78e79d527102d08313caa278a641993acddcb894548d
     # via
-    #   llama-stack
     #   sentence-transformers
     #   tokenizers
     #   transformers
@@ -333,7 +330,9 @@ joblib==1.5.3 \
 jsonschema==4.26.0 \
     --hash=sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326 \
     --hash=sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce
-    # via llama-stack
+    # via
+    #   llama-stack
+    #   llama-stack-api
 jsonschema-specifications==2025.9.1 \
     --hash=sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe \
     --hash=sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d
@@ -352,16 +351,18 @@ librt==0.7.8 ; platform_python_implementation != 'PyPy' \
     --hash=sha256:ad64a14b1e56e702e19b24aae108f18ad1bf7777f3af5fcd39f87d0c5a814449 \
     --hash=sha256:bb7a7807523a31f03061288cc4ffc065d684c39db7644c676b47d89553c0d714
     # via mypy
-llama-stack==0.2.22 \
-    --hash=sha256:576752dedc9e9f0fb9da69f373d677d8b4f2ae4203428f676fa039b6813d8450 \
-    --hash=sha256:c6bbda6b5a4417b9a73ed36b9d581fd7ec689090ceefd084d9a078e7acbdc670
+llama-stack==0.4.3 \
+    --hash=sha256:423207eae2b640894992a9075ff9dd6300ff904ab06a49fe38cfe0bb809d4669 \
+    --hash=sha256:70d379ae9dbb5b1d0693f14054d9817aba183ffcd805133f0a4442baee132c6d
+    # via aap-rag-content
+llama-stack-api==0.4.4 \
+    --hash=sha256:3973ca3bacf86916e04e521f77e7909533eec7364d32c3eabc35dc2976dbfe7d \
+    --hash=sha256:7bbc63330ed186502dcd48f65cae014dbeb788ba5690be738c98693cfcd2f599
+    # via llama-stack
+llama-stack-client==0.4.3 \
+    --hash=sha256:97b8cc5032bad4f0cdd1b0ae992cf44f5554679d315b7c40f46deb358c041f50 \
+    --hash=sha256:cb807be258206e8fedeb5e5ceba7be7108d3badb31d74199406808c3d1679c35
     # via aap-rag-content
-llama-stack-client==0.2.22 \
-    --hash=sha256:9a0bc756b91ebd539858eeaf1f231c5e5c6900e1ea4fcced726c6717f3d27ca7 \
-    --hash=sha256:b260d73aec56fcfd8fa601b3b34c2f83c4fbcfb7261a246b02bbdf6c2da184fe
-    # via
-    #   aap-rag-content
-    #   llama-stack
 markdown-it-py==4.0.0 \
     --hash=sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147 \
     --hash=sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3
@@ -454,7 +455,9 @@ numpy==2.4.1 \
 openai==2.15.0 \
     --hash=sha256:42eb8cbb407d84770633f31bf727d4ffb4138711c670565a41663d9439174fba \
     --hash=sha256:6ae23b932cd7230f7244e52954daa6602716d6b9bf235401a107af731baea6c3
-    # via llama-stack
+    # via
+    #   llama-stack
+    #   llama-stack-api
 opentelemetry-api==1.39.1 \
     --hash=sha256:2edd8463432a7f8443edce90972169b195e7d6a05500cd29e6d13898187c9950 \
     --hash=sha256:fbde8c80e1b937a2c61f20347e91c0c18a1940cecf012d62e65a7caf08967c9c
@@ -469,7 +472,9 @@ opentelemetry-exporter-otlp-proto-common==1.39.1 \
 opentelemetry-exporter-otlp-proto-http==1.39.1 \
     --hash=sha256:31bdab9745c709ce90a49a0624c2bd445d31a28ba34275951a6a362d16a0b9cb \
     --hash=sha256:d9f5207183dd752a412c4cd564ca8875ececba13be6e9c6c370ffb752fd59985
-    # via llama-stack
+    # via
+    #   llama-stack
+    #   llama-stack-api
 opentelemetry-proto==1.39.1 \
     --hash=sha256:22cdc78efd3b3765d09e68bfbd010d4fc254c9818afd0b6b423387d9dee46007 \
     --hash=sha256:6c8e05144fc0d3ed4d22c2289c6b126e03bcd0e6a7da0f16cedd2e1c2772e2c8
@@ -481,6 +486,7 @@ opentelemetry-sdk==1.39.1 \
     --hash=sha256:cf4d4563caf7bff906c9f7967e2be22d0d6b349b908be0d90fb21c8e9c995cc6
     # via
     #   llama-stack
+    #   llama-stack-api
     #   opentelemetry-exporter-otlp-proto-http
 opentelemetry-semantic-conventions==0.60b1 \
     --hash=sha256:87c228b5a0669b748c76d76df6c364c369c28f1c465e50f661e39737e84bc953 \
@@ -569,17 +575,28 @@ protobuf==6.33.4 \
     # via
     #   googleapis-common-protos
     #   opentelemetry-proto
+psycopg2-binary==2.9.11 \
+    --hash=sha256:31b32c457a6025e74d233957cc9736742ac5a6cb196c6b68499f6bb51390bd6a \
+    --hash=sha256:62b6d93d7c0b61a1dd6197d208ab613eb7dcfdcca0a49c42ceb082257991de9d \
+    --hash=sha256:a1cf393f1cdaf6a9b57c0a719a1068ba1069f022a59b8b1fe44b006745b59757 \
+    --hash=sha256:ab8905b5dcb05bf3fb22e0cf90e10f469563486ffb6a96569e51f897c750a76a \
+    --hash=sha256:b33fabeb1fde21180479b2d4667e994de7bbf0eec22832ba5d9b5e4cf65b6c6d \
+    --hash=sha256:b6aed9e096bf63f9e75edf2581aa9a7e7186d97ab5c177aa6c87797cd591236c \
+    --hash=sha256:be9b840ac0525a283a96b556616f5b4820e0526addb8dcf6525a0fa162730be4 \
+    --hash=sha256:bf940cd7e7fec19181fdbc29d76911741153d51cab52e5c21165f3262125685e \
+    --hash=sha256:edcb3aeb11cb4bf13a2af3c53a15b3d612edeb6409047ea0b5d6a21a9d744b34 \
+    --hash=sha256:ef7a6beb4beaa62f88592ccc65df20328029d721db309cb3250b0aae0fa146c3 \
+    --hash=sha256:f090b7ddd13ca842ebfe301cd587a76a4cf0913b1e429eb92c1be5dbeb1a19bc \
+    --hash=sha256:fa0f693d3c68ae925966f0b14b8edda71696608039f4ed61b1fe9ffa468d16db
+    # via llama-stack
 pyaml==25.7.0 \
     --hash=sha256:ce5d7867cc2b455efdb9b0448324ff7b9f74d99f64650f12ca570102db6b985f \
     --hash=sha256:e113a64ec16881bf2b092e2beb84b7dcf1bd98096ad17f5f14e8fb782a75d99b
     # via llama-stack-client
 pyasn1==0.6.3 \
     --hash=sha256:697a8ecd6d98891189184ca1fa05d1bb00e2f84b5977c481452050549c8a72cf \
     --hash=sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde
-    # via
-    #   aap-rag-content
-    #   python-jose
-    #   rsa
+    # via aap-rag-content
 pycparser==2.23 ; implementation_name != 'PyPy' and platform_python_implementation != 'PyPy' \
     --hash=sha256:78816d4f24add8f10a06d6f05b4d424ad9e96cfebf68a4ddc99c65c0720d00c2 \
     --hash=sha256:e5c6e8d3fbad53479cab09ac03729e0a9faf2bee3db8208a550daf5af81a5934
@@ -590,6 +607,7 @@ pydantic==2.12.5 \
     # via
     #   fastapi
     #   llama-stack
+    #   llama-stack-api
     #   llama-stack-client
     #   openai
 pydantic-core==2.41.5 \
@@ -617,6 +635,10 @@ pygments==2.19.2 \
     --hash=sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887 \
     --hash=sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b
     # via rich
+pyjwt==2.12.1 \
+    --hash=sha256:28ca37c070cad8ba8cd9790cd940535d40274d22f80ab87f3ac6a713e6e8454c \
+    --hash=sha256:c74a7a2adf861c04d002db713dd85f84beb242228e671280bf709d765b03672b
+    # via llama-stack
 python-dateutil==2.9.0.post0 \
     --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \
     --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427
@@ -625,10 +647,6 @@ python-dotenv==1.2.1 \
     --hash=sha256:42667e897e16ab0d66954af0e60a9caa94f0fd4ecf3aaf6d2d260eec1aa36ad6 \
     --hash=sha256:b81ee9561e9ca4004139c6cbba3a238c32b03e4894671e181b671e8cb8425d61
     # via llama-stack
-python-jose==3.5.0 \
-    --hash=sha256:abd1202f23d34dfad2c3d28cb8617b90acf34132c7afd60abd0b0b7d3cb55771 \
-    --hash=sha256:fb4eaa44dbeb1c26dcc69e4bd7ec54a1cb8dd64d3b4d81ef08d90ff453f2b01b
-    # via llama-stack
 python-multipart==0.0.22 \
     --hash=sha256:2b2cd894c83d21bf49d702499531c7bafd057d730c201782048f7945d82de155 \
     --hash=sha256:7340bef99a7e0032613f56dc36027b959fd3b30a787ed62d310e951f7c3a3a58
@@ -657,6 +675,7 @@ pyyaml==6.0.3 \
     --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0
     # via
     #   huggingface-hub
+    #   llama-stack
     #   pyaml
     #   transformers
 referencing==0.37.0 \
@@ -722,10 +741,6 @@ rpds-py==0.30.0 \
     # via
     #   jsonschema
     #   referencing
-rsa==4.9.1 \
-    --hash=sha256:68635866661c6836b8d39430f97a996acbd61bfa49406748ea243539fe239762 \
-    --hash=sha256:e7bdbfdb5497da4c07dfd35530e1a902659db6ff241e39d9953cad06ebd0ae75
-    # via python-jose
 ruff==0.14.13 \
     --hash=sha256:4acdf009f32b46f6e8864af19cbf6841eaaed8638e65c8dac845aea0d703c841 \
     --hash=sha256:591a7f68860ea4e003917d19b5c4f5ac39ff558f162dc753a2c5de897fd5502c \
@@ -798,9 +813,7 @@ setuptools==80.9.0 \
 six==1.17.0 \
     --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \
     --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81
-    # via
-    #   ecdsa
-    #   python-dateutil
+    # via python-dateutil
 sniffio==1.3.1 \
     --hash=sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 \
     --hash=sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc
@@ -817,7 +830,9 @@ sqlalchemy==2.0.46 \
     --hash=sha256:cf36851ee7219c170bb0793dbc3da3e80c582e04a5437bc601bfe8c85c9216d7 \
     --hash=sha256:ea3cd46b6713a10216323cda3333514944e510aa691c945334713fca6b5279ff \
     --hash=sha256:f9c11766e7e7c0a2767dda5acb006a118640c9fc0a4104214b96269bfb78399e
-    # via aap-rag-content
+    # via
+    #   aap-rag-content
+    #   llama-stack
 starlette==0.50.0 \
     --hash=sha256:9e5391843ec9b6e472eed1365a78c8098cfceb7a74bfd4d6b1c0c0095efb3bca \
     --hash=sha256:a2a17b22203254bcbc2e1f926d2d55f3f9497f769416b3190768befe598fa3ca
@@ -881,6 +896,18 @@ torch==2.9.1+cpu ; sys_platform != 'darwin' \
     # via
     #   aap-rag-content
     #   sentence-transformers
+tornado==6.5.5 \
+    --hash=sha256:192b8f3ea91bd7f1f50c06955416ed76c6b72f96779b962f07f911b91e8d30e9 \
+    --hash=sha256:2c9a876e094109333f888539ddb2de4361743e5d21eece20688e3e351e4990a6 \
+    --hash=sha256:36abed1754faeb80fbd6e64db2758091e1320f6bba74a4cf8c09cd18ccce8aca \
+    --hash=sha256:3f54aa540bdbfee7b9eb268ead60e7d199de5021facd276819c193c0fb28ea4e \
+    --hash=sha256:435319e9e340276428bbdb4e7fa732c2d399386d1de5686cb331ec8eee754f07 \
+    --hash=sha256:487dc9cc380e29f58c7ab88f9e27cdeef04b2140862e5076a66fb6bb68bb1bfa \
+    --hash=sha256:6443a794ba961a9f619b1ae926a2e900ac20c34483eea67be4ed8f1e58d3ef7b \
+    --hash=sha256:65a7f1d46d4bb41df1ac99f5fcb685fb25c7e61613742d5108b010975a9a6521 \
+    --hash=sha256:dd3eafaaeec1c7f2f8fdcd5f964e8907ad788fe8a5a32c4426fbbdda621223b7 \
+    --hash=sha256:e74c92e8e65086b338fd56333fb9a68b9f6f2fe7ad532645a290a464bcf46be5
+    # via llama-stack
 tqdm==4.67.1 \
     --hash=sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2 \
     --hash=sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2
@@ -933,6 +960,7 @@ urllib3==2.6.3 \
     --hash=sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4
     # via
     #   aap-rag-content
+    #   llama-stack
     #   requests
     #   types-requests
 uvicorn==0.40.0 \

scripts/custom_processor-aap.py

@@ -2,9 +2,9 @@
 import json
 from pathlib import Path
 
-from lightspeed_rag_content.metadata_processor import MetadataProcessor
-from lightspeed_rag_content.document_processor import DocumentProcessor
-from lightspeed_rag_content import utils
+from aap_rag_content.metadata_processor import MetadataProcessor
+from aap_rag_content.document_processor import DocumentProcessor
+from aap_rag_content import utils
 
 # Folders where AAP product documentation markdown (.md) files are stored.
 AAP_PRODUCT_DOCS = [

src/aap_rag_content/__init__.py

@@ -0,0 +1,9 @@
+"""AAP RAG Content package for document processing and vector database management."""
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "document_processor",
+    "metadata_processor",
+    "utils",
+]