Introduce MinIO Object Store event based dataprep and retriever using Milvus and LanceDB

.github/workflows/docker/compose/dataprep-compose.yaml

@@ -3,59 +3,67 @@
 
 # this file should be run in the root of the repo
 services:
-  dataprep-redis:
-    build:
-      dockerfile: comps/dataprep/redis/langchain/Dockerfile
-    image: ${REGISTRY:-opea}/dataprep-redis:${TAG:-latest}
-  dataprep-qdrant:
-    build:
-      dockerfile: comps/dataprep/qdrant/langchain/Dockerfile
-    image: ${REGISTRY:-opea}/dataprep-qdrant:${TAG:-latest}
-  dataprep-on-ray-redis:
-    build:
-      dockerfile: comps/dataprep/redis/langchain_ray/Dockerfile
-    image: ${REGISTRY:-opea}/dataprep-on-ray-redis:${TAG:-latest}
-  dataprep-multimodal-vdms:
-    build:
-      dockerfile: comps/dataprep/vdms/multimodal_langchain/Dockerfile
-    image: ${REGISTRY:-opea}/dataprep-multimodal-vdms:${TAG:-latest}
-  dataprep-multimodal-redis:
-    build:
-      dockerfile: comps/dataprep/multimodal/redis/langchain/Dockerfile
-    image: ${REGISTRY:-opea}/dataprep-multimodal-redis:${TAG:-latest}
-  dataprep-redis-llama-index:
-    build:
-      dockerfile: comps/dataprep/redis/llama_index/Dockerfile
-    image: ${REGISTRY:-opea}/dataprep-redis-llama-index:${TAG:-latest}
-  dataprep-milvus:
-    build:
-      dockerfile: comps/dataprep/milvus/langchain/Dockerfile
-    image: ${REGISTRY:-opea}/dataprep-milvus:${TAG:-latest}
-  dataprep-pgvector:
-    build:
-      dockerfile: comps/dataprep/pgvector/langchain/Dockerfile
-    image: ${REGISTRY:-opea}/dataprep-pgvector:${TAG:-latest}
-  dataprep-pinecone:
-    build:
-      dockerfile: comps/dataprep/pinecone/langchain/Dockerfile
-    image: ${REGISTRY:-opea}/dataprep-pinecone:${TAG:-latest}
-  dataprep-vdms:
-    build:
-      dockerfile: comps/dataprep/vdms/langchain/Dockerfile
-    image: ${REGISTRY:-opea}/dataprep-vdms:${TAG:-latest}
-  dataprep-neo4j:
-    build:
-      dockerfile: comps/dataprep/neo4j/langchain/Dockerfile
-    image: ${REGISTRY:-opea}/dataprep-neo4j:${TAG:-latest}
-  dataprep-neo4j-llamaindex:
-    build:
-      dockerfile: comps/dataprep/neo4j/llama_index/Dockerfile
-    image: ${REGISTRY:-opea}/dataprep-neo4j-llamaindex:${TAG:-latest}
-  dataprep-elasticsearch:
-    build:
-      dockerfile: comps/dataprep/elasticsearch/langchain/Dockerfile
-    image: ${REGISTRY:-opea}/dataprep-elasticsearch:${TAG:-latest}
-  dataprep-opensearch:
-    build:
-      dockerfile: comps/dataprep/opensearch/langchain/Dockerfile
-    image: ${REGISTRY:-opea}/dataprep-opensearch:${TAG:-latest}
+    dataprep-redis:
+        build:
+            dockerfile: comps/dataprep/redis/langchain/Dockerfile
+        image: ${REGISTRY:-opea}/dataprep-redis:${TAG:-latest}
+    dataprep-qdrant:
+        build:
+            dockerfile: comps/dataprep/qdrant/langchain/Dockerfile
+        image: ${REGISTRY:-opea}/dataprep-qdrant:${TAG:-latest}
+    dataprep-on-ray-redis:
+        build:
+            dockerfile: comps/dataprep/redis/langchain_ray/Dockerfile
+        image: ${REGISTRY:-opea}/dataprep-on-ray-redis:${TAG:-latest}
+    dataprep-multimodal-vdms:
+        build:
+            dockerfile: comps/dataprep/vdms/multimodal_langchain/Dockerfile
+        image: ${REGISTRY:-opea}/dataprep-multimodal-vdms:${TAG:-latest}
+    dataprep-multimodal-redis:
+        build:
+            dockerfile: comps/dataprep/multimodal/redis/langchain/Dockerfile
+        image: ${REGISTRY:-opea}/dataprep-multimodal-redis:${TAG:-latest}
+    dataprep-redis-llama-index:
+        build:
+            dockerfile: comps/dataprep/redis/llama_index/Dockerfile
+        image: ${REGISTRY:-opea}/dataprep-redis-llama-index:${TAG:-latest}
+    dataprep-milvus:
+        build:
+            dockerfile: comps/dataprep/milvus/langchain/Dockerfile
+        image: ${REGISTRY:-opea}/dataprep-milvus:${TAG:-latest}
+    dataprep-pgvector:
+        build:
+            dockerfile: comps/dataprep/pgvector/langchain/Dockerfile
+        image: ${REGISTRY:-opea}/dataprep-pgvector:${TAG:-latest}
+    dataprep-pinecone:
+        build:
+            dockerfile: comps/dataprep/pinecone/langchain/Dockerfile
+        image: ${REGISTRY:-opea}/dataprep-pinecone:${TAG:-latest}
+    dataprep-vdms:
+        build:
+            dockerfile: comps/dataprep/vdms/langchain/Dockerfile
+        image: ${REGISTRY:-opea}/dataprep-vdms:${TAG:-latest}
+    dataprep-neo4j:
+        build:
+            dockerfile: comps/dataprep/neo4j/langchain/Dockerfile
+        image: ${REGISTRY:-opea}/dataprep-neo4j:${TAG:-latest}
+    dataprep-neo4j-llamaindex:
+        build:
+            dockerfile: comps/dataprep/neo4j/llama_index/Dockerfile
+        image: ${REGISTRY:-opea}/dataprep-neo4j-llamaindex:${TAG:-latest}
+    dataprep-elasticsearch:
+        build:
+            dockerfile: comps/dataprep/elasticsearch/langchain/Dockerfile
+        image: ${REGISTRY:-opea}/dataprep-elasticsearch:${TAG:-latest}
+    dataprep-opensearch:
+        build:
+            dockerfile: comps/dataprep/opensearch/langchain/Dockerfile
+        image: ${REGISTRY:-opea}/dataprep-opensearch:${TAG:-latest}
+    dataprep-minio-lancedb:
+        build:
+            dockerfile: comps/dataprep/minio/lancedb/langchain/Dockerfile
+        image: ${REGISTRY:-opea}/dataprep-minio-lancedb:${TAG:-latest}
+    dataprep-minio-milvus:
+        build:
+            dockerfile: comps/dataprep/minio/milvus/langchain/Dockerfile
+        image: ${REGISTRY:-opea}/dataprep-minio-milvus:${TAG:-latest}

.github/workflows/docker/compose/retrievers-compose.yaml

@@ -51,3 +51,7 @@ services:
     build:
       dockerfile: comps/retrievers/opensearch/langchain/Dockerfile
     image: ${REGISTRY:-opea}/retriever-opensearch:${TAG:-latest}
+  retriever-minio-lancedb:
+    build:
+      dockerfile: comps/retrievers/minio/lancedb/langchain/Dockerfile
+    image: ${REGISTRY:-opea}/retriever-minio-lancedb:${TAG:-latest}

comps/dataprep/minio/__init__.py

@@ -0,0 +1,2 @@
+# Copyright (c) 2015-2024 MinIO, Inc.
+# SPDX-License-Identifier: Apache-2.0

comps/dataprep/minio/lancedb/langchain/Dockerfile

@@ -0,0 +1,38 @@
+# Copyright (c) 2015-2024 MinIO, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+FROM python:3.11-slim
+
+ENV LANG=C.UTF-8
+
+ARG ARCH="cpu"
+
+RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \
+    build-essential \
+    default-jre \
+    libgl1-mesa-glx \
+    libjemalloc-dev \
+    tesseract-ocr
+
+RUN useradd -m -s /bin/bash user && \
+    mkdir -p /home/user && \
+    chown -R user /home/user/
+
+USER user
+
+COPY comps /home/user/comps
+
+RUN pip install --no-cache-dir --upgrade pip setuptools && \
+    if [ ${ARCH} = "cpu" ]; then pip install --no-cache-dir torch torchvision --index-url https://download.pytorch.org/whl/cpu; fi && \
+    pip install --no-cache-dir -r /home/user/comps/dataprep/minio/lancedb/langchain/requirements.txt
+
+ENV PYTHONPATH=$PYTHONPATH:/home/user
+
+USER root
+
+RUN mkdir -p /home/user/comps/dataprep/minio/lancedb/langchain/uploaded_files && chown -R user /home/user/comps/dataprep/minio/lancedb/langchain/uploaded_files
+
+USER user
+WORKDIR /home/user/comps/dataprep/minio/lancedb/langchain
+
+ENTRYPOINT ["python", "prepare_doc_lancedb.py"]