Merge pull request #15 from dataforgoodfr/test/ingestion_100_papers

Test/ingestion 100 papers

Author: SaboniAmine

.env.example

@@ -0,0 +1,9 @@
+PG_DATABASE_URL = "postgresql+psycopg2://************:**********@*************-postgresql.services.clever-cloud.com:7327/************"
+LLM_INFERENCE_URL = "http://172.17.0.1:11434/v1/"
+LLM_INFERENCE_MODEL = "llama3.2:3b"
+LLM_INFERENCE_API_KEY = "ollama"
+EMBEDDING_MODEL_URL = "http://172.17.0.1:11434/v1/"
+EMBEDDING_MODEL = "snowflake-arctic-embed2"
+EMBEDDING_MODEL_API_KEY = "ollama"
+COLLECTION_ID = 1
+USER_ID = "123456"

.github/workflows/build_kotaemon_service.yaml

@@ -0,0 +1,61 @@
+name: main
+
+on:
+  push:
+    branches:
+      - "*"
+    tags:
+      - "*"
+  workflow_dispatch:
+
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  main:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Determine tag
+        id: tag
+        run: |
+          if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then
+            echo "tag=latest" >> $GITHUB_OUTPUT
+          elif [[ "${{ github.ref }}" == refs/tags/* ]]; then
+            echo "tag=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
+          else
+            # For other branches, use branch name
+            echo "tag=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT
+          fi
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+      - name: Build and push Docker image
+        run: |
+          docker buildx build \
+          --push \
+          --platform=linux/amd64,linux/arm64,linux/armhf \
+          -t ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.tag }} -f rag_system/kotaemon/Dockerfile ./
+      - name: Also create tagged image if tag exists on main branch
+        if: startsWith(github.ref, 'refs/tags/')
+        run: |
+          TAG=${GITHUB_REF#refs/tags/}
+          docker buildx build \
+          --push \
+          --platform=linux/amd64,linux/arm64,linux/armhf \
+          -t ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:$TAG -f ./Dockerfile ./

.gitignore

@@ -169,3 +169,6 @@ cython_debug/
 # the flow stuff
 .theflow
 kotaemon-custom/kotaemon/ktem_app_data
+
+# secret files
+secret*

Dockerfile

@@ -0,0 +1,91 @@
+# Lite version
+FROM python:3.10-slim AS lite
+
+# Common dependencies
+RUN apt-get update -qqy && \
+    apt-get install -y --no-install-recommends \
+        ssh \
+        git \
+        gcc \
+        g++ \
+        poppler-utils \
+        libpoppler-dev \
+        unzip \
+        curl \
+        cargo
+
+# Setup args
+ARG TARGETPLATFORM
+ARG TARGETARCH
+
+# Set environment variables
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+ENV PYTHONIOENCODING=UTF-8
+ENV TARGETARCH=${TARGETARCH}
+
+# Create working directory
+WORKDIR /app
+
+# Download pdfjs
+RUN ls
+ADD rag_system/kotaemon/scripts/download_pdfjs.sh /app/scripts/download_pdfjs.sh
+RUN chmod +x /app/scripts/download_pdfjs.sh
+ENV PDFJS_PREBUILT_DIR="/app/libs/ktem/ktem/assets/prebuilt/pdfjs-dist"
+RUN bash /app/scripts/download_pdfjs.sh $PDFJS_PREBUILT_DIR
+
+# Copy contents
+COPY rag_system /app
+COPY rag_system/kotaemon/launch.sh /app/launch.sh
+COPY rag_system/kotaemon/.env.example /app/.env
+
+WORKDIR /app/kotaemon
+
+# Install pip packages
+RUN pip install -e "libs/kotaemon" \
+    && pip install -e "libs/ktem" \
+    && pip install -e "libs/pipelineblocks" \
+    && pip install "pdfservices-sdk@git+https://github.com/niallcm/pdfservices-python-sdk.git@bump-and-unfreeze-requirements"
+
+# Clean up
+RUN apt-get autoremove \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* \
+    && rm -rf ~/.cache
+
+ENTRYPOINT ["sh", "/app/launch.sh"]
+
+# Full version
+FROM lite AS full
+
+# Additional dependencies for full version
+RUN apt-get update -qqy && \
+    apt-get install -y --no-install-recommends \
+        tesseract-ocr \
+        tesseract-ocr-jpn \
+        libsm6 \
+        libxext6 \
+        libreoffice \
+        ffmpeg \
+        libmagic-dev
+
+# Install torch and torchvision for unstructured
+RUN pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
+RUN pip install psycopg2-binary logfire pydantic==2.10.6
+
+# Install additional pip packages
+RUN pip install -e "libs/kotaemon[adv]" \
+    && pip install unstructured[all-docs]
+
+# Install lightRAG
+ENV USE_LIGHTRAG=false
+
+RUN pip install "docling<=2.5.2"
+
+# Clean up
+RUN apt-get autoremove \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* \
+    && rm -rf ~/.cache
+
+CMD ["sh", "/app/launch.sh"]