LanceDB as Default Vector Database Backend

.devcontainer/Dockerfile

@@ -6,42 +6,28 @@
 ARG BASE_IMG=nvcr.io/nvidia/cuda
 ARG BASE_IMG_TAG=12.4.1-base-ubuntu22.04
 
-# Use NVIDIA cuda
 FROM $BASE_IMG:$BASE_IMG_TAG AS base
 
-ARG RELEASE_TYPE="dev"
-ARG VERSION=""
-ARG VERSION_REV="0"
-
-# Install necessary dependencies using apt-get
 RUN apt-get update && apt-get install -y \
-      wget \
       bzip2 \
       ca-certificates \
       curl \
+      git \
       libgl1-mesa-glx \
       vim \
-      git \
+      wget \
     && apt-get clean
 
-RUN wget -O Miniforge3.sh "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" -O /tmp/miniforge.sh \
-    && bash /tmp/miniforge.sh -b -p /opt/conda \
-    && rm /tmp/miniforge.sh
-
-# Add conda to the PATH
-ENV PATH=/opt/conda/bin:$PATH
+RUN curl -LsSf https://astral.sh/uv/install.sh | sh
 
-# Install Mamba, a faster alternative to conda, within the base environment
-RUN conda install -y mamba -n base -c conda-forge
+ENV PATH=/root/.local/bin:$PATH
+ENV UV_LINK_MODE=copy
 
-COPY conda/environments/nv_ingest_environment.yml /workspace/nv_ingest_environment.yml
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv python install 3.12 \
+    && uv venv --python 3.12 /opt/nv_ingest_runtime
 
-# Create nv_ingest base environment
-RUN mamba env create -f /workspace/nv_ingest_environment.yml \
-    && conda clean --all --yes
+ENV VIRTUAL_ENV=/opt/nv_ingest_runtime
+ENV PATH=/opt/nv_ingest_runtime/bin:/root/.local/bin:$PATH
 
-# Set default shell to bash
 SHELL ["/bin/bash", "-c"]
-
-# Activate the environment (make it default for subsequent commands)
-RUN echo "source activate nv_ingest_runtime" >> ~/.bashrc

.devcontainer/README.md

@@ -17,7 +17,7 @@ limitations under the License.
 
 # NV-Ingest Devcontainer
 
-The nv-ingest devcontainer is provided as a quick-to-set-up development and exploration environment for use with [Visual Studio Code](https://code.visualstudio.com) (Code). The devcontainer is a lightweight container which mounts-in a Conda environment with cached packages, alleviating long Conda download times on subsequent launches. It provides a simple framework for adding developer-centric [scripts](#development-scripts), and incorporates some helpful Code plugins.
+The nv-ingest devcontainer is provided as a quick-to-set-up development and exploration environment for use with [Visual Studio Code](https://code.visualstudio.com) (Code). The devcontainer is a lightweight container with a pre-created `uv` Python environment and package cache for fast launches. It provides a simple framework for adding developer-centric [scripts](#development-scripts), and incorporates some helpful Code plugins.
 
 > [!Note]
 > NV-Ingest is also known as NVIDIA Ingest and NeMo Retriever extraction.

.devcontainer/devcontainer.json

@@ -82,7 +82,7 @@
           "stkb.rewrap" // Wrapping all text in any language
         ],
         "settings": {
-          "cmake.cmakePath": "/tmp/.current-conda-env/bin/cmake",
+          "cmake.cmakePath": "/usr/bin/cmake",
           "C_Cpp.intelliSenseEngine": "disabled",
           "python.terminal.activateEnvironment": false,
           "files.watcherExclude": {

.dockerignore

@@ -1,2 +1,43 @@
 # Ensure user local scripts are not included in resulting docker images that are built
 scripts/private_local
+
+# Exclude large or permission-restricted paths from build context
+.volumes/
+**/.volumes/
+models/
+outputs/
+triton_models/
+
+.cursor/
+tools/private_local/
+test_results/
+sweep_results/
+artifacts/
+
+lancedb_*/
+tools/harness/lancedb/
+
+history.db
+history.db-shm
+history.db-wal
+
+dist/
+
+.venv/
+.env/
+.env.local/
+.env.development.local/
+.env.test.local/
+.env.production.local/
+
+.env.development/
+.env.test/
+.env.production/
+
+# Keep build context small and avoid permission issues
+.git/
+**/__pycache__/
+**/*.pyc
+*.egg-info/
+**/*.egg-info/
+nemo_retriever/.venv/

.github/actions/determine-version/action.yml

@@ -0,0 +1,54 @@
+name: Determine Version
+description: Determine version string from inputs or generate from date
+
+inputs:
+  version:
+    description: 'Explicit version string (leave empty for auto-generation)'
+    required: false
+    default: ''
+  date-format:
+    description: 'Date format for auto-generated version'
+    required: false
+    default: '%Y.%m.%d'
+  branch-name:
+    description: 'Branch name to extract version from (for release branches)'
+    required: false
+    default: ''
+
+outputs:
+  version:
+    description: 'Determined version string'
+    value: ${{ steps.determine.outputs.version }}
+
+runs:
+  using: composite
+  steps:
+    - name: Determine version
+      id: determine
+      shell: bash
+      run: |
+        VERSION=""
+        INPUT_VERSION="${{ inputs.version }}"
+        INPUT_BRANCH_NAME="${{ inputs.branch-name }}"
+        INPUT_DATE_FORMAT="${{ inputs.date-format }}"
+
+        # Priority 1: Explicit version input
+        if [ -n "${INPUT_VERSION}" ]; then
+          VERSION="${INPUT_VERSION}"
+          echo "Using explicit version: $VERSION"
+
+        # Priority 2: Extract from branch name (for release/* branches)
+        elif [ -n "${INPUT_BRANCH_NAME}" ]; then
+          if [[ "${INPUT_BRANCH_NAME}" == release/* ]]; then
+            VERSION="${INPUT_BRANCH_NAME#release/}"
+            echo "Extracted version from branch: $VERSION"
+          fi
+        fi
+
+        # Priority 3: Generate from date
+        if [ -z "$VERSION" ]; then
+          VERSION="$(date +"${INPUT_DATE_FORMAT}")"
+          echo "Generated version from date: $VERSION"
+        fi
+
+        echo "version=$VERSION" >> $GITHUB_OUTPUT

.github/actions/docker-login-ngc/action.yml

@@ -0,0 +1,19 @@
+name: Login to NGC Registry
+description: Authenticate with NVIDIA NGC Container Registry
+
+inputs:
+  registry:
+    description: 'Registry URL'
+    required: false
+    default: 'nvcr.io'
+  password:
+    description: 'NGC API token'
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - name: Log in to NGC Registry
+      shell: bash
+      run: |
+        echo "${{ inputs.password }}" | docker login ${{ inputs.registry }} --username "\$oauthtoken" --password-stdin

.github/actions/setup-docker-buildx/action.yml

@@ -0,0 +1,24 @@
+name: Setup Docker Buildx
+description: Set up Docker Buildx with optional QEMU for multi-platform builds
+
+inputs:
+  use-qemu:
+    description: 'Enable QEMU emulation for cross-platform builds'
+    required: false
+    default: 'false'
+  platforms:
+    description: 'Platforms to support (comma-separated)'
+    required: false
+    default: 'linux/amd64'
+
+runs:
+  using: composite
+  steps:
+    - name: Set up QEMU
+      if: ${{ inputs.use-qemu == 'true' }}
+      uses: docker/setup-qemu-action@v3
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+      with:
+        platforms: ${{ inputs.platforms }}

.github/workflows/ci-main.yml

@@ -0,0 +1,68 @@
+name: Main Branch CI
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  # Pre-commit checks
+  pre-commit:
+    name: Pre-commit Checks
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v3
+
+      - uses: pre-commit/action@v3.0.1
+
+  # Build standard Docker image
+  docker-build:
+    name: Build Docker Image (amd64)
+    uses: ./.github/workflows/reusable-docker-build.yml
+    with:
+      platform: 'linux/amd64'
+      target: 'runtime'
+      push: false
+      tags: 'nv-ingest:main-${{ github.sha }}'
+      base-image: 'ubuntu'
+      base-image-tag: 'jammy-20250415.1'
+      runner: 'linux-large-disk'
+    secrets:
+      HF_ACCESS_TOKEN: ${{ secrets.HF_ACCESS_TOKEN }}
+
+  # Test Docker image
+  docker-test:
+    name: Test Docker Image
+    needs: docker-build
+    uses: ./.github/workflows/reusable-docker-test.yml
+    with:
+      image-tag: 'nv-ingest:main-${{ github.sha }}'
+      platform: 'linux/amd64'
+      test-selection: 'full'
+      pytest-markers: 'not integration'
+      coverage: true
+      runner: 'linux-large-disk'
+
+  # Test library mode
+  library-mode-test:
+    name: Test Library Mode
+    uses: ./.github/workflows/reusable-integration-test.yml
+    with:
+      runner: 'ubuntu-latest'
+      timeout-minutes: 60
+      source-ref: 'main'
+    secrets:
+      AUDIO_FUNCTION_ID: ${{ secrets.AUDIO_FUNCTION_ID }}
+      EMBEDDING_NIM_MODEL_NAME: ${{ secrets.EMBEDDING_NIM_MODEL_NAME }}
+      NEMOTRON_PARSE_MODEL_NAME: ${{ secrets.NEMOTRON_PARSE_MODEL_NAME }}
+      NGC_API_KEY: ${{ secrets.NGC_API_KEY }}
+      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
+      PADDLE_HTTP_ENDPOINT: ${{ secrets.PADDLE_HTTP_ENDPOINT }}
+      VLM_CAPTION_ENDPOINT: ${{ secrets.VLM_CAPTION_ENDPOINT }}
+      VLM_CAPTION_MODEL_NAME: ${{ secrets.VLM_CAPTION_MODEL_NAME }}
+      YOLOX_GRAPHIC_ELEMENTS_HTTP_ENDPOINT: ${{ secrets.YOLOX_GRAPHIC_ELEMENTS_HTTP_ENDPOINT }}
+      YOLOX_HTTP_ENDPOINT: ${{ secrets.YOLOX_HTTP_ENDPOINT }}
+      YOLOX_TABLE_STRUCTURE_HTTP_ENDPOINT: ${{ secrets.YOLOX_TABLE_STRUCTURE_HTTP_ENDPOINT }}

.github/workflows/ci-pull-request.yml

@@ -0,0 +1,110 @@
+name: Pull Request Validation
+
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+  pull_request_target:
+    types:
+      - opened
+      - synchronize
+      - reopened
+      - labeled
+
+concurrency:
+  group: ${{ github.workflow }}-pr-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+jobs:
+  # Fast pre-commit checks (runs first)
+  pre-commit:
+    name: Pre-commit Checks
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v3
+
+      - uses: pre-commit/action@v3.0.1
+
+  # Docker build + test for x86_64 (single job so built image is available locally)
+  docker-build-and-test:
+    name: Build & Test Docker (amd64)
+    uses: ./.github/workflows/reusable-docker-build-and-test.yml
+    with:
+      platform: 'linux/amd64'
+      target: 'test'
+      tags: 'nv-ingest:pr-${{ github.event.pull_request.number }}'
+      base-image: 'ubuntu'
+      base-image-tag: 'jammy-20250415.1'
+      test-selection: 'full'
+      pytest-markers: 'not integration'
+      coverage: true
+      runner: 'linux-large-disk'
+    secrets:
+      HF_ACCESS_TOKEN: ${{ secrets.HF_ACCESS_TOKEN }}
+
+  # Library mode integration tests (requires approval for external contributors)
+  library-mode-check-access:
+    name: Check Library Mode Test Access
+    if: github.event_name == 'pull_request_target'
+    runs-on: ubuntu-latest
+    outputs:
+      has-access: ${{ steps.check.outputs.has-access }}
+    steps:
+      - name: Check access
+        id: check
+        run: |
+          HAS_ACCESS="false"
+          if [[ "${{ github.event.pull_request.author_association }}" == "MEMBER" ]] || \
+             [[ "${{ github.event.pull_request.author_association }}" == "COLLABORATOR" ]] || \
+             [[ "${{ github.event.pull_request.author_association }}" == "OWNER" ]] || \
+             [[ "${{ contains(github.event.pull_request.labels.*.name, 'ok-to-test') }}" == "true" ]]; then
+            HAS_ACCESS="true"
+          fi
+          echo "has-access=$HAS_ACCESS" >> $GITHUB_OUTPUT
+
+  library-mode-test:
+    name: Test Library Mode
+    needs: library-mode-check-access
+    if: |
+      github.event_name == 'pull_request_target' &&
+      needs.library-mode-check-access.outputs.has-access == 'true'
+    uses: ./.github/workflows/reusable-integration-test.yml
+    with:
+      runner: 'ubuntu-latest'
+      timeout-minutes: 60
+      source-ref: ${{ github.event.pull_request.head.sha }}
+    secrets:
+      AUDIO_FUNCTION_ID: ${{ secrets.AUDIO_FUNCTION_ID }}
+      EMBEDDING_NIM_MODEL_NAME: ${{ secrets.EMBEDDING_NIM_MODEL_NAME }}
+      NEMOTRON_PARSE_MODEL_NAME: ${{ secrets.NEMOTRON_PARSE_MODEL_NAME }}
+      NGC_API_KEY: ${{ secrets.NGC_API_KEY }}
+      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
+      PADDLE_HTTP_ENDPOINT: ${{ secrets.PADDLE_HTTP_ENDPOINT }}
+      VLM_CAPTION_ENDPOINT: ${{ secrets.VLM_CAPTION_ENDPOINT }}
+      VLM_CAPTION_MODEL_NAME: ${{ secrets.VLM_CAPTION_MODEL_NAME }}
+      YOLOX_GRAPHIC_ELEMENTS_HTTP_ENDPOINT: ${{ secrets.YOLOX_GRAPHIC_ELEMENTS_HTTP_ENDPOINT }}
+      YOLOX_HTTP_ENDPOINT: ${{ secrets.YOLOX_HTTP_ENDPOINT }}
+      YOLOX_TABLE_STRUCTURE_HTTP_ENDPOINT: ${{ secrets.YOLOX_TABLE_STRUCTURE_HTTP_ENDPOINT }}
+
+  # Summary job
+  pr-validation-complete:
+    name: PR Validation Complete
+    needs:
+      - pre-commit
+      - docker-build-and-test
+    runs-on: ubuntu-latest
+    if: always()
+    steps:
+      - name: Check job results
+        run: |
+          if [[ "${{ needs.pre-commit.result }}" != "success" ]] || \
+             [[ "${{ needs.docker-build-and-test.result }}" != "success" ]]; then
+            echo "One or more required jobs failed"
+            exit 1
+          fi
+          echo "All required PR checks passed!"