docker-compose.yml

# Copyright 2026 Snowflake Inc.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

services:
  # Verify Docker VM has enough memory for the full stack
  check-resources:
    image: alpine:latest
    container_name: check-resources
    entrypoint: /bin/sh
    command:
      - "-c"
      - |
        total_kb=$$(awk '/MemTotal/ {print $$2}' /proc/meminfo)
        total_gb=$$((total_kb / 1024 / 1024))
        required_gb=16
        echo "Docker VM memory: $${total_gb} GB ($${total_kb} kB)"
        if [ "$$total_gb" -lt "$$required_gb" ]; then
          echo "WARNING: At least $${required_gb} GB recommended."
          echo "Increase memory in Docker Desktop > Settings > Resources."
          echo "Continuing anyway — some services may run out of memory."
        else
          echo "Memory check passed."
        fi
    networks:
      - iceberg-net

  # MinIO - S3-compatible object storage
  minio:
    image: quay.io/minio/minio:latest
    container_name: minio
    ports:
      - "9000:9000"
      - "9001:9001"
    environment:
      MINIO_ROOT_USER: admin
      MINIO_ROOT_PASSWORD: password
    command: server /data --console-address ":9001"
    volumes:
      - ./data/minio:/data
    depends_on:
      check-resources:
        condition: service_completed_successfully
    networks:
      - iceberg-net
    healthcheck:
      test: ["CMD", "curl", "http://127.0.0.1:9000/minio/health/live"]
      interval: 1s
      timeout: 10s

  # Polaris Iceberg REST Catalog (following official Apache guide)
  polaris:
    image: apache/polaris:latest
    container_name: polaris-catalog
    ports:
      - "8181:8181"
      - "5005:5005"
    environment:
      JAVA_DEBUG: "true"
      JAVA_DEBUG_PORT: "*:5005"
      AWS_REGION: us-west-2
      AWS_ACCESS_KEY_ID: admin
      AWS_SECRET_ACCESS_KEY: password
      # Bootstrap credentials: REALM,CLIENT_ID,CLIENT_SECRET
      POLARIS_BOOTSTRAP_CREDENTIALS: "POLARIS,root,s3cr3t"
      polaris.realm-context.realms: "POLARIS"
      quarkus.otel.sdk.disabled: "true"
    volumes:
      - ./data/polaris:/var/polaris
    depends_on:
      minio:
        condition: service_healthy
    networks:
      - iceberg-net
    healthcheck:
      test: ["CMD", "curl", "http://localhost:8182/q/health"]
      interval: 2s
      timeout: 10s
      retries: 10
      start_period: 10s

  # Setup MinIO bucket
  setup_bucket:
    image: quay.io/minio/mc:latest
    container_name: setup-minio
    depends_on:
      minio:
        condition: service_healthy
    entrypoint: "/bin/sh"
    command:
      - "-c"
      - >-
        echo Creating MinIO bucket...;
        mc alias set pol http://minio:9000 admin password;
        mc mb pol/warehouse || true;
        mc ls pol;
        echo Bucket setup complete.
    networks:
      - iceberg-net

  # Setup Polaris catalog
  polaris-setup:
    image: alpine/curl
    container_name: setup-polaris
    depends_on:
      polaris:
        condition: service_healthy
    environment:
      - CLIENT_ID=root
      - CLIENT_SECRET=s3cr3t
    volumes:
      - ./polaris:/setup
    entrypoint: "/bin/sh"
    command:
      - "-c"
      - >-
        apk add --no-cache jq;
        chmod +x /setup/*.sh;
        /setup/bootstrap-catalog.sh;
    networks:
      - iceberg-net

  # Jupyter Notebook with PySpark and Iceberg (Spark Connect architecture)
  jupyter:
    image: quay.io/jupyter/pyspark-notebook:spark-${SPARK_VERSION}
    container_name: jupyter-spark
    ports:
      - "8888:8888"
      - "4040:4040"
      - "15002:15002"
    depends_on:
      check-resources:
        condition: service_completed_successfully
    environment:
      - JUPYTER_ENABLE_LAB=yes
      - SPARK_REMOTE=sc://localhost:15002
      - POLARIS_URI=http://polaris:8181
      - ICEBERG_VERSION=${ICEBERG_VERSION}
      - ICEBERG_SPARK_RUNTIME_VERSION=${ICEBERG_SPARK_RUNTIME_VERSION}
      - SCALA_VERSION=${SCALA_VERSION}
      - AWS_SDK_VERSION=${AWS_SDK_VERSION}
      - POLARIS_CLIENT_ID=${POLARIS_CLIENT_ID}
      - POLARIS_CLIENT_SECRET=${POLARIS_CLIENT_SECRET}
      - AWS_REGION=irrelevant
      - SPARK_CONF_DIR=/home/jovyan/.sparkconf
    volumes:
      - ./notebooks:/home/jovyan/work
      - ./data/jupyter:/data/jupyter
      - ./spark-defaults.conf.template:/tmp/spark-defaults.conf.template:ro
      - ./generate-spark-config.py:/tmp/generate-spark-config.py:ro
      - ./jars:/opt/spark-jars:ro
    command:
      - bash
      - -c
      - |
        set -e
        mkdir -p /home/jovyan/.sparkconf
        python3 /tmp/generate-spark-config.py /home/jovyan/.sparkconf/spark-defaults.conf
        pip install --quiet pyiceberg[pyarrow,pandas] trino sqlalchemy matplotlib seaborn requests boto3
        echo 'Starting Spark Connect server...'
        (unset SPARK_REMOTE && /usr/local/spark/bin/spark-submit --class org.apache.spark.sql.connect.service.SparkConnectServer --master 'local[*]' --conf spark.testing=true > /tmp/spark-connect.log 2>&1) &
        echo 'Waiting for Spark Connect server on port 15002...'
        for i in $$(seq 1 60); do
          python3 -c "import socket; s=socket.socket(); s.connect(('localhost', 15002)); s.close()" 2>/dev/null && break || sleep 2
        done
        echo 'Spark Connect server is ready'
        exec jupyter lab --ip=0.0.0.0 --port=8888 --no-browser --NotebookApp.token='' --NotebookApp.password='' --NotebookApp.allow_origin='*'
    networks:
      - iceberg-net
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8888/api"]
      interval: 10s
      timeout: 10s
      retries: 5
      start_period: 180s

  # Trino
  trino:
    image: trinodb/trino:${TRINO_VERSION}
    container_name: trino
    ports:
      - "8080:8080"
    volumes:
      - ./trino/catalog:/etc/trino/catalog
      - ./trino/config:/etc/trino
      - ./data/trino:/data/trino
    depends_on:
      polaris:
        condition: service_healthy
      polaris-setup:
        condition: service_completed_successfully
    networks:
      - iceberg-net

networks:
  iceberg-net:
    driver: bridge