docker-compose.yml

services:
  # ============================================
  # Storage Layer
  # ============================================

  minio:
    image: minio/minio:latest
    container_name: minio
    environment:
      MINIO_ROOT_USER: ${MINIO_ROOT_USER}
      MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
    ports:
      - "${MINIO_PORT}:9000"
      - "${MINIO_CONSOLE_PORT}:9001"
    volumes:
      - minio_data:/bucket_data
    command: server /bucket_data --console-address ":9001"
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
      interval: 30s
      timeout: 20s
      retries: 3
    networks:
      - surimi-network

  minio-init:
    image: minio/mc:latest
    container_name: minio-init
    depends_on:
      minio:
        condition: service_healthy
    environment:
      MINIO_ROOT_USER: ${MINIO_ROOT_USER}
      MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
      MINIO_ENDPOINT: http://minio:9000
      MINIO_BUCKET: ${MINIO_BUCKET}
      PROCESSED_PREFIX: ${PROCESSED_PREFIX}
    volumes:
      - ./minio/minio_bootstrap.sh:/bootstrap/minio_bootstrap.sh:ro
    networks:
      - surimi-network
    entrypoint: ["/bin/sh", "/bootstrap/minio_bootstrap.sh"]
    restart: "no"

  # ============================================
  # Metadata & Database Layer
  # ============================================

  postgres:
    image: postgres:14-alpine
    container_name: postgres
    environment:
      POSTGRES_DB: ${POSTGRES_DB}
      POSTGRES_USER: ${POSTGRES_USER}
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
    ports:
      - "${POSTGRES_HOST_PORT}:5432"
    volumes:
      - postgres_data:/var/lib/postgresql/data
      - ./postgres/postgres-init.sql:/docker-entrypoint-initdb.d/init.sql
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER}"]
      interval: 10s
      timeout: 5s
      retries: 5
    networks:
      - surimi-network

  # ============================================
  # Hive Metastore (for Trino catalog)
  # ============================================

  hive-metastore:
    build: ./hive-metastore
    container_name: hive-metastore
    entrypoint: /opt/hive/entrypoint.sh
    environment:
      SERVICE_NAME: metastore
      DB_TYPE: postgres
      METASTORE_CONNECTION_URL: ${METASTORE_CONNECTION_URL}
      METASTORE_CONNECTION_USER: ${METASTORE_CONNECTION_USER}
      METASTORE_CONNECTION_PASSWORD: ${METASTORE_CONNECTION_PASSWORD}
      METASTORE_CONNECTION_DRIVER_JAR_URL: ${METASTORE_CONNECTION_DRIVER_URL}
      HADOOP_HOME: ${HADOOP_HOME}
      HADOOP_CONF_DIR: ${HADOOP_CONF_DIR}
      HADOOP_CLASSPATH: /opt/hadoop/share/hadoop/tools/lib/*
      # S3A configuration
      FS_S3A_ENDPOINT: ${FS_S3A_ENDPOINT}
      FS_S3A_ACCESS_KEY: ${FS_S3A_ACCESS_KEY}
      FS_S3A_SECRET_KEY: ${FS_S3A_SECRET_KEY}
      FS_S3A_PATH_STYLE_ACCESS: ${FS_S3A_PATH_STYLE_ACCESS}
      FS_S3A_CONNECTION_SSL_ENABLED: ${FS_S3A_CONNECTION_SSL_ENABLED}
    ports:
      - "${HIVE_METASTORE_PORT}:9083"
    depends_on:
      postgres:
        condition: service_healthy
    volumes:
      - hive_data:/opt/hive/data/warehouse
      - minio_data:/bucket_data
      - ./hive-metastore/hive-site.xml.template:/opt/hive/conf/hive-site.xml.template
      - ./trino/etc/core-site.xml:/opt/hadoop/conf/core-site.xml
      - ./trino/etc/core-site.xml.template:/opt/hadoop/conf/core-site.xml.template
    networks:
      - surimi-network
    healthcheck:
      test: ["CMD-SHELL", "bash -c 'echo > /dev/tcp/localhost/9083' && echo 'healthy' || exit 1"]
      interval: 10s
      timeout: 10s
      retries: 5

  # ============================================
  # Trino (SQL Query Engine)
  # ============================================

  trino:
    image: trinodb/trino:478
    container_name: trino
    ports:
      - "${TRINO_HOST_PORT}:8080"
    environment:
      TRINO_JVM_MAX_HEAP: ${TRINO_JVM_MAX_HEAP}
      HADOOP_CONF_DIR: /etc/hadoop/conf
      FS_S3A_ENDPOINT: ${FS_S3A_ENDPOINT}
      FS_S3A_ACCESS_KEY: ${FS_S3A_ACCESS_KEY}
      FS_S3A_SECRET_KEY: ${FS_S3A_SECRET_KEY}
      FS_S3A_PATH_STYLE_ACCESS: ${FS_S3A_PATH_STYLE_ACCESS}
      FS_S3A_CONNECTION_SSL_ENABLED: ${FS_S3A_CONNECTION_SSL_ENABLED}
      S3_REGION: ${S3_REGION:-us-east-1}
    entrypoint: ["/etc/trino/entrypoint.sh"]
    volumes:
      - ./trino/etc:/etc/trino
      - ./trino/etc:/etc/hadoop/conf
      - trino_data:/var/trino/data
      - minio_data:/bucket_data
    depends_on:
      hive-metastore:
        condition: service_healthy
      minio:
        condition: service_healthy
    networks:
      - surimi-network
    healthcheck:
      test: ["CMD-SHELL", "curl -f http://localhost:8080/ui/ || exit 1"]
      interval: 10s
      timeout: 10s
      start_period: 30s
      retries: 5

  # ============================================
  # Analytics & BI
  # ============================================

  superset:
    build: ./superset
    container_name: superset
    environment:
      SUPERSET_SECRET_KEY: ${SUPERSET_SECRET_KEY}
      SQLALCHEMY_DATABASE_URI: ${SUPERSET_DB_URI}
      TRINO_HOST: ${TRINO_HOST}
      TRINO_PORT: ${TRINO_PORT}
      TRINO_CATALOG: ${TRINO_CATALOG}
      TRINO_SCHEMA: ${TRINO_SCHEMA}
      TRINO_USER: ${TRINO_USER}
      # Keycloak SSO — set SUPERSET_AUTH_PROVIDER=keycloak in .env to enable
      AUTH_PROVIDER: ${SUPERSET_AUTH_PROVIDER:-db}
      OAUTH_CLIENT_ID: ${SUPERSET_OAUTH_CLIENT_ID}
      OAUTH_CLIENT_SECRET: ${SUPERSET_OAUTH_CLIENT_SECRET}
      OAUTH_SCOPE: ${SUPERSET_OAUTH_SCOPE}
      OAUTH_BASE_URL: ${SUPERSET_OAUTH_BASE_URL}
      OAUTH_TOKEN_URL: ${SUPERSET_OAUTH_TOKEN_URL}
      OAUTH_AUTHORIZE_URL: ${SUPERSET_OAUTH_AUTHORIZE_URL}
      OAUTH_USERDATA_URL: ${SUPERSET_OAUTH_USERDATA_URL}
      OAUTH_JWKS_URL: ${SUPERSET_OAUTH_JWKS_URL}
    ports:
      - "${SUPERSET_PORT}:8088"
    depends_on:
      postgres:
        condition: service_healthy
      trino:
        condition: service_healthy
    volumes:
      - superset_data:/var/lib/superset
    networks:
      - surimi-network
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8088/health"]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 60s

  # ============================================
  # Orchestration
  # ============================================

  airflow-db:
    image: postgres:14-alpine
    container_name: airflow-db
    environment:
      POSTGRES_DB: ${AIRFLOW_DB_NAME}
      POSTGRES_USER: ${AIRFLOW_DB_USER}
      POSTGRES_PASSWORD: ${AIRFLOW_DB_PASSWORD}
    volumes:
      - airflow_db_data:/var/lib/postgresql/data
    networks:
      - surimi-network

  airflow-init:
    build:
      context: ./airflow
      dockerfile: Dockerfile
    platform: ${DOCKER_DEFAULT_PLATFORM}
    image: surimi-airflow:2.11.0
    container_name: airflow-init
    command: >
      bash -c "airflow db migrate &&
      airflow users create
      --username '${AIRFLOW_ADMIN_USER}'
      --firstname '${AIRFLOW_ADMIN_FIRSTNAME}'
      --lastname '${AIRFLOW_ADMIN_LASTNAME}'
      --role Admin
      --email '${AIRFLOW_ADMIN_EMAIL}'
      --password '${AIRFLOW_ADMIN_PASSWORD}' || true"
    environment:
      AIRFLOW__CORE__DAGS_FOLDER: ${AIRFLOW__CORE__DAGS_FOLDER}
      AIRFLOW__CORE__LOAD_EXAMPLES: ${AIRFLOW__CORE__LOAD_EXAMPLES}
      AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: ${AIRFLOW__DATABASE__SQL_ALCHEMY_CONN}
      AIRFLOW__CORE__FERNET_KEY: ${AIRFLOW_FERNET_KEY}
    volumes:
      - ./airflow/dags:/opt/airflow/dags
      - ./airflow/airflow.cfg:/opt/airflow/airflow.cfg:ro
    depends_on:
      - airflow-db
    networks:
      - surimi-network

  airflow-scheduler:
    image: surimi-airflow:2.11.0
    container_name: airflow-scheduler
    command: scheduler
    environment:
      AIRFLOW__CORE__DAGS_FOLDER: ${AIRFLOW__CORE__DAGS_FOLDER}
      AIRFLOW__CORE__LOAD_EXAMPLES: ${AIRFLOW__CORE__LOAD_EXAMPLES}
      AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: ${AIRFLOW__DATABASE__SQL_ALCHEMY_CONN}
      AIRFLOW__CORE__FERNET_KEY: ${AIRFLOW_FERNET_KEY}
      AIRFLOW__CORE__EXECUTOR: LocalExecutor
      AIRFLOW__CORE__PARALLELISM: 32
      AIRFLOW__CORE__DAG_CONCURRENCY: 16
      # Task execution settings for Airflow 3.x
      AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK: "true"
      SUPERVISOR_RPC_SOCKET: "unix:///tmp/supervisor.sock"
      # Environment variables for ingestion
      MINIO_ENDPOINT: ${MINIO_ENDPOINT:-minio:9000}
      MINIO_ACCESS_KEY: ${MINIO_ROOT_USER}
      MINIO_SECRET_KEY: ${MINIO_ROOT_PASSWORD}
      POSTGRES_HOST: ${POSTGRES_HOST}
      POSTGRES_PORT: ${POSTGRES_PORT}
      POSTGRES_DB: ${POSTGRES_DB}
      POSTGRES_USER: ${POSTGRES_USER}
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
      TRINO_HOST: trino
      TRINO_PORT: 8080
      TRINO_USER: admin
      # DataHub settings
      DATAHUB_GMS_URL: ${DATAHUB_GMS_URL:-http://surimi-datagui-datahub-gms-1:8080}
      DATAHUB_ENABLED: ${DATAHUB_ENABLED:-true}
    volumes:
      - ./airflow/dags:/opt/airflow/dags
      - ./airflow/logs:/opt/airflow/logs
      - ./airflow/plugins:/opt/airflow/plugins
      - ./airflow/airflow.cfg:/opt/airflow/airflow.cfg:ro
    depends_on:
      airflow-init:
        condition: service_completed_successfully
    networks:
      - surimi-network
    healthcheck:
      test: ["CMD-SHELL", "airflow jobs check --job-type SchedulerJob --hostname \"$${HOSTNAME}\""]
      interval: 30s
      timeout: 10s
      retries: 5

  airflow-webserver:
    image: surimi-airflow:2.11.0
    container_name: airflow-webserver
    command: webserver
    environment:
      AIRFLOW__CORE__DAGS_FOLDER: ${AIRFLOW__CORE__DAGS_FOLDER}
      AIRFLOW__CORE__LOAD_EXAMPLES: ${AIRFLOW__CORE__LOAD_EXAMPLES}
      AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: ${AIRFLOW__DATABASE__SQL_ALCHEMY_CONN}
      AIRFLOW__CORE__FERNET_KEY: ${AIRFLOW_FERNET_KEY}
      AIRFLOW__CORE__EXECUTOR: LocalExecutor
      AIRFLOW__CORE__PARALLELISM: 32
      AIRFLOW__CORE__DAG_CONCURRENCY: 16
      # Environment variables for ingestion
      MINIO_ENDPOINT: ${MINIO_ENDPOINT:-minio:9000}
      MINIO_ACCESS_KEY: ${MINIO_ROOT_USER}
      MINIO_SECRET_KEY: ${MINIO_ROOT_PASSWORD}
      POSTGRES_HOST: ${POSTGRES_HOST}
      POSTGRES_PORT: ${POSTGRES_PORT}
      POSTGRES_DB: ${POSTGRES_DB}
      POSTGRES_USER: ${POSTGRES_USER}
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
      TRINO_HOST: trino
      TRINO_PORT: 8080
      TRINO_USER: admin
      # DataHub settings
      DATAHUB_GMS_URL: ${DATAHUB_GMS_URL:-http://surimi-datagui-datahub-gms-1:8080}
      DATAHUB_ENABLED: ${DATAHUB_ENABLED:-true}
    ports:
      - "${AIRFLOW_WEBSERVER_PORT}:8080"
    volumes:
      - ./airflow/dags:/opt/airflow/dags
      - ./airflow/logs:/opt/airflow/logs
      - ./airflow/plugins:/opt/airflow/plugins
      - ./airflow/airflow.cfg:/opt/airflow/airflow.cfg:ro
    depends_on:
      airflow-init:
        condition: service_completed_successfully
    networks:
      - surimi-network
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8080/"]
      interval: 30s
      timeout: 10s
      retries: 5

  airflow-triggerer:
    image: surimi-airflow:2.11.0
    container_name: airflow-triggerer
    command: triggerer
    environment:
      AIRFLOW__CORE__DAGS_FOLDER: ${AIRFLOW__CORE__DAGS_FOLDER}
      AIRFLOW__CORE__LOAD_EXAMPLES: ${AIRFLOW__CORE__LOAD_EXAMPLES}
      AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: ${AIRFLOW__DATABASE__SQL_ALCHEMY_CONN}
      AIRFLOW__CORE__FERNET_KEY: ${AIRFLOW_FERNET_KEY}
      AIRFLOW__CORE__EXECUTOR: LocalExecutor
    volumes:
      - ./airflow/dags:/opt/airflow/dags
      - ./airflow/logs:/opt/airflow/logs
      - ./airflow/plugins:/opt/airflow/plugins
      - ./airflow/airflow.cfg:/opt/airflow/airflow.cfg:ro
    depends_on:
      airflow-init:
        condition: service_completed_successfully
    networks:
      - surimi-network
    healthcheck:
      test: ["CMD-SHELL", "airflow jobs check --job-type TriggererJob --hostname \"$${HOSTNAME}\""]
      interval: 30s
      timeout: 10s
      retries: 5

  # ============================================
  # Keycloak (Identity Provider) — profile: debug
  # Start with: docker compose --profile debug up -d
  # Admin UI: http://localhost:8180  (admin / keycloak)
  # ============================================

  keycloak:
    image: quay.io/keycloak/keycloak:25.0
    container_name: keycloak
    profiles:
      - debug
    # --import-realm imports /opt/keycloak/data/import/*.json on every fresh start
    command: start-dev --import-realm --features=scripts
    environment:
      KEYCLOAK_ADMIN: ${KEYCLOAK_ADMIN_USER:-root}
      KEYCLOAK_ADMIN_PASSWORD: ${KEYCLOAK_ADMIN_PASSWORD:-root}
      KC_HTTP_ENABLED: "true"
      # KC_HOSTNAME_STRICT: "true"
      KC_HOSTNAME: ${KEYCLOAK_PUBLIC_URL:-http://localhost:${KEYCLOAK_PORT:-8180}}
      # Keep the admin console accessible at localhost so the UI works at http://localhost:8180
      KC_HOSTNAME_ADMIN: ${KEYCLOAK_PUBLIC_URL:-http://localhost:${KEYCLOAK_PORT:-8180}}
      KC_LOG_LEVEL: ${KEYCLOAK_LOG_LEVEL:-INFO}
    ports:
      - "${KEYCLOAK_PORT:-8180}:8080"
    volumes:
      - keycloak_data:/opt/keycloak/data
      - ./keycloak/surimi-realm.json:/opt/keycloak/data/import/surimi-realm.json:ro
    networks:
      - surimi-network
    healthcheck:
      test: ["CMD-SHELL", "curl -sf http://localhost:8080/realms/master || exit 1"]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 90s

  # ============================================
  # JupyterHub (Multi-User, OAuth, per-user containers)
  # ============================================

  jupyterhub-notebook:
    build:
      context: ./jupyterhub
      dockerfile: notebook.Dockerfile
    image: surimi-jupyter-notebook:latest
    command: ["true"]
    networks:
      - surimi-network

  jupyterhub:
    build: ./jupyterhub
    container_name: jupyterhub
    environment:
      AUTH_PROVIDER: ${JUPYTERHUB_AUTH_PROVIDER:-keycloak}
      OAUTH_CLIENT_ID: ${JUPYTERHUB_OAUTH_CLIENT_ID:-jupyterhub}
      OAUTH_CLIENT_SECRET: ${JUPYTERHUB_OAUTH_CLIENT_SECRET:-jupyterhub-secret}
      OAUTH_AUTHORIZE_URL: ${JUPYTERHUB_OAUTH_AUTHORIZE_URL:-http://localhost:8180/realms/surimi/protocol/openid-connect/auth}
      OAUTH_CALLBACK_URL: ${JUPYTERHUB_OAUTH_CALLBACK_URL:-http://localhost:8000/hub/oauth_callback}
      OAUTH_TOKEN_URL: ${JUPYTERHUB_OAUTH_TOKEN_URL:-http://keycloak:8080/realms/surimi/protocol/openid-connect/token}
      OAUTH_USERDATA_URL: ${JUPYTERHUB_OAUTH_USERDATA_URL:-http://keycloak:8080/realms/surimi/protocol/openid-connect/userinfo}
      OAUTH_USERNAME_KEY: ${JUPYTERHUB_OAUTH_USERNAME_KEY:-preferred_username}
      OAUTH_SCOPE: ${JUPYTERHUB_OAUTH_SCOPE:-openid profile email}
      OAUTH_ALLOWED_GROUPS: ${JUPYTERHUB_OAUTH_ALLOWED_GROUPS:-jupyterhub_users,jupyterhub_admins}
      OAUTH_ADMIN_GROUPS: ${JUPYTERHUB_OAUTH_ADMIN_GROUPS:-jupyterhub_admins}
      JUPYTERHUB_ADMIN_USERS: ${JUPYTERHUB_ADMIN_USERS}
      JUPYTERHUB_NOTEBOOK_IMAGE: ${JUPYTERHUB_NOTEBOOK_IMAGE}
      JUPYTERHUB_SHARED_NOTEBOOKS_PATH: ${JUPYTERHUB_SHARED_NOTEBOOKS_PATH}
      JUPYTERHUB_SHARED_RW_NOTEBOOKS_PATH: ${JUPYTERHUB_SHARED_RW_NOTEBOOKS_PATH}
      JUPYTERHUB_USER_VOLUMES_PATH: ${JUPYTERHUB_USER_VOLUMES_PATH}
      MINIO_ROOT_USER: ${MINIO_ROOT_USER}
      MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
      MINIO_USER_SECRET_SALT: ${MINIO_USER_SECRET_SALT}
      MINIO_ENDPOINT: ${MINIO_HOST}
      MINIO_BUCKET: ${MINIO_BUCKET}
      MINIO_SECURE: ${MINIO_SECURE}
    ports:
      - "${JUPYTERHUB_PORT}:8000"
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - ./jupyterhub/jupyterhub_config.py:/srv/jupyterhub/jupyterhub_config.py:ro
      - jupyterhub_data:/srv/jupyterhub/data
    depends_on:
      - trino
      - minio
    networks:
      - surimi-network
    command: jupyterhub -f /srv/jupyterhub/jupyterhub_config.py

volumes:
  minio_data:
  postgres_data:
  hive_data:
  superset_data:
  airflow_db_data:
  ingestion_logs:
  jupyterhub_data:
  trino_data:
  keycloak_data:

networks:
  surimi-network:
    driver: bridge
    name: surimi-network