docker-compose.yml

# PanDA Compose — self-contained PanDA stack for development and CI
#
# Component dependency order:
#   postgres + activemq + mariadb  →  panda-server + panda-jedi  →  harvester
#
# Usage:
#   cp .env.example .env    # fill in secrets
#   docker compose up -d
#   ./scripts/healthcheck.sh
#
# Container images mirror those used in panda-k8s Helm charts.

services:

  # ────────────────────────────────────────────────────────────────
  # PostgreSQL — PanDA server + JEDI database
  # Image ships with the PanDA schema pre-installed.
  # ────────────────────────────────────────────────────────────────
  postgres:
    image: ghcr.io/pandawms/panda-database:latest
    restart: unless-stopped
    environment:
      # The init script (panda_db_init.sh) connects as 'postgres' superuser.
      # It creates the 'panda' role using PANDA_DB_PASSWORD.
      POSTGRES_USER: postgres
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres_secret}
      POSTGRES_DB: ${PANDA_DB_NAME:-panda_db}
      PANDA_DB_PASSWORD: ${PANDA_DB_PASSWORD:-panda_secret}
    volumes:
      - panda-db-data:/var/lib/postgresql/data
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U postgres -d ${PANDA_DB_NAME:-panda_db}"]
      interval: 20s
      timeout: 10s
      retries: 15
      start_period: 60s

  # ────────────────────────────────────────────────────────────────
  # ActiveMQ — message broker for PanDA server ↔ JEDI communication
  # The image CMD defaults to "sleep infinity"; override to start broker.
  # No JAAS auth in dev — default config accepts any STOMP credentials.
  # ────────────────────────────────────────────────────────────────
  activemq:
    image: ghcr.io/pandawms/panda-activemq:latest
    restart: unless-stopped
    command: ["/opt/activemq/bin/run-activemq-services"]
    environment:
      # Disable JDK 18 cgroup v2 detection bug (JDK-8281631) that crashes the JVM
      JAVA_TOOL_OPTIONS: "-XX:-UseContainerSupport"
    ports:
      - "61613:61613"   # STOMP (used by panda-server / harvester)
      - "61616:61616"   # OpenWire
      - "8161:8161"     # Web console (admin)
    volumes:
      - activemq-data:/appdata/activemq
    healthcheck:
      test: ["CMD-SHELL", "nc -z localhost 61613"]
      interval: 15s
      timeout: 5s
      retries: 10

  # ────────────────────────────────────────────────────────────────
  # PanDA server + JEDI (unified image)
  # HTTP  → localhost:25080
  # HTTPS → localhost:25443
  #
  # The image CMD defaults to "sleep infinity". We mirror the k8s
  # statefulset startup pattern: copy sandbox scripts to /data/panda/,
  # run process_template.py to expand ${VAR} in *.template files,
  # then start httpd via run-panda-services.
  # ────────────────────────────────────────────────────────────────
  panda-server:
    image: ghcr.io/pandawms/panda-server:latest
    restart: unless-stopped
    command:
      - /bin/bash
      - -c
      - |
        cp /opt/panda/sandbox/* /data/panda/
        chmod +x /data/panda/init-panda /data/panda/run-panda-crons /data/panda/run-panda-services
        mkdir -p /var/log/panda/pandacache/jedilog
        chmod -R 777 /var/log/panda/pandacache || true
        if [ -f /opt/panda/sandbox/panda_server-httpd.conf ]; then
          mv /opt/panda/etc/panda/panda_server-httpd.conf /opt/panda/etc/panda/panda_server-httpd.conf.back || true
          ln -sf /opt/panda/sandbox/panda_server-httpd.conf /opt/panda/etc/panda/panda_server-httpd.conf
        fi
        /opt/panda/bin/python /data/panda/process_template.py
        # Remove stale lock files left by a previous container run (the writable
        # layer is reused across restarts so these files persist).
        rm -f ${PANDA_LOCK_DIR:-/var/run/panda}/panda_daemon.lock \
              ${PANDA_LOCK_DIR:-/var/run/panda}/panda_daemon.pid \
              ${PANDA_LOCK_DIR:-/var/run/panda}/panda_daemon.pid.lock
        /data/panda/run-panda-services
        sleep infinity & wait
    depends_on:
      postgres:
        condition: service_healthy
      activemq:
        condition: service_healthy
    environment:
      # Database
      PANDA_DB_BACKEND: ${PANDA_DB_BACKEND:-postgres}
      PANDA_DB_HOST: ${PANDA_DB_HOST:-postgres}
      PANDA_DB_PORT: ${PANDA_DB_PORT:-5432}
      PANDA_DB_USER: ${PANDA_DB_USER:-panda}
      PANDA_DB_PASSWORD: ${PANDA_DB_PASSWORD:-panda_secret}
      PANDA_DB_NAME: ${PANDA_DB_NAME:-panda_db}
      PANDA_DB_SCHEMAPANDA: ${PANDA_DB_SCHEMAPANDA:-DOMA_PANDA}
      PANDA_DB_SCHEMAMETA: ${PANDA_DB_SCHEMAMETA:-DOMA_PANDAMETA}
      PANDA_DB_SCHEMAPANDAARCH: ${PANDA_DB_SCHEMAPANDAARCH:-DOMA_PANDAARCH}
      PANDA_DB_SCHEMAJEDI: ${PANDA_DB_SCHEMAJEDI:-DOMA_PANDA}
      PANDA_DB_SCHEMADEFT: ${PANDA_DB_SCHEMADEFT:-DOMA_DEFT}
      # Messaging
      PANDA_ACTIVEMQ_LIST: ${PANDA_ACTIVEMQ_LIST:-activemq:61613}
      PANDA_ACTIVEMQ_PASSWD_panda: ${PANDA_ACTIVEMQ_PASSWD_panda:-panda_mq_secret}
      PANDA_ACTIVEMQ_PASSWD_jedi: ${PANDA_ACTIVEMQ_PASSWD_jedi:-jedi_mq_secret}
      # Server — run as current user to avoid atlpan/zp user-switching in Docker
      PANDA_NO_ROOT: "1"
      PANDA_AUTH: ${PANDA_AUTH:-None}
      PANDA_VERIFY_HOST: "off"
      PANDA_BEHIND_REAL_LB: "1"
      PANDA_SERVER_CONF_PORT: ${PANDA_SERVER_CONF_PORT:-80}
      PANDA_SERVER_CONF_MIN_WORKERS: ${PANDA_SERVER_CONF_MIN_WORKERS:-1}
      PANDA_SERVER_CONF_MAX_WORKERS: ${PANDA_SERVER_CONF_MAX_WORKERS:-4}
      PANDA_SERVER_CONF_MAX_CONNECTIONS: ${PANDA_SERVER_CONF_MAX_CONNECTIONS:-200}
      PANDA_SERVER_CONF_THREADS_PER_CHILD: ${PANDA_SERVER_CONF_THREADS_PER_CHILD:-1}
      PANDA_SERVER_CONF_MAX_BACKLOG: ${PANDA_SERVER_CONF_MAX_BACKLOG:-511}
      PANDA_SERVER_CONF_SERVERNAME: ${PANDA_SERVER_CONF_SERVERNAME:-localhost}
      PANDA_SERVER_CONF_NUM_WSGI: ${PANDA_SERVER_CONF_NUM_WSGI:-2}
      PANDA_SERVER_CONF_NUM_WSGI_THREAD: ${PANDA_SERVER_CONF_NUM_WSGI_THREAD:-1}
      # Proxy (optional)
      http_proxy: ${HTTP_PROXY:-}
      https_proxy: ${HTTPS_PROXY:-}
    volumes:
      - ./config/panda/panda_common.cfg:/etc/panda/panda_common.cfg:ro
      - ./config/panda/panda_server.cfg:/etc/panda/panda_server.cfg:ro
      - ./config/panda/sandbox:/opt/panda/sandbox:ro
      - panda-server-logs:/var/log/panda
      - panda-cache:/var/cache/pandaserver
    ports:
      - "25080:80"    # HTTP — used by panda-client
    healthcheck:
      test: ["CMD-SHELL", "curl -sf http://localhost:80/server/panda/isAlive | grep -q 'alive=yes' || exit 1"]
      interval: 20s
      timeout: 10s
      retries: 15
      start_period: 60s

  # ────────────────────────────────────────────────────────────────
  # PanDA JEDI — workload management daemon
  # Same sandbox startup pattern as panda-server.
  # ────────────────────────────────────────────────────────────────
  panda-jedi:
    image: ghcr.io/pandawms/panda-jedi:latest
    restart: unless-stopped
    command:
      - /bin/bash
      - -c
      - |
        cp /opt/panda/sandbox/* /data/panda/
        chmod +x /data/panda/init-jedi /data/panda/run-jedi-crons /data/panda/run-jedi-services
        /opt/panda/bin/python /data/panda/process_template.py
        /data/panda/run-jedi-services
        sleep infinity & wait
    depends_on:
      panda-server:
        condition: service_healthy
      init:
        condition: service_completed_successfully
    environment:
      PANDA_DB_BACKEND: ${PANDA_DB_BACKEND:-postgres}
      PANDA_DB_HOST: ${PANDA_DB_HOST:-postgres}
      PANDA_DB_PORT: ${PANDA_DB_PORT:-5432}
      PANDA_DB_USER: ${PANDA_DB_USER:-panda}
      PANDA_DB_PASSWORD: ${PANDA_DB_PASSWORD:-panda_secret}
      PANDA_DB_NAME: ${PANDA_DB_NAME:-panda_db}
      PANDA_DB_SCHEMAPANDA: ${PANDA_DB_SCHEMAPANDA:-DOMA_PANDA}
      PANDA_DB_SCHEMAMETA: ${PANDA_DB_SCHEMAMETA:-DOMA_PANDAMETA}
      PANDA_DB_SCHEMAPANDAARCH: ${PANDA_DB_SCHEMAPANDAARCH:-DOMA_PANDAARCH}
      PANDA_DB_SCHEMAJEDI: ${PANDA_DB_SCHEMAJEDI:-DOMA_PANDA}
      PANDA_DB_SCHEMADEFT: ${PANDA_DB_SCHEMADEFT:-DOMA_DEFT}
      PANDA_ACTIVEMQ_LIST: ${PANDA_ACTIVEMQ_LIST:-activemq:61613}
      PANDA_ACTIVEMQ_PASSWD_panda: ${PANDA_ACTIVEMQ_PASSWD_panda:-panda_mq_secret}
      PANDA_ACTIVEMQ_PASSWD_jedi: ${PANDA_ACTIVEMQ_PASSWD_jedi:-jedi_mq_secret}
      PANDA_NO_ROOT: "1"
      PYTHONPATH: /opt/panda/lib/python3.11/site-packages/pandacommon
      PANDA_URL: ${PANDA_URL:-http://panda-server:80/server/panda}
      PANDA_URL_SSL: ${PANDA_URL_SSL:-https://panda-server:443/server/panda}
      PANDA_VERIFY_HOST: "off"
      http_proxy: ${HTTP_PROXY:-}
      https_proxy: ${HTTPS_PROXY:-}
    volumes:
      - ./config/panda/panda_common.cfg:/etc/panda/panda_common.cfg:ro
      - ./config/panda/panda_server.cfg:/etc/panda/panda_server.cfg:ro
      - ./config/panda/panda_jedi.cfg:/etc/panda/panda_jedi.cfg:ro
      - ./config/jedi/sandbox:/opt/panda/sandbox:ro
      - panda-jedi-logs:/var/log/panda
    healthcheck:
      test: ["CMD-SHELL", "test -f /var/run/panda/panda_jedi.pid && kill -0 $(cat /var/run/panda/panda_jedi.pid) 2>/dev/null || exit 1"]
      interval: 20s
      timeout: 5s
      retries: 10
      start_period: 60s

  # ────────────────────────────────────────────────────────────────
  # MariaDB — Harvester database
  # ────────────────────────────────────────────────────────────────
  mariadb:
    image: mariadb:10.11
    restart: unless-stopped
    environment:
      MARIADB_USER: ${HARVESTER_DB_USER:-harvester}
      MARIADB_PASSWORD: ${HARVESTER_DB_PASSWORD:-harvester_secret}
      MARIADB_DATABASE: ${HARVESTER_DB_NAME:-harvester_db}
      MARIADB_ROOT_PASSWORD: ${HARVESTER_DB_PASSWORD:-harvester_secret}
    volumes:
      - harvester-db-data:/var/lib/mysql
    healthcheck:
      test: ["CMD-SHELL", "mysqladmin ping -u${HARVESTER_DB_USER:-harvester} -p${HARVESTER_DB_PASSWORD:-harvester_secret} --silent"]
      interval: 10s
      timeout: 5s
      retries: 10

  # ────────────────────────────────────────────────────────────────
  # Init — one-shot service that registers PANDA_COMPOSE_LOCAL in
  # the PanDA server schedconfig so JEDI can route jobs to it.
  # Runs after panda-server is healthy; exits 0 on success.
  # ────────────────────────────────────────────────────────────────
  init:
    image: postgres:15
    restart: "no"
    depends_on:
      postgres:
        condition: service_healthy
      panda-server:
        condition: service_healthy
    volumes:
      - ./scripts:/scripts:ro
    command:
      - /bin/bash
      - /scripts/setup-queue.sh
    environment:
      PGHOST: postgres
      PGPORT: "5432"
      PGUSER: postgres
      PGPASSWORD: ${POSTGRES_PASSWORD:-postgres_secret}
      PGDATABASE: ${PANDA_DB_NAME:-panda_db}
      PANDA_DB_USER: panda
      PANDA_DB_PASSWORD: ${PANDA_DB_PASSWORD:-panda_secret}

  # ────────────────────────────────────────────────────────────────
  # Harvester — resource-facing pilot submission service.
  # Runs subprocess_submitter / subprocess_monitor plugins (mounted
  # from ./scripts/) so jobs execute as local subprocesses with no
  # external batch system required.
  # ────────────────────────────────────────────────────────────────
  harvester:
    image: ghcr.io/hsf/harvester:latest
    restart: unless-stopped
    command:
      - /bin/bash
      - -c
      - |
        source /opt/harvester/bin/activate
        pip install -q 'docker==7.1.0'
        mkdir -p /var/log/harvester /var/run/panda /harvester/workers
        python -u $$SITE_PACKAGES_PATH/pandaharvester/harvesterbody/master.py --foreground &
        echo $$! > /var/run/panda/panda_harvester.pid
        wait
    depends_on:
      panda-server:
        condition: service_healthy
      mariadb:
        condition: service_healthy
    healthcheck:
      test: ["CMD-SHELL", "kill -0 $$(cat /var/run/panda/panda_harvester.pid 2>/dev/null) 2>/dev/null"]
      interval: 10s
      timeout: 5s
      retries: 30
      start_period: 30s
    environment:
      VIRTUAL_ENV: /opt/harvester
      SITE_PACKAGES_PATH: /opt/harvester/lib/python3.11/site-packages
      PYTHONPATH: /harvester/plugins:/opt/harvester/lib/python3.11/site-packages/pandacommon
      PANDA_URL: ${PANDA_URL:-http://panda-server:80/server/panda}
      PANDA_URL_SSL: ${PANDA_URL_SSL:-http://panda-server:80/server/panda}
      PANDA_AUTH: "None"
      PANDA_VERIFY_HOST: "off"
      HARVESTER_DB_USER: ${HARVESTER_DB_USER:-harvester}
      HARVESTER_DB_PASSWORD: ${HARVESTER_DB_PASSWORD:-harvester_secret}
      HARVESTER_DB_NAME: ${HARVESTER_DB_NAME:-harvester_db}
      HARVESTER_DB_HOST: mariadb
      HARVESTER_EXPERIMENT: ${HARVESTER_EXPERIMENT:-wlcg}
      http_proxy: ${HTTP_PROXY:-}
      https_proxy: ${HTTPS_PROXY:-}
    volumes:
      - ./config/harvester/panda_harvester.cfg:/etc/panda/panda_harvester.cfg:ro
      - ./config/harvester/panda_common.cfg:/etc/panda/panda_common.cfg:ro
      - ./config/harvester/panda_queues.cfg:/etc/harvester/panda_queues.cfg:ro
      - ./scripts/subprocess_submitter.py:/harvester/plugins/subprocess_submitter.py:ro
      - ./scripts/subprocess_monitor.py:/harvester/plugins/subprocess_monitor.py:ro
      - ./config/harvester/plugins/docker_submitter.py:/harvester/plugins/docker_submitter.py:ro
      - ./config/harvester/plugins/docker_monitor.py:/harvester/plugins/docker_monitor.py:ro
      - ./scripts/panda-worker.sh:/harvester/panda-worker.sh:ro
      - /var/run/docker.sock:/var/run/docker.sock
      - harvester-logs:/var/log/harvester
      - harvester-workdir:/harvester

volumes:
  panda-db-data:
  panda-server-logs:
  panda-jedi-logs:
  panda-cache:
  activemq-data:
  harvester-db-data:
  harvester-logs:
  harvester-workdir: