docker-compose.yml

services:
  tpu-watchdogs:
    build:
      context: .
      dockerfile: docker/TPUWatchdog.dockerfile
    container_name: "PHANTOM-tpu-watchdogs"
    restart: unless-stopped
    user: "${UID:-1000}:${GID:-1000}"
    environment:
      - HF_TOKEN=${HF_TOKEN}
      - WANDB_API_KEY=${WANDB_API_KEY}
      - GITHUB_TOKEN=${GITHUB_TOKEN}
      - GOOGLE_APPLICATION_CREDENTIALS=/secrets/gcp-sa.json
      - GCP_ACCOUNT=${GCP_ACCOUNT:-}
      - WATCHDOG_CONFIG_PATTERN=${WATCHDOG_CONFIG_PATTERN:-v[46]*.conf}
      - CLOUDSDK_CONFIG=/.config/gcloud
    volumes:
      - ~/.config/gcloud:/.config/gcloud:rw
      - ./secrets/gcp-sa.json:/secrets/gcp-sa.json:ro

  tensorboard-rl:
    image: tensorflow/tensorflow:latest
    container_name: "PHANTOM-tensorboard-rl"
    ports:
      - "6007:6006"
    volumes:
      - ./sim/rl/runs:/logs
    command: tensorboard --logdir=/logs --host=0.0.0.0 --port=6006
    restart: unless-stopped

  tensorboard-ml:
    image: tensorflow/tensorflow:latest
    container_name: "PHANTOM-tensorboard-ml"
    ports:
      - "6006:6006"
    volumes:
      - ./experiments/ml/runs:/logs
    command: tensorboard --logdir=/logs --host=0.0.0.0 --port=6006
    restart: unless-stopped

  backend:
    container_name: "PHANTOM-backend"
    build:
      context: .
      dockerfile: docker/backend.Dockerfile
    ports:
      - "${BACKEND_PORT:-5000}:5000"
    environment:
      - KAFKA_HOST=kafka
      - KAFKA_PORT=29092
      - BACKEND_PORT=5000
      - NEXT_PUBLIC_SUPABASE_URL=${NEXT_PUBLIC_SUPABASE_URL}
      - NEXT_PUBLIC_SUPABASE_ANON_KEY=${NEXT_PUBLIC_SUPABASE_ANON_KEY}
    depends_on:
      - kafka
    restart: unless-stopped

  redis:
    container_name: "PHANTOM-redis"
    build:
      context: ./docker
      dockerfile: Redis.dockerfile
    ports:
      - "${REDIS_PORT:-6378}:6379"
    volumes:
      - phantom_redis_data:/data
    restart: unless-stopped

  zookeeper:
    container_name: "PHANTOM-zookeeper"
    build:
      context: ./docker
      dockerfile: Zookeeper.dockerfile
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
    ports:
      - "2181:2181"

  kafka:
    container_name: "PHANTOM-kafka"
    build:
      context: ./docker
      dockerfile: Kafka.dockerfile
    depends_on:
      - zookeeper
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:29092,PLAINTEXT_HOST://0.0.0.0:9092
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
    ports:
      - "${KAFKA_PORT:-9092}:9092"
    volumes:
      - phantom_kafka_data:/var/lib/kafka/data

  redpanda-console:
    container_name: "PHANTOM-redpanda-console"
    build:
      context: ./docker
      dockerfile: RedpandaConsole.dockerfile
    depends_on:
      - kafka
    environment:
      KAFKA_BROKERS: kafka:29092
    ports:
      - "${REDPANDA_CONSOLE_PORT:-8080}:8080"
    restart: unless-stopped

  postgres:
    container_name: "PHANTOM-postgres"
    image: postgres:13
    environment:
      - POSTGRES_USER=airflow
      - POSTGRES_PASSWORD=airflow
      - POSTGRES_DB=airflow
    ports:
      - "5433:5432"
    volumes:
      - postgres_data:/var/lib/postgresql/data
    restart: unless-stopped

  airflow-init:
    container_name: "PHANTOM-airflow-init"
    build:
      context: .
      dockerfile: docker/Airflow.dockerfile
    depends_on:
      - postgres
    environment:
      - AIRFLOW__CORE__EXECUTOR=LocalExecutor
      - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@postgres/airflow
      - AIRFLOW__CORE__FERNET_KEY=${AIRFLOW_FERNET_KEY}
      - AIRFLOW__CORE__LOAD_EXAMPLES=false
      - AIRFLOW__CORE__ENABLE_XCOM_PICKLING=true
      - AIRFLOW__CORE__PARALLELISM=16
      - AIRFLOW__CORE__DAG_CONCURRENCY=8
      - AIRFLOW__CORE__MAX_ACTIVE_RUNS_PER_DAG=4
      - _AIRFLOW_DB_MIGRATE=true
      - _AIRFLOW_WWW_USER_CREATE=true
      - _AIRFLOW_WWW_USER_USERNAME=admin
      - _AIRFLOW_WWW_USER_PASSWORD=admin
      - REDIS_HOST=redis
      - REDIS_PORT=6379
    command: version
    restart: "no"

  airflow-webserver:
    container_name: "PHANTOM-airflow-webserver"
    build:
      context: .
      dockerfile: docker/Airflow.dockerfile
    depends_on:
      - postgres
      - airflow-init
      - redis
    environment:
      - AIRFLOW__CORE__EXECUTOR=LocalExecutor
      - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@postgres/airflow
      - AIRFLOW__CORE__FERNET_KEY=${AIRFLOW_FERNET_KEY}
      - AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION=true
      - AIRFLOW__CORE__LOAD_EXAMPLES=false
      - AIRFLOW__CORE__ENABLE_XCOM_PICKLING=true
      - AIRFLOW__CORE__PARALLELISM=16
      - AIRFLOW__CORE__DAG_CONCURRENCY=8
      - AIRFLOW__CORE__MAX_ACTIVE_RUNS_PER_DAG=4
      - AIRFLOW__SCHEDULER__MIN_FILE_PROCESS_INTERVAL=30
      - AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL=60
      - AIRFLOW__WEBSERVER__EXPOSE_CONFIG=true
      - AIRFLOW__WEBSERVER__SECRET_KEY=${AIRFLOW_SECRET_KEY}
      - AIRFLOW__API__AUTH_BACKENDS=airflow.api.auth.backend.basic_auth
      - KAFKA_HOST=kafka
      - KAFKA_PORT=29092
      - BACKEND_URL=http://backend:5000
      - NEXT_PUBLIC_SUPABASE_URL=${NEXT_PUBLIC_SUPABASE_URL}
      - NEXT_PUBLIC_SUPABASE_ANON_KEY=${NEXT_PUBLIC_SUPABASE_ANON_KEY}
      - REDIS_HOST=redis
      - REDIS_PORT=6379
    ports:
      - "${AIRFLOW_WEBSERVER_PORT:-8085}:8080"
    command: webserver
    restart: unless-stopped
    healthcheck:
      test: ["CMD", "curl", "--fail", "http://localhost:8080/health"]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 30s

  airflow-scheduler:
    container_name: "PHANTOM-airflow-scheduler"
    build:
      context: .
      dockerfile: docker/Airflow.dockerfile
    depends_on:
      airflow-webserver:
        condition: service_healthy
      redis:
        condition: service_started
    environment:
      - AIRFLOW__CORE__EXECUTOR=LocalExecutor
      - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@postgres/airflow
      - AIRFLOW__CORE__FERNET_KEY=${AIRFLOW_FERNET_KEY}
      - AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION=true
      - AIRFLOW__CORE__LOAD_EXAMPLES=false
      - AIRFLOW__CORE__ENABLE_XCOM_PICKLING=true
      - AIRFLOW__CORE__PARALLELISM=16
      - AIRFLOW__CORE__DAG_CONCURRENCY=8
      - AIRFLOW__CORE__MAX_ACTIVE_RUNS_PER_DAG=4
      - AIRFLOW__SCHEDULER__MIN_FILE_PROCESS_INTERVAL=30
      - AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL=60
      - AIRFLOW__SCHEDULER__PARSING_PROCESSES=2
      - AIRFLOW__WEBSERVER__SECRET_KEY=${AIRFLOW_SECRET_KEY}
      - AIRFLOW__API__AUTH_BACKENDS=airflow.api.auth.backend.basic_auth
      - KAFKA_HOST=kafka
      - KAFKA_PORT=29092
      - BACKEND_URL=http://backend:5000
      - NEXT_PUBLIC_SUPABASE_URL=${NEXT_PUBLIC_SUPABASE_URL}
      - NEXT_PUBLIC_SUPABASE_ANON_KEY=${NEXT_PUBLIC_SUPABASE_ANON_KEY}
      - REDIS_HOST=redis
      - REDIS_PORT=6379
    command: scheduler
    restart: unless-stopped
    healthcheck:
      test: ["CMD-SHELL", 'airflow jobs check --job-type SchedulerJob --hostname "$${HOSTNAME}"']
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 30s

  pricing-provider:
    container_name: "PHANTOM-pricing-provider"
    build:
      context: .
      dockerfile: docker/Provider.dockerfile
    depends_on:
      - redis
      - kafka
    environment:
      - PROVIDER_PORT=5001
      - REDIS_HOST=redis
      - REDIS_PORT=6379
      - KAFKA_HOST=kafka
      - KAFKA_PORT=29092
      - NEXT_PUBLIC_SUPABASE_URL=${NEXT_PUBLIC_SUPABASE_URL}
      - NEXT_PUBLIC_SUPABASE_ANON_KEY=${NEXT_PUBLIC_SUPABASE_ANON_KEY}
      - BACKEND_URL=http://localhost:5000
    ports:
      - "${PROVIDER_PORT:-5001}:5001"
    restart: unless-stopped

volumes:
  phantom_kafka_data:
  phantom_redis_data:
  postgres_data: