compose.yml

# CHAOS-3142 incident, 2026-07-25: this file previously shared the compose
# project name `dev-health` with the repo-root `compose.yml` (a separate,
# divergent file with an incompatible postgres definition -- different
# image, role/database names, PGDATA path, and port). Because Docker Compose
# tracks service ownership by (project name, service name), an `up` from
# EITHER file could recreate the OTHER file's already-running containers in
# place, discarding whatever they had (see the postgres image/PGDATA history
# below). Naming this project `dev-health-ops` makes that collision
# structurally impossible: this file now owns its own containers, network,
# and volumes, entirely separate from whatever the repo-root compose.yml
# manages. That also means this file's postgres always starts from its own
# empty volume -- it is no longer a way to attach to another project's data.
name: dev-health-ops

volumes:
  postgres_data:
    driver: local
  clickhouse_data:
    driver: local

services:
  postgres:
    # CHAOS-3142 incident, 2026-07-25: the immediate trigger was this file
    # sharing its compose project name with the repo-root compose.yml (see
    # the `name:` comment above) -- an `up` from either file could recreate
    # the other's already-running postgres container in place. But this
    # image was also unpinned (`latest`) and PGDATA was misspelled
    # (`PG_DATA`, which the postgres image never reads), so the container
    # that got recreated started an entirely new, empty cluster in whatever
    # subdirectory that image version's own default PGDATA happened to be,
    # rather than failing loudly or reusing the real one. Both defects are
    # fixed independently of the project-name fix, so this file's own
    # isolated volume can't suffer the same silent-relocation failure mode
    # on some future `up`. Pinned to the same Postgres 18 Alpine digest
    # already pinned in internal/testsupport/containers/harness.go, so the
    # dev stack and the Go integration-test harness agree on one Postgres
    # build. The repo-root compose.yml already got PGDATA right under
    # CHAOS-1771 (`PGDATA: /var/lib/postgresql/data/pgdata`, with its own
    # comment on the same anonymous-volume trap) -- this file adopts the
    # identical reasoning so the two agree rather than looking like
    # independent guesses.
    image: postgres:18-alpine@sha256:9a8afca54e7861fd90fab5fdf4c42477a6b1cb7d293595148e674e0a3181de15
    container_name: postgres
    restart: always
    environment:
      POSTGRES_USER: postgres
      POSTGRES_PASSWORD: postgres
      POSTGRES_DB: postgres
      # Must be a subdirectory of the volume mount (postgres_data is mounted
      # at the parent /var/lib/postgresql/), never the mount root itself --
      # otherwise the image's lost+found and other reserved entries live
      # alongside PGDATA and a future image-default change silently
      # relocates the cluster again. Previously misspelled `PG_DATA`, which
      # the postgres image never reads, so this had no effect for seven
      # months (commit 581ae01e6) while every cluster was created at the
      # image's own shifting default path.
      PGDATA: /var/lib/postgresql/pgdata
      RIVER_DOMAIN_DATABASE_ROLE: ${RIVER_DOMAIN_DATABASE_ROLE:-devhealth_domain}
      RIVER_QUEUE_DATABASE_ROLE: ${RIVER_QUEUE_DATABASE_ROLE:-devhealth_queue}
      # CHAOS-3142: the coordinator role of the CHAOS-3033 Option B split.
      # docker/init-extra-dbs.sh already reads these two (defaulting to
      # devhealth_coordinator) to create the login and grant CONNECT; without
      # them here an operator override never reaches the script that
      # provisions it.
      RIVER_COORDINATOR_DATABASE_ROLE: ${RIVER_COORDINATOR_DATABASE_ROLE:-devhealth_coordinator}
      # Local development credentials only. Production roles are provisioned
      # through the database provider/secret manager before migration.
      RIVER_DOMAIN_DATABASE_PASSWORD: ${RIVER_DOMAIN_DATABASE_PASSWORD:-devhealth_domain}
      RIVER_QUEUE_DATABASE_PASSWORD: ${RIVER_QUEUE_DATABASE_PASSWORD:-devhealth_queue}
      RIVER_COORDINATOR_DATABASE_PASSWORD: ${RIVER_COORDINATOR_DATABASE_PASSWORD:-devhealth_coordinator}
    ports:
      - "5555:5432"
    volumes:
      - postgres_data:/var/lib/postgresql/
      - ./docker/init-extra-dbs.sh:/docker-entrypoint-initdb.d/init-extra-dbs.sh:ro
    healthcheck:
      test: [ "CMD-SHELL", "pg_isready -U postgres" ]
      interval: 10s
      timeout: 5s
      retries: 3
      start_period: 30s

  pgbouncer:
    # CHAOS-2065: transaction-mode connection pooler in front of postgres.
    # Multiplexes many client connections onto a small server-connection set so
    # the worker fleet cannot exhaust postgres max_connections. The app runs with
    # PGBOUNCER_TRANSACTION_MODE=true (disables asyncpg prepared statements).
    image: edoburu/pgbouncer:latest
    container_name: pgbouncer
    restart: unless-stopped
    environment:
      DB_HOST: postgres
      DB_PORT: "5432"
      DB_USER: postgres
      DB_PASSWORD: postgres
      DB_NAME: postgres
      LISTEN_PORT: "6432"
      POOL_MODE: transaction
      AUTH_TYPE: scram-sha-256
      MAX_CLIENT_CONN: "1000"
      DEFAULT_POOL_SIZE: "25"
      MIN_POOL_SIZE: "5"
      RESERVE_POOL_SIZE: "5"
      ADMIN_USERS: postgres
    ports:
      - "6432:6432"
    depends_on:
      postgres:
        condition: service_healthy

  clickhouse:
    image: clickhouse/clickhouse-server:latest@sha256:d7556a3841027651307b5aa08d72b5c467d0241d3db5b67d9e158ef3975626f5
    container_name: clickhouse
    restart: on-failure
    environment:
      CLICKHOUSE_LISTEN_HOST: 0.0.0.0
      CLICKHOUSE_USER: ch
      CLICKHOUSE_PASSWORD: ch
      CLICKHOUSE_DB: default
    ports:
      - "8123:8123"
      - "9000:9000"
    ulimits:
      nofile:
        soft: 262144
        hard: 262144
    volumes:
      - clickhouse_data:/var/lib/clickhouse
    healthcheck:
      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:8123/ping" ]
      interval: 10s
      timeout: 5s
      retries: 3
      start_period: 30s

  valkey:
    image: valkey/valkey:9-alpine@sha256:ee91f7a174ac4d6a6b0685b3a60e321f0a9dbbb691f9b0e285be2ba1d1be8328
    container_name: valkey
    restart: unless-stopped
    ports:
      - "6379:6379"
    healthcheck:
      test: [ "CMD", "valkey-cli", "ping" ]
      interval: 10s
      timeout: 5s
      retries: 3
      start_period: 5s

  migrate:
    build:
      context: .
      dockerfile: ./docker/Dockerfile
      target: api
      args:
        SETUPTOOLS_SCM_PRETEND_VERSION: 0.0.0
    container_name: dev-health-migrate
    restart: "no"
    entrypoint:
      - sh
      - -c
      - >-
        if [ -z "$${MIGRATION_DATABASE_URI:-}" ];
        then unset MIGRATION_DATABASE_URI; fi;
        if [ -n "$$POSTGRES_URI" ]
        || [ "$${MIGRATION_DATABASE_URI+x}" = x ]
        || [ "$${MIGRATION_DATABASE_URI_FILE+x}" = x ];
        then dev-hops migrate postgres; fi
        && dev-hops migrate clickhouse
    environment:
      POSTGRES_URI: postgresql+asyncpg://postgres:postgres@postgres:5432/${POSTGRES_DB:-postgres}
      MIGRATION_DATABASE_URI: ${MIGRATION_DATABASE_URI:-}
      DEV_HEALTH_ALLOW_CELERY_RIVER_CUTOVER: ${DEV_HEALTH_ALLOW_CELERY_RIVER_CUTOVER:-}
      RIVER_DATABASE_SCHEMA: ${RIVER_DATABASE_SCHEMA:-river}
      RIVER_DOMAIN_DATABASE_ROLE: ${RIVER_DOMAIN_DATABASE_ROLE:-devhealth_domain}
      RIVER_QUEUE_DATABASE_ROLE: ${RIVER_QUEUE_DATABASE_ROLE:-devhealth_queue}
      CLICKHOUSE_URI: clickhouse://ch:ch@clickhouse:8123/${CLICKHOUSE_DB:-default}
      LOG_LEVEL: ${LOG_LEVEL:-INFO}
    depends_on:
      postgres:
        condition: service_healthy
      clickhouse:
        condition: service_healthy

  api:
    build:
      context: .
      dockerfile: ./docker/Dockerfile
      target: api
      args:
        SETUPTOOLS_SCM_PRETEND_VERSION: 0.0.0
    container_name: dev-health-api
    command: |
      --host 0.0.0.0 --port 8000 --reload
    environment: &env
      SETUPTOOLS_SCM_PRETEND_VERSION_FOR_DEV_HEALTH_OPS: "0.0.0"
      PYTHONPATH: /app/src
      # GraphQL analytics endpoint available at /graphql
      GRAPHQL_QUERY_TIMEOUT: "30"
      # CHAOS-2065: runtime DB traffic flows through PgBouncer (transaction mode).
      # Run schema migrations against postgres:5432 directly -- see
      # docs/ops/database-connection-pooling.md.
      DATABASE_URI: "postgresql+asyncpg://postgres:postgres@pgbouncer:6432/postgres"
      PGBOUNCER_TRANSACTION_MODE: "true"
      CLICKHOUSE_URI: "clickhouse://ch:ch@clickhouse:8123/default"
      REDIS_URL: "redis://valkey:6379/1"
      # Pre-existing gap fixed alongside CHAOS-3142: declared in
      # .env.example but was never actually referenced by this file, so no
      # override ever reached any container -- provider credential
      # decryption (src/dev_health_ops/core/encryption.py, read by
      # workers/task_utils.py for every provider) silently had no key in
      # local dev. The Go worker's `sync` profile decrypts the same stored
      # credentials with the same variable name
      # (SETTINGS_ENCRYPTION_KEY, deploy/go-workers/profiles.json) wherever
      # it runs, so this value stays byte-identical to whatever a Go
      # topology alongside this stack is given.
      SETTINGS_ENCRYPTION_KEY: ${SETTINGS_ENCRYPTION_KEY:-}
      # CHAOS-2299: route sync dispatch to per-provider queues. Safe to enable
      # here because the worker -Q lists below consume sync.<provider> in the
      # SAME change. Bare-image deployments must follow the two-phase rollout
      # in workers/queues.py: expand consumer -Q lists first, then flip this.
      PROVIDER_SYNC_QUEUES_ENABLED: "true"
      # CHAOS-3142/CHAOS-3123: the two route-ready provider/dataset switches.
      # Read by BOTH this Python producer gate
      # (ProviderUnitRouteSwitches.from_environment,
      # src/dev_health_ops/workers/provider_unit_route.py) and the Go worker
      # (config.Config.WorkerLaunchDarklyFeatureFlagsEnabled /
      # WorkerGithubRepoMetadataEnabled, cmd/dev-health-worker/dependencies.go
      # workerRouteSwitches) from the SAME env var names, so one override
      # flips both sides coherently wherever the Go worker runs -- a unit
      # this Python gate routes to River with the switch off here but on for
      # the Go worker (or vice versa) finds no handler. Both default to
      # "false"; only flip alongside a reviewed route release.
      WORKER_LAUNCHDARKLY_FEATURE_FLAGS_ENABLED: ${WORKER_LAUNCHDARKLY_FEATURE_FLAGS_ENABLED:-false}
      WORKER_GITHUB_REPO_METADATA_ENABLED: ${WORKER_GITHUB_REPO_METADATA_ENABLED:-false}
      AUTO_RUN_MIGRATIONS: "false"
      STRIPE_SECRET_KEY: ${STRIPE_SECRET_KEY:-}
      STRIPE_WEBHOOK_SECRET: ${STRIPE_WEBHOOK_SECRET:-}
      STRIPE_PRICE_ID_TEAM: ${STRIPE_PRICE_ID_TEAM:-}
      STRIPE_PRICE_ID_ENTERPRISE: ${STRIPE_PRICE_ID_ENTERPRISE:-}
      LICENSE_PRIVATE_KEY: ${LICENSE_PRIVATE_KEY:-}
      # Email (Resend)
      EMAIL_PROVIDER: ${EMAIL_PROVIDER:-console}
      EMAIL_API_KEY: ${EMAIL_API_KEY:-}
      EMAIL_FROM_ADDRESS: ${EMAIL_FROM_ADDRESS:-noreply@example.com}
      # Observability — OpenTelemetry (traces and metrics → SigNoz)
      OTEL_ENABLED: ${OTEL_ENABLED:-false}
      OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://signoz-otel-collector:4317}
      OTEL_SERVICE_NAME: ${OTEL_SERVICE_NAME:-dev-health-ops}
      OTEL_METRIC_EXPORT_INTERVAL: ${OTEL_METRIC_EXPORT_INTERVAL:-60000}
      # Observability — Error reporting (Sentry SDK → BugSink)
      SENTRY_DSN: ${SENTRY_DSN:-}
      SENTRY_ENVIRONMENT: ${SENTRY_ENVIRONMENT:-development}
      SENTRY_TRACES_RATE: ${SENTRY_TRACES_RATE:-0}
      SENTRY_SEND_PII: ${SENTRY_SEND_PII:-true}
    depends_on:
      postgres:
        condition: service_healthy
      clickhouse:
        condition: service_healthy
      valkey:
        condition: service_healthy
      migrate:
        condition: service_completed_successfully
      pgbouncer:
        condition: service_started
    ports:
      - "8000:8000"
    volumes:
      - ./:/app
    working_dir: /app
    healthcheck:
      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:8000/ready" ]
      interval: 15s
      timeout: 5s
      retries: 3
      start_period: 30s

  billing-edge:
    build:
      context: .
      dockerfile: ./docker/Dockerfile
      target: api
      args:
        SETUPTOOLS_SCM_PRETEND_VERSION: 0.0.0
    container_name: dev-health-billing-edge
    entrypoint:
      - python
      - -m
      - uvicorn
    command:
      - dev_health_ops.api.billing_edge:app
      - --host
      - 0.0.0.0
      - --port
      - "8000"
      - --reload
      - --reload-dir
      - /app/src/dev_health_ops/api
    environment:
      <<: *env
      POSTGRES_URI: "postgresql+asyncpg://postgres:postgres@pgbouncer:6432/postgres"
      AUTO_RUN_MIGRATIONS: "false"
    depends_on:
      postgres:
        condition: service_healthy
      migrate:
        condition: service_completed_successfully
      pgbouncer:
        condition: service_started
    ports:
      - "8010:8000"
    volumes:
      - ./:/app
    working_dir: /app
    healthcheck:
      test: [ "CMD", "wget", "--spider", "-q", "http://localhost:8000/ready" ]
      interval: 15s
      timeout: 5s
      retries: 3
      start_period: 30s

  # ---------------------------------------------------------------------------
  # Celery workers — split topology (CHAOS-2278)
  #
  # One shared pool let ~250s blocking stream consumers (ingest) and
  # multi-minute syncs starve latency-sensitive work (Sync Now, webhooks)
  # for hours. Three pools isolate the workload classes:
  #   worker        — latency-sensitive: default,sync,sync.<provider>,webhooks,reports
  #                     (per-provider sync queues, CHAOS-2299)
  #   worker-ingest — blocking stream consumers, isolated at concurrency 1
  #   worker-heavy  — long batch jobs: metrics,backfill
  # `monitoring` (queue-depth telemetry) is consumed by BOTH worker and
  # worker-heavy so telemetry survives one pool flooding (CHAOS-2299).
  # ---------------------------------------------------------------------------
  worker: &worker-base
    build:
      context: .
      dockerfile: ./docker/Dockerfile
      target: runner
      args:
        SETUPTOOLS_SCM_PRETEND_VERSION: 0.0.0
    container_name: worker
    entrypoint: ["celery"]
    command: -A dev_health_ops.workers.celery_app worker --loglevel=info --disable-prefetch -Q default,sync,sync.github,sync.gitlab,sync.linear,sync.jira,sync.launchdarkly,sync.github.light,sync.github.medium,sync.gitlab.light,sync.gitlab.medium,sync.jira.medium,sync.linear.medium,webhooks,reports,scheduler,monitoring --concurrency=${WORKER_CONCURRENCY:-4}
    environment:
      <<: *env
      CELERY_BROKER_URL: redis://valkey:6379/0
      CELERY_RESULT_BACKEND: redis://valkey:6379/0
      AUTO_RUN_MIGRATIONS: "false"
      # Email (Resend) — worker sends billing emails via Celery task
      EMAIL_PROVIDER: ${EMAIL_PROVIDER:-console}
      EMAIL_API_KEY: ${EMAIL_API_KEY:-}
      EMAIL_FROM_ADDRESS: ${EMAIL_FROM_ADDRESS:-noreply@example.com}
    depends_on:
      valkey:
        condition: service_healthy
      clickhouse:
        condition: service_healthy
      migrate:
        condition: service_completed_successfully
      pgbouncer:
        condition: service_started
    volumes:
      - ./:/app
    working_dir: /app

  worker-ingest:
    <<: *worker-base
    container_name: worker-ingest
    command: -A dev_health_ops.workers.celery_app worker --loglevel=info --disable-prefetch -Q ingest --concurrency=1

  # CHAOS-2693 D8/master-spec CC11: dedicated queue/container, not folded
  # into worker-ingest -- external-ingest is customer-facing (spiky,
  # potentially larger batches) and must not share throughput with an
  # unrelated internal consumer backlog, nor vice versa.
  # DEPLOYMENT INVARIANT: exactly ONE replica at --concurrency=1. The
  # reclaim/redelivery design (StreamConsumer.enable_reclaim,
  # reclaim_idle_ms=900_000) assumes a single logical consumer identity
  # draining the PEL; scaling replicas requires revisiting reclaim
  # semantics first. tests/test_compose_config.py asserts this flag shape.
  worker-external-ingest:
    <<: *worker-base
    container_name: worker-external-ingest
    command: -A dev_health_ops.workers.celery_app worker --loglevel=info --disable-prefetch -Q external-ingest --concurrency=1

  worker-heavy:
    <<: *worker-base
    container_name: worker-heavy
    # `monitoring` is consumed here AND by `worker` above: queue telemetry
    # survives either pool being saturated or down.
    command: -A dev_health_ops.workers.celery_app worker --loglevel=info --disable-prefetch -Q metrics,backfill,scheduler,monitoring,sync.github.heavy,sync.gitlab.heavy --concurrency=${WORKER_HEAVY_CONCURRENCY:-2}

  beat:
    <<: *worker-base
    container_name: beat
    command: -A dev_health_ops.workers.celery_app beat --loglevel=info --schedule=/tmp/celerybeat-schedule

  # ---------------------------------------------------------------------------
  # Observability — SigNoz (distributed tracing + metrics)
  # ---------------------------------------------------------------------------
  # Run SigNoz separately from its own compose file:
  #   git clone https://github.com/SigNoz/signoz.git && cd signoz/deploy
  #   docker compose -f docker/clickhouse-setup/docker-compose.yaml up -d
  #
  # Then connect this stack to SigNoz's network:
  #   docker network connect clickhouse-setup_default dev-health-api
  #
  # UI:         http://localhost:8080  (from SigNoz stack)
  # OTLP gRPC:  signoz-otel-collector:4317
  # OTLP HTTP:  signoz-otel-collector:4318
  # ---------------------------------------------------------------------------

  # ---------------------------------------------------------------------------
  # Observability — BugSink (error tracking / Sentry-compatible)
  # ---------------------------------------------------------------------------
  # Web UI: http://localhost:8800
  # Single container, uses the existing postgres (database: bugsink).
  # Set SENTRY_DSN to a project DSN from the BugSink UI to enable error
  # reporting — the Sentry SDK sends events to BugSink transparently.
  # ---------------------------------------------------------------------------
  bugsink:
    image: bugsink/bugsink:latest
    container_name: bugsink
    restart: unless-stopped
    environment:
      DATABASE_URL: "postgresql://postgres:postgres@postgres:5432/bugsink"
      SECRET_KEY: ${BUGSINK_SECRET_KEY:?BUGSINK_SECRET_KEY must be set}
      BASE_URL: ${BUGSINK_BASE_URL:-http://localhost:8800}
      PORT: "8000"
      CREATE_SUPERUSER: ${BUGSINK_CREATE_SUPERUSER:-}
    depends_on:
      postgres:
        condition: service_healthy
    ports:
      - "8800:8000"