docker-compose.yml

# Complete SN118 validator stack. Configure the root .env, then run:
#   ./scripts/validator-compose.sh up -d
# DittoBench now lives in this monorepo. The wrapper resolves one repository
# revision and feeds it to both the local build context and the binary identity;
# managed releases replace the source-build fallback with their release commit.
x-dittobench-revision: &dittobench-revision ${DITTOBENCH_SOURCE_REVISION:-source-build}
x-dittobench-build-context: &dittobench-build-context ${DITTOBENCH_BUILD_CONTEXT:-.}
x-dittobench-source-identity: &dittobench-source-identity ${DITTOBENCH_SOURCE_IDENTITY:-source:source-build}
x-dittobench-software-version: &dittobench-software-version ${DITTOBENCH_SOFTWARE_VERSION:-source-build}
# Shared secret for the validator -> scorer inference control plane
# (POST/DELETE /v1/inference/session*). The scorer joins sandbox-docker's
# network namespace while the validator stays on the Compose bridge, so a
# control call arrives from a 172.x peer and NEVER on loopback: without this
# token every benchmark ticket dies at `prepare` with 401 "inference control plane
# unavailable". One anchor feeds both sides so they can never drift apart.
# The default keeps already-deployed validators working with no operator
# action; the control plane is not miner-reachable (the sandbox INPUT chain
# admits only 11434/11435/11436, never the scorer API port 8000), so this
# token separates the control plane from the rest of the private Compose
# network rather than from untrusted harness code. Operators who want a
# per-host secret set DITTOBENCH_BROKER_CONTROL_TOKEN in .env.
x-broker-control-token: &broker-control-token ${DITTOBENCH_BROKER_CONTROL_TOKEN:-ditto-sn118-stack-broker-control}

services:
  pylon:
    image: ${PYLON_IMAGE:-ditto-subnet-pylon:local}
    pull_policy: build
    build:
      context: .
      dockerfile: Dockerfile.pylon
      args:
        PYLON_BASE_IMAGE: docker.io/backenddevelopersltd/bittensor-pylon@sha256:ba6ba5e3e7b32683f90039eb5a37a565d5479f034326b40288869f826f7362c7
      additional_contexts:
        turbobt: https://github.com/ditto-assistant/turbobt.git?ref=refs/heads/ditto/subscription-id-str&checksum=d36d1a28e761ff9d743bbcc83d091309c4d3599e
    platform: linux/amd64
    restart: unless-stopped
    environment:
      # This repository is SN118; netuid is not an operator setting. Pylon
      # reuses the validator settings for its network, wallet, and hotkey. One
      # PYLON_TOKEN guards both the open-access reads and the identity write.
      PYLON_BITTENSOR_NETWORK: ${SUBTENSOR_NETWORK:-finney}
      PYLON_BITTENSOR_ARCHIVE_NETWORK: ${SUBTENSOR_NETWORK:-finney}
      PYLON_OPEN_ACCESS_TOKEN: ${PYLON_TOKEN}
      PYLON_IDENTITIES: '["ditto"]'
      PYLON_ID_DITTO_WALLET_NAME: ${VALIDATOR_WALLET_NAME}
      PYLON_ID_DITTO_HOTKEY_NAME: ${VALIDATOR_WALLET_HOTKEY}
      PYLON_ID_DITTO_NETUID: "118"
      PYLON_ID_DITTO_TOKEN: ${PYLON_TOKEN}
      PYLON_DATABASE_PATH: /data/pylon.db
      PYLON_METRICS_TOKEN: ${PYLON_METRICS_TOKEN:-}
    security_opt:
      - no-new-privileges:true
    # Pylon runs as root, but operator hotkeys are normally mode 0600 and may
    # be owned by a non-root host user. Restore only the capability needed to
    # traverse/read the read-only wallet mount after dropping all capabilities.
    cap_add:
      - DAC_READ_SEARCH
    cap_drop:
      - ALL
    volumes:
      - type: bind
        source: ${DITTO_BITTENSOR_WALLETS_DIR:-~/.bittensor/wallets}
        target: /root/.bittensor/wallets
        read_only: true
        bind:
          create_host_path: false
      - pylon-data:/data

  # The scorer receives an isolated nested Docker daemon. This temporarily
  # retains the v0.41 rootful-DinD runtime contract so validators with the
  # original stack updater can adopt current benchmark releases. The daemon is
  # still isolated from the host Docker socket and its miner networks retain
  # the broker-only egress policy installed by the entrypoint.
  sandbox-docker:
    build:
      context: .
      dockerfile: Dockerfile.sandbox-docker
    restart: unless-stopped
    privileged: true
    environment:
      DOCKER_HOST: tcp://127.0.0.1:2375
      DOCKER_TLS_CERTDIR: ""
      DITTOBENCH_OPENROUTER_SHIM_PORT: "11437"
      DITTOBENCH_OPENROUTER_SHIM_CA_BUNDLE_PATH: /var/lib/dittobench-openrouter-shim/ca-bundle.pem
      # This trusted root process owns initialization of the private V9
      # projection volume before the non-root scorer becomes healthy.
      DITTOBENCH_PRIVATE_ARTIFACT_DIR: /var/lib/dittobench-private-artifacts
    extra_hosts:
      - host.docker.internal:127.0.0.1
    volumes:
      - sandbox-docker-rootful-data:/var/lib/docker
      # Shared only with dittobench-api. The scorer writes a public CA bundle;
      # the nested daemon mounts it read-only into miner containers. No private
      # key is written to this volume.
      - openrouter-shim-ca:/var/lib/dittobench-openrouter-shim
      # Validator-private V9 replay evidence. This is deliberately absent from
      # every miner container and from the public validator service.
      - dittobench-private-artifacts:/var/lib/dittobench-private-artifacts
    healthcheck:
      test: [CMD-SHELL, "docker info >/dev/null 2>&1"]
      interval: 10s
      timeout: 15s
      retries: 30

  dittobench-api:
    build:
      # Build the scorer from the monorepo service at the same reviewed commit.
      context: *dittobench-build-context
      dockerfile: services/dittobench-api/Dockerfile
      target: sandbox
      # Stamp the pinned identity INTO the image. dittobench-api links these
      # into the binary, so /v1/capabilities reports what the running code IS
      # rather than what its environment claims. Because the build arguments
      # are part of the build, changing the pin also invalidates the cached
      # image instead of silently reusing it.
      args:
        DITTOBENCH_SOURCE_SHA: *dittobench-revision
        DITTOBENCH_SOFTWARE_VERSION: *dittobench-software-version
    restart: unless-stopped
    network_mode: service:sandbox-docker
    # The API and nested daemon share a network namespace. The daemon maps
    # host.docker.internal to that namespace's loopback for the local broker.
    depends_on:
      sandbox-docker:
        condition: service_healthy
    environment:
      DOCKER_HOST: tcp://127.0.0.1:2375
      DITTOBENCH_SANDBOX_HARDEN: "1"
      DITTOBENCH_SANDBOX_EGRESS_NETWORK: ditto-sandbox
      # Production opt-in for platform-verified screened archives. Keep the
      # broader private-harness URL bypass disabled.
      DITTOBENCH_ALLOW_SCREENED_IMAGES: "1"
      # Legacy, operator-asserted identity. A stamped image ignores these and
      # reports its linked-in values; they remain only so an unstamped image
      # keeps answering /v1/capabilities. They come from the same anchors as the
      # build arguments above, so they can never disagree with the pin.
      DITTOBENCH_SOFTWARE_VERSION: *dittobench-software-version
      DITTOBENCH_SOURCE_SHA: *dittobench-revision
      DITTOBENCH_SANDBOX_SECCOMP_PROFILE: ${DITTOBENCH_SANDBOX_SECCOMP_PROFILE:-}
      DITTOBENCH_SANDBOX_APPARMOR_PROFILE: ${DITTOBENCH_SANDBOX_APPARMOR_PROFILE:-}
      # Scorer-side run capacity. Rides the same variable as the validator's
      # advertised slot count so the two can never disagree: the worker fails
      # closed (claims nothing at all) when the scorer advertises fewer full-run
      # slots than the worker has configured. The compose default is the
      # production value; the platform's operator cap decides how many of these
      # slots actually receive tickets.
      DITTOBENCH_MAX_CONCURRENT_RUNS: ${VALIDATOR_BENCHMARK_CAPACITY:-8}
      # Hosted v7 parallelism. Both default inside the scorer image to the
      # values dittobench-api#104 ships (4 cases, 8 concurrent embeddings per
      # run); these anchors exist so a host with unusual headroom, or one being
      # debugged, can pin them without a rebuild.
      #
      # Neither is the operator's ramp lever. The fleet-wide dial is the
      # platform's inference-concurrency board (ditto-platform#436), which is
      # changed from backroom and takes effect within seconds with no release.
      # Leave these unset unless a specific host needs to differ.
      #
      # These parallelise work WITHIN one benchmark run, so unlike
      # VALIDATOR_BENCHMARK_CAPACITY they add no sandbox: no extra memory and
      # no extra disk. On a host already near its memory or disk ceiling this
      # is the safe knob and slot capacity is not.
      DITTOBENCH_V7_CASE_CONCURRENCY: ${VALIDATOR_V7_CASE_CONCURRENCY:-}
      DITTOBENCH_V7_EMBEDDING_CONCURRENCY: ${VALIDATOR_V7_EMBEDDING_CONCURRENCY:-}
      DITTOBENCH_REQUIRE_TICKET_INFERENCE: "true"
      # Compatibility for harnesses that compiled OpenRouter's public HTTPS URL
      # instead of reading the injected ticket gateway. The scorer terminates a
      # validator-local certificate on 11437; sandbox-docker maps only LOCAL
      # openrouter.ai:443 traffic there while preserving the sandbox source IP.
      DITTOBENCH_OPENROUTER_SHIM_PORT: "11437"
      DITTOBENCH_OPENROUTER_SHIM_CA_BUNDLE_PATH: /var/lib/dittobench-openrouter-shim/ca-bundle.pem
      # The sandbox-docker entrypoint initializes this shared volume as a real
      # 0700 directory owned by the scorer's uid before its healthcheck passes.
      DITTOBENCH_PRIVATE_ARTIFACT_DIR: /var/lib/dittobench-private-artifacts
      # Authorizes the validator's session prepare/activate/cancel calls. Same
      # anchor as VALIDATOR_DITTOBENCH_CONTROL_TOKEN on the validator.
      DITTOBENCH_BROKER_CONTROL_TOKEN: *broker-control-token
      # The broker refuses an activation whose proxy_url does not byte-match
      # this value, and treats an unset/malformed value as "refuse everything"
      # (configuredPlatformProxyURL returns "" -> activate always 401s). It
      # must therefore equal the proxy_url the platform mints in the grant
      # exchange, which is DITTO_INFERENCE_PUBLIC_BASE_URL joined to
      # /api/v1/inference/chat/completions.
      #
      # This is a TWO-SIDED contract: the value below and the platform's
      # DITTO_INFERENCE_PUBLIC_BASE_URL must be changed together, or every
      # activation 401s fleet-wide. See infra PR changing the prod platform to
      # https://dittobench.ai.
      #
      # Deliberately NOT derived from VALIDATOR_PLATFORM_API_URL: operators
      # commonly set that with a trailing slash, and the resulting "//api/..."
      # path fails the broker's exact-path check. For the same reason the base
      # below has no trailing slash.
      DITTOBENCH_PLATFORM_INFERENCE_PROXY_URL: ${DITTOBENCH_PLATFORM_INFERENCE_PROXY_URL:-https://dittobench.ai/api/v1/inference/chat/completions}
      # Preserve the canonical grant URL above for exact activation checks, but
      # carry the validator-owned broker traffic over Platform's direct origin.
      # This keeps benchmark inference off the dashboard's Cloudflare path;
      # miners receive neither URL nor the ticket bearer.
      DITTOBENCH_PLATFORM_INFERENCE_TRANSPORT_URL: ${DITTOBENCH_PLATFORM_INFERENCE_TRANSPORT_URL:-https://platform-api.heyditto.ai/api/v1/inference/chat/completions}
      # Hosted embeddings share the same ticket grant and platform boundary.
      # This URL is consumed only by the trusted local broker; miner containers
      # receive the source-bound broker URL, never this upstream or a bearer.
      DITTOBENCH_EMBEDDING_UPSTREAM_URL: ${DITTOBENCH_EMBEDDING_UPSTREAM_URL:-https://dittobench.ai/api/v1/inference/embeddings}
    read_only: true
    tmpfs:
      - /tmp
    security_opt:
      - no-new-privileges:true
    cap_drop:
      - ALL
    volumes:
      - openrouter-shim-ca:/var/lib/dittobench-openrouter-shim
      - dittobench-private-artifacts:/var/lib/dittobench-private-artifacts
    healthcheck:
      test: ["CMD", "wget", "-q", "-O", "-", "http://127.0.0.1:8000/health"]
      interval: 10s
      timeout: 5s
      retries: 12

  ditto-subnet:
    # Local builds remain the default operator path. The opt-in updater passes
    # an immutable GHCR digest explicitly and overrides this build policy.
    image: ${DITTO_SUBNET_IMAGE:-ditto-subnet-validator:local}
    pull_policy: build
    build:
      context: .
      dockerfile: Dockerfile
      args:
        DITTO_REVISION: ${DITTO_SOURCE_REVISION:-local}
        VALIDATOR_COMPATIBILITY_EPOCH: "2"
        VALIDATOR_HEARTBEAT_PROTOCOL: "18"
    restart: unless-stopped
    stop_grace_period: 80m
    environment:
      NETUID: "118"
      SUBTENSOR_NETWORK: ${SUBTENSOR_NETWORK:-finney}
      VALIDATOR_PLATFORM_API_URL: ${VALIDATOR_PLATFORM_API_URL}
      VALIDATOR_DITTOBENCH_API_URL: http://sandbox-docker:8000
      VALIDATOR_DITTOBENCH_CAPABILITIES_TIMEOUT_SECONDS: "3"
      # Presented as `Authorization: Bearer` on the inference control plane
      # only. Same anchor as the scorer's DITTOBENCH_BROKER_CONTROL_TOKEN; it
      # is never sent to the platform, Pylon, or any other host.
      VALIDATOR_DITTOBENCH_CONTROL_TOKEN: *broker-control-token
      # The PINNED scorer revision links its identity into the binary, so the
      # only acceptable answer from /v1/capabilities is a binary-derived
      # revision. An environment-asserted revision from this pin means the
      # running image is not the pinned one — the exact shape of the v0.29.3
      # incident, where a recreated container wore the new pin's environment on
      # top of an older binary. Committed next to the pin and moved with it:
      # leave it off for a pin whose scorer predates build-time stamping,
      # otherwise every validator would degrade on a revision that cannot
      # possibly satisfy it.
      VALIDATOR_SCORER_REQUIRE_BINARY_PROVENANCE: "1"
      # Heartbeat v10+ per-component health probes. Private-network URLs only —
      # they are configuration, never part of the public payload. The scorer
      # shares sandbox-docker's network namespace, so this endpoint proves that
      # namespace remains reachable without probing the retired inference
      # listener on port 11434.
      VALIDATOR_SANDBOX_DOCKER_PROBE_URL: http://sandbox-docker:8000/health
      VALIDATOR_STACK_PROBE_TIMEOUT_SECONDS: "2"
      VALIDATOR_STACK_HEALTH_CACHE_SECONDS: "60"
      VALIDATOR_HOTKEY: ${VALIDATOR_HOTKEY}
      VALIDATOR_WALLET_NAME: ${VALIDATOR_WALLET_NAME}
      VALIDATOR_WALLET_HOTKEY: ${VALIDATOR_WALLET_HOTKEY}
      PYLON_URL: http://pylon:8000
      PYLON_IDENTITY_NAME: ditto
      PYLON_TOKEN: ${PYLON_TOKEN}
      VALIDATOR_RUN_SIZE: "full"
      VALIDATOR_SWEEP_SECONDS: "30"
      VALIDATOR_EPOCH_SECONDS: "3600"
      VALIDATOR_QUEUE_LIMIT: "50"
      # Heartbeat v10+ advertises every active slot. The scorer independently
      # publishes the same bound and the worker fails closed on mismatch.
      #
      # This is the CAPABILITY the host offers, not the concurrency it will run.
      # How many of these slots actually receive tickets is decided by the
      # platform's validator-slot cap (an append-only revision setting changed
      # from backroom, default 2), so concurrency can be raised gradually and
      # reverted instantly without a release. Advertising the protocol maximum
      # here gives the operator room to ramp without shipping a new stack; idle
      # slots only add lightweight polls to each sweep.
      VALIDATOR_BENCHMARK_CAPACITY: ${VALIDATOR_BENCHMARK_CAPACITY:-8}
      # Self-imposed resource ceilings. At or above these the worker stops
      # claiming tickets and reports admission=resource_constrained, while
      # still heartbeating and still finishing what it holds. High on purpose:
      # this stops a host that is about to fail, it does not throttle a busy
      # one. 0 disables; anything else must be a multiple of 5 in [50, 100],
      # because heartbeat host metrics ride a five-point grid.
      VALIDATOR_DISK_PERCENT_CEILING: ${VALIDATOR_DISK_PERCENT_CEILING:-95}
      VALIDATOR_MEMORY_PERCENT_CEILING: ${VALIDATOR_MEMORY_PERCENT_CEILING:-95}
      # A pinned CPU is a working benchmark host, not a failing one.
      VALIDATOR_CPU_PERCENT_CEILING: ${VALIDATOR_CPU_PERCENT_CEILING:-0}
      VALIDATOR_INFERENCE_PROXY_REQUIRED: "true"
      # The platform serves its inference plane under its own public hostname
      # (DITTO_INFERENCE_PUBLIC_BASE_URL), which need not equal the API host
      # above. A ticket's exchange URL must match one of the two exactly.
      # Production mints https://dittobench.ai, so that is the default here:
      # shipping this empty makes the allowlist a no-op and the validator
      # refuses every v7 ticket with "exchange URL is not the platform".
      # Override only when pointing at an environment that mints another host
      # (e.g. dev); it must stay byte-identical to that platform's
      # DITTO_INFERENCE_PUBLIC_BASE_URL.
      VALIDATOR_PLATFORM_INFERENCE_BASE_URL: ${VALIDATOR_PLATFORM_INFERENCE_BASE_URL:-https://dittobench.ai}
      VALIDATOR_DITTOBENCH_POLL_SECONDS: "10"
      VALIDATOR_DITTOBENCH_TIMEOUT_SECONDS: "4500"
      VALIDATOR_EXPECTED_COMPATIBILITY_EPOCH: "2"
      # Source/manual installs must never inherit managed-release identity from
      # an operator shell or .env file. The immutable release renderer replaces
      # this with authenticated descriptor values for managed stacks.
      # v2 retains source-build fallback; v3 requires screened images by contract.
      VALIDATOR_SCREENED_IMAGES: "1"
      VALIDATOR_SANDBOX_EGRESS_RESTRICTED: "1"
      VALIDATOR_EXECUTOR_ISOLATION: privileged_dind
      VALIDATOR_STACK_UPDATER: ${VALIDATOR_STACK_AUTO_UPDATE:-false}
      VALIDATOR_STACK_MODE: source
      VALIDATOR_STACK_COMPOSE_SCHEMA: "1"
      VALIDATOR_STACK_COMPONENT_DITTO_SUBNET: ${DITTO_SOURCE_IDENTITY:-version:source-build}
      VALIDATOR_STACK_COMPONENT_DITTOBENCH_API: *dittobench-source-identity
      VALIDATOR_STACK_COMPONENT_SANDBOX_DOCKER: ${DITTO_SOURCE_IDENTITY:-version:source-build}
      VALIDATOR_STACK_COMPONENT_PYLON: source:d36d1a28e761ff9d743bbcc83d091309c4d3599e
      VALIDATOR_START_DRAINED: ${VALIDATOR_START_DRAINED:-false}
      VALIDATOR_BOOTSTRAP_TOKEN: ${VALIDATOR_BOOTSTRAP_TOKEN:-manual}
      VALIDATOR_HTTP_TIMEOUT_SECONDS: "30"
      VALIDATOR_MIN_STAKE_TAO: "0"
      VALIDATOR_DITTOBENCH_MOCK: "false"
      VALIDATOR_LOG_LEVEL: ${VALIDATOR_LOG_LEVEL:-INFO}
      WANDB_MODE: ${WANDB_MODE:-disabled}
      WANDB_PROJECT: ${WANDB_PROJECT:-ditto-sn118}
      WANDB_ENTITY: ${WANDB_ENTITY:-}
      WANDB_RUN_NAME: ${WANDB_RUN_NAME:-}
      WANDB_API_KEY: ${WANDB_API_KEY:-}
      WANDB_DIR: /tmp/wandb
      WANDB_DATA_DIR: /tmp/wandb-data
      WANDB_CACHE_DIR: /tmp/wandb-cache
      UV_CACHE_DIR: /tmp/uv-cache
    depends_on:
      pylon:
        condition: service_started
      dittobench-api:
        condition: service_healthy
    read_only: true
    tmpfs:
      - /tmp
      # bittensor creates runtime directories (miners, metagraphs) on import;
      # keep them ephemeral while the wallet bind mount below stays read-only.
      - /root/.bittensor
    security_opt:
      - no-new-privileges:true
    # The validator runs as root, but the host-owned hotkey is mode 0600.
    # Restore read/search permission bypass only for this read-only key mount.
    cap_add:
      - DAC_READ_SEARCH
    cap_drop:
      - ALL
    volumes:
      # The root filesystem stays read-only. The updater gives every forced
      # candidate/rollback recreation a new token, so only that container's
      # successful USR2 acknowledgement survives a Docker/host restart.
      - type: volume
        source: validator-update-bootstrap
        target: /var/lib/ditto-validator-update
        read_only: false
      - type: bind
        source: ${DITTO_BITTENSOR_WALLETS_DIR:-~/.bittensor/wallets}/${VALIDATOR_WALLET_NAME}/hotkeys/${VALIDATOR_WALLET_HOTKEY}
        target: /root/.bittensor/wallets/${VALIDATOR_WALLET_NAME}/hotkeys/${VALIDATOR_WALLET_HOTKEY}
        read_only: true
        bind:
          create_host_path: false

volumes:
  pylon-data:
  openrouter-shim-ca:
  dittobench-private-artifacts:
  sandbox-docker-rootful-data:
  validator-update-bootstrap: