Merge pull request #42 from usherlabs/develop

Bump master for telemtry, in-TEE proto handling, etc.

Author: rsoury

.env.sample

@@ -1,5 +1,5 @@
 
-# Required for CLI and Optional for the Broker Library
+# Required for CLI and broker
 
 CEX_BROKER_BYBIT_API_KEY=***********************
 CEX_BROKER_BINANCE_API_KEY=****************************************
@@ -8,12 +8,39 @@ CEX_BROKER_BINANCE_API_SECRET=**************************************************
 LOG_LEVEL=debug
 CEX_BROKER_SANDBOX_MODE=true
 
-# OpenTelemetry Configuration
-# Use OTEL_EXPORTER_OTLP_ENDPOINT for the full OTLP endpoint (preferred)
+# OpenTelemetry (optional)
 OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
 OTEL_SERVICE_NAME=cex-broker
 
 # Or use legacy host/port configuration
 # CEX_BROKER_OTEL_HOST=otel-collector
 # CEX_BROKER_OTEL_PORT=4318
-# CEX_BROKER_OTEL_PROTOCOL=http
+# CEX_BROKER_OTEL_PROTOCOL=http
+
+# Travel-rule deposit auto-clear reconciler (only used when a policy has
+# travelRule.rule[].deposits.enabled). One RPC URL per Binance network code
+# (suffix must match the deposit's `network`, e.g. ARBITRUM), used to prove a
+# frozen deposit's on-chain sender before auto-submitting its questionnaire. A
+# frozen deposit on a network with no RPC configured is left frozen (fail-closed).
+# These are intentionally NOT CEX_BROKER_-prefixed so the credential scan skips them.
+# TRAVEL_RULE_RPC_URL_ARBITRUM=https://arb1.arbitrum.io/rpc
+# Optional overrides (defaults shown):
+# TRAVEL_RULE_DEPOSIT_POLL_ACTIVE_SECS=60
+# TRAVEL_RULE_DEPOSIT_POLL_IDLE_SECS=600
+# TRAVEL_RULE_QUESTIONNAIRE_COUNTRY=AU
+
+# ClickHouse research / market data archive (optional)
+# Broker → forwarder
+CEX_BROKER_ARCHIVE_ENABLED=false
+CEX_BROKER_ARCHIVE_FORWARDER_URL=http://localhost:8090/archive
+# When enabled, this must point to persistent writable storage. A path only in
+# the container filesystem is not durable across container replacement.
+CEX_BROKER_ARCHIVE_DEAD_LETTER_PATH=./archive-loss.jsonl
+CEX_BROKER_DEPLOYMENT_ID=local-dev
+# OHLCV collector: required JSON array config and reconnect bootstrap coverage.
+# 1000 one-minute bars cover roughly 16 hours on every (re)subscription.
+CEX_BROKER_OHLCV_ARCHIVE_BOOTSTRAP_LIMIT=1000
+CEX_BROKER_OHLCV_COLLECTOR_CONFIG=./ohlcv-subscriptions.json
+# ClickHouse (forwarder, candle-viewer, Python)
+CLICKHOUSE_HOST=localhost
+CLICKHOUSE_PORT=8123

.github/workflows/ci.yml

@@ -4,9 +4,9 @@ name: CI
 
 on:
   push:
-    branches: [master]
+    branches: [master, develop]
   pull_request:
-    branches: [master]
+    branches: [master, develop]
 
 jobs:
   test:
@@ -31,4 +31,10 @@ jobs:
         run: bunx @biomejs/biome lint .
 
       - name: Run tests
-        run: bun test
+        run: bun test
+
+      # Gate PRs on the same strict build the publish workflow runs on tag push
+      # (dts-bundle-generator + strict tsc). Without this, develop can merge code
+      # that only fails at publish time, blocking releases.
+      - name: Build project
+        run: bun run build

.github/workflows/publish-archive-forwarder.yml

@@ -0,0 +1,57 @@
+# Publishes the archive-forwarder image (services/archive-forwarder).
+# The forwarder is repo-runtime only — it is NOT part of the npm package
+# (files: ["dist"]), so it ships as its own container image.
+#
+# Triggers: semver tags (alongside the broker publish) and manual dispatch
+# (to publish from a branch without cutting an npm release).
+
+name: Publish Archive Forwarder
+
+on:
+  push:
+    tags:
+      - 'v[0-9]*.[0-9]*.[0-9]*'
+      - 'v[0-9]*.[0-9]*.[0-9]*-*'
+  workflow_dispatch:
+
+env:
+  IMAGE_NAME: ghcr.io/usherlabs/cex-broker-archive-forwarder
+
+permissions:
+  contents: read
+  packages: write
+
+jobs:
+  publish-docker:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.IMAGE_NAME }}
+          tags: |
+            type=match,pattern=v(\d+\.\d+\.\d+.*),group=1
+            type=sha
+            type=raw,value=latest,enable={{is_default_branch}}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: services/archive-forwarder/Dockerfile
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}

.github/workflows/publish-ohlcv-collector.yml

@@ -0,0 +1,57 @@
+# Publishes the OHLCV collector image (services/ohlcv-collector).
+# The collector is repo-runtime only — it is NOT part of the npm package
+# (files: ["dist"]), so it ships as its own container image.
+#
+# Triggers: semver tags (alongside the broker publish) and manual dispatch
+# (to publish from a branch without cutting an npm release).
+
+name: Publish OHLCV Collector
+
+on:
+  push:
+    tags:
+      - 'v[0-9]*.[0-9]*.[0-9]*'
+      - 'v[0-9]*.[0-9]*.[0-9]*-*'
+  workflow_dispatch:
+
+env:
+  IMAGE_NAME: ghcr.io/usherlabs/cex-broker-ohlcv-collector
+
+permissions:
+  contents: read
+  packages: write
+
+jobs:
+  publish-ohlcv-collector:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.IMAGE_NAME }}
+          tags: |
+            type=match,pattern=v(\d+\.\d+\.\d+.*),group=1
+            type=sha
+            type=raw,value=latest,enable={{is_default_branch}}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: services/ohlcv-collector/Dockerfile
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}

.github/workflows/publish.yml

@@ -7,6 +7,10 @@ on:
     tags:
       - 'v[0-9]*.[0-9]*.[0-9]*' # Matches semver tags in the format of v1.2.3
       - 'v[0-9]*.[0-9]*.[0-9]*-*' # Matches semver tags in the format of v1.2.3-beta
+  # Recovery path: republish from a branch when a tag's run failed for
+  # workflow-only reasons (a rerun executes the workflow file at the tag's
+  # commit, so a fixed workflow can never rerun under the original tag).
+  workflow_dispatch:
 
 env:
   IMAGE_NAME: ghcr.io/usherlabs/cex-broker
@@ -32,12 +36,17 @@ jobs:
       - name: Setup Node.js for npm publishing
         uses: actions/setup-node@v4
         with:
-          node-version: 20
+          node-version: 24
           check-latest: true
           registry-url: "https://registry.npmjs.org"
 
-      - name: Ensure latest npm
-        run: npm install -g npm@latest
+      # This package publishes via npm trusted publishing (OIDC, id-token
+      # permission) — no NPM_TOKEN secret exists. OIDC needs npm >= 11.5.1,
+      # newer than any runner-bundled npm; without it the placeholder token is
+      # sent and the registry PUT 404s. Pinned to a major because npm@latest
+      # broke every tag run when npm 12 dropped the then-pinned Node 20.
+      - name: Install npm with trusted-publishing support
+        run: npm install -g npm@12
 
       - name: Install dependencies
         run: bun install
@@ -74,20 +83,19 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Extract metadata
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.IMAGE_NAME }}
-          tags: |
-            type=match,pattern=v(\d+\.\d+\.\d+.*),group=1
-  
+      # The release convention tags the version-bump commit, so package.json
+      # is the version authority on both tag pushes and dispatch republishes
+      # (a git-tag-derived name is empty on workflow_dispatch).
+      - name: Read package version
+        id: version
+        run: echo "version=$(jq -r .version package.json)" >> "$GITHUB_OUTPUT"
+
       - name: Build and push Docker image
         uses: docker/build-push-action@v6
         with:
           context: .
           file: Dockerfile
           push: true
           tags: |
-            ${{ env.IMAGE_NAME }}:${{ fromJSON(steps.meta.outputs.json).tag-names[0] }}
+            ${{ env.IMAGE_NAME }}:${{ steps.version.outputs.version }}
             ${{ env.IMAGE_NAME }}:latest

.gitignore

@@ -27,6 +27,12 @@ report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
 .cache
 *.tsbuildinfo
 
+# research outputs
+research/python/examples/output/
+research/output/
+__pycache__
+*.egg-info
+
 # IntelliJ based IDEs
 .idea
 
@@ -36,5 +42,10 @@ build
 /proto/**
 src/proto/*.ts
 src/proto/**/*.ts
+!src/proto/node.descriptor.ts
+
+src/assets/proto/**
 
-src/assets/proto/**
+# Worktrees, Agents
+.codex
+.emdash

AGENTS.md

@@ -0,0 +1,18 @@
+## Agent Orchestrator (ao) Session
+
+You are running inside an Agent Orchestrator managed workspace.
+
+## Source layout (cex-broker)
+
+- `src/server.ts` — gRPC registration and handler wiring only; do not add domain logic here.
+- `src/handlers/` — RPC dispatch (`execute-action/`, `subscribe/`).
+- `src/helpers/` — domain and shared utilities (`shared/`, `grpc/`, `order-book.ts`, etc.).
+- Dependency direction: `server` → `handlers` → `helpers`. Helpers must not import from `server` or `handlers`.
+- Import concrete helper modules (e.g. `helpers/deposit`); avoid growing `helpers/index.ts` with server utilities.
+Session metadata is updated automatically via shell wrappers.
+
+If automatic updates fail, you can manually update metadata:
+```bash
+~/.ao/bin/ao-metadata-helper.sh  # sourced automatically
+# Then call: update_ao_metadata <key> <value>
+```

Dockerfile

@@ -6,6 +6,13 @@ RUN apt-get update -y \
   && apt-get install -y --no-install-recommends ca-certificates curl \
   && rm -rf /var/lib/apt/lists/*
 
-RUN bun install --global @usherlabs/cex-broker@0.2.6
+COPY package.json bun.lock ./
+COPY patches ./patches
+RUN bun install --frozen-lockfile
 
-CMD ["cex-broker"]
+COPY build.ts proto-gen.sh tsconfig.json ./
+COPY scripts ./scripts
+COPY src ./src
+RUN bun run build
+
+CMD ["bun", "./dist/commands/cli.js"]

POLICY.md

@@ -110,7 +110,17 @@ Accepted values:
 - **A network/chain identifier** — e.g. `"ARBITRUM"`, `"BEP20"`, `"ETH"`, `"SOL"`. The value must match what the exchange uses for that chain.
 - **`"*"`** — wildcard; matches any network.
 
-Even if the policy allows a network, the selected exchange must also support that network for the currency or the request will still fail at execution time.
+The broker normalizes common operator aliases before matching policy:
+
+| Operator alias | Broker network id |
+|----------------|-------------------|
+| `ARB`, `ARBITRUM` | `ARBITRUM` |
+| `ETH`, `ERC20`, `ETHEREUM` | `ETHEREUM` |
+| `BNB`, `BSC`, `BEP20` | `BNB` |
+
+Even if the policy allows a normalized network, the selected exchange must also
+support that network for the currency or the request will still fail at
+execution time.
 
 ---
 
@@ -191,6 +201,46 @@ Common rejection reasons:
 - address not whitelisted
 - token not in `coins` for the matched rule
 
+### Narrow Binance/MEXC USDC BEP20 corridor example
+
+Use a dedicated policy for treasury corridors instead of relying on broad
+exchange/network rules. The example
+`policy/policy.binance-mexc-usdc-bep20.example.json` permits only USDC over the
+normalized `BNB` network family (`BNB`, `BSC`, or `BEP20`) between Binance and
+MEXC:
+
+```json
+{
+  "withdraw": {
+    "rule": [
+      {
+        "exchange": "BINANCE",
+        "network": "BEP20",
+        "coins": ["USDC"],
+        "whitelist": ["0x1111111111111111111111111111111111111111"]
+      },
+      {
+        "exchange": "MEXC",
+        "network": "BEP20",
+        "coins": ["USDC"],
+        "whitelist": ["0x2222222222222222222222222222222222222222"]
+      }
+    ]
+  },
+  "deposit": {
+    "rule": [
+      { "exchange": "MEXC", "network": "BEP20", "coins": ["USDC"] },
+      { "exchange": "BINANCE", "network": "BEP20", "coins": ["USDC"] }
+    ]
+  }
+}
+```
+
+This policy does not authorize volatile inventory transfer. If a treasury
+ceremony chooses remote volatile acquisition, that acquired-asset transfer must
+be explicitly requested, live-discovered, policy-approved, cost-gated, and
+attested by the caller.
+
 ---
 
 ## Order policy (`order.rule`)