diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3cc8171..27ef871 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,6 +12,20 @@ jobs: test: runs-on: ubuntu-latest + services: + clickhouse: + image: clickhouse/clickhouse-server:24.8 + env: + CLICKHOUSE_USER: default + CLICKHOUSE_PASSWORD: integration-test + ports: + - 8123:8123 + options: >- + --health-cmd "wget --no-verbose --tries=1 --spider http://localhost:8123/ping || exit 1" + --health-interval 5s + --health-timeout 3s + --health-retries 20 + steps: - name: Checkout repo uses: actions/checkout@v3 @@ -32,6 +46,11 @@ jobs: - name: Run tests run: bun test + env: + CLICKHOUSE_REQUIRED: "1" + CLICKHOUSE_TEST_URL: http://localhost:8123 + CLICKHOUSE_USER: default + CLICKHOUSE_PASSWORD: integration-test # Gate PRs on the same strict build the publish workflow runs on tag push # (dts-bundle-generator + strict tsc). Without this, develop can merge code diff --git a/.gitignore b/.gitignore index 332a746..57157ec 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,7 @@ report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json .eslintcache .cache *.tsbuildinfo +archive-forwarder-spool.sqlite* # research outputs research/python/examples/output/ diff --git a/README.md b/README.md index 940a4a4..91f0bf6 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,10 @@ A high-performance gRPC-based cryptocurrency exchange broker service that provid - API keys for supported exchanges (e.g., Binance, Bybit, etc.) - Optional: Verity prover URL for zero-knowledge proof integration +## Service architecture + +See [SERVICES_ARCHITECTURE.md](SERVICES_ARCHITECTURE.md) for the authoritative boundaries between the full broker, operator market-data collector, archive-forwarder, research viewer, offline tools, and externally owned systems. + ## 🛠️ Installation 1. **Clone the repository:** @@ -97,8 +101,15 @@ CLICKHOUSE_PORT=8123 bun run start-candle-viewer # http://localhost:8091 Dev watchers: `dev:candle-viewer`, `dev:archive-forwarder`, `dev:archive-watch` (see [research/README.md](research/README.md)). +Maker `hb_runtime` delivery uses the forwarder's durable strategy-only SQLite +acceptance boundary. See [docs/archive-forwarder-durable-acceptance.md](docs/archive-forwarder-durable-acceptance.md) +for the wire versions, fixed quota/retention, retry ownership, health semantics, +and required production volume. + Key env vars: `CEX_BROKER_ARCHIVE_ENABLED=true`, `CEX_BROKER_ARCHIVE_FORWARDER_URL`, `CEX_BROKER_ARCHIVE_DEAD_LETTER_PATH`, and `CEX_BROKER_DEPLOYMENT_ID`. The archive is disabled for every enable value except the exact string `true`. Production durability requires the dead-letter file to reside on persistent writable storage or a mounted volume; a container-local ephemeral path is not durable. +Canonical replay capture for FIET-901/FIET-903 adds deployment-owned archive roles, capture bundles, four-feed collector configuration, deterministic checksums, conflict-blocking replay views, and mandatory legacy-to-canonical table migration before canonical-only deployment. Credential resolution remains environment-loaded broker first, request metadata second, and supported public fallback last. See [docs/canonical-market-data-replay.md](docs/canonical-market-data-replay.md) for the complete deployment and cutover contract. + #### Wallet-authenticated exchanges Some exchanges (for example Hyperliquid, Vertex, Paradex, and Derive) authenticate with an on-chain wallet instead of exchange-issued API keys. The broker keeps the same `API_KEY` / `API_SECRET` interface and maps credentials internally based on each exchange's CCXT `requiredCredentials`: diff --git a/docker/clickhouse-research.compose.yml b/docker/clickhouse-research.compose.yml index 16c8b0f..8f132bd 100644 --- a/docker/clickhouse-research.compose.yml +++ b/docker/clickhouse-research.compose.yml @@ -5,6 +5,9 @@ services: ports: - "8123:8123" - "9000:9000" + environment: + - CLICKHOUSE_USER=default + - CLICKHOUSE_PASSWORD volumes: - clickhouse_data:/var/lib/clickhouse - ../schema/clickhouse/market_data.sql:/docker-entrypoint-initdb.d/01_market_data.sql:ro @@ -24,19 +27,22 @@ services: ports: - "8090:8090" environment: - ARCHIVE_FORWARDER_PORT: "8090" - CLICKHOUSE_HOST: clickhouse - CLICKHOUSE_PORT: "8123" - CLICKHOUSE_USER: default - CLICKHOUSE_PASSWORD: "" - CLICKHOUSE_DATABASE: market_data + - ARCHIVE_FORWARDER_PORT=8090 + - CLICKHOUSE_HOST=clickhouse + - CLICKHOUSE_PORT=8123 + - CLICKHOUSE_USER=default + - CLICKHOUSE_PASSWORD + - CLICKHOUSE_DATABASE=market_data + - ARCHIVE_FORWARDER_SPOOL_PATH=/var/lib/archive-forwarder/spool.sqlite volumes: - ..:/app + - archive_forwarder_spool:/var/lib/archive-forwarder depends_on: - clickhouse volumes: clickhouse_data: + archive_forwarder_spool: networks: default: diff --git a/docs/archive-forwarder-durable-acceptance.md b/docs/archive-forwarder-durable-acceptance.md new file mode 100644 index 0000000..f10d239 --- /dev/null +++ b/docs/archive-forwarder-durable-acceptance.md @@ -0,0 +1,96 @@ +# Maker archive-forwarder durable acceptance + +## Purpose and ownership boundary + +Maker's runtime archive bridge makes one non-blocking HTTP attempt. The archive +forwarder therefore owns a conforming request only after it commits the request +to its SQLite spool. It then returns HTTP 202 and drains each represented table +to ClickHouse independently. Maker does not retry an acknowledged batch. + +This path is deliberately narrow: `source=hb_runtime` and exclusively these five +tables: + +- `strategy_data.policy_evaluation_events` +- `strategy_data.strategy_policy_snapshots` +- `strategy_data.market_identity` +- `strategy_data.symbol_mapping` +- `strategy_data.inventory_settlement_events` + +Mixed requests are rejected atomically. Strategy tables under another source +are also rejected. `broker_read`, `broker_write`, account, execution, market-data, +and Maker replay requests keep their existing direct synchronous ClickHouse path +and do not consume spool capacity. + +## Wire versions + +The envelope requires non-empty `source`, non-empty `deployment_id`, and 1–1000 +rows. Missing/empty `schema_version` and version `1` use the legacy contract. +Version `2` additionally requires non-empty `producer_id`, `producer_run_id`, +`stream_name`, and `archive_event_id`, plus positive integer `stream_seq` and +`seq`. Unknown versions and row/envelope provenance mismatches return HTTP 400. + +`test/fixtures/archive_forwarder_envelope.json` is byte-identical to the fixture +at Maker commit `563594435853c88cca5b187b8c999f845e31136b`; its pinned SHA-256 is +`784f647e048052a6c3382309b1a86abfbe08bc162363ead9fc88eaa1ba3d50c9`. + +## Spool storage and fixed service limits + +The forwarder uses Bun SQLite with WAL, foreign keys, a 5-second busy timeout, +and `synchronous=FULL`. Local runs default to: + +```text +./archive-forwarder-spool.sqlite +``` + +Production must configure and persist: + +```text +ARCHIVE_FORWARDER_SPOOL_PATH=/var/lib/archive-forwarder/spool.sqlite +``` + +Mount `/var/lib/archive-forwarder` on durable storage. The supplied Dockerfile +and compose service do so. This is an archive-forwarder service option, not a +core broker `CEX_BROKER_*` option. + +Quota is fixed at 1 GiB of deterministic payload/metadata accounting. Retention +is fixed at 72 hours from admission. Neither has an environment override. Quota +reservation, expiry cleanup, batch insertion, and per-table work insertion are +transactional. + +## Responses and retry behavior + +- `202`: the complete Maker batch is durably owned; ClickHouse completion may be pending. +- `400`: invalid envelope, table/source mix, provenance, or schema version/identity. +- `429`: accepting the batch would exceed the fixed spool quota; ownership was not accepted. +- `503`: the spool is missing, corrupt, unwritable, or cannot commit; ownership was not accepted. + +Transient ClickHouse failures retry after 1, 2, 4, 8, 16, 32, then 60 seconds, +with ±20% jitter, through retention expiry. Completion is tracked per table, so +a failed table never replays successful siblings. Schema and authentication +failures become terminal and remain visible until expiry instead of hot-looping. + +Each batch/table work item persists one stable SHA-256 deduplication token. The +worker passes it as `insert_deduplication_token`, while every strategy table has +`non_replicated_deduplication_window=1000000`. A crash after a ClickHouse commit +but before SQLite completion therefore retries with the same token. + +## Health and operations + +`GET /health` reports ClickHouse/schema readiness separately from spool +writability and exposes queued batches/work, terminal/expired work, accounted +bytes, oldest age, and the last bounded error class. + +- ClickHouse down + spool healthy: HTTP 200, `status=degraded`, durable admission available. +- ClickHouse ready + spool healthy: HTTP 200, `status=ok`. +- Spool unhealthy: HTTP 503, `status=unavailable`, even when ClickHouse is reachable. + +On restart, the service attempts to open/migrate the spool before listening and +resumes incomplete work after ClickHouse schema readiness. If the persisted file +is unavailable or corrupt, the service remains up only to expose unavailable +health and the existing direct path; strategy admission fails with 503 until an +operator preserves/repairs the file and restarts. Never delete or replace a spool +file with pending work during rollback. Stop admission, preserve the volume, and +retain the additive ClickHouse columns. + +The production feed/archive observation window is a separate FIET-937 cutover +gate. Repository conformance checks do not claim that observation has run. diff --git a/docs/canonical-market-data-replay.md b/docs/canonical-market-data-replay.md new file mode 100644 index 0000000..481c4a7 --- /dev/null +++ b/docs/canonical-market-data-replay.md @@ -0,0 +1,126 @@ +# Canonical CEX market-data replay archive + +This is the deployment and operations contract for the CEX-broker portions of FIET-901 and FIET-903. FIET-903's broker RPC/capability/current-snapshot/live-stream sub-scope was completed by `cex-broker-order-book-depth-sourcing`; this change adds capture, storage, migration, and replay integrity. It does not add or gate RPCs. + +## Deployment identity and credential precedence + +TEE and non-TEE deployments run the same broker binary and register the full `ExecuteAction` and `Subscribe` service. Archive role does not reduce that service surface. + +Set the deployment-owned archive identity: + +```env +CEX_BROKER_ARCHIVE_ENABLED=true +CEX_BROKER_ARCHIVE_SOURCE=broker_read +CEX_BROKER_DEPLOYMENT_ID=market-reader-eu-1 +CEX_BROKER_CAPTURE_BUNDLE_ID=cex-2026-08-03-eu-1 +CEX_BROKER_MARKET_CAPTURE_ENVIRONMENT=production +CEX_BROKER_ARCHIVE_FORWARDER_URL=http://archive-forwarder:8090/archive +CEX_BROKER_ARCHIVE_DEAD_LETTER_PATH=/var/lib/cex-broker/archive-loss.jsonl +``` + +`CEX_BROKER_ARCHIVE_SOURCE` is closed to `broker_read|broker_write` and defaults to `broker_write` for existing deployments. The writer stamps this immutable value into envelopes, rows, and loss records. It is never inferred from API-key presence. Production FIET-901 collector startup requires `broker_read` and a non-empty deployment-owned capture bundle. Development capture explicitly uses `CEX_BROKER_MARKET_CAPTURE_ENVIRONMENT=development` and generates a `development:` bundle when none is supplied. + +Credential resolution uses the broker's established fixed precedence and requires no archive-specific credential configuration: + +1. Use a matching broker account loaded from `.env`/deployment configuration. +2. If no matching environment broker exists, use a complete request `api-key` and `api-secret` pair. +3. If neither source exists, construct a credentialless exchange only for operations that already support public access. + +When environment and request credentials are both present, the environment-loaded broker wins. The broker does not classify keys as public, read-only, or write-capable and does not introduce credential-source or permission-attestation settings. For a non-TEE reader, ensure every credential that deployment or its trusted callers may supply has exchange-side trading and withdrawal permissions disabled. Those exchange permissions—not archive source or credential location—establish effective privilege. + +## Four-feed collector + +The collector is an independent keep-alive client of a separately deployed full broker. Set its broker target and point its canonical configuration at a JSON document containing only feed intent: + +```env +CEX_BROKER_URL=cex-broker.internal:8086 +CEX_BROKER_MARKET_DATA_COLLECTOR_CONFIG=/etc/cex-broker/market-data-subscriptions.json +``` + +```json +{ + "subscriptions": [ + { "exchange": "binance", "symbol": "BTC/USDT", "feed": "ORDERBOOK", "depthLimit": 50 }, + { "exchange": "binance", "symbol": "BTC/USDT", "feed": "TICKER" }, + { "exchange": "binance", "symbol": "BTC/USDT", "feed": "TRADES" }, + { "exchange": "binance", "symbol": "BTC/USDT", "feed": "OHLCV", "timeframe": "1m", "bootstrapLimit": 100 } + ] +} +``` + +Run it with `bun run start-market-data-collector`. The collector starts no loopback broker, loads no CEX credentials, sends no API-key metadata, owns no archive writer, and does not connect to ClickHouse. The remote broker resolves its environment-first credentials and attaches the production environment, deployment, capture bundle, source, and integrity provenance configured in the preceding section. Collector JSON containing `environment`, `captureBundleId`, or other archive identity is rejected. + +Each entry has an independent reconnect supervisor and health state. OHLCV retains bootstrap/catch-up and stamps `broker_bootstrap_fetch_v1` separately from live capture. ORDERBOOK, TICKER, and TRADES record unrecoverable gaps after reconnect rather than synthesize missing events. + +`CEX_BROKER_OHLCV_COLLECTOR_CONFIG` remains supported as an OHLCV-only array. Move those entries into the canonical document and add `feed: "OHLCV"`; archive identity stays on the broker. The `start-ohlcv-collector` command and existing service directory/image remain compatibility wrappers for both formats. + +An external CCXT or Hummingbot fallback is an optional out-of-band producer of the shared capture contract, not a broker-collector implementation. It must declare its provider, versioned fallback source mode, reason, configured exchange, and configured pair. Cross-venue or cross-pair substitution is rejected. + +## Capture and integrity contract + +All canonical rows carry source, deployment, capture bundle, exchange, trading pair/source symbol, provider, feed, source mode, source/received timestamps, raw capture ID/scope, schema version, checksum algorithm, raw checksum, normalized checksum, and provenance completeness. The current versions are: + +- schema: `1.0.0` +- checksum: `sha256-canonical-json-v1` +- construction: sampled top-N snapshots for live/current broker order books + +Raw payloads are redacted before identity/checksum calculation. Canonical JSON sorts object keys, uses finite plain-decimal numbers, normalizes negative zero, omits undefined object values, and excludes checksum fields from their own projections. The TypeScript fixture is `test/fixtures/canonical-market-capture-v1.json`; `research/hummingbot/canonical_capture_fixture.py` is the matching Maker-side verifier. + +Exact L2 is future-facing and non-blocking for this delivery. The broker reports it unsupported and never silently labels sampled evidence exact. A future exact producer must supply complete continuity proof. + +## ClickHouse and replay + +Canonical storage is: + +- `market_data.cex_stream_events` for the redacted raw ledger +- `market_data.cex_ticker_events`, `market_data.cex_trades`, and `market_data.cex_ohlcv` +- append-only `market_data.cex_order_book_levels` and `market_data.cex_order_book_depth_summary` + +`cex_ohlcv` uses `ReplacingMergeTree(broker_version)`; `cex_ohlcv_closed` applies `FINAL` and closed-bar semantics. Order-book physical duplicates remain auditable. The `_canonical` views expose one checksum-consistent logical row, while `_conflicts` expose keys with multiple checksums. Same-batch conflicts are rejected by the forwarder; cross-batch conflicts remain stored and must block the affected replay bundle. + +Use `schema/clickhouse/canonical_market_data_replay.sql` for bounded bundle/exchange/pair/source-time replay. Its conflict preflights must return no rows before consuming the canonical views. + +The retained FIET-907 reference exporter materializes a conflict-free order-book window directly from ClickHouse to Maker-compatible Parquet files: + +```bash +CEX_BROKER_REPLAY_EXPORT_DIRECTORY=/tmp/maker-capture \ +CEX_BROKER_REPLAY_CAPTURE_BUNDLE_IDS=cex-2026-08-03-eu-1 \ +CEX_BROKER_REPLAY_EXCHANGE=binance \ +CEX_BROKER_REPLAY_TRADING_PAIR=BTC-USDT \ +CEX_BROKER_REPLAY_START_TIME_MS=1785715200000 \ +CEX_BROKER_REPLAY_END_TIME_MS=1785801600000 \ +bun scripts/export-canonical-orderbook-parquet.ts + +uv run --project research/python --extra dev \ + python research/hummingbot/order_book_parquet_fixture.py \ + /tmp/maker-capture/order_book_levels.parquet \ + /tmp/maker-capture/order_book_depth_summary.parquet +``` + +The exporter refuses to overwrite existing files or export a selected window with an order-book checksum conflict. The Python verifier checks capture-core field presence and recomputes every normalized-row checksum from the Parquet values. Neither tool calls the broker or an exchange. Fixture materialization, coverage reports, and replay-bundle assembly are owned by [FIET-907](https://linear.app/usherlabs/issue/FIET-907/clickhouse-backtest-fixture-materializers-and-coveragereplay-bundles), not by the live capture runtime. + +For complete strategy-pair validation, point `CEX_BROKER_REPLAY_VALIDATION_CONFIG` at a JSON document whose `windows` array contains `captureBundleIds`, `exchange`, `tradingPair`, `startTimeMs`, and `endTimeMs`, then run `bun scripts/validate-canonical-market-replay.ts`. Every configured window must contain raw and normalized ORDERBOOK, TICKER, TRADES, and OHLCV evidence; any missing feed or checksum conflict fails validation. + +## Migration, cutover, rollback, and deployment observation + +The upgraded broker always writes the latest canonical schema. There is no runtime legacy/dual/canonical write setting. Upgrading an existing legacy deployment therefore requires the ClickHouse table migration before the new broker version is deployed. + +Follow `schema/clickhouse/migrations/canonical_market_data_replay_cutover.sql` phase by phase: apply canonical DDL, quiesce legacy writers, migrate every retained bounded window, validate parity, switch consumers, and only then deploy the canonical-only broker. + +```bash +CEX_BROKER_MIGRATION_START_TIME_MS=1700000000000 \ +CEX_BROKER_MIGRATION_END_TIME_MS=1700086400000 \ +bun run scripts/migrate-legacy-market-data-to-canonical.ts + +# Repeat after reviewing dry-run counts: +CEX_BROKER_CANONICAL_MIGRATION_CONFIRM=true \ +CEX_BROKER_MIGRATION_START_TIME_MS=1700000000000 \ +CEX_BROKER_MIGRATION_END_TIME_MS=1700086400000 \ +bun run scripts/migrate-legacy-market-data-to-canonical.ts +``` + +The script reads `market_data.orderbook_snapshots` and `market_data.candles` directly from ClickHouse and writes their canonical equivalents. It never reads fixture files and never calls a broker or exchange. Legacy migration stamps `legacy_migration_v1` and `provenance_complete=0`; unavailable bundle/raw ID/raw scope/raw checksum remain `NULL`. Reruns preserve identical logical checksums: order-book canonical views collapse agreeing physical deliveries and OHLCV replacement semantics select the recorded broker version. + +Before cutover, run the parity and replay queries for every configured pair/window and complete the FIET-937 production observation window. Record feed health, last-frame age, reconnects, unrecoverable gaps, received/archived/invalid/sampled rows, queue saturation, journaled rows, checksum conflicts, parity mismatches, and Maker replay consumption. Any unaccounted row, conflict, parity mismatch, stale feed, or persistent journal growth blocks deployment cutover. This operational gate is not an implementation-completion requirement for the archived OpenSpec change and repository checks do not claim that it has occurred. + +Rollback stops the upgraded broker, restores retained legacy names if necessary, and rolls back to the previous legacy-writing application version. The runbook uses renames rather than drops; canonical and legacy base data remain recoverable throughout the retention period. diff --git a/openspec/changes/archive/2026-06-05-cex-broker-server-modularization/README.md b/openspec/changes/archive/2026-06-05-cex-broker-server-modularization/README.md index 6dce5ad..ec8f2d1 100644 --- a/openspec/changes/archive/2026-06-05-cex-broker-server-modularization/README.md +++ b/openspec/changes/archive/2026-06-05-cex-broker-server-modularization/README.md @@ -1,3 +1,14 @@ # cex-broker-server-modularization -Establish canonical `src/helpers/` and `src/handlers/` layout, extract `server.ts` helpers, deduplicate shared primitives, and shrink `server.ts` to gRPC wiring and dispatch only. \ No newline at end of file +Establish canonical `src/helpers/` and `src/handlers/` layout, extract `server.ts` helpers, deduplicate shared primitives, and shrink `server.ts` to gRPC wiring and dispatch only. + +## Post-archive compliance note (2026-08-03) + +This change remains archived because its primary architectural outcome is implemented and the change is stale. The following known gaps are accepted as deferred follow-up work; archival does not assert that these requirements are fully satisfied: + +- `src/handlers/subscribe/handler.ts` still contains an inline `error instanceof Error ? error.message : ...` expression instead of using `getErrorMessage`. +- `src/helpers/treasury-discovery.ts` still performs a local record-like object guard instead of using the shared `isRecord` helper. +- Dedicated unit-test evidence is incomplete for some extracted pure helpers, including `parsePayload`, `depositField`, `callArgs`, and `buildTransferNetworkEvidence`. +- Repository history records the implementation phases in one implementation commit, so independent phased delivery cannot be demonstrated retrospectively. + +At review time, the full test suite passed (538 tests), TypeScript validation passed, strict OpenSpec validation passed, and `src/server.ts` remained within its 400-line budget. The synced main specification remains authoritative, and the gaps above may be revisited in future changes. diff --git a/openspec/changes/archive/2026-08-03-canonical-cex-market-data-replay-archive/.openspec.yaml b/openspec/changes/archive/2026-08-03-canonical-cex-market-data-replay-archive/.openspec.yaml new file mode 100644 index 0000000..d658936 --- /dev/null +++ b/openspec/changes/archive/2026-08-03-canonical-cex-market-data-replay-archive/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-02 diff --git a/openspec/changes/archive/2026-08-03-canonical-cex-market-data-replay-archive/design.md b/openspec/changes/archive/2026-08-03-canonical-cex-market-data-replay-archive/design.md new file mode 100644 index 0000000..7ad1a68 --- /dev/null +++ b/openspec/changes/archive/2026-08-03-canonical-cex-market-data-replay-archive/design.md @@ -0,0 +1,173 @@ +## Context + +The broker already normalizes and streams ORDERBOOK, TICKER, TRADES, and OHLCV data through `Subscribe`, and its archive writer provides bounded queues, batching, retries, metrics, and a persistent loss-journal contract. The completed `cex-broker-order-book-depth-sourcing` change also provides Maker-compatible current snapshot, capability, live stream, and typed historical-unsupported behavior. + +The remaining gap is the archive contract. Common tags and the forwarder envelope currently hardcode `broker_write`; order books are stored as arrays in `market_data.orderbook_snapshots`; OHLCV is stored in `market_data.candles`; and capture bundles, provider provenance, raw-capture identity, and normalized-row checksums are absent. The existing production collector supervises OHLCV only. + +This change crosses handlers, helpers, collector and forwarder services, ClickHouse schemas, deployment configuration, and Maker contract fixtures. Domain logic remains below `src/server.ts`: server code only wires deployment configuration into handlers and helpers, preserving the repository dependency direction `server -> handlers -> helpers`. + +## Goals / Non-Goals + +**Goals:** + +- Produce replay-auditable CEX market-data rows for all four required public feeds. +- Distinguish `broker_read` and `broker_write` archival provenance without maintaining separate RPC surfaces or binaries. +- Preserve the existing environment-first credential resolution order and leave effective privilege to exchange key permissions. +- Introduce canonical ClickHouse level, summary, and OHLCV tables compatible with Maker replay semantics. +- Make capture and normalized-row integrity reproducible across TypeScript and Python. +- Preserve current stream delivery and trading behavior when the archive sink is unavailable. +- Migrate legacy tables without fabricating evidence that was not captured. + +**Non-Goals:** + +- Add read/write RPC action gating, a `readEnabled`/`writeEnabled` switch, or a reduced market-read service surface. +- Change the existing protobuf service or remove generic `Action.Call`. +- Implement exact L2 reconstruction or claim exact continuity for sampled websocket snapshots. +- Implement a historical order-book provider where current capabilities truthfully report unsupported. +- Implement native Hummingbot inside the TypeScript broker; an approved Hummingbot fallback is a separate producer of the shared contract. +- Route DEX, protocol, or on-chain facts through the CEX archive. +- Move Maker coverage planning, prior-as-of joins, or thesis-validity decisions into cex-broker. +- Introduce credential profiles, credential-source policy, permission-attestation configuration, or archive-specific credential routing. +- Keep a runtime legacy/dual/canonical archive write selector after the canonical schema upgrade. +- Define or implement Maker `hb_runtime` delivery durability, `strategy_data.*` schema-version validation, spool recovery, or retry ownership. The dedicated `maker-archive-forwarder-conformance` change owns that cross-service contract. + +## Decisions + +### 1. One full broker service; credentials establish effective privilege + +TEE and non-TEE deployments use the same binary and register the same `ExecuteAction` and `Subscribe` implementations. No handler is removed based on deployment environment or archive role. + +Credential selection is deduced from the sources actually present, in this fixed order: + +1. A matching broker account loaded from environment configuration. +2. Complete request `api-key` and `api-secret` metadata when no matching environment broker exists. +3. Credentialless public exchange construction only where the existing operation supports public access. + +An environment-loaded broker always wins when both environment and request credentials are present. This change adds no credential profile, credential-source policy, or permission-attestation configuration. The selected exchange key's permissions establish effective privilege, so a non-TEE read deployment must receive only keys whose exchange-side permissions match its intended read-only posture regardless of whether they arrive from environment configuration or request metadata. + +Alternative considered: add broker-side credential profiles or reject one credential source by deployment policy. That duplicates existing credential routing, changes accepted requests, and is intentionally excluded from this archival change. + +### 2. Archive source is explicit and independent of operation type + +The archive writer is constructed with one immutable source identity. Common archive tags receive that source rather than importing a write-only constant, and the HTTP envelope uses the writer's configured source rather than a literal. The forwarder validates that envelope and row sources agree. + +Source is not inferred from TEE state, the selected exchange method, whether credentials are present, or whether an operation happens to be read-only. A write-capable deployment can still archive market observations as `broker_write`; the broker/archiver hosting FIET-901 production subscriptions must be explicitly configured as `broker_read`. FIET-901 production capture startup or deployment validation fails before opening subscriptions when that source is not `broker_read`. + +Alternative considered: infer source from `Subscribe` versus `ExecuteAction`. That misclassifies market streams observed by write brokers and cannot describe an explicitly deployed capture role. + +### 3. Capture context is created by trusted deployment or collector configuration + +A shared market-capture context carries: + +- source and deployment id; +- capture bundle id; +- exchange, trading pair/source symbol, asset type, and feed; +- provider and versioned source mode; +- source and received timestamps; +- raw-capture identity/scope/checksum, schema version, and raw/normalized checksum algorithm versions. + +The production collector or deployment configuration owns `capture_bundle_id`; it is not accepted from provider payloads. Production startup requires a non-empty value. Bundle rotation remains an orchestration concern, allowing one replay window to reference one or more finite capture bundles without coupling broker row normalization to a Maker run id. + +Request-controlled values are limited to feed selection parameters and the existing optional in-flight credential metadata already supported by the broker contract. Archive source, deployment, and capture authority remain server-side. + +### 4. Capture-core fields are separate from Maker materialization fields + +ClickHouse and Maker parquet share an immutable capture core: market identity, provider, timestamps, evidence modes, depth, raw/snapshot identities, schema version, and checksums. Maker may extend exported rows with run id, coverage-report path, effective as-of lag, future-leakage status, and assumption haircuts. + +Maker extensions do not participate in the broker's raw-capture or normalized-row checksum. This avoids inventing run-specific facts during live ingestion and permits multiple replay runs to reuse the same capture. The retained direct ClickHouse-to-Parquet exporter is a FIET-907 reference tool and never enters the live broker capture path. + +Alternative considered: copy Maker's current parquet schema verbatim into ClickHouse. That would put fields unknown at capture time into the source-of-record tables and blur immutable evidence with derived replay assessment. + +### 5. `cex_stream_events` is the raw-capture ledger + +`market_data.cex_stream_events` is extended to hold the redacted broker-visible payload or reproducible capture metadata plus `raw_capture_id`, `raw_capture_scope`, checksum algorithm/version, and raw checksum. Normalized ticker, trade, OHLCV, order-book level, and summary rows reference that identity. + +For current CCXT-based feeds, `raw_capture_scope` states that the retained object is CCXT-normalized or broker-visible; it does not claim to contain exchange websocket bytes. This provides honest reproducibility without introducing a second generic raw-event table. + +Alternative considered: create a separate raw-order-book table. Reusing the generic stream ledger keeps capture identity consistent across all feeds and reduces duplicate payload contracts. + +### 6. Checksums use a versioned cross-language canonical projection + +Raw checksums are SHA-256 over a canonical serialization of the redacted broker-visible payload. Raw capture identity is derived from stable market/capture identity fields and the raw checksum, making retries reproducible. + +Normalized row checksums are SHA-256 over a versioned semantic projection containing immutable contract fields and normalized feed values. The checksum field itself, ClickHouse ingestion metadata, retry counters, and Maker run extensions are excluded. Numeric and timestamp canonicalization is specified explicitly and locked with TypeScript/Python golden fixtures before production cutover. + +Alternative considered: hash ordinary `JSON.stringify` output. Object ordering and numeric representation are not a sufficiently durable cross-language contract. + +### 7. The collector becomes a feed-neutral supervisor + +The existing OHLCV collector supervision pattern is generalized into a market-data collector with one independent supervisor per configured exchange, pair, feed, and relevant option set. It connects to the deployed full broker and keeps subscriptions alive; the broker's Subscribe handler remains the capture boundary. + +Each supervisor exposes feed health, last frame, reconnect count, and archival counters. OHLCV retains bootstrap/catch-up behavior. ORDERBOOK, TICKER, and TRADES record explicit gaps when providers cannot replay a disconnected interval rather than synthesizing data. + +The existing OHLCV configuration and image entry point may remain as a compatibility wrapper during migration, but canonical production configuration describes a set of feeds rather than OHLCV-only rows. + +The collector does not implement direct CCXT or native Hummingbot fallback producers in this change. If a separately deployed producer writes the shared capture contract, ingestion validates its exchange/pair identity, provider, source mode, and fallback reason and rejects cross-venue substitution. + +### 8. Canonical tables favor row-oriented replay shapes + +The canonical ClickHouse outputs are: + +- `market_data.cex_stream_events` for raw capture metadata/payloads; +- `market_data.cex_order_book_levels` for one side/level row; +- `market_data.cex_order_book_depth_summary` for one snapshot summary; +- `market_data.cex_ticker_events`; +- `market_data.cex_trades`; +- `market_data.cex_ohlcv`. + +Level and summary tables use deterministic snapshot identity and repeat the capture-core provenance necessary for independent parquet export. ClickHouse-native timestamp and numeric types may differ physically from Arrow types, but exported values and field semantics must satisfy the shared fixtures. + +The order-book level and summary base tables remain append-only `MergeTree` evidence stores so retries and checksum conflicts are not erased by background replacement merges. Their physical `ORDER BY` keys begin with exchange, trading pair, capture bundle, and source time and include the deterministic logical identity: + +- level: capture bundle, exchange, pair, raw capture, snapshot, schema version, side, and level index; +- summary: capture bundle, exchange, pair, raw capture, snapshot, and schema version. + +`market_data.cex_order_book_levels_canonical` and `market_data.cex_order_book_depth_summary_canonical` expose one logical row only when all physical deliveries for that logical key agree on `normalized_row_checksum`. Separate conflict views group physical rows by logical key and report more than one distinct checksum. Conflicted keys are excluded from canonical views, emit an operational alert/metric, and make replay validation fail for the affected capture bundle. The forwarder rejects a same-batch logical-key conflict before insertion; cross-batch conflicts remain preserved and visible in ClickHouse. + +`market_data.cex_ohlcv` uses `ReplacingMergeTree(broker_version)` because forming and closed versions of the same candle are legitimate state updates. Its canonical closed-candle view uses `FINAL`, preserving the established candle query semantics. + +Alternative considered: use `ReplacingMergeTree` for order-book rows. That can collapse conflicting deliveries before an integrity detector observes them, so append-only evidence plus canonical/conflict views is preferred. + +### 9. Sampled top-N remains the implemented order-book evidence mode + +Live and current broker order books are classified as `sampled_top_n_snapshot` unless a future implementation supplies a valid initial snapshot, ordered deltas, complete sequence range, clean continuity status, and `exact_l2_reconstruction_complete=true`. Exact mode uses fail-fast gap semantics. + +An explicit exact request returns the existing typed unsupported result when capability is false. Capability discovery may lead the caller to select sampled mode on a subsequent request, but the broker never silently downgrades an explicit exact run. + +Source-mode and gap-policy values are versioned closed registries shared with Maker. Unknown values fail validation rather than entering ClickHouse as free-form evidence claims. + +### 10. Existing archive durability machinery is extended, not replaced + +Market rows continue through the bounded `BrokerExecutionArchiver` queue and archive forwarder. The writer becomes source-aware, and metrics add source/feed labels within bounded-cardinality sets. The forwarder allowlist and DDL are expanded for canonical tables. + +Sink failures remain asynchronous to stream delivery. Queue overflow, terminal retries, and bounded shutdown continue to use the persistent loss journal, preserving the current strategy-runtime isolation. + +## Risks / Trade-offs + +- [A non-TEE deployment receives a write-capable key] -> Restrict permissions at the exchange and control which secrets trusted callers or deployment configuration can supply; the broker does not infer or attest key scope. +- [Environment and request credentials are both present] -> Select the environment-loaded broker deterministically and lock the precedence with handler regression tests. +- [A reviewer reintroduces credential-policy configuration as archival scope] -> Keep the no-profile/no-policy decision normative in this design, delta spec, tasks, and operations documentation. +- [TypeScript and Python checksum results diverge] -> Freeze numeric/timestamp canonicalization with shared golden fixtures before enabling production writes. +- [Per-level rows and retained raw payloads increase volume] -> Keep configurable top-N sampling, bounded payloads, batching, partitioning, retention policy, and feed-volume metrics. +- [Legacy writes race the table migration] -> Quiesce legacy writers before migrating every retained partition and block upgraded deployment until parity validation passes. +- [Legacy migration appears provenance-complete] -> Stamp a legacy source mode and explicit incomplete-provenance flag; leave unavailable raw identity/checksum fields null. +- [Capture bundles become unbounded] -> Require deployment/orchestration ownership and documented rotation; replay manifests may combine multiple finite bundles. +- [Schema changes drift from Maker] -> Maintain shared capture-core fixtures and make contract tests a required check in both repositories. +- [Duplicate order-book deliveries disagree] -> Preserve physical rows in append-only tables, exclude conflicts from canonical views, alert on conflict views, and fail replay validation for affected bundles. + +## Migration Plan + +1. Freeze the capture-core schema, enum registry, canonicalization algorithm, and TypeScript/Python golden fixtures. +2. Make common archive tags, writer envelopes, loss-journal entries, and forwarder validation source-aware while defaulting existing deployments to `broker_write` for compatibility. +3. Apply forwarder support and ClickHouse DDL for the canonical tables while the existing legacy-writing broker remains deployed. +4. Quiesce legacy writers and run the bounded ClickHouse table-to-table migration for every retained order-book and candle partition with explicit legacy/incomplete provenance. +5. Validate migration parity, canonical/conflict views, and replay queries. Any missing row, mismatch, or conflict blocks deployment of the upgraded broker. +6. Switch consumers to canonical tables and deploy the upgraded broker and generalized collector. The upgraded version writes only the latest canonical schema; the FIET-901 collector uses `broker_read`, a required capture bundle, and the complete strategy pair/feed configuration. +7. Treat the documented production observation window for feed health, queue behavior, checksum verification, and Maker consumption as an external deployment cutover gate owned by FIET-937. It is not an OpenSpec implementation-completion task and repository tests do not claim it has occurred. +8. Retain compatibility views and untouched legacy base data for the agreed retention period. + +Rollback stops the upgraded broker, restores retained legacy query names when necessary, and rolls the application back to the previous legacy-writing version. Canonical and legacy data remain intact, and affected capture bundles remain identifiable. There is no runtime archive write-mode switch. + +## Open Questions + +No architecture question blocks implementation. Deployment owners must choose the production capture-bundle rotation interval and retention window before rollout; the broker contract only requires that each production row carry a valid deployment-owned bundle id. diff --git a/openspec/changes/archive/2026-08-03-canonical-cex-market-data-replay-archive/proposal.md b/openspec/changes/archive/2026-08-03-canonical-cex-market-data-replay-archive/proposal.md new file mode 100644 index 0000000..b98f402 --- /dev/null +++ b/openspec/changes/archive/2026-08-03-canonical-cex-market-data-replay-archive/proposal.md @@ -0,0 +1,38 @@ +## Why + +The broker already exposes the required CEX market-data streams and truthful order-book RPC capabilities, but its archive plane still identifies all rows as `broker_write`, uses legacy ClickHouse table shapes, and cannot prove which capture bundle or normalized rows were used by replay. FIET-901 and the remaining FIET-903 storage scope need a canonical, provenance-complete ingestion contract that production capture and Maker replay can share. + +## What Changes + +- Add deployment-controlled archive provenance for `broker_read` and `broker_write` without changing which RPCs the broker service registers. +- Keep the same full broker service in TEE and non-TEE deployments and preserve the established credential resolution order: a matching environment-loaded broker takes precedence, request metadata is used only when no environment broker exists, and public construction is the final fallback for supported public operations. Effective privilege comes from the selected exchange key's permissions, not broker-side profiles or gates. +- Define capture-bundle, provider, source-mode, construction-mode, gap-policy, raw-capture, schema-version, and deterministic checksum fields shared across CEX market-data outputs. +- Generalize production collection from OHLCV-only supervision to configured `ORDERBOOK`, `TICKER`, `TRADES`, and `OHLCV` feeds for every strategy exchange/pair. +- Keep direct CCXT/Hummingbot fallback producers outside this broker implementation while requiring any externally supplied fallback rows admitted to the shared contract to preserve venue/pair provenance. +- Add canonical ClickHouse outputs for order-book levels, order-book depth summaries, and OHLCV, and upgrade stream, ticker, and trade outputs with replay provenance. +- Normalize sampled top-N order books deterministically and retain truthful evidence semantics; exact L2 and broker historical snapshots remain unsupported unless their advertised capability is implemented and continuity is proven. +- Require a bounded ClickHouse table-to-table migration of legacy `market_data.orderbook_snapshots` and `market_data.candles` before deploying the upgraded canonical-only writer, without fabricating unavailable legacy provenance. +- Add cross-language contract fixtures and end-to-end validation from broker capture through ClickHouse to Maker-compatible replay rows. +- Retain the direct ClickHouse-to-Parquet reference exporter as a FIET-907 consumer-side tool; it is not part of the broker runtime or live capture path. +- Leave Maker `strategy_data.*` archive-forwarder delivery, schema-version validation, and durable acceptance to the dedicated `maker-archive-forwarder-conformance` change. + +## Capabilities + +### New Capabilities + +- `cex-market-data-replay-capture`: Production collection and archival of broker-read CEX streams with bundle identity, provider provenance, checksums, established credential precedence, and non-blocking durability. +- `cex-order-book-replay-archive`: Canonical order-book level and depth-summary tables, deterministic normalization, evidence-quality rules, integrity metadata, and legacy table migration. + +### Modified Capabilities + +- None. + +## Impact + +- Archive source types, common tags, batching envelopes, loss journals, and archive-forwarder validation. +- `Subscribe` market-data capture helpers and the existing OHLCV collector service, generalized to all required public market feeds. +- ClickHouse market-data schemas, supported-table allowlists, compatibility views, migrations, and research/replay queries. +- Order-book normalization and archive row builders; the completed order-book RPC/capability behavior remains backward compatible. +- Documentation and regression coverage for environment-first credential precedence without new credential profile, source-policy, or attestation configuration. +- Maker-facing schema fixtures, a retained FIET-907 reference exporter, and integration tests for canonical parquet-compatible replay rows. +- The Maker `hb_runtime` forwarder wire contract and spool are explicitly outside this change and tracked by `maker-archive-forwarder-conformance`. diff --git a/openspec/changes/archive/2026-08-03-canonical-cex-market-data-replay-archive/specs/cex-market-data-replay-capture/spec.md b/openspec/changes/archive/2026-08-03-canonical-cex-market-data-replay-archive/specs/cex-market-data-replay-capture/spec.md new file mode 100644 index 0000000..0a2a05e --- /dev/null +++ b/openspec/changes/archive/2026-08-03-canonical-cex-market-data-replay-archive/specs/cex-market-data-replay-capture/spec.md @@ -0,0 +1,207 @@ +## ADDED Requirements + +### Requirement: Broker deployments retain the full service contract +The system SHALL register the same `ExecuteAction` and `Subscribe` RPC contract in TEE and non-TEE broker deployments, and SHALL NOT use an RPC/action allowlist as the mechanism that distinguishes read-only from write-capable deployments. + +#### Scenario: Non-TEE broker starts with the full service +- **WHEN** a non-TEE broker is started with exchange credentials whose permissions are read-only +- **THEN** it MUST register the same broker RPCs and action handlers as a TEE broker +- **AND** public market-data calls MUST remain available through the normal broker contract + +#### Scenario: Deployment archive role does not change RPC registration +- **WHEN** the archive source is configured as `broker_read` or `broker_write` +- **THEN** the selected value MUST affect archival provenance only +- **AND** it MUST NOT add or remove broker RPC methods + +### Requirement: Credential resolution follows fixed source precedence +The system SHALL preserve the broker's established credential resolution order without adding archive-specific credential profiles, credential-source policy, or permission-attestation configuration. A matching environment-loaded broker SHALL take precedence over request-supplied credentials, request metadata SHALL be considered only when no matching environment broker exists, and credentialless public construction SHALL remain the final fallback only for operations that already support public access. + +The selected exchange key's actual permissions SHALL establish effective privilege. The broker SHALL NOT infer a read-only or write-capable profile from archive source, deployment type, or credential location. + +#### Scenario: Environment and request credentials are both present +- **WHEN** a matching environment-loaded broker exists and a request also supplies `api-key` or `api-secret` metadata +- **THEN** the environment-loaded broker MUST be selected +- **AND** request credentials MUST NOT replace it + +#### Scenario: Only request credentials are present +- **WHEN** no matching environment-loaded broker exists and a request supplies a complete supported credential pair +- **THEN** the broker MUST construct the exchange from the request credentials using the existing in-flight path +- **AND** archival configuration MUST NOT reject that source + +#### Scenario: No credentials are present for a public operation +- **WHEN** neither environment nor request credentials are available and the requested operation supports public exchange access +- **THEN** the broker MAY construct a credentialless public exchange +- **AND** it MUST retain the same RPC and action surface + +#### Scenario: A non-TEE deployment is intended to be read-only +- **WHEN** credentials are supplied through either supported source +- **THEN** deployment and secret provisioning MUST restrict the exchange key to the intended permissions +- **AND** the archive plane MUST NOT add a credential profile, source-policy, or attestation environment variable to classify that key + +### Requirement: Archive source identity is deployment controlled +The archive plane SHALL support the closed source identities `broker_read` and `broker_write`, selected from immutable deployment configuration and propagated consistently to every row and batch envelope. + +#### Scenario: Read broker archives a market frame +- **WHEN** a broker configured with archive source `broker_read` archives a market-data frame +- **THEN** every normalized row and its archive envelope MUST contain `source = "broker_read"` +- **AND** clients MUST NOT be able to override that source through request fields or metadata + +#### Scenario: Envelope and row sources disagree +- **WHEN** an archive batch contains a row whose source differs from the envelope source +- **THEN** the archive forwarder MUST reject the inconsistent batch or row before ClickHouse insertion +- **AND** it MUST emit a bounded diagnostic without exposing credentials or raw secrets + +#### Scenario: FIET-901 production capture uses the read archive source +- **WHEN** the broker/archiver hosting FIET-901 production subscriptions starts with an archive source other than `broker_read` +- **THEN** production capture startup or deployment validation MUST fail before opening subscriptions +- **AND** unrelated TEE or write-broker deployments MAY continue to archive their own market observations as `broker_write` + +### Requirement: Production captures have bundle and provider provenance +Every production market-data row SHALL identify the capture bundle, deployment, exchange, trading pair, provider, feed, source mode, source timestamp, broker-received timestamp, schema version, raw-capture identity and scope, raw checksum and algorithm version, and normalized-row checksum and algorithm version using the versioned replay-capture contract. + +#### Scenario: Collector opens a configured feed +- **WHEN** the production collector opens a feed for a configured exchange and pair +- **THEN** it MUST attach the deployment-owned `capture_bundle_id` to every raw-capture and normalized row produced by that feed +- **AND** it MUST record the exact provider, feed-specific source mode, schema version, raw-capture integrity fields, and normalized-row integrity fields rather than infer them during replay + +#### Scenario: Production bundle identity is missing +- **WHEN** production collection starts without a non-empty capture bundle identity +- **THEN** startup MUST fail configuration validation +- **AND** it MUST NOT archive ungrouped production rows + +### Requirement: External fallback producers conform without expanding broker scope +The capture contract SHALL permit separately deployed direct CCXT or Hummingbot fallback producers, but this broker and collector change SHALL NOT require implementing or invoking those producers for FIET-901 completion. + +#### Scenario: Broker feed is unavailable +- **WHEN** the broker collector cannot obtain a required feed and no separately deployed fallback producer is configured +- **THEN** it MUST record the feed as unavailable +- **AND** it MUST NOT instantiate an in-process direct CCXT or Hummingbot fallback as part of this change + +#### Scenario: External fallback row is admitted +- **WHEN** an approved external fallback producer submits a row to the shared capture contract +- **THEN** ingestion MUST validate and preserve the configured exchange and pair +- **AND** the row MUST record the fallback provider, a versioned fallback source mode, and the fallback reason + +#### Scenario: External fallback crosses venues +- **WHEN** a fallback row identifies an exchange different from the configured strategy exchange +- **THEN** ingestion MUST reject the row +- **AND** it MUST NOT substitute or relabel the cross-venue data + +### Requirement: Raw captures and normalized rows have reproducible integrity +The archive plane SHALL assign each broker-visible capture a reproducible raw-capture identity and checksum, and SHALL assign every normalized output row a deterministic row checksum under a versioned canonicalization algorithm. + +#### Scenario: CCXT-normalized object is the earliest available capture +- **WHEN** the broker receives a CCXT-normalized object rather than exchange wire bytes +- **THEN** `raw_capture_scope` MUST identify the object as broker-visible or CCXT-normalized +- **AND** metadata MUST NOT claim that its checksum represents the original exchange wire frame + +#### Scenario: Raw capture checksum is recomputed +- **WHEN** the retained capture payload and checksum algorithm version are supplied to the verifier +- **THEN** canonical serialization and hashing MUST reproduce the stored raw-capture checksum + +#### Scenario: Normalized row checksum is recomputed across languages +- **WHEN** TypeScript and Python normalize the same contract fixture +- **THEN** both implementations MUST produce the same normalized-row checksum +- **AND** the checksum projection MUST exclude its own checksum field and ingestion-only mutable fields + +#### Scenario: Capture contains credential-like data +- **WHEN** a provider payload contains headers, keys, secrets, signatures, or credential metadata +- **THEN** the archive plane MUST redact those fields before retained-payload serialization +- **AND** no integrity field may embed the secret value + +### Requirement: The production collector supervises all required CEX feeds +The production collector SHALL supervise configured `ORDERBOOK`, `TICKER`, `TRADES`, and `OHLCV` subscriptions independently for every strategy exchange and pair. + +#### Scenario: Strategy capture configuration is loaded +- **WHEN** production collection starts with strategy exchange/pair/feed configuration +- **THEN** it MUST validate that each required feed has a supervisor and required options such as depth limit or timeframe +- **AND** failure of one feed MUST NOT terminate supervisors for other feeds or pairs + +#### Scenario: Live stream disconnects +- **WHEN** an order-book, ticker, trade, or OHLCV stream ends or errors unexpectedly +- **THEN** its supervisor MUST reconnect with bounded backoff and jitter +- **AND** it MUST expose reconnect count, last-frame time, and current feed-health state + +#### Scenario: Provider catch-up is unavailable +- **WHEN** a live order-book, ticker, or trade stream reconnects and the provider cannot replay the missing interval +- **THEN** the collector MUST record an explicit gap +- **AND** it MUST NOT synthesize missing market events + +#### Scenario: OHLCV bootstrap is supported +- **WHEN** an OHLCV subscription starts with a configured bootstrap window +- **THEN** the collector MUST fetch and archive available historical bars before or alongside the live stream +- **AND** bootstrap rows MUST use a source mode distinct from live stream rows + +### Requirement: Market capture remains outside the strategy execution dependency path +Market-data archival SHALL use bounded asynchronous queues, bounded batches, retry policy, backpressure telemetry, and persistent loss journaling without making strategy execution or stream delivery depend on ClickHouse availability. + +#### Scenario: ClickHouse or forwarder is unavailable +- **WHEN** the archive sink is temporarily unavailable +- **THEN** market-data stream delivery MUST continue independently +- **AND** archive delivery MUST retry within configured bounds before journaling undeliverable rows + +#### Scenario: Archive queue reaches capacity +- **WHEN** the bounded archive queue cannot accept another row +- **THEN** the configured shedding or durability policy MUST be applied deterministically +- **AND** queue saturation and lost/journaled row counts MUST be observable by feed and source + +#### Scenario: Broker shuts down with pending rows +- **WHEN** shutdown begins while archive rows remain queued +- **THEN** the broker MUST attempt a bounded flush +- **AND** rows still undeliverable at the deadline MUST be written to the configured persistent loss journal + +### Requirement: Canonical non-order-book market tables preserve replay provenance +The archive plane SHALL write ticker, trade, OHLCV, and raw stream records to `market_data.cex_ticker_events`, `market_data.cex_trades`, `market_data.cex_ohlcv`, and `market_data.cex_stream_events` with the common replay-capture fields and deterministic checksums. + +#### Scenario: OHLCV row is archived +- **WHEN** an OHLCV bar is normalized for archive +- **THEN** it MUST be written to `market_data.cex_ohlcv` with timeframe, open time, OHLCV values, closure/version state, capture provenance, and normalized-row checksum + +#### Scenario: Ticker or trade row is archived +- **WHEN** a ticker snapshot or public trade is normalized for archive +- **THEN** it MUST retain its feed-specific values and common capture provenance +- **AND** it MUST link to the raw capture from which it was normalized + +#### Scenario: Raw stream event is retained +- **WHEN** a broker-visible feed payload is accepted for production archive +- **THEN** `market_data.cex_stream_events` MUST record its redacted payload or reproducible capture metadata, raw-capture identity, raw checksum, and schema version + +### Requirement: Legacy candle storage requires table migration before upgrade +The system SHALL migrate retained rows from `market_data.candles` to `market_data.cex_ohlcv` through a bounded ClickHouse table-to-table migration before deploying the upgraded canonical-only broker. The upgraded broker SHALL always write the latest canonical schema and SHALL NOT expose a runtime legacy/dual/canonical write mode. + +#### Scenario: Existing deployment is prepared for upgrade +- **WHEN** an existing broker writes `market_data.candles` and the canonical-only version is scheduled for deployment +- **THEN** operators MUST apply canonical DDL, quiesce legacy writers, migrate every retained bounded partition, and validate parity before deploying the upgraded broker +- **AND** any missing row or value mismatch MUST block the upgrade + +#### Scenario: Legacy candle lacks raw capture provenance +- **WHEN** a `market_data.candles` row is migrated and its original capture bundle, raw identity, or raw checksum is unavailable +- **THEN** the canonical row MUST record legacy source mode and incomplete provenance +- **AND** unavailable raw provenance fields MUST remain null rather than be fabricated + +#### Scenario: Canonical-only broker is deployed +- **WHEN** canonical OHLCV migration and replay validation pass for every retained migration window +- **THEN** the upgraded broker MUST write new OHLCV captures only to `market_data.cex_ohlcv` +- **AND** legacy candle query names and closed-candle semantics MUST remain available through documented compatibility views for the migration retention period + +#### Scenario: Candle cutover is rolled back +- **WHEN** a blocking integrity or replay defect is found after canonical OHLCV deployment +- **THEN** operators MUST stop the upgraded broker and MAY restore the retained legacy names before rolling back to the previous legacy-writing application version +- **AND** the defect and affected capture bundles MUST remain diagnosable + +### Requirement: Capture output is replay-consumable +The system SHALL prove that canonical ClickHouse rows can be selected by exchange, pair, capture bundle, and replay window and converted into Maker-compatible inputs without losing source identity or checksum verifiability. + +#### Scenario: Replay selects a capture window +- **WHEN** a replay consumer queries one or more capture bundles for an exchange, pair, and time window +- **THEN** the query MUST return only matching source-time rows +- **AND** every returned normalized row MUST retain enough metadata to verify its capture linkage and checksum + +#### Scenario: Cross-language contract fixture is validated +- **WHEN** CI runs the broker-to-Maker replay contract fixtures +- **THEN** the ClickHouse row projection and Maker reader MUST agree on field semantics, timestamps, source modes, and checksum values + +#### Scenario: FIET-907 materializes Parquet fixtures +- **WHEN** the retained reference exporter produces Maker-compatible fixtures +- **THEN** it MUST query canonical ClickHouse views directly without calling the broker or a connected exchange +- **AND** fixture materialization, coverage, and replay-bundle assembly MUST remain FIET-907 consumer-side ownership diff --git a/openspec/changes/archive/2026-08-03-canonical-cex-market-data-replay-archive/specs/cex-order-book-replay-archive/spec.md b/openspec/changes/archive/2026-08-03-canonical-cex-market-data-replay-archive/specs/cex-order-book-replay-archive/spec.md new file mode 100644 index 0000000..e116836 --- /dev/null +++ b/openspec/changes/archive/2026-08-03-canonical-cex-market-data-replay-archive/specs/cex-order-book-replay-archive/spec.md @@ -0,0 +1,156 @@ +## ADDED Requirements + +### Requirement: Canonical order-book levels use one row per level +The archive plane SHALL write sampled or reconstructed order-book depth to `market_data.cex_order_book_levels` using one row per snapshot, side, and level index with a parquet-compatible capture-core schema. + +#### Scenario: Sampled snapshot is normalized +- **WHEN** a valid top-N snapshot containing bids and asks is accepted for archive +- **THEN** the system MUST emit one level row for every retained bid and ask +- **AND** each row MUST include snapshot identity, side, zero-based level index, price, amount, mid-price, spread from mid, sequence when available, and common capture provenance + +#### Scenario: Level depth exceeds the configured limit +- **WHEN** a provider returns more levels than the configured archive depth limit +- **THEN** normalization MUST retain no more than that limit on each side +- **AND** the retained depth limit and actual side counts MUST remain auditable + +### Requirement: Canonical depth summaries are deterministic +The archive plane SHALL write one `market_data.cex_order_book_depth_summary` row per retained snapshot containing top-of-book, spread, staleness, level counts, configured measurement bands, and depth by band. + +#### Scenario: Summary is derived from canonical levels +- **WHEN** canonical levels are produced for a snapshot +- **THEN** the summary best bid, best ask, mid-price, spread, counts, and band depths MUST be computed from those retained levels +- **AND** repeated normalization of the same capture and contract version MUST produce the same summary values and checksum + +#### Scenario: Measurement bands are configured +- **WHEN** bid or ask measurement bands are configured +- **THEN** bands MUST be normalized into a deterministic ascending unique order +- **AND** depth for each band MUST be calculated using the documented base-amount convention + +### Requirement: Order-book normalization rejects invalid evidence +Order-book normalization SHALL reject missing sides, non-finite or non-positive values, non-monotonic levels, crossed or locked books, invalid timestamps, and invalid depth configuration before canonical rows are archived. + +#### Scenario: Book is crossed or locked +- **WHEN** the normalized best bid is greater than or equal to the normalized best ask +- **THEN** canonical order-book rows MUST NOT be emitted +- **AND** an invalid-row diagnostic MUST identify the reason without leaking provider secrets + +#### Scenario: Side ordering is invalid +- **WHEN** bids are not strictly descending or asks are not strictly ascending after provider normalization +- **THEN** the entire snapshot MUST be rejected before canonical rows are emitted +- **AND** the broker MUST NOT silently reorder the captured levels + +#### Scenario: Price or amount is invalid +- **WHEN** a retained level has a non-finite, zero, or negative price or amount +- **THEN** the snapshot MUST fail canonical validation +- **AND** partial valid levels MUST NOT be presented as a complete snapshot without an explicit invalid-evidence classification + +### Requirement: Order-book construction modes are evidence honest +The system SHALL classify order-book rows with versioned construction mode, source mode, and gap policy values and SHALL treat sampled top-N snapshots separately from exact L2 reconstruction. + +#### Scenario: Live broker snapshot is captured +- **WHEN** the broker archives `Subscribe(ORDERBOOK)` output without complete snapshot-plus-delta continuity evidence +- **THEN** it MUST record `construction_mode = "sampled_top_n_snapshot"` +- **AND** it MUST use the broker live-sampling source mode rather than claim exact L2 reconstruction + +#### Scenario: Exact L2 capability is unavailable +- **WHEN** the broker cannot prove initial snapshot identity, complete ordered sequence ranges, clean continuity, and valid reconstruction +- **THEN** it MUST advertise exact L2 support as false +- **AND** exact-L2 rows MUST NOT be written + +#### Scenario: Exact L2 is explicitly requested but unsupported +- **WHEN** a caller explicitly requests exact L2 reconstruction and the capability is unavailable +- **THEN** the request MUST return the typed unsupported result +- **AND** it MUST NOT silently downgrade that request to sampled evidence + +#### Scenario: Historical snapshots are unsupported +- **WHEN** broker historical order-book capability is unavailable +- **THEN** the existing typed historical unsupported result MUST remain truthful +- **AND** current or live snapshots MUST NOT be relabeled as historical evidence + +### Requirement: Exact L2 evidence carries continuity proof +If exact L2 support is implemented in the future, every exact reconstruction SHALL carry an initial snapshot identity, sequence start and end, continuity status, exact-completion flag, and fail-fast gap policy sufficient to audit the claim. + +#### Scenario: Exact reconstruction completes cleanly +- **WHEN** an initial snapshot and all ordered deltas reconstruct a valid book with no gaps +- **THEN** `exact_l2_reconstruction_complete` MAY be true only with the complete continuity metadata +- **AND** canonical levels and summary MUST reference the proven reconstruction identity + +#### Scenario: Sequence continuity breaks +- **WHEN** an expected delta, update identifier, or ordering marker is missing +- **THEN** exact reconstruction MUST fail under `gap_policy = "fail_fast"` +- **AND** no exact-L2 canonical rows may be emitted for the affected reconstruction + +### Requirement: Order-book rows link to raw captures and bundles +Every new canonical order-book level and summary row SHALL link to its capture bundle, raw capture, snapshot identity, provider, exchange, pair, replay-capture schema version, and integrity fields. + +#### Scenario: One captured snapshot produces canonical rows +- **WHEN** one broker-visible snapshot is normalized +- **THEN** its raw event, all level rows, and its summary row MUST share the same `capture_bundle_id` and `raw_capture_id` +- **AND** level and summary rows MUST carry individually reproducible normalized-row checksums + +#### Scenario: Snapshot identity is recomputed +- **WHEN** the same exchange, pair, source timestamp, sequence, retained levels, and normalization version are supplied again +- **THEN** the deterministic snapshot identity MUST be reproduced +- **AND** unrelated deployment ingestion timestamps MUST NOT change that identity + +### Requirement: ClickHouse and parquet share a capture-core contract +The canonical ClickHouse order-book tables and Maker parquet outputs SHALL share names and semantics for capture-core fields while allowing Maker materialization to add run-scoped coverage and assumption fields separately. + +#### Scenario: Broker rows are exported for Maker +- **WHEN** canonical ClickHouse rows are converted into `order_book_levels.parquet` and `order_book_depth_summary.parquet` +- **THEN** capture-core fields MUST map without semantic reinterpretation +- **AND** exchange, pair, provider, timestamps, construction mode, source mode, gap policy, depth, identities, and checksums MUST be preserved + +#### Scenario: Maker adds replay-run metadata +- **WHEN** Maker coverage planning adds run id, coverage report, as-of lag, future-leakage, or assumption fields +- **THEN** those fields MUST be treated as a materialized extension of the immutable broker capture +- **AND** they MUST NOT alter the original capture checksum or provenance + +### Requirement: Canonical order-book insertion is idempotent +Canonical order-book storage SHALL retain physical deliveries in append-only `MergeTree` base tables and SHALL expose deterministic canonical and conflict views over explicit logical keys so retries remain auditable without producing ambiguous replay rows. + +The level logical key SHALL consist of capture bundle, exchange, trading pair, raw capture, snapshot, schema version, side, and level index. The summary logical key SHALL use the same fields except side and level index. + +#### Scenario: Same level row is delivered twice +- **WHEN** the archive forwarder retries a level row with the same capture, snapshot, side, level index, schema version, and checksum +- **THEN** `market_data.cex_order_book_levels_canonical` MUST resolve it as one logical level +- **AND** both physical deliveries MUST remain available in the base evidence table + +#### Scenario: Same summary row is delivered twice +- **WHEN** a summary row is retried with the same capture and snapshot identity +- **THEN** `market_data.cex_order_book_depth_summary_canonical` MUST resolve it as one logical summary +- **AND** both physical deliveries MUST remain available in the base evidence table + +#### Scenario: Logical key has conflicting checksums in one batch +- **WHEN** one archive batch contains the same level or summary logical key with more than one normalized-row checksum +- **THEN** the forwarder MUST reject the conflicting table batch before insertion +- **AND** the producer retry and loss-journal policy MUST account for the rejected rows + +#### Scenario: Logical key has conflicting checksums across batches +- **WHEN** physical deliveries from different archive batches reuse one logical key with different normalized-row checksums +- **THEN** the corresponding ClickHouse conflict view MUST expose the key and distinct checksums +- **AND** canonical views MUST exclude the conflicted key +- **AND** replay validation MUST fail the affected capture bundle + +### Requirement: Legacy order-book storage requires table migration before upgrade +The system SHALL migrate retained rows from `market_data.orderbook_snapshots` to the canonical level and summary tables through a bounded ClickHouse table-to-table migration before deploying the upgraded canonical-only broker. The upgraded broker SHALL always write the latest canonical schema and SHALL NOT expose a runtime legacy/dual/canonical write mode. + +#### Scenario: Existing deployment is prepared for upgrade +- **WHEN** an existing broker writes `market_data.orderbook_snapshots` and the canonical-only version is scheduled for deployment +- **THEN** operators MUST apply canonical DDL, quiesce legacy writers, migrate every retained bounded partition, and validate parity before deploying the upgraded broker +- **AND** any missing row or value mismatch MUST block the upgrade + +#### Scenario: Legacy snapshot lacks raw capture identity +- **WHEN** a legacy row is migrated and its original raw capture or checksum is unavailable +- **THEN** the canonical row MUST record legacy source mode and incomplete provenance +- **AND** it MUST leave unavailable raw integrity fields null rather than invent them + +#### Scenario: Canonical-only broker is deployed +- **WHEN** canonical-table migration and replay validation pass for every retained migration window +- **THEN** the upgraded broker MUST write new order-book captures only to the canonical raw, level, and summary tables +- **AND** legacy query names MUST remain available through documented compatibility views for the migration retention period + +#### Scenario: Canonical cutover is rolled back +- **WHEN** a blocking integrity or replay defect is found after canonical deployment +- **THEN** operators MUST stop the upgraded broker and MAY restore the retained legacy names before rolling back to the previous legacy-writing application version +- **AND** the defect and affected capture bundles MUST remain diagnosable diff --git a/openspec/changes/archive/2026-08-03-canonical-cex-market-data-replay-archive/tasks.md b/openspec/changes/archive/2026-08-03-canonical-cex-market-data-replay-archive/tasks.md new file mode 100644 index 0000000..03245da --- /dev/null +++ b/openspec/changes/archive/2026-08-03-canonical-cex-market-data-replay-archive/tasks.md @@ -0,0 +1,109 @@ +## 1. Capture Contract And Golden Fixtures + +- [x] 1.1 Define the versioned capture-core field registry, including source, deployment, capture bundle, market identity, provider, source mode, timestamps, raw identity/scope, schema version, and integrity fields. +- [x] 1.2 Define and test the closed archive-source, feed, source-mode, construction-mode, and gap-policy registries, including explicit legacy migration values. +- [x] 1.3 Define the canonical timestamp and numeric representation used by raw and normalized checksum projections. +- [x] 1.4 Add a deterministic canonical serialization and SHA-256 helper with unit tests for key ordering, numeric edge cases, timestamp normalization, omitted mutable fields, and checksum self-exclusion. +- [x] 1.5 Add redacted raw-capture identity/checksum helpers that distinguish CCXT-normalized objects from exchange wire frames. +- [x] 1.6 Add versioned TypeScript golden fixtures covering stream, ticker, trade, OHLCV, order-book level, and order-book summary rows with expected identities and checksums. +- [x] 1.7 Add or coordinate the matching Python/Maker fixture verifier and prove it reproduces every capture-core checksum. + +## 2. Archive Source And Credential Resolution + +- [x] 2.1 Add `broker_read` and `broker_write` archive source types and deployment configuration, retaining `broker_write` as the compatibility default for existing deployments. +- [x] 2.2 Refactor common archive tags to accept the immutable writer/deployment source instead of importing a write-only constant. +- [x] 2.3 Make archive HTTP envelopes and loss-journal records use the configured writer source and add tests for both source values. +- [x] 2.4 Validate envelope/row source consistency in the archive forwarder and add bounded error/metric tests for mismatches. +- [x] 2.5 Preserve the fixed credential resolution order: matching environment-loaded broker first, complete request metadata only when no environment broker exists, and credentialless public construction last for operations that already support it. +- [x] 2.6 Add handler regression tests proving environment-loaded credentials cannot be replaced by request credentials and credentials are never logged or archived. +- [x] 2.7 Remove archive-specific credential profiles, credential-source policy, and permission-attestation configuration; document that selected exchange-key permissions establish effective privilege. +- [x] 2.8 Add regression coverage proving TEE/non-TEE and archive source do not change registered RPCs or action handlers. + +## 3. Capture Context And Stream Ledger + +- [x] 3.1 Introduce a shared market-capture context in helpers and thread it through handlers without adding domain logic to `src/server.ts`. +- [x] 3.2 Require and validate a deployment/collector-owned `capture_bundle_id` for production capture while preserving an explicit development-only configuration path. +- [x] 3.3 Resolve and record actual provider identity and versioned source mode for broker live, current snapshot, and OHLCV bootstrap inputs. +- [x] 3.4 Extend `cex_stream_events` row construction with capture-core provenance, redacted payload metadata, raw identity/scope, checksum algorithm/version, and raw checksum. +- [x] 3.5 Archive raw ledger entries for ORDERBOOK, TICKER, TRADES, and OHLCV frames and link every normalized row to the corresponding raw capture. +- [x] 3.6 Add tests proving capture source, deployment, bundle, and provider provenance cannot be overridden by provider payload or untrusted request fields. +- [x] 3.7 Validate exchange/pair, provider, source mode, and fallback reason when an external fallback producer is admitted, without implementing direct CCXT or Hummingbot fallback inside the broker collector. + +## 4. Canonical ClickHouse Schemas + +- [x] 4.1 Add append-only `MergeTree` DDL for `market_data.cex_order_book_levels` with capture-core, evidence, level, checksum, partitioning, retention, and the documented logical key in its ordering strategy. +- [x] 4.2 Add append-only `MergeTree` DDL for `market_data.cex_order_book_depth_summary` with capture-core, evidence, summary, checksum, partitioning, retention, and the documented logical key in its ordering strategy. +- [x] 4.3 Add `cex_order_book_levels_canonical` and `cex_order_book_depth_summary_canonical` dedup views plus conflict views that preserve physical duplicates, exclude conflicting logical keys, and expose distinct checksums. +- [x] 4.4 Add `market_data.cex_ohlcv` as `ReplacingMergeTree(broker_version)` with canonical provenance, integrity, timeframe, bar values, deterministic keys, and a `FINAL` closed-candle compatibility view. +- [x] 4.5 Add capture-core and raw-link fields to `cex_stream_events`, `cex_ticker_events`, and `cex_trades` using backward-compatible schema migration statements. +- [x] 4.6 Extend broker archive table types, market-table guards, archive-forwarder allowlists, and per-table limits for all canonical tables. +- [x] 4.7 Add ClickHouse schema-application and insert integration tests for canonical tables, canonical/conflict views, and nullable legacy provenance cases. +- [x] 4.8 Reject same-batch order-book logical-key/checksum conflicts in the archive forwarder and verify producer retry/loss-journal accounting. +- [x] 4.9 Add canonical replay-window queries that use deduplicated views, filter by capture bundle/exchange/pair/feed/source time, and fail affected bundles when conflict views are non-empty. + +## 5. Canonical Market Row Builders + +- [x] 5.1 Upgrade ticker, trade, and raw stream builders to emit capture-core fields and deterministic normalized-row checksums. +- [x] 5.2 Replace canonical candle output with a `cex_ohlcv` row builder while retaining migration-only legacy candle parsing. +- [x] 5.3 Add OHLCV tests for bootstrap versus live source modes, forming/closed bar replacement semantics, raw linkage, and checksums. +- [x] 5.4 Implement deterministic order-book validation that rejects the entire snapshot for missing sides, non-positive/non-finite values, non-monotonic side ordering, crossed/locked books, invalid timestamps, or invalid depth without silently reordering levels. +- [x] 5.5 Implement deterministic snapshot identity plus one-row-per-level output with side, level index, price, amount, notional, mid-price, spread, sequence, and checksums. +- [x] 5.6 Implement one-row-per-snapshot depth summaries with best bid/ask, spread, staleness, counts, deterministic measurement bands, band depth, and checksum. +- [x] 5.7 Add order-book tests for invalid evidence, depth truncation, measurement bands, idempotent identities, raw linkage, and TypeScript/Maker fixture parity. +- [x] 5.8 Enforce sampled top-N classification for current live captures and test that exact L2 cannot be emitted without complete continuity proof. +- [x] 5.9 Preserve typed historical/exact unsupported behavior and add regression tests proving explicit exact requests are never silently downgraded. + +## 6. Subscribe And Snapshot Capture Integration + +- [x] 6.1 Route ORDERBOOK subscription captures through the canonical raw, level, and summary builders while preserving existing stream response compatibility. +- [x] 6.2 Route TICKER, TRADES, and OHLCV subscription captures through the upgraded raw and normalized builders. +- [x] 6.3 Archive typed current order-book snapshot responses through the canonical path when a trusted production capture context is configured. +- [x] 6.4 Keep all capture calls asynchronous to gRPC stream delivery and add tests showing archive failures do not fail or delay successful response frames. +- [x] 6.5 Add bounded-cardinality metrics for received, sampled-out, archived, invalid, queue-saturated, journaled, and checksum-conflict rows by source and feed. + +## 7. Multi-Feed Production Collector + +- [x] 7.1 Generalize OHLCV-only collector configuration into validated exchange/pair/feed entries with order-book depth and OHLCV timeframe/bootstrap options. +- [x] 7.2 Implement independent supervisors for ORDERBOOK, TICKER, TRADES, and OHLCV while reusing the existing bounded reconnect/backoff lifecycle. +- [x] 7.3 Pass trusted capture-bundle and feed provenance from collector configuration into broker capture context and fail FIET-901 production startup/deployment validation unless the hosting broker archiver source is `broker_read`. +- [x] 7.4 Add feed-health state, last-frame time, reconnect count, and explicit unrecoverable-gap metrics and logs. +- [x] 7.5 Preserve OHLCV bootstrap/catch-up and distinguish bootstrap source mode from live source mode. +- [x] 7.6 Record gaps rather than synthesizing missed ORDERBOOK, TICKER, or TRADES events when provider catch-up is unavailable. +- [x] 7.7 Retain an OHLCV-compatible configuration/image wrapper during migration and document its replacement path. +- [x] 7.8 Add collector tests for concurrent pairs/feeds, isolated supervisor failure, reconnection, shutdown, missing bundle rejection, non-`broker_read` production rejection, and configuration validation. + +## 8. Legacy Migration And Cutover + +- [x] 8.1 Require this cutover order for existing deployments: apply canonical DDL, quiesce legacy writers, migrate every retained legacy window in ClickHouse, validate parity, switch consumers, and only then deploy the canonical-only broker. +- [x] 8.2 Add parity queries and metrics comparing legacy snapshots/candles with canonical summaries, levels, and OHLCV rows. +- [x] 8.3 Add idempotent table-to-table migration paths from `orderbook_snapshots` and `candles` that stamp legacy source mode and incomplete provenance while leaving unavailable capture bundles/raw identities/raw checksums null. +- [x] 8.4 Leave unavailable legacy raw identities/checksums null and test that migrated rows cannot be mistaken for provenance-complete captures. +- [x] 8.5 Add compatibility views for legacy order-book queries plus `candles`/`candles_closed` names and closure semantics for the agreed migration retention period. +- [x] 8.6 Document and test cutover and rollback procedures without a runtime legacy/dual/canonical archive write mode and without dropping legacy or canonical data. + +## 9. End-To-End And Operational Verification + +- [x] 9.1 Add an archive-forwarder/ClickHouse integration test that captures all four feeds and verifies raw-to-normalized linkage and checksum recomputation. +- [x] 9.2 Add a fault-injection test proving forwarder/ClickHouse unavailability does not terminate stream delivery and that retry or persistent loss journaling accounts for every failed row. +- [x] 9.3 Add idempotency tests proving physical level/summary duplicates remain auditable, canonical views return one checksum-consistent row, same-batch conflicts are rejected, and cross-batch conflicts remain observable and block replay. +- [x] 9.4 Retain the direct ClickHouse-to-Parquet reference exporter and checksum verifier, and record fixture materialization, coverage, and replay-bundle ownership under FIET-907. +- [x] 9.5 Validate a complete replay query across configured strategy pairs, capture bundles, and replay windows, including failure when a selected bundle has an order-book checksum conflict. +- [x] 9.6 Add non-destructive regression coverage for environment-first credential precedence, request-metadata fallback, and public fallback without credential profile, source-policy, or permission-attestation configuration. +Production observation remains a deployment cutover gate under FIET-937. It is deliberately excluded from this repository implementation checklist and is not claimed complete by this change. + +## 10. Documentation And Final Validation + +- [x] 10.1 Document archive source, capture bundle, fixed environment-first credential precedence, external-fallback scope, feed configuration, checksum version, canonical/conflict views, FIET-907 exporter ownership, and mandatory ClickHouse migration in the broker README and service documentation. +- [x] 10.2 Document that TEE and non-TEE use the same full RPC surface and that non-TEE read-only posture depends on the actual selected exchange-key permissions rather than broker-side credential classification. +- [x] 10.3 Record FIET-903's broker RPC/capability sub-scope as complete and link the remaining canonical storage work to this change. +- [x] 10.4 Run the repository unit, integration, type-check, lint, and formatting commands required by CI. +- [x] 10.5 Run `openspec validate canonical-cex-market-data-replay-archive --strict` and resolve every validation error before implementation handoff. + +## 11. Design Correction Audit + +- [x] 11.1 Retain the direct ClickHouse-to-Parquet reference tools and reference them in FIET-907 as consumer-side materialization utilities outside the broker runtime. +- [x] 11.2 Remove the credential-profile, credential-source-policy, and permission-attestation implementation and lock the environment-first precedence rule in code, tests, design, delta specs, and operations documentation. +- [x] 11.3 Rename the legacy backfill implementation and configuration to migration terminology and document that it reads and writes ClickHouse tables without broker, exchange, or fixture-file input. +- [x] 11.4 Remove runtime archive write modes, make upgraded writers canonical-only, and require the bounded legacy-to-canonical ClickHouse migration before deployment. +- [x] 11.5 Record these decisions normatively so they survive OpenSpec sync/archive and are visible to subsequent human and agent reviewers. +- [x] 11.6 Re-run repository checks and strict OpenSpec validation after the correction. diff --git a/openspec/changes/cex-broker-order-book-depth-sourcing/.openspec.yaml b/openspec/changes/archive/2026-08-03-cex-broker-order-book-depth-sourcing/.openspec.yaml similarity index 100% rename from openspec/changes/cex-broker-order-book-depth-sourcing/.openspec.yaml rename to openspec/changes/archive/2026-08-03-cex-broker-order-book-depth-sourcing/.openspec.yaml diff --git a/openspec/changes/cex-broker-order-book-depth-sourcing/README.md b/openspec/changes/archive/2026-08-03-cex-broker-order-book-depth-sourcing/README.md similarity index 100% rename from openspec/changes/cex-broker-order-book-depth-sourcing/README.md rename to openspec/changes/archive/2026-08-03-cex-broker-order-book-depth-sourcing/README.md diff --git a/openspec/changes/cex-broker-order-book-depth-sourcing/design.md b/openspec/changes/archive/2026-08-03-cex-broker-order-book-depth-sourcing/design.md similarity index 100% rename from openspec/changes/cex-broker-order-book-depth-sourcing/design.md rename to openspec/changes/archive/2026-08-03-cex-broker-order-book-depth-sourcing/design.md diff --git a/openspec/changes/cex-broker-order-book-depth-sourcing/proposal.md b/openspec/changes/archive/2026-08-03-cex-broker-order-book-depth-sourcing/proposal.md similarity index 100% rename from openspec/changes/cex-broker-order-book-depth-sourcing/proposal.md rename to openspec/changes/archive/2026-08-03-cex-broker-order-book-depth-sourcing/proposal.md diff --git a/openspec/changes/cex-broker-order-book-depth-sourcing/specs/cex-broker-order-book-depth-sourcing/spec.md b/openspec/changes/archive/2026-08-03-cex-broker-order-book-depth-sourcing/specs/cex-broker-order-book-depth-sourcing/spec.md similarity index 100% rename from openspec/changes/cex-broker-order-book-depth-sourcing/specs/cex-broker-order-book-depth-sourcing/spec.md rename to openspec/changes/archive/2026-08-03-cex-broker-order-book-depth-sourcing/specs/cex-broker-order-book-depth-sourcing/spec.md diff --git a/openspec/changes/cex-broker-order-book-depth-sourcing/tasks.md b/openspec/changes/archive/2026-08-03-cex-broker-order-book-depth-sourcing/tasks.md similarity index 100% rename from openspec/changes/cex-broker-order-book-depth-sourcing/tasks.md rename to openspec/changes/archive/2026-08-03-cex-broker-order-book-depth-sourcing/tasks.md diff --git a/openspec/changes/maker-archive-forwarder-conformance/.openspec.yaml b/openspec/changes/maker-archive-forwarder-conformance/.openspec.yaml new file mode 100644 index 0000000..e08b5f8 --- /dev/null +++ b/openspec/changes/maker-archive-forwarder-conformance/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-03 diff --git a/openspec/changes/maker-archive-forwarder-conformance/design.md b/openspec/changes/maker-archive-forwarder-conformance/design.md new file mode 100644 index 0000000..1dbd636 --- /dev/null +++ b/openspec/changes/maker-archive-forwarder-conformance/design.md @@ -0,0 +1,98 @@ +## Context + +Maker's `ArchiveEmitter` sends one HTTP attempt from a bounded, non-blocking in-memory queue. It cannot safely retry after an ambiguous response because the CEX Broker forwarder currently groups a request by table and inserts those groups sequentially: some tables may commit before a later table fails. The forwarder returns 200 only after synchronous ClickHouse completion and has no restart-recoverable ownership boundary. + +The pinned cross-repository contract is Maker commit `563594435853c88cca5b187b8c999f845e31136b`, especially `packages/hb-maker-shared/tests/fixtures/archive_forwarder_envelope.json`, `archive_emitter.py`, and the upstream schema fixture. The five replay-critical targets are `strategy_data.policy_evaluation_events`, `strategy_data.strategy_policy_snapshots`, `strategy_data.market_identity`, `strategy_data.symbol_mapping`, and `strategy_data.inventory_settlement_events`. + +The archive forwarder also accepts market-data, broker-account, and broker-execution rows. Those producers already have their own queue/retry/loss-journal contracts and must not inherit Maker-specific latency, quota, or retention behavior. + +## Goals / Non-Goals + +**Goals:** + +- Establish durable forwarder ownership before Maker discards an emitted batch. +- Keep multi-table retries independent and idempotent across process restarts. +- Enforce one versioned Maker wire/schema contract in CEX Broker and Maker tests. +- Bound disk use and retention with observable, typed admission failures. +- Keep health truthful during ClickHouse outages and spool failures. +- Coordinate a CEX release and Maker dependency/evidence update. + +**Non-Goals:** + +- Change Maker's non-blocking controller-tick architecture or add producer retries. +- Spool `broker_read`, `broker_write`, `maker_replay`, account, execution, or market-data traffic. +- Add broker RPCs, handler gates, credential policy, or core `CEX_BROKER_*` settings. +- Make the spool quota or retention operator-configurable. +- Replace ClickHouse, materialize Parquet, or perform FIET-937's production observation window. + +## Decisions + +### 1. Strategy-only admission is a closed contract + +A request uses the durable path only when `source=hb_runtime` and every row targets one of the five `strategy_data.*` tables. A mixed strategy/non-strategy `hb_runtime` request is rejected, as is a strategy table under any other source. This avoids silently splitting one producer request across ownership models. + +Alternative considered: spool every archive row. That would duplicate the broker writers' existing durability machinery, couple unrelated failure domains, and make a Maker quota incident affect trading-path archives. + +### 2. SQLite is the acceptance boundary + +The forwarder uses Bun's built-in `bun:sqlite`. Admission serializes the validated envelope canonically and commits a batch row plus one pending work item per represented table in a single transaction. Only after the transaction is durable does the server return HTTP 202. + +SQLite runs with WAL, foreign keys, a busy timeout, and full synchronous durability. The database path is the sole new runtime option: `ARCHIVE_FORWARDER_SPOOL_PATH`, defaulting locally to `./archive-forwarder-spool.sqlite`. Production deployment mounts that path on persistent storage. The archive forwarder option is deliberately not a `CEX_BROKER_*` variable. + +Alternative considered: a JSONL spool. It is easy to append but requires custom indexes, atomic per-table completion, compaction, corruption recovery, and quota accounting that SQLite already supplies. + +### 3. Capacity and retention are fixed contract values + +The spool admits at most 1 GiB of accounted payload and metadata and retains pending work for at most 72 hours. These values are constants, not environment knobs. Admission performs expiry cleanup and quota reservation in the same serialized transaction. Quota exhaustion returns 429 without accepting ownership; an unavailable/unwritable spool returns 503. + +Expired work is terminally recorded in bounded telemetry before deletion. The API never returns 202 for a batch that was not durably committed. + +Alternative considered: configurable values. Cross-service completion needs one predictable delivery envelope; deployment-specific tuning would make Maker's effective loss window unknowable. + +### 4. Retry state is per batch and table + +Each admitted batch is split into at most five work records. A successful table insert marks only that work record complete. Failed tables retry without replaying successful siblings. Once all work completes, the batch and its work records are deleted transactionally. + +Transient failures retry after 1, 2, 4, 8, 16, 32, then 60 seconds, capped at 60 seconds, with ±20% jitter until expiry. Schema/authentication and other permanent ClickHouse errors are retained as terminal failures until expiry and exposed through health/telemetry rather than hot-looped. + +Each table work item receives a stable SHA-256 deduplication token derived from its persisted batch identity and table name. The ClickHouse insert sets `insert_deduplication_token`; strategy tables enable a sufficiently large non-replicated deduplication window so an ambiguous insert can be retried without duplicating rows. + +Alternative considered: whole-envelope retry. It is simpler but repeats already-successful tables and cannot isolate one poisoned table. + +### 5. Wire versions are explicit and backward compatible + +Missing/empty `schema_version` and version `1` remain accepted as legacy rows. Version `2` requires non-empty `producer_id`, `producer_run_id`, `stream_name`, and `archive_event_id`, plus positive integer `stream_seq` and `seq`. Unknown versions fail with HTTP 400. Envelope `source` and `deployment_id` must be non-empty, rows must be non-empty and no more than 1000, and row source/deployment values must agree when supplied. + +The Maker golden envelope is copied byte-for-byte into the CEX fixture and guarded by a cross-repository checksum/equality test. Schema changes are additive: v2 identity fields are added with compatibility defaults, along with the pinned policy and identity/mapping columns. Historical v1 rows remain readable. + +### 6. Health distinguishes acceptance from drainage + +Spool health tests open/write/transaction capability and exposes queued batches, queued table work, accounted bytes, oldest age, expired/terminal counts, and last error class using bounded labels. ClickHouse-down plus spool-healthy returns HTTP 200 with degraded status because the forwarder can still durably accept Maker work. Spool-unhealthy returns HTTP 503. Non-strategy direct traffic retains its existing synchronous response contract. + +### 7. Release evidence closes both repositories + +CEX Broker publishes the smallest unused patch above `0.2.36`, with package and image using the same version and recorded digest. Mandatory CI includes contract, SQLite fault/restart, and real ClickHouse 24.8 tests. Maker then updates its package/lock pin, reruns its strict validation, records the CEX PR/Actions/version/digest evidence, and completes task 8.6. FIET-909 receives the conformance evidence; FIET-937 retains the separate production observation gate. + +## Risks / Trade-offs + +- [SQLite file is on ephemeral storage] -> Production compose/runbooks require a persistent mount and health reports the resolved path class without leaking secrets. +- [Quota fills during a long ClickHouse outage] -> Fixed admission accounting returns 429 before ownership and exposes depth/oldest-age metrics for alerting. +- [ClickHouse commits then the worker crashes] -> Stable per-table deduplication tokens and non-replicated dedup settings suppress retry duplicates. +- [A malformed Maker row poisons retries] -> Validate the versioned contract before admission and classify permanent ClickHouse errors as terminal rather than hot-looping. +- [Schema drift reappears] -> Pin the exact Maker fixture and require both repositories' tests against the same v2 field set. +- [SQLite corruption prevents recovery] -> Fail closed with 503, report spool unhealthy, preserve the file for operator recovery, and never fall back to memory-only acceptance. +- [Fixed retention expires evidence during an extended outage] -> Alert on oldest age and terminal expiry; 72 hours is the explicit service SLO rather than hidden indefinite retention. + +## Migration Plan + +1. Apply additive `strategy_data.*` DDL and deduplication settings on ClickHouse 24.8. +2. Deploy the forwarder with a persistent `ARCHIVE_FORWARDER_SPOOL_PATH`; startup validates/migrates the SQLite schema before listening. +3. Verify legacy/v1/v2 contract cases and an all-five-table v2 envelope against real ClickHouse. +4. Exercise ClickHouse outage, restart recovery, partial-table retry, quota, retention, and corruption/write-failure behavior. +5. Publish the coordinated CEX Broker package/image version and evidence. +6. Update Maker's dependency/lock/evidence at the pinned branch and complete its remaining conformance task. +7. Roll back the application only after stopping admission and preserving the spool volume; the additive ClickHouse columns remain compatible with the prior forwarder. + +## Open Questions + +None. The wire shape, ownership boundary, storage choice, quota, retention, retry schedule, and release boundary are fixed by this change. diff --git a/openspec/changes/maker-archive-forwarder-conformance/proposal.md b/openspec/changes/maker-archive-forwarder-conformance/proposal.md new file mode 100644 index 0000000..0920a6d --- /dev/null +++ b/openspec/changes/maker-archive-forwarder-conformance/proposal.md @@ -0,0 +1,32 @@ +## Why + +Maker emits replay-critical `hb_runtime` observations once and does not own downstream retries, while the archive forwarder currently acknowledges only after direct ClickHouse insertion and can partially commit a multi-table batch. The CEX Broker must provide a versioned, durable acceptance boundary so Maker delivery is recoverable, schema-conformant, and independently retryable without changing the broker's other archive paths. + +## What Changes + +- Pin the Maker-to-forwarder envelope, five-table allowlist, schema-version rules, and v2 producer/stream identities using a shared golden fixture. +- Add an atomic SQLite spool for conforming `hb_runtime` strategy batches, acknowledge durable acceptance with HTTP 202, and keep all non-strategy archive traffic on the existing direct ClickHouse path. +- Enforce a fixed 1 GiB spool quota and 72-hour retention, restart recovery, per-table completion, stable ClickHouse deduplication tokens, isolated exponential retry, and explicit 429/503 failure semantics. +- Add additive `strategy_data.*` schema-v2 columns and ClickHouse non-replicated deduplication settings without rewriting historical rows. +- Expose spool health and bounded telemetry; ClickHouse unavailability degrades health while a writable spool remains available and does not make admission unavailable. +- Make real ClickHouse 24.8 and SQLite durability/fault coverage mandatory in CI, and publish matching CEX Broker package/image version evidence for Maker task 8.6. +- Keep the production observation window as FIET-937's deployment cutover gate rather than an OpenSpec implementation task. + +## Capabilities + +### New Capabilities + +- `strategy-runtime-archive-ingestion`: Versioned Maker `hb_runtime` envelope, table, field, source-isolation, and ClickHouse schema contract. +- `archive-forwarder-durable-acceptance`: Durable strategy-only admission, bounded spool, retry, restart recovery, health, and operational error contract. + +### Modified Capabilities + +- None. + +## Impact + +- `services/archive-forwarder/` request validation, health, telemetry, ClickHouse insertion, and a new Bun SQLite spool/worker. +- `schema/clickhouse/strategy_data.sql`, the Maker golden fixture, Docker deployment mounts, and archive-forwarder operations documentation. +- CI gains a required ClickHouse 24.8 service and non-skipping integration contract. +- CEX Broker package/image release metadata and the Maker dependency/lock/evidence record are coordinated across repositories. +- No protobuf, broker RPC, credential selection, trading handler, market-data archival, or core `CEX_BROKER_*` environment configuration changes. diff --git a/openspec/changes/maker-archive-forwarder-conformance/specs/archive-forwarder-durable-acceptance/spec.md b/openspec/changes/maker-archive-forwarder-conformance/specs/archive-forwarder-durable-acceptance/spec.md new file mode 100644 index 0000000..dc1b4c9 --- /dev/null +++ b/openspec/changes/maker-archive-forwarder-conformance/specs/archive-forwarder-durable-acceptance/spec.md @@ -0,0 +1,123 @@ +## ADDED Requirements + +### Requirement: Accepted Maker batches are durable before acknowledgement + +The forwarder MUST atomically persist a validated Maker envelope and one pending work item per represented strategy table before returning HTTP 202. It MUST NOT acknowledge acceptance when the spool transaction has not committed. + +#### Scenario: Strategy batch is durably admitted +- **WHEN** a conforming Maker batch commits to the SQLite spool +- **THEN** the forwarder MUST return HTTP 202 without waiting for ClickHouse + +#### Scenario: ClickHouse is unavailable during admission +- **WHEN** ClickHouse is unavailable but the spool is writable and within quota +- **THEN** the forwarder MUST still durably admit the batch and return HTTP 202 + +#### Scenario: Spool is unavailable +- **WHEN** the spool cannot open, write, or commit +- **THEN** the forwarder MUST return HTTP 503 and MUST NOT claim ownership + +### Requirement: Spool bounds are fixed and atomic + +The strategy spool SHALL use a fixed 1 GiB quota and fixed 72-hour retention. Admission accounting, expiry cleanup, and quota reservation MUST be serialized so concurrent requests cannot over-admit. Neither value SHALL be configurable through environment variables. + +#### Scenario: Admission fits quota +- **WHEN** the committed spool bytes plus the new batch's deterministic accounted bytes do not exceed 1 GiB +- **THEN** the batch MUST be eligible for atomic admission + +#### Scenario: Admission exceeds quota +- **WHEN** accepting a batch would exceed the fixed quota +- **THEN** the forwarder MUST return HTTP 429 without persisting any part of that batch + +#### Scenario: Work reaches retention limit +- **WHEN** pending or terminal work reaches 72 hours since admission +- **THEN** it MUST be expired transactionally and the loss MUST be recorded in bounded telemetry + +### Requirement: Spool storage is restart recoverable + +The forwarder MUST use Bun SQLite with WAL, foreign keys, a busy timeout, and full synchronous durability. `ARCHIVE_FORWARDER_SPOOL_PATH` SHALL select the database path and SHALL default to `./archive-forwarder-spool.sqlite`; production documentation MUST require persistent storage for that path. + +#### Scenario: Forwarder restarts with pending work +- **WHEN** a process exits after acknowledgement and restarts with the same spool path +- **THEN** every incomplete table work item MUST remain eligible for drainage + +#### Scenario: Local path is not configured +- **WHEN** the service starts without `ARCHIVE_FORWARDER_SPOOL_PATH` +- **THEN** it MUST use `./archive-forwarder-spool.sqlite` + +#### Scenario: SQLite is corrupt or read-only +- **WHEN** startup or a health transaction detects corruption or inability to write +- **THEN** strategy admission MUST fail closed with HTTP 503 and health MUST report the spool unhealthy + +### Requirement: Delivery retries are isolated and idempotent + +The worker MUST track completion independently for each table in a batch. Transient failures MUST retry after exponential delays starting at 1 second, doubling to a 60-second cap, with ±20 percent jitter until retention expiry. Successful siblings MUST NOT be retried. + +Each table work item MUST use one stable ClickHouse `insert_deduplication_token`, and each strategy table MUST enable non-replicated deduplication over a window large enough for the admitted delivery horizon. + +#### Scenario: One table fails after siblings succeed +- **WHEN** a multi-table batch inserts some table groups and one table returns a transient error +- **THEN** only the failed table work item MUST be rescheduled + +#### Scenario: Insert outcome is ambiguous across restart +- **WHEN** ClickHouse commits a table insert but the worker exits before recording completion +- **THEN** the restarted retry MUST use the same deduplication token and MUST not create duplicate logical delivery rows + +#### Scenario: Permanent ClickHouse failure occurs +- **WHEN** a table insert fails with a schema, authentication, or other classified permanent error +- **THEN** the work MUST become terminal, MUST NOT hot-loop, and MUST remain observable until expiry + +#### Scenario: All table work completes +- **WHEN** every represented table work item is marked complete +- **THEN** the spool MUST transactionally remove the completed batch and its work records + +### Requirement: Health reports acceptance capacity separately from ClickHouse drainage + +Health MUST expose spool writability, queued batches, queued table work, accounted bytes, oldest age, terminal/expired work, and a bounded last-error class. ClickHouse unavailability with a healthy spool MUST return HTTP 200 with degraded status; an unhealthy spool MUST return HTTP 503. + +#### Scenario: ClickHouse is down and spool is healthy +- **WHEN** the health endpoint cannot ping ClickHouse but can validate spool writability +- **THEN** it MUST return HTTP 200 and report degraded drainage with durable admission available + +#### Scenario: Spool is unhealthy +- **WHEN** the health endpoint cannot validate the spool +- **THEN** it MUST return HTTP 503 even if ClickHouse is reachable + +### Requirement: Durable acceptance emits bounded operational telemetry + +The forwarder MUST record bounded-cardinality counters and gauges for admitted/rejected strategy batches and rows, quota rejections, spool failures, pending work, accounted bytes, oldest age, retry attempts, table completions, terminal failures, expirations, and last successful drain. + +#### Scenario: Retry telemetry is emitted +- **WHEN** one table work item is rescheduled +- **THEN** retry metrics MUST use only approved table and bounded error-class labels + +#### Scenario: Untrusted request values are submitted +- **WHEN** a client supplies arbitrary source, table, deployment, or error text +- **THEN** unbounded values MUST NOT become persistent metric labels + +### Requirement: CI proves the durability and ClickHouse contract + +Required CI MUST run the legacy/v1/v2/mixed/unknown/missing-identity/all-five-table matrix, SQLite quota/expiry/restart/corruption/write-failure/partial-retry tests, and real ClickHouse 24.8 schema/insert/deduplication tests. ClickHouse integration tests MUST fail rather than skip when the required CI service is unavailable. + +#### Scenario: ClickHouse service is missing in CI +- **WHEN** the mandatory integration job cannot reach ClickHouse 24.8 +- **THEN** the job MUST fail + +#### Scenario: Durable fault matrix passes +- **WHEN** repository CI completes successfully +- **THEN** it MUST provide evidence for quota, retention, restart recovery, partial retry isolation, spool failure, and deduplicated real inserts + +### Requirement: Release evidence closes the cross-service contract + +CEX Broker and Maker completion MUST reference a published CEX Broker package and image with the same unused patch version above `0.2.36`, an immutable image digest, the CEX PR and required Actions evidence, Maker's updated dependency and lockfile, and strict validation in both repositories. + +#### Scenario: CEX conformance release is published +- **WHEN** the CEX implementation and required CI pass +- **THEN** package and image evidence MUST identify the same version and immutable digest + +#### Scenario: Maker task 8.6 is completed +- **WHEN** Maker consumes the conformance release and its strict checks pass +- **THEN** its OpenSpec evidence MUST record the CEX commit/PR/Actions/version/digest and task 8.6 MAY be marked complete + +#### Scenario: Production observation has not run +- **WHEN** repository conformance is complete but FIET-937's observation window remains pending +- **THEN** this change MAY complete while FIET-937 continues to block deployment cutover diff --git a/openspec/changes/maker-archive-forwarder-conformance/specs/strategy-runtime-archive-ingestion/spec.md b/openspec/changes/maker-archive-forwarder-conformance/specs/strategy-runtime-archive-ingestion/spec.md new file mode 100644 index 0000000..3794d50 --- /dev/null +++ b/openspec/changes/maker-archive-forwarder-conformance/specs/strategy-runtime-archive-ingestion/spec.md @@ -0,0 +1,95 @@ +## ADDED Requirements + +### Requirement: Strategy runtime requests use a closed envelope contract + +The archive forwarder MUST accept a strategy runtime request only when its non-empty envelope source is `hb_runtime`, its deployment id is non-empty, it contains between one and 1000 rows, and every row targets an approved strategy runtime table. + +The approved table set SHALL be exactly `strategy_data.policy_evaluation_events`, `strategy_data.strategy_policy_snapshots`, `strategy_data.market_identity`, `strategy_data.symbol_mapping`, and `strategy_data.inventory_settlement_events`. + +#### Scenario: All five strategy tables are submitted +- **WHEN** one `hb_runtime` envelope contains conforming rows for all five approved tables +- **THEN** the request MUST pass strategy contract validation + +#### Scenario: Strategy and non-strategy rows are mixed +- **WHEN** an `hb_runtime` envelope contains an approved strategy table and any non-strategy table +- **THEN** the entire request MUST be rejected with HTTP 400 before durable admission or ClickHouse insertion + +#### Scenario: Strategy table uses another source +- **WHEN** an envelope under `broker_read`, `broker_write`, or another source targets an approved strategy table +- **THEN** the entire request MUST be rejected with HTTP 400 + +#### Scenario: Envelope identity is empty +- **WHEN** source or deployment id is missing, empty, or whitespace-only +- **THEN** the request MUST be rejected with HTTP 400 + +### Requirement: Strategy schema versions are validated before ownership + +Each strategy row MUST use a missing/empty legacy version, version `1`, or version `2`. Unknown versions MUST be rejected with HTTP 400 before the forwarder accepts ownership. + +Version `2` rows MUST contain non-empty `producer_id`, `producer_run_id`, `stream_name`, and `archive_event_id`, and positive UInt64-compatible integer `stream_seq` and `seq` values. + +#### Scenario: Legacy schema version is omitted +- **WHEN** a conforming strategy row omits `schema_version` or supplies an empty value +- **THEN** the forwarder MUST admit it under the legacy compatibility contract + +#### Scenario: Version one row is submitted +- **WHEN** a conforming strategy row supplies `schema_version=1` +- **THEN** the forwarder MUST admit it without requiring v2 producer and stream fields + +#### Scenario: Version two identity is complete +- **WHEN** a version `2` row supplies every required producer, stream, event, and sequence identity +- **THEN** the forwarder MUST admit it + +#### Scenario: Version two identity is incomplete +- **WHEN** any required v2 identity is blank, missing, non-integral, zero, negative, or outside UInt64 range +- **THEN** the entire request MUST be rejected with HTTP 400 + +#### Scenario: Unknown version is submitted +- **WHEN** a strategy row supplies any schema version other than legacy/empty, `1`, or `2` +- **THEN** the entire request MUST be rejected with HTTP 400 + +### Requirement: Envelope and row provenance agree + +When a strategy row supplies `source` or `deployment_id`, its value MUST equal the corresponding envelope value. Credentials and broker configuration MUST NOT supply or override these Maker identities. + +#### Scenario: Row source differs from envelope +- **WHEN** a row declares a source other than the envelope source +- **THEN** the entire request MUST be rejected before admission + +#### Scenario: Row deployment differs from envelope +- **WHEN** a row declares a deployment id other than the envelope deployment id +- **THEN** the entire request MUST be rejected before admission + +### Requirement: Strategy tables implement the pinned additive schema + +All five strategy tables MUST expose compatibility-defaulted `producer_id`, `producer_run_id`, `stream_name`, `stream_seq`, `seq`, and `archive_event_id` columns. Policy rows MUST expose the pinned policy decision fields, and policy snapshots, market identity, and symbol mapping MUST expose the pinned content/revision/market mapping fields. + +DDL upgrades MUST use additive idempotent migration statements so existing v1 data remains readable and a new deployment writes the latest schema. + +#### Scenario: Existing v1 tables are upgraded +- **WHEN** schema initialization runs against strategy tables created by the prior release +- **THEN** every pinned v2 column MUST be added without rewriting or dropping historical rows + +#### Scenario: Fresh strategy database is initialized +- **WHEN** schema initialization runs on an empty ClickHouse 24.8 database +- **THEN** all five tables MUST be insert-compatible with the pinned Maker v2 rows + +### Requirement: The Maker fixture is the cross-repository wire evidence + +The CEX Broker contract fixture MUST equal the pinned Maker `archive_forwarder_envelope.json` and contract tests MUST validate every fixture row without local field translation. + +#### Scenario: Pinned Maker fixture is checked in CEX Broker +- **WHEN** the cross-repository contract test runs +- **THEN** it MUST prove fixture equality and successful v2 validation + +#### Scenario: Fixture contract drifts +- **WHEN** Maker or CEX changes a table, schema version, or required identity without the matching repository change +- **THEN** a required contract check MUST fail + +### Requirement: Non-strategy archive behavior remains compatible + +Requests that contain no strategy table and do not use `hb_runtime` MUST retain the existing direct ClickHouse insertion and synchronous success/failure contract. + +#### Scenario: Broker market rows are submitted +- **WHEN** a valid `broker_read` or `broker_write` request contains only supported non-strategy rows +- **THEN** the forwarder MUST insert them through the existing direct path and MUST NOT consume strategy spool quota diff --git a/openspec/changes/maker-archive-forwarder-conformance/tasks.md b/openspec/changes/maker-archive-forwarder-conformance/tasks.md new file mode 100644 index 0000000..e1f6df5 --- /dev/null +++ b/openspec/changes/maker-archive-forwarder-conformance/tasks.md @@ -0,0 +1,50 @@ +## 1. Cross-Repository Contract + +- [x] 1.1 Copy the pinned Maker v2 archive envelope fixture byte-for-byte and add a fixture equality guard against Maker commit `563594435853c88cca5b187b8c999f845e31136b`. +- [x] 1.2 Add contract tests for legacy/empty, v1, v2, unknown version, missing v2 identity, mixed table, mixed source, empty envelope identity, and all-five-table requests. +- [x] 1.3 Implement a closed `hb_runtime` strategy request classifier and versioned row validator without changing non-strategy routing. + +## 2. Strategy ClickHouse Schema + +- [x] 2.1 Add the six common v2 producer/stream identity columns to all five strategy table create statements and additive migrations. +- [x] 2.2 Add the pinned policy-decision columns and content/revision/market-mapping columns to their respective tables and migrations. +- [x] 2.3 Enable a non-replicated insert deduplication window on all five strategy tables and make schema application idempotent on legacy databases. +- [x] 2.4 Add real ClickHouse tests for fresh schema, legacy upgrade, v1/v2 inserts, all five tables, and stable-token deduplication. + +## 3. Durable SQLite Admission + +- [x] 3.1 Add a Bun SQLite spool schema with atomic batch/work records, deterministic byte accounting, per-table state, dedupe tokens, and startup migration. +- [x] 3.2 Configure WAL, foreign keys, busy timeout, and full synchronous durability; add `ARCHIVE_FORWARDER_SPOOL_PATH` with local fallback `./archive-forwarder-spool.sqlite`. +- [x] 3.3 Enforce fixed 1 GiB quota and 72-hour retention in serialized admission/cleanup transactions with no environment overrides. +- [x] 3.4 Return HTTP 202 only after durable strategy admission, HTTP 429 on quota exhaustion, HTTP 503 on spool failure, and HTTP 400 for contract failures. +- [x] 3.5 Preserve direct synchronous ClickHouse behavior for every non-strategy request. + +## 4. Drainage And Recovery + +- [x] 4.1 Extend the ClickHouse inserter to accept a stable per-table `insert_deduplication_token`. +- [x] 4.2 Implement a worker that drains due table work independently and transactionally removes fully completed batches. +- [x] 4.3 Implement transient retry delays of 1/2/4/8/16/32/60 seconds with ±20 percent jitter through expiry. +- [x] 4.4 Classify permanent failures as terminal and retain them observably until expiry without hot-looping. +- [x] 4.5 Start and stop the worker with the forwarder lifecycle and recover pending work from the configured spool after restart. +- [x] 4.6 Add deterministic tests for partial success, isolated retry, ambiguous commit dedupe token reuse, restart recovery, expiry, and completion cleanup. + +## 5. Health, Telemetry, And Deployment + +- [x] 5.1 Extend health with spool writability and bounded queue/age/byte/terminal state; return degraded HTTP 200 for ClickHouse-down/spool-healthy and 503 for spool-unhealthy. +- [x] 5.2 Add bounded telemetry for admissions, rejections, quota, spool failures, depth, bytes, oldest age, retry, completion, terminal failure, expiry, and successful drain. +- [x] 5.3 Add fault tests for quota boundaries, concurrent admission, corruption, read-only/write/commit failures, and telemetry label bounds. +- [x] 5.4 Document the ownership boundary, fixed SLOs, retry/error semantics, local fallback, and required production persistent volume. +- [x] 5.5 Update Docker/compose deployment to persist the spool without introducing any new core `CEX_BROKER_*` environment option. + +## 6. Mandatory Verification + +- [x] 6.1 Add a required ClickHouse 24.8 CI service/job and make integration availability failure fatal in CI. +- [x] 6.2 Run the forwarder unit/contract/fault suite, real ClickHouse integration suite, repository tests, lint, formatting check, type-check/build, and strict OpenSpec validation. +- [x] 6.3 Verify the implementation against both new capability specs and resolve every critical or warning-level divergence. + +## 7. Coordinated Release And Maker Closure + +- [ ] 7.1 Select the smallest unused patch above `0.2.36`, update CEX package/image metadata, and publish matching version/digest evidence after required Actions pass. +- [ ] 7.2 Update Maker's CEX Broker dependency and lockfile, run Maker strict checks, record CEX commit/PR/Actions/version/digest evidence, and complete Maker task 8.6. +- [ ] 7.3 Update FIET-937 with the retained production observation cutover gate, link FIET-901/FIET-903 to the scope transition, and comment FIET-909 with conformance evidence while leaving FIET-924 closed. +- [ ] 7.4 Sync and archive this change only after both repositories satisfy the cross-service definition of done. diff --git a/openspec/specs/cex-broker-order-book-depth-sourcing/spec.md b/openspec/specs/cex-broker-order-book-depth-sourcing/spec.md new file mode 100644 index 0000000..de61a00 --- /dev/null +++ b/openspec/specs/cex-broker-order-book-depth-sourcing/spec.md @@ -0,0 +1,140 @@ +# cex-broker-order-book-depth-sourcing + +## Purpose + +The cex-broker SHALL provide HB-compatible broker order-book capability discovery, normalized current snapshots, backward-compatible live stream metadata, typed historical snapshot results, and truthful provider-scoped capability reporting without exposing credentials or private provider metadata. + +## Requirements + +### Requirement: HB-compatible order-book Call methods +The broker SHALL expose HB-compatible order-book operations through `ExecuteAction` with `Action.Call` without removing the existing generic CCXT Call behavior. + +#### Scenario: Maker method payload dispatches order-book capability +- **WHEN** a client sends `Action.Call` with payload field `method = "fetch_order_book_capability"` +- **THEN** the broker MUST return a JSON capability object for the requested `cex`, `symbol`, `depthLimit`, and `constructionMode` +- **AND** the broker MUST NOT attempt to invoke a CCXT method named `fetch_order_book_capability` + +#### Scenario: Maker method payload dispatches current snapshot +- **WHEN** a client sends `Action.Call` with payload field `method = "fetch_order_book_snapshot"` +- **THEN** the broker MUST fetch or construct a current top-N order-book snapshot for the requested exchange and symbol +- **AND** the response MUST be decodable by Maker's `fetch_order_book_snapshot` helper + +#### Scenario: Maker method payload dispatches historical snapshots +- **WHEN** a client sends `Action.Call` with payload field `method = "fetch_historical_order_book_snapshots"` +- **THEN** the broker MUST return either typed historical snapshots or a typed unsupported historical response +- **AND** the client MUST NOT need to infer historical support from a gRPC transport failure or an empty stream + +#### Scenario: Existing generic Call behavior remains available +- **WHEN** a client sends `Action.Call` for a non-order-book CCXT method using the existing generic call payload shape +- **THEN** the broker MUST continue to validate and invoke the generic CCXT call according to existing behavior + +#### Scenario: Malformed order-book Call payload is rejected +- **WHEN** an order-book Call payload omits required fields or provides an invalid `depthLimit`, time window, cadence, or construction mode +- **THEN** the broker MUST fail the request with a typed validation error +- **AND** the broker MUST NOT call the provider with partially validated input + +### Requirement: Current order-book snapshots are normalized top-N payloads +The broker SHALL return current order-book snapshots using a normalized JSON object that preserves raw `bids` and `asks` arrays and includes source metadata required by Maker HB strategy compatibility. + +#### Scenario: Current snapshot includes required fields +- **WHEN** `fetch_order_book_snapshot` succeeds for an exchange, symbol, and depth limit +- **THEN** the response MUST include `bids`, `asks`, `timestamp`, `receivedTimestamp`, `exchange`, `symbol`, and `depthLimit` +- **AND** each bid and ask level MUST be a two-item numeric `[price, amount]` array + +#### Scenario: Sequence metadata is preserved when available +- **WHEN** the provider order book includes `sequence`, `updateId`, `lastUpdateId`, or `nonce` +- **THEN** the broker MUST include an equivalent sequence/update identifier in the normalized snapshot + +#### Scenario: Snapshot depth honors requested depth limit +- **WHEN** a client requests `depthLimit = N` +- **THEN** the broker MUST request or truncate each side to at most N levels +- **AND** it MUST report `depthLimit = N` in the response + +#### Scenario: Exchange and symbol are not substituted +- **WHEN** a client requests an order-book snapshot for a specific `cex` and `symbol` +- **THEN** the response MUST identify the same exchange and symbol +- **AND** the broker MUST NOT silently substitute another exchange or trading pair + +### Requirement: Live ORDERBOOK stream remains backward compatible +The broker SHALL preserve existing `Subscribe(ORDERBOOK)` behavior while allowing enriched order-book metadata for Maker live envelope compatibility. + +#### Scenario: Existing raw order-book consumers still decode stream data +- **WHEN** a client subscribes to `Subscribe(ORDERBOOK)` +- **THEN** each successful frame's `data` JSON MUST continue to contain `bids` and `asks` arrays at the top level +- **AND** old clients that only read `bids` and `asks` MUST continue to work + +#### Scenario: Stream frames include metadata for new clients +- **WHEN** the provider returns an order-book update with timestamp or sequence metadata +- **THEN** the broker MUST include equivalent metadata in the frame `data` JSON without removing the existing top-level `bids` and `asks` +- **AND** the `SubscribeResponse.timestamp` MUST record the broker received timestamp + +#### Scenario: Omitted or legacy subscription type remains orderbook +- **WHEN** a client omits subscription type or sends `NO_ACTION` +- **THEN** the broker MUST continue resolving the request to `ORDERBOOK` +- **AND** this compatibility behavior MUST be covered by tests + +#### Scenario: Stream error frames remain explicit +- **WHEN** the broker cannot continue an order-book stream +- **THEN** it MUST emit an error frame or stream error that identifies the failure +- **AND** it MUST NOT emit an empty successful order-book frame as a failure substitute + +### Requirement: Historical order-book snapshots return typed success or typed unsupported +The broker SHALL define historical sampled top-N order-book behavior as a typed result contract even when no provider currently supports the requested historical data. + +#### Scenario: Historical snapshot success payload +- **WHEN** the broker supports sampled top-N historical snapshots for the requested exchange, symbol, start, end, cadence, construction mode, and depth limit +- **THEN** the response MUST include `exchange`, `symbol`, and a `snapshots` array +- **AND** each snapshot MUST satisfy the normalized current snapshot field requirements + +#### Scenario: Historical unsupported payload +- **WHEN** the broker does not support the requested historical snapshots +- **THEN** the response MUST include `exchange`, `symbol`, `unsupported = true`, and `unsupportedReason = "historical_order_book_provider_unsupported"` +- **AND** the broker MUST return this as a successful `ActionResponse.result` JSON payload rather than a transport failure + +#### Scenario: Live book is not used as historical evidence +- **WHEN** a client requests historical snapshots for a past time window +- **THEN** the broker MUST NOT satisfy the request by sampling the current live order book after the requested window +- **AND** unavailable historical support MUST be reported as typed unsupported + +#### Scenario: Exact reconstruction request is unsupported without continuity +- **WHEN** a client requests `constructionMode = "exact_l2_reconstruction"` and the broker lacks a validated snapshot-plus-delta reconstruction path +- **THEN** the broker MUST return typed unsupported or report exact reconstruction as unsupported in capability discovery +- **AND** it MUST NOT downgrade the request to sampled top-N without the client changing construction mode + +### Requirement: Capability discovery is truthful and provider-scoped +The broker SHALL report order-book capabilities conservatively from actual broker/provider behavior for the requested exchange, symbol, construction mode, and depth limit. + +#### Scenario: Current and live support reflect provider methods +- **WHEN** `fetch_order_book_capability` is called +- **THEN** `supportsCurrentSnapshot` MUST be true only when the broker can fetch a current order-book snapshot for the requested exchange and symbol +- **AND** `supportsLiveStream` MUST be true only when the broker can stream order-book updates for the requested exchange and symbol + +#### Scenario: Historical support is not inferred from current support +- **WHEN** the provider supports current order-book snapshots but does not support historical sampled top-N snapshots for the requested parameters +- **THEN** the broker MUST set `supportsHistoricalSnapshots = false` +- **AND** it MUST set `supportsSampledTopN = false` for that historical capability + +#### Scenario: Exact L2 reconstruction is not advertised prematurely +- **WHEN** the broker has not implemented snapshot-plus-delta reconstruction with sequence-continuity validation +- **THEN** `supportsExactL2Reconstruction` MUST be false + +#### Scenario: MEXC capability uses CCXT provider identity +- **WHEN** capability is requested for MEXC +- **THEN** the broker MUST report a CCXT-backed provider identity for current/live order-book support when using CCXT +- **AND** historical support MUST remain false unless real CCXT-backed historical top-N snapshots are implemented for the requested parameters + +#### Scenario: Binance native Hummingbot capability is not fabricated +- **WHEN** capability is requested for Binance in this TypeScript broker +- **THEN** the broker MUST NOT advertise native Hummingbot historical support unless a real native Hummingbot adapter is implemented +- **AND** it MUST report CCXT-backed broker support only for capabilities actually available through the broker + +### Requirement: Order-book responses do not leak secrets +The broker SHALL keep order-book response payloads free of raw credentials, API secrets, authorization metadata, and private provider configuration. + +#### Scenario: Snapshot response excludes credentials +- **WHEN** a current, live, historical, or capability order-book response is emitted +- **THEN** the response MUST NOT include API keys, API secrets, request signing material, authorization headers, or secret-backed metadata values + +#### Scenario: Unsupported response excludes credentials +- **WHEN** a historical request returns typed unsupported +- **THEN** the unsupported response MUST include only non-secret diagnostic fields such as exchange, symbol, provider identity, and unsupported reason diff --git a/openspec/specs/cex-market-data-replay-capture/spec.md b/openspec/specs/cex-market-data-replay-capture/spec.md new file mode 100644 index 0000000..99baab0 --- /dev/null +++ b/openspec/specs/cex-market-data-replay-capture/spec.md @@ -0,0 +1,210 @@ +# cex-market-data-replay-capture Specification + +## Purpose +TBD - created by archiving change canonical-cex-market-data-replay-archive. Update Purpose after archive. +## Requirements +### Requirement: Broker deployments retain the full service contract +The system SHALL register the same `ExecuteAction` and `Subscribe` RPC contract in TEE and non-TEE broker deployments, and SHALL NOT use an RPC/action allowlist as the mechanism that distinguishes read-only from write-capable deployments. + +#### Scenario: Non-TEE broker starts with the full service +- **WHEN** a non-TEE broker is started with exchange credentials whose permissions are read-only +- **THEN** it MUST register the same broker RPCs and action handlers as a TEE broker +- **AND** public market-data calls MUST remain available through the normal broker contract + +#### Scenario: Deployment archive role does not change RPC registration +- **WHEN** the archive source is configured as `broker_read` or `broker_write` +- **THEN** the selected value MUST affect archival provenance only +- **AND** it MUST NOT add or remove broker RPC methods + +### Requirement: Credential resolution follows fixed source precedence +The system SHALL preserve the broker's established credential resolution order without adding archive-specific credential profiles, credential-source policy, or permission-attestation configuration. A matching environment-loaded broker SHALL take precedence over request-supplied credentials, request metadata SHALL be considered only when no matching environment broker exists, and credentialless public construction SHALL remain the final fallback only for operations that already support public access. + +The selected exchange key's actual permissions SHALL establish effective privilege. The broker SHALL NOT infer a read-only or write-capable profile from archive source, deployment type, or credential location. + +#### Scenario: Environment and request credentials are both present +- **WHEN** a matching environment-loaded broker exists and a request also supplies `api-key` or `api-secret` metadata +- **THEN** the environment-loaded broker MUST be selected +- **AND** request credentials MUST NOT replace it + +#### Scenario: Only request credentials are present +- **WHEN** no matching environment-loaded broker exists and a request supplies a complete supported credential pair +- **THEN** the broker MUST construct the exchange from the request credentials using the existing in-flight path +- **AND** archival configuration MUST NOT reject that source + +#### Scenario: No credentials are present for a public operation +- **WHEN** neither environment nor request credentials are available and the requested operation supports public exchange access +- **THEN** the broker MAY construct a credentialless public exchange +- **AND** it MUST retain the same RPC and action surface + +#### Scenario: A non-TEE deployment is intended to be read-only +- **WHEN** credentials are supplied through either supported source +- **THEN** deployment and secret provisioning MUST restrict the exchange key to the intended permissions +- **AND** the archive plane MUST NOT add a credential profile, source-policy, or attestation environment variable to classify that key + +### Requirement: Archive source identity is deployment controlled +The archive plane SHALL support the closed source identities `broker_read` and `broker_write`, selected from immutable deployment configuration and propagated consistently to every row and batch envelope. + +#### Scenario: Read broker archives a market frame +- **WHEN** a broker configured with archive source `broker_read` archives a market-data frame +- **THEN** every normalized row and its archive envelope MUST contain `source = "broker_read"` +- **AND** clients MUST NOT be able to override that source through request fields or metadata + +#### Scenario: Envelope and row sources disagree +- **WHEN** an archive batch contains a row whose source differs from the envelope source +- **THEN** the archive forwarder MUST reject the inconsistent batch or row before ClickHouse insertion +- **AND** it MUST emit a bounded diagnostic without exposing credentials or raw secrets + +#### Scenario: FIET-901 production capture uses the read archive source +- **WHEN** the broker/archiver hosting FIET-901 production subscriptions starts with an archive source other than `broker_read` +- **THEN** production capture startup or deployment validation MUST fail before opening subscriptions +- **AND** unrelated TEE or write-broker deployments MAY continue to archive their own market observations as `broker_write` + +### Requirement: Production captures have bundle and provider provenance +Every production market-data row SHALL identify the capture bundle, deployment, exchange, trading pair, provider, feed, source mode, source timestamp, broker-received timestamp, schema version, raw-capture identity and scope, raw checksum and algorithm version, and normalized-row checksum and algorithm version using the versioned replay-capture contract. + +#### Scenario: Collector opens a configured feed +- **WHEN** the production collector opens a feed for a configured exchange and pair +- **THEN** it MUST attach the deployment-owned `capture_bundle_id` to every raw-capture and normalized row produced by that feed +- **AND** it MUST record the exact provider, feed-specific source mode, schema version, raw-capture integrity fields, and normalized-row integrity fields rather than infer them during replay + +#### Scenario: Production bundle identity is missing +- **WHEN** production collection starts without a non-empty capture bundle identity +- **THEN** startup MUST fail configuration validation +- **AND** it MUST NOT archive ungrouped production rows + +### Requirement: External fallback producers conform without expanding broker scope +The capture contract SHALL permit separately deployed direct CCXT or Hummingbot fallback producers, but this broker and collector change SHALL NOT require implementing or invoking those producers for FIET-901 completion. + +#### Scenario: Broker feed is unavailable +- **WHEN** the broker collector cannot obtain a required feed and no separately deployed fallback producer is configured +- **THEN** it MUST record the feed as unavailable +- **AND** it MUST NOT instantiate an in-process direct CCXT or Hummingbot fallback as part of this change + +#### Scenario: External fallback row is admitted +- **WHEN** an approved external fallback producer submits a row to the shared capture contract +- **THEN** ingestion MUST validate and preserve the configured exchange and pair +- **AND** the row MUST record the fallback provider, a versioned fallback source mode, and the fallback reason + +#### Scenario: External fallback crosses venues +- **WHEN** a fallback row identifies an exchange different from the configured strategy exchange +- **THEN** ingestion MUST reject the row +- **AND** it MUST NOT substitute or relabel the cross-venue data + +### Requirement: Raw captures and normalized rows have reproducible integrity +The archive plane SHALL assign each broker-visible capture a reproducible raw-capture identity and checksum, and SHALL assign every normalized output row a deterministic row checksum under a versioned canonicalization algorithm. + +#### Scenario: CCXT-normalized object is the earliest available capture +- **WHEN** the broker receives a CCXT-normalized object rather than exchange wire bytes +- **THEN** `raw_capture_scope` MUST identify the object as broker-visible or CCXT-normalized +- **AND** metadata MUST NOT claim that its checksum represents the original exchange wire frame + +#### Scenario: Raw capture checksum is recomputed +- **WHEN** the retained capture payload and checksum algorithm version are supplied to the verifier +- **THEN** canonical serialization and hashing MUST reproduce the stored raw-capture checksum + +#### Scenario: Normalized row checksum is recomputed across languages +- **WHEN** TypeScript and Python normalize the same contract fixture +- **THEN** both implementations MUST produce the same normalized-row checksum +- **AND** the checksum projection MUST exclude its own checksum field and ingestion-only mutable fields + +#### Scenario: Capture contains credential-like data +- **WHEN** a provider payload contains headers, keys, secrets, signatures, or credential metadata +- **THEN** the archive plane MUST redact those fields before retained-payload serialization +- **AND** no integrity field may embed the secret value + +### Requirement: The production collector supervises all required CEX feeds +The production collector SHALL supervise configured `ORDERBOOK`, `TICKER`, `TRADES`, and `OHLCV` subscriptions independently for every strategy exchange and pair. + +#### Scenario: Strategy capture configuration is loaded +- **WHEN** production collection starts with strategy exchange/pair/feed configuration +- **THEN** it MUST validate that each required feed has a supervisor and required options such as depth limit or timeframe +- **AND** failure of one feed MUST NOT terminate supervisors for other feeds or pairs + +#### Scenario: Live stream disconnects +- **WHEN** an order-book, ticker, trade, or OHLCV stream ends or errors unexpectedly +- **THEN** its supervisor MUST reconnect with bounded backoff and jitter +- **AND** it MUST expose reconnect count, last-frame time, and current feed-health state + +#### Scenario: Provider catch-up is unavailable +- **WHEN** a live order-book, ticker, or trade stream reconnects and the provider cannot replay the missing interval +- **THEN** the collector MUST record an explicit gap +- **AND** it MUST NOT synthesize missing market events + +#### Scenario: OHLCV bootstrap is supported +- **WHEN** an OHLCV subscription starts with a configured bootstrap window +- **THEN** the collector MUST fetch and archive available historical bars before or alongside the live stream +- **AND** bootstrap rows MUST use a source mode distinct from live stream rows + +### Requirement: Market capture remains outside the strategy execution dependency path +Market-data archival SHALL use bounded asynchronous queues, bounded batches, retry policy, backpressure telemetry, and persistent loss journaling without making strategy execution or stream delivery depend on ClickHouse availability. + +#### Scenario: ClickHouse or forwarder is unavailable +- **WHEN** the archive sink is temporarily unavailable +- **THEN** market-data stream delivery MUST continue independently +- **AND** archive delivery MUST retry within configured bounds before journaling undeliverable rows + +#### Scenario: Archive queue reaches capacity +- **WHEN** the bounded archive queue cannot accept another row +- **THEN** the configured shedding or durability policy MUST be applied deterministically +- **AND** queue saturation and lost/journaled row counts MUST be observable by feed and source + +#### Scenario: Broker shuts down with pending rows +- **WHEN** shutdown begins while archive rows remain queued +- **THEN** the broker MUST attempt a bounded flush +- **AND** rows still undeliverable at the deadline MUST be written to the configured persistent loss journal + +### Requirement: Canonical non-order-book market tables preserve replay provenance +The archive plane SHALL write ticker, trade, OHLCV, and raw stream records to `market_data.cex_ticker_events`, `market_data.cex_trades`, `market_data.cex_ohlcv`, and `market_data.cex_stream_events` with the common replay-capture fields and deterministic checksums. + +#### Scenario: OHLCV row is archived +- **WHEN** an OHLCV bar is normalized for archive +- **THEN** it MUST be written to `market_data.cex_ohlcv` with timeframe, open time, OHLCV values, closure/version state, capture provenance, and normalized-row checksum + +#### Scenario: Ticker or trade row is archived +- **WHEN** a ticker snapshot or public trade is normalized for archive +- **THEN** it MUST retain its feed-specific values and common capture provenance +- **AND** it MUST link to the raw capture from which it was normalized + +#### Scenario: Raw stream event is retained +- **WHEN** a broker-visible feed payload is accepted for production archive +- **THEN** `market_data.cex_stream_events` MUST record its redacted payload or reproducible capture metadata, raw-capture identity, raw checksum, and schema version + +### Requirement: Legacy candle storage requires table migration before upgrade +The system SHALL migrate retained rows from `market_data.candles` to `market_data.cex_ohlcv` through a bounded ClickHouse table-to-table migration before deploying the upgraded canonical-only broker. The upgraded broker SHALL always write the latest canonical schema and SHALL NOT expose a runtime legacy/dual/canonical write mode. + +#### Scenario: Existing deployment is prepared for upgrade +- **WHEN** an existing broker writes `market_data.candles` and the canonical-only version is scheduled for deployment +- **THEN** operators MUST apply canonical DDL, quiesce legacy writers, migrate every retained bounded partition, and validate parity before deploying the upgraded broker +- **AND** any missing row or value mismatch MUST block the upgrade + +#### Scenario: Legacy candle lacks raw capture provenance +- **WHEN** a `market_data.candles` row is migrated and its original capture bundle, raw identity, or raw checksum is unavailable +- **THEN** the canonical row MUST record legacy source mode and incomplete provenance +- **AND** unavailable raw provenance fields MUST remain null rather than be fabricated + +#### Scenario: Canonical-only broker is deployed +- **WHEN** canonical OHLCV migration and replay validation pass for every retained migration window +- **THEN** the upgraded broker MUST write new OHLCV captures only to `market_data.cex_ohlcv` +- **AND** legacy candle query names and closed-candle semantics MUST remain available through documented compatibility views for the migration retention period + +#### Scenario: Candle cutover is rolled back +- **WHEN** a blocking integrity or replay defect is found after canonical OHLCV deployment +- **THEN** operators MUST stop the upgraded broker and MAY restore the retained legacy names before rolling back to the previous legacy-writing application version +- **AND** the defect and affected capture bundles MUST remain diagnosable + +### Requirement: Capture output is replay-consumable +The system SHALL prove that canonical ClickHouse rows can be selected by exchange, pair, capture bundle, and replay window and converted into Maker-compatible inputs without losing source identity or checksum verifiability. + +#### Scenario: Replay selects a capture window +- **WHEN** a replay consumer queries one or more capture bundles for an exchange, pair, and time window +- **THEN** the query MUST return only matching source-time rows +- **AND** every returned normalized row MUST retain enough metadata to verify its capture linkage and checksum + +#### Scenario: Cross-language contract fixture is validated +- **WHEN** CI runs the broker-to-Maker replay contract fixtures +- **THEN** the ClickHouse row projection and Maker reader MUST agree on field semantics, timestamps, source modes, and checksum values + +#### Scenario: FIET-907 materializes Parquet fixtures +- **WHEN** the retained reference exporter produces Maker-compatible fixtures +- **THEN** it MUST query canonical ClickHouse views directly without calling the broker or a connected exchange +- **AND** fixture materialization, coverage, and replay-bundle assembly MUST remain FIET-907 consumer-side ownership diff --git a/openspec/specs/cex-order-book-replay-archive/spec.md b/openspec/specs/cex-order-book-replay-archive/spec.md new file mode 100644 index 0000000..c5c8bf2 --- /dev/null +++ b/openspec/specs/cex-order-book-replay-archive/spec.md @@ -0,0 +1,159 @@ +# cex-order-book-replay-archive Specification + +## Purpose +TBD - created by archiving change canonical-cex-market-data-replay-archive. Update Purpose after archive. +## Requirements +### Requirement: Canonical order-book levels use one row per level +The archive plane SHALL write sampled or reconstructed order-book depth to `market_data.cex_order_book_levels` using one row per snapshot, side, and level index with a parquet-compatible capture-core schema. + +#### Scenario: Sampled snapshot is normalized +- **WHEN** a valid top-N snapshot containing bids and asks is accepted for archive +- **THEN** the system MUST emit one level row for every retained bid and ask +- **AND** each row MUST include snapshot identity, side, zero-based level index, price, amount, mid-price, spread from mid, sequence when available, and common capture provenance + +#### Scenario: Level depth exceeds the configured limit +- **WHEN** a provider returns more levels than the configured archive depth limit +- **THEN** normalization MUST retain no more than that limit on each side +- **AND** the retained depth limit and actual side counts MUST remain auditable + +### Requirement: Canonical depth summaries are deterministic +The archive plane SHALL write one `market_data.cex_order_book_depth_summary` row per retained snapshot containing top-of-book, spread, staleness, level counts, configured measurement bands, and depth by band. + +#### Scenario: Summary is derived from canonical levels +- **WHEN** canonical levels are produced for a snapshot +- **THEN** the summary best bid, best ask, mid-price, spread, counts, and band depths MUST be computed from those retained levels +- **AND** repeated normalization of the same capture and contract version MUST produce the same summary values and checksum + +#### Scenario: Measurement bands are configured +- **WHEN** bid or ask measurement bands are configured +- **THEN** bands MUST be normalized into a deterministic ascending unique order +- **AND** depth for each band MUST be calculated using the documented base-amount convention + +### Requirement: Order-book normalization rejects invalid evidence +Order-book normalization SHALL reject missing sides, non-finite or non-positive values, non-monotonic levels, crossed or locked books, invalid timestamps, and invalid depth configuration before canonical rows are archived. + +#### Scenario: Book is crossed or locked +- **WHEN** the normalized best bid is greater than or equal to the normalized best ask +- **THEN** canonical order-book rows MUST NOT be emitted +- **AND** an invalid-row diagnostic MUST identify the reason without leaking provider secrets + +#### Scenario: Side ordering is invalid +- **WHEN** bids are not strictly descending or asks are not strictly ascending after provider normalization +- **THEN** the entire snapshot MUST be rejected before canonical rows are emitted +- **AND** the broker MUST NOT silently reorder the captured levels + +#### Scenario: Price or amount is invalid +- **WHEN** a retained level has a non-finite, zero, or negative price or amount +- **THEN** the snapshot MUST fail canonical validation +- **AND** partial valid levels MUST NOT be presented as a complete snapshot without an explicit invalid-evidence classification + +### Requirement: Order-book construction modes are evidence honest +The system SHALL classify order-book rows with versioned construction mode, source mode, and gap policy values and SHALL treat sampled top-N snapshots separately from exact L2 reconstruction. + +#### Scenario: Live broker snapshot is captured +- **WHEN** the broker archives `Subscribe(ORDERBOOK)` output without complete snapshot-plus-delta continuity evidence +- **THEN** it MUST record `construction_mode = "sampled_top_n_snapshot"` +- **AND** it MUST use the broker live-sampling source mode rather than claim exact L2 reconstruction + +#### Scenario: Exact L2 capability is unavailable +- **WHEN** the broker cannot prove initial snapshot identity, complete ordered sequence ranges, clean continuity, and valid reconstruction +- **THEN** it MUST advertise exact L2 support as false +- **AND** exact-L2 rows MUST NOT be written + +#### Scenario: Exact L2 is explicitly requested but unsupported +- **WHEN** a caller explicitly requests exact L2 reconstruction and the capability is unavailable +- **THEN** the request MUST return the typed unsupported result +- **AND** it MUST NOT silently downgrade that request to sampled evidence + +#### Scenario: Historical snapshots are unsupported +- **WHEN** broker historical order-book capability is unavailable +- **THEN** the existing typed historical unsupported result MUST remain truthful +- **AND** current or live snapshots MUST NOT be relabeled as historical evidence + +### Requirement: Exact L2 evidence carries continuity proof +If exact L2 support is implemented in the future, every exact reconstruction SHALL carry an initial snapshot identity, sequence start and end, continuity status, exact-completion flag, and fail-fast gap policy sufficient to audit the claim. + +#### Scenario: Exact reconstruction completes cleanly +- **WHEN** an initial snapshot and all ordered deltas reconstruct a valid book with no gaps +- **THEN** `exact_l2_reconstruction_complete` MAY be true only with the complete continuity metadata +- **AND** canonical levels and summary MUST reference the proven reconstruction identity + +#### Scenario: Sequence continuity breaks +- **WHEN** an expected delta, update identifier, or ordering marker is missing +- **THEN** exact reconstruction MUST fail under `gap_policy = "fail_fast"` +- **AND** no exact-L2 canonical rows may be emitted for the affected reconstruction + +### Requirement: Order-book rows link to raw captures and bundles +Every new canonical order-book level and summary row SHALL link to its capture bundle, raw capture, snapshot identity, provider, exchange, pair, replay-capture schema version, and integrity fields. + +#### Scenario: One captured snapshot produces canonical rows +- **WHEN** one broker-visible snapshot is normalized +- **THEN** its raw event, all level rows, and its summary row MUST share the same `capture_bundle_id` and `raw_capture_id` +- **AND** level and summary rows MUST carry individually reproducible normalized-row checksums + +#### Scenario: Snapshot identity is recomputed +- **WHEN** the same exchange, pair, source timestamp, sequence, retained levels, and normalization version are supplied again +- **THEN** the deterministic snapshot identity MUST be reproduced +- **AND** unrelated deployment ingestion timestamps MUST NOT change that identity + +### Requirement: ClickHouse and parquet share a capture-core contract +The canonical ClickHouse order-book tables and Maker parquet outputs SHALL share names and semantics for capture-core fields while allowing Maker materialization to add run-scoped coverage and assumption fields separately. + +#### Scenario: Broker rows are exported for Maker +- **WHEN** canonical ClickHouse rows are converted into `order_book_levels.parquet` and `order_book_depth_summary.parquet` +- **THEN** capture-core fields MUST map without semantic reinterpretation +- **AND** exchange, pair, provider, timestamps, construction mode, source mode, gap policy, depth, identities, and checksums MUST be preserved + +#### Scenario: Maker adds replay-run metadata +- **WHEN** Maker coverage planning adds run id, coverage report, as-of lag, future-leakage, or assumption fields +- **THEN** those fields MUST be treated as a materialized extension of the immutable broker capture +- **AND** they MUST NOT alter the original capture checksum or provenance + +### Requirement: Canonical order-book insertion is idempotent +Canonical order-book storage SHALL retain physical deliveries in append-only `MergeTree` base tables and SHALL expose deterministic canonical and conflict views over explicit logical keys so retries remain auditable without producing ambiguous replay rows. + +The level logical key SHALL consist of capture bundle, exchange, trading pair, raw capture, snapshot, schema version, side, and level index. The summary logical key SHALL use the same fields except side and level index. + +#### Scenario: Same level row is delivered twice +- **WHEN** the archive forwarder retries a level row with the same capture, snapshot, side, level index, schema version, and checksum +- **THEN** `market_data.cex_order_book_levels_canonical` MUST resolve it as one logical level +- **AND** both physical deliveries MUST remain available in the base evidence table + +#### Scenario: Same summary row is delivered twice +- **WHEN** a summary row is retried with the same capture and snapshot identity +- **THEN** `market_data.cex_order_book_depth_summary_canonical` MUST resolve it as one logical summary +- **AND** both physical deliveries MUST remain available in the base evidence table + +#### Scenario: Logical key has conflicting checksums in one batch +- **WHEN** one archive batch contains the same level or summary logical key with more than one normalized-row checksum +- **THEN** the forwarder MUST reject the conflicting table batch before insertion +- **AND** the producer retry and loss-journal policy MUST account for the rejected rows + +#### Scenario: Logical key has conflicting checksums across batches +- **WHEN** physical deliveries from different archive batches reuse one logical key with different normalized-row checksums +- **THEN** the corresponding ClickHouse conflict view MUST expose the key and distinct checksums +- **AND** canonical views MUST exclude the conflicted key +- **AND** replay validation MUST fail the affected capture bundle + +### Requirement: Legacy order-book storage requires table migration before upgrade +The system SHALL migrate retained rows from `market_data.orderbook_snapshots` to the canonical level and summary tables through a bounded ClickHouse table-to-table migration before deploying the upgraded canonical-only broker. The upgraded broker SHALL always write the latest canonical schema and SHALL NOT expose a runtime legacy/dual/canonical write mode. + +#### Scenario: Existing deployment is prepared for upgrade +- **WHEN** an existing broker writes `market_data.orderbook_snapshots` and the canonical-only version is scheduled for deployment +- **THEN** operators MUST apply canonical DDL, quiesce legacy writers, migrate every retained bounded partition, and validate parity before deploying the upgraded broker +- **AND** any missing row or value mismatch MUST block the upgrade + +#### Scenario: Legacy snapshot lacks raw capture identity +- **WHEN** a legacy row is migrated and its original raw capture or checksum is unavailable +- **THEN** the canonical row MUST record legacy source mode and incomplete provenance +- **AND** it MUST leave unavailable raw integrity fields null rather than invent them + +#### Scenario: Canonical-only broker is deployed +- **WHEN** canonical-table migration and replay validation pass for every retained migration window +- **THEN** the upgraded broker MUST write new order-book captures only to the canonical raw, level, and summary tables +- **AND** legacy query names MUST remain available through documented compatibility views for the migration retention period + +#### Scenario: Canonical cutover is rolled back +- **WHEN** a blocking integrity or replay defect is found after canonical deployment +- **THEN** operators MUST stop the upgraded broker and MAY restore the retained legacy names before rolling back to the previous legacy-writing application version +- **AND** the defect and affected capture bundles MUST remain diagnosable diff --git a/package.json b/package.json index a1cf6bf..b2486c7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@usherlabs/cex-broker", - "version": "0.2.36", + "version": "0.2.37", "description": "Unified gRPC API to CEXs by Usher Labs.", "repository": { "type": "git", diff --git a/research/hummingbot/canonical_capture_fixture.py b/research/hummingbot/canonical_capture_fixture.py new file mode 100644 index 0000000..1c51cf3 --- /dev/null +++ b/research/hummingbot/canonical_capture_fixture.py @@ -0,0 +1,285 @@ +"""Maker-side verifier for cex-broker canonical capture fixture v1.""" + +from __future__ import annotations + +import hashlib +import json +import math +from decimal import Decimal +from pathlib import Path +from typing import Any + +_CHECKSUM_FIELDS = {"normalized_row_checksum", "raw_checksum", "checksum"} + + + +def _number(value: int | float | Decimal) -> str: + if isinstance(value, Decimal): + if not value.is_finite(): + raise ValueError("canonical numbers must be finite") + if value == 0: + return "0" + rendered = format(value, "f") + if "." in rendered: + rendered = rendered.rstrip("0").rstrip(".") + return rendered + if isinstance(value, float): + if not math.isfinite(value): + raise ValueError("canonical numbers must be finite") + if value == 0: + return "0" + if value.is_integer(): + return str(int(value)) + rendered = repr(value).lower() + else: + return str(value) + if "e" not in rendered: + return rendered + return format(Decimal(rendered), "f") + + +def canonical_serialize(value: Any) -> str: + if value is None: + return "null" + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, (int, float, Decimal)): + return _number(value) + if isinstance(value, str): + return json.dumps(value, ensure_ascii=False, separators=(",", ":")) + if isinstance(value, list): + return "[" + ",".join(canonical_serialize(item) for item in value) + "]" + if isinstance(value, dict): + return "{" + ",".join( + f"{json.dumps(key, ensure_ascii=False)}:{canonical_serialize(value[key])}" + for key in sorted(value) + ) + "}" + raise TypeError(f"unsupported canonical value type: {type(value).__name__}") + + +def _without_checksums(value: Any) -> Any: + if isinstance(value, list): + return [_without_checksums(item) for item in value] + if isinstance(value, dict): + return { + key: _without_checksums(item) + for key, item in value.items() + if key not in _CHECKSUM_FIELDS + } + return value + + +def sha256_canonical(value: Any) -> str: + encoded = canonical_serialize(_without_checksums(value)).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def _context(fixture: dict[str, Any], capture: dict[str, Any]) -> dict[str, Any]: + return { + **fixture["context"], + "feed": capture["feed"], + "sourceMode": capture["sourceMode"], + **({"timeframe": capture["timeframe"]} if "timeframe" in capture else {}), + } + + +def _raw(fixture: dict[str, Any], capture: dict[str, Any]) -> dict[str, Any]: + context = _context(fixture, capture) + # cex-broker redaction preserves object payloads and wraps root arrays so the + # persisted evidence always has an object-shaped JSON envelope. + redacted_payload = ( + {"items": capture["payload"]} + if isinstance(capture["payload"], list) + else capture["payload"] + ) + raw_checksum = sha256_canonical(redacted_payload) + raw_capture_id = sha256_canonical({ + "capture_bundle_id": context["captureBundleId"], + "exchange": context["exchange"].lower(), + "feed": context["feed"], + "raw_capture_scope": "ccxt_normalized_object", + "raw_payload_sha256": raw_checksum, + "schema_version": context["schemaVersion"], + "source_mode": context["sourceMode"], + "source_symbol": context["symbol"], + "source_time_ms": capture["eventTimeMs"], + }) + return { + "rawCaptureId": raw_capture_id, + "rawCaptureScope": "ccxt_normalized_object", + "rawChecksum": raw_checksum, + "eventTimeMs": capture["eventTimeMs"], + "receivedTimeMs": capture["receivedTimeMs"], + } + + +def _core(context: dict[str, Any], raw: dict[str, Any]) -> dict[str, Any]: + return { + "source": context["source"], + "deployment_id": context["deploymentId"], + "capture_bundle_id": context["captureBundleId"], + "exchange": context["exchange"].lower(), + "symbol": context["symbol"], + "trading_pair": context["symbol"].replace("/", "-"), + "source_symbol": context["symbol"], + "asset_type": context["assetType"], + "feed": context["feed"], + "provider": context["provider"], + "source_mode": context["sourceMode"], + "source_time_ms": raw["eventTimeMs"], + "received_time_ms": raw["receivedTimeMs"], + "raw_capture_id": raw["rawCaptureId"], + "raw_capture_scope": raw["rawCaptureScope"], + "schema_version": context["schemaVersion"], + "checksum_algorithm": context["checksumAlgorithm"], + "raw_checksum": raw["rawChecksum"], + "provenance_complete": 1, + } + + +def _verify_capture( + fixture: dict[str, Any], + name: str, + row: dict[str, Any], + raw: dict[str, Any], +) -> list[str]: + expected = fixture[name]["expected"] + actual = { + "raw_capture_id": raw["rawCaptureId"], + "raw_checksum": raw["rawChecksum"], + "normalized_row_checksum": sha256_canonical(row), + } + return [ + f"{name}.{field}: expected {expected[field]}, got {value}" + for field, value in actual.items() + if expected[field] != value + ] + + +def verify_fixture(path: str | Path) -> list[str]: + fixture = json.loads(Path(path).read_text(encoding="utf-8")) + failures: list[str] = [] + + stream = fixture["stream"] + stream_context = _context(fixture, stream) + stream_raw = _raw(fixture, stream) + stream_row = { + **_core(stream_context, stream_raw), + "stream_type": stream_context["feed"], + "event_time_ms": stream_raw["eventTimeMs"], + "payload_encoding": "canonical_json_v1", + "payload_json": canonical_serialize(stream["payload"]), + } + failures += _verify_capture(fixture, "stream", stream_row, stream_raw) + + for name, field_map in ( + ("ticker", { + "eventTimeMs": "event_time_ms", "last": "last", "bid": "bid", + "ask": "ask", "high": "high", "low": "low", "open": "open", + "close": "close", "baseVolume": "base_volume", + "quoteVolume": "quote_volume", "change": "change", + "percentage": "percentage", + }), + ("trade", { + "tradeId": "trade_id", "eventTimeMs": "event_time_ms", + "side": "side", "price": "price", "amount": "amount", + "cost": "cost", "takerOrMaker": "taker_or_maker", + }), + ): + capture = fixture[name] + context = _context(fixture, capture) + raw = _raw(fixture, capture) + row = {**_core(context, raw)} + for source_field, row_field in field_map.items(): + if source_field in capture["normalized"]: + row[row_field] = capture["normalized"][source_field] + if name == "ticker": + row["source_time_ms"] = capture["normalized"]["eventTimeMs"] + else: + row["source_time_ms"] = capture["normalized"]["eventTimeMs"] + failures += _verify_capture(fixture, name, row, raw) + + ohlcv = fixture["ohlcv"] + ohlcv_context = _context(fixture, ohlcv) + ohlcv_raw = _raw(fixture, ohlcv) + bar = ohlcv["bar"] + ohlcv_row = { + **_core(ohlcv_context, ohlcv_raw), + "source_time_ms": bar["openTimeMs"], + "timeframe": ohlcv_context["timeframe"], + "open_time_ms": bar["openTimeMs"], + "open": bar["open"], "high": bar["high"], "low": bar["low"], + "close": bar["close"], "volume": bar["volume"], + "is_closed": 1 if ohlcv["isClosed"] else 0, + "broker_version": ohlcv["brokerVersion"], + } + failures += _verify_capture(fixture, "ohlcv", ohlcv_row, ohlcv_raw) + + book = fixture["orderbook"] + book_context = _context(fixture, book) + book_raw = _raw(fixture, book) + bids = [{"price": level[0], "amount": level[1]} for level in book["payload"]["bids"]] + asks = [{"price": level[0], "amount": level[1]} for level in book["payload"]["asks"]] + best_bid, best_ask = bids[0], asks[0] + mid = (best_bid["price"] + best_ask["price"]) / 2 + spread = best_ask["price"] - best_bid["price"] + snapshot_id = sha256_canonical({ + "exchange": book_context["exchange"].lower(), + "trading_pair": book_context["symbol"].replace("/", "-"), + "source_time_ms": book["eventTimeMs"], + "sequence": book["payload"]["sequence"], + "depth_limit": book["depthLimit"], + "bids": bids, + "asks": asks, + "schema_version": book_context["schemaVersion"], + }) + common = { + **_core(book_context, book_raw), + "source_time_ms": book["eventTimeMs"], + "received_time_ms": book["receivedTimeMs"], + "snapshot_id": snapshot_id, + "construction_mode": "sampled_top_n_snapshot", + "gap_policy": "record_gap", + "depth_limit": book["depthLimit"], + "sequence": book["payload"]["sequence"], + "exact_l2_reconstruction_complete": 0, + } + first_level = { + **common, "side": "bid", "level_index": 0, + "price": best_bid["price"], "amount": best_bid["amount"], + "notional": best_bid["price"] * best_bid["amount"], + "mid_price": mid, + "spread_from_mid_bps": abs((best_bid["price"] - mid) / mid) * 10_000, + } + bands = [10, 25, 50, 100] + summary = { + **common, + "best_bid": best_bid["price"], "best_ask": best_ask["price"], + "best_bid_amount": best_bid["amount"], "best_ask_amount": best_ask["amount"], + "mid_price": mid, "spread": spread, "spread_bps": spread / mid * 10_000, + "staleness_ms": book["receivedTimeMs"] - book["eventTimeMs"], + "bid_level_count": len(bids), "ask_level_count": len(asks), + "measurement_bands_bps": bands, + "bid_depth_by_band": [ + sum(level["amount"] for level in bids if level["price"] >= best_bid["price"] * (1 - band / 10_000)) + for band in bands + ], + "ask_depth_by_band": [ + sum(level["amount"] for level in asks if level["price"] <= best_ask["price"] * (1 + band / 10_000)) + for band in bands + ], + } + expected = book["expected"] + actual_book = { + "raw_capture_id": book_raw["rawCaptureId"], + "raw_checksum": book_raw["rawChecksum"], + "snapshot_id": snapshot_id, + "level_normalized_row_checksum": sha256_canonical(first_level), + "summary_normalized_row_checksum": sha256_canonical(summary), + } + failures += [ + f"orderbook.{field}: expected {expected[field]}, got {value}" + for field, value in actual_book.items() + if expected[field] != value + ] + return failures diff --git a/research/hummingbot/order_book_parquet_fixture.py b/research/hummingbot/order_book_parquet_fixture.py new file mode 100644 index 0000000..0426dac --- /dev/null +++ b/research/hummingbot/order_book_parquet_fixture.py @@ -0,0 +1,77 @@ +"""Maker-side capture-core and checksum verifier for broker Parquet exports.""" + +from __future__ import annotations + +import argparse +from pathlib import Path +from typing import Any, Iterable + +from canonical_capture_fixture import sha256_canonical + +_CAPTURE_CORE_FIELDS = { + "source", "deployment_id", "capture_bundle_id", "exchange", "symbol", + "trading_pair", "source_symbol", "asset_type", "feed", "provider", + "source_mode", "source_time_ms", "received_time_ms", "raw_capture_id", + "raw_capture_scope", "schema_version", "checksum_algorithm", "raw_checksum", + "provenance_complete", "snapshot_id", "construction_mode", "gap_policy", + "depth_limit", "sequence", "exact_l2_reconstruction_complete", + "normalized_row_checksum", +} +_LEVEL_FIELDS = { + "side", "level_index", "price", "amount", "notional", "mid_price", + "spread_from_mid_bps", +} +_SUMMARY_FIELDS = { + "best_bid", "best_ask", "best_bid_amount", "best_ask_amount", "mid_price", + "spread", "spread_bps", "staleness_ms", "bid_level_count", + "ask_level_count", "measurement_bands_bps", "bid_depth_by_band", + "ask_depth_by_band", +} + + +def verify_order_book_rows( + rows: Iterable[dict[str, Any]], row_kind: str, +) -> list[str]: + required = _CAPTURE_CORE_FIELDS | ( + _LEVEL_FIELDS if row_kind == "levels" else _SUMMARY_FIELDS + ) + failures: list[str] = [] + for index, row in enumerate(rows): + missing = sorted(required - row.keys()) + if missing: + failures.append(f"{row_kind}[{index}] missing fields: {','.join(missing)}") + continue + if row["feed"] != "ORDERBOOK": + failures.append(f"{row_kind}[{index}] feed is not ORDERBOOK") + if row["checksum_algorithm"] != "sha256-canonical-json-v1": + failures.append(f"{row_kind}[{index}] checksum algorithm is unsupported") + actual = sha256_canonical(row) + if row["normalized_row_checksum"] != actual: + failures.append( + f"{row_kind}[{index}] checksum mismatch: " + f"expected {row['normalized_row_checksum']}, got {actual}" + ) + return failures + + +def verify_order_book_parquet( + levels_path: str | Path, summary_path: str | Path, +) -> list[str]: + import pyarrow.parquet as parquet + + levels = parquet.read_table(levels_path).to_pylist() + summaries = parquet.read_table(summary_path).to_pylist() + return [ + *verify_order_book_rows(levels, "levels"), + *verify_order_book_rows(summaries, "summary"), + ] + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("levels") + parser.add_argument("summary") + args = parser.parse_args() + errors = verify_order_book_parquet(args.levels, args.summary) + if errors: + raise SystemExit("\n".join(errors)) diff --git a/research/python/pyproject.toml b/research/python/pyproject.toml index bdadf5e..39bf325 100644 --- a/research/python/pyproject.toml +++ b/research/python/pyproject.toml @@ -6,6 +6,7 @@ requires-python = ">=3.11" dependencies = [ "clickhouse-connect>=0.7.0", "pandas>=2.0.0", + "pyarrow>=14.0.0", "pyyaml>=6.0", ] diff --git a/research/python/tests/test_canonical_capture_fixture.py b/research/python/tests/test_canonical_capture_fixture.py new file mode 100644 index 0000000..4974574 --- /dev/null +++ b/research/python/tests/test_canonical_capture_fixture.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +import sys +from decimal import Decimal +from pathlib import Path + +_REPO_ROOT = Path(__file__).resolve().parents[3] +_HUMMINGBOT = _REPO_ROOT / "research" / "hummingbot" +sys.path.insert(0, str(_HUMMINGBOT)) + +from canonical_capture_fixture import ( # noqa: E402 + canonical_serialize, + sha256_canonical, + verify_fixture, +) +from order_book_parquet_fixture import verify_order_book_parquet # noqa: E402 + + +def test_python_reproduces_typescript_golden_capture_checksums(): + fixture = _REPO_ROOT / "test" / "fixtures" / "canonical-market-capture-v1.json" + assert verify_fixture(fixture) == [] + + +def test_python_canonical_number_and_key_representation(): + assert canonical_serialize({"z": -0.0, "a": 1e-7}) == '{"a":0.0000001,"z":0}' + assert canonical_serialize(Decimal("100.000000000000000000")) == "100" + + +def test_maker_parquet_reader_preserves_capture_core_and_checksums(tmp_path): + import pyarrow as pa + import pyarrow.parquet as parquet + + common = { + "source": "broker_read", "deployment_id": "test", "capture_bundle_id": "bundle", + "exchange": "binance", "symbol": "BTC/USDT", "trading_pair": "BTC-USDT", + "source_symbol": "BTC/USDT", "asset_type": "spot", "feed": "ORDERBOOK", + "provider": "ccxt:binance", "source_mode": "broker_live_sampling_v1", + "source_time_ms": 1_700_000_000_000, "received_time_ms": 1_700_000_000_010, + "raw_capture_id": "raw-id", "raw_capture_scope": "ccxt_normalized_object", + "schema_version": "1.0.0", "checksum_algorithm": "sha256-canonical-json-v1", + "raw_checksum": "raw-checksum", "provenance_complete": 1, + "snapshot_id": "snapshot-id", "construction_mode": "sampled_top_n_snapshot", + "gap_policy": "record_gap", "depth_limit": 1, "sequence": 7, + "exact_l2_reconstruction_complete": 0, + } + level = { + **common, "side": "bid", "level_index": 0, "price": Decimal("100"), + "amount": Decimal("2"), "notional": Decimal("200"), + "mid_price": Decimal("100.5"), "spread_from_mid_bps": 49.75124378109453, + } + level["normalized_row_checksum"] = sha256_canonical(level) + summary = { + **common, "best_bid": Decimal("100"), "best_ask": Decimal("101"), + "best_bid_amount": Decimal("2"), "best_ask_amount": Decimal("3"), + "mid_price": Decimal("100.5"), "spread": Decimal("1"), + "spread_bps": 99.50248756218906, "staleness_ms": 10, + "bid_level_count": 1, "ask_level_count": 1, + "measurement_bands_bps": [10, 25], + "bid_depth_by_band": [Decimal("2"), Decimal("2")], + "ask_depth_by_band": [Decimal("3"), Decimal("3")], + } + summary["normalized_row_checksum"] = sha256_canonical(summary) + levels_path = tmp_path / "order_book_levels.parquet" + summary_path = tmp_path / "order_book_depth_summary.parquet" + parquet.write_table(pa.Table.from_pylist([level]), levels_path) + parquet.write_table(pa.Table.from_pylist([summary]), summary_path) + assert verify_order_book_parquet(levels_path, summary_path) == [] diff --git a/research/python/uv.lock b/research/python/uv.lock new file mode 100644 index 0000000..c76a2e8 --- /dev/null +++ b/research/python/uv.lock @@ -0,0 +1,623 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'win32'", + "python_full_version >= '3.14' and sys_platform == 'emscripten'", + "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'win32'", + "python_full_version < '3.12' and sys_platform == 'win32'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'emscripten'", + "python_full_version < '3.12' and sys_platform == 'emscripten'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] + +[[package]] +name = "backports-zstd" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8e/b5/5a873da082bd08acd6a497f7aae224e94a7c27fa8f24488089cc50a16c84/backports_zstd-1.6.0.tar.gz", hash = "sha256:80a7859ffe70bf239d7a2ce15293bdeb5b4280ff7dc326ffab312b0e254dbb24", size = 1000009, upload-time = "2026-06-14T10:50:58.555Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c5/90/428dd82228b1b6d62d5a1bf312c29e6c125af6a182fcfd82768ca179dcc7/backports_zstd-1.6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c4fc41b2df5529cad5ceb230319e82728096d4b353ce8d4df68a2ec37e291bb8", size = 437067, upload-time = "2026-06-14T10:49:28.335Z" }, + { url = "https://files.pythonhosted.org/packages/ef/48/768edf21fe33bae8d874470b1be136681d4d32eb820a32e1c98262ebe39b/backports_zstd-1.6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:83391ef5935cc0f329b1abca414ae20ffe40d335fc21a4b5e664f08a74317d5f", size = 363454, upload-time = "2026-06-14T10:49:29.784Z" }, + { url = "https://files.pythonhosted.org/packages/29/8a/d462c2e5071eb573378f0d26760f6590613086fdf59c2d3c66bdfffb9f41/backports_zstd-1.6.0-cp311-cp311-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:7d3f64c503af7b60115b97c16feaf75bd191ef2c978d5c0c7725a6682bef63c5", size = 507393, upload-time = "2026-06-14T10:49:31.077Z" }, + { url = "https://files.pythonhosted.org/packages/b9/cb/af58363b0dd0b497282ecef1fa99789b03cc1885a01a41394cad42ceeff6/backports_zstd-1.6.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0308990ffc998df3c7ed35276bde049728b5c3956203cae40d80893576a41459", size = 476957, upload-time = "2026-06-14T10:49:32.53Z" }, + { url = "https://files.pythonhosted.org/packages/e4/fd/5fbdf2275cefae95c4b3509f6db2dc372d0587ebafea342d28781d51d932/backports_zstd-1.6.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8c298785e2fadeab82342040f2d9ce764ce500e6da6a6d99a2de514e63580b5a", size = 582618, upload-time = "2026-06-14T10:49:33.723Z" }, + { url = "https://files.pythonhosted.org/packages/99/6f/7dd45c53c907ea67f635c3900b58bb3347c01dc2ded441402028aae0ef9c/backports_zstd-1.6.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ae106fe16e36efc60ab098d02478d30aa0e31e1420eb4ecf0116459253bc6361", size = 642279, upload-time = "2026-06-14T10:49:34.938Z" }, + { url = "https://files.pythonhosted.org/packages/4d/25/a9e37dd035027565fa0b7e367da50e88a6ab26e7fd413269aa118e25258b/backports_zstd-1.6.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7293fefe15f0e5852bdb4ad1e0e26f3cbd4d3e61c19f751ecc4ff34bc1eb237d", size = 492486, upload-time = "2026-06-14T10:49:36.06Z" }, + { url = "https://files.pythonhosted.org/packages/a1/52/659686bf8f7c53ea279e1c44038504b82a6901cee2f5ae83c30bbf581301/backports_zstd-1.6.0-cp311-cp311-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ece8e7288db5b827ef8c64b2f78519f1a173a8991a625978fce02eccd7654fe9", size = 566440, upload-time = "2026-06-14T10:49:37.536Z" }, + { url = "https://files.pythonhosted.org/packages/d9/1a/c7ea5a0ff607a1a6066bb7c7cb65ae20e2f85da6adc69ab77fd8943e180c/backports_zstd-1.6.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:28eef3881164f3c23ce58ed59e4684103bdd279583eb2d299858c9e9b72fde9a", size = 482899, upload-time = "2026-06-14T10:49:38.805Z" }, + { url = "https://files.pythonhosted.org/packages/83/48/bd2b91100ee4fe6bb4d816e3659cbbb0cda5dd32760d2379c54d1752ec25/backports_zstd-1.6.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:481a1e9bd8f419fdc625307aa20234687f99368c75df511ef589693c5fea4c6f", size = 510826, upload-time = "2026-06-14T10:49:40.062Z" }, + { url = "https://files.pythonhosted.org/packages/25/fe/fa28509d7ce2ad59404e7ce738a2fd858e12dfd9a896629f10330222a7fb/backports_zstd-1.6.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:3b6713371f8987a1178df93cb36f29eef191f224021e2d656b2f11ce60d26816", size = 586941, upload-time = "2026-06-14T10:49:41.305Z" }, + { url = "https://files.pythonhosted.org/packages/45/28/757daf2399aa71bb37f9f7f48b42ab03fc51c340eccfad2fec92a23f6aa3/backports_zstd-1.6.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:b0ddbcd2866b8ff1a2836e4b0e4d44788f5b992d83fac75a38cda8f9a2bee079", size = 564261, upload-time = "2026-06-14T10:49:42.49Z" }, + { url = "https://files.pythonhosted.org/packages/4e/53/9b9db30cb2c148a69c40ad7647aa787338041f3dc81c5b22113286e590e9/backports_zstd-1.6.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:2914abea516704bdafb2090acd3f15b5f9debecfabd15b8dd8285b2ad3b92209", size = 632869, upload-time = "2026-06-14T10:49:43.981Z" }, + { url = "https://files.pythonhosted.org/packages/81/a4/1692fbb88af8aaf900a53619fcc95c9e45d9ff162223a47fd672a9893c8d/backports_zstd-1.6.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:dd085eafa2aac6f883afd28210a3231f717f25409a1e44a39bb7b04c8c5b5646", size = 496496, upload-time = "2026-06-14T10:49:45.118Z" }, + { url = "https://files.pythonhosted.org/packages/93/42/c5a66c47320bd12ce84a7341330ea582d67069bdb70214bca0b6bf394cfd/backports_zstd-1.6.0-cp311-cp311-win32.whl", hash = "sha256:b81b4cf3d6e0ad7ac92bef248f49fafc954262c5fb0f7e19d6aac497e5a856b2", size = 291613, upload-time = "2026-06-14T10:49:46.473Z" }, + { url = "https://files.pythonhosted.org/packages/2a/f2/f22c19b4cdde429805ff5ac8dd77a95569a7c4cb8991741b2ff0d538f220/backports_zstd-1.6.0-cp311-cp311-win_amd64.whl", hash = "sha256:10b61850c4112952e05aa6e6cce8c9a5936fbeadb321e154216705cc76a14afa", size = 329078, upload-time = "2026-06-14T10:49:47.71Z" }, + { url = "https://files.pythonhosted.org/packages/ef/dc/e902a3f1eb92c4907b5f47f90cb3c2734ee315c4ff67179fc111343b45ba/backports_zstd-1.6.0-cp311-cp311-win_arm64.whl", hash = "sha256:068ef3d8c18815a2e3a752f766313e19910e7c50939b956923748d9c04ebcb1b", size = 291727, upload-time = "2026-06-14T10:49:48.929Z" }, + { url = "https://files.pythonhosted.org/packages/1e/bb/009af3a9532d4cc66d5385391c512210fae32ab2442605f26aca1d8d2957/backports_zstd-1.6.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0466b14723f3b7697669c00ee66fe16e30e25636b286b0a923fa86fa3d8a753c", size = 437407, upload-time = "2026-06-14T10:49:50.155Z" }, + { url = "https://files.pythonhosted.org/packages/0c/76/f7c02efde81ebb9993586f9e435d2fd1191a6f806f640e4eeb8d004493ed/backports_zstd-1.6.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1d146926e997d2d3de8212bdcbf4985344a2622ca3bec458d8908000a84fd883", size = 363519, upload-time = "2026-06-14T10:49:51.383Z" }, + { url = "https://files.pythonhosted.org/packages/2e/5e/0cf66f12472fe3e082cc4134395a7e0b8746cfb30aabd74251ce8fafa9a7/backports_zstd-1.6.0-cp312-cp312-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:460fd6b3f338c659507ae36cfd6b58ac9942a2ff233c5cf574416dfec0451a84", size = 507756, upload-time = "2026-06-14T10:49:52.497Z" }, + { url = "https://files.pythonhosted.org/packages/03/95/7ed25c90369360f96f8bfa961540845e063377c32a43b775201af66a588c/backports_zstd-1.6.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7c2b1f4a640c51130caa92cef5bf72bd3c3dbbcfbf814c37403aa0601b1811b0", size = 477578, upload-time = "2026-06-14T10:49:53.887Z" }, + { url = "https://files.pythonhosted.org/packages/e3/75/f16b1d3e33ca396525847c81d96e3de7bc74d2c6f9ca2ddee76b0c450697/backports_zstd-1.6.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:beb43e9885202c8d4f3762319ed4d5e98e197622afbff8439fbbdd81d08938b9", size = 583029, upload-time = "2026-06-14T10:49:55.132Z" }, + { url = "https://files.pythonhosted.org/packages/e1/2b/a17b111b631e1c79a0e570881c1a266c661b936585afa395435a458b1991/backports_zstd-1.6.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fbb746522ebfc11155f1cd688e2c48ef3d74125e38b63eabdaab068a055c3e88", size = 641741, upload-time = "2026-06-14T10:49:56.42Z" }, + { url = "https://files.pythonhosted.org/packages/6b/b2/d17b2722c636d64b4e77ddc68d8d0625719d39f94021be8719a218af4c0a/backports_zstd-1.6.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a99710fbb225d459d66def4dc2bb2cd4a9a0bdc8b799fc0621cfdd863be9c93", size = 495554, upload-time = "2026-06-14T10:49:57.652Z" }, + { url = "https://files.pythonhosted.org/packages/63/12/2853e8b6c03f03795b6548ea61f82cc104d4f7ff2523a04bc69f46984663/backports_zstd-1.6.0-cp312-cp312-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f69365ee2b836939137de024a302395a1cb8654fb6dc5ffef6381105259c8f87", size = 570027, upload-time = "2026-06-14T10:49:59.003Z" }, + { url = "https://files.pythonhosted.org/packages/18/aa/83f37b81f3b8c6ea035bf260ec374648bd59372894c02323dc9de3cbdf77/backports_zstd-1.6.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:66cf8038893c7708ec345ffb3ac63c775d10f430f323ac2f0334fdb6a397c57c", size = 483594, upload-time = "2026-06-14T10:50:00.49Z" }, + { url = "https://files.pythonhosted.org/packages/f5/6a/d77f8cd2ff642d3b3652c1ccab5b6583114dbf10f8cb0143531357c83998/backports_zstd-1.6.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e514c71ca72f3b56bd8fbda1a6a5b7d1100a2764b42a3c74a38841f25f9b00ab", size = 511206, upload-time = "2026-06-14T10:50:01.86Z" }, + { url = "https://files.pythonhosted.org/packages/56/b2/99a60fe4d1aac8053769d2463271d5df37a7c11c387072fdbb0b16aed7f7/backports_zstd-1.6.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:7741e44f7938ec94f9a52678c8d19b7bc548522ffdc39c9e4481af8db545fa9a", size = 587416, upload-time = "2026-06-14T10:50:03.236Z" }, + { url = "https://files.pythonhosted.org/packages/ec/1e/a9c003fe4d14bd4bf671598d4c7dcc1cef51e3513d9d7111ba1d07b6f07b/backports_zstd-1.6.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:97e8a9674652496c7612b528085dd5a296c052a2edc466ca1bfb7b0b27820413", size = 567615, upload-time = "2026-06-14T10:50:04.524Z" }, + { url = "https://files.pythonhosted.org/packages/ec/b9/955bd604f692c550c7cb66d00bd7691ead5c86df8ebd23d7254eeaa90789/backports_zstd-1.6.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:23a793f2fed4dbf0517319759a2cded0b0dd8e8d3797fe30badd5693e320c175", size = 632269, upload-time = "2026-06-14T10:50:05.86Z" }, + { url = "https://files.pythonhosted.org/packages/18/d7/9f61f612f8a4193484c78a1f26db82a50141234189885113ef0085a8a961/backports_zstd-1.6.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b951113113ed4b8d173418a4f155c14b739dace626b3fa3f82be1831958d39e4", size = 500066, upload-time = "2026-06-14T10:50:07.446Z" }, + { url = "https://files.pythonhosted.org/packages/81/a3/19fb8c48d94139481c5ccaf2fb54c31b543fa635fd7bd7399aadd15752ac/backports_zstd-1.6.0-cp312-cp312-win32.whl", hash = "sha256:6430b34a2ae6fcc604672f4f913102563473d9a015bdca1ce8c95041cc1f2677", size = 291825, upload-time = "2026-06-14T10:50:08.762Z" }, + { url = "https://files.pythonhosted.org/packages/58/38/40ba081c6c71f0f22c64d3d54b912ad75a4e6812caa1397cbb15b5693b12/backports_zstd-1.6.0-cp312-cp312-win_amd64.whl", hash = "sha256:08793876172551a930ce4d65c712cd516184d1a97070d4a1193e05bf0cf7040d", size = 329201, upload-time = "2026-06-14T10:50:09.979Z" }, + { url = "https://files.pythonhosted.org/packages/2b/6c/f7116dd2edc6f960545f0d8616939eae3a20031b3b6669697d4f9fd83b2e/backports_zstd-1.6.0-cp312-cp312-win_arm64.whl", hash = "sha256:03b7c59c71f7a597e2bcb3f8368371e9a660a1bdf1c37afc1f1ad1496a013c19", size = 291901, upload-time = "2026-06-14T10:50:11.198Z" }, + { url = "https://files.pythonhosted.org/packages/38/06/c430537d59c55d49bcd15ecf4b1aa965453219caad810a4f2b484816f4be/backports_zstd-1.6.0-cp313-cp313-android_24_arm64_v8a.whl", hash = "sha256:2ace939e4d620e119423606f2d3d7115f8707733bf57f279ad9a9383f875986f", size = 400327, upload-time = "2026-06-14T10:50:12.446Z" }, + { url = "https://files.pythonhosted.org/packages/36/48/2f8323bb0e3ebba88b54877a2979afeb83983fb2ca572f09ad61aae2d3a0/backports_zstd-1.6.0-cp313-cp313-android_24_x86_64.whl", hash = "sha256:4c68a9ed2df0cca51d774c521e68a34d2e3d9ebfc687ef8096adfd4f345b551d", size = 454276, upload-time = "2026-06-14T10:50:13.667Z" }, + { url = "https://files.pythonhosted.org/packages/7c/39/87a665244a65f5b87a06b848c29a8cce07e91d59c5988ee2a32c0293a21c/backports_zstd-1.6.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:30576f49b82328ec8af16c11100efe52ca88526f71bbe100ef6b4e707dc13bf2", size = 357457, upload-time = "2026-06-14T10:50:14.906Z" }, + { url = "https://files.pythonhosted.org/packages/7f/8b/854d4a47bb8b7a48bfb2ed381c7b03a70efb4fc49f0e4a1509b38a2e1727/backports_zstd-1.6.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:b4bddfcfb6679215d6f4dc5f79a1f9301af339480d70527a14b57a1f2e6b6cbf", size = 366139, upload-time = "2026-06-14T10:50:16.399Z" }, + { url = "https://files.pythonhosted.org/packages/8f/de/c3af43eb8df6f2581e157e18a3e0121eadb826055b2fde3f91ec188689cb/backports_zstd-1.6.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:65048ed08c5124f05ff9f355ab9703014bb2dbe7f8d9948ce193685b1775f442", size = 446683, upload-time = "2026-06-14T10:50:17.633Z" }, + { url = "https://files.pythonhosted.org/packages/5c/39/87cf3d883d386c10ac52f5322604fb9afdd204229f4c47d4a820a839b8ff/backports_zstd-1.6.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5918fc6b31437208721276964323933cd86077b8d5b469c59c1b3fd2c8220a05", size = 436869, upload-time = "2026-06-14T10:50:19.113Z" }, + { url = "https://files.pythonhosted.org/packages/5e/b6/9479e6f0f18824ad38e8d7dd85161ab0842a198be669421232925bb30960/backports_zstd-1.6.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4b6c8b02ab0ccb2431bb7bc238be91d158b308915e7b07937388e540466fe7e7", size = 363090, upload-time = "2026-06-14T10:50:20.302Z" }, + { url = "https://files.pythonhosted.org/packages/d9/74/a5e98fe108e17c91d9bc590a19e77f5d47d579e34d3f5bc098a949d6c27c/backports_zstd-1.6.0-cp313-cp313-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:711e6b98f8924e8b4a61ff97ab6321f33de024e1ed6a32f5123763aeda8459be", size = 507070, upload-time = "2026-06-14T10:50:21.536Z" }, + { url = "https://files.pythonhosted.org/packages/69/f5/392bb7dce7363b77bc5403060f418fad438b9cfdd3edd10d65cee7d8fd11/backports_zstd-1.6.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2ba9ac10fc393e5123a08802e0e895a107cb4a66b9973d2844dbd8a343111e59", size = 477200, upload-time = "2026-06-14T10:50:22.91Z" }, + { url = "https://files.pythonhosted.org/packages/e4/4d/dfb665806ba4f74bc48071d32006843b53568c4a17ff627a3061de5eaa09/backports_zstd-1.6.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f723219335387d7546412d8141e0303590600949b4184a1391a0c6a3c756058", size = 582724, upload-time = "2026-06-14T10:50:24.28Z" }, + { url = "https://files.pythonhosted.org/packages/57/b2/beeca7393a8310debd82ee2f0ce5c1801e8d7cb673f7f226f4a0866ca238/backports_zstd-1.6.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:64b94d7a836568926a3309ff510c7f8261b881b341fd4992cabf4f0998878f8a", size = 643493, upload-time = "2026-06-14T10:50:25.736Z" }, + { url = "https://files.pythonhosted.org/packages/38/26/ce90e9eed6f25aaa4a4fa305a2aaf2d2ad81fd69de8eb248ddd91c80d1e0/backports_zstd-1.6.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e39258a09b1c7ca70b5e94a5c5ccfe4700b4250b8077cfeab31d0f79565d4c9b", size = 492190, upload-time = "2026-06-14T10:50:27.205Z" }, + { url = "https://files.pythonhosted.org/packages/17/9b/37b9b146df1f5452419a96071a7017cbac212ec9b137d7a88ca46dc2aa9e/backports_zstd-1.6.0-cp313-cp313-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:15b1aae0f64cd742df4bba1d989d0a09a6ec619202543fdba684640454541fd3", size = 567432, upload-time = "2026-06-14T10:50:28.386Z" }, + { url = "https://files.pythonhosted.org/packages/06/66/81b30991be83237529f36335ac3682bce26409064b906ac6122874575196/backports_zstd-1.6.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:25b5ddc789480072551af571a746e9500356b2aff0499861cf2ca07ea7431e68", size = 483021, upload-time = "2026-06-14T10:50:29.654Z" }, + { url = "https://files.pythonhosted.org/packages/49/2a/792c65dcc1e45eb0c1bdc012ee94b84867186bfe27a860d0813bd216f03b/backports_zstd-1.6.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:a13cfa3410a75e4cb87abdb669aaf79da861cb79299159054ff8f77b9671bc40", size = 510596, upload-time = "2026-06-14T10:50:31.657Z" }, + { url = "https://files.pythonhosted.org/packages/1d/22/01b92a600505620e4cb5f20429e181f30458b7207ca8b52ca5ca6068c35f/backports_zstd-1.6.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:2ddab55a5f54dec8acfad68ef70f1c704fd21919990ddc238afbd6f496e61c6a", size = 587143, upload-time = "2026-06-14T10:50:32.868Z" }, + { url = "https://files.pythonhosted.org/packages/d8/60/4672f5110b9eb01388cc6225a739e3a5fcd749a63a9c4c1450a04fa27113/backports_zstd-1.6.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:fa305a84087e10d7a85e8a8a3dcba8cdbda4868f2180173b264b7b488fd37c55", size = 565238, upload-time = "2026-06-14T10:50:34.173Z" }, + { url = "https://files.pythonhosted.org/packages/5c/3b/19928d60ea7d25820bf12ef88de74534ca85b56ff7cf13c1b0e74e3a3d7c/backports_zstd-1.6.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:df27b57d214a3124fbe4e933ef5a903d4567f154260d9aece8c797a987f2a205", size = 633970, upload-time = "2026-06-14T10:50:35.506Z" }, + { url = "https://files.pythonhosted.org/packages/df/97/c4cecb3e0ff53563ef9819f0395d919ceaae9c5147392ac23bac7afdb20f/backports_zstd-1.6.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:28fecd73459d74910ae1987ab84b7bef690d3dd860948430dd5555108b006daf", size = 496539, upload-time = "2026-06-14T10:50:37.015Z" }, + { url = "https://files.pythonhosted.org/packages/eb/f4/46b2f29d2938a80e56e61a19f11ab093f531a9f8cd0ec8eeaac1246bcd99/backports_zstd-1.6.0-cp313-cp313-win32.whl", hash = "sha256:3e689af303df287142770abe3a48bbefd24dab4a09da5807d0e1fa8c75bab026", size = 291451, upload-time = "2026-06-14T10:50:38.518Z" }, + { url = "https://files.pythonhosted.org/packages/d1/ad/b529f92166da61f496621345f95d2dc583c8ca5ac553c084a4ef6c12cd71/backports_zstd-1.6.0-cp313-cp313-win_amd64.whl", hash = "sha256:b067b1ef9c8e41fb0882c828aa37829938b5c0dab067eca72b23fc24c563b9da", size = 329023, upload-time = "2026-06-14T10:50:39.742Z" }, + { url = "https://files.pythonhosted.org/packages/30/d8/6be904d20345fbebec583ca83676e01f30c76118b283eb666d8ec8291ca1/backports_zstd-1.6.0-cp313-cp313-win_arm64.whl", hash = "sha256:a838296f5b84c920172fb579cac894d255c1fc25457c7234613ddcfa385e49b7", size = 291636, upload-time = "2026-06-14T10:50:41.004Z" }, + { url = "https://files.pythonhosted.org/packages/e8/09/898fe2f8196fa7ab825f5fed786c68581fdac7d23a8e20baa0cc01cb2f0b/backports_zstd-1.6.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:aeef8563b82ed4af328f98e5041c1b4800d86f68f857ffd1577d4d47dc9aa6cd", size = 411023, upload-time = "2026-06-14T10:50:50.286Z" }, + { url = "https://files.pythonhosted.org/packages/6e/ad/6ad9af1596ab5f284bb53954be41396e13d23c81cdfe3d945402e8ee0215/backports_zstd-1.6.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:9cb75e33131946fabd6319061df3b8b1d588fe0963183280e9b5f49f7772fc09", size = 340554, upload-time = "2026-06-14T10:50:51.523Z" }, + { url = "https://files.pythonhosted.org/packages/f0/00/f083d7c8a4ee5d0bb21b4d3144e76de9f655ca4dd0bffcb95baa5bc47a62/backports_zstd-1.6.0-pp311-pypy311_pp73-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:ef132cfb638e9a86bd5dc07fb4e1cb895bc55bce6bb5e759366e8b160d0747e2", size = 421694, upload-time = "2026-06-14T10:50:52.917Z" }, + { url = "https://files.pythonhosted.org/packages/41/d7/693b20f3ccae2e05d166f98fe55b1657451170b72c804ed9f6b98df520be/backports_zstd-1.6.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab70eace272d6f122b121c057e436709b50a28abf30d97aab28433c08f4a4095", size = 395237, upload-time = "2026-06-14T10:50:54.448Z" }, + { url = "https://files.pythonhosted.org/packages/53/a1/484e0f9ec994bd2285d6747e7c8028350f1a177e9210bc57637898042d3b/backports_zstd-1.6.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:17efb3d11137de5166dd51eedab9c36ad633402acba386eee8d715213ea47e49", size = 415201, upload-time = "2026-06-14T10:50:55.854Z" }, + { url = "https://files.pythonhosted.org/packages/3c/56/70860ece85cd49b564305cbc22bf6c4183975427ff6dfe2097e855f5dd5e/backports_zstd-1.6.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:994167ff6551b9c1ce226e0aab16295b98c94507b5701aa60d2c32b7d50796b1", size = 315721, upload-time = "2026-06-14T10:50:57.074Z" }, +] + +[[package]] +name = "certifi" +version = "2026.7.22" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/c2/24167ea9858356b47a87a50d39908bfdb72ceeefe0041586e704e5376b3a/certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55", size = 138112, upload-time = "2026-07-22T03:35:12.644Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775", size = 136983, upload-time = "2026-07-22T03:35:11.276Z" }, +] + +[[package]] +name = "cex-broker-research" +version = "0.1.0" +source = { virtual = "." } +dependencies = [ + { name = "clickhouse-connect" }, + { name = "pandas" }, + { name = "pyarrow" }, + { name = "pyyaml" }, +] + +[package.optional-dependencies] +dev = [ + { name = "pytest" }, +] + +[package.metadata] +requires-dist = [ + { name = "clickhouse-connect", specifier = ">=0.7.0" }, + { name = "pandas", specifier = ">=2.0.0" }, + { name = "pyarrow", specifier = ">=14.0.0" }, + { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0" }, + { name = "pyyaml", specifier = ">=6.0" }, +] +provides-extras = ["dev"] + +[[package]] +name = "clickhouse-connect" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "backports-zstd", marker = "python_full_version < '3.14'" }, + { name = "certifi" }, + { name = "lz4" }, + { name = "tzdata", marker = "sys_platform == 'win32'" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/31/aeda81546adc2004742933ed7b741716e3581fea5e4fd6c4dd8a196f82cd/clickhouse_connect-1.6.0.tar.gz", hash = "sha256:e4eb8b83b848f4ca4b984899acd62b0a60f310cbaa559d83519b7477968392ed", size = 193965, upload-time = "2026-07-23T18:03:16.783Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5f/9e/80810656d9d5ce1357222ad5a326a9c569030ebec5b46351e86716227e96/clickhouse_connect-1.6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:effd020411e4669f21d8b9f7f95d2342145ff6a2e32b49eae416fbd733334e53", size = 387549, upload-time = "2026-07-23T18:01:56.54Z" }, + { url = "https://files.pythonhosted.org/packages/a1/85/ec9f6359dfcf0a918c6926b948f079bbda02f95be06a7f6762aa352ecd5c/clickhouse_connect-1.6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f8957fa2fd400d1f002dded822b3e667b7941e48107d8d2470e1935a12af9937", size = 376692, upload-time = "2026-07-23T18:01:58.112Z" }, + { url = "https://files.pythonhosted.org/packages/f6/e8/12acfd15c40c9c3ac4d1c6fc413eec3ca3782adb70cf00ecf021abc12556/clickhouse_connect-1.6.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ca3d4b70d1fa92e62ab130dec495b470de6e0e56a944cd3c8ea049cb0f137c20", size = 1323910, upload-time = "2026-07-23T18:01:59.525Z" }, + { url = "https://files.pythonhosted.org/packages/c2/f1/dda7de7bce2f72f263a725bf3e3cabcf81f13608cd6f1a71152f59d005e5/clickhouse_connect-1.6.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ed28787dd00a34b003fe24de4953e15b172058d875279336d4ee5d8f585acc50", size = 1334678, upload-time = "2026-07-23T18:02:01.097Z" }, + { url = "https://files.pythonhosted.org/packages/a4/03/004af313f0edb521b12967f3d80cb8ac465c1a06c964f3ba305297b7872e/clickhouse_connect-1.6.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d56348f1a39c8bd872106c4af92faa532fbdd8d66e72c588e9b4dd676ca2dbbd", size = 1312144, upload-time = "2026-07-23T18:02:02.698Z" }, + { url = "https://files.pythonhosted.org/packages/fe/bd/24b758e548e34a3a0e661ddd43ffb981629b4ca9b1d0bbf89567bb261a75/clickhouse_connect-1.6.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d541a1a75f08d62b7e0059f77efeff26f2b0829fa0d543e1e1edcce270c78a06", size = 1342925, upload-time = "2026-07-23T18:02:04.008Z" }, + { url = "https://files.pythonhosted.org/packages/79/1d/10f1d37e1feb8840e959dc554ca8cfb5c8b2b0eea93c3e2c0b55a38f43a9/clickhouse_connect-1.6.0-cp311-cp311-win32.whl", hash = "sha256:2d5888d76be3bf1ddea5a210522572d66abcaf651b45ff7c5a1014ec08fe7565", size = 347963, upload-time = "2026-07-23T18:02:05.424Z" }, + { url = "https://files.pythonhosted.org/packages/12/e1/ebe765c6d4173f20ccee002e41289d30f2e40203cedee9e3ee0450ee4cd7/clickhouse_connect-1.6.0-cp311-cp311-win_amd64.whl", hash = "sha256:5135f91841e9df5827faa24f3b2436d4ac8242e74a88a8d47922e31a4383b712", size = 368380, upload-time = "2026-07-23T18:02:06.729Z" }, + { url = "https://files.pythonhosted.org/packages/0b/ea/0c1eaafcb78d5e33d13b0ca531f5f4ee0e1101ea2eb8bfd8a22dde2bdd0f/clickhouse_connect-1.6.0-cp311-cp311-win_arm64.whl", hash = "sha256:fe2736f4cb520c4403f1f6b078ed3ed6f41a6ee8972328082240001853d8639e", size = 350903, upload-time = "2026-07-23T18:02:08.17Z" }, + { url = "https://files.pythonhosted.org/packages/23/0e/9c6a165ecfd0708df9bec011b6adedb2e627b5778b041f7e35ca4bf942b9/clickhouse_connect-1.6.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7043480f1459ddd791480e3ea560a1def81dfd6462eab0c9ada54946e72918af", size = 385950, upload-time = "2026-07-23T18:02:09.475Z" }, + { url = "https://files.pythonhosted.org/packages/ea/8d/11521bebc3c89c8f47e94bdaa5abea49b15946c1d0b9d1464e0efd6623d9/clickhouse_connect-1.6.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:064144e1a7f6075ade3c13b8ce49517ca92f86c0c4842bd2abb52a475c026025", size = 374386, upload-time = "2026-07-23T18:02:10.959Z" }, + { url = "https://files.pythonhosted.org/packages/ea/1f/66ed0c9fa39be881f6e1b5b0f570564f6a03b5ee938c614a9e7aa967f63b/clickhouse_connect-1.6.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:069f105173cd04263d3bb710381edae341dc9a3aa42fb4673017a617a5a1237a", size = 1335168, upload-time = "2026-07-23T18:02:12.247Z" }, + { url = "https://files.pythonhosted.org/packages/c7/bb/78a23f75d3750e10a6740c9643505f4b5c5e2cdb2fee558b3f2af4803b39/clickhouse_connect-1.6.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:30bb77e7aa842550d5265c4aaa3e432b01440fd116d63e7e79ae79c380971ade", size = 1358890, upload-time = "2026-07-23T18:02:13.807Z" }, + { url = "https://files.pythonhosted.org/packages/90/91/e809ac0953394009137eba72bb2f727c6a63e55a1c019a07a8f834589ae2/clickhouse_connect-1.6.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4372dbcbd7b1c897c8d7062832468f82af27fa731c7488421ec7f0a882e1580b", size = 1312423, upload-time = "2026-07-23T18:02:15.314Z" }, + { url = "https://files.pythonhosted.org/packages/66/6b/3cf46e7c270852673307f61d5206875beb6a367853b4d0787bd6d97fb01a/clickhouse_connect-1.6.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a5285b9005d5a3ba41d6f148c957e4df8e37a38ad27835a4ad9c31fe21083149", size = 1354991, upload-time = "2026-07-23T18:02:17.216Z" }, + { url = "https://files.pythonhosted.org/packages/12/1e/793672cd06e32ba16d09e11b687651bda05c9e82e8a64c34c5ebb9f0a7a6/clickhouse_connect-1.6.0-cp312-cp312-win32.whl", hash = "sha256:bd8de68d176a807fd16b85c3c5639c25bbdc81c203217f9e985e9560357fecd5", size = 349053, upload-time = "2026-07-23T18:02:18.736Z" }, + { url = "https://files.pythonhosted.org/packages/45/72/e939a153b6266a4ae4ef32820567a5cc5aa4069b3adbc59b02b861fd140f/clickhouse_connect-1.6.0-cp312-cp312-win_amd64.whl", hash = "sha256:2f8b8060a8277509db77b14a01ed70683a0d9becba7a9994a5e59e42e9f98b37", size = 368135, upload-time = "2026-07-23T18:02:20.235Z" }, + { url = "https://files.pythonhosted.org/packages/fe/4e/c5bf9989371204cfeca68b7f1663509b62662fd25134cd31b81be3cae2cc/clickhouse_connect-1.6.0-cp312-cp312-win_arm64.whl", hash = "sha256:cf66e5bcfba8a183e40485efcfc80329ec75104e22dd02485fe73c0ab9073b19", size = 349140, upload-time = "2026-07-23T18:02:21.935Z" }, + { url = "https://files.pythonhosted.org/packages/e0/58/6daab3e2c2e5e12bb3f658a665c8b36b635b45bf2bb879c429ab736455ba/clickhouse_connect-1.6.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8128dfe4a17aef265d5ba2bb5f2f5a0d5fa76e6aca357c71a17da378c1b30ea6", size = 384027, upload-time = "2026-07-23T18:02:23.482Z" }, + { url = "https://files.pythonhosted.org/packages/f5/6e/abc3575964f6f737274619362cdf83a9c1f21d75febff9eda6301167c15e/clickhouse_connect-1.6.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8d4303987b36ca075cbbe2cff3439ad017f56e1cf2bd936f83b2e6be222ae62f", size = 372729, upload-time = "2026-07-23T18:02:24.855Z" }, + { url = "https://files.pythonhosted.org/packages/8a/eb/afc65f349ef9b38a865ffe58b8fcc9e5bcb14e5a0aa1fb663f07e9c71f99/clickhouse_connect-1.6.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a34e72d1ae6f71be491b2d44bd10d082d04c9e2e2a04d4774498e965d933b21d", size = 1306563, upload-time = "2026-07-23T18:02:26.36Z" }, + { url = "https://files.pythonhosted.org/packages/1a/12/3a6d1b11c01bf936af1168868a2d953e72494537ef142d247fe1fa62d1da/clickhouse_connect-1.6.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:444564bad257b8923005431d661e347e5360cd686aae67091de3796a465cfec1", size = 1330861, upload-time = "2026-07-23T18:02:28.135Z" }, + { url = "https://files.pythonhosted.org/packages/33/0c/31ce83989fa10606c7108c04caf1d9b662048918e9f2ac0aa2a4fa407929/clickhouse_connect-1.6.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:794735a62d0a46688d50652876bcb50116c5e912577e9a469ef6ed1aa52b4d7b", size = 1283562, upload-time = "2026-07-23T18:02:29.82Z" }, + { url = "https://files.pythonhosted.org/packages/c5/53/9613b4b70b66d662e1a0cffecd55fce5eedc398c3aefd08422298c3e88ab/clickhouse_connect-1.6.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:75390e6ff6b398a88e0cac4907f785763fc9e476314cbafcfb24553bdbd92d28", size = 1326848, upload-time = "2026-07-23T18:02:31.67Z" }, + { url = "https://files.pythonhosted.org/packages/93/1e/6569d51fea552e69af33feec06f3236c450329ef8f9d19b107888ca574fd/clickhouse_connect-1.6.0-cp313-cp313-win32.whl", hash = "sha256:1a84eb6d545e2647a51b3b4ffa0da4a49f308aaaa096e58c44fb749c171a7d4a", size = 348445, upload-time = "2026-07-23T18:02:33.188Z" }, + { url = "https://files.pythonhosted.org/packages/42/8a/0f13f9f02af1c564fab34346c1496a8254eaa5cb81f10111e72550ea5472/clickhouse_connect-1.6.0-cp313-cp313-win_amd64.whl", hash = "sha256:483e34a7841dbf3d2863e784a129503cc37a910a05464f2f48ce3e9b2f0fc007", size = 366668, upload-time = "2026-07-23T18:02:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/20/46/75a37ee92929327870ab371e3186074cd4806ed16cd5ae604cfd0f54f674/clickhouse_connect-1.6.0-cp313-cp313-win_arm64.whl", hash = "sha256:cd4c1dee496ad3a811237ff8066084c18cdaa46a046f9e0eeacce107ac29c83c", size = 348490, upload-time = "2026-07-23T18:02:36.489Z" }, + { url = "https://files.pythonhosted.org/packages/ba/2a/4d4c036fcea08ed797991d3e74eb8550d8c10219d095867cb8a6030cd90d/clickhouse_connect-1.6.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:c62bd3cc58aa0f6d4cac7012d6e5698cdc23111e3938a809596a5af8459470bd", size = 384372, upload-time = "2026-07-23T18:02:38.043Z" }, + { url = "https://files.pythonhosted.org/packages/d1/38/4ea401d123541348f90026f7e0387e324fe624a880773c318c30d30832b1/clickhouse_connect-1.6.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:51905921e7c98a11e7551663d263f28a715b14fdda191569fc71ddcde964ce52", size = 373618, upload-time = "2026-07-23T18:02:39.512Z" }, + { url = "https://files.pythonhosted.org/packages/87/4f/853d131e34cd85cf66ac5a9fdbaef80920c359873e216ac2310417d042e4/clickhouse_connect-1.6.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c924a2dea34578eda62e99a95988134ddaf4f66aa3f842ebd0980973b6acad56", size = 1304307, upload-time = "2026-07-23T18:02:41.209Z" }, + { url = "https://files.pythonhosted.org/packages/11/c8/2c59851c115242556d1b10629dde02b19f3fd232ab69654e483aa5a809e3/clickhouse_connect-1.6.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:163eb12e7aaeefe9e51a483d3ad40502e15ac2ef94cf1f5b7fe8e9f5303d161b", size = 1320249, upload-time = "2026-07-23T18:02:42.736Z" }, + { url = "https://files.pythonhosted.org/packages/1d/47/b4ed658107e42ff6168428dbc2f02ecf6466dd8f810411038a834de4799f/clickhouse_connect-1.6.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:34969b2a57d9372921aef2b0b51fb858d81ce952c5249b6cb5a0d459e846854c", size = 1283036, upload-time = "2026-07-23T18:02:44.508Z" }, + { url = "https://files.pythonhosted.org/packages/7c/aa/06e5d48c902af21d08a8a1162e50ce6aae510cac0d8bf6c1dc1c03ac618f/clickhouse_connect-1.6.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6bcdb6039bb55870229ec46784840f1420362e3979c74abdc6a53d1ae95718eb", size = 1317414, upload-time = "2026-07-23T18:02:46.511Z" }, + { url = "https://files.pythonhosted.org/packages/2e/12/31f99f76437dca6fbbb33f064a36b2ba3b621d2a317e6770a46b2868b90f/clickhouse_connect-1.6.0-cp314-cp314-win32.whl", hash = "sha256:bdd153037df79b684a30372395681bd673b562e9d6c5159c3a08bffc38ad8185", size = 351328, upload-time = "2026-07-23T18:02:48.19Z" }, + { url = "https://files.pythonhosted.org/packages/72/36/0843ddb2bd27c4b2412895a6fd951ec3fd9cf674f058cae90fa678650e37/clickhouse_connect-1.6.0-cp314-cp314-win_amd64.whl", hash = "sha256:9a7af9bf7b992e5098ed9a0c2edf18d19c6acb2ccc77ee1d2da4e7e5d9dc7ddc", size = 371279, upload-time = "2026-07-23T18:02:49.731Z" }, + { url = "https://files.pythonhosted.org/packages/02/a2/6355a74d18bc79d8c5ba9bb5886689b93fc9c67380d2eda06c267bd339ea/clickhouse_connect-1.6.0-cp314-cp314-win_arm64.whl", hash = "sha256:3cc6bdeafc743739f809671eb3c8fe9af1701981a43c624d9f5bea1e1409e5c4", size = 353282, upload-time = "2026-07-23T18:02:51.418Z" }, + { url = "https://files.pythonhosted.org/packages/e1/8b/c4d9a9e1bb21132323caab2784f1e04595c37945b6f3acb1644914dbba6c/clickhouse_connect-1.6.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:9090c972f3d3ab40fc046913cf51c611345d603c922483e3239633f2cda16a3c", size = 402355, upload-time = "2026-07-23T18:02:52.904Z" }, + { url = "https://files.pythonhosted.org/packages/5a/01/544eb85b743556328d596a8ac11f115ae1192a0378c82e25d42d3b960525/clickhouse_connect-1.6.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9e64e332983fb268350abc76119c14da7b0640373a77ecf4f310240c3fdb4b6d", size = 394992, upload-time = "2026-07-23T18:02:54.52Z" }, + { url = "https://files.pythonhosted.org/packages/2c/e7/8594f336a47f39ea801c4d3bedf949f1ed50277fafe40edf4e58e62153f0/clickhouse_connect-1.6.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c793706c87ab04b29a58e5bc2c0f53806c257859e9ceb859b2840c5aabe56402", size = 1392111, upload-time = "2026-07-23T18:02:56.481Z" }, + { url = "https://files.pythonhosted.org/packages/22/bf/c85f78c4063b51b454ccd071b8d703c46a869a6edb77b718a558b05f5500/clickhouse_connect-1.6.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9dac7ce419ced91f035bd26a86dd49067461547cb9e3b9b1d7c5975b460b650c", size = 1376316, upload-time = "2026-07-23T18:02:58.221Z" }, + { url = "https://files.pythonhosted.org/packages/37/26/818d83db4d5b5a208956c600737d3d1e987eca8c104d3b25e6450c42aada/clickhouse_connect-1.6.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:01367cc75ddd3081f750ccba65c252385feff72f38f68613ea02dd056d047922", size = 1349149, upload-time = "2026-07-23T18:03:00.262Z" }, + { url = "https://files.pythonhosted.org/packages/8b/0c/7021c7227df21bdce7059e7fd8275c70452d364987b8ace945366289eb39/clickhouse_connect-1.6.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:508d3b7cc893b5d889b92702cc7dc4a1c975f59ee8da383054076969e4cb17c3", size = 1366133, upload-time = "2026-07-23T18:03:01.956Z" }, + { url = "https://files.pythonhosted.org/packages/21/83/016567c0212d9684789d2d9d2565feaf03a4fe5bb3a8127be57a37ab6263/clickhouse_connect-1.6.0-cp314-cp314t-win32.whl", hash = "sha256:30a183095bc367dcc4695869754e36ce2177264d0ef79421990ee24700810ee0", size = 364631, upload-time = "2026-07-23T18:03:03.695Z" }, + { url = "https://files.pythonhosted.org/packages/2f/a8/c90ea156f7753ce1034c44a2cddcbac3ca7e0bcc0c5e18a12f52ff286cab/clickhouse_connect-1.6.0-cp314-cp314t-win_amd64.whl", hash = "sha256:79caafbd2b1364d4b63c2e7c625cb424c8572b9cedc62b4ac2289cfbdf875487", size = 385961, upload-time = "2026-07-23T18:03:05.313Z" }, + { url = "https://files.pythonhosted.org/packages/a8/74/1fcd2854aed712204d62eb1365e42962cb1dfd6ecfad8948bf30742cda13/clickhouse_connect-1.6.0-cp314-cp314t-win_arm64.whl", hash = "sha256:fe4bdd4e09f52bff46c2f920877a08a26a44d91fdb5fc0be76edb67395440192", size = 364215, upload-time = "2026-07-23T18:03:06.936Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "lz4" +version = "4.4.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/57/51/f1b86d93029f418033dddf9b9f79c8d2641e7454080478ee2aab5123173e/lz4-4.4.5.tar.gz", hash = "sha256:5f0b9e53c1e82e88c10d7c180069363980136b9d7a8306c4dca4f760d60c39f0", size = 172886, upload-time = "2025-11-03T13:02:36.061Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/93/5b/6edcd23319d9e28b1bedf32768c3d1fd56eed8223960a2c47dacd2cec2af/lz4-4.4.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d6da84a26b3aa5da13a62e4b89ab36a396e9327de8cd48b436a3467077f8ccd4", size = 207391, upload-time = "2025-11-03T13:01:36.644Z" }, + { url = "https://files.pythonhosted.org/packages/34/36/5f9b772e85b3d5769367a79973b8030afad0d6b724444083bad09becd66f/lz4-4.4.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:61d0ee03e6c616f4a8b69987d03d514e8896c8b1b7cc7598ad029e5c6aedfd43", size = 207146, upload-time = "2025-11-03T13:01:37.928Z" }, + { url = "https://files.pythonhosted.org/packages/04/f4/f66da5647c0d72592081a37c8775feacc3d14d2625bbdaabd6307c274565/lz4-4.4.5-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:33dd86cea8375d8e5dd001e41f321d0a4b1eb7985f39be1b6a4f466cd480b8a7", size = 1292623, upload-time = "2025-11-03T13:01:39.341Z" }, + { url = "https://files.pythonhosted.org/packages/85/fc/5df0f17467cdda0cad464a9197a447027879197761b55faad7ca29c29a04/lz4-4.4.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:609a69c68e7cfcfa9d894dc06be13f2e00761485b62df4e2472f1b66f7b405fb", size = 1279982, upload-time = "2025-11-03T13:01:40.816Z" }, + { url = "https://files.pythonhosted.org/packages/25/3b/b55cb577aa148ed4e383e9700c36f70b651cd434e1c07568f0a86c9d5fbb/lz4-4.4.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:75419bb1a559af00250b8f1360d508444e80ed4b26d9d40ec5b09fe7875cb989", size = 1368674, upload-time = "2025-11-03T13:01:42.118Z" }, + { url = "https://files.pythonhosted.org/packages/fb/31/e97e8c74c59ea479598e5c55cbe0b1334f03ee74ca97726e872944ed42df/lz4-4.4.5-cp311-cp311-win32.whl", hash = "sha256:12233624f1bc2cebc414f9efb3113a03e89acce3ab6f72035577bc61b270d24d", size = 88168, upload-time = "2025-11-03T13:01:43.282Z" }, + { url = "https://files.pythonhosted.org/packages/18/47/715865a6c7071f417bef9b57c8644f29cb7a55b77742bd5d93a609274e7e/lz4-4.4.5-cp311-cp311-win_amd64.whl", hash = "sha256:8a842ead8ca7c0ee2f396ca5d878c4c40439a527ebad2b996b0444f0074ed004", size = 99491, upload-time = "2025-11-03T13:01:44.167Z" }, + { url = "https://files.pythonhosted.org/packages/14/e7/ac120c2ca8caec5c945e6356ada2aa5cfabd83a01e3170f264a5c42c8231/lz4-4.4.5-cp311-cp311-win_arm64.whl", hash = "sha256:83bc23ef65b6ae44f3287c38cbf82c269e2e96a26e560aa551735883388dcc4b", size = 91271, upload-time = "2025-11-03T13:01:45.016Z" }, + { url = "https://files.pythonhosted.org/packages/1b/ac/016e4f6de37d806f7cc8f13add0a46c9a7cfc41a5ddc2bc831d7954cf1ce/lz4-4.4.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:df5aa4cead2044bab83e0ebae56e0944cc7fcc1505c7787e9e1057d6d549897e", size = 207163, upload-time = "2025-11-03T13:01:45.895Z" }, + { url = "https://files.pythonhosted.org/packages/8d/df/0fadac6e5bd31b6f34a1a8dbd4db6a7606e70715387c27368586455b7fc9/lz4-4.4.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6d0bf51e7745484d2092b3a51ae6eb58c3bd3ce0300cf2b2c14f76c536d5697a", size = 207150, upload-time = "2025-11-03T13:01:47.205Z" }, + { url = "https://files.pythonhosted.org/packages/b7/17/34e36cc49bb16ca73fb57fbd4c5eaa61760c6b64bce91fcb4e0f4a97f852/lz4-4.4.5-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:7b62f94b523c251cf32aa4ab555f14d39bd1a9df385b72443fd76d7c7fb051f5", size = 1292045, upload-time = "2025-11-03T13:01:48.667Z" }, + { url = "https://files.pythonhosted.org/packages/90/1c/b1d8e3741e9fc89ed3b5f7ef5f22586c07ed6bb04e8343c2e98f0fa7ff04/lz4-4.4.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2c3ea562c3af274264444819ae9b14dbbf1ab070aff214a05e97db6896c7597e", size = 1279546, upload-time = "2025-11-03T13:01:50.159Z" }, + { url = "https://files.pythonhosted.org/packages/55/d9/e3867222474f6c1b76e89f3bd914595af69f55bf2c1866e984c548afdc15/lz4-4.4.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:24092635f47538b392c4eaeff14c7270d2c8e806bf4be2a6446a378591c5e69e", size = 1368249, upload-time = "2025-11-03T13:01:51.273Z" }, + { url = "https://files.pythonhosted.org/packages/b2/e7/d667d337367686311c38b580d1ca3d5a23a6617e129f26becd4f5dc458df/lz4-4.4.5-cp312-cp312-win32.whl", hash = "sha256:214e37cfe270948ea7eb777229e211c601a3e0875541c1035ab408fbceaddf50", size = 88189, upload-time = "2025-11-03T13:01:52.605Z" }, + { url = "https://files.pythonhosted.org/packages/a5/0b/a54cd7406995ab097fceb907c7eb13a6ddd49e0b231e448f1a81a50af65c/lz4-4.4.5-cp312-cp312-win_amd64.whl", hash = "sha256:713a777de88a73425cf08eb11f742cd2c98628e79a8673d6a52e3c5f0c116f33", size = 99497, upload-time = "2025-11-03T13:01:53.477Z" }, + { url = "https://files.pythonhosted.org/packages/6a/7e/dc28a952e4bfa32ca16fa2eb026e7a6ce5d1411fcd5986cd08c74ec187b9/lz4-4.4.5-cp312-cp312-win_arm64.whl", hash = "sha256:a88cbb729cc333334ccfb52f070463c21560fca63afcf636a9f160a55fac3301", size = 91279, upload-time = "2025-11-03T13:01:54.419Z" }, + { url = "https://files.pythonhosted.org/packages/2f/46/08fd8ef19b782f301d56a9ccfd7dafec5fd4fc1a9f017cf22a1accb585d7/lz4-4.4.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:6bb05416444fafea170b07181bc70640975ecc2a8c92b3b658c554119519716c", size = 207171, upload-time = "2025-11-03T13:01:56.595Z" }, + { url = "https://files.pythonhosted.org/packages/8f/3f/ea3334e59de30871d773963997ecdba96c4584c5f8007fd83cfc8f1ee935/lz4-4.4.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b424df1076e40d4e884cfcc4c77d815368b7fb9ebcd7e634f937725cd9a8a72a", size = 207163, upload-time = "2025-11-03T13:01:57.721Z" }, + { url = "https://files.pythonhosted.org/packages/41/7b/7b3a2a0feb998969f4793c650bb16eff5b06e80d1f7bff867feb332f2af2/lz4-4.4.5-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:216ca0c6c90719731c64f41cfbd6f27a736d7e50a10b70fad2a9c9b262ec923d", size = 1292136, upload-time = "2025-11-03T13:02:00.375Z" }, + { url = "https://files.pythonhosted.org/packages/89/d1/f1d259352227bb1c185288dd694121ea303e43404aa77560b879c90e7073/lz4-4.4.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:533298d208b58b651662dd972f52d807d48915176e5b032fb4f8c3b6f5fe535c", size = 1279639, upload-time = "2025-11-03T13:02:01.649Z" }, + { url = "https://files.pythonhosted.org/packages/d2/fb/ba9256c48266a09012ed1d9b0253b9aa4fe9cdff094f8febf5b26a4aa2a2/lz4-4.4.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:451039b609b9a88a934800b5fc6ee401c89ad9c175abf2f4d9f8b2e4ef1afc64", size = 1368257, upload-time = "2025-11-03T13:02:03.35Z" }, + { url = "https://files.pythonhosted.org/packages/a5/6d/dee32a9430c8b0e01bbb4537573cabd00555827f1a0a42d4e24ca803935c/lz4-4.4.5-cp313-cp313-win32.whl", hash = "sha256:a5f197ffa6fc0e93207b0af71b302e0a2f6f29982e5de0fbda61606dd3a55832", size = 88191, upload-time = "2025-11-03T13:02:04.406Z" }, + { url = "https://files.pythonhosted.org/packages/18/e0/f06028aea741bbecb2a7e9648f4643235279a770c7ffaf70bd4860c73661/lz4-4.4.5-cp313-cp313-win_amd64.whl", hash = "sha256:da68497f78953017deb20edff0dba95641cc86e7423dfadf7c0264e1ac60dc22", size = 99502, upload-time = "2025-11-03T13:02:05.886Z" }, + { url = "https://files.pythonhosted.org/packages/61/72/5bef44afb303e56078676b9f2486f13173a3c1e7f17eaac1793538174817/lz4-4.4.5-cp313-cp313-win_arm64.whl", hash = "sha256:c1cfa663468a189dab510ab231aad030970593f997746d7a324d40104db0d0a9", size = 91285, upload-time = "2025-11-03T13:02:06.77Z" }, + { url = "https://files.pythonhosted.org/packages/49/55/6a5c2952971af73f15ed4ebfdd69774b454bd0dc905b289082ca8664fba1/lz4-4.4.5-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:67531da3b62f49c939e09d56492baf397175ff39926d0bd5bd2d191ac2bff95f", size = 207348, upload-time = "2025-11-03T13:02:08.117Z" }, + { url = "https://files.pythonhosted.org/packages/4e/d7/fd62cbdbdccc35341e83aabdb3f6d5c19be2687d0a4eaf6457ddf53bba64/lz4-4.4.5-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:a1acbbba9edbcbb982bc2cac5e7108f0f553aebac1040fbec67a011a45afa1ba", size = 207340, upload-time = "2025-11-03T13:02:09.152Z" }, + { url = "https://files.pythonhosted.org/packages/77/69/225ffadaacb4b0e0eb5fd263541edd938f16cd21fe1eae3cd6d5b6a259dc/lz4-4.4.5-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:a482eecc0b7829c89b498fda883dbd50e98153a116de612ee7c111c8bcf82d1d", size = 1293398, upload-time = "2025-11-03T13:02:10.272Z" }, + { url = "https://files.pythonhosted.org/packages/c6/9e/2ce59ba4a21ea5dc43460cba6f34584e187328019abc0e66698f2b66c881/lz4-4.4.5-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e099ddfaa88f59dd8d36c8a3c66bd982b4984edf127eb18e30bb49bdba68ce67", size = 1281209, upload-time = "2025-11-03T13:02:12.091Z" }, + { url = "https://files.pythonhosted.org/packages/80/4f/4d946bd1624ec229b386a3bc8e7a85fa9a963d67d0a62043f0af0978d3da/lz4-4.4.5-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a2af2897333b421360fdcce895c6f6281dc3fab018d19d341cf64d043fc8d90d", size = 1369406, upload-time = "2025-11-03T13:02:13.683Z" }, + { url = "https://files.pythonhosted.org/packages/02/a2/d429ba4720a9064722698b4b754fb93e42e625f1318b8fe834086c7c783b/lz4-4.4.5-cp313-cp313t-win32.whl", hash = "sha256:66c5de72bf4988e1b284ebdd6524c4bead2c507a2d7f172201572bac6f593901", size = 88325, upload-time = "2025-11-03T13:02:14.743Z" }, + { url = "https://files.pythonhosted.org/packages/4b/85/7ba10c9b97c06af6c8f7032ec942ff127558863df52d866019ce9d2425cf/lz4-4.4.5-cp313-cp313t-win_amd64.whl", hash = "sha256:cdd4bdcbaf35056086d910d219106f6a04e1ab0daa40ec0eeef1626c27d0fddb", size = 99643, upload-time = "2025-11-03T13:02:15.978Z" }, + { url = "https://files.pythonhosted.org/packages/77/4d/a175459fb29f909e13e57c8f475181ad8085d8d7869bd8ad99033e3ee5fa/lz4-4.4.5-cp313-cp313t-win_arm64.whl", hash = "sha256:28ccaeb7c5222454cd5f60fcd152564205bcb801bd80e125949d2dfbadc76bbd", size = 91504, upload-time = "2025-11-03T13:02:17.313Z" }, + { url = "https://files.pythonhosted.org/packages/63/9c/70bdbdb9f54053a308b200b4678afd13efd0eafb6ddcbb7f00077213c2e5/lz4-4.4.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:c216b6d5275fc060c6280936bb3bb0e0be6126afb08abccde27eed23dead135f", size = 207586, upload-time = "2025-11-03T13:02:18.263Z" }, + { url = "https://files.pythonhosted.org/packages/b6/cb/bfead8f437741ce51e14b3c7d404e3a1f6b409c440bad9b8f3945d4c40a7/lz4-4.4.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c8e71b14938082ebaf78144f3b3917ac715f72d14c076f384a4c062df96f9df6", size = 207161, upload-time = "2025-11-03T13:02:19.286Z" }, + { url = "https://files.pythonhosted.org/packages/e7/18/b192b2ce465dfbeabc4fc957ece7a1d34aded0d95a588862f1c8a86ac448/lz4-4.4.5-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:9b5e6abca8df9f9bdc5c3085f33ff32cdc86ed04c65e0355506d46a5ac19b6e9", size = 1292415, upload-time = "2025-11-03T13:02:20.829Z" }, + { url = "https://files.pythonhosted.org/packages/67/79/a4e91872ab60f5e89bfad3e996ea7dc74a30f27253faf95865771225ccba/lz4-4.4.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3b84a42da86e8ad8537aabef062e7f661f4a877d1c74d65606c49d835d36d668", size = 1279920, upload-time = "2025-11-03T13:02:22.013Z" }, + { url = "https://files.pythonhosted.org/packages/f1/01/d52c7b11eaa286d49dae619c0eec4aabc0bf3cda7a7467eb77c62c4471f3/lz4-4.4.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0bba042ec5a61fa77c7e380351a61cb768277801240249841defd2ff0a10742f", size = 1368661, upload-time = "2025-11-03T13:02:23.208Z" }, + { url = "https://files.pythonhosted.org/packages/f7/da/137ddeea14c2cb86864838277b2607d09f8253f152156a07f84e11768a28/lz4-4.4.5-cp314-cp314-win32.whl", hash = "sha256:bd85d118316b53ed73956435bee1997bd06cc66dd2fa74073e3b1322bd520a67", size = 90139, upload-time = "2025-11-03T13:02:24.301Z" }, + { url = "https://files.pythonhosted.org/packages/18/2c/8332080fd293f8337779a440b3a143f85e374311705d243439a3349b81ad/lz4-4.4.5-cp314-cp314-win_amd64.whl", hash = "sha256:92159782a4502858a21e0079d77cdcaade23e8a5d252ddf46b0652604300d7be", size = 101497, upload-time = "2025-11-03T13:02:25.187Z" }, + { url = "https://files.pythonhosted.org/packages/ca/28/2635a8141c9a4f4bc23f5135a92bbcf48d928d8ca094088c962df1879d64/lz4-4.4.5-cp314-cp314-win_arm64.whl", hash = "sha256:d994b87abaa7a88ceb7a37c90f547b8284ff9da694e6afcfaa8568d739faf3f7", size = 93812, upload-time = "2025-11-03T13:02:26.133Z" }, +] + +[[package]] +name = "numpy" +version = "2.4.6" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.12' and sys_platform == 'win32'", + "python_full_version < '3.12' and sys_platform == 'emscripten'", + "python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] +sdist = { url = "https://files.pythonhosted.org/packages/d0/ad/fed0499ce6a338d2a03ebae59cd15093910c8875328855781952abf6c2fe/numpy-2.4.6.tar.gz", hash = "sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda", size = 20735807, upload-time = "2026-05-18T23:37:14.07Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/49/ec46835a70be8fa6446c495126ac84fdb28cb2558e1620ffb87a10c8b64c/numpy-2.4.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0280e0356c0829a18d9de1cb7eee50ec22ca639878d7240307ca0943d73cd2c4", size = 16969194, upload-time = "2026-05-18T23:33:13.503Z" }, + { url = "https://files.pythonhosted.org/packages/0e/0d/f5957185c0ee2f3e12f78715aa9e3b353fd83633316c8532b38faa37e3f6/numpy-2.4.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:110f8b71aacb688ec69062bb7f6938a0f8acb01b7c1c4beb453c65b6d234584d", size = 14964111, upload-time = "2026-05-18T23:33:17.795Z" }, + { url = "https://files.pythonhosted.org/packages/ad/40/40a40ee0ddf7ceb782c49af278894b686e586d65d8c1889c8b5da01a3d7d/numpy-2.4.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:4cfe66903cc32a9921a6733d96b19bb6abf310397581bbad89c228f5abaf0ee8", size = 5469159, upload-time = "2026-05-18T23:33:20.654Z" }, + { url = "https://files.pythonhosted.org/packages/63/13/f9a8046535cb21deae82f8d03de9617e08882d274fad2539630761888228/numpy-2.4.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:8155154c7c691289fe18f510b5d4657c68c67989f293f0535a91360392ff6538", size = 6798936, upload-time = "2026-05-18T23:33:22.987Z" }, + { url = "https://files.pythonhosted.org/packages/33/a8/6fa8c1a345a8c85dbb21932c447bee07c30a2c2a3f31e369c0a84b300147/numpy-2.4.6-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ab0a9c4ffb1a6d95ef519fe4247dba8eb6b18ad93999f76b7f657039acabd47", size = 15966692, upload-time = "2026-05-18T23:33:26.62Z" }, + { url = "https://files.pythonhosted.org/packages/02/03/74fe2a4cb3817d94d86402f2506554130a2f01414e299b5a843e5a8a957f/numpy-2.4.6-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:89cd468399cfd2504718f0ba50e410dca55a170b61a02ad92bb18c8a65186e93", size = 16918164, upload-time = "2026-05-18T23:33:29.955Z" }, + { url = "https://files.pythonhosted.org/packages/c5/80/3615be3313f7e7696609bc194b9f0101da809df79e859bdb84e0cd043f46/numpy-2.4.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c2d37ab77531417474168eb79d6d80b14f821a966818505d03013d0833edb7a8", size = 17322877, upload-time = "2026-05-18T23:33:34.724Z" }, + { url = "https://files.pythonhosted.org/packages/ca/ac/a691e0fe2675e370d0e08ff905adc49a1c8830e8cae03efe4477e92cd55d/numpy-2.4.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f407cb6b8e9d6d8c626bc73c945db1706035af8fd632295547bf1c9e46d092d6", size = 18651487, upload-time = "2026-05-18T23:33:38.217Z" }, + { url = "https://files.pythonhosted.org/packages/15/a7/9bc1cd626d7bf6869bfedf27b91b6ab5dd607758bf8e959d6fa80c6a59cb/numpy-2.4.6-cp311-cp311-win32.whl", hash = "sha256:ddea102b48f9e339f3948bf22040944184627a30fdf7f858667673b9c5f033c8", size = 6233945, upload-time = "2026-05-18T23:33:41.331Z" }, + { url = "https://files.pythonhosted.org/packages/c5/31/7fc6239c12bce7e931463251cca4426c465e1876ba3cc785402ef4dd8f4e/numpy-2.4.6-cp311-cp311-win_amd64.whl", hash = "sha256:1e254a00cdf42b1e4d5b3d68d33af63268d41340d8885df2ab6470f2e1500147", size = 12608406, upload-time = "2026-05-18T23:33:44.131Z" }, + { url = "https://files.pythonhosted.org/packages/27/83/140f85a466595a16382996a1bf06b2b54bcd597488921b0c9daaeeda72af/numpy-2.4.6-cp311-cp311-win_arm64.whl", hash = "sha256:ed9749eef4cbd126da3dc1d6bcb3a57f5eb7ac6a6484146bdbf743f552dfc577", size = 10479528, upload-time = "2026-05-18T23:33:50.725Z" }, + { url = "https://files.pythonhosted.org/packages/95/2a/3d7b5ac8aac24feaf9ad7ed58f45b0bbc06d37e4338ae84c9f2298b570f9/numpy-2.4.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:001fbb8e08d942dd57599e781f2472269ee7f2755fae407b4f67b2f0b17da3f1", size = 16689119, upload-time = "2026-05-18T23:33:54.065Z" }, + { url = "https://files.pythonhosted.org/packages/ea/12/92c4c131527599e8288d6918e888d88726f84d805d784b771f32408aeaef/numpy-2.4.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ebfb099f8dcf083deef3ac1ca4c1503f387cf76296fcb3816b66f5ecb5f54fdb", size = 14699246, upload-time = "2026-05-18T23:33:57.621Z" }, + { url = "https://files.pythonhosted.org/packages/ad/fe/c0a6b7b2ca128a8fb228575147073b660656734b8ebe4d76c8fd748dcc79/numpy-2.4.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:3213d622a0283a39a93d188f3cf72b26862df52fbb4ca3697f51705016523d41", size = 5204410, upload-time = "2026-05-18T23:34:00.302Z" }, + { url = "https://files.pythonhosted.org/packages/f3/d4/9770d14ba719432bb90a421bfd443872ed0f70f7264b64bec12ea363d5fd/numpy-2.4.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:357cc07a6d7b0b182ff02249616a03742827ebb1277546b5c7cd7f7620a45698", size = 6551240, upload-time = "2026-05-18T23:34:02.852Z" }, + { url = "https://files.pythonhosted.org/packages/c9/c6/50a46a6205feba2343f1d6d17438107c5dc491ed1c736e6ea68689fd906b/numpy-2.4.6-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f9fb9157b4ce2971008323afe46053787b526ef624fea915b261468a8421a0f", size = 15671012, upload-time = "2026-05-18T23:34:05.485Z" }, + { url = "https://files.pythonhosted.org/packages/99/60/14115e6364fa676c5397c2ad3004e527e9aa487abf5d0706ec81bbd08529/numpy-2.4.6-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90f9849678c75fe7afa2d348ac842c168b0a4d3d61919687216dfc547976d853", size = 16645538, upload-time = "2026-05-18T23:34:09.265Z" }, + { url = "https://files.pythonhosted.org/packages/ae/c5/693cbe59e57db94d2231fa519ca3978dc9e19da5a8f088588f5c6e947ff2/numpy-2.4.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c1a2af6c6ef86344a6b0db6b97834208bf598db514f2b155042439b62605601a", size = 17020706, upload-time = "2026-05-18T23:34:13.053Z" }, + { url = "https://files.pythonhosted.org/packages/ef/fc/85b7c4eff9b4966ade25c2273cf7e7012e92366c032058653934b37de044/numpy-2.4.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e5805d5a22fd19c8ccff10a9561f9df94436b0545619ea579db2d3c35294bce2", size = 18368541, upload-time = "2026-05-18T23:34:17.024Z" }, + { url = "https://files.pythonhosted.org/packages/f6/81/e1b27545deedce7f4a0b348618c6b62d74e36a4dc9ccd42f3eb2f85eee32/numpy-2.4.6-cp312-cp312-win32.whl", hash = "sha256:e3eeb0aabd6bd5ce64faae67e9935203a6991b4bc2a485a767fbafb2c5125f45", size = 5962825, upload-time = "2026-05-18T23:34:20.3Z" }, + { url = "https://files.pythonhosted.org/packages/ab/ca/feab00bd44aa5fe1ad2c18f08b4d3bb92e26484b0b1d1443897809ed528c/numpy-2.4.6-cp312-cp312-win_amd64.whl", hash = "sha256:d8e8286dd7cea7895157318d1b91cdacac64c479f3cbc8dce548331728484751", size = 12321687, upload-time = "2026-05-18T23:34:23.095Z" }, + { url = "https://files.pythonhosted.org/packages/63/cf/5a6d34850a39d1093558564f77ee8e8e0bee5061151b8f05a55711001ec7/numpy-2.4.6-cp312-cp312-win_arm64.whl", hash = "sha256:4081eb135ac24158bd51cdfbef16f1c64df7063b1143f24731387137c092bec8", size = 10221482, upload-time = "2026-05-18T23:34:25.876Z" }, + { url = "https://files.pythonhosted.org/packages/fb/82/bdab26d7438c6791ca31b7c024ca37c1eab8b726ba236129005cd4a06e45/numpy-2.4.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:511dbaf848decaaaf4b4ca48032619fb3138710c4bf7da7617765edad1ef96b0", size = 16684648, upload-time = "2026-05-18T23:34:29.41Z" }, + { url = "https://files.pythonhosted.org/packages/1b/30/a80189bcc7f5e4258b3fbc3968d909d1756f54d023299ecc39ad6fdb9ef8/numpy-2.4.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bf162abab1c1a736333192707cef898e735a5ca00f38f27eeedf44b39d9e85eb", size = 14693902, upload-time = "2026-05-18T23:34:33.013Z" }, + { url = "https://files.pythonhosted.org/packages/97/12/70b5d0d7c15e1ebb8a6a84a8caa1d19e181d84fb58bb6d70aca29099dec1/numpy-2.4.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:043191bfa8eab18c776647b62723ac9dddece59743b13f49b2016094129c2b3f", size = 5198992, upload-time = "2026-05-18T23:34:36.132Z" }, + { url = "https://files.pythonhosted.org/packages/ba/8c/ebd2a8f8a83541f8d38cc5667e8c2b69cecfd30da6e45693e8158857d44b/numpy-2.4.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:6180d8b35af935aed8ece3a85e0a43f87393ae0ac87c8d2c8bd2c993f7270ef3", size = 6546944, upload-time = "2026-05-18T23:34:38.484Z" }, + { url = "https://files.pythonhosted.org/packages/bb/c5/7b863a97a91671a0338f4253bd3b5a3d3852f0692dae91711c9f4a10e787/numpy-2.4.6-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:72fbe16c6fac95aedf5937fa873445cec2110be35d8a4e9433d7501fd98dae6b", size = 15669392, upload-time = "2026-05-18T23:34:41.257Z" }, + { url = "https://files.pythonhosted.org/packages/a5/9d/3584b9984ca4c047aea75214ce1a4c4c73d849bd71b604264b7f5653f8a8/numpy-2.4.6-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a7830bab239b79cda9c08c2da014761cafb48da6150e1da17ac06283f43b6089", size = 16633220, upload-time = "2026-05-18T23:34:45.075Z" }, + { url = "https://files.pythonhosted.org/packages/05/ae/7c67fba23bd98caec7c99261f3a16072ade14813486b0282cb29846de832/numpy-2.4.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ef4aea96ce4d3b074422cb4f2f64e216bf9e213004bb58ecfdf50ea02ea8eb9a", size = 17020800, upload-time = "2026-05-18T23:34:49.065Z" }, + { url = "https://files.pythonhosted.org/packages/d9/5d/3b6725cb31d983c5e66916f5d36f6d7e5521129e4c4404d64f918292a5b6/numpy-2.4.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dfa20cc6ca228e6b155b11da03825975ce66aea520985dbbddf0f2a5a495c605", size = 18357600, upload-time = "2026-05-18T23:34:52.709Z" }, + { url = "https://files.pythonhosted.org/packages/f7/da/2ccc6c2fe8898dee01d90c75c5f5f914a23daf99e3e0f59516a08760c8b5/numpy-2.4.6-cp313-cp313-win32.whl", hash = "sha256:56b39e5e0622a09a25bf5baf62f4bcf0cb8a41ae6e2819cf49bbc5a74c083f91", size = 5961134, upload-time = "2026-05-18T23:34:55.618Z" }, + { url = "https://files.pythonhosted.org/packages/b5/cd/9cc4dc876fb065d5c220aae4d5e14826b2715331bb7618ce1fb07a679d99/numpy-2.4.6-cp313-cp313-win_amd64.whl", hash = "sha256:c4fc99836233ea196540b17ab0983aff60ed07941751930f5f4d05bc3b3b7359", size = 12318598, upload-time = "2026-05-18T23:34:58.928Z" }, + { url = "https://files.pythonhosted.org/packages/39/1e/c0bcba1f8694116485fe28fd1be698c278fcda4141c5b0e53a2aed8b12a8/numpy-2.4.6-cp313-cp313-win_arm64.whl", hash = "sha256:a7c711e21628b52034bb5ab8d1bce291f752fcc5e92accc615778acee1ff4778", size = 10222272, upload-time = "2026-05-18T23:35:02.167Z" }, + { url = "https://files.pythonhosted.org/packages/63/6d/cc5619247c8f4204e507f5883528372e4ac4bb189e579fb859a12e480b1f/numpy-2.4.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:112b06a867b235ef466ed3508ddf0238050df9c727cafb5301ac385b899189a1", size = 14821197, upload-time = "2026-05-18T23:35:05.468Z" }, + { url = "https://files.pythonhosted.org/packages/00/58/f1c39161c87d9e9bed660f1ed4bafc0e403d5ec9650b6dd77aead07d489b/numpy-2.4.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:eaf7fa2de5c0be8ae6ff8e9bea2ccd725e980541244521d8d4b5f3354a27babe", size = 5326287, upload-time = "2026-05-18T23:35:08.693Z" }, + { url = "https://files.pythonhosted.org/packages/af/57/3917ab0fd97f271a8694513581b8a36c655f111c446852c302f04ccdb6fc/numpy-2.4.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:7265a2f3d436e54ef9f2b52b5c937e6be778781bd97a590319d7348f1c1ca997", size = 6646763, upload-time = "2026-05-18T23:35:11.459Z" }, + { url = "https://files.pythonhosted.org/packages/eb/0f/037e64c494b67581ae18193d770adef354c41f3f2c8ebf865602d949bf8f/numpy-2.4.6-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f74a575920ab21fe304421a3fc28793d82e299cae9eccb37084e9fc7f3617c20", size = 15728070, upload-time = "2026-05-18T23:35:14.79Z" }, + { url = "https://files.pythonhosted.org/packages/21/a6/5d2bae9c9542eb4df16dc9c46dc79c186e9bad53805dfa5399a6023c6db0/numpy-2.4.6-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ede83e07a75dd06bc501566c1eca2afc0d61677c1472ac9ad93fdee6e638a48d", size = 16681752, upload-time = "2026-05-18T23:35:18.836Z" }, + { url = "https://files.pythonhosted.org/packages/92/14/23d1dfb410ae362cd59ce53e936b1513d545eb40db3949ced632e19a459e/numpy-2.4.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:68bb27509ac1b9a3443094260f6326150663b06abe40b73a2f81160623da5b67", size = 17086024, upload-time = "2026-05-18T23:35:22.52Z" }, + { url = "https://files.pythonhosted.org/packages/4b/6e/23595a2c642cdf3bc567877064bdd7f91c8b0038a4453cf2daf7248eafe9/numpy-2.4.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:a0df0043bdb289bde1f62da130d20df23d58b45429f752bc7a8fc5325a225ecd", size = 18403398, upload-time = "2026-05-18T23:35:26.398Z" }, + { url = "https://files.pythonhosted.org/packages/8a/90/0ac3bc947217e66dec77e7cbc6a1979d1af70b6461b82f620d3bccd5e4c8/numpy-2.4.6-cp313-cp313t-win32.whl", hash = "sha256:29a287e0cf63ff528da061de6b9f64a4618da591ca1046aafc54062e40ca7eab", size = 6084971, upload-time = "2026-05-18T23:35:29.387Z" }, + { url = "https://files.pythonhosted.org/packages/77/71/5673e351671a1d2bd6063b91b44f70c0affea7d1516fa7a6572941ba4aa1/numpy-2.4.6-cp313-cp313t-win_amd64.whl", hash = "sha256:25c692919ac5a01f170a3bfcd62d745b24fd095c353d50812637d6fcab442e75", size = 12458532, upload-time = "2026-05-18T23:35:32.175Z" }, + { url = "https://files.pythonhosted.org/packages/3f/88/19d3503c5046e688f049274b27a3ef3d771152fa80d3ba3d01a3dff61abe/numpy-2.4.6-cp313-cp313t-win_arm64.whl", hash = "sha256:1e978ec1e8bd0e0e4de6bb75de9d30cbb74db6b6a2bb727618613703ca0167dd", size = 10291881, upload-time = "2026-05-18T23:35:35.465Z" }, + { url = "https://files.pythonhosted.org/packages/f8/91/3ab2044d05fd16d343c5ac2e69b127f1b2854040dd20b193257c78028bd3/numpy-2.4.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:06ca2f61ec4385a07a6977c55ba998a4466c123642b4a32694d3128fce18c079", size = 16683458, upload-time = "2026-05-18T23:35:38.353Z" }, + { url = "https://files.pythonhosted.org/packages/8e/62/764ce66fa4147ae6d73071a3abf804ffe606f174618697c571acdf26a7c9/numpy-2.4.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:38efbc8de75c7a0fc1ac190162d892787f3f47b57cc291231aafee36b80982b7", size = 14704559, upload-time = "2026-05-18T23:35:42.14Z" }, + { url = "https://files.pythonhosted.org/packages/60/61/23f27c172f022e04025b7dc2367f4d63c1a398120607ec896228649a6f48/numpy-2.4.6-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:d581b735e177fdcdce6fed8e7e8880a3fb6ee4e3653a3ac6af01c6f4c03effc5", size = 5209716, upload-time = "2026-05-18T23:35:45.377Z" }, + { url = "https://files.pythonhosted.org/packages/03/71/21cf70dc6ea3e3acb95fc53a265b2fc248b981f0194ceb5b475271b8809d/numpy-2.4.6-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:0a041d3d761dc3c35cc56ce0351506a02bcbc25f7b169f652435141a17db9096", size = 6543947, upload-time = "2026-05-18T23:35:47.926Z" }, + { url = "https://files.pythonhosted.org/packages/d5/91/64288395ee1799bd2e0b04a305dce9666da90c961e1f3fe982a05ee1c036/numpy-2.4.6-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40fdc1ae7125e518ea98e53e69a4ebc27e1fd50510c47b7ea130cf21e5e1d42b", size = 15685197, upload-time = "2026-05-18T23:35:50.863Z" }, + { url = "https://files.pythonhosted.org/packages/f3/eb/ebffaa97dc55502df69584a8f0dcf07f69a3e0b3e2323670a2722db9aa39/numpy-2.4.6-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a2c306dea656c12c68f51f4cea133cbe78ca7435eb28c735eac1d3ebe73be6e8", size = 16638245, upload-time = "2026-05-18T23:35:54.752Z" }, + { url = "https://files.pythonhosted.org/packages/b8/0b/54f9da33128d7e350fab89c7455902eeae70349ee52bddb448dc4a576f45/numpy-2.4.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:33111801a01c12a8a1e3721f0a9232f8cfc8ae2c6b7098167e6f623c6073f402", size = 17036587, upload-time = "2026-05-18T23:35:58.355Z" }, + { url = "https://files.pythonhosted.org/packages/b6/f0/fdebc1052db1cc37c64beb22072d67cd6d1c71adca1299f53dec2b5e20d3/numpy-2.4.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ae506e6902902557576a26ff33eda8695e7ecb3cb36c3b573a0765dee114ebdb", size = 18363226, upload-time = "2026-05-18T23:36:02.845Z" }, + { url = "https://files.pythonhosted.org/packages/aa/b4/298628d98c72b57e57f7165ae6a481a1deaf6f3c28262a6e4c739c275930/numpy-2.4.6-cp314-cp314-win32.whl", hash = "sha256:aaf159caa35993cb1f56fb9b8e4610d35758e7ca005412eb1daa856a78c9c4b1", size = 6010196, upload-time = "2026-05-18T23:36:05.92Z" }, + { url = "https://files.pythonhosted.org/packages/df/ac/46de6dda46478f7942f839e094970be2d4a861e005c4b3bf07c92e291a09/numpy-2.4.6-cp314-cp314-win_amd64.whl", hash = "sha256:b507f5c4c1d508876d1819b6bf9a49d365b96320b5d4993426b33a23ca4b8261", size = 12450334, upload-time = "2026-05-18T23:36:09.107Z" }, + { url = "https://files.pythonhosted.org/packages/78/92/b8b798ac784102c0da830d2257d59358e3d3d90d1e2b3f2575dad976c5cf/numpy-2.4.6-cp314-cp314-win_arm64.whl", hash = "sha256:6f41ae150c4e32db4f3310cdaf64b1593a03dbabe29eec77fc9b50fe64061df6", size = 10495678, upload-time = "2026-05-18T23:36:12.766Z" }, + { url = "https://files.pythonhosted.org/packages/30/34/ec28d1aa8115971537c01469ab2011ee96827930f0a124de1000cc2a7ed7/numpy-2.4.6-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ece3d2cfe132e7d51f44a832b303895e6f2d499c5e74dfbdb06ee246147a304a", size = 14823672, upload-time = "2026-05-18T23:36:16.473Z" }, + { url = "https://files.pythonhosted.org/packages/16/bd/f6d1fede4e54e8042a7ff97bb495510f3c220f94bcd9e8b228e87c92cc0d/numpy-2.4.6-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:e3e5193ef5a3dc73bceee50f7fdc2c90dbb76c42df8d8fae3d1067a583df579e", size = 5328731, upload-time = "2026-05-18T23:36:19.767Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f0/e105b9e2fd728a9910103884decd6951d9dd73896b914a98d9a231de02ee/numpy-2.4.6-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:17f9ade344e7d9b464a084d69bcf18fc691cb1db67c62ed80820bf4926d78f0e", size = 6649805, upload-time = "2026-05-18T23:36:22.266Z" }, + { url = "https://files.pythonhosted.org/packages/82/dd/1206a7ca6ab15e3f02069707ca96222e202af681bb73756da7527f3cb837/numpy-2.4.6-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9cd5ffd25db4e7ba6a375693b3fc0fc1791ec636c17db3720da19bde7180ec43", size = 15730496, upload-time = "2026-05-18T23:36:25.713Z" }, + { url = "https://files.pythonhosted.org/packages/51/e7/38d3ea825dcab85a591734decb2f6c67caa7c8367d374df1a1c3842f9b07/numpy-2.4.6-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d92c3819208a60205a12a245c91ad70cb0a85336659b19b834205573ac8456e", size = 16679616, upload-time = "2026-05-18T23:36:29.652Z" }, + { url = "https://files.pythonhosted.org/packages/93/b7/caabfdf53edf663e0b4eb74d7d405d83baef09eb5e83bcd32d601d72b93e/numpy-2.4.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e85b752a1e912b70eaad4fafbd4d1238007ab221de2009b9a2f5ae7461239895", size = 17085145, upload-time = "2026-05-18T23:36:33.449Z" }, + { url = "https://files.pythonhosted.org/packages/f9/45/68d7c33a6bcf3e5aa3bdbd57a367e6f615286dfd6482f97e8ffeb734306e/numpy-2.4.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:29cb7f67d10b479ff07c17d33e39f78c07f71c40ef30d63c153d340e96cd3fb4", size = 18403813, upload-time = "2026-05-18T23:36:37.369Z" }, + { url = "https://files.pythonhosted.org/packages/9c/50/0753655aa844c99cd9e018aacf76f130f1bd81d881bb74bc0aef5d73a8ba/numpy-2.4.6-cp314-cp314t-win32.whl", hash = "sha256:260a5d70215b61ab4fadf5c7baacd64821842975eea312125ed3c39a6391b063", size = 6156982, upload-time = "2026-05-18T23:36:40.817Z" }, + { url = "https://files.pythonhosted.org/packages/b2/d4/7c67becf668f973cb490cec3e98dfd799d866f9c989a54d355672cfa0db6/numpy-2.4.6-cp314-cp314t-win_amd64.whl", hash = "sha256:81a1cca95ed5bb92aa8b10dd2cdc9a0d3853a50fad926c28b5d7e8ea54389627", size = 12638908, upload-time = "2026-05-18T23:36:43.996Z" }, + { url = "https://files.pythonhosted.org/packages/43/bb/e1c71a4295b1b1d1393d50dbb4f2a36283c6859d9d3892e84f00ec5a91d5/numpy-2.4.6-cp314-cp314t-win_arm64.whl", hash = "sha256:0c9136e14ed34a9e343a31c533d78a9813a69a3148332bce5e9821cb2f996e66", size = 10565867, upload-time = "2026-05-18T23:36:47.114Z" }, + { url = "https://files.pythonhosted.org/packages/de/12/b422cc84439adc0d00de605bf4a308890ae5c26f2c71fbd73e5d08fbb0dd/numpy-2.4.6-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:55cced7c52e981362f708ad635198e97a752dfba412cc03c23bbf3bd8d5cd662", size = 16847511, upload-time = "2026-05-18T23:36:50.673Z" }, + { url = "https://files.pythonhosted.org/packages/44/53/f481bef68011740f8849418d82db07230e825013f31f4eef5ba5b805316a/numpy-2.4.6-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:d6da64deb6b8ed903e7560180a92f2d804ee1ba5eeb849ac2748b8c1aba1f6d7", size = 14889064, upload-time = "2026-05-18T23:36:53.879Z" }, + { url = "https://files.pythonhosted.org/packages/7f/57/42ed575c10ced8af951d426bc4e1f8aff16fd851db33f067036215a7f860/numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_arm64.whl", hash = "sha256:68a5124b13fa6cc2086764a20005d30bc0548146f7f5322f02fce212ca14317f", size = 5394157, upload-time = "2026-05-18T23:36:57.194Z" }, + { url = "https://files.pythonhosted.org/packages/6a/ef/f66cc724fcc36c1e364c67f51ae9146090b8b584f27d58b97fdae3edd737/numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_x86_64.whl", hash = "sha256:948424b06129ce883307e8cff868c31396d8dc7630a59c61d70d98dbe70f222c", size = 6708728, upload-time = "2026-05-18T23:36:59.575Z" }, + { url = "https://files.pythonhosted.org/packages/1a/9c/c531f2293b91265d8b48e9b329f54fdd7ffae73cb4134ea10cca4237e9cc/numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbbdb29840ca3d91ee0fece42fc29278886d908280bfec0a5846c6f901a3eb0", size = 15798374, upload-time = "2026-05-18T23:37:02.674Z" }, + { url = "https://files.pythonhosted.org/packages/1a/b0/413077f6b1153ed3cba361401c6783bbad6114804a000cc22eb71c13e190/numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ad03c0965fb3c692200e74d458ca28c1dbb4ce96f9a479a8aa041ad5fabca02", size = 16747286, upload-time = "2026-05-18T23:37:06.327Z" }, + { url = "https://files.pythonhosted.org/packages/15/ce/e5ec180bc41812edcd8daeb8639d205622c0e8c02259d8ab25a0201b3c2a/numpy-2.4.6-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:2803abfebfc990042cd494d8ce2d5f82e9d847af6d35ec486923aa19dbad5e73", size = 12504263, upload-time = "2026-05-18T23:37:09.715Z" }, +] + +[[package]] +name = "numpy" +version = "2.5.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'win32'", + "python_full_version >= '3.14' and sys_platform == 'emscripten'", + "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'win32'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'emscripten'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] +sdist = { url = "https://files.pythonhosted.org/packages/22/fd/89965aa4ac08c74998539fcbf24fa3540f3e15237fbeb6bcf9c908f4aade/numpy-2.5.1.tar.gz", hash = "sha256:a48a113e6afea91f5608793bafa7ef2ad481fefbda87ec5069f483de61cb9fa3", size = 20755553, upload-time = "2026-07-04T17:08:00.933Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/7b/14687aa674250e5e546f616f486b0d56d3631cd5b2415739141ce40bdcea/numpy-2.5.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2c889b56fe48b1018f764b0eec8df59ab654e9148aa91faa12596043500de277", size = 16801574, upload-time = "2026-07-04T17:06:12.423Z" }, + { url = "https://files.pythonhosted.org/packages/e1/19/cc5bb2a3f2913d27d6dbb2c78d25921fabaedc6741d4a5a615a11f3c5bf3/numpy-2.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab451b59c5643c570974c43aef780703ef1d3b4965d2be07afd530615a9358d1", size = 11772250, upload-time = "2026-07-04T17:06:15.726Z" }, + { url = "https://files.pythonhosted.org/packages/42/77/fdf34a71dd30f54979b18603bee915e0aaf825b07afe79acd60b04b691e2/numpy-2.5.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:78798bd5b9ad744056af8efa90e3b9ddaa53272a0848a483084a1cc0a13b2dc0", size = 5331516, upload-time = "2026-07-04T17:06:17.913Z" }, + { url = "https://files.pythonhosted.org/packages/ce/e2/eb7efa015b4cce41e2517bf182a7fce0d7d5b9d9ed76a29bfa0f4fe4505c/numpy-2.5.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:2ae0ca40bcb22d6ba59c1dfd5446f49940b0f2d821fde133f10dda11f816b84e", size = 6664863, upload-time = "2026-07-04T17:06:20.02Z" }, + { url = "https://files.pythonhosted.org/packages/a9/4b/a2b32dd94ee9ffbeecb28152240042a3949db33b1c834d44090b80e1b3b8/numpy-2.5.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:61ac47e772e6b8ea489e1d2f441a34c5c3ac17327e7ce294cbdf535795ad4e75", size = 15167977, upload-time = "2026-07-04T17:06:21.621Z" }, + { url = "https://files.pythonhosted.org/packages/b8/a9/6e73d68500f80773f65f0654ea932019d6694329a0eb0ed0533de38df376/numpy-2.5.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:59fda5e192b570217ec2580c96f00e9a7e12ef6866a900eb089b62c1a32545ca", size = 16672469, upload-time = "2026-07-04T17:06:24.064Z" }, + { url = "https://files.pythonhosted.org/packages/24/7d/ad3e59015135f5261c95fd4cafeff159c955febd83a99a1d9250c4233815/numpy-2.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f7119ebff1a9829e9f431a4f9d28e703023bb6b9fe7c8f724467dbfc27c94ab3", size = 16527531, upload-time = "2026-07-04T17:06:26.69Z" }, + { url = "https://files.pythonhosted.org/packages/83/d0/a39b2fbcde9cb17a1dac678f254b33a6336298af9df338824c685425d5e8/numpy-2.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e824c2acf8862052246be5a44c15da1777940c60d010dd2aab897824d9c430f9", size = 18431940, upload-time = "2026-07-04T17:06:29.521Z" }, + { url = "https://files.pythonhosted.org/packages/04/12/cff070947791c1ed425ff76413189adbdc2fbe215eba7ce7fa454a03c7f8/numpy-2.5.1-cp312-cp312-win32.whl", hash = "sha256:08d60c810432eb83360958dea0999ac4cfb94531ea8efcbf0b7f277c2068aeb2", size = 6066764, upload-time = "2026-07-04T17:06:32.571Z" }, + { url = "https://files.pythonhosted.org/packages/65/66/53f31807a48a750f9d748da273bc3fcedd12b27ff1f3e373bfec55ef2dc0/numpy-2.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:f7d60026c0bdb1380e83bfa7a0419c4577ee4b9a08880afcb6dadeb74c649fa2", size = 12430966, upload-time = "2026-07-04T17:06:34.926Z" }, + { url = "https://files.pythonhosted.org/packages/2b/2a/d1a88066b1c14186f5d3c0d18c94f17b064511982bab0578d49ee9d43c29/numpy-2.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:17a25e09640602e10bc8de0e6fa2b3fd68eedd84ba6d7842dc8f32f9ab87bd0b", size = 10350488, upload-time = "2026-07-04T17:06:37.785Z" }, + { url = "https://files.pythonhosted.org/packages/eb/07/ec2a3f0c91761581d4b7104a740791800025983f9a4dc4e73f91a99aeac4/numpy-2.5.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0bfebd8695f9863592fe744be833a258120b14a9f39da255e8aa8fade2c0ddd1", size = 16796419, upload-time = "2026-07-04T17:06:40.37Z" }, + { url = "https://files.pythonhosted.org/packages/ab/ab/ddb499fc4f8780354395face5b65c7fd107bcd6e1d667a5f07d046956f6f/numpy-2.5.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:30b44a6b53a7ae63c54c089a8726e5563ed302716c5b7ccc85afade40b0e7ff6", size = 11765832, upload-time = "2026-07-04T17:06:42.768Z" }, + { url = "https://files.pythonhosted.org/packages/88/b3/3c28c558a09fc72100c646dac6d2fce8e834c471b0edca01a29996706117/numpy-2.5.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:6165343f81b56ef8f514f396989e529b61d9dc709b99421b07e9f3e698e2287d", size = 5325143, upload-time = "2026-07-04T17:06:45.466Z" }, + { url = "https://files.pythonhosted.org/packages/5e/0e/ce19b985bb15c596f4f05954e76cccc77c845083b3b8f938a6c68e523128/numpy-2.5.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:4939237038ada79308dda3204ac6462df056b5672b2e25db1149cf873668b3e1", size = 6659749, upload-time = "2026-07-04T17:06:47.288Z" }, + { url = "https://files.pythonhosted.org/packages/2e/20/1ee6614d64332a1bba6411f38e68cb79eec1b2459e20a623777c5c5492a2/numpy-2.5.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c6759f538fb912fc46de0a6b1758ccf7b57bc7c7ebebc23974fdac3de8db0cd", size = 15164716, upload-time = "2026-07-04T17:06:49.494Z" }, + { url = "https://files.pythonhosted.org/packages/ed/a7/2bcd3fdbb87804755c35b729bf8709d62025c5f4cfd7d5b2415997097515/numpy-2.5.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9726558e8db4a5bf7929a70ae50f63abda4daf0efe810e3bfbab95976f75fc1a", size = 16661440, upload-time = "2026-07-04T17:06:52.061Z" }, + { url = "https://files.pythonhosted.org/packages/fc/d7/a41e3310c886fe457d36e670bbf24fae411aca8a7b6ad92a32afd924077c/numpy-2.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3935f3b419b244a02732676fa5317a9193cc596a4c0646db07e5b421229ac9f7", size = 16526305, upload-time = "2026-07-04T17:06:54.605Z" }, + { url = "https://files.pythonhosted.org/packages/53/75/4333a9a707c1edd3a4e1a0c58eca52c0f31e55089fa80db02b5565b24df7/numpy-2.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dc932a65ded7ce9013d120845a2514dcccb1a67bfc8deb8d37633762951904a6", size = 18423008, upload-time = "2026-07-04T17:06:57.54Z" }, + { url = "https://files.pythonhosted.org/packages/ee/90/e314a32b1c11a2ffe818ddad3a57b50b4b6e1b6c487192eb50cdef0415d0/numpy-2.5.1-cp313-cp313-win32.whl", hash = "sha256:4b4ff1608417eb7a59da7b967bbb798cacfe071d2caf526a24281cd562072ed9", size = 6063885, upload-time = "2026-07-04T17:07:00.14Z" }, + { url = "https://files.pythonhosted.org/packages/10/70/800b3fca480af32df9e8ea9f3d4a0c8feb4b32d7f195d174eabbda4829ad/numpy-2.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:6c3fe51bc6a16453d452997053454f309e8e0ed7b42d6b361ce4ac8c32913d74", size = 12425674, upload-time = "2026-07-04T17:07:02.387Z" }, + { url = "https://files.pythonhosted.org/packages/8b/0b/196350c122f50f6ca56846f2d71efd5e0d24b7b2e07355e019b2e2c7a11e/numpy-2.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:f7feb014281029e628ba2d5a007407443b06e418b6fe451d1e2adcbc8eba0107", size = 10350256, upload-time = "2026-07-04T17:07:04.878Z" }, + { url = "https://files.pythonhosted.org/packages/db/f4/731b6085a83faf6ca843394cbd5e217280c214399f7e8b21b9f552af0ae2/numpy-2.5.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:7c786fe9a5bbe360022e584c5a34cf6b54265c71bd7ec8ac3d8fec38968071f8", size = 16795063, upload-time = "2026-07-04T17:07:07.374Z" }, + { url = "https://files.pythonhosted.org/packages/bf/64/0e215f2048dd11a55bb989ed41b3585ef57452404e638d703a211a3e4157/numpy-2.5.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:32985c896d897419ef8da6917872d80b78ad0ea26d85b23245c7366ffde76d75", size = 11776652, upload-time = "2026-07-04T17:07:09.907Z" }, + { url = "https://files.pythonhosted.org/packages/b5/59/2b844c7a6e9deff69b404a66221e1542937734f65d5e6e39411876053862/numpy-2.5.1-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:efd736408cc97c79b9e6917338dfc8f06013b2274f992e96b1d9a81a71e2a2c2", size = 5335944, upload-time = "2026-07-04T17:07:12.227Z" }, + { url = "https://files.pythonhosted.org/packages/86/51/9bf7cb2cabcebc9e017e4ec7e6322b378317a542c08b4cb68479c1efc716/numpy-2.5.1-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:ab84dc6b074fa881cae55bea94cc4f68e285181ba7f32497bf7dee6b1496165b", size = 6656266, upload-time = "2026-07-04T17:07:14.368Z" }, + { url = "https://files.pythonhosted.org/packages/83/3e/fb7615b211b82a32f44d5180a6d421b61f84d4fadd578b48ba4ac34e189f/numpy-2.5.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:caf3e317d33d60c37986b452613f4ab51246d0691350c03d0cb4a898627f4a95", size = 15179720, upload-time = "2026-07-04T17:07:16.272Z" }, + { url = "https://files.pythonhosted.org/packages/41/5f/0f992cb24560673496c5d68de61913b57166ce530ffda07c1f280e0cc464/numpy-2.5.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:54ad769f17bc2d833b620851989f62054fb9ab93c969d9e1dc3c8e3d56beea21", size = 16664835, upload-time = "2026-07-04T17:07:19.021Z" }, + { url = "https://files.pythonhosted.org/packages/a2/2f/97d6475ee91afe2587797d09446f9d3e475ad4cb681662d824809327b75a/numpy-2.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c12afb53450fa976d4c681c50a7423729a4c51c0465ed9f32b8a9cabbc472373", size = 16539135, upload-time = "2026-07-04T17:07:22.015Z" }, + { url = "https://files.pythonhosted.org/packages/c4/5b/4db81e4ba0be7e2776b1de68c82aa862c7f8ec27e1b4927d4ae075e20678/numpy-2.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e8c11c405efc5ff6816d5983c96cdfa215bab3428961243af3ff59b228490438", size = 18426684, upload-time = "2026-07-04T17:07:24.941Z" }, + { url = "https://files.pythonhosted.org/packages/1f/64/c0ba2d90724d450279a7df8f32057241070250a26a7e2b5337d77347f481/numpy-2.5.1-cp314-cp314-win32.whl", hash = "sha256:f2479a47f8d5932d1718168a681ad6e536a9df484c83cfcf9de365e164537ace", size = 6116103, upload-time = "2026-07-04T17:07:27.622Z" }, + { url = "https://files.pythonhosted.org/packages/c1/1a/837f9ed7405adcd7a40538792eb169eddd8fa5630c16a1ef49dae71a30f4/numpy-2.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:24d0eb82c0541d3415a33425db64ae439dffccd7b4dbcb30e7c35120205c506a", size = 12562177, upload-time = "2026-07-04T17:07:29.887Z" }, + { url = "https://files.pythonhosted.org/packages/22/ed/49707938b6dd0a78a9178dd93227dc89e4c11af47f5c798d70366e8d0483/numpy-2.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:5a4c988b38d261deeeaad9954e3deb091ad905c94e8bb6708654ef1d97f286b0", size = 10627739, upload-time = "2026-07-04T17:07:32.568Z" }, + { url = "https://files.pythonhosted.org/packages/a6/c7/bb4b882cfe7f299cbc8b66e42e7dd78cf9d14e40f9469fc5e3db7e15b3bd/numpy-2.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a33276be12fa045805f477f22482088b66bb758ffbe89a9d21457de863a32e22", size = 11894709, upload-time = "2026-07-04T17:07:34.941Z" }, + { url = "https://files.pythonhosted.org/packages/40/3f/5af7f4a7f6224aef48017aa82bb6174c7a659d724be0c75017b7e64a55b4/numpy-2.5.1-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:f089d7b00756190aacf1f5d34bdf38c3c430ac82b4f868f8cede73380460fce7", size = 5453810, upload-time = "2026-07-04T17:07:37.495Z" }, + { url = "https://files.pythonhosted.org/packages/20/c9/3474309bc94d634d3f9c3eddf03250ecb8c22cd948ef16fef69a77cc5d7b/numpy-2.5.1-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:09e9bfd8d2cf479c7d174804fb3811c53a8e9f20a37444008606b57d6b7a826d", size = 6761189, upload-time = "2026-07-04T17:07:39.563Z" }, + { url = "https://files.pythonhosted.org/packages/90/8a/558ae39fdd55d7e7f7fef9a84a6e964ac6b23edbd2a07e52bb084500507d/numpy-2.5.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e68d8dd1e7eba712948f2053a29ec86917bc70ba1358df869d9f06649ef9cf09", size = 15225039, upload-time = "2026-07-04T17:07:41.682Z" }, + { url = "https://files.pythonhosted.org/packages/63/27/ca7392b2d030277bdf0273e7d23255b3ee57d57a7c170a6f4fb3981e1e5d/numpy-2.5.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:99d5095fa265a0c4152e7bb12759e14381ef5496152f1ce58f44bdf55c44beb4", size = 16701306, upload-time = "2026-07-04T17:07:44.611Z" }, + { url = "https://files.pythonhosted.org/packages/02/42/03d53ae7996c44d4374a8262e9dc41671fd56cbb98f7d47ef85cf5da4c6b/numpy-2.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ab87a91b3cc3382b8956095bd8f95e00cf679bb81554339be1a2ba404a1473c1", size = 16589955, upload-time = "2026-07-04T17:07:47.694Z" }, + { url = "https://files.pythonhosted.org/packages/7b/15/6c1784ae469640e65db111e9a34b3d0f14d91e8a38b9ce34810ced370dbb/numpy-2.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:224ca51130ef7da85bea2191625181cb4f337f9cb64b471f10c1a12aa8b60077", size = 18464252, upload-time = "2026-07-04T17:07:50.684Z" }, + { url = "https://files.pythonhosted.org/packages/94/a8/f98e50356cf167df656c526c2dfeec2d7dde182f2a3da4b458a5938e2776/numpy-2.5.1-cp314-cp314t-win32.whl", hash = "sha256:6eab239876581b2b3c5a242281b6007bbdbcd1c7085d7709bb57c5929b11e6bf", size = 6263298, upload-time = "2026-07-04T17:07:53.445Z" }, + { url = "https://files.pythonhosted.org/packages/72/ac/96ae880cdecad0b3275d9359fcec72667b49a4863c9f12942e43679dda02/numpy-2.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:83ce9c80d5b521b0d77ddcbe5447c218d247929b6cc056ca5351342accfff0af", size = 12748623, upload-time = "2026-07-04T17:07:55.384Z" }, + { url = "https://files.pythonhosted.org/packages/a1/5a/4d2b1601df3602dba7a14f3348ba9bfe94a18adb428e693df6154c293831/numpy-2.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:5a6db61f9aaa57e369905c67d852045d3c4f7126405b29d09b19dec118e9c9cb", size = 10697674, upload-time = "2026-07-04T17:07:58.506Z" }, +] + +[[package]] +name = "packaging" +version = "26.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, +] + +[[package]] +name = "pandas" +version = "3.0.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "python-dateutil" }, + { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/be/4f/5f3422a2afec5ffc46308b79e53291365a93748b498ac2e58bead0197916/pandas-3.0.5.tar.gz", hash = "sha256:dca3734d6ab7c906e6730f0788b0a1dbb9f2467731f9711f77995c8e9d62d712", size = 4658219, upload-time = "2026-07-22T22:19:28.819Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/ef/f1fd7431d635bf20015489bf0bd69c17fff1018de773540f651455a3916b/pandas-3.0.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2946e77e4a53cd248cbde631a12f0e51c8324ce354c3eba4d20147c1ad6f4282", size = 10397178, upload-time = "2026-07-22T22:17:48.274Z" }, + { url = "https://files.pythonhosted.org/packages/31/b4/0eafac990a431561187694126de01f9b12559549b4d86360c0c4bd870fde/pandas-3.0.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:71ecc8fb7ed1a7aa4392316b5309a6347e8e7f832f38fd897846b3a1457a9298", size = 9990736, upload-time = "2026-07-22T22:17:52.388Z" }, + { url = "https://files.pythonhosted.org/packages/de/21/359880af3ea9b7cb23bea5b51e8e70ef3866c03be09da9a2787e18e330a8/pandas-3.0.5-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b173f5951ff6b8b0ec7675e20dff3c97b7e7a57dfcce387c2d7c5afe87cb7899", size = 10814438, upload-time = "2026-07-22T22:17:54.708Z" }, + { url = "https://files.pythonhosted.org/packages/d1/50/d6cc4d7e508bbccf5d6027314a8312bc7ac73d0ec7f195f53838daafab40/pandas-3.0.5-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2c0cf1dd9b55a22d105fc46c1b489af3bd42264fcba7c66297bf47a9a1d9c78a", size = 11323634, upload-time = "2026-07-22T22:17:56.858Z" }, + { url = "https://files.pythonhosted.org/packages/70/2b/d5f0a8c90dd0ae04e64ba53b871afb796ec026b615086d382ddc2ade729b/pandas-3.0.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0fac0010c75e4efb6b99e249c183a8993ce0dc95c240f9b120a5e67c727b7928", size = 11850860, upload-time = "2026-07-22T22:17:59.1Z" }, + { url = "https://files.pythonhosted.org/packages/5c/30/183aec2e19adf778a98d29b5729a0a68f4cc4ebf9b9c3b70d0297355bcb1/pandas-3.0.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:08d24fe11a17dc33bd6e937dc9c665f9cba08fbdc9f657f405713515febe300d", size = 12411100, upload-time = "2026-07-22T22:18:01.485Z" }, + { url = "https://files.pythonhosted.org/packages/fa/9a/31f4983f191af51ab2a8f2d0c7b33dff3a84da26533f982fff02c2f9e28b/pandas-3.0.5-cp311-cp311-win_amd64.whl", hash = "sha256:b1261758dfb6cf12c3cff8300e21cefad30e7ec709abb4c24ac7318e6a52462a", size = 9968804, upload-time = "2026-07-22T22:18:03.903Z" }, + { url = "https://files.pythonhosted.org/packages/49/97/7886c89a39045c69ad82cbceaf3343810480c8ef49a216319ce8183860a6/pandas-3.0.5-cp311-cp311-win_arm64.whl", hash = "sha256:679f4e85b30ddb1515458ab1e788d3e260eae369b1f78da7a3aa4cac8ebf4a2a", size = 9205447, upload-time = "2026-07-22T22:18:06.134Z" }, + { url = "https://files.pythonhosted.org/packages/1c/54/1dc810ea558d1320b597aa140a514f2fdf1d2ea09c38cf556f13ea712ec9/pandas-3.0.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fa290c16964d4963fbfbc358928239cf3bd755b20e988ce944877def2f44471d", size = 10411717, upload-time = "2026-07-22T22:18:08.307Z" }, + { url = "https://files.pythonhosted.org/packages/68/56/fbe81c09195924d8b7b8d4461a20458fe80a6a5ed6b24f0314da684277e1/pandas-3.0.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c2e26bb46934b8a2ca0c3de1d3d606fc5f6746584791b2db264d58cf370e08dc", size = 9957095, upload-time = "2026-07-22T22:18:10.6Z" }, + { url = "https://files.pythonhosted.org/packages/e0/51/fac252f4a913ed5eabf3c11b880a9e8d5a6c10f0b2129d0462212d238b4d/pandas-3.0.5-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:73fa87b08a7ef706f8aafda39ddaccf2a99047bea62d8c88a0361bcafb2237bc", size = 10485458, upload-time = "2026-07-22T22:18:12.834Z" }, + { url = "https://files.pythonhosted.org/packages/12/98/e976540c1addf70442be7842a18cf70884a964abbf69442504f4d2939989/pandas-3.0.5-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d373ce03ffd84010ed9839fa73672a9c8256990532e158440c0085db7d914b34", size = 10998091, upload-time = "2026-07-22T22:18:15.209Z" }, + { url = "https://files.pythonhosted.org/packages/a4/8c/1f29b5be8d3fc47dd7567eb167fabba2085879b31e0287ce7cba6d3d2ff4/pandas-3.0.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2a29c53d85ea98c5e792c59ef82ee9fbe6ca902c0d0adb6b23f45ef894cd7bf6", size = 11499501, upload-time = "2026-07-22T22:18:17.689Z" }, + { url = "https://files.pythonhosted.org/packages/9d/e2/bd9c98ad2df7b38bde002adde4cdf353519da51881634323b126c55997f9/pandas-3.0.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a5ad3b02ed6bc7d7ae9b70804b2c6aa31827489d150f8e623ce82491b82085d7", size = 12060559, upload-time = "2026-07-22T22:18:20.147Z" }, + { url = "https://files.pythonhosted.org/packages/f3/9a/ffbd852d58bd74a617fe2f8ee6a58a96982271ce41cf981eab22190b4a4b/pandas-3.0.5-cp312-cp312-pyemscripten_2024_0_wasm32.whl", hash = "sha256:b2acb4650527eec6822c3dadb2b771277b65e7dae7a267d4bccf65fd1bb3fbce", size = 7197652, upload-time = "2026-07-22T22:18:22.502Z" }, + { url = "https://files.pythonhosted.org/packages/70/b5/d2d3e9ae73362ba4229651b0ee1455cf78073a1ce585f6ff693782ce263e/pandas-3.0.5-cp312-cp312-win_amd64.whl", hash = "sha256:80a611068e8a3ac23f7398c6c14eb46dc974e5cc9997f653e2dcfd1da74edd41", size = 9831691, upload-time = "2026-07-22T22:18:24.534Z" }, + { url = "https://files.pythonhosted.org/packages/52/51/dea1e89d6a6796b9c43f85a09b484ee03edb8a4c4842e73e200a8c11301c/pandas-3.0.5-cp312-cp312-win_arm64.whl", hash = "sha256:25ff585b972a18ef1fe9ffa3ac6544d9950508aa76832e5147640b6022821e49", size = 9105796, upload-time = "2026-07-22T22:18:27.064Z" }, + { url = "https://files.pythonhosted.org/packages/bf/09/7b95c4a0025227d6f118c4039b423412ac6a982db02864166185d812fbc7/pandas-3.0.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c1c05a767fe8e5b4fe9e1c29806829c582052eaedb9120a3da83ba3f69e24a5b", size = 10385742, upload-time = "2026-07-22T22:18:29.346Z" }, + { url = "https://files.pythonhosted.org/packages/8d/0c/dc78fd8c4da477b4b5e8ad37295af352190d21ef63a9ee1bc071753074cc/pandas-3.0.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b86765f268b56f7e665b93bce9d5df69dee7f99e595cf8fb839483ab315942a3", size = 9932067, upload-time = "2026-07-22T22:18:31.833Z" }, + { url = "https://files.pythonhosted.org/packages/3e/71/3592c055cf44df9808550f9368ceda80ff2b224d355ef73fe251dcda1802/pandas-3.0.5-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c597ecf5616b5c420372c1d4d4c00dbbfba7398bea857dcc984347e1ea48417b", size = 10466756, upload-time = "2026-07-22T22:18:34.195Z" }, + { url = "https://files.pythonhosted.org/packages/e3/70/4363150359f95b4cb4bcbb34ca23572bb5495749a621a8f3d5a1ddfd293c/pandas-3.0.5-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4b11c36e218331d0387cbe3a0a5f75162357a1d92d57b2b08a336ff94b19b2be", size = 10938525, upload-time = "2026-07-22T22:18:36.81Z" }, + { url = "https://files.pythonhosted.org/packages/f7/d0/317e7a0c67c0e69fa905a0161409397a7dc2d46ff611f6ca4803352c042b/pandas-3.0.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cf52e1f61d229496da17dc7ab54acdee627357e7008fd4fecba3d0ba2937fa58", size = 11489303, upload-time = "2026-07-22T22:18:39.287Z" }, + { url = "https://files.pythonhosted.org/packages/f1/8d/36dade89b49e4f9d5cbdbe863772581f98c0c6d78fc39ad4c557f6f2e17e/pandas-3.0.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:db172144bb56422bd157812f3b021eacc255451470b31e2c633c349490a1cfee", size = 11989004, upload-time = "2026-07-22T22:18:42.208Z" }, + { url = "https://files.pythonhosted.org/packages/9c/ba/18c4ec8a746e177da05a9e7a7963781d8ea195780724f854601b6ebd6b78/pandas-3.0.5-cp313-cp313-win_amd64.whl", hash = "sha256:0d298e951f23016ce4699951d044ae6418dbc91bf68cefca0f77666fcbb4e5c6", size = 9826896, upload-time = "2026-07-22T22:18:44.539Z" }, + { url = "https://files.pythonhosted.org/packages/de/ec/28a57266b753799a87b8bc79e7887ac6fd981b8c6d2978a0b7e7b6bd708c/pandas-3.0.5-cp313-cp313-win_arm64.whl", hash = "sha256:66266d3442a5e8b3c90274c2b8b230bee42dd1c286bc822cc2f9f2c7e12b883e", size = 9094790, upload-time = "2026-07-22T22:18:47.468Z" }, + { url = "https://files.pythonhosted.org/packages/51/2f/cf6aae281264f4463f0875bcbb15fd2bb6d291cc535187dad1732475e4a9/pandas-3.0.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2f264fc46911cc8131a7322a16199bbf8e353d27c10bb211f5bd0c814324dc36", size = 10390034, upload-time = "2026-07-22T22:18:49.818Z" }, + { url = "https://files.pythonhosted.org/packages/06/ec/5189518c7a7659c4bdcc6b1eb32c46c6f3c86b0661ffd84143d1112c7732/pandas-3.0.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:53730687fcd161883b24e10411c06d6a4c0f2275d2faf3bb2bc25deb4ba8007c", size = 9980065, upload-time = "2026-07-22T22:18:52.249Z" }, + { url = "https://files.pythonhosted.org/packages/ea/f1/598503ce8d7e3c35601e0747ba288c7864baae66380725bc12f13f884dfe/pandas-3.0.5-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:960d3ebcf249f75206899fcd2c6de53f736b7265759ced0d3e559df0b8b709b0", size = 10545532, upload-time = "2026-07-22T22:18:54.813Z" }, + { url = "https://files.pythonhosted.org/packages/fa/de/ceae2adf7034e07e9910299fe412e1819c4f0dd520700a888bcb03625448/pandas-3.0.5-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9e94c2c5ca43bd3ca32bf64d32308887b65e5f9bfd8023ea52755107a999f93b", size = 10963120, upload-time = "2026-07-22T22:18:57.42Z" }, + { url = "https://files.pythonhosted.org/packages/66/25/86e0f4451874eb79e688deeebe3c451fec4557f8952005818d800ee8ac7e/pandas-3.0.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e819dd5f62966b481a8cb649d3299ebd886a1ea91ed5a99bf7ce77c98d18ab94", size = 11563178, upload-time = "2026-07-22T22:18:59.729Z" }, + { url = "https://files.pythonhosted.org/packages/f3/45/8643daa3b4147e433adfcccefdd0380d3aad79d86b15d8999730fe1944d5/pandas-3.0.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:3c5ed2e7c06e91d340dfd091d7934f9bc82e4a36b95f647f090b9d1c9ac649da", size = 12028708, upload-time = "2026-07-22T22:19:02.164Z" }, + { url = "https://files.pythonhosted.org/packages/96/58/ad979ae617615576e8aafd569c9d4b62f1191d896e38f51d66ba06f3b89a/pandas-3.0.5-cp314-cp314-win_amd64.whl", hash = "sha256:cd8f7c6dc98527058ee6264219343f5392240a6f1bfa654fc5d79023020d0c92", size = 9951806, upload-time = "2026-07-22T22:19:04.596Z" }, + { url = "https://files.pythonhosted.org/packages/69/32/7ac03886b304049a9d2625ee88f59af760d8a93bd30ed9239bce7b9869a8/pandas-3.0.5-cp314-cp314-win_arm64.whl", hash = "sha256:5183427f5a8156d480f30333777bc978be93650a49a7c01db26adffe95b31e85", size = 9238297, upload-time = "2026-07-22T22:19:06.836Z" }, + { url = "https://files.pythonhosted.org/packages/be/ed/1d1f2ee5547d5167face2376d11c8b2a4c7bfff5a416ee7a9046891fab1e/pandas-3.0.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:303da736987d481074ca720ada325f8bd80c64ebc2d45ed79b29df3aaa4a26ca", size = 10849690, upload-time = "2026-07-22T22:19:09.391Z" }, + { url = "https://files.pythonhosted.org/packages/57/55/17e17152e98fbb0c4b1e562bc65387a2f20a80db0f4a86bf8d3a0e4248d4/pandas-3.0.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:3b2801bbb049d0136f6c213eae02b5fca969384fc2064dd728d8620552aa49da", size = 10509945, upload-time = "2026-07-22T22:19:11.773Z" }, + { url = "https://files.pythonhosted.org/packages/88/90/817d44dbf83facf9556f33576d9af0a241981e7bb5c00606c0bcb5df8dda/pandas-3.0.5-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cce3a9d11d2b1f82c69a27ec1f4948a170e2c403c4bbfa8cca62e3fdebe2ef3a", size = 10392197, upload-time = "2026-07-22T22:19:14.024Z" }, + { url = "https://files.pythonhosted.org/packages/f1/da/889f00c0a6f5aa1545add70abbf01502dff87ab577adb855bd631c54d2f2/pandas-3.0.5-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ef01af4d8dc6cd2c8d6c7736f149574ef93fe043811eeb5e445f2647154b5040", size = 10862726, upload-time = "2026-07-22T22:19:16.351Z" }, + { url = "https://files.pythonhosted.org/packages/bc/98/f1e934fb3c98fce859c6147c6785816c7b5b9ab7821115c5d8c4de9842b9/pandas-3.0.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e2759e890db96dfcffdbd9b86c3c2cb6afaf58def482820317e06163ec1066cd", size = 11414864, upload-time = "2026-07-22T22:19:18.981Z" }, + { url = "https://files.pythonhosted.org/packages/fe/be/d448af7d657d82e1888dd8551f79c6d6fb161080b5b9752d84d910ec2319/pandas-3.0.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b58b1b39d46a5862e3fb18f50d1a201398619d16a0f9f73f57eea5583cf0e63c", size = 11925105, upload-time = "2026-07-22T22:19:21.515Z" }, + { url = "https://files.pythonhosted.org/packages/29/c1/ccb4238212c8c4f496c584f3044d94e0c030ed8e1d68999db46c91c2242f/pandas-3.0.5-cp314-cp314t-win_amd64.whl", hash = "sha256:1c10461f6eeb35d8f05b6184c65c8b9991663b66c46b1d559b682cb34ae7c6ea", size = 10387612, upload-time = "2026-07-22T22:19:24.257Z" }, + { url = "https://files.pythonhosted.org/packages/d2/cf/6a51b2c38980e04c279fd2fa908a1b0982064e860444acfca4ec2e2c8359/pandas-3.0.5-cp314-cp314t-win_arm64.whl", hash = "sha256:3c5015fd1730fbf883647e88068176c839c102cea883ba1769a6f4593bfc1f8c", size = 9509776, upload-time = "2026-07-22T22:19:26.694Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pyarrow" +version = "25.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/27/f3/95428098d1fa7d04432fb750eed06b41304c2f6a5d3319985e64db2d9d41/pyarrow-25.0.0.tar.gz", hash = "sha256:d2d697008b5ec06d75952ef260c2e9a8a0f6ccfce24266c04c9c8ade927cb3b4", size = 1199181, upload-time = "2026-07-10T08:29:50.116Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/98/ae2b5acf9876dbeffa6f320776242c52caab062df55c8ac5501ed2679e74/pyarrow-25.0.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:2e3b6544e26e393fe2cd530f523e36c1c8d3c345bbbb60cca3fd866be8322517", size = 35939080, upload-time = "2026-07-10T08:26:04.53Z" }, + { url = "https://files.pythonhosted.org/packages/80/09/3de2a968edbd496c86cb8b932cdbee2d4b08c4a28e9884a15e5c705a646b/pyarrow-25.0.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:b724d127783b4c19f088fcdfc844cbc318809246a30307bcabd5ed02045e890e", size = 37633420, upload-time = "2026-07-10T08:26:10.354Z" }, + { url = "https://files.pythonhosted.org/packages/19/86/8399243a4ce080426ec37db18d5e29148b7ec960a8a8c7f9059a7bf6ef0a/pyarrow-25.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:244f98a595f70fa4fd35faa7508c4ae67e14a173397a4b3b49d2b3c360fb0062", size = 46861050, upload-time = "2026-07-10T08:26:16.397Z" }, + { url = "https://files.pythonhosted.org/packages/7b/79/72d704b02bc5fc6d06954d76a0208c1e79cad3ab370f6d6a91ffe5078870/pyarrow-25.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:0222f0071d13313962a88d21bf28b80d355ac39d81bfa6ff3fe00eeaf748e4be", size = 50056458, upload-time = "2026-07-10T08:26:23.271Z" }, + { url = "https://files.pythonhosted.org/packages/06/5d/3c31a60b6403d63cad2e0f829096f5fc5763a129ead4207a5d4690b96448/pyarrow-25.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b58726f118c079f9d4ed7e904975d4f15fd69d0741ba511a4e2dcaa4ef16354f", size = 49957793, upload-time = "2026-07-10T08:26:30.232Z" }, + { url = "https://files.pythonhosted.org/packages/34/f7/8f8a019061f9863a831915329264372a87ed25eaf9109ce56eb0e84012c5/pyarrow-25.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:38a2c887cb3883e241b70201688db34133b6dfadd04f03c8f9213df53770c18e", size = 53100544, upload-time = "2026-07-10T08:26:36.414Z" }, + { url = "https://files.pythonhosted.org/packages/f1/e2/738071e95c5ddad7b3dfc12f569ffa992db89d7d7b4a95258fd184191249/pyarrow-25.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:161649d60a7a46c613a19fd795763ea8a88c36ba997dd99d9bc66e6794ee36e8", size = 27848311, upload-time = "2026-07-10T08:26:41.429Z" }, + { url = "https://files.pythonhosted.org/packages/73/44/fdd3a4377807b7dcabe2d4b5aa99dbbc98e2e5df3f1ca4e7f0aec492d987/pyarrow-25.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:149730a3d1f0fb59d663a0b8aa210adfd9c17c27cd94a0d143e60daea8320d4e", size = 35850884, upload-time = "2026-07-10T08:26:47.357Z" }, + { url = "https://files.pythonhosted.org/packages/bf/71/9f053177a7709b8c90abb00a2375b916286f9f0d6cfb21a5cadd4ef811e8/pyarrow-25.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:0721332c30fdd453fdd1fc203b2ac1f4c9db5aea28fa38d41f2574c4b068b9ec", size = 37616197, upload-time = "2026-07-10T08:26:53.564Z" }, + { url = "https://files.pythonhosted.org/packages/95/1a/22bfb6597dcdc861fa83c39c06e1457cb56f698940eff42fbb25de30e8e5/pyarrow-25.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:fa1482b3da10cac2d4db6e26b81da543e237616af2ef6d466018b31ca586496f", size = 46841966, upload-time = "2026-07-10T08:27:07.685Z" }, + { url = "https://files.pythonhosted.org/packages/55/0e/cd705c042bc4fe7022478db577fcab4abdcfabb9bc37ab7a75556b3fcb2b/pyarrow-25.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:5d1dbf24e151042f2fa3c129563f65d66674128868496fb008c4272b16bdf778", size = 50088993, upload-time = "2026-07-10T08:27:14.268Z" }, + { url = "https://files.pythonhosted.org/packages/98/ee/d822e1ee31fe31ec5d057210e0605c950b975dcd8d9a332976cc859a9df8/pyarrow-25.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:20887a762dd61dcc530f93a140840ab1f6aa7836b33270e42d627ab3cf11e537", size = 49941005, upload-time = "2026-07-10T08:27:21.274Z" }, + { url = "https://files.pythonhosted.org/packages/33/1b/207a90cc64619a095eb75a263ae069735f2810056d43c667befd573ec083/pyarrow-25.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58d1ab556b0cea1c93fdb799b24ad58adb2f2a2788dbce782a94f64ae1a5cc9b", size = 53112355, upload-time = "2026-07-10T08:27:27.911Z" }, + { url = "https://files.pythonhosted.org/packages/7e/fe/81d1e5f8beed15c01e98649d5c6e2167b67fd395884a2488f18bf1cf0dba/pyarrow-25.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:3f356afe61186395c861d5cd63dc21ff7d5fa335012a4668d979257df7fea0f5", size = 27945954, upload-time = "2026-07-10T08:27:32.903Z" }, + { url = "https://files.pythonhosted.org/packages/6c/c8/098ce17d778fd9d29e40bb8c5f19a40cc90c3f0b46c9057b0d7993f42f54/pyarrow-25.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:8831a3ba52fa7cdb78d368d968b1dcd06171e6dff5461e16d90de91d371e47bc", size = 35844549, upload-time = "2026-07-10T08:27:37.956Z" }, + { url = "https://files.pythonhosted.org/packages/bc/66/24c28877219abf6263d909b1592c97ff82c59f13a59acbed11fc87c0654f/pyarrow-25.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:5f4bacb60f91dd2fca6c52f1b9a0012cd090e0294f1f781dc1881a247a352f8e", size = 37610397, upload-time = "2026-07-10T08:27:43.803Z" }, + { url = "https://files.pythonhosted.org/packages/53/55/6d1d5f5aff317ec5de9421594679ed51ed828fe7e2ce209327f819d801e4/pyarrow-25.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:59516c822d5fd8e544aaa0dfe72f36fed5d4c24ea8390aab1bcd31d7e959c6be", size = 46841701, upload-time = "2026-07-10T08:27:49.741Z" }, + { url = "https://files.pythonhosted.org/packages/b5/5d/f790fb6965ab54c9da0dda7856abc75fd0d7648d865f8d603c111d203a64/pyarrow-25.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:6f9dbd83e91c239a1f5ee7ce13f108b5f6c0efbe40a4375260d8f08b43ad05e9", size = 50090118, upload-time = "2026-07-10T08:27:56.051Z" }, + { url = "https://files.pythonhosted.org/packages/0c/8c/faf025357ebf31bc96777f234277aa31e2aeca6dd4ecaa391f29085473c2/pyarrow-25.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:18dcc8cc50b5e72eae6fcbfc6c8776c21a007176b27a3cdec5c2f5bcf126708d", size = 49945559, upload-time = "2026-07-10T08:28:01.927Z" }, + { url = "https://files.pythonhosted.org/packages/07/a1/bd051871708ea99a5e0fc711926c26c6f2c6d0130c7aaac8093e34998af6/pyarrow-25.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4ec1895a87aa834c3b99b7a1e758747eb8bb57f922b32c0e0fa04afb8d6998b1", size = 53114238, upload-time = "2026-07-10T08:28:08.594Z" }, + { url = "https://files.pythonhosted.org/packages/7c/31/737f0c3cffcd6af647849477d1dd68045deac2e3963c3f9f211bedc48540/pyarrow-25.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:77c8d1ae46a44b4006e8db1cc977bbcc6ce4873c92f74137d68e45503b97fb18", size = 27861162, upload-time = "2026-07-10T08:28:12.975Z" }, + { url = "https://files.pythonhosted.org/packages/55/c7/581ccbcdb3d897eb2893328d68db3d52eca373bf2a7e964d0a6276b8e85b/pyarrow-25.0.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:72132b9a8a0a1840197794d4dea26080069b6b0981c116bc078762dc9691b21b", size = 35878945, upload-time = "2026-07-10T08:28:18.222Z" }, + { url = "https://files.pythonhosted.org/packages/64/d1/ccb01db7329ea0411ef4fbd9b62a04d3268b36777d4e758d5e39b91ddeab/pyarrow-25.0.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:e009ef945e498dca2f050ea10d2e9764cb44017254826fc4574fdb8d2530173b", size = 37630854, upload-time = "2026-07-10T08:28:23.452Z" }, + { url = "https://files.pythonhosted.org/packages/af/9f/2d81ba89d1e4198d0cb25fe7529de936830fdaec0db926bb52a1ef7080d4/pyarrow-25.0.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:f57a39dbcb416345401c2e77a4373669b45fd111a1768e6cf267a7a0607ff0ec", size = 46905617, upload-time = "2026-07-10T08:28:29.376Z" }, + { url = "https://files.pythonhosted.org/packages/6a/29/0ed312ec800fb536f93783215126cee4b8977dcfeccba6f0f44df0cc87d7/pyarrow-25.0.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:447df764beb07c544f0178a5f6b70ef44b9ecf382b3cdfad4c2d7867353c3887", size = 50119765, upload-time = "2026-07-10T08:28:35.826Z" }, + { url = "https://files.pythonhosted.org/packages/ca/88/cab5063ba0c4d46a9f6b4b7eb1c9029dc0302d65cd5ab3510c949a386568/pyarrow-25.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ac5dfeee59f9ceb4d45ba76e83b026c38c24334135bb329d8274baa49cec3c62", size = 50027563, upload-time = "2026-07-10T08:28:43.848Z" }, + { url = "https://files.pythonhosted.org/packages/7b/fb/4d24f1b7fe2e042dc4ef315ef75e4e702d8e46fe10c37e63caff00502b03/pyarrow-25.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f0f100dacf2c0f400601664a79d1a907ced4740514bb2b00917341038e2ce76f", size = 53162437, upload-time = "2026-07-10T08:28:52.819Z" }, + { url = "https://files.pythonhosted.org/packages/fa/65/da20806de93ca6ee91e72cb6a9b08b3ac890b46efc8d94a7326c651c4c81/pyarrow-25.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:2e093efbecb5317372f819228fa4b4e6157eee48d3f0a7b0303705ebf81a7104", size = 28613262, upload-time = "2026-07-10T08:29:47.544Z" }, + { url = "https://files.pythonhosted.org/packages/86/9f/c632afb1d3ef4a7814cee236718235f3a47eac46e97eb87df40f550b6b48/pyarrow-25.0.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:26be35b80780d2d21f4bae3d568b1666337c3a89722cc1794c956a77017cb24e", size = 36120702, upload-time = "2026-07-10T08:28:59.577Z" }, + { url = "https://files.pythonhosted.org/packages/36/0a/093d53a0e72ad06e45d6443e00651bbc2d21af4211295086cbf4d873d3b9/pyarrow-25.0.0-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:6f4812bfbf11ca7d8faf59eb8fff8bf4dd25ce3a38b62baa010cc17a0926d1b2", size = 37750674, upload-time = "2026-07-10T08:29:06.916Z" }, + { url = "https://files.pythonhosted.org/packages/8a/18/b37fc31a69cff4bdfb8842683def5612f551b93fff6f44375e4a4a6a5535/pyarrow-25.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:b8af8ceedf0c9c160fd2b63440f2d205b9404db85866c1217bfea601de7cfb50", size = 46912304, upload-time = "2026-07-10T08:29:14.656Z" }, + { url = "https://files.pythonhosted.org/packages/32/35/5cae19ba72493e5598022468b56f6a5571f399f485bf412f157356476caa/pyarrow-25.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:c70a5fd9a82bd1a702fd482bdc62d38dcb672fb2b449b1d7c0d7d1f4be7b7bfe", size = 50073652, upload-time = "2026-07-10T08:29:22.467Z" }, + { url = "https://files.pythonhosted.org/packages/2e/a5/ddd508424bdfd5e6945765e9e2ffc687e2f6115972badc8ecf423076c407/pyarrow-25.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0490a7f8b38ffe11cc26526b50c65d111cb54ddac3717cec781806793f1244dc", size = 50058654, upload-time = "2026-07-10T08:29:29.689Z" }, + { url = "https://files.pythonhosted.org/packages/5f/a4/324d0db203ff5eebe8694ec2d6ec5a23f9aaa5d02e5b8c692914c518c33c/pyarrow-25.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e83916bbcf380866b4e14255850b33323ff678dc9758411d0409cdd2523880b0", size = 53140153, upload-time = "2026-07-10T08:29:36.041Z" }, + { url = "https://files.pythonhosted.org/packages/bd/8d/d236e9c82fe315f9128885c8be3ec719f41965a1eb6b6f4b42470904cd41/pyarrow-25.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:13240f0d3dc5932ccd0bfa90cd76d835680b9d94a7661c635df4b703d40ce849", size = 28743657, upload-time = "2026-07-10T08:29:42.742Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826, upload-time = "2025-09-25T21:31:58.655Z" }, + { url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577, upload-time = "2025-09-25T21:32:00.088Z" }, + { url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556, upload-time = "2025-09-25T21:32:01.31Z" }, + { url = "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", size = 882114, upload-time = "2025-09-25T21:32:03.376Z" }, + { url = "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", size = 806638, upload-time = "2025-09-25T21:32:04.553Z" }, + { url = "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", size = 767463, upload-time = "2025-09-25T21:32:06.152Z" }, + { url = "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", size = 794986, upload-time = "2025-09-25T21:32:07.367Z" }, + { url = "https://files.pythonhosted.org/packages/45/91/47a6e1c42d9ee337c4839208f30d9f09caa9f720ec7582917b264defc875/pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", size = 142543, upload-time = "2025-09-25T21:32:08.95Z" }, + { url = "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", size = 158763, upload-time = "2025-09-25T21:32:09.96Z" }, + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "tzdata" +version = "2026.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/92/ff/5a28bdfd8c3ebec42564ac7d0e54ca3db65044a9314a97f9564fa7a1e926/tzdata-2026.3.tar.gz", hash = "sha256:4a1518b8993086a7982523e071643f3c0e5f213e75b21318e78bcabfff9d1415", size = 198674, upload-time = "2026-07-10T08:50:37.887Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl", hash = "sha256:dc096730c87af6cab1b171c9d532be840741ff5d459015e7f6947bd7d7e54931", size = 348168, upload-time = "2026-07-10T08:50:36.46Z" }, +] + +[[package]] +name = "urllib3" +version = "2.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, +] diff --git a/schema/clickhouse/canonical_market_data_replay.sql b/schema/clickhouse/canonical_market_data_replay.sql new file mode 100644 index 0000000..03f502d --- /dev/null +++ b/schema/clickhouse/canonical_market_data_replay.sql @@ -0,0 +1,85 @@ +-- Canonical replay-window contract. +-- +-- Run the two conflict preflight queries first. A replay orchestrator MUST fail +-- the selected bundle when either query returns a row; it must not consume a +-- partially conflicted bundle. Query parameters use ClickHouse HTTP-client +-- syntax and deliberately constrain bundle, venue, pair, and source-time range. + +SELECT + 'levels' AS conflict_table, + capture_bundle_id, + exchange, + trading_pair, + snapshot_id, + side, + level_index, + distinct_checksums +FROM market_data.cex_order_book_levels_conflicts +WHERE capture_bundle_id IN ({capture_bundle_ids:Array(String)}) + AND exchange = {exchange:String} + AND trading_pair = {trading_pair:String} + AND snapshot_id IN + ( + SELECT snapshot_id + FROM market_data.cex_order_book_depth_summary + WHERE capture_bundle_id IN ({capture_bundle_ids:Array(String)}) + AND exchange = {exchange:String} + AND trading_pair = {trading_pair:String} + AND source_time_ms >= {start_time_ms:UInt64} + AND source_time_ms < {end_time_ms:UInt64} + ); + +SELECT + 'summary' AS conflict_table, + capture_bundle_id, + exchange, + trading_pair, + snapshot_id, + distinct_checksums +FROM market_data.cex_order_book_depth_summary_conflicts +WHERE capture_bundle_id IN ({capture_bundle_ids:Array(String)}) + AND exchange = {exchange:String} + AND trading_pair = {trading_pair:String} + AND snapshot_id IN + ( + SELECT snapshot_id + FROM market_data.cex_order_book_depth_summary + WHERE capture_bundle_id IN ({capture_bundle_ids:Array(String)}) + AND exchange = {exchange:String} + AND trading_pair = {trading_pair:String} + AND source_time_ms >= {start_time_ms:UInt64} + AND source_time_ms < {end_time_ms:UInt64} + ); + +-- Replay levels. Only checksum-consistent logical rows are exposed here. +SELECT * +FROM market_data.cex_order_book_levels_canonical +WHERE capture_bundle_id IN ({capture_bundle_ids:Array(String)}) + AND exchange = {exchange:String} + AND trading_pair = {trading_pair:String} + AND feed = 'ORDERBOOK' + AND source_time_ms >= {start_time_ms:UInt64} + AND source_time_ms < {end_time_ms:UInt64} +ORDER BY source_time_ms, snapshot_id, side, level_index; + +-- Replay summaries. +SELECT * +FROM market_data.cex_order_book_depth_summary_canonical +WHERE capture_bundle_id IN ({capture_bundle_ids:Array(String)}) + AND exchange = {exchange:String} + AND trading_pair = {trading_pair:String} + AND feed = 'ORDERBOOK' + AND source_time_ms >= {start_time_ms:UInt64} + AND source_time_ms < {end_time_ms:UInt64} +ORDER BY source_time_ms, snapshot_id; + +-- Replay OHLCV using replacement semantics for forming/closed updates. +SELECT * +FROM market_data.cex_ohlcv FINAL +WHERE capture_bundle_id IN ({capture_bundle_ids:Array(String)}) + AND exchange = {exchange:String} + AND trading_pair = {trading_pair:String} + AND feed = 'OHLCV' + AND source_time_ms >= {start_time_ms:UInt64} + AND source_time_ms < {end_time_ms:UInt64} +ORDER BY source_time_ms, timeframe; diff --git a/schema/clickhouse/market_data.sql b/schema/clickhouse/market_data.sql index 71f8121..07d9bd4 100644 --- a/schema/clickhouse/market_data.sql +++ b/schema/clickhouse/market_data.sql @@ -213,3 +213,253 @@ ENGINE = MergeTree PARTITION BY toYYYYMM(fromUnixTimestamp64Milli(event_time_ms)) ORDER BY (exchange, asset_type, symbol, event_time_ms, trade_id) TTL toDateTime(fromUnixTimestamp64Milli(event_time_ms)) + INTERVAL 90 DAY; + +-- Replay capture-core columns are added idempotently so this schema upgrades +-- installations created before canonical replay capture was introduced. +ALTER TABLE market_data.cex_stream_events ADD COLUMN IF NOT EXISTS capture_bundle_id Nullable(String) AFTER deployment_id; +ALTER TABLE market_data.cex_stream_events ADD COLUMN IF NOT EXISTS trading_pair LowCardinality(String) AFTER symbol; +ALTER TABLE market_data.cex_stream_events ADD COLUMN IF NOT EXISTS source_symbol String AFTER trading_pair; +ALTER TABLE market_data.cex_stream_events ADD COLUMN IF NOT EXISTS feed LowCardinality(String) AFTER stream_type; +ALTER TABLE market_data.cex_stream_events ADD COLUMN IF NOT EXISTS provider LowCardinality(String) AFTER feed; +ALTER TABLE market_data.cex_stream_events ADD COLUMN IF NOT EXISTS source_mode LowCardinality(String) AFTER provider; +ALTER TABLE market_data.cex_stream_events ADD COLUMN IF NOT EXISTS source_time_ms UInt64 AFTER source_mode; +ALTER TABLE market_data.cex_stream_events ADD COLUMN IF NOT EXISTS raw_capture_id Nullable(String) AFTER received_time_ms; +ALTER TABLE market_data.cex_stream_events ADD COLUMN IF NOT EXISTS raw_capture_scope LowCardinality(Nullable(String)) AFTER raw_capture_id; +ALTER TABLE market_data.cex_stream_events ADD COLUMN IF NOT EXISTS schema_version LowCardinality(String) AFTER raw_capture_scope; +ALTER TABLE market_data.cex_stream_events ADD COLUMN IF NOT EXISTS checksum_algorithm LowCardinality(String) AFTER schema_version; +ALTER TABLE market_data.cex_stream_events ADD COLUMN IF NOT EXISTS raw_checksum Nullable(String) AFTER checksum_algorithm; +ALTER TABLE market_data.cex_stream_events ADD COLUMN IF NOT EXISTS provenance_complete UInt8 DEFAULT 0 AFTER raw_checksum; +ALTER TABLE market_data.cex_stream_events ADD COLUMN IF NOT EXISTS payload_encoding LowCardinality(String) DEFAULT 'legacy_json' AFTER provenance_complete; +ALTER TABLE market_data.cex_stream_events ADD COLUMN IF NOT EXISTS normalized_row_checksum String DEFAULT '' AFTER payload_json; + +ALTER TABLE market_data.cex_ticker_events ADD COLUMN IF NOT EXISTS capture_bundle_id Nullable(String) AFTER deployment_id; +ALTER TABLE market_data.cex_ticker_events ADD COLUMN IF NOT EXISTS trading_pair LowCardinality(String) AFTER symbol; +ALTER TABLE market_data.cex_ticker_events ADD COLUMN IF NOT EXISTS source_symbol String AFTER trading_pair; +ALTER TABLE market_data.cex_ticker_events ADD COLUMN IF NOT EXISTS feed LowCardinality(String) DEFAULT 'TICKER' AFTER source_symbol; +ALTER TABLE market_data.cex_ticker_events ADD COLUMN IF NOT EXISTS provider LowCardinality(String) AFTER feed; +ALTER TABLE market_data.cex_ticker_events ADD COLUMN IF NOT EXISTS source_mode LowCardinality(String) AFTER provider; +ALTER TABLE market_data.cex_ticker_events ADD COLUMN IF NOT EXISTS source_time_ms UInt64 AFTER source_mode; +ALTER TABLE market_data.cex_ticker_events ADD COLUMN IF NOT EXISTS raw_capture_id Nullable(String) AFTER received_time_ms; +ALTER TABLE market_data.cex_ticker_events ADD COLUMN IF NOT EXISTS raw_capture_scope LowCardinality(Nullable(String)) AFTER raw_capture_id; +ALTER TABLE market_data.cex_ticker_events ADD COLUMN IF NOT EXISTS schema_version LowCardinality(String) AFTER raw_capture_scope; +ALTER TABLE market_data.cex_ticker_events ADD COLUMN IF NOT EXISTS checksum_algorithm LowCardinality(String) AFTER schema_version; +ALTER TABLE market_data.cex_ticker_events ADD COLUMN IF NOT EXISTS raw_checksum Nullable(String) AFTER checksum_algorithm; +ALTER TABLE market_data.cex_ticker_events ADD COLUMN IF NOT EXISTS provenance_complete UInt8 DEFAULT 0 AFTER raw_checksum; +ALTER TABLE market_data.cex_ticker_events ADD COLUMN IF NOT EXISTS normalized_row_checksum String DEFAULT '' AFTER payload_json; + +ALTER TABLE market_data.cex_trades ADD COLUMN IF NOT EXISTS capture_bundle_id Nullable(String) AFTER deployment_id; +ALTER TABLE market_data.cex_trades ADD COLUMN IF NOT EXISTS trading_pair LowCardinality(String) AFTER symbol; +ALTER TABLE market_data.cex_trades ADD COLUMN IF NOT EXISTS source_symbol String AFTER trading_pair; +ALTER TABLE market_data.cex_trades ADD COLUMN IF NOT EXISTS feed LowCardinality(String) DEFAULT 'TRADES' AFTER source_symbol; +ALTER TABLE market_data.cex_trades ADD COLUMN IF NOT EXISTS provider LowCardinality(String) AFTER feed; +ALTER TABLE market_data.cex_trades ADD COLUMN IF NOT EXISTS source_mode LowCardinality(String) AFTER provider; +ALTER TABLE market_data.cex_trades ADD COLUMN IF NOT EXISTS source_time_ms UInt64 AFTER source_mode; +ALTER TABLE market_data.cex_trades ADD COLUMN IF NOT EXISTS raw_capture_id Nullable(String) AFTER received_time_ms; +ALTER TABLE market_data.cex_trades ADD COLUMN IF NOT EXISTS raw_capture_scope LowCardinality(Nullable(String)) AFTER raw_capture_id; +ALTER TABLE market_data.cex_trades ADD COLUMN IF NOT EXISTS schema_version LowCardinality(String) AFTER raw_capture_scope; +ALTER TABLE market_data.cex_trades ADD COLUMN IF NOT EXISTS checksum_algorithm LowCardinality(String) AFTER schema_version; +ALTER TABLE market_data.cex_trades ADD COLUMN IF NOT EXISTS raw_checksum Nullable(String) AFTER checksum_algorithm; +ALTER TABLE market_data.cex_trades ADD COLUMN IF NOT EXISTS provenance_complete UInt8 DEFAULT 0 AFTER raw_checksum; +ALTER TABLE market_data.cex_trades ADD COLUMN IF NOT EXISTS normalized_row_checksum String DEFAULT '' AFTER taker_or_maker; + +-- Append-only physical evidence. Canonical and conflict views below implement +-- query-time idempotency without erasing disagreeing deliveries. +CREATE TABLE IF NOT EXISTS market_data.cex_order_book_levels +( + source LowCardinality(String), + deployment_id LowCardinality(String), + capture_bundle_id Nullable(String), + exchange LowCardinality(String), + symbol LowCardinality(String), + trading_pair LowCardinality(String), + source_symbol String, + asset_type LowCardinality(String), + feed LowCardinality(String), + provider LowCardinality(String), + source_mode LowCardinality(String), + source_time_ms UInt64, + received_time_ms UInt64, + raw_capture_id Nullable(String), + raw_capture_scope LowCardinality(Nullable(String)), + schema_version LowCardinality(String), + checksum_algorithm LowCardinality(String), + raw_checksum Nullable(String), + provenance_complete UInt8, + snapshot_id String, + construction_mode LowCardinality(String), + gap_policy LowCardinality(String), + depth_limit UInt16, + sequence Nullable(UInt64), + exact_l2_reconstruction_complete UInt8, + side LowCardinality(String), + level_index UInt16, + price Decimal(38, 18), + amount Decimal(38, 18), + notional Decimal(38, 18), + mid_price Decimal(38, 18), + spread_from_mid_bps Float64, + normalized_row_checksum String +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(fromUnixTimestamp64Milli(source_time_ms)) +ORDER BY (exchange, trading_pair, capture_bundle_id, source_time_ms, raw_capture_id, snapshot_id, schema_version, side, level_index) +TTL toDateTime(fromUnixTimestamp64Milli(source_time_ms)) + INTERVAL 90 DAY +SETTINGS allow_nullable_key = 1; + +CREATE TABLE IF NOT EXISTS market_data.cex_order_book_depth_summary +( + source LowCardinality(String), + deployment_id LowCardinality(String), + capture_bundle_id Nullable(String), + exchange LowCardinality(String), + symbol LowCardinality(String), + trading_pair LowCardinality(String), + source_symbol String, + asset_type LowCardinality(String), + feed LowCardinality(String), + provider LowCardinality(String), + source_mode LowCardinality(String), + source_time_ms UInt64, + received_time_ms UInt64, + raw_capture_id Nullable(String), + raw_capture_scope LowCardinality(Nullable(String)), + schema_version LowCardinality(String), + checksum_algorithm LowCardinality(String), + raw_checksum Nullable(String), + provenance_complete UInt8, + snapshot_id String, + construction_mode LowCardinality(String), + gap_policy LowCardinality(String), + depth_limit UInt16, + sequence Nullable(UInt64), + exact_l2_reconstruction_complete UInt8, + best_bid Decimal(38, 18), + best_ask Decimal(38, 18), + best_bid_amount Decimal(38, 18), + best_ask_amount Decimal(38, 18), + mid_price Decimal(38, 18), + spread Decimal(38, 18), + spread_bps Float64, + staleness_ms UInt64, + bid_level_count UInt16, + ask_level_count UInt16, + measurement_bands_bps Array(UInt32), + bid_depth_by_band Array(Decimal(38, 18)), + ask_depth_by_band Array(Decimal(38, 18)), + normalized_row_checksum String +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(fromUnixTimestamp64Milli(source_time_ms)) +ORDER BY (exchange, trading_pair, capture_bundle_id, source_time_ms, raw_capture_id, snapshot_id, schema_version) +TTL toDateTime(fromUnixTimestamp64Milli(source_time_ms)) + INTERVAL 90 DAY +SETTINGS allow_nullable_key = 1; + +CREATE VIEW IF NOT EXISTS market_data.cex_order_book_levels_conflicts AS +SELECT + capture_bundle_id, exchange, trading_pair, raw_capture_id, snapshot_id, + schema_version, side, level_index, + groupUniqArray(normalized_row_checksum) AS distinct_checksums, + count() AS physical_row_count +FROM market_data.cex_order_book_levels +GROUP BY capture_bundle_id, exchange, trading_pair, raw_capture_id, snapshot_id, + schema_version, side, level_index +HAVING uniqExact(normalized_row_checksum) > 1; + +CREATE VIEW IF NOT EXISTS market_data.cex_order_book_levels_canonical AS +SELECT DISTINCT evidence.* +FROM market_data.cex_order_book_levels AS evidence +INNER JOIN +( + SELECT + capture_bundle_id, exchange, trading_pair, raw_capture_id, snapshot_id, + schema_version, side, level_index, + any(normalized_row_checksum) AS agreed_checksum + FROM market_data.cex_order_book_levels + GROUP BY capture_bundle_id, exchange, trading_pair, raw_capture_id, + snapshot_id, schema_version, side, level_index + HAVING uniqExact(normalized_row_checksum) = 1 +) AS consistent +ON evidence.capture_bundle_id IS NOT DISTINCT FROM consistent.capture_bundle_id +AND evidence.exchange = consistent.exchange +AND evidence.trading_pair = consistent.trading_pair +AND evidence.raw_capture_id IS NOT DISTINCT FROM consistent.raw_capture_id +AND evidence.snapshot_id = consistent.snapshot_id +AND evidence.schema_version = consistent.schema_version +AND evidence.side = consistent.side +AND evidence.level_index = consistent.level_index +AND evidence.normalized_row_checksum = consistent.agreed_checksum; + +CREATE VIEW IF NOT EXISTS market_data.cex_order_book_depth_summary_conflicts AS +SELECT + capture_bundle_id, exchange, trading_pair, raw_capture_id, snapshot_id, + schema_version, + groupUniqArray(normalized_row_checksum) AS distinct_checksums, + count() AS physical_row_count +FROM market_data.cex_order_book_depth_summary +GROUP BY capture_bundle_id, exchange, trading_pair, raw_capture_id, snapshot_id, + schema_version +HAVING uniqExact(normalized_row_checksum) > 1; + +CREATE VIEW IF NOT EXISTS market_data.cex_order_book_depth_summary_canonical AS +SELECT DISTINCT evidence.* +FROM market_data.cex_order_book_depth_summary AS evidence +INNER JOIN +( + SELECT + capture_bundle_id, exchange, trading_pair, raw_capture_id, snapshot_id, + schema_version, any(normalized_row_checksum) AS agreed_checksum + FROM market_data.cex_order_book_depth_summary + GROUP BY capture_bundle_id, exchange, trading_pair, raw_capture_id, + snapshot_id, schema_version + HAVING uniqExact(normalized_row_checksum) = 1 +) AS consistent +ON evidence.capture_bundle_id IS NOT DISTINCT FROM consistent.capture_bundle_id +AND evidence.exchange = consistent.exchange +AND evidence.trading_pair = consistent.trading_pair +AND evidence.raw_capture_id IS NOT DISTINCT FROM consistent.raw_capture_id +AND evidence.snapshot_id = consistent.snapshot_id +AND evidence.schema_version = consistent.schema_version +AND evidence.normalized_row_checksum = consistent.agreed_checksum; + +CREATE TABLE IF NOT EXISTS market_data.cex_ohlcv +( + source LowCardinality(String), + deployment_id LowCardinality(String), + capture_bundle_id Nullable(String), + exchange LowCardinality(String), + symbol LowCardinality(String), + trading_pair LowCardinality(String), + source_symbol String, + asset_type LowCardinality(String), + feed LowCardinality(String), + provider LowCardinality(String), + source_mode LowCardinality(String), + source_time_ms UInt64, + received_time_ms UInt64, + raw_capture_id Nullable(String), + raw_capture_scope LowCardinality(Nullable(String)), + schema_version LowCardinality(String), + checksum_algorithm LowCardinality(String), + raw_checksum Nullable(String), + provenance_complete UInt8, + timeframe LowCardinality(String), + open_time_ms UInt64, + open Decimal(38, 18), + high Decimal(38, 18), + low Decimal(38, 18), + close Decimal(38, 18), + volume Decimal(38, 18), + quote_volume Nullable(Decimal(38, 18)), + is_closed UInt8, + broker_version UInt64, + normalized_row_checksum String +) +ENGINE = ReplacingMergeTree(broker_version) +PARTITION BY toYYYYMM(fromUnixTimestamp64Milli(open_time_ms)) +ORDER BY (exchange, trading_pair, timeframe, open_time_ms, schema_version) +SETTINGS allow_nullable_key = 1; + +CREATE VIEW IF NOT EXISTS market_data.cex_ohlcv_closed AS +SELECT * +FROM market_data.cex_ohlcv FINAL +WHERE is_closed = 1; diff --git a/schema/clickhouse/migrations/canonical_market_data_replay_cutover.sql b/schema/clickhouse/migrations/canonical_market_data_replay_cutover.sql new file mode 100644 index 0000000..470fdbf --- /dev/null +++ b/schema/clickhouse/migrations/canonical_market_data_replay_cutover.sql @@ -0,0 +1,133 @@ +-- Operator-run table migration template. This file is intentionally not part +-- of automatic schema startup. Existing deployments write the legacy schema; +-- the upgraded broker writes only the latest canonical schema. No phase drops +-- a legacy or canonical table. + +-- PHASE 1: apply schema/clickhouse/market_data.sql while the existing broker +-- version remains deployed, then quiesce legacy writers for the migration +-- window. Do not deploy the canonical-only broker until all following checks +-- pass. + +-- PHASE 2: migrate every retained bounded source-time partition with +-- scripts/migrate-legacy-market-data-to-canonical.ts. Re-running a window is safe: +-- append-only order-book duplicates collapse in canonical views when their +-- checksums agree, and cex_ohlcv uses ReplacingMergeTree(broker_version). +-- Migrated rows use source_mode=legacy_migration_v1, +-- provenance_complete=0, and NULL bundle/raw identity/raw checksum. + +-- PHASE 3: parity checks. Zero mismatches are required before the FIET-937 +-- deployment observation window and canonical cutover. +SELECT + legacy.exchange, + replaceAll(legacy.symbol, '/', '-') AS trading_pair, + legacy.event_time_ms, + count() AS legacy_rows, + countIf(canonical.snapshot_id != '') AS canonical_rows, + countIf( + legacy.best_bid != canonical.best_bid OR + legacy.best_ask != canonical.best_ask OR + legacy.bid_levels != canonical.bid_level_count OR + legacy.ask_levels != canonical.ask_level_count + ) AS value_mismatches +FROM market_data.orderbook_snapshots AS legacy +LEFT JOIN market_data.cex_order_book_depth_summary_canonical AS canonical + ON canonical.exchange = legacy.exchange + AND canonical.trading_pair = replaceAll(legacy.symbol, '/', '-') + AND canonical.source_time_ms = legacy.event_time_ms + AND canonical.source_mode = 'legacy_migration_v1' +WHERE legacy.event_time_ms >= {start_time_ms:UInt64} + AND legacy.event_time_ms < {end_time_ms:UInt64} +GROUP BY legacy.exchange, trading_pair, legacy.event_time_ms +HAVING canonical_rows != legacy_rows OR value_mismatches != 0; + +SELECT + legacy.exchange, + replaceAll(legacy.symbol, '/', '-') AS trading_pair, + legacy.timeframe, + legacy.open_time_ms, + countIf( + legacy.open != canonical.open OR + legacy.high != canonical.high OR + legacy.low != canonical.low OR + legacy.close != canonical.close OR + legacy.volume != canonical.volume OR + legacy.is_closed != canonical.is_closed OR + legacy.broker_version != canonical.broker_version + ) AS value_mismatches +FROM market_data.candles FINAL AS legacy +LEFT JOIN market_data.cex_ohlcv FINAL AS canonical + ON canonical.exchange = legacy.exchange + AND canonical.trading_pair = replaceAll(legacy.symbol, '/', '-') + AND canonical.timeframe = legacy.timeframe + AND canonical.open_time_ms = legacy.open_time_ms + AND canonical.source_mode = 'legacy_migration_v1' +WHERE legacy.open_time_ms >= {start_time_ms:UInt64} + AND legacy.open_time_ms < {end_time_ms:UInt64} +GROUP BY legacy.exchange, trading_pair, legacy.timeframe, legacy.open_time_ms +HAVING value_mismatches != 0; + +-- PHASE 4: validate schema/clickhouse/canonical_market_data_replay.sql for every +-- retained pair and migration window. Any missing row or mismatch blocks the +-- upgraded deployment. + +-- PHASE 5: switch consumers to canonical names, then deploy the upgraded broker. +-- The upgraded broker writes only the latest canonical schema. The following +-- optional compatibility cutover preserves the old query names. +-- Execute as a reviewed maintenance operation, not during service startup. +-- +-- RENAME TABLE +-- market_data.orderbook_snapshots TO market_data.orderbook_snapshots_legacy, +-- market_data.candles TO market_data.candles_legacy, +-- market_data.candles_closed TO market_data.candles_closed_legacy; +-- +-- CREATE VIEW market_data.orderbook_snapshots AS +-- SELECT +-- any(summary.source) AS source, +-- any(summary.deployment_id) AS deployment_id, +-- '' AS account_selector, +-- summary.exchange, +-- any(summary.asset_type) AS asset_type, +-- any(summary.source_symbol) AS symbol, +-- summary.source_time_ms AS event_time_ms, +-- any(summary.received_time_ms) AS received_time_ms, +-- any(summary.best_bid) AS best_bid, +-- any(summary.best_ask) AS best_ask, +-- any(summary.best_bid_amount) AS bid_size, +-- any(summary.best_ask_amount) AS ask_size, +-- any(summary.mid_price) AS mid, +-- any(summary.spread_bps) AS spread_bps, +-- any(summary.depth_limit) AS depth_limit, +-- any(summary.bid_level_count) AS bid_levels, +-- any(summary.ask_level_count) AS ask_levels, +-- arrayMap(x -> x.2, arraySort(x -> x.1, groupArrayIf((level.level_index, level.price), level.side = 'bid'))) AS bids_price, +-- arrayMap(x -> x.2, arraySort(x -> x.1, groupArrayIf((level.level_index, level.amount), level.side = 'bid'))) AS bids_size, +-- arrayMap(x -> x.2, arraySort(x -> x.1, groupArrayIf((level.level_index, level.price), level.side = 'ask'))) AS asks_price, +-- arrayMap(x -> x.2, arraySort(x -> x.1, groupArrayIf((level.level_index, level.amount), level.side = 'ask'))) AS asks_size, +-- any(summary.sequence) AS sequence +-- FROM market_data.cex_order_book_depth_summary_canonical AS summary +-- INNER JOIN market_data.cex_order_book_levels_canonical AS level USING +-- (capture_bundle_id, exchange, trading_pair, raw_capture_id, snapshot_id, schema_version) +-- GROUP BY summary.exchange, summary.trading_pair, summary.source_time_ms, summary.snapshot_id; +-- +-- CREATE VIEW market_data.candles AS +-- SELECT +-- source, deployment_id, '' AS account_selector, exchange, asset_type, +-- source_symbol AS symbol, timeframe, open_time_ms, open, high, low, close, +-- volume, quote_volume, is_closed, broker_version +-- FROM market_data.cex_ohlcv FINAL; +-- +-- CREATE VIEW market_data.candles_closed AS +-- SELECT * FROM market_data.candles WHERE is_closed = 1; + +-- ROLLBACK: stop the upgraded broker. If compatibility names were switched, +-- rename—not drop—the canonical views, then restore the retained legacy objects +-- before rolling the application back to the previous legacy-writing version: +-- +-- RENAME TABLE +-- market_data.orderbook_snapshots TO market_data.orderbook_snapshots_canonical_compat, +-- market_data.candles TO market_data.candles_canonical_compat, +-- market_data.candles_closed TO market_data.candles_closed_canonical_compat, +-- market_data.orderbook_snapshots_legacy TO market_data.orderbook_snapshots, +-- market_data.candles_legacy TO market_data.candles, +-- market_data.candles_closed_legacy TO market_data.candles_closed; +-- Canonical base tables and all affected capture bundles remain intact. diff --git a/schema/clickhouse/strategy_data.sql b/schema/clickhouse/strategy_data.sql index a9790d8..fa7a914 100644 --- a/schema/clickhouse/strategy_data.sql +++ b/schema/clickhouse/strategy_data.sql @@ -1,36 +1,16 @@ --- Strategy runtime archive tables (HB external-strategy bridge, FIET-924). +-- Strategy runtime archive tables (Maker hb_runtime bridge; FIET-909/FIET-924). -- --- Cross-repo contract between this forwarder (DDL + validation) and the --- fiet-maker HB bridge (row producer). Column and table NAMES are the contract: --- do not rename them here without changing the producer in lockstep. --- Derived from Linear specs FIET-904 (policy clock), FIET-905 (policy/identity --- snapshots), FIET-906 (inventory/settlement), FIET-909 (provenance). +-- Maker performs one non-blocking HTTP attempt. For source=hb_runtime and the +-- five tables below, the archive forwarder accepts ownership only after an +-- atomic SQLite spool commit, returns 202, and retries each table independently. +-- All other archive sources remain on the direct ClickHouse path. -- --- Rows arrive through the archive forwarder (HTTP POST /archive) with the batch --- envelope source = "hb_runtime". --- --- NO TTL: replay-critical strategy history, retained indefinitely. --- --- Delivery contract: the producer is AT-MOST-ONCE over a bounded in-memory --- queue. It drops the oldest row when the queue is full and discards a batch --- after two failed POST attempts, so rows can be lost silently. To make loss --- distinguishable from "the event never happened", every row carries `seq`: a --- monotonic counter starting at 1, scoped to one (controller_id, run_id) and --- shared across ALL tables in this database. Gap detection is therefore a UNION --- of these tables filtered by run_id, looking for holes in `seq`; the producer's --- heartbeat carries dropped_rows/failed_batches as the aggregate counterpart. --- A hole proves an allocated row was lost. The converse does not hold: paths --- that skip emission entirely (e.g. blocked Layer12 ticks bypassing the archive --- tap) never allocate a `seq`, so their absence leaves no hole. --- --- Adding a column here is a POISON PILL if it ships after the producer. Inserts --- use JSONEachRow, so a row carrying a column ClickHouse does not have fails the --- whole per-table batch; the forwarder returns non-2xx and the producer drops it. --- Deploy the forwarder (which applies this file at startup) BEFORE the producer. +-- The v2 producer/stream fields are additive and defaulted so historical v1 +-- rows remain readable. Deploy this schema/forwarder before a v2 producer. +-- NO TTL: replay-critical strategy history is retained indefinitely. CREATE DATABASE IF NOT EXISTS strategy_data; --- One row per control-loop evaluation that produces a Layer12 decision (FIET-904). CREATE TABLE IF NOT EXISTS strategy_data.policy_evaluation_events ( event_time_ms Int64, @@ -45,27 +25,32 @@ CREATE TABLE IF NOT EXISTS strategy_data.policy_evaluation_events trading_pair LowCardinality(String), market_id String, run_id String, - -- Per-run gap-detection counter; see the delivery contract at the top. - seq UInt64, - + producer_id String DEFAULT '', + producer_run_id String DEFAULT '', + stream_name LowCardinality(String) DEFAULT '', + stream_seq UInt64 DEFAULT 0, + seq UInt64 DEFAULT 0, + archive_event_id String DEFAULT '', + tick_id String DEFAULT '', + policy_revision String DEFAULT '', + decision_stage LowCardinality(String) DEFAULT '', + decision_outcome LowCardinality(String) DEFAULT '', + decision_reason String DEFAULT '', + source_clock String DEFAULT '', + action_ids_json String DEFAULT '[]', policy_epoch String, fidelity LowCardinality(String), lag_ms Int64, fallback_reason String, - -- Durable replay provenance cursor ("block::log:"), empty when not derived from a chain cursor. source_cursor String, decision_kind LowCardinality(String), payload_json String ) ENGINE = MergeTree PARTITION BY toYYYYMM(fromUnixTimestamp64Milli(event_time_ms)) -ORDER BY (controller_id, trading_pair, event_time_ms); +ORDER BY (controller_id, trading_pair, event_time_ms) +SETTINGS non_replicated_deduplication_window = 1000000; -ALTER TABLE strategy_data.policy_evaluation_events -ADD COLUMN IF NOT EXISTS source_cursor String AFTER fallback_reason; - --- Append-only versioned snapshots of the effective controller/ladder config --- on start and on config/policy change, hash-gated (FIET-905). CREATE TABLE IF NOT EXISTS strategy_data.strategy_policy_snapshots ( event_time_ms Int64, @@ -80,9 +65,23 @@ CREATE TABLE IF NOT EXISTS strategy_data.strategy_policy_snapshots trading_pair LowCardinality(String), market_id String, run_id String, - -- Per-run gap-detection counter; see the delivery contract at the top. - seq UInt64, - + producer_id String DEFAULT '', + producer_run_id String DEFAULT '', + stream_name LowCardinality(String) DEFAULT '', + stream_seq UInt64 DEFAULT 0, + seq UInt64 DEFAULT 0, + archive_event_id String DEFAULT '', + content_hash String DEFAULT '', + revision String DEFAULT '', + active_from_ms Int64 DEFAULT 0, + canonical_market_id String DEFAULT '', + connector_id String DEFAULT '', + canonical_exchange String DEFAULT '', + canonical_trading_pair String DEFAULT '', + source_symbol String DEFAULT '', + base_asset LowCardinality(String) DEFAULT '', + quote_asset LowCardinality(String) DEFAULT '', + access_policy_id String DEFAULT '', snapshot_reason LowCardinality(String), policy_epoch String, config_file_path String, @@ -91,10 +90,9 @@ CREATE TABLE IF NOT EXISTS strategy_data.strategy_policy_snapshots ) ENGINE = MergeTree PARTITION BY toYYYYMM(fromUnixTimestamp64Milli(event_time_ms)) -ORDER BY (controller_id, trading_pair, event_time_ms); +ORDER BY (controller_id, trading_pair, event_time_ms) +SETTINGS non_replicated_deduplication_window = 1000000; --- Append-only market identity snapshots keyed by canonical core pool id; the --- latest row with event_time_ms <= t is the identity active at t (FIET-905). CREATE TABLE IF NOT EXISTS strategy_data.market_identity ( event_time_ms Int64, @@ -109,9 +107,23 @@ CREATE TABLE IF NOT EXISTS strategy_data.market_identity trading_pair LowCardinality(String), market_id String, run_id String, - -- Per-run gap-detection counter; see the delivery contract at the top. - seq UInt64, - + producer_id String DEFAULT '', + producer_run_id String DEFAULT '', + stream_name LowCardinality(String) DEFAULT '', + stream_seq UInt64 DEFAULT 0, + seq UInt64 DEFAULT 0, + archive_event_id String DEFAULT '', + content_hash String DEFAULT '', + revision String DEFAULT '', + active_from_ms Int64 DEFAULT 0, + canonical_market_id String DEFAULT '', + connector_id String DEFAULT '', + canonical_exchange String DEFAULT '', + canonical_trading_pair String DEFAULT '', + source_symbol String DEFAULT '', + base_asset LowCardinality(String) DEFAULT '', + quote_asset LowCardinality(String) DEFAULT '', + access_policy_id String DEFAULT '', snapshot_reason LowCardinality(String), source_hash String, core_pool_id String, @@ -120,10 +132,9 @@ CREATE TABLE IF NOT EXISTS strategy_data.market_identity ) ENGINE = MergeTree PARTITION BY toYYYYMM(fromUnixTimestamp64Milli(event_time_ms)) -ORDER BY (canonical_core_pool_id, event_time_ms); +ORDER BY (canonical_core_pool_id, event_time_ms) +SETTINGS non_replicated_deduplication_window = 1000000; --- Append-only CEX routeability snapshots keyed by exchange + trading_pair --- (FIET-905). CREATE TABLE IF NOT EXISTS strategy_data.symbol_mapping ( event_time_ms Int64, @@ -138,19 +149,32 @@ CREATE TABLE IF NOT EXISTS strategy_data.symbol_mapping trading_pair LowCardinality(String), market_id String, run_id String, - -- Per-run gap-detection counter; see the delivery contract at the top. - seq UInt64, - + producer_id String DEFAULT '', + producer_run_id String DEFAULT '', + stream_name LowCardinality(String) DEFAULT '', + stream_seq UInt64 DEFAULT 0, + seq UInt64 DEFAULT 0, + archive_event_id String DEFAULT '', + content_hash String DEFAULT '', + revision String DEFAULT '', + active_from_ms Int64 DEFAULT 0, + canonical_market_id String DEFAULT '', + connector_id String DEFAULT '', + canonical_exchange String DEFAULT '', + canonical_trading_pair String DEFAULT '', + source_symbol String DEFAULT '', + base_asset LowCardinality(String) DEFAULT '', + quote_asset LowCardinality(String) DEFAULT '', + access_policy_id String DEFAULT '', snapshot_reason LowCardinality(String), source_hash String, payload_json String ) ENGINE = MergeTree PARTITION BY toYYYYMM(fromUnixTimestamp64Milli(event_time_ms)) -ORDER BY (exchange, trading_pair, event_time_ms); +ORDER BY (exchange, trading_pair, event_time_ms) +SETTINGS non_replicated_deduplication_window = 1000000; --- Inventory snapshots (and, later, reservation/funding facts) on change with a --- periodic heartbeat, hash-gated (FIET-906). CREATE TABLE IF NOT EXISTS strategy_data.inventory_settlement_events ( event_time_ms Int64, @@ -165,9 +189,12 @@ CREATE TABLE IF NOT EXISTS strategy_data.inventory_settlement_events trading_pair LowCardinality(String), market_id String, run_id String, - -- Per-run gap-detection counter; see the delivery contract at the top. - seq UInt64, - + producer_id String DEFAULT '', + producer_run_id String DEFAULT '', + stream_name LowCardinality(String) DEFAULT '', + stream_seq UInt64 DEFAULT 0, + seq UInt64 DEFAULT 0, + archive_event_id String DEFAULT '', event_kind LowCardinality(String), token LowCardinality(String), account LowCardinality(String), @@ -177,21 +204,99 @@ CREATE TABLE IF NOT EXISTS strategy_data.inventory_settlement_events ) ENGINE = MergeTree PARTITION BY toYYYYMM(fromUnixTimestamp64Milli(event_time_ms)) -ORDER BY (controller_id, trading_pair, event_time_ms); +ORDER BY (controller_id, trading_pair, event_time_ms) +SETTINGS non_replicated_deduplication_window = 1000000; --- Backfill the gap-detection counter onto already-created tables. Existing rows --- keep seq = 0; only runs that start after the producer ships allocate real values. +-- Additive migration for tables created by the v1 forwarder. Existing values +-- remain intact; defaults represent unavailable producer/stream evidence. ALTER TABLE strategy_data.policy_evaluation_events -ADD COLUMN IF NOT EXISTS seq UInt64 AFTER run_id; + ADD COLUMN IF NOT EXISTS producer_id String DEFAULT '' AFTER run_id, + ADD COLUMN IF NOT EXISTS producer_run_id String DEFAULT '' AFTER producer_id, + ADD COLUMN IF NOT EXISTS stream_name LowCardinality(String) DEFAULT '' AFTER producer_run_id, + ADD COLUMN IF NOT EXISTS stream_seq UInt64 DEFAULT 0 AFTER stream_name, + ADD COLUMN IF NOT EXISTS seq UInt64 DEFAULT 0 AFTER stream_seq, + ADD COLUMN IF NOT EXISTS archive_event_id String DEFAULT '' AFTER seq, + ADD COLUMN IF NOT EXISTS tick_id String DEFAULT '' AFTER archive_event_id, + ADD COLUMN IF NOT EXISTS policy_revision String DEFAULT '' AFTER tick_id, + ADD COLUMN IF NOT EXISTS decision_stage LowCardinality(String) DEFAULT '' AFTER policy_revision, + ADD COLUMN IF NOT EXISTS decision_outcome LowCardinality(String) DEFAULT '' AFTER decision_stage, + ADD COLUMN IF NOT EXISTS decision_reason String DEFAULT '' AFTER decision_outcome, + ADD COLUMN IF NOT EXISTS source_clock String DEFAULT '' AFTER decision_reason, + ADD COLUMN IF NOT EXISTS action_ids_json String DEFAULT '[]' AFTER source_clock, + ADD COLUMN IF NOT EXISTS source_cursor String AFTER fallback_reason; ALTER TABLE strategy_data.strategy_policy_snapshots -ADD COLUMN IF NOT EXISTS seq UInt64 AFTER run_id; + ADD COLUMN IF NOT EXISTS producer_id String DEFAULT '' AFTER run_id, + ADD COLUMN IF NOT EXISTS producer_run_id String DEFAULT '' AFTER producer_id, + ADD COLUMN IF NOT EXISTS stream_name LowCardinality(String) DEFAULT '' AFTER producer_run_id, + ADD COLUMN IF NOT EXISTS stream_seq UInt64 DEFAULT 0 AFTER stream_name, + ADD COLUMN IF NOT EXISTS seq UInt64 DEFAULT 0 AFTER stream_seq, + ADD COLUMN IF NOT EXISTS archive_event_id String DEFAULT '' AFTER seq, + ADD COLUMN IF NOT EXISTS content_hash String DEFAULT '' AFTER archive_event_id, + ADD COLUMN IF NOT EXISTS revision String DEFAULT '' AFTER content_hash, + ADD COLUMN IF NOT EXISTS active_from_ms Int64 DEFAULT 0 AFTER revision, + ADD COLUMN IF NOT EXISTS canonical_market_id String DEFAULT '' AFTER active_from_ms, + ADD COLUMN IF NOT EXISTS connector_id String DEFAULT '' AFTER canonical_market_id, + ADD COLUMN IF NOT EXISTS canonical_exchange String DEFAULT '' AFTER connector_id, + ADD COLUMN IF NOT EXISTS canonical_trading_pair String DEFAULT '' AFTER canonical_exchange, + ADD COLUMN IF NOT EXISTS source_symbol String DEFAULT '' AFTER canonical_trading_pair, + ADD COLUMN IF NOT EXISTS base_asset LowCardinality(String) DEFAULT '' AFTER source_symbol, + ADD COLUMN IF NOT EXISTS quote_asset LowCardinality(String) DEFAULT '' AFTER base_asset, + ADD COLUMN IF NOT EXISTS access_policy_id String DEFAULT '' AFTER quote_asset; ALTER TABLE strategy_data.market_identity -ADD COLUMN IF NOT EXISTS seq UInt64 AFTER run_id; + ADD COLUMN IF NOT EXISTS producer_id String DEFAULT '' AFTER run_id, + ADD COLUMN IF NOT EXISTS producer_run_id String DEFAULT '' AFTER producer_id, + ADD COLUMN IF NOT EXISTS stream_name LowCardinality(String) DEFAULT '' AFTER producer_run_id, + ADD COLUMN IF NOT EXISTS stream_seq UInt64 DEFAULT 0 AFTER stream_name, + ADD COLUMN IF NOT EXISTS seq UInt64 DEFAULT 0 AFTER stream_seq, + ADD COLUMN IF NOT EXISTS archive_event_id String DEFAULT '' AFTER seq, + ADD COLUMN IF NOT EXISTS content_hash String DEFAULT '' AFTER archive_event_id, + ADD COLUMN IF NOT EXISTS revision String DEFAULT '' AFTER content_hash, + ADD COLUMN IF NOT EXISTS active_from_ms Int64 DEFAULT 0 AFTER revision, + ADD COLUMN IF NOT EXISTS canonical_market_id String DEFAULT '' AFTER active_from_ms, + ADD COLUMN IF NOT EXISTS connector_id String DEFAULT '' AFTER canonical_market_id, + ADD COLUMN IF NOT EXISTS canonical_exchange String DEFAULT '' AFTER connector_id, + ADD COLUMN IF NOT EXISTS canonical_trading_pair String DEFAULT '' AFTER canonical_exchange, + ADD COLUMN IF NOT EXISTS source_symbol String DEFAULT '' AFTER canonical_trading_pair, + ADD COLUMN IF NOT EXISTS base_asset LowCardinality(String) DEFAULT '' AFTER source_symbol, + ADD COLUMN IF NOT EXISTS quote_asset LowCardinality(String) DEFAULT '' AFTER base_asset, + ADD COLUMN IF NOT EXISTS access_policy_id String DEFAULT '' AFTER quote_asset; ALTER TABLE strategy_data.symbol_mapping -ADD COLUMN IF NOT EXISTS seq UInt64 AFTER run_id; + ADD COLUMN IF NOT EXISTS producer_id String DEFAULT '' AFTER run_id, + ADD COLUMN IF NOT EXISTS producer_run_id String DEFAULT '' AFTER producer_id, + ADD COLUMN IF NOT EXISTS stream_name LowCardinality(String) DEFAULT '' AFTER producer_run_id, + ADD COLUMN IF NOT EXISTS stream_seq UInt64 DEFAULT 0 AFTER stream_name, + ADD COLUMN IF NOT EXISTS seq UInt64 DEFAULT 0 AFTER stream_seq, + ADD COLUMN IF NOT EXISTS archive_event_id String DEFAULT '' AFTER seq, + ADD COLUMN IF NOT EXISTS content_hash String DEFAULT '' AFTER archive_event_id, + ADD COLUMN IF NOT EXISTS revision String DEFAULT '' AFTER content_hash, + ADD COLUMN IF NOT EXISTS active_from_ms Int64 DEFAULT 0 AFTER revision, + ADD COLUMN IF NOT EXISTS canonical_market_id String DEFAULT '' AFTER active_from_ms, + ADD COLUMN IF NOT EXISTS connector_id String DEFAULT '' AFTER canonical_market_id, + ADD COLUMN IF NOT EXISTS canonical_exchange String DEFAULT '' AFTER connector_id, + ADD COLUMN IF NOT EXISTS canonical_trading_pair String DEFAULT '' AFTER canonical_exchange, + ADD COLUMN IF NOT EXISTS source_symbol String DEFAULT '' AFTER canonical_trading_pair, + ADD COLUMN IF NOT EXISTS base_asset LowCardinality(String) DEFAULT '' AFTER source_symbol, + ADD COLUMN IF NOT EXISTS quote_asset LowCardinality(String) DEFAULT '' AFTER base_asset, + ADD COLUMN IF NOT EXISTS access_policy_id String DEFAULT '' AFTER quote_asset; + +ALTER TABLE strategy_data.inventory_settlement_events + ADD COLUMN IF NOT EXISTS producer_id String DEFAULT '' AFTER run_id, + ADD COLUMN IF NOT EXISTS producer_run_id String DEFAULT '' AFTER producer_id, + ADD COLUMN IF NOT EXISTS stream_name LowCardinality(String) DEFAULT '' AFTER producer_run_id, + ADD COLUMN IF NOT EXISTS stream_seq UInt64 DEFAULT 0 AFTER stream_name, + ADD COLUMN IF NOT EXISTS seq UInt64 DEFAULT 0 AFTER stream_seq, + ADD COLUMN IF NOT EXISTS archive_event_id String DEFAULT '' AFTER seq; +ALTER TABLE strategy_data.policy_evaluation_events + MODIFY SETTING non_replicated_deduplication_window = 1000000; +ALTER TABLE strategy_data.strategy_policy_snapshots + MODIFY SETTING non_replicated_deduplication_window = 1000000; +ALTER TABLE strategy_data.market_identity + MODIFY SETTING non_replicated_deduplication_window = 1000000; +ALTER TABLE strategy_data.symbol_mapping + MODIFY SETTING non_replicated_deduplication_window = 1000000; ALTER TABLE strategy_data.inventory_settlement_events -ADD COLUMN IF NOT EXISTS seq UInt64 AFTER run_id; + MODIFY SETTING non_replicated_deduplication_window = 1000000; diff --git a/scripts/export-canonical-orderbook-parquet.ts b/scripts/export-canonical-orderbook-parquet.ts new file mode 100644 index 0000000..d5a9537 --- /dev/null +++ b/scripts/export-canonical-orderbook-parquet.ts @@ -0,0 +1,303 @@ +#!/usr/bin/env bun +import { mkdir, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +export type CanonicalMarketReplayWindow = { + clickhouseUrl: string; + captureBundleIds: string[]; + exchange: string; + tradingPair: string; + startTimeMs: number; + endTimeMs: number; + username?: string; + password?: string; +}; + +export type CanonicalOrderBookParquetExport = CanonicalMarketReplayWindow & { + outputDirectory: string; +}; + +export type CanonicalOrderBookParquetExportResult = { + levelsPath: string; + summaryPath: string; + levelRows: number; + summaryRows: number; +}; + +export type CanonicalMarketReplayValidation = { + rawRowsByFeed: Record<"ORDERBOOK" | "TICKER" | "TRADES" | "OHLCV", number>; + normalizedRows: { + levels: number; + summaries: number; + tickers: number; + trades: number; + ohlcv: number; + }; +}; + +type HttpQuery = { + endpoint: URL; + headers: Record; + parameters: Record; +}; + +function clickHouseArray(values: string[]): string { + return `[${values + .map( + (value) => `'${value.replaceAll("\\", "\\\\").replaceAll("'", "\\'")}'`, + ) + .join(",")}]`; +} + +function validateInput(input: CanonicalMarketReplayWindow): void { + if (input.captureBundleIds.length === 0) { + throw new Error("At least one capture bundle is required"); + } + if (input.captureBundleIds.some((value) => value.trim().length === 0)) { + throw new Error("Capture bundle ids must be non-empty"); + } + if (!input.exchange.trim() || !input.tradingPair.trim()) { + throw new Error("Exchange and trading pair are required"); + } + if ( + !Number.isSafeInteger(input.startTimeMs) || + !Number.isSafeInteger(input.endTimeMs) || + input.startTimeMs < 0 || + input.endTimeMs <= input.startTimeMs + ) { + throw new Error("Replay export requires a bounded increasing time window"); + } +} + +function httpQuery(input: CanonicalMarketReplayWindow): HttpQuery { + const endpoint = new URL(input.clickhouseUrl); + const embeddedUsername = decodeURIComponent(endpoint.username); + const embeddedPassword = decodeURIComponent(endpoint.password); + endpoint.username = ""; + endpoint.password = ""; + endpoint.searchParams.set("database", "market_data"); + return { + endpoint, + headers: { + "X-ClickHouse-User": input.username || embeddedUsername || "default", + "X-ClickHouse-Key": input.password ?? embeddedPassword, + }, + parameters: { + capture_bundle_ids: clickHouseArray(input.captureBundleIds), + exchange: input.exchange, + trading_pair: input.tradingPair, + start_time_ms: String(input.startTimeMs), + end_time_ms: String(input.endTimeMs), + }, + }; +} + +async function executeQuery( + http: HttpQuery, + query: string, + format: "JSONEachRow" | "Parquet", +): Promise { + const endpoint = new URL(http.endpoint); + for (const [name, value] of Object.entries(http.parameters)) { + endpoint.searchParams.set(`param_${name}`, value); + } + const response = await fetch(endpoint, { + method: "POST", + headers: http.headers, + body: `${query.trim()}\nFORMAT ${format}`, + }); + const bytes = new Uint8Array(await response.arrayBuffer()); + if (!response.ok) { + const detail = new TextDecoder().decode(bytes).slice(0, 2_000); + throw new Error( + `ClickHouse replay export failed (${response.status}): ${detail}`, + ); + } + return bytes; +} + +const WINDOW_FILTER = ` +capture_bundle_id IN ({capture_bundle_ids:Array(String)}) +AND exchange = {exchange:String} +AND trading_pair = {trading_pair:String} +AND source_time_ms >= {start_time_ms:UInt64} +AND source_time_ms < {end_time_ms:UInt64} +`; + +async function rowCount(http: HttpQuery, table: string): Promise { + const bytes = await executeQuery( + http, + `SELECT count() AS rows FROM ${table} WHERE ${WINDOW_FILTER}`, + "JSONEachRow", + ); + const parsed = JSON.parse(new TextDecoder().decode(bytes)) as { + rows: string; + }; + return Number(parsed.rows); +} + +async function assertConflictFree(http: HttpQuery): Promise { + const bytes = await executeQuery( + http, + `SELECT count() AS conflicts + FROM + ( + SELECT snapshot_id + FROM market_data.cex_order_book_levels_conflicts + WHERE capture_bundle_id IN ({capture_bundle_ids:Array(String)}) + AND exchange = {exchange:String} + AND trading_pair = {trading_pair:String} + UNION ALL + SELECT snapshot_id + FROM market_data.cex_order_book_depth_summary_conflicts + WHERE capture_bundle_id IN ({capture_bundle_ids:Array(String)}) + AND exchange = {exchange:String} + AND trading_pair = {trading_pair:String} + ) AS conflicts + WHERE snapshot_id IN + ( + SELECT snapshot_id + FROM market_data.cex_order_book_depth_summary + WHERE ${WINDOW_FILTER} + )`, + "JSONEachRow", + ); + const parsed = JSON.parse(new TextDecoder().decode(bytes)) as { + conflicts: string; + }; + if (Number(parsed.conflicts) > 0) { + throw new Error( + "Canonical order-book export blocked by checksum conflicts in the selected replay window", + ); + } +} + +function parseJsonEachRow(bytes: Uint8Array): T[] { + const text = new TextDecoder().decode(bytes).trim(); + return text ? text.split("\n").map((line) => JSON.parse(line) as T) : []; +} + +export async function validateCanonicalMarketReplayWindow( + input: CanonicalMarketReplayWindow, +): Promise { + validateInput(input); + const http = httpQuery(input); + await assertConflictFree(http); + const [rawBytes, levels, summaries, tickers, trades, ohlcv] = + await Promise.all([ + executeQuery( + http, + `SELECT feed, count() AS rows + FROM market_data.cex_stream_events + WHERE ${WINDOW_FILTER} + GROUP BY feed`, + "JSONEachRow", + ), + rowCount(http, "market_data.cex_order_book_levels_canonical"), + rowCount(http, "market_data.cex_order_book_depth_summary_canonical"), + rowCount(http, "market_data.cex_ticker_events"), + rowCount(http, "market_data.cex_trades"), + rowCount(http, "market_data.cex_ohlcv FINAL"), + ]); + const rawRowsByFeed = { + ORDERBOOK: 0, + TICKER: 0, + TRADES: 0, + OHLCV: 0, + }; + for (const row of parseJsonEachRow<{ feed: string; rows: string }>( + rawBytes, + )) { + if (row.feed in rawRowsByFeed) { + rawRowsByFeed[row.feed as keyof typeof rawRowsByFeed] = Number(row.rows); + } + } + const normalizedRows = { levels, summaries, tickers, trades, ohlcv }; + const missingRawFeeds = Object.entries(rawRowsByFeed) + .filter(([, rows]) => rows === 0) + .map(([feed]) => feed); + const missingNormalizedFeeds = Object.entries(normalizedRows) + .filter(([, rows]) => rows === 0) + .map(([feed]) => feed); + if (missingRawFeeds.length > 0 || missingNormalizedFeeds.length > 0) { + throw new Error( + `Replay window is incomplete; missing raw feeds: ${missingRawFeeds.join(",") || "none"}; missing normalized rows: ${missingNormalizedFeeds.join(",") || "none"}`, + ); + } + return { rawRowsByFeed, normalizedRows }; +} + +function assertParquet(bytes: Uint8Array, label: string): void { + const decoder = new TextDecoder(); + if ( + bytes.length < 8 || + decoder.decode(bytes.subarray(0, 4)) !== "PAR1" || + decoder.decode(bytes.subarray(-4)) !== "PAR1" + ) { + throw new Error(`${label} export did not produce a valid Parquet envelope`); + } +} + +export async function exportCanonicalOrderBookParquet( + input: CanonicalOrderBookParquetExport, +): Promise { + validateInput(input); + const http = httpQuery(input); + await assertConflictFree(http); + const levelsTable = "market_data.cex_order_book_levels_canonical"; + const summaryTable = "market_data.cex_order_book_depth_summary_canonical"; + const [levelRows, summaryRows, levels, summary] = await Promise.all([ + rowCount(http, levelsTable), + rowCount(http, summaryTable), + executeQuery( + http, + `SELECT * FROM ${levelsTable} WHERE ${WINDOW_FILTER} + ORDER BY source_time_ms, snapshot_id, side, level_index`, + "Parquet", + ), + executeQuery( + http, + `SELECT * FROM ${summaryTable} WHERE ${WINDOW_FILTER} + ORDER BY source_time_ms, snapshot_id`, + "Parquet", + ), + ]); + assertParquet(levels, "Order-book levels"); + assertParquet(summary, "Order-book depth summary"); + await mkdir(input.outputDirectory, { recursive: true, mode: 0o700 }); + const levelsPath = join(input.outputDirectory, "order_book_levels.parquet"); + const summaryPath = join( + input.outputDirectory, + "order_book_depth_summary.parquet", + ); + await Promise.all([ + writeFile(levelsPath, levels, { flag: "wx", mode: 0o600 }), + writeFile(summaryPath, summary, { flag: "wx", mode: 0o600 }), + ]); + return { levelsPath, summaryPath, levelRows, summaryRows }; +} + +function requiredEnv(name: string): string { + const value = process.env[name]?.trim(); + if (!value) throw new Error(`${name} is required`); + return value; +} + +if (import.meta.main) { + const result = await exportCanonicalOrderBookParquet({ + clickhouseUrl: + process.env.CLICKHOUSE_URL?.trim() || + `http://${process.env.CLICKHOUSE_HOST?.trim() || "localhost"}:${process.env.CLICKHOUSE_PORT?.trim() || "8123"}`, + username: process.env.CLICKHOUSE_USER?.trim(), + password: process.env.CLICKHOUSE_PASSWORD, + outputDirectory: requiredEnv("CEX_BROKER_REPLAY_EXPORT_DIRECTORY"), + captureBundleIds: requiredEnv("CEX_BROKER_REPLAY_CAPTURE_BUNDLE_IDS") + .split(",") + .map((value) => value.trim()), + exchange: requiredEnv("CEX_BROKER_REPLAY_EXCHANGE"), + tradingPair: requiredEnv("CEX_BROKER_REPLAY_TRADING_PAIR"), + startTimeMs: Number(requiredEnv("CEX_BROKER_REPLAY_START_TIME_MS")), + endTimeMs: Number(requiredEnv("CEX_BROKER_REPLAY_END_TIME_MS")), + }); + console.info(JSON.stringify(result)); +} diff --git a/scripts/migrate-legacy-market-data-to-canonical.ts b/scripts/migrate-legacy-market-data-to-canonical.ts new file mode 100644 index 0000000..643e017 --- /dev/null +++ b/scripts/migrate-legacy-market-data-to-canonical.ts @@ -0,0 +1,160 @@ +#!/usr/bin/env bun +import { createClient } from "@clickhouse/client"; +import type { + BrokerArchiveRow, + BrokerArchiveSource, +} from "../src/helpers/broker-execution-archive/types"; +import { + buildLegacyOhlcvMigrationRow, + buildLegacyOrderBookMigrationRows, + type LegacyCandle, + type LegacyOrderBookSnapshot, +} from "../src/helpers/market-data-archive/legacy-migration"; + +const startTimeMs = Number(process.env.CEX_BROKER_MIGRATION_START_TIME_MS); +const endTimeMs = Number(process.env.CEX_BROKER_MIGRATION_END_TIME_MS); +if ( + !Number.isSafeInteger(startTimeMs) || + !Number.isSafeInteger(endTimeMs) || + startTimeMs < 0 || + endTimeMs <= startTimeMs +) { + throw new Error( + "CEX_BROKER_MIGRATION_START_TIME_MS and CEX_BROKER_MIGRATION_END_TIME_MS must define a bounded increasing window", + ); +} + +const clickhouse = createClient({ + url: + process.env.CLICKHOUSE_URL?.trim() || + `http://${process.env.CLICKHOUSE_HOST?.trim() || "localhost"}:${process.env.CLICKHOUSE_PORT?.trim() || "8123"}`, + database: "market_data", +}); +const confirmed = process.env.CEX_BROKER_CANONICAL_MIGRATION_CONFIRM === "true"; + +function numberField(row: Record, field: string): number { + const value = Number(row[field]); + if (!Number.isFinite(value)) throw new Error(`Invalid legacy ${field}`); + return value; +} + +function numberArray(row: Record, field: string): number[] { + const value = row[field]; + if (!Array.isArray(value)) throw new Error(`Invalid legacy ${field}`); + return value.map((entry) => Number(entry)); +} + +function sourceField(value: unknown): BrokerArchiveSource { + return value === "broker_read" ? "broker_read" : "broker_write"; +} + +function orderBookInput(row: Record): LegacyOrderBookSnapshot { + return { + source: sourceField(row.source), + deployment_id: String(row.deployment_id), + account_selector: String(row.account_selector ?? ""), + exchange: String(row.exchange), + asset_type: + row.asset_type === "swap" || row.asset_type === "future" + ? row.asset_type + : "spot", + symbol: String(row.symbol), + event_time_ms: numberField(row, "event_time_ms"), + received_time_ms: numberField(row, "received_time_ms"), + depth_limit: numberField(row, "depth_limit"), + bids_price: numberArray(row, "bids_price"), + bids_size: numberArray(row, "bids_size"), + asks_price: numberArray(row, "asks_price"), + asks_size: numberArray(row, "asks_size"), + sequence: + row.sequence === null || row.sequence === undefined + ? undefined + : numberField(row, "sequence"), + }; +} + +function candleInput(row: Record): LegacyCandle { + return { + source: sourceField(row.source), + deployment_id: String(row.deployment_id), + account_selector: String(row.account_selector ?? ""), + exchange: String(row.exchange), + asset_type: + row.asset_type === "swap" || row.asset_type === "future" + ? row.asset_type + : "spot", + symbol: String(row.symbol), + timeframe: String(row.timeframe), + open_time_ms: numberField(row, "open_time_ms"), + open: numberField(row, "open"), + high: numberField(row, "high"), + low: numberField(row, "low"), + close: numberField(row, "close"), + volume: numberField(row, "volume"), + quote_volume: + row.quote_volume === null || row.quote_volume === undefined + ? undefined + : numberField(row, "quote_volume"), + is_closed: numberField(row, "is_closed") === 1 ? 1 : 0, + broker_version: numberField(row, "broker_version"), + }; +} + +async function readLegacyRows( + table: "orderbook_snapshots" | "candles", + timeField: "event_time_ms" | "open_time_ms", +): Promise[]> { + const result = await clickhouse.query({ + query: ` + SELECT * FROM market_data.${table}${table === "candles" ? " FINAL" : ""} + WHERE ${timeField} >= {start:UInt64} AND ${timeField} < {end:UInt64} + ORDER BY exchange, symbol, ${timeField} + `, + query_params: { start: startTimeMs, end: endTimeMs }, + format: "JSONEachRow", + }); + return (await result.json()) as Record[]; +} + +async function insertRows(rows: BrokerArchiveRow[]): Promise { + const byTable = new Map(); + for (const entry of rows) { + const entries = byTable.get(entry.table) ?? []; + entries.push(entry); + byTable.set(entry.table, entries); + } + for (const [table, entries] of byTable) { + await clickhouse.insert({ + table, + values: entries.map(({ row }) => row), + format: "JSONEachRow", + }); + } +} + +try { + const [legacyBooks, legacyCandles] = await Promise.all([ + readLegacyRows("orderbook_snapshots", "event_time_ms"), + readLegacyRows("candles", "open_time_ms"), + ]); + const canonicalRows = [ + ...legacyBooks.flatMap((row) => + buildLegacyOrderBookMigrationRows(orderBookInput(row)), + ), + ...legacyCandles.map((row) => + buildLegacyOhlcvMigrationRow(candleInput(row)), + ), + ]; + console.info( + JSON.stringify({ + window: { start_time_ms: startTimeMs, end_time_ms: endTimeMs }, + legacy_order_books: legacyBooks.length, + legacy_candles: legacyCandles.length, + canonical_rows: canonicalRows.length, + mode: confirmed ? "write" : "dry_run", + }), + ); + if (confirmed) await insertRows(canonicalRows); +} finally { + await clickhouse.close(); +} diff --git a/scripts/validate-canonical-market-replay.ts b/scripts/validate-canonical-market-replay.ts new file mode 100644 index 0000000..5111548 --- /dev/null +++ b/scripts/validate-canonical-market-replay.ts @@ -0,0 +1,81 @@ +#!/usr/bin/env bun +import { z } from "zod"; +import { + type CanonicalMarketReplayWindow, + validateCanonicalMarketReplayWindow, +} from "./export-canonical-orderbook-parquet"; + +const replayWindowSchema = z + .object({ + captureBundleIds: z.array(z.string().trim().min(1)).min(1), + exchange: z.string().trim().min(1), + tradingPair: z.string().trim().min(1), + startTimeMs: z.number().int().nonnegative(), + endTimeMs: z.number().int().positive(), + }) + .strict() + .refine((window) => window.endTimeMs > window.startTimeMs, { + message: "endTimeMs must be greater than startTimeMs", + path: ["endTimeMs"], + }); + +const replayConfigSchema = z + .object({ + windows: z + .array(replayWindowSchema) + .min(1, "at least one validation window is required"), + }) + .strict(); + +export type ReplayValidationWindow = z.infer; + +export function parseReplayValidationConfig( + input: unknown, +): ReplayValidationWindow[] { + const result = replayConfigSchema.safeParse(input); + if (!result.success) { + const detail = result.error.issues + .map((issue) => `${issue.path.join(".") || "config"}: ${issue.message}`) + .join("; "); + throw new Error(`Invalid replay validation config: ${detail}`); + } + return result.data.windows; +} + +async function loadConfig(path: string): Promise { + let input: unknown; + try { + input = JSON.parse(await Bun.file(path).text()); + } catch (error) { + throw new Error(`Unable to read replay validation config at ${path}`, { + cause: error, + }); + } + return parseReplayValidationConfig(input); +} + +if (import.meta.main) { + const configPath = process.env.CEX_BROKER_REPLAY_VALIDATION_CONFIG?.trim(); + if (!configPath) { + throw new Error("CEX_BROKER_REPLAY_VALIDATION_CONFIG is required"); + } + const clickhouseUrl = + process.env.CLICKHOUSE_URL?.trim() || + `http://${process.env.CLICKHOUSE_HOST?.trim() || "localhost"}:${process.env.CLICKHOUSE_PORT?.trim() || "8123"}`; + const shared = { + clickhouseUrl, + username: process.env.CLICKHOUSE_USER?.trim(), + password: process.env.CLICKHOUSE_PASSWORD, + }; + const validated = []; + for (const window of await loadConfig(configPath)) { + const input: CanonicalMarketReplayWindow = { ...shared, ...window }; + validated.push({ + window, + result: await validateCanonicalMarketReplayWindow(input), + }); + } + console.info( + JSON.stringify({ status: "valid", windows: validated }, null, 2), + ); +} diff --git a/services/archive-forwarder/Dockerfile b/services/archive-forwarder/Dockerfile index cce0994..a31f59b 100644 --- a/services/archive-forwarder/Dockerfile +++ b/services/archive-forwarder/Dockerfile @@ -1,6 +1,7 @@ # Archive forwarder: receives cex-broker archive batches over HTTP (POST /archive) # and writes them to ClickHouse (market_data, broker_execution, strategy_data). -# It self-initializes the schema on boot and fail-closes if that fails. +# It self-initializes the durable spool before listening and retries ClickHouse +# schema initialization without giving up durable Maker admission. # # Build from the repository ROOT so the schema/ directory is in the build context: # docker build -f services/archive-forwarder/Dockerfile -t cex-broker-archive-forwarder . @@ -22,6 +23,8 @@ COPY src/helpers/otel.ts src/helpers/logger.ts ./src/helpers/ COPY schema/clickhouse ./schema/clickhouse ENV ARCHIVE_FORWARDER_PORT=8090 +ENV ARCHIVE_FORWARDER_SPOOL_PATH=/var/lib/archive-forwarder/spool.sqlite +VOLUME ["/var/lib/archive-forwarder"] EXPOSE 8090 CMD ["bun", "run", "services/archive-forwarder/index.ts"] diff --git a/services/archive-forwarder/config.ts b/services/archive-forwarder/config.ts index 3477de0..d57c4ec 100644 --- a/services/archive-forwarder/config.ts +++ b/services/archive-forwarder/config.ts @@ -9,6 +9,7 @@ export type ClickHouseConfig = { export type ForwarderConfig = { port: number; authToken?: string; + spoolPath: string; clickhouse: ClickHouseConfig; }; @@ -25,6 +26,9 @@ export function loadForwarderConfig(): ForwarderConfig { return { port: parsePort(process.env.ARCHIVE_FORWARDER_PORT, 8090), authToken: authToken || undefined, + spoolPath: + process.env.ARCHIVE_FORWARDER_SPOOL_PATH?.trim() || + DEFAULT_STRATEGY_SPOOL_PATH, clickhouse: { host: process.env.CLICKHOUSE_HOST?.trim() || "localhost", port: parsePort(process.env.CLICKHOUSE_PORT, 8123), @@ -34,3 +38,4 @@ export function loadForwarderConfig(): ForwarderConfig { }, }; } +import { DEFAULT_STRATEGY_SPOOL_PATH } from "./strategy-spool"; diff --git a/services/archive-forwarder/health.ts b/services/archive-forwarder/health.ts index 1d7895a..39d67b4 100644 --- a/services/archive-forwarder/health.ts +++ b/services/archive-forwarder/health.ts @@ -1,4 +1,30 @@ import type { ClickHouseClient } from "@clickhouse/client"; +import type { StrategySpoolStats } from "./strategy-spool"; + +export type ForwarderHealthInput = { + clickhouseOk: boolean; + spoolOk: boolean; + spool: StrategySpoolStats | null; +}; + +export function evaluateForwarderHealth(input: ForwarderHealthInput) { + const status = !input.spoolOk + ? "unavailable" + : input.clickhouseOk + ? "ok" + : "degraded"; + return { + statusCode: input.spoolOk ? 200 : 503, + body: { + status, + clickhouse: input.clickhouseOk, + durableAdmission: input.spoolOk, + spool: input.spoolOk + ? { healthy: true, ...input.spool } + : { healthy: false }, + }, + }; +} export async function pingClickHouse( client: ClickHouseClient, diff --git a/services/archive-forwarder/index.ts b/services/archive-forwarder/index.ts index ef076f0..9efaaf9 100644 --- a/services/archive-forwarder/index.ts +++ b/services/archive-forwarder/index.ts @@ -1,10 +1,15 @@ import { createClient } from "@clickhouse/client"; import { loadForwarderConfig } from "./config"; import { createClickHouseInserter } from "./insert"; -import { pingClickHouse } from "./health"; +import { evaluateForwarderHealth, pingClickHouse } from "./health"; import { handleArchiveRequest } from "./request"; import { ensureArchiveSchema } from "./schema"; import { createArchiveForwarderTelemetry } from "./telemetry"; +import { + StrategyArchiveSpool, + StrategySpoolUnavailableError, +} from "./strategy-spool"; +import { StrategySpoolWorker } from "./strategy-worker"; const config = loadForwarderConfig(); const clickhouse = createClient({ @@ -21,33 +26,82 @@ const clickhouse = createClient({ }); const inserter = createClickHouseInserter(clickhouse); const telemetry = createArchiveForwarderTelemetry(); +let spool: StrategyArchiveSpool | undefined; +let worker: StrategySpoolWorker | undefined; try { - await ensureArchiveSchema(clickhouse); - console.log( - "ClickHouse archive schema ensured (market_data, broker_execution, broker_account, strategy_data)", - ); + spool = new StrategyArchiveSpool({ path: config.spoolPath }); + worker = new StrategySpoolWorker({ spool, inserter, telemetry }); } catch (error) { - console.error("Failed to ensure ClickHouse schema:", error); - process.exit(1); + console.error("Failed to initialize durable strategy archive spool:", error); + telemetry.recordStrategyAdmissionRejected("spool_unavailable"); } +let schemaReady = false; +let schemaRetryTimer: ReturnType | undefined; + +async function ensureSchemaAndStartDrainage(): Promise { + try { + await ensureArchiveSchema(clickhouse); + schemaReady = true; + worker?.start(); + console.log( + "ClickHouse archive schema ensured (market_data, broker_execution, broker_account, strategy_data)", + ); + } catch (error) { + schemaReady = false; + console.error("Failed to ensure ClickHouse schema; durable admission remains available:", error); + schemaRetryTimer = setTimeout(() => { + void ensureSchemaAndStartDrainage(); + }, 5_000); + schemaRetryTimer.unref?.(); + } +} + +void ensureSchemaAndStartDrainage(); + const server = Bun.serve({ port: config.port, async fetch(request) { const url = new URL(request.url); if (request.method === "GET" && url.pathname === "/health") { - const clickhouseOk = await pingClickHouse(clickhouse); - return Response.json( - { status: clickhouseOk ? "ok" : "degraded", clickhouse: clickhouseOk }, - { status: clickhouseOk ? 200 : 503 }, - ); + const clickhouseOk = schemaReady && (await pingClickHouse(clickhouse)); + if (!spool) { + const health = evaluateForwarderHealth({ + clickhouseOk, + spoolOk: false, + spool: null, + }); + return Response.json(health.body, { status: health.statusCode }); + } + try { + const spoolStats = spool.assertWritable(); + telemetry.recordStrategySpoolStats(spoolStats); + const health = evaluateForwarderHealth({ + clickhouseOk, + spoolOk: true, + spool: spoolStats, + }); + return Response.json(health.body, { status: health.statusCode }); + } catch (error) { + telemetry.recordStrategyAdmissionRejected("spool_unavailable"); + const health = evaluateForwarderHealth({ + clickhouseOk, + spoolOk: false, + spool: null, + }); + if (!(error instanceof StrategySpoolUnavailableError)) { + console.error("Strategy spool health check failed:", error); + } + return Response.json(health.body, { status: health.statusCode }); + } } if (request.method === "POST" && url.pathname === "/archive") { return handleArchiveRequest(request, { authToken: config.authToken, inserter, + spool, telemetry, }); } @@ -62,3 +116,18 @@ console.log( console.log( `ClickHouse target: ${config.clickhouse.host}:${config.clickhouse.port}/${config.clickhouse.database}`, ); +console.log(`Strategy archive spool: ${config.spoolPath}`); + +let shuttingDown = false; +async function shutdown(): Promise { + if (shuttingDown) return; + shuttingDown = true; + if (schemaRetryTimer) clearTimeout(schemaRetryTimer); + worker?.stop(); + server.stop(true); + await clickhouse.close(); + spool?.close(); +} + +process.once("SIGTERM", () => void shutdown()); +process.once("SIGINT", () => void shutdown()); diff --git a/services/archive-forwarder/insert.ts b/services/archive-forwarder/insert.ts index 196fa53..7c0be9b 100644 --- a/services/archive-forwarder/insert.ts +++ b/services/archive-forwarder/insert.ts @@ -6,6 +6,7 @@ import { isSupportedTable } from "./types"; export type RowInserter = ( table: SupportedTable, rows: Record[], + options?: { deduplicationToken?: string }, ) => Promise; export function groupRowsByTable( @@ -67,11 +68,18 @@ export async function insertArchiveRows( export function createClickHouseInserter( client: ClickHouseClient, ): RowInserter { - return async (table, rows) => { + return async (table, rows, options) => { await client.insert({ table, values: rows, format: "JSONEachRow", + ...(options?.deduplicationToken + ? { + clickhouse_settings: { + insert_deduplication_token: options.deduplicationToken, + }, + } + : {}), }); }; } diff --git a/services/archive-forwarder/limits.ts b/services/archive-forwarder/limits.ts index 6bd9135..738c5b1 100644 --- a/services/archive-forwarder/limits.ts +++ b/services/archive-forwarder/limits.ts @@ -1,5 +1,28 @@ export const MAX_ARCHIVE_BODY_BYTES = 5 * 1024 * 1024; export const MAX_ARCHIVE_ROWS = 1_000; +export const MAX_ARCHIVE_ROWS_BY_TABLE: Readonly> = { + "market_data.cex_stream_events": 500, + "market_data.cex_ticker_events": 500, + "market_data.cex_trades": 1_000, + "market_data.cex_ohlcv": 500, + "market_data.cex_order_book_levels": 1_000, + "market_data.cex_order_book_depth_summary": 500, +}; + +export function findTableRowLimitViolation( + rows: readonly { table: string }[], +): { table: string; maxRows: number; received: number } | undefined { + const counts = new Map(); + for (const { table } of rows) { + const received = (counts.get(table) ?? 0) + 1; + counts.set(table, received); + const maxRows = MAX_ARCHIVE_ROWS_BY_TABLE[table]; + if (maxRows !== undefined && received > maxRows) { + return { table, maxRows, received }; + } + } + return undefined; +} export function isArchiveBodyTooLarge(contentLength: string | null): boolean { if (!contentLength) { diff --git a/services/archive-forwarder/request.ts b/services/archive-forwarder/request.ts index c3dbd0f..e849a91 100644 --- a/services/archive-forwarder/request.ts +++ b/services/archive-forwarder/request.ts @@ -2,15 +2,26 @@ import { isArchiveRequestAuthorized } from "./auth"; import type { RowInserter } from "./insert"; import { MAX_ARCHIVE_ROWS, + findTableRowLimitViolation, isArchiveBodyTooLarge, readBoundedArchiveBody, } from "./limits"; import { handleArchiveBatch, parseArchiveBatchRequest } from "./router"; +import { + classifyStrategyArchiveBatch, + validateStrategyArchiveBatch, +} from "./strategy-contract"; +import { + StrategySpoolQuotaError, + StrategySpoolUnavailableError, + type StrategyArchiveSpool, +} from "./strategy-spool"; import type { ArchiveForwarderTelemetry } from "./telemetry"; export type ArchiveRequestDependencies = { authToken?: string; inserter: RowInserter; + spool?: Pick; telemetry: ArchiveForwarderTelemetry; }; @@ -46,6 +57,25 @@ export async function handleArchiveRequest( return Response.json({ error: "Invalid JSON body" }, { status: 400 }); } + const strategyClassification = classifyStrategyArchiveBatch(body); + if ( + strategyClassification === "invalid_strategy_mix" || + strategyClassification === "invalid_strategy_source" + ) { + dependencies.telemetry.recordStrategyAdmissionRejected("invalid_contract"); + return Response.json( + { error: "Invalid strategy archive source or table mix" }, + { status: 400 }, + ); + } + if (strategyClassification === "strategy") { + const validation = validateStrategyArchiveBatch(body); + if (!validation.ok) { + dependencies.telemetry.recordStrategyAdmissionRejected("invalid_contract"); + return Response.json({ error: validation.error }, { status: 400 }); + } + } + const parsed = parseArchiveBatchRequest(body); if (!parsed.ok) { return Response.json( @@ -56,6 +86,10 @@ export async function handleArchiveRequest( if (parsed.rejectedRowCount > 0) { dependencies.telemetry.recordRejectedRows(parsed.rejectedRowsByTable); + dependencies.telemetry.recordChecksumConflicts( + parsed.batch.source, + parsed.checksumConflictsByTable, + ); // Name the offending tables: an unknown table (e.g. a forgotten // SUPPORTED_TABLES entry for a new archive table) would otherwise reject // the whole batch with only a count, hiding which table is at fault. @@ -83,6 +117,52 @@ export async function handleArchiveRequest( { status: 413 }, ); } + const tableLimit = findTableRowLimitViolation(parsed.batch.rows); + if (tableLimit) { + return Response.json( + { error: "Too many archive rows for table", ...tableLimit }, + { status: 413 }, + ); + } + + if (strategyClassification === "strategy") { + if (!dependencies.spool) { + dependencies.telemetry.recordStrategyAdmissionRejected("spool_unavailable"); + return Response.json( + { error: "Strategy archive spool unavailable" }, + { status: 503 }, + ); + } + try { + const admitted = dependencies.spool.admit(parsed.batch); + dependencies.telemetry.recordStrategyAdmission(parsed.batch.rows.length); + return Response.json( + { + ok: true, + accepted: true, + batchId: admitted.batchId, + rows: parsed.batch.rows.length, + }, + { status: 202 }, + ); + } catch (error) { + if (error instanceof StrategySpoolQuotaError) { + dependencies.telemetry.recordStrategyAdmissionRejected("quota"); + return Response.json( + { error: "Strategy archive spool quota exceeded" }, + { status: 429 }, + ); + } + if (!(error instanceof StrategySpoolUnavailableError)) { + console.error("Strategy archive spool admission failed:", error); + } + dependencies.telemetry.recordStrategyAdmissionRejected("spool_unavailable"); + return Response.json( + { error: "Strategy archive spool unavailable" }, + { status: 503 }, + ); + } + } try { const result = await handleArchiveBatch( diff --git a/services/archive-forwarder/router.ts b/services/archive-forwarder/router.ts index 1f4b273..2ace142 100644 --- a/services/archive-forwarder/router.ts +++ b/services/archive-forwarder/router.ts @@ -14,10 +14,14 @@ export type ParsedArchiveBatch = // A rejected row with no string `table` contributes "(malformed)". rejectedTables: string[]; rejectedRowsByTable: Record; + checksumConflictsByTable: Record; } | { ok: false }; -function isValidArchiveRow(entry: unknown): entry is ArchiveBatchRequest["rows"][number] { +function isValidArchiveRow( + entry: unknown, + envelopeSource?: string, +): entry is ArchiveBatchRequest["rows"][number] { if (!entry || typeof entry !== "object" || Array.isArray(entry)) { return false; } @@ -32,7 +36,77 @@ function isValidArchiveRow(entry: unknown): entry is ArchiveBatchRequest["rows"] ) { return false; } - return isSupportedTable(row.table); + if (!isSupportedTable(row.table)) { + return false; + } + const rowSource = (row.row as Record).source; + return !( + envelopeSource !== undefined && + typeof rowSource === "string" && + rowSource !== envelopeSource + ); +} + +const ORDER_BOOK_LOGICAL_KEYS = { + "market_data.cex_order_book_levels": [ + "capture_bundle_id", + "exchange", + "trading_pair", + "raw_capture_id", + "snapshot_id", + "schema_version", + "side", + "level_index", + ], + "market_data.cex_order_book_depth_summary": [ + "capture_bundle_id", + "exchange", + "trading_pair", + "raw_capture_id", + "snapshot_id", + "schema_version", + ], +} as const; + +function conflictingOrderBookRows( + rows: ArchiveBatchRequest["rows"], +): Set { + const groups = new Map< + string, + { checksums: Set; rows: ArchiveBatchRequest["rows"] } + >(); + for (const entry of rows) { + const keys = + ORDER_BOOK_LOGICAL_KEYS[ + entry.table as keyof typeof ORDER_BOOK_LOGICAL_KEYS + ]; + if (!keys) continue; + const checksum = entry.row.normalized_row_checksum; + const values = keys.map((key) => entry.row[key]); + if ( + typeof checksum !== "string" || + values.some( + (value) => typeof value !== "string" && typeof value !== "number", + ) + ) { + continue; + } + const logicalKey = `${entry.table}\u0000${JSON.stringify(values)}`; + const group = groups.get(logicalKey) ?? { + checksums: new Set(), + rows: [], + }; + group.checksums.add(checksum); + group.rows.push(entry); + groups.set(logicalKey, group); + } + const conflicts = new Set(); + for (const group of groups.values()) { + if (group.checksums.size > 1) { + for (const entry of group.rows) conflicts.add(entry); + } + } + return conflicts; } export function parseArchiveBatchRequest(body: unknown): ParsedArchiveBatch { @@ -50,8 +124,16 @@ export function parseArchiveBatchRequest(body: unknown): ParsedArchiveBatch { return { ok: false }; } const inputRowCount = record.rows.length; - const rows = record.rows.filter(isValidArchiveRow); - const rejectedRowsByTable = countRejectedRowsByTable(record.rows); + const structurallyValidRows = record.rows.filter((entry) => + isValidArchiveRow(entry, record.source as string), + ); + const conflicts = conflictingOrderBookRows(structurallyValidRows); + const rows = structurallyValidRows.filter((entry) => !conflicts.has(entry)); + const rejectedRowsByTable = countRejectedRowsByTable( + record.rows, + record.source as string, + conflicts, + ); return { ok: true, batch: { @@ -63,13 +145,31 @@ export function parseArchiveBatchRequest(body: unknown): ParsedArchiveBatch { rejectedRowCount: inputRowCount - rows.length, rejectedTables: Object.keys(rejectedRowsByTable), rejectedRowsByTable, + checksumConflictsByTable: countRowsByTable(conflicts), }; } -function countRejectedRowsByTable(rows: unknown[]): Record { +function countRowsByTable( + rows: ReadonlySet, +): Record { + const counts = new Map(); + for (const { table } of rows) { + counts.set(table, (counts.get(table) ?? 0) + 1); + } + return Object.fromEntries(counts); +} + +function countRejectedRowsByTable( + rows: unknown[], + envelopeSource: string, + conflicts: ReadonlySet, +): Record { const counts = new Map(); for (const entry of rows) { - if (isValidArchiveRow(entry)) { + if ( + isValidArchiveRow(entry, envelopeSource) && + !conflicts.has(entry) + ) { continue; } const table = (entry as { table?: unknown } | null)?.table; diff --git a/services/archive-forwarder/strategy-contract.ts b/services/archive-forwarder/strategy-contract.ts new file mode 100644 index 0000000..7d62385 --- /dev/null +++ b/services/archive-forwarder/strategy-contract.ts @@ -0,0 +1,172 @@ +import type { ArchiveBatchRequest, ArchiveRow } from "./types"; + +export const STRATEGY_ARCHIVE_SOURCE = "hb_runtime"; + +export const STRATEGY_ARCHIVE_TABLES = [ + "strategy_data.policy_evaluation_events", + "strategy_data.strategy_policy_snapshots", + "strategy_data.market_identity", + "strategy_data.symbol_mapping", + "strategy_data.inventory_settlement_events", +] as const; + +export type StrategyArchiveTable = (typeof STRATEGY_ARCHIVE_TABLES)[number]; + +export type StrategyBatchClassification = + | "strategy" + | "direct" + | "invalid_strategy_source" + | "invalid_strategy_mix"; + +export type StrategyContractValidation = + | { ok: true } + | { ok: false; error: string }; + +export function isStrategyArchiveTable( + table: unknown, +): table is StrategyArchiveTable { + return ( + typeof table === "string" && + (STRATEGY_ARCHIVE_TABLES as readonly string[]).includes(table) + ); +} + +function rawRows(value: unknown): unknown[] { + if (!value || typeof value !== "object") return []; + const rows = (value as { rows?: unknown }).rows; + return Array.isArray(rows) ? rows : []; +} + +function rawSource(value: unknown): unknown { + return value && typeof value === "object" + ? (value as { source?: unknown }).source + : undefined; +} + +function entryTable(entry: unknown): unknown { + return entry && typeof entry === "object" + ? (entry as { table?: unknown }).table + : undefined; +} + +export function classifyStrategyArchiveBatch( + value: unknown, +): StrategyBatchClassification { + const rows = rawRows(value); + const source = rawSource(value); + const hasStrategyRows = rows.some((entry) => + isStrategyArchiveTable(entryTable(entry)), + ); + + if (source === STRATEGY_ARCHIVE_SOURCE) { + return rows.length > 0 && rows.every((entry) => + isStrategyArchiveTable(entryTable(entry)), + ) + ? "strategy" + : "invalid_strategy_mix"; + } + return hasStrategyRows ? "invalid_strategy_source" : "direct"; +} + +function nonEmptyString(value: unknown): value is string { + return typeof value === "string" && value.trim().length > 0; +} + +function positiveSequence(value: unknown): boolean { + if ( + typeof value === "number" && + Number.isSafeInteger(value) && + value > 0 + ) { + return true; + } + if (typeof value !== "string" || !/^[1-9]\d*$/.test(value)) return false; + return BigInt(value) <= 18_446_744_073_709_551_615n; +} + +function validateV2Row(row: Record): string | undefined { + for (const field of [ + "producer_id", + "producer_run_id", + "stream_name", + "archive_event_id", + ] as const) { + if (!nonEmptyString(row[field])) return `Missing v2 field: ${field}`; + } + for (const field of ["stream_seq", "seq"] as const) { + if (!positiveSequence(row[field])) return `Invalid v2 field: ${field}`; + } + return undefined; +} + +function validateStrategyRow( + entry: unknown, + source: string, + deploymentId: string, +): string | undefined { + if (!entry || typeof entry !== "object" || Array.isArray(entry)) { + return "Malformed strategy row entry"; + } + const archiveRow = entry as Partial; + if (!isStrategyArchiveTable(archiveRow.table)) { + return "Unsupported strategy table"; + } + if ( + !archiveRow.row || + typeof archiveRow.row !== "object" || + Array.isArray(archiveRow.row) + ) { + return "Malformed strategy row body"; + } + const row = archiveRow.row; + if (row.source !== undefined && row.source !== source) { + return "Row source does not match envelope"; + } + if ( + row.deployment_id !== undefined && + row.deployment_id !== deploymentId + ) { + return "Row deployment does not match envelope"; + } + + const version = row.schema_version; + if (version === undefined || version === "" || version === "1") { + return undefined; + } + if (version !== "2") return "Unsupported strategy schema version"; + return validateV2Row(row); +} + +export function validateStrategyArchiveBatch( + value: unknown, +): StrategyContractValidation { + if (!value || typeof value !== "object" || Array.isArray(value)) { + return { ok: false, error: "Malformed strategy envelope" }; + } + const envelope = value as Partial; + if (!nonEmptyString(envelope.source)) { + return { ok: false, error: "Missing strategy envelope source" }; + } + if (!nonEmptyString(envelope.deployment_id)) { + return { ok: false, error: "Missing strategy deployment id" }; + } + if (!Array.isArray(envelope.rows) || envelope.rows.length === 0) { + return { ok: false, error: "Strategy envelope rows must be non-empty" }; + } + if (envelope.rows.length > 1_000) { + return { ok: false, error: "Too many strategy rows" }; + } + const classification = classifyStrategyArchiveBatch(envelope); + if (classification !== "strategy") { + return { ok: false, error: "Invalid strategy source or table mix" }; + } + for (const entry of envelope.rows) { + const error = validateStrategyRow( + entry, + envelope.source, + envelope.deployment_id, + ); + if (error) return { ok: false, error }; + } + return { ok: true }; +} diff --git a/services/archive-forwarder/strategy-spool.ts b/services/archive-forwarder/strategy-spool.ts new file mode 100644 index 0000000..080132f --- /dev/null +++ b/services/archive-forwarder/strategy-spool.ts @@ -0,0 +1,429 @@ +import { Database } from "bun:sqlite"; +import { createHash, randomUUID } from "node:crypto"; +import { groupRowsByTable } from "./insert"; +import type { StrategyArchiveTable } from "./strategy-contract"; +import type { ArchiveBatchRequest } from "./types"; + +export const STRATEGY_SPOOL_MAX_BYTES = 1_073_741_824; +export const STRATEGY_SPOOL_RETENTION_MS = 72 * 60 * 60 * 1_000; +export const DEFAULT_STRATEGY_SPOOL_PATH = "./archive-forwarder-spool.sqlite"; + +const WORK_METADATA_BYTES = 256; + +function canonicalSerialize(value: unknown): string { + if (value === null) return "null"; + if (typeof value === "string" || typeof value === "boolean") { + return JSON.stringify(value); + } + if (typeof value === "number") { + if (!Number.isFinite(value)) throw new Error("Spool values must be finite"); + return Object.is(value, -0) ? "0" : JSON.stringify(value); + } + if (Array.isArray(value)) { + return `[${value.map(canonicalSerialize).join(",")}]`; + } + if (typeof value === "object") { + return `{${Object.entries(value as Record) + .filter(([, entry]) => entry !== undefined) + .sort(([left], [right]) => left.localeCompare(right)) + .map( + ([key, entry]) => + `${JSON.stringify(key)}:${canonicalSerialize(entry)}`, + ) + .join(",")}}`; + } + throw new Error(`Unsupported spool value: ${typeof value}`); +} + +type SpoolLimits = { + maxBytes: number; + retentionMs: number; +}; + +export type StrategySpoolOptions = { + path?: string; + now?: () => number; + /** Deterministic test seam. Runtime construction always uses fixed constants. */ + limits?: Partial; +}; + +export type StrategySpoolAdmission = { + batchId: string; + accountedBytes: number; +}; + +export type StrategySpoolWork = { + batchId: string; + table: StrategyArchiveTable; + rows: Record[]; + dedupeToken: string; + attemptCount: number; + admittedAtMs: number; + expiresAtMs: number; +}; + +export type StrategySpoolStats = { + queuedBatches: number; + queuedWork: number; + terminalWork: number; + expiredWork: number; + accountedBytes: number; + oldestAgeMs: number; + lastErrorClass: string; +}; + +export class StrategySpoolQuotaError extends Error { + constructor() { + super("Strategy archive spool quota exceeded"); + this.name = "StrategySpoolQuotaError"; + } +} + +export class StrategySpoolUnavailableError extends Error { + constructor(message: string, options?: ErrorOptions) { + super(message, options); + this.name = "StrategySpoolUnavailableError"; + } +} + +type WorkRow = { + batch_id: string; + table_name: StrategyArchiveTable; + rows_json: string; + dedupe_token: string; + attempt_count: number; + admitted_at_ms: number; + expires_at_ms: number; +}; + +function utf8Bytes(value: string): number { + return new TextEncoder().encode(value).byteLength; +} + +function dedupeToken(batchId: string, table: string): string { + return createHash("sha256") + .update(`maker-archive-forwarder-v1\0${batchId}\0${table}`) + .digest("hex"); +} + +function groupedRows(batch: ArchiveBatchRequest) { + return [...groupRowsByTable(batch.rows)].map(([table, rows]) => ({ + table: table as StrategyArchiveTable, + rows, + rowsJson: canonicalSerialize(rows), + })); +} + +export class StrategyArchiveSpool { + private readonly database: Database; + private readonly now: () => number; + private readonly limits: SpoolLimits; + public readonly path: string; + + constructor(options: StrategySpoolOptions = {}) { + this.path = options.path?.trim() || DEFAULT_STRATEGY_SPOOL_PATH; + this.now = options.now ?? Date.now; + this.limits = { + maxBytes: options.limits?.maxBytes ?? STRATEGY_SPOOL_MAX_BYTES, + retentionMs: + options.limits?.retentionMs ?? STRATEGY_SPOOL_RETENTION_MS, + }; + try { + this.database = new Database(this.path, { create: true, strict: true }); + this.configure(); + this.migrate(); + } catch (error) { + throw new StrategySpoolUnavailableError( + `Unable to initialize strategy archive spool at ${this.path}`, + { cause: error }, + ); + } + } + + private configure(): void { + this.database.exec("PRAGMA journal_mode = WAL"); + this.database.exec("PRAGMA foreign_keys = ON"); + this.database.exec("PRAGMA busy_timeout = 5000"); + this.database.exec("PRAGMA synchronous = FULL"); + } + + private migrate(): void { + this.database.exec(` + CREATE TABLE IF NOT EXISTS strategy_spool_batches ( + batch_id TEXT PRIMARY KEY, + source TEXT NOT NULL, + deployment_id TEXT NOT NULL, + admitted_at_ms INTEGER NOT NULL, + expires_at_ms INTEGER NOT NULL, + accounted_bytes INTEGER NOT NULL CHECK (accounted_bytes > 0), + envelope_json TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS strategy_spool_work ( + batch_id TEXT NOT NULL REFERENCES strategy_spool_batches(batch_id) ON DELETE CASCADE, + table_name TEXT NOT NULL, + rows_json TEXT NOT NULL, + dedupe_token TEXT NOT NULL, + state TEXT NOT NULL DEFAULT 'pending' CHECK (state IN ('pending', 'terminal')), + attempt_count INTEGER NOT NULL DEFAULT 0, + next_attempt_at_ms INTEGER NOT NULL, + last_error_class TEXT NOT NULL DEFAULT '', + last_error_at_ms INTEGER, + PRIMARY KEY (batch_id, table_name) + ); + CREATE INDEX IF NOT EXISTS strategy_spool_work_due + ON strategy_spool_work(state, next_attempt_at_ms); + CREATE TABLE IF NOT EXISTS strategy_spool_health ( + id INTEGER PRIMARY KEY CHECK (id = 1), + last_checked_at_ms INTEGER NOT NULL + ); + INSERT OR IGNORE INTO strategy_spool_health(id, last_checked_at_ms) VALUES (1, 0); + CREATE TABLE IF NOT EXISTS strategy_spool_counters ( + name TEXT PRIMARY KEY, + value INTEGER NOT NULL DEFAULT 0 + ); + INSERT OR IGNORE INTO strategy_spool_counters(name, value) VALUES ('expired_work', 0); + `); + } + + public accountedBytes(batch: ArchiveBatchRequest): number { + const envelopeJson = canonicalSerialize(batch); + return groupedRows(batch).reduce( + (total, group) => + total + + utf8Bytes(group.table) + + utf8Bytes(group.rowsJson) + + 64 + + WORK_METADATA_BYTES, + utf8Bytes(envelopeJson), + ); + } + + public admit(batch: ArchiveBatchRequest): StrategySpoolAdmission { + const admittedAtMs = this.now(); + const expiresAtMs = admittedAtMs + this.limits.retentionMs; + const batchId = randomUUID(); + const envelopeJson = canonicalSerialize(batch); + const groups = groupedRows(batch); + const accountedBytes = this.accountedBytes(batch); + try { + const transaction = this.database.transaction(() => { + this.deleteExpired(admittedAtMs); + const current = this.database + .query<{ used: number }, []>( + "SELECT COALESCE(SUM(accounted_bytes), 0) AS used FROM strategy_spool_batches", + ) + .get()?.used ?? 0; + if (current + accountedBytes > this.limits.maxBytes) { + throw new StrategySpoolQuotaError(); + } + this.database + .query( + `INSERT INTO strategy_spool_batches + (batch_id, source, deployment_id, admitted_at_ms, expires_at_ms, accounted_bytes, envelope_json) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)`, + ) + .run( + batchId, + batch.source, + batch.deployment_id, + admittedAtMs, + expiresAtMs, + accountedBytes, + envelopeJson, + ); + for (const group of groups) { + this.database + .query( + `INSERT INTO strategy_spool_work + (batch_id, table_name, rows_json, dedupe_token, next_attempt_at_ms) + VALUES (?1, ?2, ?3, ?4, ?5)`, + ) + .run( + batchId, + group.table, + group.rowsJson, + dedupeToken(batchId, group.table), + admittedAtMs, + ); + } + }); + transaction.immediate(); + return { batchId, accountedBytes }; + } catch (error) { + if (error instanceof StrategySpoolQuotaError) throw error; + throw new StrategySpoolUnavailableError( + "Unable to commit strategy archive batch", + { cause: error }, + ); + } + } + + public dueWork(limit = 100): StrategySpoolWork[] { + const rows = this.database + .query( + `SELECT w.batch_id, w.table_name, w.rows_json, w.dedupe_token, + w.attempt_count, b.admitted_at_ms, b.expires_at_ms + FROM strategy_spool_work w + JOIN strategy_spool_batches b ON b.batch_id = w.batch_id + WHERE w.state = 'pending' AND w.next_attempt_at_ms <= ?1 AND b.expires_at_ms > ?1 + ORDER BY w.next_attempt_at_ms, b.admitted_at_ms, w.table_name + LIMIT ?2`, + ) + .all(this.now(), limit); + return rows.map((row) => ({ + batchId: row.batch_id, + table: row.table_name, + rows: JSON.parse(row.rows_json) as Record[], + dedupeToken: row.dedupe_token, + attemptCount: row.attempt_count, + admittedAtMs: row.admitted_at_ms, + expiresAtMs: row.expires_at_ms, + })); + } + + public complete(work: Pick): void { + const transaction = this.database.transaction(() => { + this.database + .query( + "DELETE FROM strategy_spool_work WHERE batch_id = ?1 AND table_name = ?2", + ) + .run(work.batchId, work.table); + this.database + .query( + `DELETE FROM strategy_spool_batches + WHERE batch_id = ?1 + AND NOT EXISTS (SELECT 1 FROM strategy_spool_work WHERE batch_id = ?1)`, + ) + .run(work.batchId); + }); + transaction.immediate(); + } + + public reschedule( + work: Pick, + failure: { nextAttemptAtMs: number; errorClass: string }, + ): void { + this.database + .query( + `UPDATE strategy_spool_work + SET attempt_count = attempt_count + 1, + next_attempt_at_ms = ?3, + last_error_class = ?4, + last_error_at_ms = ?5 + WHERE batch_id = ?1 AND table_name = ?2`, + ) + .run( + work.batchId, + work.table, + failure.nextAttemptAtMs, + failure.errorClass, + this.now(), + ); + } + + public markTerminal( + work: Pick, + errorClass: string, + ): void { + this.database + .query( + `UPDATE strategy_spool_work + SET state = 'terminal', attempt_count = attempt_count + 1, + last_error_class = ?3, last_error_at_ms = ?4 + WHERE batch_id = ?1 AND table_name = ?2`, + ) + .run(work.batchId, work.table, errorClass, this.now()); + } + + private deleteExpired(now: number): number { + const count = + this.database + .query<{ count: number }, [number]>( + `SELECT COUNT(*) AS count FROM strategy_spool_work w + JOIN strategy_spool_batches b ON b.batch_id = w.batch_id + WHERE b.expires_at_ms <= ?1`, + ) + .get(now)?.count ?? 0; + this.database + .query("DELETE FROM strategy_spool_batches WHERE expires_at_ms <= ?1") + .run(now); + if (count > 0) { + this.database + .query( + "UPDATE strategy_spool_counters SET value = value + ?1 WHERE name = 'expired_work'", + ) + .run(count); + } + return count; + } + + public expire(): number { + const transaction = this.database.transaction(() => + this.deleteExpired(this.now()), + ); + return transaction.immediate(); + } + + public stats(): StrategySpoolStats { + const now = this.now(); + const row = this.database + .query< + { + queued_batches: number; + queued_work: number; + terminal_work: number; + expired_work: number; + accounted_bytes: number; + oldest_admitted_at_ms: number | null; + last_error_class: string | null; + }, + [] + >( + `SELECT + (SELECT COUNT(*) FROM strategy_spool_batches) AS queued_batches, + (SELECT COUNT(*) FROM strategy_spool_work) AS queued_work, + (SELECT COUNT(*) FROM strategy_spool_work WHERE state = 'terminal') AS terminal_work, + (SELECT value FROM strategy_spool_counters WHERE name = 'expired_work') AS expired_work, + (SELECT COALESCE(SUM(accounted_bytes), 0) FROM strategy_spool_batches) AS accounted_bytes, + (SELECT MIN(admitted_at_ms) FROM strategy_spool_batches) AS oldest_admitted_at_ms, + (SELECT last_error_class FROM strategy_spool_work + WHERE last_error_class <> '' ORDER BY last_error_at_ms DESC LIMIT 1) AS last_error_class`, + ) + .get(); + return { + queuedBatches: row?.queued_batches ?? 0, + queuedWork: row?.queued_work ?? 0, + terminalWork: row?.terminal_work ?? 0, + expiredWork: row?.expired_work ?? 0, + accountedBytes: row?.accounted_bytes ?? 0, + oldestAgeMs: + row?.oldest_admitted_at_ms == null + ? 0 + : Math.max(0, now - row.oldest_admitted_at_ms), + lastErrorClass: row?.last_error_class ?? "none", + }; + } + + public assertWritable(): StrategySpoolStats { + try { + const transaction = this.database.transaction(() => { + this.database + .query( + "UPDATE strategy_spool_health SET last_checked_at_ms = ?1 WHERE id = 1", + ) + .run(this.now()); + }); + transaction.immediate(); + return this.stats(); + } catch (error) { + throw new StrategySpoolUnavailableError( + "Strategy archive spool is not writable", + { cause: error }, + ); + } + } + + public close(): void { + this.database.close(); + } +} diff --git a/services/archive-forwarder/strategy-worker.ts b/services/archive-forwarder/strategy-worker.ts new file mode 100644 index 0000000..98f4b1f --- /dev/null +++ b/services/archive-forwarder/strategy-worker.ts @@ -0,0 +1,136 @@ +import type { RowInserter } from "./insert"; +import type { ArchiveForwarderTelemetry } from "./telemetry"; +import { classifyInsertError } from "./telemetry"; +import type { StrategyArchiveSpool } from "./strategy-spool"; + +const RETRY_CAP_MS = 60_000; +const RETRY_JITTER = 0.2; + +export function retryDelayMs( + attemptCount: number, + random: () => number = Math.random, +): number { + const base = Math.min(RETRY_CAP_MS, 1_000 * 2 ** attemptCount); + const jitterFactor = 1 - RETRY_JITTER + random() * RETRY_JITTER * 2; + return Math.round(base * jitterFactor); +} + +function isPermanentError(errorClass: string): boolean { + return errorClass === "schema" || errorClass === "authentication"; +} + +export type StrategyWorkerOptions = { + spool: StrategyArchiveSpool; + inserter: RowInserter; + telemetry?: ArchiveForwarderTelemetry; + now?: () => number; + random?: () => number; + pollIntervalMs?: number; +}; + +export type StrategyDrainResult = { + completed: number; + retried: number; + terminal: number; + expired: number; +}; + +export class StrategySpoolWorker { + private readonly spool: StrategyArchiveSpool; + private readonly inserter: RowInserter; + private readonly telemetry?: ArchiveForwarderTelemetry; + private readonly now: () => number; + private readonly random: () => number; + private readonly pollIntervalMs: number; + private timer?: ReturnType; + private stopped = true; + private draining = false; + + constructor(options: StrategyWorkerOptions) { + this.spool = options.spool; + this.inserter = options.inserter; + this.telemetry = options.telemetry; + this.now = options.now ?? Date.now; + this.random = options.random ?? Math.random; + this.pollIntervalMs = options.pollIntervalMs ?? 250; + } + + public async drainOnce(): Promise { + const result: StrategyDrainResult = { + completed: 0, + retried: 0, + terminal: 0, + expired: this.spool.expire(), + }; + if (result.expired > 0) { + this.telemetry?.recordStrategyExpired(result.expired); + this.telemetry?.recordStrategySpoolStats(this.spool.stats()); + } + for (const work of this.spool.dueWork()) { + try { + await this.inserter(work.table, work.rows, { + deduplicationToken: work.dedupeToken, + }); + this.spool.complete(work); + this.telemetry?.recordRowsInserted(work.table, work.rows.length); + this.telemetry?.recordSuccessfulFlush(); + this.telemetry?.recordStrategyTableCompletion(work.table); + result.completed += 1; + } catch (error) { + const errorClass = classifyInsertError(error); + this.telemetry?.recordInsertFailure(work.table, error); + if (isPermanentError(errorClass)) { + this.spool.markTerminal(work, errorClass); + this.telemetry?.recordStrategyTerminalFailure( + work.table, + errorClass, + ); + result.terminal += 1; + continue; + } + this.spool.reschedule(work, { + nextAttemptAtMs: + this.now() + retryDelayMs(work.attemptCount, this.random), + errorClass, + }); + this.telemetry?.recordStrategyRetry(work.table, errorClass); + result.retried += 1; + } + } + this.telemetry?.recordStrategySpoolStats(this.spool.stats()); + return result; + } + + public start(): void { + if (!this.stopped) return; + this.stopped = false; + this.schedule(0); + } + + private schedule(delayMs: number): void { + if (this.stopped) return; + this.timer = setTimeout(() => { + void this.tick(); + }, delayMs); + this.timer.unref?.(); + } + + private async tick(): Promise { + if (this.stopped || this.draining) return; + this.draining = true; + try { + await this.drainOnce(); + } catch (error) { + console.error("Strategy archive spool drain failed:", error); + } finally { + this.draining = false; + this.schedule(this.pollIntervalMs); + } + } + + public stop(): void { + this.stopped = true; + if (this.timer) clearTimeout(this.timer); + this.timer = undefined; + } +} diff --git a/services/archive-forwarder/telemetry.ts b/services/archive-forwarder/telemetry.ts index 68226ff..0aaf068 100644 --- a/services/archive-forwarder/telemetry.ts +++ b/services/archive-forwarder/telemetry.ts @@ -7,15 +7,52 @@ import { MALFORMED_TABLE_LABEL, UNSUPPORTED_TABLE_LABEL, } from "./types"; +import type { StrategySpoolStats } from "./strategy-spool"; export const ARCHIVE_FORWARDER_METRICS = { rowsInserted: "archive_forwarder_rows_inserted_total", rowsRejected: "archive_forwarder_rows_rejected_total", insertFailures: "archive_forwarder_insert_failures_total", + checksumConflicts: "archive_forwarder_checksum_conflict_rows_total", lastSuccessfulFlush: "archive_forwarder_last_successful_flush_timestamp_seconds", + strategyBatchesAdmitted: + "archive_forwarder_strategy_batches_admitted_total", + strategyRowsAdmitted: "archive_forwarder_strategy_rows_admitted_total", + strategyAdmissionsRejected: + "archive_forwarder_strategy_admissions_rejected_total", + strategySpoolPendingBatches: + "archive_forwarder_strategy_spool_pending_batches", + strategySpoolPendingWork: "archive_forwarder_strategy_spool_pending_work", + strategySpoolTerminalWork: "archive_forwarder_strategy_spool_terminal_work", + strategySpoolExpiredWork: "archive_forwarder_strategy_spool_expired_work", + strategySpoolBytes: "archive_forwarder_strategy_spool_accounted_bytes", + strategySpoolOldestAge: + "archive_forwarder_strategy_spool_oldest_age_milliseconds", + strategyRetries: "archive_forwarder_strategy_retries_total", + strategyTableCompletions: + "archive_forwarder_strategy_table_completions_total", + strategyTerminalFailures: + "archive_forwarder_strategy_terminal_failures_total", + strategyExpiredWork: "archive_forwarder_strategy_expired_work_total", + lastSuccessfulStrategyDrain: + "archive_forwarder_last_successful_strategy_drain_timestamp_seconds", } as const; +const STRATEGY_REJECTION_REASONS = new Set([ + "invalid_contract", + "quota", + "spool_unavailable", +]); + +const INSERT_ERROR_CLASSES = new Set([ + "timeout", + "connection", + "authentication", + "schema", + "unknown", +]); + export type ArchiveMetricsRecorder = Pick< OtelMetrics, "recordCounter" | "setObservableGauge" @@ -77,6 +114,23 @@ export class ArchiveForwarderTelemetry { ); } + public recordChecksumConflicts( + source: string, + rowsByTable: Readonly>, + ): void { + const boundedSource = + source === "broker_read" || source === "broker_write" ? source : "other"; + for (const [table, count] of Object.entries(rowsByTable)) { + this.bestEffort(() => + this.metrics.recordCounter( + ARCHIVE_FORWARDER_METRICS.checksumConflicts, + count, + { source: boundedSource, feed: "ORDERBOOK", table }, + ), + ); + } + } + public recordSuccessfulFlush(completedAt: Date = new Date()): void { this.bestEffort(() => this.metrics.setObservableGauge( @@ -87,6 +141,107 @@ export class ArchiveForwarderTelemetry { ); } + public recordStrategyAdmission(rowCount: number): void { + this.bestEffort(() => + this.metrics.recordCounter( + ARCHIVE_FORWARDER_METRICS.strategyBatchesAdmitted, + 1, + {}, + ), + ); + this.bestEffort(() => + this.metrics.recordCounter( + ARCHIVE_FORWARDER_METRICS.strategyRowsAdmitted, + rowCount, + {}, + ), + ); + } + + public recordStrategyAdmissionRejected(reason: string): void { + this.bestEffort(() => + this.metrics.recordCounter( + ARCHIVE_FORWARDER_METRICS.strategyAdmissionsRejected, + 1, + { reason: STRATEGY_REJECTION_REASONS.has(reason) ? reason : "other" }, + ), + ); + } + + public recordStrategySpoolStats(stats: StrategySpoolStats): void { + for (const [name, value] of [ + [ARCHIVE_FORWARDER_METRICS.strategySpoolPendingBatches, stats.queuedBatches], + [ARCHIVE_FORWARDER_METRICS.strategySpoolPendingWork, stats.queuedWork], + [ARCHIVE_FORWARDER_METRICS.strategySpoolTerminalWork, stats.terminalWork], + [ARCHIVE_FORWARDER_METRICS.strategySpoolExpiredWork, stats.expiredWork], + [ARCHIVE_FORWARDER_METRICS.strategySpoolBytes, stats.accountedBytes], + [ARCHIVE_FORWARDER_METRICS.strategySpoolOldestAge, stats.oldestAgeMs], + ] as const) { + this.bestEffort(() => this.metrics.setObservableGauge(name, value, {})); + } + } + + public recordStrategyRetry(table: string, errorClass: string): void { + this.bestEffort(() => + this.metrics.recordCounter( + ARCHIVE_FORWARDER_METRICS.strategyRetries, + 1, + { + table: isSupportedTable(table) ? table : UNSUPPORTED_TABLE_LABEL, + error_class: INSERT_ERROR_CLASSES.has(errorClass) + ? errorClass + : "unknown", + }, + ), + ); + } + + public recordStrategyTableCompletion( + table: string, + completedAt: Date = new Date(), + ): void { + this.bestEffort(() => + this.metrics.recordCounter( + ARCHIVE_FORWARDER_METRICS.strategyTableCompletions, + 1, + { table: isSupportedTable(table) ? table : UNSUPPORTED_TABLE_LABEL }, + ), + ); + this.bestEffort(() => + this.metrics.setObservableGauge( + ARCHIVE_FORWARDER_METRICS.lastSuccessfulStrategyDrain, + Math.floor(completedAt.getTime() / 1_000), + {}, + ), + ); + } + + public recordStrategyExpired(count: number): void { + if (count <= 0) return; + this.bestEffort(() => + this.metrics.recordCounter( + ARCHIVE_FORWARDER_METRICS.strategyExpiredWork, + count, + {}, + ), + ); + } + + public recordStrategyTerminalFailure(table: string, errorClass: string): void { + this.bestEffort(() => + this.metrics.recordCounter( + ARCHIVE_FORWARDER_METRICS.strategyTerminalFailures, + 1, + { + table: isSupportedTable(table) ? table : UNSUPPORTED_TABLE_LABEL, + error_class: INSERT_ERROR_CLASSES.has(errorClass) + ? errorClass + : "unknown", + }, + ), + ); + } + private bestEffort(action: () => void | Promise): void { try { const result = action(); diff --git a/services/archive-forwarder/types.ts b/services/archive-forwarder/types.ts index 454af63..bffcd22 100644 --- a/services/archive-forwarder/types.ts +++ b/services/archive-forwarder/types.ts @@ -23,6 +23,9 @@ export const SUPPORTED_TABLES = [ "market_data.cex_stream_events", "market_data.cex_ticker_events", "market_data.cex_trades", + "market_data.cex_ohlcv", + "market_data.cex_order_book_levels", + "market_data.cex_order_book_depth_summary", "broker_execution.order_events", "broker_execution.market_metadata_snapshots", "broker_execution.transfer_events", diff --git a/services/ohlcv-collector/collector.ts b/services/ohlcv-collector/collector.ts index 2287869..cf23b94 100644 --- a/services/ohlcv-collector/collector.ts +++ b/services/ohlcv-collector/collector.ts @@ -3,7 +3,7 @@ import { SubscriptionType } from "../../src/helpers/constants"; import { log } from "../../src/helpers/logger"; import type { OtelMetrics } from "../../src/helpers/otel"; import { CEX_BROKER_PACKAGE_DEFINITION } from "../../src/proto-package-definition"; -import type { OhlcvSubscription } from "./config"; +import type { MarketDataSubscription, OhlcvSubscription } from "./config"; type SubscribeResponse = { data: string; @@ -22,11 +22,22 @@ type CollectorMetrics = Pick; export type OhlcvCollectorOptions = { brokerUrl: string; - subscriptions: OhlcvSubscription[]; + subscriptions: Array; metrics?: CollectorMetrics; retry?: Partial; }; +type CollectorSubscription = + | (OhlcvSubscription & { feed: "OHLCV"; bootstrapLimit?: number }) + | MarketDataSubscription; + +export type CollectorFeedHealth = { + state: "connecting" | "healthy" | "backoff" | "stopped"; + lastFrameAtMs?: number; + reconnectCount: number; + gapCount: number; +}; + type RetryPolicy = { initialDelayMs: number; maxDelayMs: number; @@ -58,11 +69,14 @@ const grpcObject = grpc.loadPackageDefinition( }; }; -function pairLabels(subscription: OhlcvSubscription) { +function pairLabels(subscription: CollectorSubscription) { return { exchange: subscription.exchange, symbol: subscription.symbol, - timeframe: subscription.timeframe, + feed: subscription.feed, + ...(subscription.feed === "OHLCV" + ? { timeframe: subscription.timeframe } + : {}), }; } @@ -101,13 +115,18 @@ function waitForDelay(delayMs: number, signal: AbortSignal): Promise { export class OhlcvCollector { readonly #client: SubscribeClient; - readonly #subscriptions: OhlcvSubscription[]; + readonly #subscriptions: CollectorSubscription[]; readonly #metrics?: CollectorMetrics; readonly #retry: RetryPolicy; + readonly #health = new Map(); #started = false; constructor(options: OhlcvCollectorOptions) { - this.#subscriptions = options.subscriptions; + this.#subscriptions = options.subscriptions.map((subscription) => + "feed" in subscription + ? subscription + : { ...subscription, feed: "OHLCV" as const }, + ); this.#metrics = options.metrics; this.#retry = { ...DEFAULT_RETRY_POLICY, ...options.retry }; this.#client = new grpcObject.cex_broker.cex_service( @@ -116,6 +135,30 @@ export class OhlcvCollector { ); } + getHealthSnapshot(): Readonly> { + return Object.fromEntries( + [...this.#health.entries()].map(([key, value]) => [key, { ...value }]), + ); + } + + #healthKey(subscription: CollectorSubscription): string { + return `${subscription.exchange}:${subscription.symbol}:${subscription.feed}`; + } + + #updateHealth( + subscription: CollectorSubscription, + update: Partial, + ): void { + const key = this.#healthKey(subscription); + this.#health.set(key, { + state: "connecting", + reconnectCount: 0, + gapCount: 0, + ...(this.#health.get(key) ?? {}), + ...update, + }); + } + async run(signal: AbortSignal): Promise { if (this.#started) { throw new Error("OHLCV collector can only be started once"); @@ -128,12 +171,15 @@ export class OhlcvCollector { ), ); } finally { + for (const subscription of this.#subscriptions) { + this.#updateHealth(subscription, { state: "stopped" }); + } this.#client.close(); } } async #keepSupervisorAlive( - subscription: OhlcvSubscription, + subscription: CollectorSubscription, signal: AbortSignal, ): Promise { while (!signal.aborted) { @@ -153,7 +199,7 @@ export class OhlcvCollector { } async #supervise( - subscription: OhlcvSubscription, + subscription: CollectorSubscription, signal: AbortSignal, ): Promise { let consecutiveFailures = 0; @@ -161,15 +207,39 @@ export class OhlcvCollector { const labels = pairLabels(subscription); while (!signal.aborted) { + this.#updateHealth(subscription, { state: "connecting" }); if (subscriptionCount > 0) { + const health = this.#health.get(this.#healthKey(subscription)); + this.#updateHealth(subscription, { + reconnectCount: (health?.reconnectCount ?? 0) + 1, + }); void this.#metrics?.recordCounter( - "cex_ohlcv_collector_reconnects_total", + "cex_market_data_collector_reconnects_total", 1, labels, ); + if (subscription.feed === "OHLCV") { + void this.#metrics?.recordCounter( + "cex_ohlcv_collector_reconnects_total", + 1, + labels, + ); + } else { + const gapCount = (health?.gapCount ?? 0) + 1; + this.#updateHealth(subscription, { gapCount }); + void this.#metrics?.recordCounter( + "cex_market_data_collector_unrecoverable_gaps_total", + 1, + labels, + ); + log.warn("Market-data collector recorded unrecoverable feed gap", { + ...labels, + gap_count: gapCount, + }); + } } subscriptionCount += 1; - log.info("OHLCV collector subscription opened", { + log.info("Market-data collector subscription opened", { ...labels, subscription_attempt: subscriptionCount, }); @@ -185,11 +255,12 @@ export class OhlcvCollector { } consecutiveFailures += 1; const delayMs = this.#retryDelay(consecutiveFailures); + this.#updateHealth(subscription, { state: "backoff" }); const errorFields = result.reason === "error" ? { grpc_code: result.error.code, error: result.error.message } : {}; - log.warn("OHLCV collector stream closed; reconnect scheduled", { + log.warn("Market-data collector stream closed; reconnect scheduled", { ...labels, reason: result.reason, delay_ms: delayMs, @@ -202,7 +273,7 @@ export class OhlcvCollector { } #openStream( - subscription: OhlcvSubscription, + subscription: CollectorSubscription, signal: AbortSignal, ): Promise { return new Promise((resolve) => { @@ -222,11 +293,24 @@ export class OhlcvCollector { }; try { + const options = + subscription.feed === "ORDERBOOK" + ? { depthLimit: String(subscription.depthLimit) } + : subscription.feed === "OHLCV" + ? { + timeframe: subscription.timeframe, + ...(subscription.bootstrapLimit === undefined + ? {} + : { + bootstrapLimit: String(subscription.bootstrapLimit), + }), + } + : {}; stream = this.#client.Subscribe({ cex: subscription.exchange, symbol: subscription.symbol, - type: SubscriptionType.OHLCV, - options: { timeframe: subscription.timeframe }, + type: SubscriptionType[subscription.feed], + options, }); } catch (error) { settle({ reason: "error", error: error as grpc.ServiceError }); @@ -234,18 +318,34 @@ export class OhlcvCollector { } stream.on("data", (response) => { - const bars = countOhlcvBars(response.data); - if (bars === 0) { + const frames = + subscription.feed === "OHLCV" + ? countOhlcvBars(response.data) + : response.data.length > 0 + ? 1 + : 0; + if (frames === 0) { return; } + this.#updateHealth(subscription, { + state: "healthy", + lastFrameAtMs: Date.now(), + }); void this.#metrics?.recordCounter( - "cex_ohlcv_collector_bars_received_total", - bars, + "cex_market_data_collector_frames_received_total", + frames, pairLabels(subscription), ); - log.debug("OHLCV collector bars received", { + if (subscription.feed === "OHLCV") { + void this.#metrics?.recordCounter( + "cex_ohlcv_collector_bars_received_total", + frames, + pairLabels(subscription), + ); + } + log.debug("Market-data collector frames received", { ...pairLabels(subscription), - bars, + frames, }); }); stream.on("error", (error: grpc.ServiceError) => { @@ -270,3 +370,5 @@ export class OhlcvCollector { return Math.max(0, Math.round(baseDelay * jitter)); } } + +export const MarketDataCollector = OhlcvCollector; diff --git a/services/ohlcv-collector/config.ts b/services/ohlcv-collector/config.ts index 0954ce6..8f07ab2 100644 --- a/services/ohlcv-collector/config.ts +++ b/services/ohlcv-collector/config.ts @@ -1,6 +1,138 @@ import { z } from "zod"; export const OHLCV_COLLECTOR_CONFIG_ENV = "CEX_BROKER_OHLCV_COLLECTOR_CONFIG"; +export const MARKET_DATA_COLLECTOR_CONFIG_ENV = + "CEX_BROKER_MARKET_DATA_COLLECTOR_CONFIG"; + +const exchangeSchema = z + .string() + .trim() + .min(1) + .transform((value) => value.toLowerCase()); +const symbolSchema = z.string().trim().min(1); + +const marketSubscriptionSchema = z.discriminatedUnion("feed", [ + z + .object({ + exchange: exchangeSchema, + symbol: symbolSchema, + feed: z.literal("ORDERBOOK"), + depthLimit: z.number().int().min(1).max(500), + }) + .strict(), + z + .object({ + exchange: exchangeSchema, + symbol: symbolSchema, + feed: z.literal("TICKER"), + }) + .strict(), + z + .object({ + exchange: exchangeSchema, + symbol: symbolSchema, + feed: z.literal("TRADES"), + }) + .strict(), + z + .object({ + exchange: exchangeSchema, + symbol: symbolSchema, + feed: z.literal("OHLCV"), + timeframe: z.string().trim().min(1), + bootstrapLimit: z.number().int().min(0).max(1_000).default(100), + }) + .strict(), +]); + +const marketConfigSchema = z + .object({ + captureBundleId: z.string().trim().optional(), + environment: z.enum(["development", "production"]).default("production"), + subscriptions: z.array(marketSubscriptionSchema).min(1), + }) + .strict() + .superRefine((config, context) => { + if (config.environment === "production" && !config.captureBundleId) { + context.addIssue({ + code: "custom", + message: "captureBundleId is required in production", + path: ["captureBundleId"], + }); + } + const seen = new Set(); + for (const [index, subscription] of config.subscriptions.entries()) { + const options = + subscription.feed === "ORDERBOOK" + ? String(subscription.depthLimit) + : subscription.feed === "OHLCV" + ? `${subscription.timeframe}:${subscription.bootstrapLimit}` + : ""; + const key = `${subscription.exchange}\u0000${subscription.symbol}\u0000${subscription.feed}\u0000${options}`; + if (seen.has(key)) { + context.addIssue({ + code: "custom", + message: "duplicate feed subscription", + path: ["subscriptions", index], + }); + } + seen.add(key); + } + }); + +export type MarketDataSubscription = z.infer; +export type MarketDataCollectorConfig = z.infer; + +function formatZodIssues(error: z.ZodError): string { + return error.issues + .map((issue) => { + const path = issue.path.length > 0 ? issue.path.join(".") : "config"; + return `${path}: ${issue.message}`; + }) + .join("; "); +} + +export function parseMarketDataCollectorConfig( + input: unknown, +): MarketDataCollectorConfig { + const result = marketConfigSchema.safeParse(input); + if (!result.success) { + throw new Error( + `Invalid market-data collector config: ${formatZodIssues(result.error)}`, + ); + } + return result.data; +} + +export async function loadMarketDataCollectorConfig( + configPath = process.env[MARKET_DATA_COLLECTOR_CONFIG_ENV], +): Promise { + const path = configPath?.trim(); + if (!path) { + throw new Error( + `${MARKET_DATA_COLLECTOR_CONFIG_ENV} must point to a JSON file`, + ); + } + let contents: string; + try { + contents = await Bun.file(path).text(); + } catch (error) { + throw new Error(`Failed to read market-data collector config at ${path}`, { + cause: error, + }); + } + try { + return parseMarketDataCollectorConfig(JSON.parse(contents)); + } catch (error) { + if (error instanceof SyntaxError) { + throw new Error( + `Market-data collector config at ${path} is not valid JSON`, + { cause: error }, + ); + } + throw error; + } +} const subscriptionSchema = z .object({ @@ -37,13 +169,9 @@ export type OhlcvSubscription = z.infer; export function parseOhlcvCollectorConfig(input: unknown): OhlcvSubscription[] { const result = configSchema.safeParse(input); if (!result.success) { - const details = result.error.issues - .map((issue) => { - const path = issue.path.length > 0 ? issue.path.join(".") : "config"; - return `${path}: ${issue.message}`; - }) - .join("; "); - throw new Error(`Invalid OHLCV collector config: ${details}`); + throw new Error( + `Invalid OHLCV collector config: ${formatZodIssues(result.error)}`, + ); } return result.data; } diff --git a/services/ohlcv-collector/index.ts b/services/ohlcv-collector/index.ts index db4de62..76e22fc 100644 --- a/services/ohlcv-collector/index.ts +++ b/services/ohlcv-collector/index.ts @@ -2,6 +2,7 @@ import * as grpc from "@grpc/grpc-js"; import { SubscribeBrokerLifecycle } from "../../src/handlers/subscribe"; import { createBrokerExecutionArchiverFromEnv } from "../../src/helpers/broker-execution-archive"; import { log } from "../../src/helpers/logger"; +import { validateProductionCollectorArchive } from "../../src/helpers/market-data-archive/capture-context"; import { createOtelLogsFromEnv, createOtelMetricsFromEnv, @@ -9,7 +10,12 @@ import { import { getServer } from "../../src/server"; import type { PolicyConfig } from "../../src/types"; import { OhlcvCollector } from "./collector"; -import { loadOhlcvCollectorConfig } from "./config"; +import { + loadMarketDataCollectorConfig, + loadOhlcvCollectorConfig, + MARKET_DATA_COLLECTOR_CONFIG_ENV, + type MarketDataSubscription, +} from "./config"; const PUBLIC_ONLY_POLICY: PolicyConfig = { withdraw: { rule: [] }, @@ -71,10 +77,34 @@ function bindPublicBroker(server: grpc.Server): Promise { } async function run(): Promise { - const subscriptions = await loadOhlcvCollectorConfig(); + let subscriptions: MarketDataSubscription[]; + let productionCaptureBundleId: string | undefined; + if (process.env[MARKET_DATA_COLLECTOR_CONFIG_ENV]?.trim()) { + const config = await loadMarketDataCollectorConfig(); + subscriptions = config.subscriptions; + process.env.CEX_BROKER_MARKET_CAPTURE_ENVIRONMENT = config.environment; + if (config.captureBundleId) { + process.env.CEX_BROKER_CAPTURE_BUNDLE_ID = config.captureBundleId; + } + if (config.environment === "production") { + productionCaptureBundleId = config.captureBundleId; + } + } else { + subscriptions = (await loadOhlcvCollectorConfig()).map((subscription) => ({ + ...subscription, + feed: "OHLCV" as const, + bootstrapLimit: 100, + })); + } const metrics = createOtelMetricsFromEnv(); const otelLogs = createOtelLogsFromEnv(); const archiver = createBrokerExecutionArchiverFromEnv(otelLogs, metrics); + if (productionCaptureBundleId) { + validateProductionCollectorArchive({ + source: archiver.getSource(), + captureBundleId: productionCaptureBundleId, + }); + } const subscribeBrokerLifecycle = new SubscribeBrokerLifecycle(); const server = getServer( PUBLIC_ONLY_POLICY, @@ -104,6 +134,7 @@ async function run(): Promise { log.info("OHLCV collector service started", { broker_url: brokerUrl, subscriptions: subscriptions.length, + feeds: [...new Set(subscriptions.map(({ feed }) => feed))], bootstrap_limit: process.env.CEX_BROKER_OHLCV_ARCHIVE_BOOTSTRAP_LIMIT ?? "100", }); diff --git a/src/handlers/execute-action/handler.ts b/src/handlers/execute-action/handler.ts index 96cc390..1147df2 100644 --- a/src/handlers/execute-action/handler.ts +++ b/src/handlers/execute-action/handler.ts @@ -2,7 +2,11 @@ import type { Metadata } from "@grpc/grpc-js"; import * as grpc from "@grpc/grpc-js"; import type { Exchange } from "@usherlabs/ccxt"; import { authenticateRequest } from "../../helpers/auth"; -import { type BrokerPoolEntry, createBroker } from "../../helpers/broker"; +import { + type BrokerPoolEntry, + createBroker, + createPublicBroker, +} from "../../helpers/broker"; import { type BrokerExecutionArchiver, WithdrawalObservationTracker, @@ -11,6 +15,7 @@ import { Action, getActionName, resolveAction } from "../../helpers/constants"; import { selectBrokerAccountForCex } from "../../helpers/grpc/broker"; import { log } from "../../helpers/logger"; import type { OrderActivityTracker } from "../../helpers/order-activity-tracker"; +import { isOrderBookCallMethod } from "../../helpers/order-book"; import type { OtelMetrics } from "../../helpers/otel"; import { safeLogError } from "../../helpers/shared/errors"; import { @@ -35,6 +40,14 @@ export type ExecuteActionDeps = { withdrawalObservationTracker?: WithdrawalObservationTracker; }; +function isPublicMarketDataAction( + action: ReturnType, + payload: Record | undefined, +): boolean { + if (action !== Action.Call) return false; + return isOrderBookCallMethod(payload?.method ?? payload?.functionName); +} + export function createExecuteActionHandler(deps: ExecuteActionDeps) { const { policy, @@ -48,7 +61,6 @@ export function createExecuteActionHandler(deps: ExecuteActionDeps) { } = deps; const withdrawalObservationTracker = deps.withdrawalObservationTracker ?? new WithdrawalObservationTracker(); - return async ( call: grpc.ServerUnaryCall, callback: grpc.sendUnaryData, @@ -122,6 +134,21 @@ export function createExecuteActionHandler(deps: ExecuteActionDeps) { brokers, metadata, ); + const broker = + selectedBrokerAccount?.exchange ?? + createBroker(normalizedCex, call.metadata) ?? + (isPublicMarketDataAction(action, call.request.payload) + ? createPublicBroker(normalizedCex) + : null); + if (!broker) { + return wrappedCallback( + { + code: grpc.status.UNAUTHENTICATED, + message: `This Exchange is not registered and No API metadata was found`, + }, + null, + ); + } const verity = { proof: "" }; const applyVerityToBroker = (targetBroker: Exchange) => { @@ -151,9 +178,7 @@ export function createExecuteActionHandler(deps: ExecuteActionDeps) { cex, symbol, selectedBrokerAccount, - broker: - selectedBrokerAccount?.exchange ?? - (createBroker(normalizedCex, metadata) as Exchange), + broker, verity, applyVerityToBroker, useVerity, @@ -169,20 +194,6 @@ export function createExecuteActionHandler(deps: ExecuteActionDeps) { if (handled) return; } - const broker = - selectedBrokerAccount?.exchange ?? - createBroker(normalizedCex, metadata); - - if (!broker) { - return wrappedCallback( - { - code: grpc.status.UNAUTHENTICATED, - message: `This Exchange is not registered and No API metadata was found`, - }, - null, - ); - } - applyVerityToBroker(broker); const ctx: ExecuteActionContext = { ...preludeCtx, broker }; diff --git a/src/handlers/execute-action/order-book-call.ts b/src/handlers/execute-action/order-book-call.ts index 4c47f18..5cf0e0d 100644 --- a/src/handlers/execute-action/order-book-call.ts +++ b/src/handlers/execute-action/order-book-call.ts @@ -1,6 +1,6 @@ import * as grpc from "@grpc/grpc-js"; -import { createBroker, createPublicBroker } from "../../helpers"; import { mapCcxtErrorToGrpcStatus } from "../../helpers/grpc/status"; +import { archiveOrderbookInBackground } from "../../helpers/market-data-archive/capture"; import { buildHistoricalOrderBookUnsupported, buildOrderBookCapability, @@ -36,10 +36,7 @@ export async function handleOrderBookCall( return false; } - const orderBookBroker = - ctx.selectedBrokerAccount?.exchange ?? - createBroker(ctx.normalizedCex, ctx.metadata) ?? - createPublicBroker(ctx.normalizedCex); + const orderBookBroker = ctx.broker; if (!orderBookBroker) { ctx.wrappedCallback( { @@ -103,17 +100,32 @@ export async function handleOrderBookCall( orderBookPayload.symbol, orderBookPayload.depthLimit, ); + const snapshot = normalizeOrderBookSnapshot(rawOrderBook, { + exchange: orderBookPayload.exchange, + symbol: orderBookPayload.symbol, + depthLimit: orderBookPayload.depthLimit, + receivedTimestamp, + }); ctx.wrappedCallback(null, { proof: ctx.verity.proof, - result: JSON.stringify( - normalizeOrderBookSnapshot(rawOrderBook, { - exchange: orderBookPayload.exchange, - symbol: orderBookPayload.symbol, - depthLimit: orderBookPayload.depthLimit, - receivedTimestamp, - }), - ), + result: JSON.stringify(snapshot), }); + archiveOrderbookInBackground( + ctx.brokerArchiver, + ctx.otelMetrics, + { + exchange: orderBookPayload.exchange, + symbol: orderBookPayload.symbol, + assetType: "spot", + accountSelector: ctx.selectedBrokerAccount?.label, + deploymentId: ctx.brokerArchiver?.getDeploymentId() ?? "unarchived", + snapshot, + }, + { + sourceMode: "broker_current_snapshot_v1", + depthLimit: orderBookPayload.depthLimit, + }, + ); } catch (error: unknown) { safeLogError("Order-book Call failed", error); ctx.wrappedCallback( diff --git a/src/handlers/subscribe/handler.ts b/src/handlers/subscribe/handler.ts index 14e3912..f464b45 100644 --- a/src/handlers/subscribe/handler.ts +++ b/src/handlers/subscribe/handler.ts @@ -311,7 +311,6 @@ export function createSubscribeHandler(deps: SubscribeDeps) { const { brokers, whitelistIps, otelMetrics, brokerArchiver } = deps; const brokerLifecycle = deps.brokerLifecycle ?? new SubscribeBrokerLifecycle(); - return async (call: SubscribeCall) => { const subscribeStartTime = Date.now(); let streamClosed = false; diff --git a/src/helpers/broker-execution-archive/index.ts b/src/helpers/broker-execution-archive/index.ts index 42deedb..4b99a19 100644 --- a/src/helpers/broker-execution-archive/index.ts +++ b/src/helpers/broker-execution-archive/index.ts @@ -33,9 +33,11 @@ export { ACCOUNT_BALANCE_PRECISION_BASIS, ACCOUNT_BALANCE_SCOPE, ARCHIVE_SCHEMA_VERSION, + BROKER_READ_SOURCE, BROKER_WRITE_SOURCE, type BrokerArchiveCommonTags, type BrokerArchiveRow, + type BrokerArchiveSource, type BrokerArchiveTable, type OrderArchiveAction, type SubscribeArchiveType, diff --git a/src/helpers/broker-execution-archive/rows.ts b/src/helpers/broker-execution-archive/rows.ts index 9f4dd47..3d6ea25 100644 --- a/src/helpers/broker-execution-archive/rows.ts +++ b/src/helpers/broker-execution-archive/rows.ts @@ -9,6 +9,7 @@ import { BROKER_WRITE_SOURCE, type BrokerArchiveCommonTags, type BrokerArchiveRow, + type BrokerArchiveSource, FILL_EVENT_KIND, type OrderArchiveAction, type SubscribeArchiveType, @@ -252,6 +253,7 @@ function compactUndefined( } export function buildCommonArchiveTags(input: { + source?: BrokerArchiveSource; deploymentId: string; accountSelector?: string; exchange: string; @@ -259,7 +261,7 @@ export function buildCommonArchiveTags(input: { brokerObservedTimestamp?: string; }): BrokerArchiveCommonTags { return { - source: BROKER_WRITE_SOURCE, + source: input.source ?? BROKER_WRITE_SOURCE, deployment_id: input.deploymentId, account_selector: input.accountSelector ?? "unknown", exchange: input.exchange.trim().toLowerCase() || "unknown", diff --git a/src/helpers/broker-execution-archive/types.ts b/src/helpers/broker-execution-archive/types.ts index 54f09aa..f59e895 100644 --- a/src/helpers/broker-execution-archive/types.ts +++ b/src/helpers/broker-execution-archive/types.ts @@ -1,4 +1,8 @@ +export const BROKER_READ_SOURCE = "broker_read" as const; export const BROKER_WRITE_SOURCE = "broker_write" as const; +export type BrokerArchiveSource = + | typeof BROKER_READ_SOURCE + | typeof BROKER_WRITE_SOURCE; // Bumped only on a breaking broker archive column-shape change. Stamped onto // transfer, fill, and account-balance rows so a reader can identify their layout. @@ -14,7 +18,10 @@ export type BrokerArchiveTable = | "market_data.candles" | "market_data.cex_stream_events" | "market_data.cex_ticker_events" - | "market_data.cex_trades"; + | "market_data.cex_trades" + | "market_data.cex_ohlcv" + | "market_data.cex_order_book_levels" + | "market_data.cex_order_book_depth_summary"; export type BrokerArchiveRow = { table: BrokerArchiveTable; @@ -22,7 +29,7 @@ export type BrokerArchiveRow = { }; export type BrokerArchiveCommonTags = { - source: typeof BROKER_WRITE_SOURCE; + source: BrokerArchiveSource; deployment_id: string; account_selector: string; exchange: string; diff --git a/src/helpers/broker-execution-archive/writer.ts b/src/helpers/broker-execution-archive/writer.ts index a6c19b8..5c77fc0 100644 --- a/src/helpers/broker-execution-archive/writer.ts +++ b/src/helpers/broker-execution-archive/writer.ts @@ -5,7 +5,12 @@ import { SeverityNumber } from "@opentelemetry/api-logs"; import { log } from "../logger"; import type { OtelLogs, OtelMetrics } from "../otel"; import { REDACTED_ERROR_MESSAGE } from "../shared/errors"; -import type { BrokerArchiveRow, BrokerArchiveTable } from "./types"; +import { + BROKER_WRITE_SOURCE, + type BrokerArchiveRow, + type BrokerArchiveSource, + type BrokerArchiveTable, +} from "./types"; const BROKER_EXECUTION_ARCHIVE_TABLES = new Set([ "broker_execution.order_events", @@ -21,6 +26,7 @@ export function isBrokerExecutionArchiveTable( } export type BrokerExecutionArchiverOptions = { + source?: BrokerArchiveSource; deploymentId?: string; otelLogs?: OtelLogs; otelMetrics?: OtelMetrics; @@ -62,11 +68,36 @@ type ArchiveLossReason = "queue_shed" | "shutdown_forwarder_failure"; type ArchiveLossRecord = { timestamp: string; + source: BrokerArchiveSource; deployment_id: string; reason: ArchiveLossReason; payload: BrokerArchiveRow; }; +const MARKET_FEEDS = new Set(["ORDERBOOK", "TICKER", "TRADES", "OHLCV"]); + +function archiveFeed(row: BrokerArchiveRow): string { + const declared = row.row.feed ?? row.row.stream_type; + if (typeof declared === "string" && MARKET_FEEDS.has(declared)) { + return declared; + } + if ( + row.table === "market_data.orderbook_snapshots" || + row.table.startsWith("market_data.cex_order_book_") + ) { + return "ORDERBOOK"; + } + if ( + row.table === "market_data.candles" || + row.table === "market_data.cex_ohlcv" + ) { + return "OHLCV"; + } + if (row.table === "market_data.cex_ticker_events") return "TICKER"; + if (row.table === "market_data.cex_trades") return "TRADES"; + return "NON_MARKET"; +} + export function isArchiveOtelLogsEnabled(): boolean { return process.env.CEX_BROKER_ARCHIVE_OTEL_LOGS_ENABLED === "true"; } @@ -75,7 +106,20 @@ export function resolveArchiveForwarderUrlFromEnv(): string | undefined { return process.env.CEX_BROKER_ARCHIVE_FORWARDER_URL?.trim() || undefined; } +export function resolveArchiveSourceFromEnv( + value = process.env.CEX_BROKER_ARCHIVE_SOURCE, +): BrokerArchiveSource { + const source = value?.trim() || BROKER_WRITE_SOURCE; + if (source !== "broker_read" && source !== "broker_write") { + throw new Error( + "CEX_BROKER_ARCHIVE_SOURCE must be broker_read or broker_write", + ); + } + return source; +} + export class BrokerExecutionArchiver { + private readonly source: BrokerArchiveSource; private readonly deploymentId: string; private readonly otelLogs?: OtelLogs; private readonly otelMetrics?: OtelMetrics; @@ -96,12 +140,14 @@ export class BrokerExecutionArchiver { private flushTimer: ReturnType | null = null; private flushInFlight: Promise | null = null; private lastShedWarnAtMs = 0; + private closing = false; private closed = false; private readonly enabled: boolean; private readonly forwarderAuthToken?: string; private constructor(options: { enabled: boolean; + source?: BrokerArchiveSource; deploymentId?: string; otelLogs?: OtelLogs; otelMetrics?: OtelMetrics; @@ -112,6 +158,7 @@ export class BrokerExecutionArchiver { flushIntervalMs?: number; forwarderTimeoutMs?: number; }) { + this.source = options.source ?? BROKER_WRITE_SOURCE; this.deploymentId = options.deploymentId?.trim() || process.env.CEX_BROKER_DEPLOYMENT_ID?.trim() || @@ -157,6 +204,7 @@ export class BrokerExecutionArchiver { this.flushTimer.unref?.(); log.info("Broker execution archive enabled", { enabled: true, + source: this.source, otel_mirror_enabled: Boolean(this.otelLogs?.isOtelEnabled()), }); } catch (error) { @@ -179,8 +227,12 @@ export class BrokerExecutionArchiver { return this.deploymentId; } + getSource(): BrokerArchiveSource { + return this.source; + } + isEnabled(): boolean { - return this.enabled && !this.closed; + return this.enabled && !this.closing && !this.closed; } canPersistMarketMetadataSnapshot(): boolean { @@ -195,6 +247,13 @@ export class BrokerExecutionArchiver { if (!this.enabled || this.closed) { return; } + // Archive role is deployment identity. Stamp it at the durability boundary + // so no request metadata or legacy row-builder default can contradict the + // immutable envelope source. + const archiveRow: BrokerArchiveRow = { + table: row.table, + row: { ...row.row, source: this.source }, + }; if (this.queue.length >= this.maxQueueSize) { const shedRow = this.queue[0]; if (shedRow) { @@ -204,6 +263,13 @@ export class BrokerExecutionArchiver { this.stats.shed += 1; void this.recordArchiveMetric("cex_archive_rows_shed_total", { table: shedRow?.table ?? "unknown", + source: this.source, + feed: shedRow ? archiveFeed(shedRow) : "NON_MARKET", + }); + void this.recordArchiveMetric("cex_archive_queue_saturated_rows_total", { + table: shedRow?.table ?? "unknown", + source: this.source, + feed: shedRow ? archiveFeed(shedRow) : "NON_MARKET", }); // The durable journal is authoritative; this rate-limited warning makes // sustained queue pressure visible without becoming another loss sink. @@ -217,10 +283,12 @@ export class BrokerExecutionArchiver { this.lastShedWarnAtMs = now; } } - this.queue.push(row); + this.queue.push(archiveRow); this.stats.enqueued += 1; void this.recordArchiveMetric("cex_archive_rows_enqueued_total", { - table: row.table, + table: archiveRow.table, + source: this.source, + feed: archiveFeed(archiveRow), }); if (this.queue.length >= this.batchSize) { void this.flush(); @@ -244,12 +312,21 @@ export class BrokerExecutionArchiver { .then(() => undefined) .finally(() => { this.flushInFlight = null; + if ( + !this.closed && + !this.closing && + this.enabled && + this.queue.length >= this.batchSize + ) { + queueMicrotask(() => void this.flush()); + } }); this.flushInFlight = inFlight; return inFlight; } async close(): Promise { + this.closing = true; if (this.flushTimer) { clearInterval(this.flushTimer); this.flushTimer = null; @@ -325,6 +402,13 @@ export class BrokerExecutionArchiver { this.stats.shed += 1; void this.recordArchiveMetric("cex_archive_rows_shed_total", { table: dropped?.table ?? "unknown", + source: this.source, + feed: archiveFeed(dropped), + }); + void this.recordArchiveMetric("cex_archive_queue_saturated_rows_total", { + table: dropped.table, + source: this.source, + feed: archiveFeed(dropped), }); } } @@ -344,6 +428,7 @@ export class BrokerExecutionArchiver { const timestamp = new Date().toISOString(); const records: ArchiveLossRecord[] = rows.map((payload) => ({ timestamp, + source: this.source, deployment_id: this.deploymentId, reason, payload, @@ -357,6 +442,28 @@ export class BrokerExecutionArchiver { throw new Error(`wrote ${written} of ${bytes.length} bytes`); } fsyncSync(this.deadLetterFd); + const byFeedAndTable = new Map< + string, + { row: BrokerArchiveRow; count: number } + >(); + for (const row of rows) { + const key = `${row.table}\u0000${archiveFeed(row)}`; + const grouped = byFeedAndTable.get(key) ?? { row, count: 0 }; + grouped.count += 1; + byFeedAndTable.set(key, grouped); + } + for (const { row, count } of byFeedAndTable.values()) { + void this.recordArchiveMetric( + "cex_archive_rows_journaled_total", + { + table: row.table, + source: this.source, + feed: archiveFeed(row), + reason, + }, + count, + ); + } } catch (error) { throw new BrokerExecutionArchiveDurabilityError( `Broker execution archive failed to durably record ${reason}; affected row(s) were retained`, @@ -415,14 +522,24 @@ export class BrokerExecutionArchiver { // last-flush-success gauge (unix seconds) whose staleness is the "archive plane // stuck" signal. Fire-and-forget so metrics never gate flushing. private recordFlushHealth(batch: BrokerArchiveRow[]): void { - const countByTable = new Map(); + const countByTable = new Map< + string, + { row: BrokerArchiveRow; count: number } + >(); for (const entry of batch) { - countByTable.set(entry.table, (countByTable.get(entry.table) ?? 0) + 1); + const key = `${entry.table}\u0000${archiveFeed(entry)}`; + const grouped = countByTable.get(key) ?? { row: entry, count: 0 }; + grouped.count += 1; + countByTable.set(key, grouped); } - for (const [table, count] of countByTable) { + for (const { row, count } of countByTable.values()) { void this.recordArchiveMetric( "cex_archive_rows_flushed_total", - { table }, + { + table: row.table, + source: this.source, + feed: archiveFeed(row), + }, count, ); } @@ -483,7 +600,7 @@ export class BrokerExecutionArchiver { return Promise.resolve(); } const body = JSON.stringify({ - source: "broker_write", + source: this.source, deployment_id: this.deploymentId, rows: batch, }); @@ -580,6 +697,7 @@ export function createBrokerExecutionArchiverFromEnv( const forwarderUrl = resolveArchiveForwarderUrlFromEnv(); const archiveOtelLogs = isArchiveOtelLogsEnabled() ? otelLogs : undefined; return BrokerExecutionArchiver.create({ + source: resolveArchiveSourceFromEnv(), otelLogs: archiveOtelLogs, otelMetrics, forwarderUrl: forwarderUrl ?? "", diff --git a/src/helpers/market-data-archive/canonical-orderbook.ts b/src/helpers/market-data-archive/canonical-orderbook.ts new file mode 100644 index 0000000..0fcb265 --- /dev/null +++ b/src/helpers/market-data-archive/canonical-orderbook.ts @@ -0,0 +1,270 @@ +import type { BrokerArchiveRow } from "../broker-execution-archive/types"; +import type { + NormalizedOrderBookSnapshot, + OrderBookConstructionMode, +} from "../order-book"; +import { + captureCoreFields, + normalizeTimestampMs, + sha256Canonical, +} from "./capture-contract"; +import type { MarketCaptureContext, RawCapture } from "./types"; + +export class OrderBookValidationError extends Error { + readonly reason: string; + + constructor(reason: string) { + super(`Invalid order-book evidence: ${reason}`); + this.name = "OrderBookValidationError"; + this.reason = reason; + } +} + +type ValidLevel = { price: number; amount: number }; + +export type CanonicalOrderBookRows = { + snapshotId: string; + levels: BrokerArchiveRow[]; + summary: BrokerArchiveRow; +}; + +function parseSequence( + value: NormalizedOrderBookSnapshot["sequence"], +): number | undefined { + if (value === undefined) return undefined; + const parsed = + typeof value === "number" + ? value + : typeof value === "string" && /^\d+$/.test(value) + ? Number(value) + : Number.NaN; + if (!Number.isSafeInteger(parsed) || parsed < 0) { + throw new OrderBookValidationError( + "sequence must be a non-negative integer", + ); + } + return parsed; +} + +function validateSide( + side: "bid" | "ask", + levels: number[][], + depthLimit: number, +): ValidLevel[] { + if (levels.length === 0) { + throw new OrderBookValidationError(`${side} side is missing`); + } + const retained = levels.slice(0, depthLimit); + for (let index = 0; index < retained.length; index += 1) { + const entry = retained[index]; + const price = entry?.[0]; + const amount = entry?.[1]; + if ( + !Array.isArray(entry) || + entry.length < 2 || + price === undefined || + amount === undefined || + !Number.isFinite(price) || + !Number.isFinite(amount) || + price <= 0 || + amount <= 0 + ) { + throw new OrderBookValidationError( + `${side} level ${index} has a non-positive or non-finite price/amount`, + ); + } + if (index > 0) { + const previous = retained[index - 1]?.[0]; + if ( + previous === undefined || + (side === "bid" ? price >= previous : price <= previous) + ) { + throw new OrderBookValidationError( + `${side} levels are not strictly ${side === "bid" ? "descending" : "ascending"}`, + ); + } + } + } + return retained.map(([price, amount]) => ({ + price: price as number, + amount: amount as number, + })); +} + +function normalizedBands(input: readonly number[] | undefined): number[] { + const bands = input ?? [10, 25, 50, 100]; + for (const band of bands) { + if (!Number.isFinite(band) || band <= 0 || !Number.isInteger(band)) { + throw new OrderBookValidationError( + "measurement bands must be positive integer basis points", + ); + } + } + return [...new Set(bands)].sort((left, right) => left - right); +} + +function checksumRow(row: Record): Record { + return { ...row, normalized_row_checksum: sha256Canonical(row) }; +} + +function commonEvidenceFields( + context: MarketCaptureContext, + rawCapture: RawCapture, + input: { + snapshotId: string; + sequence?: number; + depthLimit: number; + eventTimeMs: number; + receivedTimeMs: number; + }, +): Record { + return { + ...captureCoreFields(context, rawCapture), + source_time_ms: input.eventTimeMs, + received_time_ms: input.receivedTimeMs, + snapshot_id: input.snapshotId, + construction_mode: "sampled_top_n_snapshot", + gap_policy: "record_gap", + depth_limit: input.depthLimit, + sequence: input.sequence, + exact_l2_reconstruction_complete: 0, + }; +} + +export function buildCanonicalOrderBookRows(input: { + context: MarketCaptureContext; + snapshot: NormalizedOrderBookSnapshot; + rawCapture: RawCapture; + depthLimit: number; + measurementBandsBps?: readonly number[]; + constructionMode?: OrderBookConstructionMode; +}): CanonicalOrderBookRows { + if ( + !Number.isSafeInteger(input.depthLimit) || + input.depthLimit <= 0 || + input.depthLimit > 500 + ) { + throw new OrderBookValidationError( + "depth limit must be an integer between 1 and 500", + ); + } + if (input.context.feed !== "ORDERBOOK") { + throw new OrderBookValidationError("capture context feed is not ORDERBOOK"); + } + if (input.constructionMode === "exact_l2_reconstruction") { + throw new OrderBookValidationError( + "exact L2 requires a complete continuity proof and is unsupported", + ); + } + + let eventTimeMs: number; + let receivedTimeMs: number; + try { + eventTimeMs = normalizeTimestampMs( + input.snapshot.timestamp, + "source_time_ms", + ); + receivedTimeMs = normalizeTimestampMs( + input.snapshot.receivedTimestamp, + "received_time_ms", + ); + } catch (error) { + throw new OrderBookValidationError( + error instanceof Error ? error.message : "invalid timestamp", + ); + } + if (receivedTimeMs < eventTimeMs) { + throw new OrderBookValidationError( + "received timestamp precedes source timestamp", + ); + } + + const bids = validateSide("bid", input.snapshot.bids, input.depthLimit); + const asks = validateSide("ask", input.snapshot.asks, input.depthLimit); + const bestBid = bids[0] as ValidLevel; + const bestAsk = asks[0] as ValidLevel; + if (bestBid.price >= bestAsk.price) { + throw new OrderBookValidationError("book is crossed or locked"); + } + const sequence = parseSequence(input.snapshot.sequence); + const midPrice = (bestBid.price + bestAsk.price) / 2; + const spread = bestAsk.price - bestBid.price; + const spreadBps = (spread / midPrice) * 10_000; + const snapshotId = sha256Canonical({ + exchange: input.context.exchange.trim().toLowerCase(), + trading_pair: input.context.symbol.trim().replace("/", "-"), + source_time_ms: eventTimeMs, + sequence, + depth_limit: input.depthLimit, + bids, + asks, + schema_version: input.context.schemaVersion, + }); + const common = commonEvidenceFields(input.context, input.rawCapture, { + snapshotId, + sequence, + depthLimit: input.depthLimit, + eventTimeMs, + receivedTimeMs, + }); + + const levels: BrokerArchiveRow[] = ( + [ + ["bid", bids], + ["ask", asks], + ] as const + ).flatMap(([side, sideLevels]) => + sideLevels.map(({ price, amount }, levelIndex) => { + const row = checksumRow({ + ...common, + side, + level_index: levelIndex, + price, + amount, + notional: price * amount, + mid_price: midPrice, + spread_from_mid_bps: Math.abs((price - midPrice) / midPrice) * 10_000, + }); + return { table: "market_data.cex_order_book_levels", row }; + }), + ); + + const bands = normalizedBands(input.measurementBandsBps); + const bidDepth = bands.map((band) => { + const minimumPrice = bestBid.price * (1 - band / 10_000); + return bids + .filter(({ price }) => price >= minimumPrice) + .reduce((sum, { amount }) => sum + amount, 0); + }); + const askDepth = bands.map((band) => { + const maximumPrice = bestAsk.price * (1 + band / 10_000); + return asks + .filter(({ price }) => price <= maximumPrice) + .reduce((sum, { amount }) => sum + amount, 0); + }); + const summaryRow = checksumRow({ + ...common, + best_bid: bestBid.price, + best_ask: bestAsk.price, + best_bid_amount: bestBid.amount, + best_ask_amount: bestAsk.amount, + mid_price: midPrice, + spread, + spread_bps: spreadBps, + staleness_ms: receivedTimeMs - eventTimeMs, + bid_level_count: bids.length, + ask_level_count: asks.length, + measurement_bands_bps: bands, + bid_depth_by_band: bidDepth, + ask_depth_by_band: askDepth, + }); + + return { + snapshotId, + levels, + summary: { + table: "market_data.cex_order_book_depth_summary", + row: summaryRow, + }, + }; +} diff --git a/src/helpers/market-data-archive/capture-context.ts b/src/helpers/market-data-archive/capture-context.ts new file mode 100644 index 0000000..8a6dd69 --- /dev/null +++ b/src/helpers/market-data-archive/capture-context.ts @@ -0,0 +1,113 @@ +import type { BrokerArchiveSource } from "../broker-execution-archive/types"; +import type { BrokerMarketType } from "../market-type"; +import { + CHECKSUM_ALGORITHM, + MARKET_CAPTURE_SCHEMA_VERSION, +} from "./capture-contract"; +import type { + CaptureFeed, + CaptureSourceMode, + MarketCaptureContext, +} from "./types"; + +export type CaptureEnvironment = "development" | "production"; + +export function createMarketCaptureContext(input: { + source: BrokerArchiveSource; + deploymentId: string; + captureBundleId?: string; + exchange: string; + symbol: string; + assetType: BrokerMarketType; + feed: CaptureFeed; + provider?: string; + sourceMode: CaptureSourceMode; + timeframe?: string; + accountSelector?: string; + environment?: CaptureEnvironment; +}): MarketCaptureContext { + const environment = input.environment ?? "development"; + const deploymentId = input.deploymentId.trim(); + if (!deploymentId) throw new Error("deployment_id must not be empty"); + const configuredBundle = input.captureBundleId?.trim(); + if (environment === "production" && !configuredBundle) { + throw new Error( + "capture_bundle_id is required for production market capture", + ); + } + const exchange = input.exchange.trim().toLowerCase(); + const symbol = input.symbol.trim(); + if (!exchange || !symbol) { + throw new Error("exchange and symbol are required for market capture"); + } + return { + source: input.source, + deploymentId, + captureBundleId: configuredBundle ?? `development:${deploymentId}`, + exchange, + symbol, + assetType: input.assetType, + feed: input.feed, + provider: input.provider?.trim() || `ccxt:${exchange}`, + sourceMode: input.sourceMode, + schemaVersion: MARKET_CAPTURE_SCHEMA_VERSION, + checksumAlgorithm: CHECKSUM_ALGORITHM, + provenanceComplete: true, + timeframe: input.timeframe, + accountSelector: input.accountSelector, + }; +} + +export function validateProductionCollectorArchive(input: { + source: BrokerArchiveSource; + captureBundleId?: string; +}): void { + if (input.source !== "broker_read") { + throw new Error( + "FIET-901 production collector requires CEX_BROKER_ARCHIVE_SOURCE=broker_read", + ); + } + if (!input.captureBundleId?.trim()) { + throw new Error( + "FIET-901 production collector requires a capture_bundle_id", + ); + } +} + +export function validateExternalFallbackContext(input: { + configuredExchange: string; + configuredSymbol: string; + rowExchange: string; + rowSymbol: string; + provider: string; + sourceMode: "external_ccxt_fallback_v1" | "external_hummingbot_fallback_v1"; + fallbackReason: string; +}): void { + if ( + input.configuredExchange.trim().toLowerCase() !== + input.rowExchange.trim().toLowerCase() + ) { + throw new Error("External fallback rejected: cross-venue substitution"); + } + if (input.configuredSymbol.trim() !== input.rowSymbol.trim()) { + throw new Error("External fallback rejected: cross-pair substitution"); + } + if (!input.provider.trim()) { + throw new Error("External fallback provider is required"); + } + if (!input.fallbackReason.trim()) { + throw new Error("External fallback reason is required"); + } +} + +export function captureEnvironmentFromEnv( + value = process.env.CEX_BROKER_MARKET_CAPTURE_ENVIRONMENT, +): CaptureEnvironment { + const environment = value?.trim() || "development"; + if (environment !== "development" && environment !== "production") { + throw new Error( + "CEX_BROKER_MARKET_CAPTURE_ENVIRONMENT must be development or production", + ); + } + return environment; +} diff --git a/src/helpers/market-data-archive/capture-contract.ts b/src/helpers/market-data-archive/capture-contract.ts new file mode 100644 index 0000000..5d74f0d --- /dev/null +++ b/src/helpers/market-data-archive/capture-contract.ts @@ -0,0 +1,260 @@ +import { createHash } from "node:crypto"; +import { redactStreamPayload } from "../broker-execution-archive/redact"; +import type { BrokerArchiveSource } from "../broker-execution-archive/types"; +import type { + MarketCaptureContext, + RawCapture, + RawCaptureScope, +} from "./types"; + +export const MARKET_CAPTURE_SCHEMA_VERSION = "1.0.0" as const; +export const CHECKSUM_ALGORITHM = "sha256-canonical-json-v1" as const; + +export const ARCHIVE_SOURCES = ["broker_read", "broker_write"] as const; +export const CAPTURE_FEEDS = [ + "ORDERBOOK", + "TICKER", + "TRADES", + "OHLCV", +] as const; +export const SOURCE_MODES = [ + "broker_live_stream_v1", + "broker_live_sampling_v1", + "broker_current_snapshot_v1", + "broker_bootstrap_fetch_v1", + "external_ccxt_fallback_v1", + "external_hummingbot_fallback_v1", + "legacy_migration_v1", +] as const; +export const CONSTRUCTION_MODES = [ + "sampled_top_n_snapshot", + "exact_l2_reconstruction", +] as const; +export const GAP_POLICIES = [ + "record_gap", + "ohlcv_catch_up", + "fail_fast", +] as const; +export const RAW_CAPTURE_SCOPES = [ + "ccxt_normalized_object", + "broker_visible_payload", + "exchange_wire_frame", +] as const; + +export type CaptureFeed = (typeof CAPTURE_FEEDS)[number]; +export type SourceMode = (typeof SOURCE_MODES)[number]; +export type ConstructionMode = (typeof CONSTRUCTION_MODES)[number]; +export type GapPolicy = (typeof GAP_POLICIES)[number]; + +const CHECKSUM_FIELDS = new Set([ + "normalized_row_checksum", + "raw_checksum", + "checksum", +]); + +/** + * Renders a finite JavaScript number as a plain decimal. Exponent expansion and + * negative-zero normalization are part of checksum algorithm v1 and are mirrored + * by the Python fixture verifier. + */ +export function canonicalDecimal(value: number): string { + if (!Number.isFinite(value)) { + throw new Error("Canonical numbers must be finite"); + } + if (Object.is(value, -0)) { + return "0"; + } + const rendered = String(value).toLowerCase(); + if (!rendered.includes("e")) { + return rendered; + } + const [coefficient = "0", exponentText = "0"] = rendered.split("e"); + const exponent = Number.parseInt(exponentText, 10); + const negative = coefficient.startsWith("-"); + const unsigned = negative ? coefficient.slice(1) : coefficient; + const [integer = "0", fraction = ""] = unsigned.split("."); + const digits = `${integer}${fraction}`; + const decimalIndex = integer.length + exponent; + let result: string; + if (decimalIndex <= 0) { + result = `0.${"0".repeat(-decimalIndex)}${digits}`; + } else if (decimalIndex >= digits.length) { + result = `${digits}${"0".repeat(decimalIndex - digits.length)}`; + } else { + result = `${digits.slice(0, decimalIndex)}.${digits.slice(decimalIndex)}`; + } + return negative ? `-${result}` : result; +} + +function serializeCanonical(value: unknown, stack: Set): string { + if (value === null) return "null"; + if (typeof value === "string") return JSON.stringify(value); + if (typeof value === "boolean") return value ? "true" : "false"; + if (typeof value === "number") return canonicalDecimal(value); + if (typeof value === "bigint") return value.toString(10); + if (value instanceof Date) { + if (Number.isNaN(value.getTime())) { + throw new Error("Canonical timestamps must be valid"); + } + return value.getTime().toString(10); + } + if (Array.isArray(value)) { + if (stack.has(value)) throw new Error("Canonical values must be acyclic"); + stack.add(value); + const result = `[${value + .map((entry) => + entry === undefined ? "null" : serializeCanonical(entry, stack), + ) + .join(",")}]`; + stack.delete(value); + return result; + } + if (typeof value === "object") { + if (stack.has(value)) throw new Error("Canonical values must be acyclic"); + stack.add(value); + const entries = Object.entries(value as Record) + .filter(([, entry]) => entry !== undefined) + .sort(([left], [right]) => left.localeCompare(right)); + const result = `{${entries + .map( + ([key, entry]) => + `${JSON.stringify(key)}:${serializeCanonical(entry, stack)}`, + ) + .join(",")}}`; + stack.delete(value); + return result; + } + throw new Error(`Unsupported canonical value type: ${typeof value}`); +} + +export function canonicalSerialize(value: unknown): string { + return serializeCanonical(value, new Set()); +} + +function omitChecksumFields(value: unknown): unknown { + if (Array.isArray(value)) return value.map(omitChecksumFields); + if (value && typeof value === "object" && !(value instanceof Date)) { + return Object.fromEntries( + Object.entries(value as Record) + .filter(([key]) => !CHECKSUM_FIELDS.has(key)) + .map(([key, entry]) => [key, omitChecksumFields(entry)]), + ); + } + return value; +} + +export function sha256Canonical(value: unknown): string { + return createHash("sha256") + .update(canonicalSerialize(omitChecksumFields(value))) + .digest("hex"); +} + +export function normalizeTimestampMs(value: unknown, field: string): number { + let timestamp: number; + if (value instanceof Date) { + timestamp = value.getTime(); + } else if (typeof value === "number") { + timestamp = value; + } else if (typeof value === "string" && /^\d+$/.test(value.trim())) { + timestamp = Number(value.trim()); + } else if (typeof value === "string") { + timestamp = Date.parse(value); + } else { + timestamp = Number.NaN; + } + if (!Number.isSafeInteger(timestamp) || timestamp < 0) { + throw new Error(`${field} must be a non-negative millisecond timestamp`); + } + return timestamp; +} + +function assertCaptureContext(context: MarketCaptureContext): void { + if (!(ARCHIVE_SOURCES as readonly string[]).includes(context.source)) { + throw new Error(`Unsupported archive source: ${context.source}`); + } + if (!(CAPTURE_FEEDS as readonly string[]).includes(context.feed)) { + throw new Error(`Unsupported capture feed: ${context.feed}`); + } + if (!(SOURCE_MODES as readonly string[]).includes(context.sourceMode)) { + throw new Error(`Unsupported source mode: ${context.sourceMode}`); + } + for (const [field, value] of [ + ["deployment_id", context.deploymentId], + ["capture_bundle_id", context.captureBundleId], + ["exchange", context.exchange], + ["symbol", context.symbol], + ["provider", context.provider], + ] as const) { + if (!value.trim()) throw new Error(`${field} must not be empty`); + } +} + +export function createRawCapture( + context: MarketCaptureContext, + input: { + payload: unknown; + eventTimeMs: unknown; + receivedTimeMs: unknown; + scope: RawCaptureScope; + }, +): RawCapture { + assertCaptureContext(context); + if (!(RAW_CAPTURE_SCOPES as readonly string[]).includes(input.scope)) { + throw new Error(`Unsupported raw capture scope: ${input.scope}`); + } + const eventTimeMs = normalizeTimestampMs(input.eventTimeMs, "event_time_ms"); + const receivedTimeMs = normalizeTimestampMs( + input.receivedTimeMs, + "received_time_ms", + ); + const redactedPayload = redactStreamPayload(input.payload); + const rawChecksum = sha256Canonical(redactedPayload); + const rawCaptureId = sha256Canonical({ + capture_bundle_id: context.captureBundleId, + exchange: context.exchange.trim().toLowerCase(), + feed: context.feed, + raw_capture_scope: input.scope, + raw_payload_sha256: rawChecksum, + schema_version: context.schemaVersion, + source_mode: context.sourceMode, + source_symbol: context.symbol.trim(), + source_time_ms: eventTimeMs, + }); + return { + rawCaptureId, + rawCaptureScope: input.scope, + rawChecksum, + redactedPayload, + eventTimeMs, + receivedTimeMs, + checksumAlgorithm: context.checksumAlgorithm, + }; +} + +export function captureCoreFields( + context: MarketCaptureContext, + rawCapture: RawCapture, +): Record { + assertCaptureContext(context); + return { + source: context.source satisfies BrokerArchiveSource, + deployment_id: context.deploymentId, + capture_bundle_id: context.captureBundleId, + exchange: context.exchange.trim().toLowerCase(), + symbol: context.symbol.trim(), + trading_pair: context.symbol.trim().replace("/", "-"), + source_symbol: context.symbol.trim(), + asset_type: context.assetType, + feed: context.feed, + provider: context.provider, + source_mode: context.sourceMode, + source_time_ms: rawCapture.eventTimeMs, + received_time_ms: rawCapture.receivedTimeMs, + raw_capture_id: rawCapture.rawCaptureId, + raw_capture_scope: rawCapture.rawCaptureScope, + schema_version: context.schemaVersion, + checksum_algorithm: context.checksumAlgorithm, + raw_checksum: rawCapture.rawChecksum, + provenance_complete: context.provenanceComplete ? 1 : 0, + }; +} diff --git a/src/helpers/market-data-archive/capture.ts b/src/helpers/market-data-archive/capture.ts index fa0ef9d..63dc08b 100644 --- a/src/helpers/market-data-archive/capture.ts +++ b/src/helpers/market-data-archive/capture.ts @@ -5,18 +5,32 @@ import { } from "../broker-execution-archive/writer"; import { log } from "../logger"; import type { OtelMetrics } from "../otel"; +import { + buildCanonicalOrderBookRows, + OrderBookValidationError, +} from "./canonical-orderbook"; +import { + captureEnvironmentFromEnv, + createMarketCaptureContext, +} from "./capture-context"; +import { createRawCapture } from "./capture-contract"; import { OhlcvBarTracker } from "./ohlcv-bar-tracker"; +import { getOrderbookArchiveDepthLimit } from "./orderbook-depth"; import { isMarketArchiveEnabled, OrderbookSampler } from "./orderbook-sampler"; import { extractTrades, parseTicker } from "./parse-stream"; import { - buildCandleRow, + buildCanonicalCexStreamEventRow, + buildCanonicalOhlcvRow, + buildCanonicalTickerEventRow, + buildCanonicalTradeRow, buildCexStreamEventRow, - buildCexTickerEventRow, - buildCexTradeRow, - buildOrderbookSnapshotRow, } from "./rows"; import type { + CaptureFeed, + CaptureSourceMode, CexStreamArchiveInput, + MarketArchiveContext, + MarketCaptureContext, OhlcvArchiveInput, OrderbookArchiveInput, TickerArchiveInput, @@ -40,21 +54,51 @@ async function recordWatchMetric( function watchLabels( stream: WatchStream, input: { exchange: string; symbol: string }, + archiver: BrokerExecutionArchiver | undefined, + feed: string, ): Record { return { stream, + feed, + source: archiver?.getSource() ?? "disabled", exchange: input.exchange, symbol: input.symbol, }; } +function resolveCaptureContext( + archiver: BrokerExecutionArchiver, + input: MarketArchiveContext, + feed: CaptureFeed, + sourceMode: CaptureSourceMode, +): MarketCaptureContext { + return createMarketCaptureContext({ + source: archiver.getSource(), + deploymentId: archiver.getDeploymentId(), + captureBundleId: process.env.CEX_BROKER_CAPTURE_BUNDLE_ID, + exchange: input.exchange, + symbol: input.symbol, + assetType: input.assetType, + feed, + provider: `ccxt:${input.exchange.trim().toLowerCase()}`, + sourceMode, + timeframe: input.timeframe, + accountSelector: input.accountSelector, + environment: captureEnvironmentFromEnv(), + }); +} + export function archiveOrderbookInBackground( archiver: BrokerExecutionArchiver | undefined, otelMetrics: OtelMetrics | undefined, input: OrderbookArchiveInput, - options?: { sampledOut?: boolean }, + options?: { + sampledOut?: boolean; + sourceMode?: "broker_live_sampling_v1" | "broker_current_snapshot_v1"; + depthLimit?: number; + }, ): void { - const labels = watchLabels("orderbook", input); + const labels = watchLabels("orderbook", input, archiver, "ORDERBOOK"); void recordWatchMetric( otelMetrics, "cex_watch_frames_received_total", @@ -76,17 +120,40 @@ export function archiveOrderbookInBackground( queueMicrotask(() => { try { - const row = buildOrderbookSnapshotRow(input); - if (row) { - archiver.enqueue(row); - void recordWatchMetric( - otelMetrics, - "cex_watch_frames_archived_total", - labels, - ); - } + const context = resolveCaptureContext( + archiver, + input, + "ORDERBOOK", + options?.sourceMode ?? "broker_live_sampling_v1", + ); + const rawCapture = createRawCapture(context, { + payload: input.snapshot, + eventTimeMs: input.snapshot.timestamp, + receivedTimeMs: input.snapshot.receivedTimestamp, + scope: "ccxt_normalized_object", + }); + const canonical = buildCanonicalOrderBookRows({ + context, + snapshot: input.snapshot, + rawCapture, + depthLimit: options?.depthLimit ?? getOrderbookArchiveDepthLimit(), + }); + archiver.enqueue(buildCanonicalCexStreamEventRow(context, rawCapture)); + for (const row of canonical.levels) archiver.enqueue(row); + archiver.enqueue(canonical.summary); + void recordWatchMetric( + otelMetrics, + "cex_watch_frames_archived_total", + labels, + ); } catch (error) { rethrowArchiveDurabilityError(error); + if (error instanceof OrderBookValidationError) { + void recordWatchMetric(otelMetrics, "cex_watch_frames_invalid_total", { + ...labels, + reason: error.reason, + }); + } log.warn("Failed to archive orderbook snapshot", { error }); } }); @@ -105,7 +172,7 @@ export function archiveOhlcvInBackground( tracker: OhlcvBarTracker, input: OhlcvArchiveInput, ): void { - const labels = watchLabels("ohlcv", input); + const labels = watchLabels("ohlcv", input, archiver, "OHLCV"); void recordWatchMetric( otelMetrics, "cex_watch_frames_received_total", @@ -122,15 +189,37 @@ export function archiveOhlcvInBackground( input.payload, input.receivedTimestamp, ); + const context = resolveCaptureContext( + archiver, + input, + "OHLCV", + input.sourceMode ?? "broker_live_stream_v1", + ); + const rawCapture = + candidates.length > 0 + ? createRawCapture(context, { + payload: input.payload, + eventTimeMs: + candidates[0]?.bar.openTimeMs ?? input.receivedTimestamp, + receivedTimeMs: input.receivedTimestamp, + scope: "ccxt_normalized_object", + }) + : undefined; + if (rawCapture) { + archiver.enqueue(buildCanonicalCexStreamEventRow(context, rawCapture)); + } for (const candidate of candidates) { - const row = buildCandleRow({ - context: input, - bar: candidate.bar, - isClosed: candidate.isClosed, - brokerVersion: candidate.brokerVersion, - receivedTimestamp: input.receivedTimestamp, - }); - archiver.enqueue(row); + if (rawCapture) { + archiver.enqueue( + buildCanonicalOhlcvRow({ + context, + rawCapture, + bar: candidate.bar, + isClosed: candidate.isClosed, + brokerVersion: candidate.brokerVersion, + }), + ); + } } if (candidates.length > 0) { void recordWatchMetric( @@ -162,9 +251,10 @@ function archiveMarketRowsInBackground( otelMetrics: OtelMetrics | undefined, stream: WatchStream, input: { exchange: string; symbol: string }, + feed: string, enqueueRows: () => BrokerArchiveRow[], ): void { - const labels = watchLabels(stream, input); + const labels = watchLabels(stream, input, archiver, feed); void recordWatchMetric( otelMetrics, "cex_watch_frames_received_total", @@ -200,10 +290,33 @@ export function archiveTradesInBackground( otelMetrics: OtelMetrics | undefined, input: TradesArchiveInput, ): void { - archiveMarketRowsInBackground(archiver, otelMetrics, "trades", input, () => - extractTrades(input.payload, input.receivedTimestamp).map((trade) => - buildCexTradeRow(input, trade), - ), + archiveMarketRowsInBackground( + archiver, + otelMetrics, + "trades", + input, + "TRADES", + () => { + if (!archiver) return []; + const trades = extractTrades(input.payload, input.receivedTimestamp); + if (trades.length === 0) return []; + const context = resolveCaptureContext( + archiver, + input, + "TRADES", + "broker_live_stream_v1", + ); + const raw = createRawCapture(context, { + payload: input.payload, + eventTimeMs: trades[0]?.eventTimeMs ?? input.receivedTimestamp, + receivedTimeMs: input.receivedTimestamp, + scope: "ccxt_normalized_object", + }); + return [ + buildCanonicalCexStreamEventRow(context, raw), + ...trades.map((trade) => buildCanonicalTradeRow(context, raw, trade)), + ]; + }, ); } @@ -212,10 +325,33 @@ export function archiveTickerInBackground( otelMetrics: OtelMetrics | undefined, input: TickerArchiveInput, ): void { - archiveMarketRowsInBackground(archiver, otelMetrics, "ticker", input, () => { - const ticker = parseTicker(input.payload, input.receivedTimestamp); - return ticker ? [buildCexTickerEventRow(input, ticker)] : []; - }); + archiveMarketRowsInBackground( + archiver, + otelMetrics, + "ticker", + input, + "TICKER", + () => { + const ticker = parseTicker(input.payload, input.receivedTimestamp); + if (!ticker || !archiver) return []; + const context = resolveCaptureContext( + archiver, + input, + "TICKER", + "broker_live_stream_v1", + ); + const raw = createRawCapture(context, { + payload: input.payload, + eventTimeMs: ticker.eventTimeMs, + receivedTimeMs: input.receivedTimestamp, + scope: "ccxt_normalized_object", + }); + return [ + buildCanonicalCexStreamEventRow(context, raw), + buildCanonicalTickerEventRow(context, raw, ticker), + ]; + }, + ); } export function archiveCexStreamEventInBackground( @@ -223,7 +359,17 @@ export function archiveCexStreamEventInBackground( otelMetrics: OtelMetrics | undefined, input: CexStreamArchiveInput, ): void { - archiveMarketRowsInBackground(archiver, otelMetrics, "stream", input, () => [ - buildCexStreamEventRow(input), - ]); + archiveMarketRowsInBackground( + archiver, + otelMetrics, + "stream", + input, + input.streamType, + () => [ + buildCexStreamEventRow({ + ...input, + source: archiver?.getSource(), + }), + ], + ); } diff --git a/src/helpers/market-data-archive/index.ts b/src/helpers/market-data-archive/index.ts index dad0dea..d022209 100644 --- a/src/helpers/market-data-archive/index.ts +++ b/src/helpers/market-data-archive/index.ts @@ -1,3 +1,7 @@ +export { + buildCanonicalOrderBookRows, + OrderBookValidationError, +} from "./canonical-orderbook"; export { archiveCexStreamEventInBackground, archiveOhlcvInBackground, @@ -10,6 +14,32 @@ export { createOrderbookSampler, createOrderbookTobSampler, } from "./capture"; +export { + captureEnvironmentFromEnv, + createMarketCaptureContext, + validateExternalFallbackContext, + validateProductionCollectorArchive, +} from "./capture-context"; +export { + ARCHIVE_SOURCES, + CAPTURE_FEEDS, + CHECKSUM_ALGORITHM, + CONSTRUCTION_MODES, + canonicalDecimal, + canonicalSerialize, + createRawCapture, + GAP_POLICIES, + MARKET_CAPTURE_SCHEMA_VERSION, + RAW_CAPTURE_SCOPES, + SOURCE_MODES, + sha256Canonical, +} from "./capture-contract"; +export { + buildLegacyOhlcvMigrationRow, + buildLegacyOrderBookMigrationRows, + type LegacyCandle, + type LegacyOrderBookSnapshot, +} from "./legacy-migration"; export { extractLatestOhlcvBar, extractOhlcvBars, @@ -31,24 +61,27 @@ export { } from "./orderbook-sampler"; export { extractTrades, parseTicker, parseTrade } from "./parse-stream"; export { - buildCandleRow, + buildCanonicalCexStreamEventRow, + buildCanonicalOhlcvRow, + buildCanonicalTickerEventRow, + buildCanonicalTradeRow, buildCexStreamEventRow, - buildCexTickerEventRow, - buildCexTradeRow, - buildOrderbookDepthRow, - buildOrderbookSnapshotRow, - buildOrderbookTobRow, } from "./rows"; export type { + CaptureFeed, + CaptureSourceMode, CexStreamArchiveInput, CexStreamType, MarketArchiveContext, + MarketCaptureContext, OhlcvArchiveCandidate, OhlcvArchiveInput, OrderbookArchiveInput, OrderbookSnapshotArchiveInput, OrderbookTobArchiveInput, ParsedOhlcvBar, + RawCapture, + RawCaptureScope, TickerArchiveInput, TradesArchiveInput, } from "./types"; diff --git a/src/helpers/market-data-archive/legacy-migration.ts b/src/helpers/market-data-archive/legacy-migration.ts new file mode 100644 index 0000000..b92a7d5 --- /dev/null +++ b/src/helpers/market-data-archive/legacy-migration.ts @@ -0,0 +1,171 @@ +import type { + BrokerArchiveRow, + BrokerArchiveSource, +} from "../broker-execution-archive/types"; +import type { NormalizedOrderBookSnapshot } from "../order-book"; +import { buildCanonicalOrderBookRows } from "./canonical-orderbook"; +import { + CHECKSUM_ALGORITHM, + MARKET_CAPTURE_SCHEMA_VERSION, + sha256Canonical, +} from "./capture-contract"; +import { buildCanonicalOhlcvRow } from "./rows"; +import type { MarketCaptureContext, ParsedOhlcvBar, RawCapture } from "./types"; + +export type LegacyOrderBookSnapshot = { + source?: BrokerArchiveSource; + deployment_id: string; + account_selector?: string; + exchange: string; + asset_type: "spot" | "swap" | "future"; + symbol: string; + event_time_ms: number; + received_time_ms: number; + depth_limit: number; + bids_price: number[]; + bids_size: number[]; + asks_price: number[]; + asks_size: number[]; + sequence?: number; +}; + +export type LegacyCandle = { + source?: BrokerArchiveSource; + deployment_id: string; + account_selector?: string; + exchange: string; + asset_type: "spot" | "swap" | "future"; + symbol: string; + timeframe: string; + open_time_ms: number; + open: number; + high: number; + low: number; + close: number; + volume: number; + quote_volume?: number; + is_closed: 0 | 1; + broker_version: number; +}; + +function legacyContext(input: { + source?: BrokerArchiveSource; + deployment_id: string; + account_selector?: string; + exchange: string; + asset_type: "spot" | "swap" | "future"; + symbol: string; + feed: "ORDERBOOK" | "OHLCV"; + timeframe?: string; +}): MarketCaptureContext { + return { + source: input.source ?? "broker_write", + deploymentId: input.deployment_id, + accountSelector: input.account_selector, + captureBundleId: "legacy-unavailable", + exchange: input.exchange, + symbol: input.symbol, + assetType: input.asset_type, + feed: input.feed, + provider: + input.feed === "ORDERBOOK" + ? "legacy:orderbook_snapshots" + : "legacy:candles", + sourceMode: "legacy_migration_v1", + schemaVersion: MARKET_CAPTURE_SCHEMA_VERSION, + checksumAlgorithm: CHECKSUM_ALGORITHM, + provenanceComplete: false, + timeframe: input.timeframe, + }; +} + +function unavailableRaw( + eventTimeMs: number, + receivedTimeMs: number, +): RawCapture { + return { + rawCaptureId: "legacy-unavailable", + rawCaptureScope: "ccxt_normalized_object", + rawChecksum: "legacy-unavailable", + redactedPayload: null, + eventTimeMs, + receivedTimeMs, + checksumAlgorithm: CHECKSUM_ALGORITHM, + }; +} + +function markIncomplete(row: BrokerArchiveRow): BrokerArchiveRow { + const incomplete = { + ...row.row, + capture_bundle_id: null, + raw_capture_id: null, + raw_capture_scope: null, + raw_checksum: null, + provenance_complete: 0, + source_mode: "legacy_migration_v1", + }; + return { + table: row.table, + row: { + ...incomplete, + normalized_row_checksum: sha256Canonical(incomplete), + }, + }; +} + +export function buildLegacyOrderBookMigrationRows( + legacy: LegacyOrderBookSnapshot, +): BrokerArchiveRow[] { + if ( + legacy.bids_price.length !== legacy.bids_size.length || + legacy.asks_price.length !== legacy.asks_size.length + ) { + throw new Error("Legacy order-book price and size arrays must align"); + } + const snapshot: NormalizedOrderBookSnapshot = { + bids: legacy.bids_price.map((price, index) => [ + price, + legacy.bids_size[index] as number, + ]), + asks: legacy.asks_price.map((price, index) => [ + price, + legacy.asks_size[index] as number, + ]), + timestamp: legacy.event_time_ms, + receivedTimestamp: legacy.received_time_ms, + exchange: legacy.exchange, + symbol: legacy.symbol, + depthLimit: legacy.depth_limit, + sequence: legacy.sequence, + }; + const canonical = buildCanonicalOrderBookRows({ + context: legacyContext({ ...legacy, feed: "ORDERBOOK" }), + snapshot, + rawCapture: unavailableRaw(legacy.event_time_ms, legacy.received_time_ms), + depthLimit: legacy.depth_limit, + }); + return [...canonical.levels, canonical.summary].map(markIncomplete); +} + +export function buildLegacyOhlcvMigrationRow( + legacy: LegacyCandle, +): BrokerArchiveRow { + const bar: ParsedOhlcvBar = { + openTimeMs: legacy.open_time_ms, + open: legacy.open, + high: legacy.high, + low: legacy.low, + close: legacy.close, + volume: legacy.volume, + quoteVolume: legacy.quote_volume, + }; + return markIncomplete( + buildCanonicalOhlcvRow({ + context: legacyContext({ ...legacy, feed: "OHLCV" }), + rawCapture: unavailableRaw(legacy.open_time_ms, legacy.open_time_ms), + bar, + isClosed: legacy.is_closed === 1, + brokerVersion: legacy.broker_version, + }), + ); +} diff --git a/src/helpers/market-data-archive/ohlcv-history.ts b/src/helpers/market-data-archive/ohlcv-history.ts index 0e495a6..2d2edb0 100644 --- a/src/helpers/market-data-archive/ohlcv-history.ts +++ b/src/helpers/market-data-archive/ohlcv-history.ts @@ -44,6 +44,7 @@ export async function bootstrapOhlcvHistory( const receivedTimestamp = Date.now(); archiveOhlcvInBackground(archiver, otelMetrics, tracker, { ...input, + sourceMode: "broker_bootstrap_fetch_v1", payload, receivedTimestamp, }); diff --git a/src/helpers/market-data-archive/rows.ts b/src/helpers/market-data-archive/rows.ts index 0c760c1..f0692ac 100644 --- a/src/helpers/market-data-archive/rows.ts +++ b/src/helpers/market-data-archive/rows.ts @@ -1,19 +1,17 @@ import { redactStreamPayload } from "../broker-execution-archive/redact"; import { buildCommonArchiveTags } from "../broker-execution-archive/rows"; import type { BrokerArchiveRow } from "../broker-execution-archive/types"; -import type { NormalizedOrderBookSnapshot } from "../order-book"; import { - getOrderbookArchiveDepthLimit, - splitOrderBookSide, -} from "./orderbook-depth"; + canonicalSerialize, + captureCoreFields, + sha256Canonical, +} from "./capture-contract"; import type { ParsedTicker, ParsedTrade } from "./parse-stream"; import type { CexStreamArchiveInput, - MarketArchiveContext, - OrderbookArchiveInput, + MarketCaptureContext, ParsedOhlcvBar, - TickerArchiveInput, - TradesArchiveInput, + RawCapture, } from "./types"; function compactUndefined( @@ -24,166 +22,104 @@ function compactUndefined( ); } -function scalarTimestampMs(value: number | string | boolean | null): number { - if (typeof value === "number" && Number.isFinite(value)) { - return value; - } - if (typeof value === "string") { - if (/^\d+$/.test(value)) { - const numeric = Number.parseInt(value, 10); - if (Number.isFinite(numeric)) { - return numeric; - } - } - const parsed = Date.parse(value); - if (Number.isFinite(parsed)) { - return parsed; - } - } - return Date.now(); -} - -function parseSequence( - value: number | string | boolean | undefined, -): number | undefined { - if (typeof value === "number" && Number.isFinite(value)) { - return value; - } - if (typeof value === "string" && /^\d+$/.test(value)) { - return Number.parseInt(value, 10); - } - return undefined; +function withNormalizedChecksum( + record: Record, +): Record { + const compact = compactUndefined(record); + return { + ...compact, + normalized_row_checksum: sha256Canonical(compact), + }; } -function topOfBookLevel( - levels: number[][], -): { price: number; size: number } | null { - const level = levels[0]; - if (!level || level.length < 2) { - return null; - } - const price = level[0]; - const size = level[1]; - if ( - price === undefined || - size === undefined || - !Number.isFinite(price) || - !Number.isFinite(size) - ) { - return null; - } - return { price, size }; +export function buildCanonicalCexStreamEventRow( + context: MarketCaptureContext, + rawCapture: RawCapture, +): BrokerArchiveRow { + const row = withNormalizedChecksum({ + ...captureCoreFields(context, rawCapture), + stream_type: context.feed, + event_time_ms: rawCapture.eventTimeMs, + payload_encoding: "canonical_json_v1", + payload_json: canonicalSerialize(rawCapture.redactedPayload), + }); + return { table: "market_data.cex_stream_events", row }; } -function computeSpreadBps(bestBid: number, bestAsk: number): number { - if (bestBid <= 0 || bestAsk <= 0 || bestAsk < bestBid) { - return 0; - } - const mid = (bestBid + bestAsk) / 2; - if (mid <= 0) { - return 0; +export function buildCanonicalTickerEventRow( + context: MarketCaptureContext, + rawCapture: RawCapture, + ticker: ParsedTicker, +): BrokerArchiveRow { + if (context.feed !== "TICKER") { + throw new Error("Ticker row requires a TICKER capture context"); } - return ((bestAsk - bestBid) / mid) * 10_000; -} - -function buildOrderbookArchiveTags( - input: OrderbookArchiveInput, - receivedTimeMs: number, -) { - return buildCommonArchiveTags({ - deploymentId: input.deploymentId, - accountSelector: input.accountSelector, - exchange: input.exchange, - symbol: input.symbol, - brokerObservedTimestamp: new Date(receivedTimeMs).toISOString(), + const row = withNormalizedChecksum({ + ...captureCoreFields(context, rawCapture), + source_time_ms: ticker.eventTimeMs, + event_time_ms: ticker.eventTimeMs, + last: ticker.last, + bid: ticker.bid, + ask: ticker.ask, + high: ticker.high, + low: ticker.low, + open: ticker.open, + close: ticker.close, + base_volume: ticker.baseVolume, + quote_volume: ticker.quoteVolume, + change: ticker.change, + percentage: ticker.percentage, }); + return { table: "market_data.cex_ticker_events", row }; } -export function buildOrderbookSnapshotRow( - input: OrderbookArchiveInput, -): BrokerArchiveRow | null { - const bid = topOfBookLevel(input.snapshot.bids); - const ask = topOfBookLevel(input.snapshot.asks); - if (!bid || !ask) { - return null; - } - - const archiveDepthLimit = getOrderbookArchiveDepthLimit(); - const bids = splitOrderBookSide(input.snapshot.bids, archiveDepthLimit); - const asks = splitOrderBookSide(input.snapshot.asks, archiveDepthLimit); - if (bids.prices.length === 0 || asks.prices.length === 0) { - return null; +export function buildCanonicalTradeRow( + context: MarketCaptureContext, + rawCapture: RawCapture, + trade: ParsedTrade, +): BrokerArchiveRow { + if (context.feed !== "TRADES") { + throw new Error("Trade row requires a TRADES capture context"); } - - const eventTimeMs = scalarTimestampMs(input.snapshot.timestamp); - const receivedTimeMs = input.snapshot.receivedTimestamp; - const mid = (bid.price + ask.price) / 2; - const sequence = parseSequence(input.snapshot.sequence); - - return { - table: "market_data.orderbook_snapshots", - row: compactUndefined({ - ...buildOrderbookArchiveTags(input, receivedTimeMs), - asset_type: input.assetType, - event_time_ms: eventTimeMs, - received_time_ms: receivedTimeMs, - best_bid: bid.price, - best_ask: ask.price, - bid_size: bid.size, - ask_size: ask.size, - mid, - spread_bps: computeSpreadBps(bid.price, ask.price), - depth_limit: archiveDepthLimit, - bid_levels: bids.prices.length, - ask_levels: asks.prices.length, - bids_price: bids.prices, - bids_size: bids.sizes, - asks_price: asks.prices, - asks_size: asks.sizes, - sequence, - }), - }; + const row = withNormalizedChecksum({ + ...captureCoreFields(context, rawCapture), + source_time_ms: trade.eventTimeMs, + trade_id: trade.tradeId, + event_time_ms: trade.eventTimeMs, + side: trade.side, + price: trade.price, + amount: trade.amount, + cost: trade.cost, + taker_or_maker: trade.takerOrMaker, + }); + return { table: "market_data.cex_trades", row }; } -/** @deprecated Use buildOrderbookSnapshotRow */ -export const buildOrderbookTobRow = buildOrderbookSnapshotRow; - -/** @deprecated Use buildOrderbookSnapshotRow */ -export const buildOrderbookDepthRow = buildOrderbookSnapshotRow; - -export function buildCandleRow(input: { - context: MarketArchiveContext; +export function buildCanonicalOhlcvRow(input: { + context: MarketCaptureContext; + rawCapture: RawCapture; bar: ParsedOhlcvBar; isClosed: boolean; brokerVersion: number; - receivedTimestamp: number; }): BrokerArchiveRow { - const { context, bar, isClosed, brokerVersion, receivedTimestamp } = input; - const tags = buildCommonArchiveTags({ - deploymentId: context.deploymentId, - accountSelector: context.accountSelector, - exchange: context.exchange, - symbol: context.symbol, - brokerObservedTimestamp: new Date(receivedTimestamp).toISOString(), + if (input.context.feed !== "OHLCV") { + throw new Error("OHLCV row requires an OHLCV capture context"); + } + const row = withNormalizedChecksum({ + ...captureCoreFields(input.context, input.rawCapture), + source_time_ms: input.bar.openTimeMs, + timeframe: input.context.timeframe ?? "1m", + open_time_ms: input.bar.openTimeMs, + open: input.bar.open, + high: input.bar.high, + low: input.bar.low, + close: input.bar.close, + volume: input.bar.volume, + quote_volume: input.bar.quoteVolume, + is_closed: input.isClosed ? 1 : 0, + broker_version: input.brokerVersion, }); - - return { - table: "market_data.candles", - row: compactUndefined({ - ...tags, - asset_type: context.assetType, - timeframe: context.timeframe ?? "1m", - open_time_ms: bar.openTimeMs, - open: bar.open, - high: bar.high, - low: bar.low, - close: bar.close, - volume: bar.volume, - quote_volume: bar.quoteVolume, - is_closed: isClosed ? 1 : 0, - broker_version: brokerVersion, - }), - }; + return { table: "market_data.cex_ohlcv", row }; } export function buildCexStreamEventRow( @@ -192,6 +128,7 @@ export function buildCexStreamEventRow( const receivedTimeMs = input.receivedTimestamp; const redactedPayload = redactStreamPayload(input.payload); const tags = buildCommonArchiveTags({ + source: input.source, deploymentId: input.deploymentId, accountSelector: input.accountSelector, exchange: input.exchange, @@ -211,69 +148,3 @@ export function buildCexStreamEventRow( }), }; } - -export function buildCexTickerEventRow( - input: TickerArchiveInput, - ticker: ParsedTicker, -): BrokerArchiveRow { - const receivedTimeMs = input.receivedTimestamp; - const tags = buildCommonArchiveTags({ - deploymentId: input.deploymentId, - accountSelector: input.accountSelector, - exchange: input.exchange, - symbol: input.symbol, - brokerObservedTimestamp: new Date(receivedTimeMs).toISOString(), - }); - - return { - table: "market_data.cex_ticker_events", - row: compactUndefined({ - ...tags, - asset_type: input.assetType, - event_time_ms: ticker.eventTimeMs, - received_time_ms: receivedTimeMs, - last: ticker.last, - bid: ticker.bid, - ask: ticker.ask, - high: ticker.high, - low: ticker.low, - open: ticker.open, - close: ticker.close, - base_volume: ticker.baseVolume, - quote_volume: ticker.quoteVolume, - change: ticker.change, - percentage: ticker.percentage, - payload_json: JSON.stringify(redactStreamPayload(input.payload)), - }), - }; -} - -export function buildCexTradeRow( - input: TradesArchiveInput, - trade: ParsedTrade, -): BrokerArchiveRow { - const receivedTimeMs = input.receivedTimestamp; - const tags = buildCommonArchiveTags({ - deploymentId: input.deploymentId, - accountSelector: input.accountSelector, - exchange: input.exchange, - symbol: input.symbol, - brokerObservedTimestamp: new Date(receivedTimeMs).toISOString(), - }); - - return { - table: "market_data.cex_trades", - row: compactUndefined({ - ...tags, - asset_type: input.assetType, - trade_id: trade.tradeId, - event_time_ms: trade.eventTimeMs, - received_time_ms: receivedTimeMs, - side: trade.side, - price: trade.price, - amount: trade.amount, - cost: trade.cost, - taker_or_maker: trade.takerOrMaker, - }), - }; -} diff --git a/src/helpers/market-data-archive/types.ts b/src/helpers/market-data-archive/types.ts index f57d541..6a8dc03 100644 --- a/src/helpers/market-data-archive/types.ts +++ b/src/helpers/market-data-archive/types.ts @@ -1,4 +1,7 @@ -import type { BrokerArchiveRow } from "../broker-execution-archive/types"; +import type { + BrokerArchiveRow, + BrokerArchiveSource, +} from "../broker-execution-archive/types"; import type { BrokerMarketType } from "../market-type"; import type { NormalizedOrderBookSnapshot } from "../order-book"; @@ -7,7 +10,10 @@ export type MarketArchiveTable = | "market_data.candles" | "market_data.cex_stream_events" | "market_data.cex_ticker_events" - | "market_data.cex_trades"; + | "market_data.cex_trades" + | "market_data.cex_ohlcv" + | "market_data.cex_order_book_levels" + | "market_data.cex_order_book_depth_summary"; export type CexStreamType = | "BALANCE" @@ -34,6 +40,7 @@ export type OhlcvArchiveCandidate = { }; export type MarketArchiveContext = { + source?: BrokerArchiveSource; exchange: string; symbol: string; assetType: BrokerMarketType; @@ -42,6 +49,41 @@ export type MarketArchiveContext = { deploymentId: string; }; +export type CaptureFeed = "ORDERBOOK" | "TICKER" | "TRADES" | "OHLCV"; +export type CaptureSourceMode = + | "broker_live_stream_v1" + | "broker_live_sampling_v1" + | "broker_current_snapshot_v1" + | "broker_bootstrap_fetch_v1" + | "external_ccxt_fallback_v1" + | "external_hummingbot_fallback_v1" + | "legacy_migration_v1"; +export type RawCaptureScope = + | "ccxt_normalized_object" + | "broker_visible_payload" + | "exchange_wire_frame"; + +export type MarketCaptureContext = MarketArchiveContext & { + source: BrokerArchiveSource; + captureBundleId: string; + feed: CaptureFeed; + provider: string; + sourceMode: CaptureSourceMode; + schemaVersion: string; + checksumAlgorithm: string; + provenanceComplete: boolean; +}; + +export type RawCapture = { + rawCaptureId: string; + rawCaptureScope: RawCaptureScope; + rawChecksum: string; + redactedPayload: unknown; + eventTimeMs: number; + receivedTimeMs: number; + checksumAlgorithm: string; +}; + export type OrderbookArchiveInput = MarketArchiveContext & { snapshot: NormalizedOrderBookSnapshot; }; @@ -56,6 +98,7 @@ export type OhlcvArchiveInput = MarketArchiveContext & { payload: unknown; receivedTimestamp: number; timeframe: string; + sourceMode?: "broker_live_stream_v1" | "broker_bootstrap_fetch_v1"; }; export type TradesArchiveInput = MarketArchiveContext & { @@ -81,6 +124,9 @@ const MARKET_ARCHIVE_TABLES = new Set([ "market_data.cex_stream_events", "market_data.cex_ticker_events", "market_data.cex_trades", + "market_data.cex_ohlcv", + "market_data.cex_order_book_levels", + "market_data.cex_order_book_depth_summary", ]); export function isMarketArchiveTable( diff --git a/src/index.ts b/src/index.ts index f14eea7..1de0276 100644 --- a/src/index.ts +++ b/src/index.ts @@ -246,7 +246,6 @@ export default class CEXBroker { this.otelLogs, this.otelMetrics, ); - this.loadExchangeCredentials(apiCredentials); this.whitelistIps = [ ...((config ?? { whitelistIps: [] }).whitelistIps ?? []), @@ -356,6 +355,7 @@ export default class CEXBroker { this.brokerArchiver, this.orderActivityTracker, this.withdrawalObservationTracker, + undefined, ); this.server.bindAsync( diff --git a/test/archive-forwarder-config.test.ts b/test/archive-forwarder-config.test.ts new file mode 100644 index 0000000..ffdb513 --- /dev/null +++ b/test/archive-forwarder-config.test.ts @@ -0,0 +1,28 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { loadForwarderConfig } from "../services/archive-forwarder/config"; +import { DEFAULT_STRATEGY_SPOOL_PATH } from "../services/archive-forwarder/strategy-spool"; + +const originalEnv = { ...process.env }; + +afterEach(() => { + process.env = { ...originalEnv }; +}); + +describe("archive forwarder spool configuration", () => { + test("uses the documented local fallback", () => { + delete process.env.ARCHIVE_FORWARDER_SPOOL_PATH; + expect(loadForwarderConfig().spoolPath).toBe(DEFAULT_STRATEGY_SPOOL_PATH); + }); + + test("accepts only the forwarder-specific persistent path knob", () => { + process.env.ARCHIVE_FORWARDER_SPOOL_PATH = + "/var/lib/archive-forwarder/spool.sqlite"; + expect(loadForwarderConfig().spoolPath).toBe( + "/var/lib/archive-forwarder/spool.sqlite", + ); + expect(Object.keys(loadForwarderConfig())).not.toContain("spoolQuotaBytes"); + expect(Object.keys(loadForwarderConfig())).not.toContain( + "spoolRetentionMs", + ); + }); +}); diff --git a/test/archive-forwarder-health.test.ts b/test/archive-forwarder-health.test.ts new file mode 100644 index 0000000..1648eef --- /dev/null +++ b/test/archive-forwarder-health.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, test } from "bun:test"; +import { evaluateForwarderHealth } from "../services/archive-forwarder/health"; + +const spoolStats = { + queuedBatches: 2, + queuedWork: 3, + terminalWork: 1, + expiredWork: 4, + accountedBytes: 12_345, + oldestAgeMs: 6_000, + lastErrorClass: "connection", +}; + +describe("archive forwarder health contract", () => { + test("is degraded but available while ClickHouse is down and spool is healthy", () => { + expect( + evaluateForwarderHealth({ + clickhouseOk: false, + spoolOk: true, + spool: spoolStats, + }), + ).toEqual({ + statusCode: 200, + body: { + status: "degraded", + clickhouse: false, + durableAdmission: true, + spool: { healthy: true, ...spoolStats }, + }, + }); + }); + + test("returns unavailable whenever the durable spool is unhealthy", () => { + const health = evaluateForwarderHealth({ + clickhouseOk: true, + spoolOk: false, + spool: null, + }); + expect(health.statusCode).toBe(503); + expect(health.body).toMatchObject({ + status: "unavailable", + clickhouse: true, + durableAdmission: false, + spool: { healthy: false }, + }); + }); + + test("reports ok only when ClickHouse and the spool are healthy", () => { + const health = evaluateForwarderHealth({ + clickhouseOk: true, + spoolOk: true, + spool: { ...spoolStats, queuedBatches: 0, queuedWork: 0 }, + }); + expect(health.statusCode).toBe(200); + expect(health.body.status).toBe("ok"); + }); +}); diff --git a/test/archive-forwarder-spool.test.ts b/test/archive-forwarder-spool.test.ts new file mode 100644 index 0000000..0695d13 --- /dev/null +++ b/test/archive-forwarder-spool.test.ts @@ -0,0 +1,206 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { chmodSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { + StrategyArchiveSpool, + StrategySpoolQuotaError, + StrategySpoolUnavailableError, +} from "../services/archive-forwarder/strategy-spool"; +import type { ArchiveBatchRequest } from "../services/archive-forwarder/types"; + +const tempDirs: string[] = []; + +afterEach(() => { + for (const directory of tempDirs.splice(0)) { + rmSync(directory, { recursive: true, force: true }); + } +}); + +function spoolPath(): string { + const directory = mkdtempSync(path.join(tmpdir(), "cex-forwarder-spool-")); + tempDirs.push(directory); + return path.join(directory, "spool.sqlite"); +} + +function strategyBatch( + tables = [ + "strategy_data.policy_evaluation_events", + "strategy_data.inventory_settlement_events", + ], +): ArchiveBatchRequest { + return { + source: "hb_runtime", + deployment_id: "maker-a", + rows: tables.map((table, index) => ({ + table, + row: { + source: "hb_runtime", + deployment_id: "maker-a", + schema_version: "1", + seq: index + 1, + payload_json: JSON.stringify({ index }), + }, + })), + }; +} + +describe("strategy archive SQLite spool", () => { + test("atomically admits one work item per represented table", () => { + const spool = new StrategyArchiveSpool({ path: spoolPath() }); + const admitted = spool.admit(strategyBatch()); + const stats = spool.stats(); + const due = spool.dueWork(); + + expect(admitted.accountedBytes).toBeGreaterThan(0); + expect(stats.queuedBatches).toBe(1); + expect(stats.queuedWork).toBe(2); + expect(stats.accountedBytes).toBe(admitted.accountedBytes); + expect(due.map((work) => work.table).sort()).toEqual([ + "strategy_data.inventory_settlement_events", + "strategy_data.policy_evaluation_events", + ]); + expect(new Set(due.map((work) => work.dedupeToken)).size).toBe(2); + spool.close(); + }); + + test("preserves incomplete work and stable tokens across restart", () => { + const file = spoolPath(); + const first = new StrategyArchiveSpool({ path: file }); + first.admit(strategyBatch()); + const before = first.dueWork(); + first.complete(before[0]); + const remainingToken = first.dueWork()[0].dedupeToken; + first.close(); + + const restarted = new StrategyArchiveSpool({ path: file }); + expect(restarted.stats()).toMatchObject({ + queuedBatches: 1, + queuedWork: 1, + }); + expect(restarted.dueWork()[0].dedupeToken).toBe(remainingToken); + restarted.close(); + }); + + test("deletes a batch only after every table completes", () => { + const spool = new StrategyArchiveSpool({ path: spoolPath() }); + spool.admit(strategyBatch()); + const [first, second] = spool.dueWork(); + spool.complete(first); + expect(spool.stats()).toMatchObject({ queuedBatches: 1, queuedWork: 1 }); + spool.complete(second); + expect(spool.stats()).toMatchObject({ + queuedBatches: 0, + queuedWork: 0, + accountedBytes: 0, + }); + spool.close(); + }); + + test("rejects atomically when deterministic accounting exceeds quota", () => { + const probe = new StrategyArchiveSpool({ path: ":memory:" }); + const required = probe.accountedBytes(strategyBatch()); + probe.close(); + const spool = new StrategyArchiveSpool({ + path: spoolPath(), + limits: { maxBytes: required - 1 }, + }); + expect(() => spool.admit(strategyBatch())).toThrow(StrategySpoolQuotaError); + expect(spool.stats()).toMatchObject({ queuedBatches: 0, queuedWork: 0 }); + spool.close(); + }); + + test("expires work at the fixed retention boundary", () => { + let now = 10_000; + const spool = new StrategyArchiveSpool({ + path: spoolPath(), + now: () => now, + limits: { retentionMs: 1_000 }, + }); + spool.admit(strategyBatch()); + now = 10_999; + expect(spool.expire()).toBe(0); + now = 11_000; + expect(spool.expire()).toBe(2); + expect(spool.stats()).toMatchObject({ queuedBatches: 0, queuedWork: 0 }); + spool.close(); + }); + + test("reschedules transient work and retains terminal failures", () => { + let now = 10_000; + const spool = new StrategyArchiveSpool({ + path: spoolPath(), + now: () => now, + }); + spool.admit(strategyBatch(["strategy_data.policy_evaluation_events"])); + const work = spool.dueWork()[0]; + spool.reschedule(work, { + nextAttemptAtMs: 11_000, + errorClass: "connection", + }); + expect(spool.dueWork()).toHaveLength(0); + now = 11_000; + expect(spool.dueWork()[0].attemptCount).toBe(1); + spool.markTerminal(spool.dueWork()[0], "schema"); + expect(spool.dueWork()).toHaveLength(0); + expect(spool.stats()).toMatchObject({ terminalWork: 1, queuedWork: 1 }); + spool.close(); + }); + + test("serializes concurrent quota reservation without over-admission", async () => { + const probe = new StrategyArchiveSpool({ path: ":memory:" }); + const required = probe.accountedBytes(strategyBatch()); + probe.close(); + const spool = new StrategyArchiveSpool({ + path: spoolPath(), + limits: { maxBytes: required }, + }); + const results = await Promise.allSettled( + Array.from({ length: 8 }, () => + Promise.resolve().then(() => spool.admit(strategyBatch())), + ), + ); + expect( + results.filter((result) => result.status === "fulfilled"), + ).toHaveLength(1); + expect( + results.filter((result) => result.status === "rejected"), + ).toHaveLength(7); + expect(spool.stats().queuedBatches).toBe(1); + spool.close(); + }); + + test("fails closed when the SQLite file is corrupt", () => { + const file = spoolPath(); + writeFileSync(file, "not-a-sqlite-database"); + expect(() => new StrategyArchiveSpool({ path: file })).toThrow( + StrategySpoolUnavailableError, + ); + }); + + test("wraps post-start write failures as spool unavailability", () => { + const spool = new StrategyArchiveSpool({ path: spoolPath() }); + spool.close(); + expect(() => spool.admit(strategyBatch())).toThrow( + StrategySpoolUnavailableError, + ); + }); + + test("health fails when an on-disk spool becomes read-only", () => { + const file = spoolPath(); + const spool = new StrategyArchiveSpool({ path: file }); + chmodSync(file, 0o444); + try { + // The existing connection can retain write access on some Unix filesystems; + // a reopened connection must still prove the persisted path is writable. + spool.close(); + const readOnly = new StrategyArchiveSpool({ path: file }); + expect(() => readOnly.assertWritable()).toThrow( + StrategySpoolUnavailableError, + ); + readOnly.close(); + } finally { + chmodSync(file, 0o644); + } + }); +}); diff --git a/test/archive-forwarder-strategy-contract.test.ts b/test/archive-forwarder-strategy-contract.test.ts new file mode 100644 index 0000000..16556a9 --- /dev/null +++ b/test/archive-forwarder-strategy-contract.test.ts @@ -0,0 +1,139 @@ +import { describe, expect, test } from "bun:test"; +import { createHash } from "node:crypto"; +import { + classifyStrategyArchiveBatch, + validateStrategyArchiveBatch, +} from "../services/archive-forwarder/strategy-contract"; +import fixture from "./fixtures/archive_forwarder_envelope.json"; + +const PINNED_MAKER_FIXTURE_SHA256 = + "784f647e048052a6c3382309b1a86abfbe08bc162363ead9fc88eaa1ba3d50c9"; + +const strategyTables = [ + "strategy_data.policy_evaluation_events", + "strategy_data.strategy_policy_snapshots", + "strategy_data.market_identity", + "strategy_data.symbol_mapping", + "strategy_data.inventory_settlement_events", +] as const; + +function v2Row(table: (typeof strategyTables)[number], seq: number) { + return { + table, + row: { + source: "hb_runtime", + deployment_id: "maker-a", + schema_version: "2", + producer_id: "hb_runtime:maker-a:controller-a", + producer_run_id: "run-a", + stream_name: table, + stream_seq: 1, + seq, + archive_event_id: `run-a:${table}:1`, + }, + }; +} + +function batch(rows: unknown[]) { + return { source: "hb_runtime", deployment_id: "maker-a", rows }; +} + +describe("Maker strategy archive contract", () => { + test("pins the exact Maker v2 fixture bytes and accepts every row", async () => { + const bytes = await Bun.file( + new URL("./fixtures/archive_forwarder_envelope.json", import.meta.url), + ).arrayBuffer(); + expect( + createHash("sha256").update(new Uint8Array(bytes)).digest("hex"), + ).toBe(PINNED_MAKER_FIXTURE_SHA256); + expect(validateStrategyArchiveBatch(fixture).ok).toBe(true); + }); + + test("accepts one v2 row for every approved strategy table", () => { + const rows = strategyTables.map((table, index) => v2Row(table, index + 1)); + expect(validateStrategyArchiveBatch(batch(rows))).toEqual({ ok: true }); + }); + + test.each([ + undefined, + "", + "1", + ])("accepts legacy schema version %p without v2 identity", (schemaVersion) => { + const row: Record = { + source: "hb_runtime", + deployment_id: "maker-a", + seq: 1, + }; + if (schemaVersion !== undefined) row.schema_version = schemaVersion; + expect( + validateStrategyArchiveBatch( + batch([{ table: "strategy_data.policy_evaluation_events", row }]), + ).ok, + ).toBe(true); + }); + + test.each([ + ["unknown version", { schema_version: "3" }], + ["missing producer id", { producer_id: "" }], + ["zero stream sequence", { stream_seq: 0 }], + ["fractional seq", { seq: 1.5 }], + ])("rejects v2 %s", (_name, override) => { + const entry = v2Row("strategy_data.policy_evaluation_events", 1); + Object.assign(entry.row, override); + expect(validateStrategyArchiveBatch(batch([entry])).ok).toBe(false); + }); + + test("accepts lossless decimal UInt64 strings and rejects overflow", () => { + const entry = v2Row("strategy_data.policy_evaluation_events", 1); + Object.assign(entry.row, { + stream_seq: "18446744073709551615", + seq: "18446744073709551615", + }); + expect(validateStrategyArchiveBatch(batch([entry])).ok).toBe(true); + entry.row.seq = "18446744073709551616"; + expect(validateStrategyArchiveBatch(batch([entry])).ok).toBe(false); + }); + + test("rejects empty envelope identity and empty rows", () => { + expect(validateStrategyArchiveBatch({ ...batch([]), source: " " }).ok).toBe( + false, + ); + expect( + validateStrategyArchiveBatch({ ...batch([]), deployment_id: "" }).ok, + ).toBe(false); + expect(validateStrategyArchiveBatch(batch([])).ok).toBe(false); + }); + + test("rejects mixed tables, mixed sources, and row provenance mismatch", () => { + const strategy = v2Row("strategy_data.policy_evaluation_events", 1); + expect( + validateStrategyArchiveBatch( + batch([ + strategy, + { table: "market_data.cex_trades", row: { source: "hb_runtime" } }, + ]), + ).ok, + ).toBe(false); + expect( + classifyStrategyArchiveBatch({ + source: "broker_write", + deployment_id: "maker-a", + rows: [strategy], + }), + ).toBe("invalid_strategy_source"); + strategy.row.deployment_id = "maker-b"; + expect(validateStrategyArchiveBatch(batch([strategy])).ok).toBe(false); + }); + + test("classifies non-strategy traffic for the direct path", () => { + expect( + classifyStrategyArchiveBatch({ + source: "broker_read", + deployment_id: "broker-a", + rows: [ + { table: "market_data.cex_trades", row: { source: "broker_read" } }, + ], + }), + ).toBe("direct"); + }); +}); diff --git a/test/archive-forwarder-strategy-request.test.ts b/test/archive-forwarder-strategy-request.test.ts new file mode 100644 index 0000000..3c79836 --- /dev/null +++ b/test/archive-forwarder-strategy-request.test.ts @@ -0,0 +1,108 @@ +import { describe, expect, test } from "bun:test"; +import { handleArchiveRequest } from "../services/archive-forwarder/request"; +import { StrategyArchiveSpool } from "../services/archive-forwarder/strategy-spool"; +import { + ArchiveForwarderTelemetry, + type ArchiveMetricsRecorder, +} from "../services/archive-forwarder/telemetry"; +import fixture from "./fixtures/archive_forwarder_envelope.json"; + +const noopRecorder: ArchiveMetricsRecorder = { + recordCounter: () => {}, + setObservableGauge: () => {}, +}; + +function post(body: unknown): Request { + return new Request("http://localhost/archive", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }); +} + +describe("strategy durable HTTP admission", () => { + test("returns 202 after SQLite ownership without waiting for ClickHouse", async () => { + const spool = new StrategyArchiveSpool({ path: ":memory:" }); + let insertCalled = false; + const response = await handleArchiveRequest(post(fixture), { + inserter: async () => { + insertCalled = true; + throw new Error("ClickHouse is down"); + }, + spool, + telemetry: new ArchiveForwarderTelemetry(noopRecorder), + }); + expect(response.status).toBe(202); + expect(insertCalled).toBe(false); + expect(spool.stats()).toMatchObject({ queuedBatches: 1, queuedWork: 3 }); + spool.close(); + }); + + test("keeps broker traffic on direct synchronous insertion", async () => { + const spool = new StrategyArchiveSpool({ path: ":memory:" }); + let insertCalled = false; + const response = await handleArchiveRequest( + post({ + source: "broker_read", + deployment_id: "broker-a", + rows: [ + { + table: "market_data.cex_trades", + row: { source: "broker_read", trade_id: "trade-a" }, + }, + ], + }), + { + inserter: async () => { + insertCalled = true; + }, + spool, + telemetry: new ArchiveForwarderTelemetry(noopRecorder), + }, + ); + expect(response.status).toBe(200); + expect(insertCalled).toBe(true); + expect(spool.stats().queuedBatches).toBe(0); + spool.close(); + }); + + test("rejects strategy source/table/version failures before admission", async () => { + const spool = new StrategyArchiveSpool({ path: ":memory:" }); + const invalid = structuredClone(fixture); + invalid.rows[0].row.schema_version = "99"; + const response = await handleArchiveRequest(post(invalid), { + inserter: async () => {}, + spool, + telemetry: new ArchiveForwarderTelemetry(noopRecorder), + }); + expect(response.status).toBe(400); + expect(spool.stats().queuedBatches).toBe(0); + spool.close(); + }); + + test("returns 429 when the spool cannot reserve fixed capacity", async () => { + const probe = new StrategyArchiveSpool({ path: ":memory:" }); + const required = probe.accountedBytes(fixture); + probe.close(); + const spool = new StrategyArchiveSpool({ + path: ":memory:", + limits: { maxBytes: required - 1 }, + }); + const response = await handleArchiveRequest(post(fixture), { + inserter: async () => {}, + spool, + telemetry: new ArchiveForwarderTelemetry(noopRecorder), + }); + expect(response.status).toBe(429); + expect(spool.stats().queuedBatches).toBe(0); + spool.close(); + }); + + test("returns 503 when no durable strategy spool is available", async () => { + const response = await handleArchiveRequest(post(fixture), { + inserter: async () => {}, + telemetry: new ArchiveForwarderTelemetry(noopRecorder), + }); + expect(response.status).toBe(503); + }); +}); diff --git a/test/archive-forwarder-telemetry.test.ts b/test/archive-forwarder-telemetry.test.ts index 31b354e..5a64a07 100644 --- a/test/archive-forwarder-telemetry.test.ts +++ b/test/archive-forwarder-telemetry.test.ts @@ -194,6 +194,44 @@ describe("archive forwarder telemetry", () => { expect(gauges).toHaveLength(0); }); + test("records same-batch checksum conflicts with bounded source/feed labels", async () => { + const { telemetry, counters } = createCapturingTelemetry(); + const common = { + source: "broker_write", + capture_bundle_id: "bundle-a", + exchange: "binance", + trading_pair: "BTC-USDT", + raw_capture_id: "raw-a", + snapshot_id: "snapshot-a", + schema_version: "1.0.0", + side: "bid", + level_index: 0, + }; + const response = await handleArchiveRequest( + archiveRequest([ + { + table: "market_data.cex_order_book_levels", + row: { ...common, normalized_row_checksum: "a" }, + }, + { + table: "market_data.cex_order_book_levels", + row: { ...common, normalized_row_checksum: "b" }, + }, + ]), + { inserter: async () => {}, telemetry }, + ); + expect(response.status).toBe(400); + expect(counters).toContainEqual({ + name: ARCHIVE_FORWARDER_METRICS.checksumConflicts, + value: 2, + labels: { + source: "broker_write", + feed: "ORDERBOOK", + table: "market_data.cex_order_book_levels", + }, + }); + }); + test("keeps a successful request independent of throwing telemetry", async () => { const throwingRecorder: ArchiveMetricsRecorder = { recordCounter: async () => { @@ -235,4 +273,51 @@ describe("archive forwarder telemetry", () => { expect(response.status).toBe(200); }); + + test("records durable strategy admission and spool state with bounded labels", () => { + const { telemetry, counters, gauges } = createCapturingTelemetry(); + telemetry.recordStrategyAdmission(3); + telemetry.recordStrategyAdmissionRejected("attacker-controlled-reason"); + telemetry.recordStrategyRetry( + "strategy_data.policy_evaluation_events", + "connection", + ); + telemetry.recordStrategySpoolStats({ + queuedBatches: 2, + queuedWork: 3, + terminalWork: 1, + expiredWork: 4, + accountedBytes: 100, + oldestAgeMs: 5_000, + lastErrorClass: "connection", + }); + + expect(counters).toEqual( + expect.arrayContaining([ + { + name: ARCHIVE_FORWARDER_METRICS.strategyBatchesAdmitted, + value: 1, + labels: {}, + }, + { + name: ARCHIVE_FORWARDER_METRICS.strategyRowsAdmitted, + value: 3, + labels: {}, + }, + { + name: ARCHIVE_FORWARDER_METRICS.strategyAdmissionsRejected, + value: 1, + labels: { reason: "other" }, + }, + ]), + ); + expect(gauges.map((entry) => entry.name)).toEqual( + expect.arrayContaining([ + ARCHIVE_FORWARDER_METRICS.strategySpoolPendingBatches, + ARCHIVE_FORWARDER_METRICS.strategySpoolPendingWork, + ARCHIVE_FORWARDER_METRICS.strategySpoolBytes, + ARCHIVE_FORWARDER_METRICS.strategySpoolOldestAge, + ]), + ); + }); }); diff --git a/test/archive-forwarder-worker.test.ts b/test/archive-forwarder-worker.test.ts new file mode 100644 index 0000000..d8c5665 --- /dev/null +++ b/test/archive-forwarder-worker.test.ts @@ -0,0 +1,120 @@ +import { describe, expect, test } from "bun:test"; +import type { RowInserter } from "../services/archive-forwarder/insert"; +import { StrategyArchiveSpool } from "../services/archive-forwarder/strategy-spool"; +import { + retryDelayMs, + StrategySpoolWorker, +} from "../services/archive-forwarder/strategy-worker"; +import type { ArchiveBatchRequest } from "../services/archive-forwarder/types"; + +function batch(): ArchiveBatchRequest { + return { + source: "hb_runtime", + deployment_id: "maker-a", + rows: [ + { + table: "strategy_data.policy_evaluation_events", + row: { schema_version: "1", seq: 1 }, + }, + { + table: "strategy_data.inventory_settlement_events", + row: { schema_version: "1", seq: 2 }, + }, + ], + }; +} + +describe("strategy spool retry worker", () => { + test("uses the specified exponential schedule and jitter bounds", () => { + expect(retryDelayMs(0, () => 0.5)).toBe(1_000); + expect(retryDelayMs(1, () => 0.5)).toBe(2_000); + expect(retryDelayMs(5, () => 0.5)).toBe(32_000); + expect(retryDelayMs(6, () => 0.5)).toBe(60_000); + expect(retryDelayMs(20, () => 0)).toBe(48_000); + expect(retryDelayMs(20, () => 1)).toBe(72_000); + }); + + test("inserts each table with its persisted stable deduplication token", async () => { + const spool = new StrategyArchiveSpool({ path: ":memory:" }); + spool.admit(batch()); + const calls: Array<{ table: string; token?: string }> = []; + const inserter: RowInserter = async (table, _rows, options) => { + calls.push({ table, token: options?.deduplicationToken }); + }; + const worker = new StrategySpoolWorker({ spool, inserter }); + + expect(await worker.drainOnce()).toEqual({ + completed: 2, + retried: 0, + terminal: 0, + expired: 0, + }); + expect(calls).toHaveLength(2); + expect(calls.every((call) => /^[a-f0-9]{64}$/.test(call.token ?? ""))).toBe( + true, + ); + expect(spool.stats().queuedBatches).toBe(0); + spool.close(); + }); + + test("retries only the failed sibling after the first delay", async () => { + let now = 10_000; + const spool = new StrategyArchiveSpool({ + path: ":memory:", + now: () => now, + }); + spool.admit(batch()); + const attempts = new Map(); + const inserter: RowInserter = async (table) => { + attempts.set(table, (attempts.get(table) ?? 0) + 1); + if ( + table === "strategy_data.policy_evaluation_events" && + attempts.get(table) === 1 + ) { + throw new Error("connection reset"); + } + }; + const worker = new StrategySpoolWorker({ + spool, + inserter, + now: () => now, + random: () => 0.5, + }); + + expect(await worker.drainOnce()).toMatchObject({ + completed: 1, + retried: 1, + }); + expect(spool.stats()).toMatchObject({ queuedBatches: 1, queuedWork: 1 }); + expect(await worker.drainOnce()).toMatchObject({ + completed: 0, + retried: 0, + }); + now += 1_000; + expect(await worker.drainOnce()).toMatchObject({ + completed: 1, + retried: 0, + }); + expect(attempts.get("strategy_data.inventory_settlement_events")).toBe(1); + expect(attempts.get("strategy_data.policy_evaluation_events")).toBe(2); + spool.close(); + }); + + test("makes schema and authentication failures terminal without hot-looping", async () => { + const spool = new StrategyArchiveSpool({ path: ":memory:" }); + spool.admit({ ...batch(), rows: [batch().rows[0]] }); + let calls = 0; + const worker = new StrategySpoolWorker({ + spool, + inserter: async () => { + calls += 1; + throw new Error("unknown column archive_event_id"); + }, + }); + expect(await worker.drainOnce()).toMatchObject({ terminal: 1 }); + expect(await worker.drainOnce()).toMatchObject({ terminal: 0, retried: 0 }); + expect(calls).toBe(1); + expect(spool.stats().terminalWork).toBe(1); + spool.close(); + }); +}); diff --git a/test/archive-forwarder.test.ts b/test/archive-forwarder.test.ts index f88daa5..23e235c 100644 --- a/test/archive-forwarder.test.ts +++ b/test/archive-forwarder.test.ts @@ -66,6 +66,63 @@ describe("archive forwarder batch parsing", () => { expect(parsed.batch.rows).toHaveLength(1); }); + test("rejects rows whose source disagrees with the archive envelope", () => { + const parsed = parseArchiveBatchRequest({ + source: "broker_read", + deployment_id: "deploy-a", + rows: [ + { + table: "market_data.cex_trades", + row: { source: "broker_write", trade_id: "t-1" }, + }, + ], + }); + + expect(parsed.ok).toBe(true); + if (!parsed.ok) return; + expect(parsed.batch.rows).toHaveLength(0); + expect(parsed.rejectedRowCount).toBe(1); + expect(parsed.rejectedRowsByTable).toEqual({ + "market_data.cex_trades": 1, + }); + }); + + test("rejects same-batch order-book logical-key checksum conflicts", () => { + const common = { + source: "broker_read", + capture_bundle_id: "bundle-a", + exchange: "binance", + trading_pair: "BTC-USDT", + raw_capture_id: "raw-a", + snapshot_id: "snapshot-a", + schema_version: "1.0.0", + side: "bid", + level_index: 0, + }; + const parsed = parseArchiveBatchRequest({ + source: "broker_read", + deployment_id: "deploy-a", + rows: [ + { + table: "market_data.cex_order_book_levels", + row: { ...common, normalized_row_checksum: "a" }, + }, + { + table: "market_data.cex_order_book_levels", + row: { ...common, normalized_row_checksum: "b" }, + }, + ], + }); + + expect(parsed.ok).toBe(true); + if (!parsed.ok) return; + expect(parsed.batch.rows).toHaveLength(0); + expect(parsed.rejectedRowCount).toBe(2); + expect(parsed.checksumConflictsByTable).toEqual({ + "market_data.cex_order_book_levels": 2, + }); + }); + test("accepts the broker execution and account balance archive tables", () => { expect(isSupportedTable("broker_execution.transfer_events")).toBe(true); expect(isSupportedTable("broker_execution.fill_events")).toBe(true); @@ -369,6 +426,9 @@ describe("archive forwarder schema init", () => { .filter((name): name is string => Boolean(name)); expect(createdTables).toEqual( expect.arrayContaining([ + "market_data.cex_order_book_levels", + "market_data.cex_order_book_depth_summary", + "market_data.cex_ohlcv", "broker_execution.order_events", "broker_execution.market_metadata_snapshots", "broker_execution.transfer_events", @@ -381,6 +441,28 @@ describe("archive forwarder schema init", () => { "strategy_data.inventory_settlement_events", ]), ); + const marketSchema = statements.join("\n"); + expect(marketSchema).toContain( + "ENGINE = ReplacingMergeTree(broker_version)", + ); + expect(marketSchema).toContain( + "CREATE VIEW IF NOT EXISTS market_data.cex_order_book_levels_canonical", + ); + expect(marketSchema).toContain( + "CREATE VIEW IF NOT EXISTS market_data.cex_order_book_levels_conflicts", + ); + expect(marketSchema).toContain( + "CREATE VIEW IF NOT EXISTS market_data.cex_order_book_depth_summary_canonical", + ); + expect(marketSchema).toContain( + "CREATE VIEW IF NOT EXISTS market_data.cex_order_book_depth_summary_conflicts", + ); + expect(marketSchema).toContain( + "ORDER BY (exchange, trading_pair, capture_bundle_id, source_time_ms, raw_capture_id, snapshot_id, schema_version, side, level_index)", + ); + expect(marketSchema).toContain( + "ALTER TABLE market_data.cex_stream_events ADD COLUMN IF NOT EXISTS capture_bundle_id", + ); const balanceTable = statements.find((query) => /CREATE TABLE IF NOT EXISTS\s+broker_account\.balance_snapshots/i.test( diff --git a/test/broker-execution-archive.test.ts b/test/broker-execution-archive.test.ts index cbda363..757af50 100644 --- a/test/broker-execution-archive.test.ts +++ b/test/broker-execution-archive.test.ts @@ -45,6 +45,7 @@ import { isArchiveOtelLogsEnabled, isBrokerExecutionArchiveTable, resolveArchiveForwarderUrlFromEnv, + resolveArchiveSourceFromEnv, rethrowArchiveDurabilityError, } from "../src/helpers/broker-execution-archive/writer"; import { Action } from "../src/helpers/constants"; @@ -1197,6 +1198,7 @@ describe("broker execution archiver queue", () => { try { const archiver = BrokerExecutionArchiver.create({ + source: "broker_read", forwarderUrl: server.url, deadLetterPath: createDeadLetterPath(), deploymentId: "test-deploy", @@ -1217,9 +1219,16 @@ describe("broker execution archiver queue", () => { expect(request?.headers["content-type"]).toBe("application/json"); expect(request?.headers.authorization).toBe("Bearer secret-token"); expect(request?.body).toMatchObject({ - source: "broker_write", + source: "broker_read", deployment_id: "test-deploy", + rows: [ + { + table: "broker_execution.order_events", + row: { source: "broker_read", order_id: "1" }, + }, + ], }); + expect(archiver.getSource()).toBe("broker_read"); await archiver.close(); } finally { @@ -1238,6 +1247,7 @@ describe("broker execution archiver queue", () => { const otelLogs = new MockOtelLogs(); const deadLetterPath = createDeadLetterPath(); const archiver = BrokerExecutionArchiver.create({ + source: "broker_read", forwarderUrl: server.url, deadLetterPath, otelLogs, @@ -1264,11 +1274,12 @@ describe("broker execution archiver queue", () => { expect(archiver.getQueueDepth()).toBe(2); const [loss] = readDeadLetters(deadLetterPath); expect(loss).toMatchObject({ + source: "broker_read", deployment_id: "test-deploy", reason: "queue_shed", payload: { table: "broker_execution.order_events", - row: { source: "broker_write", order_id: "1" }, + row: { source: "broker_read", order_id: "1" }, }, }); expect(new Date(String(loss?.timestamp)).toISOString()).toBe( @@ -1550,6 +1561,13 @@ describe("broker execution archiver queue", () => { }); describe("broker execution archiver env", () => { + test("uses closed archive source configuration with broker_write compatibility default", () => { + expect(resolveArchiveSourceFromEnv(undefined)).toBe("broker_write"); + expect(resolveArchiveSourceFromEnv("broker_read")).toBe("broker_read"); + expect(() => resolveArchiveSourceFromEnv("request_inferred")).toThrow( + "broker_read or broker_write", + ); + }); test("only the exact true enable flag enables archive construction", async () => { const originalEnabled = process.env.CEX_BROKER_ARCHIVE_ENABLED; const originalForwarderUrl = process.env.CEX_BROKER_ARCHIVE_FORWARDER_URL; @@ -1687,6 +1705,7 @@ describe("broker execution archiver env", () => { ); expect(enabledCall?.[1]).toEqual({ enabled: true, + source: "broker_write", otel_mirror_enabled: true, }); const startupOutput = JSON.stringify(enabledCall); diff --git a/test/canonical-market-data-contract.test.ts b/test/canonical-market-data-contract.test.ts new file mode 100644 index 0000000..91c37f3 --- /dev/null +++ b/test/canonical-market-data-contract.test.ts @@ -0,0 +1,499 @@ +import { describe, expect, test } from "bun:test"; +import { + buildCanonicalOrderBookRows, + OrderBookValidationError, +} from "../src/helpers/market-data-archive/canonical-orderbook"; +import { + createMarketCaptureContext, + validateExternalFallbackContext, + validateProductionCollectorArchive, +} from "../src/helpers/market-data-archive/capture-context"; +import { + ARCHIVE_SOURCES, + CAPTURE_FEEDS, + CHECKSUM_ALGORITHM, + CONSTRUCTION_MODES, + canonicalDecimal, + canonicalSerialize, + createRawCapture, + GAP_POLICIES, + MARKET_CAPTURE_SCHEMA_VERSION, + RAW_CAPTURE_SCOPES, + SOURCE_MODES, + sha256Canonical, +} from "../src/helpers/market-data-archive/capture-contract"; +import { + buildLegacyOhlcvMigrationRow, + buildLegacyOrderBookMigrationRows, +} from "../src/helpers/market-data-archive/legacy-migration"; +import { + buildCanonicalCexStreamEventRow, + buildCanonicalOhlcvRow, + buildCanonicalTickerEventRow, + buildCanonicalTradeRow, +} from "../src/helpers/market-data-archive/rows"; +import type { MarketCaptureContext } from "../src/helpers/market-data-archive/types"; +import type { NormalizedOrderBookSnapshot } from "../src/helpers/order-book"; + +const captureContext: MarketCaptureContext = { + source: "broker_read", + deploymentId: "collector-eu-1", + captureBundleId: "bundle-2026-08-03", + exchange: "binance", + symbol: "BTC/USDT", + assetType: "spot", + feed: "ORDERBOOK", + provider: "ccxt:binance", + sourceMode: "broker_live_sampling_v1", + schemaVersion: MARKET_CAPTURE_SCHEMA_VERSION, + checksumAlgorithm: CHECKSUM_ALGORITHM, + provenanceComplete: true, +}; + +function snapshot( + overrides: Partial = {}, +): NormalizedOrderBookSnapshot { + return { + bids: [ + [100, 1], + [99, 2], + ], + asks: [ + [101, 3], + [102, 4], + ], + timestamp: 1_700_000_000_000, + receivedTimestamp: 1_700_000_000_125, + exchange: "binance", + symbol: "BTC/USDT", + depthLimit: 2, + sequence: 42, + ...overrides, + }; +} + +describe("canonical market capture contract", () => { + test("production contexts require a deployment-owned capture bundle", () => { + expect(() => + createMarketCaptureContext({ + source: "broker_read", + deploymentId: "collector-a", + exchange: "binance", + symbol: "BTC/USDT", + assetType: "spot", + feed: "TRADES", + sourceMode: "broker_live_stream_v1", + environment: "production", + }), + ).toThrow("capture_bundle_id"); + const development = createMarketCaptureContext({ + source: "broker_write", + deploymentId: "local-a", + exchange: "binance", + symbol: "BTC/USDT", + assetType: "spot", + feed: "TRADES", + sourceMode: "broker_live_stream_v1", + environment: "development", + }); + expect(development.captureBundleId).toBe("development:local-a"); + }); + + test("FIET-901 production collector requires broker_read", () => { + expect(() => + validateProductionCollectorArchive({ + source: "broker_write", + captureBundleId: "bundle-a", + }), + ).toThrow("broker_read"); + expect(() => + validateProductionCollectorArchive({ + source: "broker_read", + captureBundleId: "bundle-a", + }), + ).not.toThrow(); + }); + + test("external fallback rows cannot cross venue or omit their reason", () => { + expect(() => + validateExternalFallbackContext({ + configuredExchange: "binance", + configuredSymbol: "BTC/USDT", + rowExchange: "kraken", + rowSymbol: "BTC/USDT", + provider: "hummingbot:kraken", + sourceMode: "external_hummingbot_fallback_v1", + fallbackReason: "primary feed unavailable", + }), + ).toThrow("cross-venue"); + expect(() => + validateExternalFallbackContext({ + configuredExchange: "binance", + configuredSymbol: "BTC/USDT", + rowExchange: "binance", + rowSymbol: "BTC/USDT", + provider: "ccxt:binance", + sourceMode: "external_ccxt_fallback_v1", + fallbackReason: "", + }), + ).toThrow("fallback reason"); + }); + + test("uses closed, versioned registries", () => { + expect(ARCHIVE_SOURCES).toEqual(["broker_read", "broker_write"]); + expect(CAPTURE_FEEDS).toEqual(["ORDERBOOK", "TICKER", "TRADES", "OHLCV"]); + expect(SOURCE_MODES).toContain("legacy_migration_v1"); + expect(CONSTRUCTION_MODES).toContain("sampled_top_n_snapshot"); + expect(GAP_POLICIES).toContain("record_gap"); + expect(RAW_CAPTURE_SCOPES).toContain("ccxt_normalized_object"); + }); + + test("canonical serialization sorts keys, normalizes numbers, and omits undefined", () => { + expect( + canonicalSerialize({ z: -0, b: 1e-7, omitted: undefined, a: [2, 1.5] }), + ).toBe('{"a":[2,1.5],"b":0.0000001,"z":0}'); + expect(canonicalDecimal(1e-7)).toBe("0.0000001"); + expect(() => canonicalSerialize({ bad: Number.NaN })).toThrow("finite"); + }); + + test("checksums are deterministic and exclude checksum fields", () => { + const left = sha256Canonical({ + b: 2, + normalized_row_checksum: "old-a", + a: 1, + }); + const right = sha256Canonical({ + a: 1, + b: 2, + normalized_row_checksum: "old-b", + }); + expect(left).toBe(right); + expect(left).toHaveLength(64); + }); + + test("raw captures redact secrets and reproduce identity", () => { + const first = createRawCapture(captureContext, { + payload: { apiSecret: "do-not-retain", bids: [[100, 1]] }, + eventTimeMs: 1_700_000_000_000, + receivedTimeMs: 1_700_000_000_125, + scope: "ccxt_normalized_object", + }); + const retried = createRawCapture(captureContext, { + payload: { bids: [[100, 1]], apiSecret: "different-secret" }, + eventTimeMs: 1_700_000_000_000, + receivedTimeMs: 1_700_000_000_125, + scope: "ccxt_normalized_object", + }); + + expect(JSON.stringify(first.redactedPayload)).not.toContain( + "do-not-retain", + ); + expect(first.rawCaptureId).toBe(retried.rawCaptureId); + expect(first.rawChecksum).toBe(retried.rawChecksum); + const changedEvidence = createRawCapture(captureContext, { + payload: { bids: [[99, 1]] }, + eventTimeMs: 1_700_000_000_000, + receivedTimeMs: 1_700_000_000_125, + scope: "ccxt_normalized_object", + }); + expect(changedEvidence.rawCaptureId).not.toBe(first.rawCaptureId); + }); + + test("canonical raw, ticker, trade, and OHLCV rows share capture linkage", () => { + const tickerContext = { ...captureContext, feed: "TICKER" as const }; + const raw = createRawCapture(tickerContext, { + payload: { last: 100, bid: 99, ask: 101 }, + eventTimeMs: 1_700_000_000_000, + receivedTimeMs: 1_700_000_000_125, + scope: "ccxt_normalized_object", + }); + const stream = buildCanonicalCexStreamEventRow(tickerContext, raw); + const ticker = buildCanonicalTickerEventRow(tickerContext, raw, { + eventTimeMs: 1_700_000_000_000, + last: 100, + bid: 99, + ask: 101, + }); + expect(stream.row.raw_capture_id).toBe(raw.rawCaptureId); + expect(ticker.row.raw_capture_id).toBe(raw.rawCaptureId); + expect(String(stream.row.normalized_row_checksum)).toHaveLength(64); + expect(String(ticker.row.normalized_row_checksum)).toHaveLength(64); + + const tradeContext = { ...captureContext, feed: "TRADES" as const }; + const tradeRaw = createRawCapture(tradeContext, { + payload: [{ id: "trade-1", price: 100, amount: 0.5 }], + eventTimeMs: 1_700_000_000_001, + receivedTimeMs: 1_700_000_000_125, + scope: "ccxt_normalized_object", + }); + const trade = buildCanonicalTradeRow(tradeContext, tradeRaw, { + tradeId: "trade-1", + eventTimeMs: 1_700_000_000_001, + side: "buy", + price: 100, + amount: 0.5, + }); + expect(trade.table).toBe("market_data.cex_trades"); + + const ohlcvContext = { + ...captureContext, + feed: "OHLCV" as const, + sourceMode: "broker_bootstrap_fetch_v1" as const, + timeframe: "1m", + }; + const ohlcvRaw = createRawCapture(ohlcvContext, { + payload: [[1_700_000_000_000, 1, 2, 0.5, 1.5, 10]], + eventTimeMs: 1_700_000_000_000, + receivedTimeMs: 1_700_000_000_125, + scope: "ccxt_normalized_object", + }); + const ohlcv = buildCanonicalOhlcvRow({ + context: ohlcvContext, + rawCapture: ohlcvRaw, + bar: { + openTimeMs: 1_700_000_000_000, + open: 1, + high: 2, + low: 0.5, + close: 1.5, + volume: 10, + }, + isClosed: true, + brokerVersion: 1_700_000_000_125, + }); + expect(ohlcv.table).toBe("market_data.cex_ohlcv"); + expect(ohlcv.row).toMatchObject({ + source_mode: "broker_bootstrap_fetch_v1", + is_closed: 1, + raw_capture_id: ohlcvRaw.rawCaptureId, + }); + }); + + test("OHLCV live updates retain one replacement key while changing evidence versions", () => { + const context = { + ...captureContext, + feed: "OHLCV" as const, + sourceMode: "broker_live_stream_v1" as const, + timeframe: "1m", + }; + const formingBar = { + openTimeMs: 1_700_000_000_000, + open: 100, + high: 101, + low: 99, + close: 100.5, + volume: 10, + }; + const formingRaw = createRawCapture(context, { + payload: formingBar, + eventTimeMs: formingBar.openTimeMs, + receivedTimeMs: 1_700_000_030_000, + scope: "ccxt_normalized_object", + }); + const forming = buildCanonicalOhlcvRow({ + context, + rawCapture: formingRaw, + bar: formingBar, + isClosed: false, + brokerVersion: 1_700_000_030_000, + }); + const closedBar = { ...formingBar, high: 102, close: 101, volume: 14 }; + const closedRaw = createRawCapture(context, { + payload: closedBar, + eventTimeMs: closedBar.openTimeMs, + receivedTimeMs: 1_700_000_060_001, + scope: "ccxt_normalized_object", + }); + const closed = buildCanonicalOhlcvRow({ + context, + rawCapture: closedRaw, + bar: closedBar, + isClosed: true, + brokerVersion: 1_700_000_060_001, + }); + + for (const logicalKey of [ + "exchange", + "trading_pair", + "timeframe", + "open_time_ms", + "schema_version", + ]) { + expect(closed.row[logicalKey]).toBe(forming.row[logicalKey]); + } + expect(forming.row.source_mode).toBe("broker_live_stream_v1"); + expect(forming.row.is_closed).toBe(0); + expect(closed.row.is_closed).toBe(1); + expect(Number(closed.row.broker_version)).toBeGreaterThan( + Number(forming.row.broker_version), + ); + expect(closed.row.raw_capture_id).not.toBe(forming.row.raw_capture_id); + expect(closed.row.normalized_row_checksum).not.toBe( + forming.row.normalized_row_checksum, + ); + }); +}); + +describe("canonical order-book normalization", () => { + test("emits one checksummed row per retained level and one summary", () => { + const raw = createRawCapture(captureContext, { + payload: snapshot(), + eventTimeMs: 1_700_000_000_000, + receivedTimeMs: 1_700_000_000_125, + scope: "ccxt_normalized_object", + }); + const result = buildCanonicalOrderBookRows({ + context: captureContext, + snapshot: snapshot(), + rawCapture: raw, + depthLimit: 2, + measurementBandsBps: [100, 50, 100], + }); + + expect(result.levels).toHaveLength(4); + expect(result.levels[0]?.table).toBe("market_data.cex_order_book_levels"); + expect(result.levels.map((entry) => entry.row.side)).toEqual([ + "bid", + "bid", + "ask", + "ask", + ]); + expect(result.levels[0]?.row).toMatchObject({ + capture_bundle_id: "bundle-2026-08-03", + raw_capture_id: raw.rawCaptureId, + construction_mode: "sampled_top_n_snapshot", + gap_policy: "record_gap", + level_index: 0, + price: 100, + amount: 1, + notional: 100, + }); + expect(result.summary.table).toBe( + "market_data.cex_order_book_depth_summary", + ); + expect(result.summary.row.measurement_bands_bps).toEqual([50, 100]); + expect(result.summary.row.bid_depth_by_band).toEqual([1, 3]); + expect(result.summary.row.ask_depth_by_band).toEqual([3, 7]); + expect(String(result.summary.row.normalized_row_checksum)).toHaveLength(64); + + const retried = buildCanonicalOrderBookRows({ + context: captureContext, + snapshot: snapshot({ receivedTimestamp: 1_700_000_009_999 }), + rawCapture: raw, + depthLimit: 2, + measurementBandsBps: [50, 100], + }); + expect(retried.snapshotId).toBe(result.snapshotId); + }); + + test.each([ + { name: "missing side", overrides: { asks: [] } }, + { name: "zero amount", overrides: { bids: [[100, 0]] } }, + { + name: "non-monotonic bids", + overrides: { + bids: [ + [99, 1], + [100, 1], + ], + }, + }, + { name: "locked book", overrides: { bids: [[101, 1]] } }, + { name: "invalid timestamp", overrides: { timestamp: -1 } }, + ])("rejects the whole $name snapshot", ({ overrides }) => { + const book = snapshot(overrides as Partial); + const raw = createRawCapture(captureContext, { + payload: book, + eventTimeMs: 1_700_000_000_000, + receivedTimeMs: 1_700_000_000_125, + scope: "ccxt_normalized_object", + }); + expect(() => + buildCanonicalOrderBookRows({ + context: captureContext, + snapshot: book, + rawCapture: raw, + depthLimit: 2, + }), + ).toThrow(OrderBookValidationError); + }); + + test("never emits exact L2 without continuity proof", () => { + const raw = createRawCapture(captureContext, { + payload: snapshot(), + eventTimeMs: 1_700_000_000_000, + receivedTimeMs: 1_700_000_000_125, + scope: "ccxt_normalized_object", + }); + expect(() => + buildCanonicalOrderBookRows({ + context: captureContext, + snapshot: snapshot(), + rawCapture: raw, + depthLimit: 2, + constructionMode: "exact_l2_reconstruction", + }), + ).toThrow("continuity proof"); + }); +}); + +describe("legacy canonical migration", () => { + test("leaves unavailable provenance null and is deterministic on rerun", () => { + const legacy = { + source: "broker_write" as const, + deployment_id: "legacy-deploy", + exchange: "binance", + asset_type: "spot" as const, + symbol: "BTC/USDT", + event_time_ms: 1_700_000_000_000, + received_time_ms: 1_700_000_000_010, + depth_limit: 2, + bids_price: [100, 99], + bids_size: [1, 2], + asks_price: [101, 102], + asks_size: [3, 4], + sequence: 42, + }; + const first = buildLegacyOrderBookMigrationRows(legacy); + const second = buildLegacyOrderBookMigrationRows(legacy); + expect(second).toEqual(first); + expect(first).toHaveLength(5); + for (const { row } of first) { + expect(row).toMatchObject({ + capture_bundle_id: null, + raw_capture_id: null, + raw_capture_scope: null, + raw_checksum: null, + provenance_complete: 0, + source_mode: "legacy_migration_v1", + }); + } + }); + + test("migrates candles into cex_ohlcv without invented raw evidence", () => { + const migrated = buildLegacyOhlcvMigrationRow({ + deployment_id: "legacy-deploy", + exchange: "binance", + asset_type: "spot", + symbol: "BTC/USDT", + timeframe: "1m", + open_time_ms: 1_700_000_000_000, + open: 100, + high: 102, + low: 99, + close: 101, + volume: 12, + is_closed: 1, + broker_version: 7, + }); + expect(migrated.table).toBe("market_data.cex_ohlcv"); + expect(migrated.row).toMatchObject({ + capture_bundle_id: null, + raw_capture_id: null, + raw_checksum: null, + provenance_complete: 0, + source_mode: "legacy_migration_v1", + is_closed: 1, + }); + }); +}); diff --git a/test/canonical-market-data-fixtures.test.ts b/test/canonical-market-data-fixtures.test.ts new file mode 100644 index 0000000..c02b781 --- /dev/null +++ b/test/canonical-market-data-fixtures.test.ts @@ -0,0 +1,147 @@ +import { describe, expect, test } from "bun:test"; +import { buildCanonicalOrderBookRows } from "../src/helpers/market-data-archive/canonical-orderbook"; +import { createRawCapture } from "../src/helpers/market-data-archive/capture-contract"; +import { + buildCanonicalCexStreamEventRow, + buildCanonicalOhlcvRow, + buildCanonicalTickerEventRow, + buildCanonicalTradeRow, +} from "../src/helpers/market-data-archive/rows"; +import type { + CaptureFeed, + CaptureSourceMode, + MarketCaptureContext, +} from "../src/helpers/market-data-archive/types"; +import fixture from "./fixtures/canonical-market-capture-v1.json" with { + type: "json", +}; + +type FixtureCapture = { + feed: CaptureFeed; + sourceMode: CaptureSourceMode; + timeframe?: string; + payload: unknown; + eventTimeMs: number; + receivedTimeMs: number; + expected: Record; +}; + +function context(capture: FixtureCapture): MarketCaptureContext { + return { + ...fixture.context, + source: "broker_read", + assetType: "spot", + feed: capture.feed, + sourceMode: capture.sourceMode, + timeframe: capture.timeframe, + }; +} + +function raw(capture: FixtureCapture) { + return createRawCapture(context(capture), { + payload: capture.payload, + eventTimeMs: capture.eventTimeMs, + receivedTimeMs: capture.receivedTimeMs, + scope: "ccxt_normalized_object", + }); +} + +function expectIdentity( + capture: FixtureCapture, + actual: { rawCaptureId: string; rawChecksum: string }, + rowChecksum: unknown, +): void { + expect(actual.rawCaptureId).toBe(capture.expected.raw_capture_id); + expect(actual.rawChecksum).toBe(capture.expected.raw_checksum); + expect(rowChecksum).toBe(capture.expected.normalized_row_checksum); +} + +describe("canonical market-data golden fixture v1", () => { + test("reproduces stream, ticker, trade, and OHLCV identities", () => { + const stream = fixture.stream as FixtureCapture; + const streamRaw = raw(stream); + expectIdentity( + stream, + streamRaw, + buildCanonicalCexStreamEventRow(context(stream), streamRaw).row + .normalized_row_checksum, + ); + + const ticker = fixture.ticker as FixtureCapture & { + normalized: Parameters[2]; + }; + const tickerRaw = raw(ticker); + expectIdentity( + ticker, + tickerRaw, + buildCanonicalTickerEventRow( + context(ticker), + tickerRaw, + ticker.normalized, + ).row.normalized_row_checksum, + ); + + const trade = fixture.trade as FixtureCapture & { + normalized: Parameters[2]; + }; + const tradeRaw = raw(trade); + expectIdentity( + trade, + tradeRaw, + buildCanonicalTradeRow(context(trade), tradeRaw, trade.normalized).row + .normalized_row_checksum, + ); + + const ohlcv = fixture.ohlcv as FixtureCapture & { + bar: Parameters[0]["bar"]; + isClosed: boolean; + brokerVersion: number; + }; + const ohlcvRaw = raw(ohlcv); + expectIdentity( + ohlcv, + ohlcvRaw, + buildCanonicalOhlcvRow({ + context: context(ohlcv), + rawCapture: ohlcvRaw, + bar: ohlcv.bar, + isClosed: ohlcv.isClosed, + brokerVersion: ohlcv.brokerVersion, + }).row.normalized_row_checksum, + ); + }); + + test("reproduces order-book level and summary identities", () => { + const capture = fixture.orderbook as FixtureCapture & { + depthLimit: number; + payload: { + bids: number[][]; + asks: number[][]; + timestamp: number; + sequence: number; + }; + }; + const rawCapture = raw(capture); + const rows = buildCanonicalOrderBookRows({ + context: context(capture), + rawCapture, + depthLimit: capture.depthLimit, + snapshot: { + ...capture.payload, + receivedTimestamp: capture.receivedTimeMs, + exchange: fixture.context.exchange, + symbol: fixture.context.symbol, + depthLimit: capture.depthLimit, + }, + }); + expect(rawCapture.rawCaptureId).toBe(capture.expected.raw_capture_id); + expect(rawCapture.rawChecksum).toBe(capture.expected.raw_checksum); + expect(rows.snapshotId).toBe(capture.expected.snapshot_id); + expect(rows.levels[0]?.row.normalized_row_checksum).toBe( + capture.expected.level_normalized_row_checksum, + ); + expect(rows.summary.row.normalized_row_checksum).toBe( + capture.expected.summary_normalized_row_checksum, + ); + }); +}); diff --git a/test/canonical-market-data-migration.test.ts b/test/canonical-market-data-migration.test.ts new file mode 100644 index 0000000..3dd4cd9 --- /dev/null +++ b/test/canonical-market-data-migration.test.ts @@ -0,0 +1,111 @@ +import { describe, expect, test } from "bun:test"; +import { parseReplayValidationConfig } from "../scripts/validate-canonical-market-replay"; + +const migrationPath = new URL( + "../schema/clickhouse/migrations/canonical_market_data_replay_cutover.sql", + import.meta.url, +); +const replayPath = new URL( + "../schema/clickhouse/canonical_market_data_replay.sql", + import.meta.url, +); +const tableMigrationPath = new URL( + "../scripts/migrate-legacy-market-data-to-canonical.ts", + import.meta.url, +); + +function executableSql(sql: string): string { + return sql + .split("\n") + .map((line) => line.replace(/--.*$/, "")) + .join("\n"); +} + +describe("canonical market-data migration contracts", () => { + test("replay validation config covers explicit strategy pair windows", () => { + expect( + parseReplayValidationConfig({ + windows: [ + { + captureBundleIds: ["bundle-a"], + exchange: "binance", + tradingPair: "BTC-USDT", + startTimeMs: 100, + endTimeMs: 200, + }, + { + captureBundleIds: ["bundle-a", "bundle-b"], + exchange: "coinbase", + tradingPair: "ETH-USD", + startTimeMs: 200, + endTimeMs: 300, + }, + ], + }), + ).toHaveLength(2); + expect(() => parseReplayValidationConfig({ windows: [] })).toThrow( + "validation window", + ); + expect(() => + parseReplayValidationConfig({ + windows: [ + { + captureBundleIds: [], + exchange: "binance", + tradingPair: "BTC-USDT", + startTimeMs: 200, + endTimeMs: 100, + }, + ], + }), + ).toThrow("Invalid replay validation config"); + }); + + test("cutover requires a table migration before canonical-only deployment", async () => { + const migration = await Bun.file(migrationPath).text(); + expect(executableSql(migration)).not.toMatch(/\bDROP\b/i); + expect(migration).not.toContain("CEX_BROKER_MARKET_ARCHIVE_WRITE_MODE"); + expect(migration).not.toContain("dual-write"); + expect(migration).toContain("quiesce legacy writers"); + expect(migration).toContain( + "scripts/migrate-legacy-market-data-to-canonical.ts", + ); + expect(migration).toContain("market_data.orderbook_snapshots_legacy"); + expect(migration).toContain("market_data.candles_legacy"); + expect(migration).toContain("market_data.candles_closed_legacy"); + expect(migration).toContain("ROLLBACK"); + expect(migration).toContain("rename—not drop"); + }); + + test("table migration is bounded, dry-run by default, and reads ClickHouse only", async () => { + const migration = await Bun.file(tableMigrationPath).text(); + expect(migration).toContain("CEX_BROKER_MIGRATION_START_TIME_MS"); + expect(migration).toContain("CEX_BROKER_MIGRATION_END_TIME_MS"); + expect(migration).toContain("CEX_BROKER_CANONICAL_MIGRATION_CONFIRM"); + expect(migration).toContain('readLegacyRows("orderbook_snapshots"'); + expect(migration).toContain('readLegacyRows("candles"'); + expect(migration).toContain("buildLegacyOrderBookMigrationRows"); + expect(migration).toContain("buildLegacyOhlcvMigrationRow"); + expect(migration).not.toContain("createBroker("); + expect(migration).not.toContain("Bun.file("); + }); + + test("replay is windowed and conflict-preflighted before canonical reads", async () => { + const replay = await Bun.file(replayPath).text(); + expect(replay.indexOf("cex_order_book_levels_conflicts")).toBeLessThan( + replay.indexOf("cex_order_book_levels_canonical"), + ); + expect( + replay.indexOf("cex_order_book_depth_summary_conflicts"), + ).toBeLessThan(replay.indexOf("cex_order_book_depth_summary_canonical")); + for (const filter of [ + "capture_bundle_ids", + "exchange", + "trading_pair", + "start_time_ms", + "end_time_ms", + ]) { + expect(replay).toContain(filter); + } + }); +}); diff --git a/test/clickhouse-schema.integration.test.ts b/test/clickhouse-schema.integration.test.ts index 4c1f6cc..b7b4525 100644 --- a/test/clickhouse-schema.integration.test.ts +++ b/test/clickhouse-schema.integration.test.ts @@ -1,12 +1,31 @@ import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { type ClickHouseClient, createClient } from "@clickhouse/client"; +import { + exportCanonicalOrderBookParquet, + validateCanonicalMarketReplayWindow, +} from "../scripts/export-canonical-orderbook-parquet"; import { createClickHouseInserter } from "../services/archive-forwarder/insert"; import { handleArchiveBatch } from "../services/archive-forwarder/router"; import { ensureArchiveSchema } from "../services/archive-forwarder/schema"; +import { buildCanonicalOrderBookRows } from "../src/helpers/market-data-archive/canonical-orderbook"; +import { createRawCapture } from "../src/helpers/market-data-archive/capture-contract"; +import { buildLegacyOhlcvMigrationRow } from "../src/helpers/market-data-archive/legacy-migration"; +import { + buildCanonicalCexStreamEventRow, + buildCanonicalOhlcvRow, + buildCanonicalTickerEventRow, + buildCanonicalTradeRow, +} from "../src/helpers/market-data-archive/rows"; +import type { MarketCaptureContext } from "../src/helpers/market-data-archive/types"; const CLICKHOUSE_URL = process.env.CLICKHOUSE_TEST_URL?.trim() || `http://${process.env.CLICKHOUSE_HOST?.trim() || "localhost"}:${process.env.CLICKHOUSE_PORT?.trim() || "18123"}`; +const CLICKHOUSE_USERNAME = process.env.CLICKHOUSE_USER?.trim() || "default"; +const CLICKHOUSE_PASSWORD = process.env.CLICKHOUSE_PASSWORD ?? ""; const TEST_DEPLOYMENT = `clickhouse-integration-test-${Date.now()}`; const TEST_EVENT_MS = 1_900_000_000_000; @@ -24,7 +43,8 @@ function requireClient(): ClickHouseClient { async function probeClickHouse(): Promise { const probe = createClient({ url: CLICKHOUSE_URL, - database: "market_data", + username: CLICKHOUSE_USERNAME, + password: CLICKHOUSE_PASSWORD, }); try { const result = await probe.query({ @@ -64,6 +84,20 @@ async function cleanupTestRows(): Promise { `, query_params: { deployment_id: TEST_DEPLOYMENT }, }); + for (const table of [ + "cex_stream_events", + "cex_ticker_events", + "cex_trades", + "cex_order_book_levels", + "cex_order_book_depth_summary", + "cex_ohlcv", + ]) { + await activeClient.command({ + query: `ALTER TABLE ${table} DELETE WHERE deployment_id = {deployment_id:String}`, + query_params: { deployment_id: TEST_DEPLOYMENT }, + }); + await activeClient.command({ query: `OPTIMIZE TABLE ${table} FINAL` }); + } await activeClient.command({ query: ` ALTER TABLE candles @@ -79,8 +113,10 @@ async function cleanupTestRows(): Promise { }); for (const table of [ "policy_evaluation_events", + "strategy_policy_snapshots", "market_identity", "symbol_mapping", + "inventory_settlement_events", ]) { await activeClient.command({ query: ` @@ -99,11 +135,27 @@ describe("ClickHouse market_data schema integration", () => { beforeAll(async () => { clickhouseAvailable = await probeClickHouse(); if (!clickhouseAvailable) { + if (process.env.CLICKHOUSE_REQUIRED === "1") { + throw new Error( + `Required ClickHouse integration service is unavailable at ${CLICKHOUSE_URL}`, + ); + } return; } + const bootstrap = createClient({ + url: CLICKHOUSE_URL, + username: CLICKHOUSE_USERNAME, + password: CLICKHOUSE_PASSWORD, + }); + await bootstrap.command({ + query: "CREATE DATABASE IF NOT EXISTS market_data", + }); + await bootstrap.close(); client = createClient({ url: CLICKHOUSE_URL, database: "market_data", + username: CLICKHOUSE_USERNAME, + password: CLICKHOUSE_PASSWORD, }); await ensureArchiveSchema(client); try { @@ -134,6 +186,428 @@ describe("ClickHouse market_data schema integration", () => { expect(await tableEngine("orderbook_tob")).toBe("View"); expect(await tableEngine("orderbook_depth")).toBe("View"); expect(await tableEngine("candles_closed")).toBe("View"); + expect(await tableEngine("cex_order_book_levels")).toBe("MergeTree"); + expect(await tableEngine("cex_order_book_depth_summary")).toBe("MergeTree"); + expect(await tableEngine("cex_order_book_levels_canonical")).toBe("View"); + expect(await tableEngine("cex_order_book_levels_conflicts")).toBe("View"); + expect(await tableEngine("cex_ohlcv")).toBe("ReplacingMergeTree"); + expect(await tableEngine("cex_ohlcv_closed")).toBe("View"); + }); + + test("canonical views deduplicate agreement, expose conflicts, and accept incomplete legacy provenance", async () => { + if (!clickhouseAvailable || !client) return; + const context: MarketCaptureContext = { + source: "broker_read", + deploymentId: TEST_DEPLOYMENT, + captureBundleId: "integration-bundle", + exchange: "binance", + symbol: "TEST/CANONICAL", + assetType: "spot", + feed: "ORDERBOOK", + provider: "ccxt:binance", + sourceMode: "broker_live_sampling_v1", + schemaVersion: "1.0.0", + checksumAlgorithm: "sha256-canonical-json-v1", + provenanceComplete: true, + }; + const snapshot = { + bids: [[100, 1]], + asks: [[101, 2]], + timestamp: TEST_EVENT_MS + 300_000, + receivedTimestamp: TEST_EVENT_MS + 300_010, + exchange: "binance", + symbol: "TEST/CANONICAL", + depthLimit: 1, + sequence: 7, + }; + const raw = createRawCapture(context, { + payload: snapshot, + eventTimeMs: snapshot.timestamp, + receivedTimeMs: snapshot.receivedTimestamp, + scope: "ccxt_normalized_object", + }); + const canonical = buildCanonicalOrderBookRows({ + context, + snapshot, + rawCapture: raw, + depthLimit: 1, + }); + const duplicateRows = [ + ...canonical.levels, + canonical.summary, + ...canonical.levels, + canonical.summary, + ]; + const inserted = await handleArchiveBatch( + createClickHouseInserter(client), + { + source: "broker_read", + deployment_id: TEST_DEPLOYMENT, + rows: duplicateRows, + }, + ); + expect(inserted.failed).toBe(0); + + const counts = await client.query({ + query: ` + SELECT + (SELECT count() FROM cex_order_book_levels WHERE snapshot_id = {snapshot:String}) AS physical, + (SELECT count() FROM cex_order_book_levels_canonical WHERE snapshot_id = {snapshot:String}) AS canonical + `, + query_params: { snapshot: canonical.snapshotId }, + format: "JSONEachRow", + }); + expect(await counts.json()).toEqual([{ physical: "4", canonical: "2" }]); + + const firstLevel = canonical.levels[0] as (typeof canonical.levels)[number]; + for (const checksum of ["conflict-a", "conflict-b"]) { + await handleArchiveBatch(createClickHouseInserter(client), { + source: "broker_read", + deployment_id: TEST_DEPLOYMENT, + rows: [ + { + ...firstLevel, + row: { + ...firstLevel.row, + normalized_row_checksum: checksum, + }, + }, + ], + }); + } + const conflicts = await client.query({ + query: ` + SELECT + (SELECT count() FROM cex_order_book_levels_conflicts + WHERE snapshot_id = {snapshot:String} AND side = {side:String} + AND level_index = {level_index:UInt16}) AS conflicts, + (SELECT count() FROM cex_order_book_levels_canonical + WHERE snapshot_id = {snapshot:String} AND side = {side:String} + AND level_index = {level_index:UInt16}) AS replay_rows + `, + query_params: { + snapshot: canonical.snapshotId, + side: firstLevel.row.side, + level_index: firstLevel.row.level_index, + }, + format: "JSONEachRow", + }); + expect(await conflicts.json()).toEqual([ + { conflicts: "1", replay_rows: "0" }, + ]); + + const legacy = buildLegacyOhlcvMigrationRow({ + deployment_id: TEST_DEPLOYMENT, + exchange: "binance", + asset_type: "spot", + symbol: "TEST/CANONICAL", + timeframe: "1m", + open_time_ms: TEST_EVENT_MS + 360_000, + open: 1, + high: 2, + low: 0.5, + close: 1.5, + volume: 10, + is_closed: 1, + broker_version: 1, + }); + await handleArchiveBatch(createClickHouseInserter(client), { + source: "broker_write", + deployment_id: TEST_DEPLOYMENT, + rows: [legacy], + }); + const provenance = await client.query({ + query: ` + SELECT isNull(capture_bundle_id) AS bundle_null, + isNull(raw_capture_id) AS raw_null, + isNull(raw_checksum) AS checksum_null, + provenance_complete, source_mode + FROM cex_ohlcv FINAL + WHERE deployment_id = {deployment:String} + `, + query_params: { deployment: TEST_DEPLOYMENT }, + format: "JSONEachRow", + }); + expect(await provenance.json()).toEqual([ + { + bundle_null: 1, + raw_null: 1, + checksum_null: 1, + provenance_complete: 0, + source_mode: "legacy_migration_v1", + }, + ]); + }); + + test("all four feeds retain raw linkage and reproducible canonical checksums", async () => { + if (!clickhouseAvailable || !client) return; + const receivedTimeMs = TEST_EVENT_MS + 500_010; + const captureBundleId = "integration-four-feed-bundle"; + const baseContext = { + source: "broker_read" as const, + deploymentId: TEST_DEPLOYMENT, + captureBundleId, + exchange: "binance", + symbol: "TEST/FOUR-FEED", + assetType: "spot" as const, + provider: "ccxt:binance", + schemaVersion: "1.0.0", + checksumAlgorithm: "sha256-canonical-json-v1", + provenanceComplete: true, + }; + const tickerContext: MarketCaptureContext = { + ...baseContext, + feed: "TICKER", + sourceMode: "broker_live_stream_v1", + }; + const ticker = { + eventTimeMs: TEST_EVENT_MS + 500_000, + last: 100.5, + bid: 100, + ask: 101, + }; + const tickerRaw = createRawCapture(tickerContext, { + payload: ticker, + eventTimeMs: ticker.eventTimeMs, + receivedTimeMs, + scope: "ccxt_normalized_object", + }); + const tickerRows = [ + buildCanonicalCexStreamEventRow(tickerContext, tickerRaw), + buildCanonicalTickerEventRow(tickerContext, tickerRaw, ticker), + ]; + + const tradesContext: MarketCaptureContext = { + ...baseContext, + feed: "TRADES", + sourceMode: "broker_live_stream_v1", + }; + const trade = { + eventTimeMs: TEST_EVENT_MS + 500_001, + tradeId: "integration-trade-1", + side: "buy", + price: 100.5, + amount: 2, + cost: 201, + }; + const tradeRaw = createRawCapture(tradesContext, { + payload: trade, + eventTimeMs: trade.eventTimeMs, + receivedTimeMs, + scope: "ccxt_normalized_object", + }); + const tradeRows = [ + buildCanonicalCexStreamEventRow(tradesContext, tradeRaw), + buildCanonicalTradeRow(tradesContext, tradeRaw, trade), + ]; + + const ohlcvContext: MarketCaptureContext = { + ...baseContext, + feed: "OHLCV", + timeframe: "1m", + sourceMode: "broker_live_stream_v1", + }; + const bar = { + openTimeMs: TEST_EVENT_MS + 480_000, + open: 100, + high: 102, + low: 99, + close: 101, + volume: 10, + }; + const ohlcvRaw = createRawCapture(ohlcvContext, { + payload: bar, + eventTimeMs: bar.openTimeMs, + receivedTimeMs, + scope: "ccxt_normalized_object", + }); + const ohlcvRows = [ + buildCanonicalCexStreamEventRow(ohlcvContext, ohlcvRaw), + buildCanonicalOhlcvRow({ + context: ohlcvContext, + rawCapture: ohlcvRaw, + bar, + isClosed: true, + brokerVersion: receivedTimeMs, + }), + ]; + + const orderBookContext: MarketCaptureContext = { + ...baseContext, + feed: "ORDERBOOK", + sourceMode: "broker_live_sampling_v1", + }; + const snapshot = { + bids: [[100, 1]], + asks: [[101, 2]], + timestamp: TEST_EVENT_MS + 500_002, + receivedTimestamp: receivedTimeMs, + exchange: "binance", + symbol: "TEST/FOUR-FEED", + depthLimit: 1, + sequence: 9, + }; + const orderBookRaw = createRawCapture(orderBookContext, { + payload: snapshot, + eventTimeMs: snapshot.timestamp, + receivedTimeMs, + scope: "ccxt_normalized_object", + }); + const orderBook = buildCanonicalOrderBookRows({ + context: orderBookContext, + snapshot, + rawCapture: orderBookRaw, + depthLimit: 1, + }); + const allRows = [ + ...tickerRows, + ...tradeRows, + ...ohlcvRows, + buildCanonicalCexStreamEventRow(orderBookContext, orderBookRaw), + ...orderBook.levels, + orderBook.summary, + ]; + + const inserted = await handleArchiveBatch( + createClickHouseInserter(client), + { + source: "broker_read", + deployment_id: TEST_DEPLOYMENT, + rows: allRows, + }, + ); + expect(inserted.inserted).toBe(allRows.length); + expect(inserted.failed).toBe(0); + + const links = await client.query({ + query: ` + SELECT feed, raw_capture_id, raw_checksum, normalized_row_checksum + FROM cex_stream_events + WHERE deployment_id = {deployment:String} + AND capture_bundle_id = {bundle:String} + ORDER BY feed + `, + query_params: { + deployment: TEST_DEPLOYMENT, + bundle: captureBundleId, + }, + format: "JSONEachRow", + }); + const rawRows = (await links.json()) as Array<{ + feed: string; + raw_capture_id: string; + raw_checksum: string; + normalized_row_checksum: string; + }>; + expect(rawRows).toHaveLength(4); + const expectedRaw = new Map( + [tickerRaw, tradeRaw, ohlcvRaw, orderBookRaw].map((raw) => [ + raw.rawCaptureId, + raw.rawChecksum, + ]), + ); + for (const row of rawRows) { + expect(row.raw_checksum).toBe(expectedRaw.get(row.raw_capture_id)); + expect(row.normalized_row_checksum).toMatch(/^[a-f0-9]{64}$/); + } + + for (const [table, expected] of [ + ["cex_ticker_events", tickerRows[1]], + ["cex_trades", tradeRows[1]], + ["cex_ohlcv", ohlcvRows[1]], + ["cex_order_book_depth_summary", orderBook.summary], + ] as const) { + const result = await client.query({ + query: ` + SELECT raw_capture_id, normalized_row_checksum + FROM ${table}${table === "cex_ohlcv" ? " FINAL" : ""} + WHERE deployment_id = {deployment:String} + AND capture_bundle_id = {bundle:String} + `, + query_params: { + deployment: TEST_DEPLOYMENT, + bundle: captureBundleId, + }, + format: "JSONEachRow", + }); + const rows = (await result.json()) as Array<{ + raw_capture_id: string; + normalized_row_checksum: string; + }>; + expect(rows).toContainEqual({ + raw_capture_id: expected?.row.raw_capture_id, + normalized_row_checksum: expected?.row.normalized_row_checksum, + }); + } + + const replay = await validateCanonicalMarketReplayWindow({ + clickhouseUrl: CLICKHOUSE_URL, + username: CLICKHOUSE_USERNAME, + password: CLICKHOUSE_PASSWORD, + captureBundleIds: [captureBundleId], + exchange: "binance", + tradingPair: "TEST-FOUR-FEED", + startTimeMs: TEST_EVENT_MS + 480_000, + endTimeMs: TEST_EVENT_MS + 500_100, + }); + expect(replay.rawRowsByFeed).toEqual({ + OHLCV: 1, + ORDERBOOK: 1, + TICKER: 1, + TRADES: 1, + }); + expect(replay.normalizedRows).toEqual({ + levels: 2, + summaries: 1, + tickers: 1, + trades: 1, + ohlcv: 1, + }); + }); + + test("replay validation blocks a configured window with a checksum conflict", async () => { + if (!clickhouseAvailable || !client) return; + await expect( + validateCanonicalMarketReplayWindow({ + clickhouseUrl: CLICKHOUSE_URL, + username: CLICKHOUSE_USERNAME, + password: CLICKHOUSE_PASSWORD, + captureBundleIds: ["integration-bundle"], + exchange: "binance", + tradingPair: "TEST-CANONICAL", + startTimeMs: TEST_EVENT_MS + 300_000, + endTimeMs: TEST_EVENT_MS + 300_100, + }), + ).rejects.toThrow("checksum conflicts"); + }); + + test("exports checksum-consistent order-book capture core as parquet", async () => { + if (!clickhouseAvailable || !client) return; + const outputDirectory = await mkdtemp( + join(tmpdir(), "cex-broker-parquet-test-"), + ); + try { + const result = await exportCanonicalOrderBookParquet({ + clickhouseUrl: CLICKHOUSE_URL, + username: CLICKHOUSE_USERNAME, + password: CLICKHOUSE_PASSWORD, + outputDirectory, + captureBundleIds: ["integration-four-feed-bundle"], + exchange: "binance", + tradingPair: "TEST-FOUR-FEED", + startTimeMs: TEST_EVENT_MS + 500_000, + endTimeMs: TEST_EVENT_MS + 500_100, + }); + expect(result.levelRows).toBe(2); + expect(result.summaryRows).toBe(1); + for (const path of [result.levelsPath, result.summaryPath]) { + const bytes = await readFile(path); + expect(bytes.subarray(0, 4).toString()).toBe("PAR1"); + expect(bytes.subarray(-4).toString()).toBe("PAR1"); + } + } finally { + await rm(outputDirectory, { recursive: true, force: true }); + } }); test("orderbook views reflect inserts into orderbook_snapshots", async () => { @@ -512,4 +986,145 @@ describe("ClickHouse market_data schema integration", () => { expect(await snapshot.json()).toEqual([{ source_hash: sourceHash }]); } }); + + test("strategy v1 tables upgrade additively to the shared v2 identity", async () => { + if (!clickhouseAvailable || !client) return; + const tables = [ + "policy_evaluation_events", + "strategy_policy_snapshots", + "market_identity", + "symbol_mapping", + "inventory_settlement_events", + ] as const; + for (const table of tables) { + for (const column of [ + "producer_id", + "producer_run_id", + "stream_name", + "stream_seq", + "archive_event_id", + ]) { + await client.command({ + query: `ALTER TABLE strategy_data.${table} DROP COLUMN IF EXISTS ${column}`, + }); + } + } + + await ensureArchiveSchema(client); + const result = await client.query({ + query: `SELECT table, count() AS columns + FROM system.columns + WHERE database = 'strategy_data' + AND table IN ({tables:Array(String)}) + AND name IN ({columns:Array(String)}) + GROUP BY table + ORDER BY table`, + query_params: { + tables: [...tables], + columns: [ + "producer_id", + "producer_run_id", + "stream_name", + "stream_seq", + "seq", + "archive_event_id", + ], + }, + format: "JSONEachRow", + }); + expect(await result.json()).toEqual( + [...tables].sort().map((table) => ({ table, columns: "6" })), + ); + }); + + test("strategy schema v2 accepts all five tables and deduplicates stable tokens", async () => { + if (!clickhouseAvailable || !client) return; + const eventMs = TEST_EVENT_MS + 240_000; + const tables = [ + "policy_evaluation_events", + "strategy_policy_snapshots", + "market_identity", + "symbol_mapping", + "inventory_settlement_events", + ] as const; + const tableFields: Record< + (typeof tables)[number], + Record + > = { + policy_evaluation_events: { + policy_epoch: "epoch-v2", + fidelity: "hb_runtime_policy_clock", + lag_ms: 0, + fallback_reason: "", + source_cursor: "block:1:log:0", + decision_kind: "quote", + }, + strategy_policy_snapshots: { + snapshot_reason: "startup", + policy_epoch: "epoch-v2", + config_file_path: "controller.yml", + source_hash: "policy-v2", + }, + market_identity: { + snapshot_reason: "startup", + source_hash: "identity-v2", + core_pool_id: "pool-v2", + canonical_core_pool_id: "pool-v2", + }, + symbol_mapping: { + snapshot_reason: "startup", + source_hash: "symbol-v2", + }, + inventory_settlement_events: { + event_kind: "inventory_snapshot", + token: "BTC", + account: "primary", + reservation_id: "", + workflow_state: "observed", + }, + }; + const inserter = createClickHouseInserter(client); + + for (const [index, table] of tables.entries()) { + const row = { + event_time_ms: eventMs + index, + emitted_at_ms: eventMs + index, + source: "hb_runtime", + deployment_id: TEST_DEPLOYMENT, + schema_version: "2", + controller_id: "controller-v2", + controller_type: "layer12", + connector_name: "binance", + exchange: "binance", + trading_pair: "BTC-USDT", + market_id: "market-v2", + run_id: "run-v2", + producer_id: "hb_runtime:test:controller-v2", + producer_run_id: "run-v2", + stream_name: `strategy_data.${table}`, + stream_seq: 1, + seq: index + 1, + archive_event_id: `run-v2:${table}:1`, + payload_json: "{}", + ...tableFields[table], + }; + const token = `integration-v2-${TEST_DEPLOYMENT}-${table}`; + await inserter(`strategy_data.${table}`, [row], { + deduplicationToken: token, + }); + await inserter(`strategy_data.${table}`, [row], { + deduplicationToken: token, + }); + } + + for (const table of tables) { + const result = await client.query({ + query: `SELECT count() AS count FROM strategy_data.${table} + WHERE deployment_id = {deployment_id:String} AND run_id = 'run-v2'`, + query_params: { deployment_id: TEST_DEPLOYMENT }, + format: "JSONEachRow", + }); + expect(await result.json()).toEqual([{ count: "1" }]); + } + }); }); diff --git a/test/fixtures/archive_forwarder_envelope.json b/test/fixtures/archive_forwarder_envelope.json index 0cdd565..dab8cb6 100644 --- a/test/fixtures/archive_forwarder_envelope.json +++ b/test/fixtures/archive_forwarder_envelope.json @@ -3,6 +3,7 @@ "rows": [ { "row": { + "archive_event_id": "run-2026-07-02T00-00-00Z:strategy_data.policy_evaluation_events:1", "connector_name": "fiet_cex", "controller_id": "layer12-live-arb-usdc-015", "controller_type": "layer12_live", @@ -16,17 +17,22 @@ "market_id": "arb-usdc-015", "payload_json": "{\"execution_role\":\"dex_lead\",\"mid_price\":\"1.0234\"}", "policy_epoch": "42", + "producer_id": "hb_runtime:arb-usdc-015:layer12-live-arb-usdc-015", + "producer_run_id": "run-2026-07-02T00-00-00Z", "run_id": "run-2026-07-02T00-00-00Z", - "schema_version": "1", + "schema_version": "2", "seq": 1, "source": "hb_runtime", "source_cursor": "block:12345680:log:3", + "stream_name": "strategy_data.policy_evaluation_events", + "stream_seq": 1, "trading_pair": "ARB-USDC" }, "table": "strategy_data.policy_evaluation_events" }, { "row": { + "archive_event_id": "run-2026-07-02T00-00-00Z:strategy_data.strategy_policy_snapshots:1", "config_file_path": "conf/controllers/layer12_live_arb_usdc_015.yml", "connector_name": "fiet_cex", "controller_id": "layer12-live-arb-usdc-015", @@ -38,12 +44,16 @@ "market_id": "arb-usdc-015", "payload_json": "{\"lower_tick_offset\":-60,\"position_type\":\"OffsetFromCurrent\",\"upper_tick_offset\":60}", "policy_epoch": "42", + "producer_id": "hb_runtime:arb-usdc-015:layer12-live-arb-usdc-015", + "producer_run_id": "run-2026-07-02T00-00-00Z", "run_id": "run-2026-07-02T00-00-00Z", - "schema_version": "1", + "schema_version": "2", "seq": 2, "snapshot_reason": "content_change", "source": "hb_runtime", "source_hash": "1ed025b4ae41b0865651beb829e16eb62119ecb867c5148858fbe4bfe35e547e", + "stream_name": "strategy_data.strategy_policy_snapshots", + "stream_seq": 1, "trading_pair": "ARB-USDC" }, "table": "strategy_data.strategy_policy_snapshots" @@ -51,6 +61,7 @@ { "row": { "account": "secondary:1", + "archive_event_id": "run-2026-07-02T00-00-00Z:strategy_data.inventory_settlement_events:1", "connector_name": "fiet_cex", "controller_id": "layer12-live-arb-usdc-015", "controller_type": "layer12_live", @@ -61,11 +72,15 @@ "exchange": "binance", "market_id": "arb-usdc-015", "payload_json": "{\"amount\":\"99.81\",\"reason\":\"fill\"}", + "producer_id": "hb_runtime:arb-usdc-015:layer12-live-arb-usdc-015", + "producer_run_id": "run-2026-07-02T00-00-00Z", "reservation_id": "resv-0001", "run_id": "run-2026-07-02T00-00-00Z", - "schema_version": "1", + "schema_version": "2", "seq": 3, "source": "hb_runtime", + "stream_name": "strategy_data.inventory_settlement_events", + "stream_seq": 1, "token": "ARB", "trading_pair": "ARB-USDC", "workflow_state": "released" diff --git a/test/fixtures/canonical-market-capture-v1.json b/test/fixtures/canonical-market-capture-v1.json new file mode 100644 index 0000000..524e67f --- /dev/null +++ b/test/fixtures/canonical-market-capture-v1.json @@ -0,0 +1,64 @@ +{ + "fixture_version": "1", + "schema_version": "1.0.0", + "checksum_algorithm": "sha256-canonical-json-v1", + "context": { + "source": "broker_read", + "deploymentId": "fixture-collector-eu-1", + "captureBundleId": "fixture-bundle-2026-08-03", + "exchange": "binance", + "symbol": "BTC/USDT", + "assetType": "spot", + "provider": "ccxt:binance", + "schemaVersion": "1.0.0", + "checksumAlgorithm": "sha256-canonical-json-v1", + "provenanceComplete": true + }, + "stream": { + "feed": "ORDERBOOK", + "sourceMode": "broker_current_snapshot_v1", + "payload": { "bids": [[100, 1]], "asks": [[101, 2]], "timestamp": 1700000000000 }, + "eventTimeMs": 1700000000000, + "receivedTimeMs": 1700000000010, + "expected": { "raw_capture_id": "448292b1e054b7ad576b56e06129fca37cdbad9bede03061d01148e94d9bfdea", "raw_checksum": "8bcac32d02d0f86948c0533e9a1b8fffa33597c306bd6b81ced4e54c6d7e5cc0", "normalized_row_checksum": "8f8ab0c1a298bb08410b07de16d4bada51dd8b26a0567e4889f522631e14f539" } + }, + "ticker": { + "feed": "TICKER", + "sourceMode": "broker_live_stream_v1", + "payload": { "symbol": "BTC/USDT", "last": 100.5, "bid": 100, "ask": 101, "timestamp": 1700000000100 }, + "eventTimeMs": 1700000000100, + "receivedTimeMs": 1700000000110, + "normalized": { "eventTimeMs": 1700000000100, "last": 100.5, "bid": 100, "ask": 101 }, + "expected": { "raw_capture_id": "ccbcc74e6aeb65af676f85f9834d8ce86869de49f77d1e1fcb839430d9b48747", "raw_checksum": "cc4815a23f75ab5ee4e09cad3dc02cf0008839e816d8970e6d85b4c53406b5e2", "normalized_row_checksum": "dbc0941239434fa9a66fba610dfaa4a8918ada58bdc7fca05c55152901ae3c29" } + }, + "trade": { + "feed": "TRADES", + "sourceMode": "broker_live_stream_v1", + "payload": [{ "id": "trade-42", "timestamp": 1700000000200, "side": "buy", "price": 100.25, "amount": 0.5, "cost": 50.125, "takerOrMaker": "taker" }], + "eventTimeMs": 1700000000200, + "receivedTimeMs": 1700000000210, + "normalized": { "tradeId": "trade-42", "eventTimeMs": 1700000000200, "side": "buy", "price": 100.25, "amount": 0.5, "cost": 50.125, "takerOrMaker": "taker" }, + "expected": { "raw_capture_id": "fcafa03d10a308ed8a71fd103300cd069766f18283e65aaa4a677eebb882d565", "raw_checksum": "775321781aa4768105277b75fe04ef2d37ac801223072ce99c3cb12352c3696d", "normalized_row_checksum": "f992c12410f493992858fc4a6774dae5d309b0ddac29267533233603249b2ea7" } + }, + "ohlcv": { + "feed": "OHLCV", + "sourceMode": "broker_bootstrap_fetch_v1", + "timeframe": "1m", + "payload": [[1700000000000, 100, 102, 99, 101, 12.5]], + "eventTimeMs": 1700000000000, + "receivedTimeMs": 1700000000300, + "bar": { "openTimeMs": 1700000000000, "open": 100, "high": 102, "low": 99, "close": 101, "volume": 12.5 }, + "isClosed": true, + "brokerVersion": 1700000000300, + "expected": { "raw_capture_id": "3990f5b88f23c4e582165f24f399930d41385c12b0bef7259a0dfd46dfc165bf", "raw_checksum": "494b1c82f7fc9aabe3fb0d1019e407ccf87f3a2af506d0d0e35b44e769569b39", "normalized_row_checksum": "56800943ab52f21998124edf504517f9f598140e5ac10831bee8109daa27269b" } + }, + "orderbook": { + "feed": "ORDERBOOK", + "sourceMode": "broker_live_sampling_v1", + "payload": { "bids": [[100, 1], [99, 2]], "asks": [[101, 3], [102, 4]], "timestamp": 1700000000400, "sequence": 42 }, + "eventTimeMs": 1700000000400, + "receivedTimeMs": 1700000000410, + "depthLimit": 2, + "expected": { "raw_capture_id": "e423d4c8e6de3a468d5a5ab3799b07c3445cbaed8be3d0287cc0b2754f6bb7bc", "raw_checksum": "c85fed94343df96134a0e380ab4ba75ae121bbbb414e85e9df748be1340eb842", "snapshot_id": "e3e33c00fde1f3d27bc19bcbf21a09377408fdbdb26755ae2f9164ae90d41730", "level_normalized_row_checksum": "67d06b8cdd5e1d66fa2d1cab679060af4c9ca8df8deb8f652f3b2d077cfebce6", "summary_normalized_row_checksum": "8d2d30f59e855a05b155cf02793cb27cd28108cfd1edb9d9ae9bfb2eb1b23bad" } + } +} diff --git a/test/market-data-archive.test.ts b/test/market-data-archive.test.ts index e4640b0..fa16d2b 100644 --- a/test/market-data-archive.test.ts +++ b/test/market-data-archive.test.ts @@ -12,72 +12,9 @@ import { extractTrades, parseTicker, } from "../src/helpers/market-data-archive/parse-stream"; -import { - buildCandleRow, - buildCexStreamEventRow, - buildCexTickerEventRow, - buildCexTradeRow, - buildOrderbookSnapshotRow, -} from "../src/helpers/market-data-archive/rows"; -import type { NormalizedOrderBookSnapshot } from "../src/helpers/order-book"; - -function createSnapshot( - overrides: Partial = {}, -): NormalizedOrderBookSnapshot { - return { - bids: [[100, 1.5]], - asks: [[101, 2]], - timestamp: 1_700_000_000_000, - receivedTimestamp: 1_700_000_000_123, - exchange: "binance", - symbol: "BTC/USDT", - depthLimit: 5, - ...overrides, - }; -} +import { buildCexStreamEventRow } from "../src/helpers/market-data-archive/rows"; describe("market data archive rows", () => { - test("buildOrderbookSnapshotRow stores TOB scalars and depth arrays", () => { - const row = buildOrderbookSnapshotRow({ - deploymentId: "deploy-a", - exchange: "binance", - symbol: "BTC/USDT", - assetType: "spot", - snapshot: createSnapshot({ - bids: [ - [100, 1.5], - [99.5, 2], - [99, 3], - ], - asks: [ - [101, 2], - [101.5, 1.5], - [102, 4], - ], - depthLimit: 3, - }), - }); - - expect(row?.table).toBe("market_data.orderbook_snapshots"); - expect(row?.row).toMatchObject({ - exchange: "binance", - symbol: "BTC/USDT", - asset_type: "spot", - best_bid: 100, - best_ask: 101, - bid_size: 1.5, - ask_size: 2, - bid_levels: 3, - ask_levels: 3, - bids_price: [100, 99.5, 99], - bids_size: [1.5, 2, 3], - asks_price: [101, 101.5, 102], - asks_size: [2, 1.5, 4], - }); - expect(row?.row.mid).toBeCloseTo(100.5); - expect(row?.row.spread_bps).toBeGreaterThan(0); - }); - test("splitOrderBookSide ignores malformed levels", () => { expect( splitOrderBookSide( @@ -87,88 +24,6 @@ describe("market data archive rows", () => { ).toEqual([100, 99]); }); - test("buildCandleRow maps OHLCV fields and closed flag", () => { - const row = buildCandleRow({ - context: { - deploymentId: "deploy-a", - exchange: "binance", - symbol: "BTC/USDT", - assetType: "swap", - timeframe: "5m", - }, - bar: { - openTimeMs: 1_700_000_000_000, - open: 1, - high: 2, - low: 0.5, - close: 1.5, - volume: 100, - }, - isClosed: true, - brokerVersion: 1_700_000_000_500, - receivedTimestamp: 1_700_000_000_500, - }); - - expect(row.table).toBe("market_data.candles"); - expect(row.row).toMatchObject({ - timeframe: "5m", - asset_type: "swap", - open_time_ms: 1_700_000_000_000, - is_closed: 1, - broker_version: 1_700_000_000_500, - }); - }); - - test("buildCexTradeRow maps trade fields", () => { - const row = buildCexTradeRow( - { - deploymentId: "deploy-a", - exchange: "binance", - symbol: "BTC/USDT", - assetType: "spot", - payload: {}, - receivedTimestamp: 1_700_000_000_500, - }, - { - tradeId: "t-1", - eventTimeMs: 1_700_000_000_000, - side: "buy", - price: 100, - amount: 0.5, - }, - ); - - expect(row.table).toBe("market_data.cex_trades"); - expect(row.row).toMatchObject({ - trade_id: "t-1", - side: "buy", - price: 100, - amount: 0.5, - }); - }); - - test("buildCexTickerEventRow maps ticker fields", () => { - const row = buildCexTickerEventRow( - { - deploymentId: "deploy-a", - exchange: "binance", - symbol: "BTC/USDT", - assetType: "spot", - payload: { last: 100 }, - receivedTimestamp: 1_700_000_000_500, - }, - { - eventTimeMs: 1_700_000_000_000, - last: 100, - bid: 99.5, - ask: 100.5, - }, - ); - - expect(row.table).toBe("market_data.cex_ticker_events"); - expect(row.row.last).toBe(100); - }); - test("buildCexStreamEventRow stores redacted stream payload", () => { const row = buildCexStreamEventRow({ deploymentId: "deploy-a", diff --git a/test/ohlcv-collector-config.test.ts b/test/ohlcv-collector-config.test.ts index 2f83028..4a294ce 100644 --- a/test/ohlcv-collector-config.test.ts +++ b/test/ohlcv-collector-config.test.ts @@ -1,10 +1,87 @@ import { describe, expect, test } from "bun:test"; import { loadOhlcvCollectorConfig, + MARKET_DATA_COLLECTOR_CONFIG_ENV, OHLCV_COLLECTOR_CONFIG_ENV, + parseMarketDataCollectorConfig, parseOhlcvCollectorConfig, } from "../services/ohlcv-collector/config"; +describe("market-data collector config", () => { + test("validates all four feed supervisors and feed-specific options", () => { + const parsed = parseMarketDataCollectorConfig({ + captureBundleId: "bundle-2026-08-03", + environment: "production", + subscriptions: [ + { + exchange: " Binance ", + symbol: "BTC/USDT", + feed: "ORDERBOOK", + depthLimit: 25, + }, + { exchange: "binance", symbol: "BTC/USDT", feed: "TICKER" }, + { exchange: "binance", symbol: "BTC/USDT", feed: "TRADES" }, + { + exchange: "binance", + symbol: "BTC/USDT", + feed: "OHLCV", + timeframe: "1m", + bootstrapLimit: 100, + }, + ], + }); + + expect(parsed.captureBundleId).toBe("bundle-2026-08-03"); + expect(parsed.subscriptions.map(({ feed }) => feed)).toEqual([ + "ORDERBOOK", + "TICKER", + "TRADES", + "OHLCV", + ]); + expect(parsed.subscriptions[0]).toMatchObject({ + exchange: "binance", + depthLimit: 25, + }); + }); + + test.each([ + { + captureBundleId: "", + environment: "production", + subscriptions: [ + { exchange: "binance", symbol: "BTC/USDT", feed: "TICKER" }, + ], + }, + { + captureBundleId: "bundle-a", + environment: "production", + subscriptions: [ + { exchange: "binance", symbol: "BTC/USDT", feed: "ORDERBOOK" }, + ], + }, + { + captureBundleId: "bundle-a", + environment: "production", + subscriptions: [ + { exchange: "binance", symbol: "BTC/USDT", feed: "OHLCV" }, + ], + }, + ])("rejects incomplete production feed configuration", (input) => { + expect(() => parseMarketDataCollectorConfig(input)).toThrow( + "Invalid market-data collector config", + ); + }); + + test("exposes canonical and legacy config environments", () => { + expect(MARKET_DATA_COLLECTOR_CONFIG_ENV).toBe( + "CEX_BROKER_MARKET_DATA_COLLECTOR_CONFIG", + ); + expect(OHLCV_COLLECTOR_CONFIG_ENV).toBe( + "CEX_BROKER_OHLCV_COLLECTOR_CONFIG", + ); + }); +}); + describe("OHLCV collector config", () => { test("parses subscriptions and defaults the timeframe to 1m", () => { expect( diff --git a/test/ohlcv-collector.test.ts b/test/ohlcv-collector.test.ts index d79c460..ce14ab5 100644 --- a/test/ohlcv-collector.test.ts +++ b/test/ohlcv-collector.test.ts @@ -95,6 +95,63 @@ class CapturingMetrics { } describe("OHLCV collector supervision", () => { + test("opens independent supervisors for all four market feeds", async () => { + const requests: SubscribeRequest[] = []; + const { server, port } = await startSubscribeServer((call) => { + requests.push(call.request); + call.write({ + data: JSON.stringify({ ok: true }), + timestamp: Date.now(), + symbol: call.request.symbol, + type: call.request.type, + }); + }); + const abort = new AbortController(); + const collector = new OhlcvCollector({ + brokerUrl: `127.0.0.1:${port}`, + subscriptions: [ + { + exchange: "binance", + symbol: "BTC/USDT", + feed: "ORDERBOOK", + depthLimit: 25, + }, + { exchange: "binance", symbol: "BTC/USDT", feed: "TICKER" }, + { exchange: "binance", symbol: "BTC/USDT", feed: "TRADES" }, + { + exchange: "binance", + symbol: "BTC/USDT", + feed: "OHLCV", + timeframe: "1m", + bootstrapLimit: 100, + }, + ], + }); + const runPromise = collector.run(abort.signal); + + try { + await waitFor(() => requests.length === 4); + expect(requests).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + type: "ORDERBOOK", + options: { depthLimit: "25" }, + }), + expect.objectContaining({ type: "TICKER", options: {} }), + expect.objectContaining({ type: "TRADES", options: {} }), + expect.objectContaining({ + type: "OHLCV", + options: { timeframe: "1m", bootstrapLimit: "100" }, + }), + ]), + ); + } finally { + abort.abort(); + await runPromise; + server.forceShutdown(); + } + }); + test.each([ { terminal: "error" as const }, { terminal: "end" as const }, @@ -154,6 +211,7 @@ describe("OHLCV collector supervision", () => { value: 1, labels: { exchange: "binance", + feed: "OHLCV", symbol: "BTC/USDT", timeframe: "1m", }, @@ -163,6 +221,7 @@ describe("OHLCV collector supervision", () => { value: 1, labels: { exchange: "binance", + feed: "OHLCV", symbol: "BTC/USDT", timeframe: "1m", }, diff --git a/test/order-book-rpc.test.ts b/test/order-book-rpc.test.ts index cea0569..22a2340 100644 --- a/test/order-book-rpc.test.ts +++ b/test/order-book-rpc.test.ts @@ -1,12 +1,16 @@ import { afterEach, describe, expect, test } from "bun:test"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import * as grpc from "@grpc/grpc-js"; import * as protoLoader from "@grpc/proto-loader"; import type { Exchange } from "@usherlabs/ccxt"; +import { BrokerExecutionArchiver } from "../src/helpers/broker-execution-archive/writer"; import { Action } from "../src/helpers/constants"; import type { BrokerPoolEntry } from "../src/helpers/index"; import { PROTO_LOADER_OPTIONS } from "../src/proto-loader-options"; import { getServer } from "../src/server"; import type { PolicyConfig } from "../src/types"; +import { startForwarderServer } from "./archive-forwarder-server"; const packageDef = protoLoader.loadSync( "src/proto/node.proto", @@ -145,18 +149,48 @@ function createClient(port: number) { ); } +async function waitFor( + predicate: () => boolean, + timeoutMs = 1_000, +): Promise { + const deadline = Date.now() + timeoutMs; + while (!predicate()) { + if (Date.now() >= deadline) + throw new Error("timed out waiting for condition"); + await Bun.sleep(5); + } +} + function executeAction( client: InstanceType, request: Record, + metadata?: grpc.Metadata, ) { return new Promise<{ result: string; proof: string }>((resolve, reject) => { - client.ExecuteAction(request, (error, response) => { + const callback = ( + error: grpc.ServiceError | null, + response?: { result: string; proof: string }, + ) => { if (error) { reject(error); return; } resolve(response as { result: string; proof: string }); - }); + }; + if (metadata) { + ( + client.ExecuteAction as unknown as ( + request: Record, + metadata: grpc.Metadata, + callback: grpc.requestCallback<{ + result: string; + proof: string; + }>, + ) => void + )(request, metadata, callback); + } else { + client.ExecuteAction(request, callback); + } }); } @@ -210,6 +244,41 @@ describe("order-book RPC compatibility", () => { } }); + test("environment credentials take precedence over request credentials", async () => { + const { exchange, calls } = createOrderBookExchange(); + server = getServer( + testPolicy, + createPool("binance", exchange), + ["*"], + false, + "", + ); + client = createClient(await bindServer(server)); + const metadata = new grpc.Metadata(); + metadata.set("api-key", "request-key-must-not-win"); + metadata.set("api-secret", "request-secret-must-not-win"); + + const response = await executeAction( + client, + { + action: Action.Call, + cex: "binance", + symbol: "BTC/USDT", + payload: { + method: "fetch_order_book_snapshot", + depthLimit: "1", + }, + }, + metadata, + ); + + expect(JSON.parse(response.result)).toMatchObject({ + exchange: "binance", + symbol: "BTC/USDT", + }); + expect(calls.fetchOrderBook).toHaveLength(1); + }); + test("dispatches Maker capability method without invoking a fake CCXT method", async () => { const { exchange, calls } = createOrderBookExchange(); server = getServer( @@ -283,6 +352,70 @@ describe("order-book RPC compatibility", () => { expect(calls.fetchOrderBook[0]).toEqual(["BTC/USDT", 1]); }); + test("archives a typed current snapshot with current-snapshot provenance", async () => { + const originalEnabled = process.env.CEX_BROKER_MARKET_ARCHIVE_ENABLED; + process.env.CEX_BROKER_MARKET_ARCHIVE_ENABLED = "true"; + const forwarder = await startForwarderServer(); + const archiver = BrokerExecutionArchiver.create({ + source: "broker_read", + deploymentId: "read-collector-test", + deadLetterPath: join(tmpdir(), `cex-rpc-${crypto.randomUUID()}.jsonl`), + forwarderUrl: forwarder.url, + batchSize: 100, + flushIntervalMs: 60_000, + }); + try { + const { exchange } = createOrderBookExchange(); + server = getServer( + testPolicy, + createPool("binance", exchange), + ["*"], + false, + "", + undefined, + archiver, + ); + client = createClient(await bindServer(server)); + + await executeAction(client, { + action: Action.Call, + cex: "binance", + symbol: "BTC/USDT", + payload: { + method: "fetch_order_book_snapshot", + depthLimit: "1", + }, + }); + await waitFor(() => archiver.getStats().enqueued === 4); + const queued = Reflect.get(archiver, "queue") as Array<{ + table: string; + row: Record; + }>; + expect(queued.map(({ table }) => table)).toEqual([ + "market_data.cex_stream_events", + "market_data.cex_order_book_levels", + "market_data.cex_order_book_levels", + "market_data.cex_order_book_depth_summary", + ]); + expect(queued.every(({ row }) => row.source === "broker_read")).toBe( + true, + ); + expect( + queued.every( + ({ row }) => row.source_mode === "broker_current_snapshot_v1", + ), + ).toBe(true); + } finally { + await archiver.close(); + await forwarder.close(); + if (originalEnabled === undefined) { + delete process.env.CEX_BROKER_MARKET_ARCHIVE_ENABLED; + } else { + process.env.CEX_BROKER_MARKET_ARCHIVE_ENABLED = originalEnabled; + } + } + }); + test("returns typed historical unsupported including exact reconstruction", async () => { const { exchange, calls } = createOrderBookExchange(); server = getServer( diff --git a/test/strategy-data-schema-contract.test.ts b/test/strategy-data-schema-contract.test.ts new file mode 100644 index 0000000..08ae6ef --- /dev/null +++ b/test/strategy-data-schema-contract.test.ts @@ -0,0 +1,77 @@ +import { describe, expect, test } from "bun:test"; + +const schemaPath = new URL( + "../schema/clickhouse/strategy_data.sql", + import.meta.url, +); + +describe("strategy_data schema v2 contract", () => { + test("fresh tables carry the six shared v2 identity fields", async () => { + const sql = await Bun.file(schemaPath).text(); + const freshSchema = sql.split("-- Additive migration")[0]; + const columnDefinitions = freshSchema + .split("\n") + .map((line) => line.trim().replace(/,$/, "")); + for (const definition of [ + "producer_id String DEFAULT ''", + "producer_run_id String DEFAULT ''", + "stream_name LowCardinality(String) DEFAULT ''", + "stream_seq UInt64 DEFAULT 0", + "seq UInt64 DEFAULT 0", + "archive_event_id String DEFAULT ''", + ]) { + expect( + columnDefinitions.filter((line) => line === definition), + ).toHaveLength(5); + } + }); + + test("applies shared fields additively to all five existing tables", async () => { + const sql = await Bun.file(schemaPath).text(); + for (const column of [ + "producer_id", + "producer_run_id", + "stream_name", + "stream_seq", + "seq", + "archive_event_id", + ]) { + expect( + sql.match(new RegExp(`ADD COLUMN IF NOT EXISTS ${column} `, "g")), + ).toHaveLength(5); + } + }); + + test("includes Maker policy and identity/mapping extension fields", async () => { + const sql = await Bun.file(schemaPath).text(); + for (const column of [ + "tick_id", + "policy_revision", + "decision_stage", + "decision_outcome", + "decision_reason", + "source_clock", + "action_ids_json", + "content_hash", + "revision", + "active_from_ms", + "canonical_market_id", + "connector_id", + "canonical_exchange", + "canonical_trading_pair", + "source_symbol", + "base_asset", + "quote_asset", + "access_policy_id", + ]) { + expect(sql).toContain(column); + } + }); + + test("enables non-replicated insert deduplication on every table", async () => { + const sql = await Bun.file(schemaPath).text(); + expect( + sql.match(/non_replicated_deduplication_window = 1000000/g), + ).toHaveLength(10); + }); +}); diff --git a/test/subscribe-handler.test.ts b/test/subscribe-handler.test.ts index 95a6c9a..4021cce 100644 --- a/test/subscribe-handler.test.ts +++ b/test/subscribe-handler.test.ts @@ -1,6 +1,6 @@ import { afterAll, describe, expect, test } from "bun:test"; import { EventEmitter } from "node:events"; -import { mkdtempSync, rmSync } from "node:fs"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import * as grpc from "@grpc/grpc-js"; @@ -36,6 +36,7 @@ type MockCallState = { writes: SubscribeResponse[]; endCount: number; destroyed: boolean; + errors: unknown[]; }; function createSubscribeCall( @@ -47,6 +48,7 @@ function createSubscribeCall( writes: [], endCount: 0, destroyed: false, + errors: [], }; const writeResults = [...(options.writeResults ?? [])]; const call = Object.assign(emitter, { @@ -70,6 +72,7 @@ function createSubscribeCall( emitter.emit("close"); }, }); + emitter.on("error", (error) => state.errors.push(error)); Object.defineProperty(call, "destroyed", { get: () => state.destroyed, }); @@ -205,6 +208,37 @@ function createThrowingExchange( } describe("subscribe handler", () => { + test("environment credentials take precedence over request credentials", async () => { + const controlledWatch = createControlledWatch(); + const exchange = { + watchTicker: controlledWatch.watch, + } as unknown as Exchange; + const { call, state } = createSubscribeCall({ + cex: "binance", + symbol: "BTC/USDT", + type: SubscriptionType.TICKER, + }); + call.metadata.set("api-key", "request-key-must-not-win"); + call.metadata.set("api-secret", "request-secret-must-not-win"); + const handler = createSubscribeHandler({ + brokers: createPool(exchange), + whitelistIps: ["*"], + }); + + const handlerPromise = handler(call); + await waitFor(() => controlledWatch.calls.length === 1); + controlledWatch.resolvers[0]?.({ last: 100 }); + await waitFor(() => state.writes.length === 1); + cancelSubscribeCall(call); + controlledWatch.resolvers[1]?.({}); + await handlerPromise; + + expect(JSON.parse(state.writes[0]?.data ?? "{}")).toMatchObject({ + last: 100, + }); + expect(controlledWatch.calls).toHaveLength(2); + }); + test("keeps a subscription active when close fires without cancellation", async () => { const controlledWatch = createControlledWatch(); const exchange = { @@ -369,7 +403,7 @@ describe("subscribe handler", () => { expect(state.endCount).toBe(1); }); - test("archives orderbook snapshot rows to the forwarder", async () => { + test("archives canonical orderbook rows without legacy dual-write", async () => { const server = await startForwarderServer(); const posts = server.requests; const originalInterval = process.env.CEX_BROKER_ORDERBOOK_INTERVAL_MS; @@ -408,10 +442,10 @@ describe("subscribe handler", () => { asks: [[101, 2]], timestamp: 1_700_000_000_000, }); - await waitFor(() => archiver.getStats().enqueued >= 1); + await waitFor(() => archiver.getStats().enqueued >= 4); // batchSize 1 auto-flushes on enqueue; wait for that post to reach the // forwarder over the real transport rather than racing the round trip. - await waitFor(() => archiver.getStats().flushed >= 1); + await waitFor(() => archiver.getStats().flushed >= 4); await waitFor(() => controlledWatch.calls.length >= 2); cancelSubscribeCall(call); controlledWatch.resolvers[1]?.({ @@ -435,17 +469,27 @@ describe("subscribe handler", () => { ); expect( rows.some((entry) => entry.table === "market_data.orderbook_snapshots"), + ).toBe(false); + expect( + rows.some( + (entry) => entry.table === "market_data.cex_order_book_levels", + ), + ).toBe(true); + expect( + rows.some( + (entry) => entry.table === "market_data.cex_order_book_depth_summary", + ), ).toBe(true); - const snapshotRow = rows.find( - (entry) => entry.table === "market_data.orderbook_snapshots", + const summaryRow = rows.find( + (entry) => entry.table === "market_data.cex_order_book_depth_summary", ); - expect(snapshotRow?.row).toMatchObject({ + expect(summaryRow?.row).toMatchObject({ exchange: "binance", - symbol: "BTC/USDT", + trading_pair: "BTC-USDT", best_bid: 100, best_ask: 101, - bids_price: [100], - asks_price: [101], + bid_level_count: 1, + ask_level_count: 1, }); await archiver.close(); @@ -464,7 +508,7 @@ describe("subscribe handler", () => { } }); - test("archives OHLCV candle rows to the forwarder", async () => { + test("archives canonical OHLCV rows without legacy dual-write", async () => { const server = await startForwarderServer(); const posts = server.requests; const originalArchiveEnabled = @@ -498,24 +542,33 @@ describe("subscribe handler", () => { const handlerPromise = handler(call); await waitFor(() => controlledWatch.calls.length === 1); controlledWatch.resolvers[0]?.([[1_700_000_000_000, 1, 2, 0.5, 1.5, 10]]); - await waitFor(() => archiver.getStats().enqueued >= 1); + await waitFor(() => archiver.getStats().enqueued >= 2); // batchSize 1 auto-flushes on enqueue; wait for that post to reach the // forwarder over the real transport rather than racing the round trip. - await waitFor(() => archiver.getStats().flushed >= 1); + await waitFor(() => archiver.getStats().flushed >= 2); await waitFor(() => controlledWatch.calls.length >= 2); cancelSubscribeCall(call); controlledWatch.resolvers[1]?.([[1_700_000_000_000, 1, 2, 0.5, 1.5, 10]]); await handlerPromise; expect(posts.length).toBeGreaterThanOrEqual(1); - const rows = (posts[0]?.body.rows ?? []) as Array<{ - table: string; - row: Record; - }>; + const rows = posts.flatMap( + (post) => + (post.body.rows ?? []) as Array<{ + table: string; + row: Record; + }>, + ); expect(rows.some((entry) => entry.table === "market_data.candles")).toBe( - true, + false, + ); + expect( + rows.some((entry) => entry.table === "market_data.cex_ohlcv"), + ).toBe(true); + const candle = rows.find( + (entry) => entry.table === "market_data.cex_ohlcv", ); - expect(rows[0]?.row).toMatchObject({ + expect(candle?.row).toMatchObject({ timeframe: "1m", open_time_ms: 1_700_000_000_000, is_closed: 0, @@ -531,4 +584,169 @@ describe("subscribe handler", () => { } } }); + + test("archives ticker and trades with raw-to-normalized capture linkage", async () => { + const server = await startForwarderServer(); + const originalEnabled = process.env.CEX_BROKER_MARKET_ARCHIVE_ENABLED; + process.env.CEX_BROKER_MARKET_ARCHIVE_ENABLED = "true"; + try { + const tickerWatch = createControlledWatch(); + const tradesWatch = createControlledWatch(); + const exchange = { + watchTicker: tickerWatch.watch, + watchTrades: tradesWatch.watch, + } as unknown as Exchange; + const archiver = BrokerExecutionArchiver.create({ + source: "broker_read", + forwarderUrl: server.url, + deadLetterPath: createDeadLetterPath(), + deploymentId: "read-capture-test", + batchSize: 1, + flushIntervalMs: 60_000, + }); + const tickerCall = createSubscribeCall({ + cex: "binance", + symbol: "BTC/USDT", + type: SubscriptionType.TICKER, + }).call; + const tradesCall = createSubscribeCall({ + cex: "binance", + symbol: "BTC/USDT", + type: SubscriptionType.TRADES, + }).call; + const handler = createSubscribeHandler({ + brokers: createPool(exchange), + whitelistIps: ["*"], + brokerArchiver: archiver, + }); + const tickerPromise = handler(tickerCall); + const tradesPromise = handler(tradesCall); + await waitFor( + () => tickerWatch.calls.length === 1 && tradesWatch.calls.length === 1, + ); + tickerWatch.resolvers[0]?.({ + timestamp: 1_700_000_000_100, + last: 100.5, + bid: 100, + ask: 101, + }); + tradesWatch.resolvers[0]?.([ + { + id: "trade-1", + timestamp: 1_700_000_000_200, + side: "buy", + price: 100.25, + amount: 0.5, + }, + ]); + await waitFor(() => archiver.getStats().flushed >= 4); + await waitFor( + () => tickerWatch.calls.length >= 2 && tradesWatch.calls.length >= 2, + ); + cancelSubscribeCall(tickerCall); + cancelSubscribeCall(tradesCall); + tickerWatch.resolvers[1]?.({}); + tradesWatch.resolvers[1]?.([]); + await Promise.all([tickerPromise, tradesPromise]); + + const rows = server.requests.flatMap( + (post) => + (post.body.rows ?? []) as Array<{ + table: string; + row: Record; + }>, + ); + for (const [feed, table] of [ + ["TICKER", "market_data.cex_ticker_events"], + ["TRADES", "market_data.cex_trades"], + ] as const) { + const raw = rows.find( + (entry) => + entry.table === "market_data.cex_stream_events" && + entry.row.feed === feed, + ); + const normalized = rows.find((entry) => entry.table === table); + expect(raw?.row.source).toBe("broker_read"); + expect(normalized?.row.raw_capture_id).toBe(raw?.row.raw_capture_id); + expect(normalized?.row.raw_checksum).toBe(raw?.row.raw_checksum); + expect(normalized?.row.normalized_row_checksum).toBeString(); + } + await archiver.close(); + } finally { + await server.close(); + if (originalEnabled === undefined) { + delete process.env.CEX_BROKER_MARKET_ARCHIVE_ENABLED; + } else { + process.env.CEX_BROKER_MARKET_ARCHIVE_ENABLED = originalEnabled; + } + } + }); + + test("keeps stream delivery successful while forwarder failure is durably journaled", async () => { + const server = await startForwarderServer(() => ({ status: 503 })); + const originalEnabled = process.env.CEX_BROKER_MARKET_ARCHIVE_ENABLED; + process.env.CEX_BROKER_MARKET_ARCHIVE_ENABLED = "true"; + const deadLetterPath = createDeadLetterPath(); + try { + const controlledWatch = createControlledWatch(); + const exchange = { + watchTicker: controlledWatch.watch, + } as unknown as Exchange; + const archiver = BrokerExecutionArchiver.create({ + source: "broker_read", + forwarderUrl: server.url, + deadLetterPath, + deploymentId: "read-fault-test", + batchSize: 1, + flushIntervalMs: 60_000, + }); + const { call, state } = createSubscribeCall({ + cex: "binance", + symbol: "BTC/USDT", + type: SubscriptionType.TICKER, + }); + const handler = createSubscribeHandler({ + brokers: createPool(exchange), + whitelistIps: ["*"], + brokerArchiver: archiver, + }); + const handlerPromise = handler(call); + await waitFor(() => controlledWatch.calls.length === 1); + controlledWatch.resolvers[0]?.({ + timestamp: 1_700_000_000_100, + last: 100.5, + bid: 100, + ask: 101, + }); + await waitFor(() => state.writes.length === 1); + expect(JSON.parse(state.writes[0]?.data ?? "{}")).toMatchObject({ + last: 100.5, + }); + await waitFor(() => server.requests.length >= 1); + cancelSubscribeCall(call); + controlledWatch.resolvers[1]?.({}); + await handlerPromise; + await archiver.close(); + + const losses = readFileSync(deadLetterPath, "utf8") + .trim() + .split("\n") + .map((line) => JSON.parse(line) as Record); + expect(losses).toHaveLength(2); + expect( + losses.every( + (loss) => + loss.source === "broker_read" && + loss.reason === "shutdown_forwarder_failure", + ), + ).toBe(true); + } finally { + await server.close(); + if (originalEnabled === undefined) { + delete process.env.CEX_BROKER_MARKET_ARCHIVE_ENABLED; + } else { + process.env.CEX_BROKER_MARKET_ARCHIVE_ENABLED = originalEnabled; + } + } + }); });