diff --git a/.github/workflows/listener-cargo-fmt-clippy.yml b/.github/workflows/listener-cargo-fmt-clippy.yml new file mode 100644 index 0000000000..9fe92d14b7 --- /dev/null +++ b/.github/workflows/listener-cargo-fmt-clippy.yml @@ -0,0 +1,67 @@ +name: listener-cargo-fmt-clippy + +on: + push: + branches: [main] + pull_request: + +permissions: {} + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +jobs: + check-changes: + name: listener-cargo-fmt-clippy/check-changes + runs-on: ubuntu-latest + outputs: + rust-changed: ${{ github.event_name == 'push' || steps.filter.outputs.rust }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + if: github.event_name != 'push' + with: + persist-credentials: 'false' + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + if: github.event_name != 'push' + id: filter + with: + filters: | + rust: + - .github/workflows/listener-cargo-fmt-clippy.yml + - Cargo.toml + - Cargo.lock + - listener/crates/** + + fmt-clippy: + name: listener-cargo-fmt-clippy/fmt-clippy + needs: check-changes + if: ${{ needs.check-changes.outputs.rust-changed == 'true' }} + permissions: + contents: 'read' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' + + - uses: dtolnay/rust-toolchain@a54c7afa936fefeb4456b2dd8068152669aa8203 + with: + toolchain: 1.91.1 + components: rustfmt, clippy + + - uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + key: ${{ runner.os }}-cargo-clippy-${{ hashFiles('**/Cargo.lock') }} + restore-keys: ${{ runner.os }}-cargo-clippy- + + - run: cargo fmt --check + working-directory: listener + + - run: > + cargo clippy --workspace --all-targets + -- -W clippy::perf -W clippy::suspicious -W clippy::style -D warnings + working-directory: listener diff --git a/.github/workflows/listener-tests.yml b/.github/workflows/listener-tests.yml new file mode 100644 index 0000000000..3f71799bb8 --- /dev/null +++ b/.github/workflows/listener-tests.yml @@ -0,0 +1,61 @@ +name: listener-tests + +on: + push: + branches: [main] + pull_request: + +permissions: {} + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +jobs: + check-changes: + name: listener-tests/check-changes + runs-on: ubuntu-latest + outputs: + rust-changed: ${{ github.event_name == 'push' || steps.filter.outputs.rust }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + if: github.event_name != 'push' + with: + persist-credentials: 'false' + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + if: github.event_name != 'push' + id: filter + with: + filters: | + rust: + - .github/workflows/listener-tests.yml + - Cargo.toml + - Cargo.lock + - listener/crates/** + + tests: + name: listener-tests/tests + needs: check-changes + if: ${{ needs.check-changes.outputs.rust-changed == 'true' }} + permissions: + contents: 'read' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' + + - uses: dtolnay/rust-toolchain@a54c7afa936fefeb4456b2dd8068152669aa8203 + with: + toolchain: 1.91.1 + + - uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + key: ${{ runner.os }}-cargo-test-${{ hashFiles('**/Cargo.lock') }} + restore-keys: ${{ runner.os }}-cargo-test- + + - run: make test + working-directory: listener diff --git a/listener/.dockerignore b/listener/.dockerignore new file mode 100644 index 0000000000..bb91a19883 --- /dev/null +++ b/listener/.dockerignore @@ -0,0 +1,11 @@ +# Exclude everything +* + +# Allow only what cargo needs to build +!Cargo.toml +!Cargo.lock +!crates/listener_core/ +!crates/shared/ +!crates/shared/broker/ +!crates/consumer/ +!crates/test-support/ diff --git a/listener/.gitignore b/listener/.gitignore new file mode 100644 index 0000000000..e3c1f61d0a --- /dev/null +++ b/listener/.gitignore @@ -0,0 +1,11 @@ +.vscode +tmp +target +.cursor +Cargo.lock +_erpc +.env + +# Helm +charts/*/charts/ +charts/*/Chart.lock \ No newline at end of file diff --git a/listener/Cargo.toml b/listener/Cargo.toml new file mode 100644 index 0000000000..b956502770 --- /dev/null +++ b/listener/Cargo.toml @@ -0,0 +1,33 @@ +[workspace] +resolver = "2" +members = [ + "crates/consumer", + "crates/listener_core", + "crates/shared/broker", + "crates/shared/telemetry", + "crates/test-support", +] + +[workspace.package] +edition = "2024" + +[workspace.dependencies] +alloy = { version = "1.6.1", features = ["full", "trie", "rpc-types", "network", "consensus", "rlp"] } +alloy-json-rpc = "1.6.1" +alloy-primitives = { version = "1", features = ["serde"] } +alloy-transport-ipc = "1.6.1" +async-trait = "0.1.89" +axum = { version = "0.8", default-features = false } +serde = "1.0.228" +serde_json = "1.0.149" +thiserror = "2.0.18" +tokio = { version = "1.49.0", features = ["rt-multi-thread", "sync", "time"] } +tokio-util = "0.7" +tracing = "0.1.44" +futures = "0.3.31" +metrics = "0.24" + +[profile.release] +opt-level = 3 +lto = true +codegen-units = 1 diff --git a/listener/Dockerfile b/listener/Dockerfile new file mode 100644 index 0000000000..e6516c889e --- /dev/null +++ b/listener/Dockerfile @@ -0,0 +1,33 @@ +# syntax=docker/dockerfile:1 + +# ── Planner (generate dependency recipe) ───────────────────────────── +FROM rust:1.92-slim-bookworm AS planner +RUN cargo install cargo-chef +WORKDIR /app +COPY . . +RUN cargo chef prepare --recipe-path recipe.json + +# ── Builder (cook deps, then build app) ────────────────────────────── +FROM rust:1.92-slim-bookworm AS builder +RUN apt-get update \ + && apt-get install -y pkg-config libssl-dev \ + && rm -rf /var/lib/apt/lists/* \ + && cargo install cargo-chef +ENV SQLX_OFFLINE=true +WORKDIR /app + +# Cook dependencies (cached unless Cargo.toml/Cargo.lock change) +COPY --from=planner /app/recipe.json recipe.json +RUN cargo chef cook --release --recipe-path recipe.json + +# Build application +COPY . . +RUN cargo build --release -p listener_core + +# ── Runtime ────────────────────────────────────────────────────────── +FROM gcr.io/distroless/cc-debian12:nonroot +COPY --from=builder /app/target/release/listener_core /app/listener_core +COPY --from=builder /app/crates/listener_core/migrations/ /app/migrations/ +WORKDIR /app +ENTRYPOINT ["/app/listener_core"] +CMD ["--config", "/config.yaml"] diff --git a/listener/Makefile b/listener/Makefile new file mode 100644 index 0000000000..c78a26632a --- /dev/null +++ b/listener/Makefile @@ -0,0 +1,63 @@ +.PHONY: test test-unit test-e2e test-e2e-amqp test-e2e-redis test-e2e-consumer clean-e2e-containers clippy \ + helm-lint + +# ── Unit tests (no Docker required) ────────────────────────────────────────── +test-unit: + cargo test --workspace + +# ── E2E helpers ─────────────────────────────────────────────────────────────── +# Named containers created by the shared-container pattern in e2e tests. +AMQP_CONTAINER := e2e-rabbitmq +REDIS_CONTAINER := e2e-redis + +# Remove named e2e containers if they exist (idempotent). +clean-e2e-containers: + @docker rm -f $(AMQP_CONTAINER) 2>/dev/null || true + @docker rm -f $(REDIS_CONTAINER) 2>/dev/null || true + +# ── broker (amqp) e2e ──────────────────────────────────────────────────────── +# Runs the AMQP e2e suite and removes the shared container afterwards, +# even if the tests fail. +test-e2e-amqp: clean-e2e-containers + cargo test -p broker --features amqp --test amqp_e2e -- --include-ignored; \ + $(MAKE) clean-e2e-containers + +# ── broker (redis) e2e ─────────────────────────────────────────────────────── +# Runs the Redis e2e suite and removes the shared container afterwards, +# even if the tests fail. +test-e2e-redis: clean-e2e-containers + cargo test -p broker --features redis --test redis_e2e -- --include-ignored; \ + $(MAKE) clean-e2e-containers + +# ── broker facade e2e ──────────────────────────────────────────────────────── +# Runs the Broker facade e2e suite (high-level API: Broker → Publisher / ConsumerBuilder) +# and removes the shared container afterwards. +test-e2e-broker: clean-e2e-containers + cargo test -p broker --features redis --test broker_e2e -- --include-ignored; \ + $(MAKE) clean-e2e-containers + +# ── consumer-lib e2e ───────────────────────────────────────────────────────── +test-e2e-consumer: clean-e2e-containers + cargo test -p consumer --test watch_e2e -- --include-ignored; \ + $(MAKE) clean-e2e-containers + +# ── Run all e2e tests ───────────────────────────────────────────────────────── +test-e2e: test-e2e-amqp test-e2e-redis test-e2e-broker test-e2e-consumer + +# ── Clippy (same flags as CI) ──────────────────────────────────────────────── +clippy: + cargo clippy --workspace --all-targets -- -W clippy::perf -W clippy::suspicious -W clippy::style -D warnings + +# ── Run everything ──────────────────────────────────────────────────────────── +test: test-unit test-e2e + +# ── Run Format ─────────────────────────────────────────────────────────────── +format: + cargo fmt + +# ── Helm Chart Lint ────────────────────────────────────────────────────────── +# Requires: ct (chart-testing), yamllint, helm +helm-lint: + @command -v ct >/dev/null 2>&1 || { echo "ct not found. Install with: brew install chart-testing"; exit 1; } + @command -v yamllint >/dev/null 2>&1 || { echo "yamllint not found. Install with: brew install yamllint"; exit 1; } + ct lint --config ct.yaml --all diff --git a/listener/README.md b/listener/README.md new file mode 100644 index 0000000000..c365ac8dd3 --- /dev/null +++ b/listener/README.md @@ -0,0 +1,111 @@ +# Listener + +Zero-event-loss blockchain listener for EVM chains. + +Polls RPC nodes in parallel, validates block hash-chains to detect reorgs, and +publishes canonical blocks/transactions/receipts to downstream consumers via +Redis Streams or RabbitMQ. Designed for chains with fast block times where +HTTP latency would otherwise cause gaps. + +``` +RPC Nodes ──► Parallel Fetchers ──► Cursor Validator ──► Broker ──► Consumers + (AsyncSlotBuffer) (hash-chain check) (Redis/AMQP) +``` + +**Core guarantees:** +- No block, transaction, or receipt is ever skipped — even across crashes. +- Reorgs (simple, back-and-forth, multi-branch) are detected and handled. +- Transient RPC/broker failures retry indefinitely with circuit breaker protection. + +## Crates + +| Crate | Purpose | +|-------|---------| +| `listener_core` | Main binary. Fetches blocks via HTTP RPC, validates hash-chains, persists metadata to PostgreSQL, publishes events. | +| `broker` | Backend-agnostic message broker. Wraps Redis Streams (consumer groups, XCLAIM, PEL) and RabbitMQ (exchanges, TTL-DLX retry) behind a unified API with retry, dead-letter, and circuit breaker semantics. | +| `primitives` | Shared Ethereum types and routing constants. | + +## Quick Start + +```bash +# 1. Start local infrastructure (PostgreSQL, Redis, RabbitMQ) +docker compose up -d + +# 2. Run tests +make test-unit # unit tests (no Docker services needed) +make test-e2e # e2e tests (spins up testcontainers) +make clippy # lint + +# 3. Run the listener +cargo run -p listener_core -- --config config/listener-1.yaml +``` + +## Local Development + +`docker compose up -d` starts PostgreSQL 17, Redis 7, and RabbitMQ 3. + +Profiles add optional services: + +```bash +docker compose --profile erpc up -d # + eRPC load balancer (:4000) +docker compose --profile monitoring up -d # + Prometheus (:9090) + Grafana (:3000) +docker compose --profile listener-1 up -d # + listener instance (Ethereum mainnet, Redis) +docker compose --profile listener-2 up -d # + listener instance (Base Sepolia, AMQP) +``` + +Combine profiles freely: `docker compose --profile listener-1 --profile monitoring up -d` + +Service endpoints: + +| Service | URL | Credentials | +|---------|-----|-------------| +| PostgreSQL | `localhost:5432` | postgres / postgres | +| Redis | `localhost:6379` | — | +| RabbitMQ | `localhost:5672` | user / pass | +| RabbitMQ UI | `localhost:15672` | user / pass | +| eRPC | `localhost:4000` | — | +| Prometheus | `localhost:9090` | — | +| Grafana | `localhost:3000` | admin / admin | + +## Configuration + +Copy and edit `config/listener-1.yaml`. The three things you must set: + +```yaml +name: listener + +database: + db_url: postgres://postgres:postgres@listener-postgres:5432/listener + migration_max_attempts: 5 + pool: + max_connections: 10 + min_connections: 2 + +broker: + broker_type: redis + broker_url: redis://listener-redis:6379 + +blockchain: + type: evm + chain_id: 1 + rpc_url: http://listener-erpc:4000/listener-indexer/evm/1 + network: ethereum-mainnet + strategy: + automatic_startup: true + block_start_on_first_start: 24572795 + range_size: 10 + loop_delay_ms: 1000 + max_parallel_requests: 10 + block_fetcher: block_receipts + batch_receipts_size_range: 10 + compute_block: false +``` + +## Docker Build + +Multi-stage distroless image. Database migrations are bundled. + +```bash +docker build -t listener . +docker compose --profile listener-1 up +``` \ No newline at end of file diff --git a/listener/benchmark/k6/README.md b/listener/benchmark/k6/README.md new file mode 100644 index 0000000000..0db21a3b1c --- /dev/null +++ b/listener/benchmark/k6/README.md @@ -0,0 +1,181 @@ +# k6 Listener Workload Benchmark + +Chain-agnostic load testing for eRPC load balancer simulating realistic blockchain listener/indexer patterns. + +## Prerequisites + +```bash +# Install k6 +brew install k6 + +# Ensure eRPC stack is running +cd ../.. +docker-compose ps +``` + +## Quick Start + +```bash +cd k6 + +# Test single chain (Sepolia) +k6 run listener-workload.js --env CHAIN_ID=11155111 + +# Test all testnets +k6 run listener-workload.js + +# Test with higher load +k6 run listener-workload.js --env CHAIN_ID=43113 --env RATE=100 +``` + +## Environment Variables + +| Variable | Description | Default | Example | +|----------|-------------|---------|---------| +| `CHAIN_ID` | Test single chain | all chains | `11155111` (Sepolia) | +| `CHAINS` | Test multiple chains (comma-separated) | all chains | `11155111,43113,97` | +| `RATE` | Requests per second | `50` | `100` | +| `DURATION` | Test duration | `10m` | `5m`, `30m` | +| `CONFIG` | Config tag for reports | `default` | `public`, `mixed` | +| `BASE_URL` | eRPC base URL | `http://localhost:4000` | `http://erpc:4000` | + +## Supported Chains + +| Chain ID | Name | Alias | +|----------|------|-------| +| 11155111 | Sepolia | Ethereum testnet | +| 43113 | Fuji | Avalanche testnet | +| 97 | BSC-Testnet | Binance Smart Chain testnet | +| 80002 | Amoy | Polygon testnet | +| 84532 | Base-Sepolia | Base testnet | + +## Method Distribution + +Realistic listener workload pattern: + +| Method | Weight | Use Case | +|--------|--------|----------| +| `eth_blockNumber` | 50% | Polling heartbeat (high frequency) | +| `eth_getBlockByNumber` | 25% | Block fetching (medium frequency) | +| `eth_getLogs` | 15% | Event scanning (variable frequency) | +| `eth_getBlockReceipts` | 8% | Receipt fetching (medium frequency) | +| `eth_getBlockByHash` | 2% | Reorg handling (low frequency) | + +## Usage Examples + +### Test Single Chain + +```bash +# Test Sepolia with public nodes +k6 run listener-workload.js --env CHAIN_ID=11155111 --env CONFIG=public + +# Test Fuji with mixed config (after switching to erpc.yaml) +k6 run listener-workload.js --env CHAIN_ID=43113 --env CONFIG=mixed +``` + +### Test Multiple Chains + +```bash +# Test 3 specific chains +k6 run listener-workload.js --env CHAINS=11155111,43113,80002 + +# Test all testnets (default) +k6 run listener-workload.js +``` + +### Custom Load Patterns + +```bash +# Higher load test (100 req/s) +k6 run listener-workload.js --env CHAIN_ID=11155111 --env RATE=100 + +# Short smoke test (2 minutes, 10 req/s) +k6 run listener-workload.js --env CHAIN_ID=80002 --env DURATION=2m --env RATE=10 + +# Long endurance test (1 hour, 25 req/s) +k6 run listener-workload.js --env DURATION=1h --env RATE=25 +``` + +### Compare Public vs Mixed Config + +```bash +# Test with public nodes +k6 run listener-workload.js --env CHAIN_ID=11155111 --env CONFIG=public + +# Switch docker-compose to erpc.yaml, then: +k6 run listener-workload.js --env CHAIN_ID=11155111 --env CONFIG=mixed + +# Compare results from the console summary (P95 latency, error rate, per-method stats) +``` + +## Understanding Results + +### Console Output + +k6 shows real-time metrics and a summary at the end: +- HTTP request duration (p50, p95, p99) +- Request rate (req/s) +- Error rate (%) +- Virtual Users (VUs) +- Per-method performance (latency_eth_blockNumber, latency_eth_getBlockByNumber, etc.) +- Error breakdown by method and chain + +## Performance Expectations + +### Public Nodes Only +- eth_blockNumber: p95 < 500ms +- eth_getBlockByNumber: p95 < 1s +- eth_getLogs: p95 < 2s +- Error rate: < 5% + +### Mixed (Public + Private Fallback) +- eth_blockNumber: p95 < 200ms +- eth_getBlockByNumber: p95 < 400ms +- eth_getLogs: p95 < 500ms +- Error rate: < 1% + +## Troubleshooting + +### High Error Rates + +Check eRPC logs and Prometheus: +```bash +docker-compose logs erpc | grep -i error +open http://localhost:9090 # Check erpc_upstream_request_errors_total +``` + +### Slow Performance + +- Increase timeouts in thresholds +- Reduce RATE +- Check if public nodes are rate-limiting + +### k6 Not Found + +```bash +brew install k6 +``` + +## Advanced Usage + +### Run with Debug Output + +```bash +k6 run listener-workload.js --env CHAIN_ID=11155111 --log-output=stdout --verbose +``` + +### Export to InfluxDB/Grafana + +```bash +k6 run listener-workload.js \ + --out influxdb=http://localhost:8086/k6 +``` + +### Run in Docker + +```bash +docker run --rm -i --network=host \ + -v $PWD:/scripts \ + grafana/k6 run /scripts/listener-workload.js \ + --env CHAIN_ID=11155111 +``` diff --git a/listener/benchmark/k6/config.js b/listener/benchmark/k6/config.js new file mode 100644 index 0000000000..225eba70cb --- /dev/null +++ b/listener/benchmark/k6/config.js @@ -0,0 +1,209 @@ +import { Counter, Trend } from 'k6/metrics'; + +// ============================================================================= +// CHAIN REGISTRY +// ============================================================================= +export const CHAIN_REGISTRY = { + 11155111: { + id: 11155111, + name: 'Sepolia', + endpoint: '/listener-indexer/evm/11155111', + blockTime: 12, + }, + 43113: { + id: 43113, + name: 'Fuji', + endpoint: '/listener-indexer/evm/43113', + blockTime: 2, + }, + 97: { + id: 97, + name: 'BSC-Testnet', + endpoint: '/listener-indexer/evm/97', + blockTime: 3, + }, + 80002: { + id: 80002, + name: 'Amoy', + endpoint: '/listener-indexer/evm/80002', + blockTime: 2, + }, + 84532: { + id: 84532, + name: 'Base-Sepolia', + endpoint: '/listener-indexer/evm/84532', + blockTime: 2, + }, +}; + +export const BASE_URL = __ENV.BASE_URL || 'http://localhost:4000'; + +// ============================================================================= +// RPC METHODS +// ============================================================================= +export const RPC_METHODS = { + ETH_BLOCK_NUMBER: 'eth_blockNumber', + ETH_GET_BLOCK_BY_NUMBER: 'eth_getBlockByNumber', + ETH_GET_BLOCK_BY_HASH: 'eth_getBlockByHash', + ETH_GET_BLOCK_RECEIPTS: 'eth_getBlockReceipts', + ETH_GET_LOGS: 'eth_getLogs', +}; + +// ============================================================================= +// CUSTOM METRICS - Separate Trend per method for proper breakdown +// ============================================================================= +export const methodLatencyBlockNumber = new Trend('latency_eth_blockNumber', true); +export const methodLatencyGetBlock = new Trend('latency_eth_getBlockByNumber', true); +export const methodLatencyGetBlockHash = new Trend('latency_eth_getBlockByHash', true); +export const methodLatencyGetReceipts = new Trend('latency_eth_getBlockReceipts', true); + +export const methodErrors = new Counter('method_errors'); +export const chainErrors = new Counter('chain_errors'); +export const responseSizes = new Trend('response_sizes'); +export const jsonRpcErrors = new Counter('jsonrpc_errors'); +export const parsingErrors = new Counter('parsing_errors'); + +// ============================================================================= +// CHAIN SELECTION HELPERS +// ============================================================================= + +/** + * Get chains to test based on environment variables + * Priority: CHAIN_ID > CHAINS > all chains + */ +export function getSelectedChains() { + // Single chain via CHAIN_ID + if (__ENV.CHAIN_ID) { + const chainId = parseInt(__ENV.CHAIN_ID); + const chain = CHAIN_REGISTRY[chainId]; + if (!chain) { + throw new Error(`Invalid CHAIN_ID: ${chainId}. Valid IDs: ${Object.keys(CHAIN_REGISTRY).join(', ')}`); + } + return [chain]; + } + + // Multiple chains via CHAINS (comma-separated) + if (__ENV.CHAINS) { + const chainIds = __ENV.CHAINS.split(',').map(id => id.trim()); + const chains = chainIds.map(id => { + const chainId = parseInt(id); + const chain = CHAIN_REGISTRY[chainId]; + if (!chain) { + throw new Error(`Invalid chain ID in CHAINS: ${id}. Valid IDs: ${Object.keys(CHAIN_REGISTRY).join(', ')}`); + } + return chain; + }); + return chains; + } + + // Default: all chains + return Object.values(CHAIN_REGISTRY); +} + +/** + * Get random chain from selected chains + */ +export function getRandomChain(chains) { + return chains[randomIntBetween(0, chains.length - 1)]; +} + +// ============================================================================= +// UTILITY FUNCTIONS +// ============================================================================= + +export function randomIntBetween(min, max) { + return Math.floor(Math.random() * (max - min + 1)) + min; +} + +export function makeRpcRequest(method, params = []) { + return { + jsonrpc: '2.0', + method: method, + params: params, + id: Math.floor(Math.random() * 100000000), + }; +} + +export function getFullUrl(chain) { + return `${BASE_URL}${chain.endpoint}`; +} + +export function truncateString(str, maxLength = 200) { + if (!str || str.length <= maxLength) return str; + const half = Math.floor(maxLength / 2); + return `${str.substring(0, half)}...${str.substring(str.length - half)}`; +} + +/** + * Record metrics for a request + * @returns {boolean} success - true if request was successful + */ +export function recordMetrics(res, method, chain, startTime) { + const duration = Date.now() - startTime; + + const tags = { + method: method, + chain: chain.name, + chainId: chain.id.toString(), + }; + + if (!res || !res.status) { + console.error(`[ERROR] No response - Method: ${method}, Chain: ${chain.name}`); + methodErrors.add(1, tags); + chainErrors.add(1, tags); + return false; + } + + // Record response size + if (res.body) { + responseSizes.add(res.body.length, tags); + } + + // Record method-specific latency + switch(method) { + case 'eth_blockNumber': + methodLatencyBlockNumber.add(duration, tags); + break; + case 'eth_getBlockByNumber': + methodLatencyGetBlock.add(duration, tags); + break; + case 'eth_getBlockByHash': + methodLatencyGetBlockHash.add(duration, tags); + break; + case 'eth_getBlockReceipts': + methodLatencyGetReceipts.add(duration, tags); + break; + } + + // Parse response and check for errors + let parsedBody = null; + try { + parsedBody = JSON.parse(res.body); + } catch (e) { + console.error(`[ERROR] Parsing failed - Method: ${method}, Chain: ${chain.name}, Body: ${truncateString(res.body, 100)}`); + parsingErrors.add(1, tags); + methodErrors.add(1, tags); + chainErrors.add(1, tags); + return false; + } + + // Check for JSON-RPC errors + if (parsedBody?.error) { + const errorTags = { + ...tags, + error_code: parsedBody.error.code?.toString() || 'unknown', + error_message: truncateString(parsedBody.error.message || '', 50), + }; + + console.error(`[ERROR] JSON-RPC error - Method: ${method}, Chain: ${chain.name}, Code: ${parsedBody.error.code}, Message: ${parsedBody.error.message}`); + + jsonRpcErrors.add(1, errorTags); + methodErrors.add(1, tags); + chainErrors.add(1, tags); + + return false; + } + + // Success if we have a result + return parsedBody?.result !== undefined; +} diff --git a/listener/benchmark/k6/listener-workload.js b/listener/benchmark/k6/listener-workload.js new file mode 100644 index 0000000000..9084c952c9 --- /dev/null +++ b/listener/benchmark/k6/listener-workload.js @@ -0,0 +1,346 @@ +import http from 'k6/http'; +import { check, sleep } from 'k6'; +import { textSummary } from 'https://jslib.k6.io/k6-summary/0.1.0/index.js'; +import { + RPC_METHODS, + getSelectedChains, + getRandomChain, + makeRpcRequest, + getFullUrl, + recordMetrics, +} from './config.js'; + +// ============================================================================= +// LISTENER WORKLOAD - Chain Agnostic +// ============================================================================= +// Simulates realistic blockchain listener/indexer load pattern +// +// Usage: +// k6 run listener-workload.js --env CHAIN_ID=11155111 +// k6 run listener-workload.js --env CHAINS=11155111,43113 +// k6 run listener-workload.js # defaults to all chains +// ============================================================================= + +// ============================================================================= +// CONFIGURATION +// ============================================================================= +const RATE = parseInt(__ENV.RATE || '50'); // Requests per second +const DURATION = __ENV.DURATION || '10m'; // Test duration +const CONFIG_TAG = __ENV.CONFIG || 'default'; // public/mixed/default + +// ============================================================================= +// METHOD DISTRIBUTION (Realistic Listener Pattern) +// ============================================================================= +// Note: eth_getLogs removed - not used by this listener +const METHOD_WEIGHTS = { + [RPC_METHODS.ETH_BLOCK_NUMBER]: 55, // Polling heartbeat (was 50%) + [RPC_METHODS.ETH_GET_BLOCK_BY_NUMBER]: 30, // Block fetching (was 25%) + [RPC_METHODS.ETH_GET_BLOCK_RECEIPTS]: 13, // Receipt fetching (was 8%) + [RPC_METHODS.ETH_GET_BLOCK_BY_HASH]: 2, // Reorg handling (was 2%) +}; + +// Select chains based on environment +const selectedChains = getSelectedChains(); + +// ============================================================================= +// K6 OPTIONS +// ============================================================================= +export const options = { + scenarios: { + listener_load: { + executor: 'constant-arrival-rate', + rate: RATE, + timeUnit: '1s', + duration: DURATION, + preAllocatedVUs: RATE * 2, + maxVUs: RATE * 4, + }, + }, + thresholds: { + 'http_req_failed': ['rate<0.05'], // Less than 5% errors + }, +}; + +// ============================================================================= +// STATE CACHE (per-VU, shared across iterations) +// ============================================================================= +const chainStateCache = new Map(); + +function initializeCache() { + for (const chain of selectedChains) { + if (!chainStateCache.has(chain.id)) { + chainStateCache.set(chain.id, { + latestBlockNumber: null, + latestBlockHash: null, + timestamp: 0, + }); + } + } +} + +// Initialize cache on VU startup +initializeCache(); + +// ============================================================================= +// MAIN TEST FUNCTION +// ============================================================================= +export default function () { + const params = { + headers: { + 'Content-Type': 'application/json', + 'Accept-Encoding': 'gzip, deflate', + }, + timeout: '60s', + }; + + // Select random chain from selected chains + const chain = getRandomChain(selectedChains); + + // Select method based on weighted distribution + const method = selectWeightedMethod(); + + const startTime = Date.now(); + let payload; + + switch (method) { + case RPC_METHODS.ETH_BLOCK_NUMBER: + payload = makeRpcRequest(method); + break; + + case RPC_METHODS.ETH_GET_BLOCK_BY_NUMBER: + const blockNum = getLatestBlockNumber(chain); + payload = makeRpcRequest(method, [blockNum, false]); + break; + + case RPC_METHODS.ETH_GET_BLOCK_BY_HASH: + const blockHash = getCachedBlockHash(chain); + if (!blockHash) { + // Skip if we don't have a cached hash yet + return; + } + payload = makeRpcRequest(method, [blockHash, false]); + break; + + case RPC_METHODS.ETH_GET_BLOCK_RECEIPTS: + const receiptBlockNum = getLatestBlockNumber(chain); + payload = makeRpcRequest(method, [receiptBlockNum]); + break; + + default: + console.error(`Unsupported method: ${method}`); + return; + } + + const requestTags = { + method: method, + chain: chain.name, + chainId: chain.id.toString(), + config: CONFIG_TAG, + }; + + // Add tags to the HTTP request params so http_req_duration gets tagged + const taggedParams = { + ...params, + tags: requestTags, + }; + + const res = http.post(getFullUrl(chain), JSON.stringify(payload), taggedParams); + const success = recordMetrics(res, method, chain, startTime); + + check(res, { + 'status is 200': (r) => r.status === 200, + 'has valid result': (r) => { + try { + const body = JSON.parse(r.body); + return body.result !== undefined && !body.error; + } catch (e) { + return false; + } + }, + }, requestTags); + + if (__ENV.DEBUG) { + console.log(`Request: method=${method}, chain=${chain.name}, duration=${Date.now() - startTime}ms, success=${success}`); + } + + // Update cache if we got a successful block response + if (success && method === RPC_METHODS.ETH_GET_BLOCK_BY_NUMBER) { + updateBlockCache(chain, res); + } + + // Small sleep to smooth request distribution + sleep(0.01); +} + +// ============================================================================= +// HELPER FUNCTIONS +// ============================================================================= + +function selectWeightedMethod() { + const rand = Math.random() * 100; + let cumulative = 0; + + for (const [method, weight] of Object.entries(METHOD_WEIGHTS)) { + cumulative += weight; + if (rand <= cumulative) { + return method; + } + } + + return RPC_METHODS.ETH_BLOCK_NUMBER; // Fallback +} + +function getLatestBlockNumber(chain) { + const cached = chainStateCache.get(chain.id); + const now = Date.now(); + + // Use cache if less than 5 seconds old + if (cached && cached.latestBlockNumber && (now - cached.timestamp) < 5000) { + return cached.latestBlockNumber; + } + + // Fetch latest block number + const payload = makeRpcRequest(RPC_METHODS.ETH_BLOCK_NUMBER); + const res = http.post(getFullUrl(chain), JSON.stringify(payload), { + headers: { 'Content-Type': 'application/json' }, + timeout: '10s', + }); + + if (res.status === 200) { + try { + const body = JSON.parse(res.body); + if (body.result) { + cached.latestBlockNumber = body.result; + cached.timestamp = now; + return body.result; + } + } catch (e) { + // Ignore parsing errors + } + } + + return 'latest'; // Fallback to 'latest' tag +} + +function getCachedBlockHash(chain) { + const cached = chainStateCache.get(chain.id); + return cached ? cached.latestBlockHash : null; +} + +function updateBlockCache(chain, res) { + try { + const body = JSON.parse(res.body); + if (body.result && body.result.hash && body.result.number) { + const cached = chainStateCache.get(chain.id); + cached.latestBlockHash = body.result.hash; + cached.latestBlockNumber = body.result.number; + cached.timestamp = Date.now(); + } + } catch (e) { + // Ignore cache update errors + } +} + +// ============================================================================= +// SUMMARY OUTPUT +// ============================================================================= +export function handleSummary(data) { + const totalReqs = data.metrics.http_reqs?.values?.count || 0; + const errorRate = data.metrics.http_req_failed?.values?.rate || 0; + const avgDuration = data.metrics.http_req_duration?.values?.avg || 0; + const p95Duration = data.metrics.http_req_duration?.values?.['p(95)'] || 0; + const p99Duration = data.metrics.http_req_duration?.values?.['p(99)'] || 0; + + console.log('\n=== Listener Workload Summary ==='); + console.log(` Chains: ${selectedChains.map(c => c.name).join(', ')}`); + console.log(` Total requests: ${totalReqs}`); + console.log(` Error rate: ${(errorRate * 100).toFixed(2)}%`); + console.log(` Avg latency: ${avgDuration.toFixed(0)}ms`); + console.log(` P95 latency: ${p95Duration.toFixed(0)}ms`); + console.log(` P99 latency: ${p99Duration.toFixed(0)}ms`); + + console.log('\n=== Method Distribution (Expected) ==='); + for (const [method, weight] of Object.entries(METHOD_WEIGHTS)) { + console.log(` ${method}: ${weight}%`); + } + + // Show per-method statistics from custom Trend metrics + console.log('\n=== Per-Method Performance ==='); + + const methodMetrics = { + 'eth_blockNumber': 'latency_eth_blockNumber', + 'eth_getBlockByNumber': 'latency_eth_getBlockByNumber', + 'eth_getBlockByHash': 'latency_eth_getBlockByHash', + 'eth_getBlockReceipts': 'latency_eth_getBlockReceipts', + }; + + for (const [method, metricName] of Object.entries(methodMetrics)) { + const metric = data.metrics[metricName]; + + if (metric && metric.values && metric.values.avg > 0) { + const avg = metric.values.avg || 0; + const min = metric.values.min || 0; + const max = metric.values.max || 0; + const p50 = metric.values.med || 0; + const p95 = metric.values['p(95)'] || 0; + const p99 = metric.values['p(99)'] || 0; + + console.log(` ${method}:`); + console.log(` - Min: ${min.toFixed(1)}ms`); + console.log(` - Avg: ${avg.toFixed(1)}ms`); + console.log(` - P50: ${p50.toFixed(1)}ms`); + console.log(` - P95: ${p95.toFixed(1)}ms`); + console.log(` - P99: ${p99.toFixed(1)}ms`); + console.log(` - Max: ${max.toFixed(1)}ms`); + } else { + console.log(` ${method}: Not tested (0% weight or no data)`); + } + } + + // Show per-method error breakdown + console.log('\n=== Error Breakdown ==='); + const totalErrors = data.metrics.method_errors?.values?.count || 0; + console.log(` Total errors: ${totalErrors}`); + + if (totalErrors > 0) { + // Extract per-method errors from tagged metrics + const methodErrorCounts = {}; + + // Parse method_errors counter with tags + if (data.metrics.method_errors?.values?.tags) { + for (const [tagKey, metricData] of Object.entries(data.metrics.method_errors.values.tags)) { + const methodMatch = tagKey.match(/method:([^,]+)/); + const chainMatch = tagKey.match(/chain:([^,]+)/); + + if (methodMatch) { + const method = methodMatch[1]; + const chain = chainMatch ? chainMatch[1] : 'unknown'; + const count = metricData.count || metricData; + + if (!methodErrorCounts[method]) { + methodErrorCounts[method] = {}; + } + methodErrorCounts[method][chain] = count; + } + } + } + + for (const [method, chainCounts] of Object.entries(methodErrorCounts)) { + console.log(` ${method}:`); + for (const [chain, count] of Object.entries(chainCounts)) { + console.log(` - ${chain}: ${count} errors`); + } + } + } + + // Show JSON-RPC error details if any + const jsonRpcErrorCount = data.metrics.jsonrpc_errors?.values?.count || 0; + if (jsonRpcErrorCount > 0) { + console.log('\n=== JSON-RPC Errors ==='); + console.log(` Total JSON-RPC errors: ${jsonRpcErrorCount}`); + } + + return { + stdout: textSummary(data, { indent: ' ', enableColors: true }), + }; +} diff --git a/listener/charts/listener/.helmignore b/listener/charts/listener/.helmignore new file mode 100644 index 0000000000..57d14664ee --- /dev/null +++ b/listener/charts/listener/.helmignore @@ -0,0 +1,25 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ +charts/*/Chart.lock +charts/*/charts/* diff --git a/listener/charts/listener/Chart.yaml b/listener/charts/listener/Chart.yaml new file mode 100644 index 0000000000..2ed8567dc8 --- /dev/null +++ b/listener/charts/listener/Chart.yaml @@ -0,0 +1,23 @@ +apiVersion: v2 +name: listener +description: >- + Helm chart for the Zama blockchain listener. + Deploys one listener instance per chain, with shared PostgreSQL, Redis/RabbitMQ + infrastructure, and an optional eRPC proxy. +type: application +version: 0.1.7 +appVersion: "0.1.0" + +dependencies: + - name: postgresql + version: "~16.4" + repository: https://charts.bitnami.com/bitnami + condition: postgresql.enabled + - name: redis + version: "~20.6" + repository: https://charts.bitnami.com/bitnami + condition: redis.enabled + - name: rabbitmq + version: "~15.1" + repository: https://charts.bitnami.com/bitnami + condition: rabbitmq.enabled diff --git a/listener/charts/listener/README.md b/listener/charts/listener/README.md new file mode 100644 index 0000000000..2e51d0712c --- /dev/null +++ b/listener/charts/listener/README.md @@ -0,0 +1,281 @@ +# Listener Helm Chart + +Deploys one blockchain listener instance per chain, with shared PostgreSQL, Redis/RabbitMQ infrastructure, and an optional [eRPC](https://github.com/erpc/erpc) proxy. + +## Architecture + +``` + +------------------+ + | eRPC proxy | + | (optional) | + +--------+---------+ + | + +-------------------+-------------------+ + | | ++--------v---------+ +----------------v--------+ +| listener:ethereum | | listener:base-sepolia | ++--------+---------+ +----------------+--------+ + | | + +-------------------+-------------------+ + | + +--------------+--------------+ + | | + +--------v--------+ +--------v----------+ + | PostgreSQL | | Redis / RabbitMQ | + +-----------------+ +-------------------+ +``` + +Each listener entry in `values.yaml` creates its own Deployment and ConfigMap. All listeners share the same database and broker infrastructure. + +## Prerequisites + +- Helm 3.x +- Kubernetes 1.24+ + +## Quick Start + +```bash +helm dependency update charts/listener +helm install listener charts/listener -n listener --create-namespace +``` + +## Configuration + +### Config merge strategy + +Both the listener and eRPC configs use a **symlink + deep-merge** pattern. Canonical config files live in `config/` at the repo root and are symlinked into `charts/listener/configs/`. The Helm templates load them via `.Files.Get` and apply overrides with `mergeOverwrite`. + +This ensures that whenever the application config changes, the Helm chart is affected and must be version-bumped. + +``` +charts/listener/configs/ + listener-default.yaml -> config/listener-default.yaml + erpc-base.yaml -> config/erpc-base.yaml + erpc-public.yaml -> config/erpc-public.yaml +``` + +--- + +### Listener + +Listener config is built from a **3-layer deep merge** (last wins): + +| Layer | Source | Purpose | +|-------|--------|---------| +| 1. Base defaults | `configs/listener-default.yaml` | All Rust `Settings` struct defaults | +| 2. Common overrides | `values.yaml` -> `commonConfig` | Operator-shared overrides (e.g., broker type) | +| 3. Per-listener | `values.yaml` -> `listeners[].config` | Chain-specific values (chain_id, rpc_url, etc.) | + +The listener `name` is auto-injected from `listeners[].name` and cannot drift. + +#### Adding a new chain + +Add an entry to `listeners[]`. Only specify fields that differ from the defaults: + +```yaml +listeners: + - name: polygon + config: + blockchain: + chain_id: 137 + rpc_url: http://listener-erpc:4000/listener-indexer/evm/137 + network: polygon-mainnet + strategy: + block_start_on_first_start: 70000000 + range_size: 50 + env: [] +``` + +Everything else (database, broker, pool settings, strategy defaults) inherits from the base file + `commonConfig`. + +#### Overriding shared settings + +Use `commonConfig` for values that apply to **all** listeners. Only add keys that differ from `config/listener-default.yaml`: + +```yaml +commonConfig: + broker: + broker_type: redis # override default amqp -> redis + database: + pool: + max_connections: 20 # increase for high-throughput clusters +``` + +#### Per-listener overrides + +The `config` block uses the **same nested structure** as the Rust config file. Any field from `config/listener-default.yaml` can be overridden: + +```yaml +listeners: + - name: ethereum + config: + broker: + ensure_publish: true # enable durability for this chain only + blockchain: + chain_id: 1 + rpc_url: http://listener-erpc:4000/listener-indexer/evm/1 + network: ethereum-mainnet + finality_depth: 128 + strategy: + block_start_on_first_start: 24572795 + range_size: 10 + max_parallel_requests: 10 +``` + +#### Per-listener resource and scheduling overrides + +Each listener can override resources, security context, and scheduling independently: + +```yaml +listeners: + - name: ethereum + config: { ... } + resources: + requests: + cpu: "2" + memory: 2Gi + nodeSelector: + dedicated: blockchain + tolerations: + - key: dedicated + value: blockchain + effect: NoSchedule +``` + +#### Adding a new config field + +When a new field is added to the Rust `Settings` struct: + +1. Add it to `config/listener-default.yaml` with its default value +2. The Helm chart picks it up automatically via the symlink +3. Per-listener overrides work immediately (same nested structure) +4. Bump `Chart.yaml` version + +--- + +### eRPC + +The eRPC proxy is an optional component that provides RPC load balancing, failover, and caching. + +#### Config profiles + +eRPC uses a **base config file** selected by the `erpc.baseConfig` field. Available profiles: + +| Profile | File | Use case | +|---------|------|----------| +| `erpc-base.yaml` | Minimal defaults | Standalone eRPC for generic apps (default) | +| `erpc-public.yaml` | Public nodes, listener-tuned | Listener clusters using public RPC endpoints | + +```yaml +erpc: + enabled: true + baseConfig: erpc-base.yaml # or erpc-public.yaml for listener clusters +``` + +#### Adding a new eRPC profile + +1. Create the config file: `config/erpc-.yaml` +2. Symlink it: `ln -s ../../../config/erpc-.yaml charts/listener/configs/erpc-.yaml` +3. Deploy with: `--set erpc.baseConfig=erpc-.yaml` + +#### Partial overrides + +Use `erpc.config` to deep-merge overrides on top of the base profile without replacing the entire config: + +```yaml +erpc: + baseConfig: erpc-public.yaml + config: + logLevel: info + server: + maxTimeout: 60s +``` + +#### Full replacement + +To completely bypass the base config file, use `--set-file`: + +```bash +helm install listener charts/listener \ + --set-file erpc.configFile=path/to/custom-erpc.yaml +``` + +This ignores both the base config and `erpc.config` overrides. + +#### Disabling eRPC + +```yaml +erpc: + enabled: false +``` + +Listener `rpc_url` values should then point directly to your RPC provider. + +--- + +### Secrets + +Sensitive values (database URL, broker URL) are injected via environment variables referencing a Kubernetes Secret. + +#### With External Secrets Operator (default) + +```yaml +externalSecret: + enabled: true # assumes Secret "listener-secrets" already exists +secretName: listener-secrets +``` + +#### Without External Secrets Operator + +```yaml +externalSecret: + enabled: false +fallbackSecret: + name: listener-secrets + data: + database-url: "postgres://postgres:postgres@listener-postgresql:5432/listener" + broker-url: "redis://listener-redis-master:6379" +``` + +--- + +### Sub-chart dependencies + +| Dependency | Default | Toggle | +|------------|---------|--------| +| PostgreSQL | enabled | `postgresql.enabled: false` | +| Redis | enabled | `redis.enabled: false` | +| RabbitMQ | disabled | `rabbitmq.enabled: true` | + +Set `enabled: false` to use externally managed services. See the [Bitnami charts documentation](https://github.com/bitnami/charts) for sub-chart configuration options. + +--- + +## Values Reference + +| Key | Default | Description | +|-----|---------|-------------| +| `image.repository` | `ghcr.io/zama-ai/listener` | Listener container image | +| `image.tag` | `""` (uses appVersion) | Image tag override | +| `commonConfig` | `{broker: {broker_type: redis}}` | Shared config overrides (merged on top of base defaults) | +| `listeners` | 2 entries (ethereum, base-sepolia) | Per-chain listener instances | +| `listeners[].name` | - | Chain name (used for Deployment/ConfigMap naming) | +| `listeners[].config` | `{}` | Chain-specific config overrides (same structure as Rust config) | +| `listeners[].env` | `[]` | Per-listener env var overrides | +| `listeners[].resources` | inherits root `resources` | Per-listener resource overrides | +| `secretName` | `listener-secrets` | K8s Secret name for sensitive values | +| `erpc.enabled` | `true` | Deploy eRPC proxy | +| `erpc.baseConfig` | `erpc-base.yaml` | Base eRPC config profile | +| `erpc.config` | `{}` | Partial overrides deep-merged on top of base config | +| `erpc.configFile` | `""` | Full config replacement (via `--set-file`) | +| `erpc.replicas` | `1` | eRPC replica count | +| `erpc.image.tag` | `0.0.63` | eRPC image version | +| `erpc.service.httpPort` | `4000` | eRPC HTTP port | +| `erpc.service.metricsPort` | `4001` | eRPC Prometheus metrics port | +| `erpc.podSecurityContext` | nonroot + `seccompProfile: RuntimeDefault` | Pod-level security context for the eRPC pod | +| `erpc.securityContext` | readOnlyRoot + capDropAll + `seccompProfile: RuntimeDefault` | Container-level security context for the eRPC container | +| `externalSecret.enabled` | `true` | Use pre-existing Secrets (ESO or manual) | +| `fallbackSecret.data` | `{}` | Secret data when ESO is disabled | +| `postgresql.enabled` | `true` | Deploy PostgreSQL sub-chart | +| `redis.enabled` | `true` | Deploy Redis sub-chart | +| `rabbitmq.enabled` | `false` | Deploy RabbitMQ sub-chart | diff --git a/listener/charts/listener/configs/erpc-base.yaml b/listener/charts/listener/configs/erpc-base.yaml new file mode 120000 index 0000000000..cd15d5fac2 --- /dev/null +++ b/listener/charts/listener/configs/erpc-base.yaml @@ -0,0 +1 @@ +../../../config/erpc-base.yaml \ No newline at end of file diff --git a/listener/charts/listener/configs/erpc-public.yaml b/listener/charts/listener/configs/erpc-public.yaml new file mode 120000 index 0000000000..147e318415 --- /dev/null +++ b/listener/charts/listener/configs/erpc-public.yaml @@ -0,0 +1 @@ +../../../config/erpc-public.yaml \ No newline at end of file diff --git a/listener/charts/listener/configs/listener-default.yaml b/listener/charts/listener/configs/listener-default.yaml new file mode 120000 index 0000000000..5ceead855d --- /dev/null +++ b/listener/charts/listener/configs/listener-default.yaml @@ -0,0 +1 @@ +../../../config/listener-default.yaml \ No newline at end of file diff --git a/listener/charts/listener/templates/_helpers.tpl b/listener/charts/listener/templates/_helpers.tpl new file mode 100644 index 0000000000..4d613d6fc3 --- /dev/null +++ b/listener/charts/listener/templates/_helpers.tpl @@ -0,0 +1,156 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "listener.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "listener.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "listener.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels (chart-level). +*/}} +{{- define "listener.labels" -}} +helm.sh/chart: {{ include "listener.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/part-of: {{ include "listener.name" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +{{- end }} + +{{/* +Per-listener fully qualified name: -- +Truncated at 63 chars for DNS compliance. +Usage: {{ include "listener.instanceName" (dict "root" . "listener" $listener) }} +*/}} +{{- define "listener.instanceName" -}} +{{- $base := include "listener.fullname" .root }} +{{- printf "%s-%s" $base .listener.name | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Per-listener labels. +Usage: {{ include "listener.instanceLabels" (dict "root" . "listener" $listener) }} +*/}} +{{- define "listener.instanceLabels" -}} +{{ include "listener.labels" .root }} +app.kubernetes.io/name: {{ include "listener.name" .root }} +app.kubernetes.io/instance: {{ .root.Release.Name }} +app.kubernetes.io/component: listener +listener.zama.ai/chain: {{ .listener.name }} +{{- $chainId := dig "blockchain" "chain_id" "" (.listener.config | default dict) }} +{{- if $chainId }} +listener.zama.ai/chain-id: {{ $chainId | quote }} +{{- end }} +{{- end }} + +{{/* +Per-listener selector labels. +Usage: {{ include "listener.instanceSelectorLabels" (dict "root" . "listener" $listener) }} +*/}} +{{- define "listener.instanceSelectorLabels" -}} +app.kubernetes.io/name: {{ include "listener.name" .root }} +app.kubernetes.io/instance: {{ .root.Release.Name }} +app.kubernetes.io/component: listener +listener.zama.ai/chain: {{ .listener.name }} +{{- end }} + +{{/* +Create the name of the service account to use. +*/}} +{{- define "listener.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "listener.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Merge root env with per-listener env. Per-listener entries win on name conflict. +Env values support tpl expressions (e.g., {{ .Values.secretName }}). +Usage: {{ include "listener.mergedEnv" (dict "root" . "listener" $listener) }} + +Strategy: emit per-listener env first, then root env entries whose name does +NOT appear in the per-listener list. All values are processed through tpl. +*/}} +{{- define "listener.mergedEnv" -}} +{{- $listenerEnv := default (list) .listener.env -}} +{{- $listenerNames := list -}} +{{- range $listenerEnv -}} + {{- $listenerNames = append $listenerNames .name -}} +{{- end -}} +{{- /* Emit per-listener env (processed through tpl) */ -}} +{{- range $listenerEnv }} +- {{ tpl (toYaml .) $.root | nindent 2 | trim }} +{{- end }} +{{- /* Emit root env entries not overridden by per-listener (processed through tpl) */ -}} +{{- range .root.Values.env }} +{{- if not (has .name $listenerNames) }} +- {{ tpl (toYaml .) $.root | nindent 2 | trim }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Effective metrics port for a listener instance. +Merges the same 3 layers as configmap.yaml and extracts telemetry.metrics_port. +Usage: {{ include "listener.metricsPort" (dict "root" $ "listener" $listener) }} +*/}} +{{- define "listener.metricsPort" -}} +{{- $defaultConfig := .root.Files.Get "configs/listener-default.yaml" | fromYaml }} +{{- $common := .root.Values.commonConfig | default dict }} +{{- $perListener := .listener.config | default dict }} +{{- $merged := mergeOverwrite (deepCopy $defaultConfig) $common $perListener }} +{{- dig "telemetry" "metrics_port" 9090 $merged }} +{{- end }} + +{{/* +eRPC fully qualified name. +*/}} +{{- define "listener.erpcName" -}} +{{- printf "%s-erpc" (include "listener.fullname" .) | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +eRPC labels. +*/}} +{{- define "listener.erpcLabels" -}} +{{ include "listener.labels" . }} +app.kubernetes.io/name: {{ include "listener.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/component: erpc +{{- end }} + +{{/* +eRPC selector labels. +*/}} +{{- define "listener.erpcSelectorLabels" -}} +app.kubernetes.io/name: {{ include "listener.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/component: erpc +{{- end }} diff --git a/listener/charts/listener/templates/configmap.yaml b/listener/charts/listener/templates/configmap.yaml new file mode 100644 index 0000000000..fc17eb1934 --- /dev/null +++ b/listener/charts/listener/templates/configmap.yaml @@ -0,0 +1,56 @@ +{{/* +Per-listener ConfigMaps. +Each listener gets its own ConfigMap built from a 3-layer deep merge: + 1. configs/listener-default.yaml (application defaults, symlinked from config/) + 2. .Values.commonConfig (operator-shared overrides) + 3. per-listener .config (chain-specific overrides) +The listener name is always injected from .name so it cannot drift. +*/}} +{{- $defaultConfig := .Files.Get "configs/listener-default.yaml" | fromYaml }} +{{- $common := .Values.commonConfig | default dict }} +{{- if .Values.listeners }} +{{- range $listener := .Values.listeners }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "listener.instanceName" (dict "root" $ "listener" $listener) }} + labels: + {{- include "listener.instanceLabels" (dict "root" $ "listener" $listener) | nindent 4 }} +data: + config.yaml: | + {{- $perListener := $listener.config | default dict }} + {{- $nameOverride := dict "name" $listener.name }} + {{- $merged := mergeOverwrite (deepCopy $defaultConfig) $common $perListener $nameOverride }} + {{- toYaml $merged | nindent 4 }} +{{- end }} +{{- end }} + +{{/* ── eRPC ConfigMap ───────────────────────────────────────────────── */}} +{{/* + Base config: configs/ (bundled in chart via symlink). + The operator picks the profile at install time (erpc-public.yaml, erpc-gateway.yaml, etc.). + Partial overrides: .Values.erpc.config (dict) is deep-merged on top. + Full replacement: --set-file erpc.configFile=path/to/full-config.yaml +*/}} +{{- if .Values.erpc.enabled }} +{{- $erpcRaw := .Files.Get (printf "configs/%s" .Values.erpc.baseConfig) }} +{{- if .Values.erpc.configFile }} + {{- $erpcRaw = .Values.erpc.configFile }} +{{- end }} +{{- if $erpcRaw }} +{{- $base := $erpcRaw | fromYaml }} +{{- $overrides := .Values.erpc.config | default dict }} +{{- $merged := mergeOverwrite $base $overrides }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "listener.erpcName" . }} + labels: + {{- include "listener.erpcLabels" . | nindent 4 }} +data: + erpc.yaml: | + {{- toYaml $merged | nindent 4 }} +{{- end }} +{{- end }} diff --git a/listener/charts/listener/templates/deployment.yaml b/listener/charts/listener/templates/deployment.yaml new file mode 100644 index 0000000000..d213b909b6 --- /dev/null +++ b/listener/charts/listener/templates/deployment.yaml @@ -0,0 +1,171 @@ +{{- if .Values.listeners }} +{{- range $listener := .Values.listeners }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "listener.instanceName" (dict "root" $ "listener" $listener) }} + labels: + {{- include "listener.instanceLabels" (dict "root" $ "listener" $listener) | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + {{- include "listener.instanceSelectorLabels" (dict "root" $ "listener" $listener) | nindent 6 }} + template: + metadata: + annotations: + checksum/config: {{ printf "%s%s%s" ($.Files.Get "configs/listener-default.yaml") (toYaml $.Values.commonConfig) (toYaml ($listener.config | default dict)) | sha256sum }} + {{- with $.Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "listener.instanceLabels" (dict "root" $ "listener" $listener) | nindent 8 }} + {{- with $.Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with $.Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "listener.serviceAccountName" $ }} + {{- with (default $.Values.podSecurityContext $listener.podSecurityContext) }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: listener + {{- with (default $.Values.securityContext $listener.securityContext) }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + image: "{{ $.Values.image.repository }}:{{ $.Values.image.tag | default $.Chart.AppVersion }}" + imagePullPolicy: {{ $.Values.image.pullPolicy }} + {{- if $.Values.metrics.enabled }} + ports: + - name: metrics + containerPort: {{ include "listener.metricsPort" (dict "root" $ "listener" $listener) }} + protocol: TCP + {{- end }} + args: + - "--config" + - "/config/config.yaml" + env: + {{- include "listener.mergedEnv" (dict "root" $ "listener" $listener) | nindent 12 }} + {{- with $.Values.envFrom }} + envFrom: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with (default $.Values.resources $listener.resources) }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: config + mountPath: /config + readOnly: true + volumes: + - name: config + configMap: + name: {{ include "listener.instanceName" (dict "root" $ "listener" $listener) }} + {{- with (default $.Values.nodeSelector $listener.nodeSelector) }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with (default $.Values.affinity $listener.affinity) }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with (default $.Values.tolerations $listener.tolerations) }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} +{{- end }} + +{{/* ── eRPC Deployment ──────────────────────────────────────────────── */}} +{{- if .Values.erpc.enabled }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "listener.erpcName" . }} + labels: + {{- include "listener.erpcLabels" . | nindent 4 }} +spec: + replicas: {{ .Values.erpc.replicas | default 1 }} + selector: + matchLabels: + {{- include "listener.erpcSelectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + checksum/config: {{ .Files.Get (printf "configs/%s" .Values.erpc.baseConfig) | sha256sum }} + labels: + {{- include "listener.erpcLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.erpc.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.erpc.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.erpc.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.erpc.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: erpc + {{- with .Values.erpc.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + image: "{{ .Values.erpc.image.repository }}:{{ .Values.erpc.image.tag }}" + imagePullPolicy: {{ .Values.erpc.image.pullPolicy }} + {{- with .Values.erpc.args }} + args: + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: {{ .Values.erpc.service.httpPort }} + protocol: TCP + - name: metrics + containerPort: {{ .Values.erpc.service.metricsPort }} + protocol: TCP + {{- with .Values.erpc.env }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.erpc.envFrom }} + envFrom: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.erpc.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: erpc-config + mountPath: /config + readOnly: true + - name: tmp + mountPath: /tmp + volumes: + - name: erpc-config + configMap: + name: {{ include "listener.erpcName" . }} + - name: tmp + emptyDir: {} +{{- end }} diff --git a/listener/charts/listener/templates/secret.yaml b/listener/charts/listener/templates/secret.yaml new file mode 100644 index 0000000000..8d7df9235e --- /dev/null +++ b/listener/charts/listener/templates/secret.yaml @@ -0,0 +1,18 @@ +{{/* +Fallback Secret — only rendered when externalSecret.enabled=false. +When using the external-secrets operator, this template is skipped entirely +and pods reference Secrets created by ExternalSecret CRs. +*/}} +{{- if not .Values.externalSecret.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.fallbackSecret.name | default "listener-secrets" }} + labels: + {{- include "listener.labels" . | nindent 4 }} +type: Opaque +stringData: + {{- range $key, $value := .Values.fallbackSecret.data }} + {{ $key }}: {{ $value | quote }} + {{- end }} +{{- end }} diff --git a/listener/charts/listener/templates/service.yaml b/listener/charts/listener/templates/service.yaml new file mode 100644 index 0000000000..814366be23 --- /dev/null +++ b/listener/charts/listener/templates/service.yaml @@ -0,0 +1,49 @@ +{{/* ── Per-listener metrics Service ──────────────────────────────── */}} +{{- if and .Values.listeners .Values.metrics.enabled }} +{{- range $listener := .Values.listeners }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "listener.instanceName" (dict "root" $ "listener" $listener) }} + labels: + {{- include "listener.instanceLabels" (dict "root" $ "listener" $listener) | nindent 4 }} + {{- with $.Values.metrics.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ $.Values.metrics.service.type | default "ClusterIP" }} + selector: + {{- include "listener.instanceSelectorLabels" (dict "root" $ "listener" $listener) | nindent 4 }} + ports: + - name: metrics + port: {{ include "listener.metricsPort" (dict "root" $ "listener" $listener) }} + targetPort: metrics + protocol: TCP +{{- end }} +{{- end }} + +{{/* ── eRPC Service ───────────────────────────────────────────────── */}} +{{- if .Values.erpc.enabled }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "listener.erpcName" . }} + labels: + {{- include "listener.erpcLabels" . | nindent 4 }} +spec: + type: {{ .Values.erpc.service.type | default "ClusterIP" }} + selector: + {{- include "listener.erpcSelectorLabels" . | nindent 4 }} + ports: + - name: http + port: {{ .Values.erpc.service.httpPort }} + targetPort: http + protocol: TCP + - name: metrics + port: {{ .Values.erpc.service.metricsPort }} + targetPort: metrics + protocol: TCP +{{- end }} diff --git a/listener/charts/listener/templates/serviceaccount.yaml b/listener/charts/listener/templates/serviceaccount.yaml new file mode 100644 index 0000000000..6c4b96cea2 --- /dev/null +++ b/listener/charts/listener/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "listener.serviceAccountName" . }} + labels: + {{- include "listener.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automount }} +{{- end }} diff --git a/listener/charts/listener/templates/servicemonitor.yaml b/listener/charts/listener/templates/servicemonitor.yaml new file mode 100644 index 0000000000..0dc467cc5c --- /dev/null +++ b/listener/charts/listener/templates/servicemonitor.yaml @@ -0,0 +1,60 @@ +{{- if and .Values.listeners .Values.metrics.enabled .Values.metrics.serviceMonitor.enabled }} +{{- range $listener := .Values.listeners }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "listener.instanceName" (dict "root" $ "listener" $listener) }} + {{- with $.Values.metrics.serviceMonitor.namespace }} + namespace: {{ . }} + {{- end }} + labels: + {{- include "listener.instanceLabels" (dict "root" $ "listener" $listener) | nindent 4 }} + {{- with $.Values.metrics.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with $.Values.metrics.serviceMonitor.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "listener.instanceSelectorLabels" (dict "root" $ "listener" $listener) | nindent 6 }} + endpoints: + - port: metrics + path: {{ $.Values.metrics.path }} + interval: {{ $.Values.metrics.serviceMonitor.interval }} + scrapeTimeout: {{ $.Values.metrics.serviceMonitor.scrapeTimeout }} +{{- end }} +{{- end }} + +{{/* ── eRPC ServiceMonitor ─────────────────────────────────────── */}} +{{- if and .Values.erpc.enabled .Values.metrics.enabled .Values.erpc.serviceMonitor.enabled }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "listener.erpcName" . }} + {{- with .Values.erpc.serviceMonitor.namespace }} + namespace: {{ . }} + {{- end }} + labels: + {{- include "listener.erpcLabels" . | nindent 4 }} + {{- with .Values.erpc.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.erpc.serviceMonitor.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "listener.erpcSelectorLabels" . | nindent 6 }} + endpoints: + - port: metrics + path: {{ .Values.erpc.serviceMonitor.path | default "/metrics" }} + interval: {{ .Values.erpc.serviceMonitor.interval | default "30s" }} + scrapeTimeout: {{ .Values.erpc.serviceMonitor.scrapeTimeout | default "10s" }} +{{- end }} diff --git a/listener/charts/listener/values.yaml b/listener/charts/listener/values.yaml new file mode 100644 index 0000000000..57cbde768d --- /dev/null +++ b/listener/charts/listener/values.yaml @@ -0,0 +1,278 @@ +# -- Listener image +image: + repository: ghcr.io/zama-ai/listener + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +# -- ServiceAccount configuration +serviceAccount: + create: true + automount: true + annotations: {} + name: "" + +# -- Shared annotations/labels applied to all listener pods +podAnnotations: {} +podLabels: {} + +# --------------------------------------------------------------------------- +# Shared config — operator-level overrides applied on top of the base defaults +# in configs/listener-default.yaml (symlinked from config/listener-default.yaml). +# +# Merge order (last wins): +# 1. configs/listener-default.yaml (application defaults from Rust code) +# 2. commonConfig (this block — shared across all listeners) +# 3. listeners[].config (chain-specific overrides) +# +# Only add keys here that DIFFER from the base defaults. The base file already +# carries every field with the Rust-matching default value. +# --------------------------------------------------------------------------- +commonConfig: + broker: + broker_type: redis + +# --------------------------------------------------------------------------- +# Shared env vars — merged into all listener pods + migration job. +# Use valueFrom.secretKeyRef to reference Secrets created by ESO or manually. +# --------------------------------------------------------------------------- +# -- Name of the K8s Secret holding sensitive values (database URL, broker URL). +# Referenced by env vars below. Must match fallbackSecret.name when ESO is +# disabled, or the Secret name created by ExternalSecret when ESO is enabled. +secretName: listener-secrets + +env: + - name: APP_DATABASE__DB_URL + valueFrom: + secretKeyRef: + name: "database-credentials" + key: database-url + # - name: APP_BROKER__BROKER_URL + # valueFrom: + # secretKeyRef: + # name: "broker-credentials" + # key: broker-url + +envFrom: [] + +# --------------------------------------------------------------------------- +# Per-chain listener instances. +# Each entry creates its own Deployment + ConfigMap. +# +# The "config" block uses the SAME nested structure as the Rust config file +# (config/listener-default.yaml). Only specify fields you want to override; +# everything else inherits from the base defaults + commonConfig. +# --------------------------------------------------------------------------- +listeners: [] + # - name: ethereum + # config: + # blockchain: + # chain_id: 1 + # rpc_url: http://listener-erpc:4000/listener-indexer/evm/1 + # network: ethereum-mainnet + # strategy: + # block_start_on_first_start: 24572795 + # range_size: 10 + # max_parallel_requests: 10 + # # -- Per-listener env overrides (merged with root env, per-listener wins) + # env: [] + # # -- Per-listener resource overrides + # # resources: {} + # # -- Per-listener security context overrides + # # podSecurityContext: {} + # # securityContext: {} + # # -- Per-listener scheduling overrides + # # nodeSelector: {} + # # tolerations: [] + # # affinity: {} + + # - name: base-sepolia + # config: + # blockchain: + # chain_id: 84532 + # rpc_url: http://listener-erpc:4000/listener-indexer/evm/84532 + # network: base-sepolia + # strategy: + # block_start_on_first_start: 38342520 + # range_size: 100 + # max_parallel_requests: 50 + # env: [] + +# --------------------------------------------------------------------------- +# Security context — hardened defaults for distroless nonroot base image. +# Overridable per-listener. +# +# seccompProfile.type=RuntimeDefault is required by the Pod Security Standard +# "restricted (strict)" profile and the Kyverno restrict-seccomp-strict policy. +# --------------------------------------------------------------------------- +podSecurityContext: + runAsNonRoot: true + runAsUser: 65534 + fsGroup: 65534 + seccompProfile: + type: RuntimeDefault + +securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + seccompProfile: + type: RuntimeDefault + +# -- Default resource requests/limits for listener pods +resources: + requests: + cpu: "1" + memory: 1Gi + limits: + cpu: "1" + memory: 1Gi + +# -- Prometheus metrics endpoint configuration. +# The metrics port is derived from the telemetry config chain +# (listener-default.yaml -> commonConfig -> listeners[].config), +# so there is no separate port setting here. +metrics: + enabled: true + path: /metrics + service: + type: ClusterIP + annotations: {} + serviceMonitor: + enabled: false + namespace: "" + interval: 30s + scrapeTimeout: 10s + labels: {} + annotations: {} + +# -- Node scheduling constraints +nodeSelector: {} +tolerations: [] +affinity: {} + +# --------------------------------------------------------------------------- +# ExternalSecret integration. +# When true, the chart assumes Secrets already exist (created by the +# external-secrets operator or manually). No Secret resource is rendered. +# When false, the fallbackSecret below is rendered instead. +# --------------------------------------------------------------------------- +externalSecret: + enabled: true + +# --------------------------------------------------------------------------- +# Fallback Secret — only rendered when externalSecret.enabled=false. +# For environments without the external-secrets operator. +# --------------------------------------------------------------------------- +fallbackSecret: + name: listener-secrets + data: {} + # database-url: "postgres://postgres:postgres@listener-postgresql:5432/listener" + # broker-url: "redis://listener-redis-master:6379" + + +# --------------------------------------------------------------------------- +# Optional sub-chart dependencies. +# Set enabled=false to use externally managed services instead. +# --------------------------------------------------------------------------- +postgresql: + enabled: true + auth: + database: listener + username: postgres + password: postgres + primary: + persistence: + size: 10Gi + +redis: + enabled: true + architecture: standalone + master: + persistence: + enabled: true + size: 1Gi + auth: + enabled: false + +rabbitmq: + enabled: false + auth: + username: user + password: pass + +# --------------------------------------------------------------------------- +# eRPC proxy (optional inline Deployment — no official Helm chart exists). +# --------------------------------------------------------------------------- +erpc: + enabled: false + # -- CLI arguments passed to the eRPC container. The image's entrypoint is + # `erpc`, which takes the config-file path as a positional argument. Without + # an explicit path eRPC searches a hardcoded set of well-known locations + # (/home/nonroot/erpc.yaml, /erpc.yaml, /root/erpc.yaml, ...) — none of which + # match the ConfigMap mount at /config/erpc.yaml below, so the rendered + # config would be silently ignored and the proxy would boot with default + # public endpoints. The default below points at the mounted ConfigMap. + args: + - /config/erpc.yaml + # -- Base eRPC config profile (must exist as a symlink in charts/listener/configs/). + # Available profiles: erpc-base.yaml (minimal), erpc-public.yaml (public nodes), etc. + baseConfig: erpc-base.yaml + image: + repository: ghcr.io/erpc/erpc + tag: "0.0.63" + pullPolicy: IfNotPresent + replicas: 1 + resources: + requests: + cpu: 250m + memory: 256Mi + limits: + cpu: "1" + memory: 512Mi + service: + type: ClusterIP + httpPort: 4000 + metricsPort: 4001 + env: [] + envFrom: [] + nodeSelector: {} + tolerations: [] + affinity: {} + # -- Security context — hardened defaults for the eRPC distroless image. + # seccompProfile.type=RuntimeDefault satisfies the Pod Security Standard + # "restricted (strict)" profile and the Kyverno restrict-seccomp-strict + # policy. Override to {} to opt out. + podSecurityContext: + runAsNonRoot: true + runAsUser: 65534 + fsGroup: 65534 + seccompProfile: + type: RuntimeDefault + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + seccompProfile: + type: RuntimeDefault + serviceMonitor: + enabled: false + namespace: "" + path: /metrics + interval: 30s + scrapeTimeout: 10s + labels: {} + annotations: {} + # -- Partial overrides deep-merged on top of configs/erpc-public.yaml. + # Example: set logLevel to info without replacing the entire config. + config: {} + # logLevel: info + # -- Full replacement (ignores base + config overrides). + # Usage: --set-file erpc.configFile=path/to/custom-erpc.yaml + configFile: "" diff --git a/listener/config/erpc-base.yaml b/listener/config/erpc-base.yaml new file mode 100644 index 0000000000..341bec4f43 --- /dev/null +++ b/listener/config/erpc-base.yaml @@ -0,0 +1,81 @@ +# ============================================================================= +# erpc-base.yaml — Minimal eRPC configuration +# ============================================================================= +# Baseline config with sane defaults for server, metrics, and rate limiting. +# No networks or upstreams defined — those are added by profile-specific +# configs (erpc-public.yaml, erpc-gateway.yaml) or via Helm value overrides. +# +# This file is the default baseConfig in the Helm chart. +# ============================================================================= + +# ============================================================================= +# SERVER +# ============================================================================= +server: + httpHostV4: 0.0.0.0 + httpPortV4: 4000 + maxTimeout: 30s + enableGzip: true + +# ============================================================================= +# METRICS (Prometheus) +# ============================================================================= +metrics: + enabled: true + hostV4: 0.0.0.0 + port: 4001 + +# ============================================================================= +# RATE LIMITERS +# ============================================================================= +rateLimiters: + store: + driver: memory + budgets: + - id: default-budget + rules: + - method: "*" + maxCount: 10000000 + period: second + +# ============================================================================= +# PROJECT +# ============================================================================= +# Single project with generic defaults. Override via Helm values (.erpc.config) +# or use a profile-specific config (--set erpc.baseConfig=erpc-public.yaml). +# ============================================================================= +projects: + - id: main + networkDefaults: + directiveDefaults: + retryEmpty: true + retryPending: false + failsafe: + - matchMethod: "*" + timeout: + duration: 15s + retry: + maxAttempts: 3 + delay: 500ms + backoffMaxDelay: 5s + backoffFactor: 2.0 + jitter: 200ms + upstreamDefaults: + jsonRpc: + supportsBatch: true + batchMaxSize: 100 + batchMaxWait: 50ms + failsafe: + - matchMethod: "*" + timeout: + duration: 10s + retry: + maxAttempts: 2 + delay: 300ms + circuitBreaker: + failureThresholdCount: 10 + failureThresholdCapacity: 20 + halfOpenAfter: 30s + successThresholdCount: 3 + successThresholdCapacity: 5 + networks: [] diff --git a/listener/config/erpc-public.yaml b/listener/config/erpc-public.yaml new file mode 100644 index 0000000000..21a3f0d0d9 --- /dev/null +++ b/listener/config/erpc-public.yaml @@ -0,0 +1,1133 @@ +logLevel: warn + +# ============================================================================= +# erpc-public.yaml — PUBLIC NODES ONLY (repository provider) +# ============================================================================= +# Uses exclusively the eRPC public endpoints repository @ +# https://evm-public-endpoints.erpc.cloud (chainlist.org, chainid.network, viem) +# +# Tuned for listener-indexer: timeouts ≤ 10s to match SemEvmRpcProvider HTTP +# client timeout. Conservative hedges (quantile 0.98, maxCount 1) to avoid +# rate-limit cascade on public nodes; no hedge for eth_blockNumber (55% volume). +# ============================================================================= + +# ============================================================================= +# SERVER +# ============================================================================= +server: + httpHostV4: 0.0.0.0 + httpPortV4: 4000 + maxTimeout: 20s # Must cover ethereum receipt budget: 3 upstreams × 5s = 15s + overhead + enableGzip: true + +# ============================================================================= +# METRICS (Prometheus) +# ============================================================================= +metrics: + enabled: true + hostV4: 0.0.0.0 + port: 4001 + +# ============================================================================= +# NO DATABASE / NO CACHE +# ============================================================================= +# Same as erpc.yaml — every request goes directly to upstreams. +# ============================================================================= + +# ============================================================================= +# RATE LIMITERS +# ============================================================================= +# Public nodes only — eRPC not the bottleneck; upstream limits drive backoff. +# ============================================================================= +rateLimiters: + store: + driver: memory + + budgets: + # Very high limits - eRPC should not be the bottleneck + # Let the public nodes' own rate limits be the constraint + - id: very-high-limits + rules: + - method: "*" + maxCount: 10000000 # Effectively unlimited, let upstream responses drive backoff + period: second + +# ============================================================================= +# PROJECT — listener-indexer (PUBLIC ONLY) +# ============================================================================= +projects: + - id: listener-indexer + + # ========================================================================= + # SCORING / ROUTING REACTIVITY + # ========================================================================= + # Public endpoints degrade quickly under shared load, so routing must react fast. + scoreMetricsWindowSize: 5m + scoreGranularity: method + scoreRefreshInterval: 10s + scorePenaltyDecayRate: 0.80 + scoreSwitchHysteresis: 0.02 + scoreMinSwitchInterval: 20s + + # ========================================================================= + # NETWORK DEFAULTS + # ========================================================================= + networkDefaults: + directiveDefaults: + retryEmpty: true + retryPending: false + skipCacheRead: true + + # ===================================================================== + # NETWORK-LEVEL FAILSAFE — Tuned for listener (10s HTTP client timeout) + # ===================================================================== + failsafe: + # No hedge for eth_blockNumber — highest volume (55%), retries sufficient + - matchMethod: "eth_blockNumber" + timeout: + duration: 7s # Network budget across retries; still under listener 10s + retry: + maxAttempts: 4 + delay: 100ms + jitter: 50ms + + - matchMethod: "eth_getBlockByNumber" + timeout: + duration: 10s # Match listener HTTP timeout + retry: + maxAttempts: 3 + delay: 500ms + backoffMaxDelay: 3s + backoffFactor: 2.0 + jitter: 150ms + + - matchMethod: "eth_getBlockByHash" + timeout: + duration: 10s # Match listener HTTP timeout + hedge: + quantile: 0.98 + minDelay: 400ms + maxDelay: 3s + maxCount: 1 + retry: + maxAttempts: 4 + delay: 150ms # Give block time to propagate before next upstream + jitter: 50ms + + - matchMethod: "eth_getBlockReceipts" + timeout: + duration: 10s # Match listener HTTP timeout + hedge: + quantile: 0.98 # Heavier call — hedge only when clearly slow + minDelay: 800ms + maxDelay: 5s + maxCount: 1 + retry: + maxAttempts: 4 + delay: 500ms + backoffMaxDelay: 5s + backoffFactor: 1.5 + + - matchMethod: "eth_getTransactionReceipt" + timeout: + duration: 10s # Match listener HTTP timeout + hedge: + quantile: 0.98 + minDelay: 500ms + maxDelay: 3s + maxCount: 1 + retry: + maxAttempts: 4 + delay: 0ms + + - matchMethod: "*" + timeout: + duration: 10s # Match listener HTTP timeout + hedge: + quantile: 0.99 + minDelay: 500ms + maxDelay: 3s + maxCount: 1 + retry: + maxAttempts: 4 + delay: 500ms + + # ========================================================================= + # UPSTREAM DEFAULTS — Tuned for public nodes + # ========================================================================= + upstreamDefaults: + jsonRpc: + supportsBatch: true + batchMaxSize: 100 # Match listener batch_receipts_size_range usage + batchMaxWait: 50ms + + routing: + scoreLatencyQuantile: 0.75 + scoreMultipliers: + - respLatency: 10.0 + errorRate: 6.0 + throttledRate: 5.0 + blockHeadLag: 8.0 + totalRequests: 0.5 + finalizationLag: 3.0 + + failsafe: + # >>>>>>>>>> BEGIN CHAOS-RESILIENT CB (2026-03-20) >>>>>>>>>> + # Previous: eth_blockNumber had NO CB, receipts CB was 80/100 (never tripped + # during chaos test — only 32 TransportFailure < 80 threshold). + # Fix: uniform 10/20 CB on ALL methods. Ejects dead upstream in ~2s. + # Rollback: restore failureThresholdCount/Capacity to original values below, + # remove CB from eth_blockNumber, restore halfOpenAfter to original values. + # Original values: eth_getBlockByNumber 25/50/20s, eth_getBlockReceipts 80/100/5m, + # eth_getBlockByHash 80/100/3m, eth_getTransactionReceipt 80/100/3m, * 80/100/5m + # ────────────────────────────────────────────────────────── + - matchMethod: "eth_blockNumber" + timeout: + duration: 3s + retry: + maxAttempts: 1 + circuitBreaker: # NEW — was missing + failureThresholdCount: 10 + failureThresholdCapacity: 20 + halfOpenAfter: 15s + successThresholdCount: 3 + successThresholdCapacity: 5 + + - matchMethod: "eth_getBlockByNumber" + timeout: + duration: 5s + retry: + maxAttempts: 1 + circuitBreaker: + failureThresholdCount: 10 # Was 25 + failureThresholdCapacity: 20 # Was 50 + halfOpenAfter: 15s # Was 20s + successThresholdCount: 3 + successThresholdCapacity: 5 # Was 10 + + - matchMethod: "eth_getBlockByHash" + timeout: + duration: 10s + retry: + maxAttempts: 3 + delay: 300ms + jitter: 100ms + circuitBreaker: + failureThresholdCount: 10 # Was 80 + failureThresholdCapacity: 20 # Was 100 + halfOpenAfter: 15s # Was 3m + successThresholdCount: 3 # Was 5 + successThresholdCapacity: 5 # Was 10 + + - matchMethod: "eth_getBlockReceipts" + timeout: + duration: 5s + retry: + maxAttempts: 1 + circuitBreaker: + failureThresholdCount: 10 # Was 80 — never tripped in chaos test + failureThresholdCapacity: 20 # Was 100 + halfOpenAfter: 15s # Was 5m + successThresholdCount: 3 + successThresholdCapacity: 5 # Was 10 + + - matchMethod: "eth_getTransactionReceipt" + timeout: + duration: 5s + retry: + maxAttempts: 1 + circuitBreaker: + failureThresholdCount: 10 # Was 80 + failureThresholdCapacity: 20 # Was 100 + halfOpenAfter: 15s # Was 3m + successThresholdCount: 3 # Was 5 + successThresholdCapacity: 5 # Was 10 + + - matchMethod: "*" + timeout: + duration: 5s + retry: + maxAttempts: 1 + circuitBreaker: + failureThresholdCount: 10 # Was 80 + failureThresholdCapacity: 20 # Was 100 + halfOpenAfter: 15s # Was 5m + successThresholdCount: 3 + successThresholdCapacity: 5 # Was 10 + # <<<<<<<<<< END CHAOS-RESILIENT CB (2026-03-20) <<<<<<<<<< + + # ========================================================================= + # NETWORKS — Same chains as erpc.yaml + # ========================================================================= + networks: + - architecture: evm + evm: + chainId: 43114 + fallbackFinalityDepth: 1024 + integrity: + enforceHighestBlock: true + enforceGetLogsBlockRange: true + enforceNonNullTaggedBlocks: true + alias: avalanche + directiveDefaults: + retryEmpty: true + + - architecture: evm + evm: + chainId: 56 + fallbackFinalityDepth: 1024 + integrity: + enforceHighestBlock: true + enforceGetLogsBlockRange: true + enforceNonNullTaggedBlocks: true + alias: binance + directiveDefaults: + retryEmpty: true + + # ───────────────────────────────────────────────────────────────────── + # Ethereum mainnet — publicnode primary, 7 fallbacks, ZERO amplification + # ───────────────────────────────────────────────────────────────────── + # Root cause of all previous failures: eRPC retries amplify load. + # Direct to publicnode = 3,207 calls. Through eRPC = 11,320 calls (3.5x). + # + # Fix: NO hedge + NO network retry. One pass through upstreams in score + # order: publicnode (100) → drpc (7) → nodies (6) → ... → blockpi (1). + # If publicnode succeeds (90%+ of requests): done, no extra calls. + # If publicnode fails: fallbacks handle it, publicnode NOT retried. + # + # Per eRPC docs: defining failsafe here REPLACES networkDefaults. + # ───────────────────────────────────────────────────────────────────── + - architecture: evm + evm: + chainId: 1 + integrity: + enforceHighestBlock: true + enforceGetLogsBlockRange: true + enforceNonNullTaggedBlocks: true + alias: ethereum + directiveDefaults: + retryEmpty: true + failsafe: + - matchMethod: "eth_blockNumber" + timeout: + duration: 10s + retry: + maxAttempts: 1 # No network retry — 8 upstreams is enough + delay: 100ms + + - matchMethod: "eth_getBlockByNumber" + timeout: + duration: 15s # 8 upstreams × ~1.5s each worst case + retry: + maxAttempts: 1 + + - matchMethod: "eth_getBlockByHash" + timeout: + duration: 10s + retry: + maxAttempts: 1 + + - matchMethod: "eth_getBlockReceipts" + timeout: + duration: 15s # 8 upstreams × ~5s timeout, but most succeed in <1s + retry: + maxAttempts: 1 # CRITICAL: no retry = no amplification + + - matchMethod: "eth_getTransactionReceipt" + timeout: + duration: 10s + retry: + maxAttempts: 1 + + - matchMethod: "*" + timeout: + duration: 10s + retry: + maxAttempts: 1 + + - architecture: evm + evm: + chainId: 137 + fallbackFinalityDepth: 128 + integrity: + enforceHighestBlock: true + enforceGetLogsBlockRange: true + enforceNonNullTaggedBlocks: true + alias: polygon + directiveDefaults: + retryEmpty: true + + - architecture: evm + evm: + chainId: 8453 + integrity: + enforceHighestBlock: true + enforceGetLogsBlockRange: true + enforceNonNullTaggedBlocks: true + alias: base + directiveDefaults: + retryEmpty: true + + # Testnets + - architecture: evm + evm: + chainId: 11155111 + fallbackFinalityDepth: 64 + integrity: + enforceHighestBlock: true + enforceGetLogsBlockRange: true + enforceNonNullTaggedBlocks: true + alias: sepolia + directiveDefaults: + retryEmpty: true + # Sepolia: metrics-based selection policy for eth_getBlockByNumber to avoid + # rate-limit exhaustion. Excludes upstreams with high error/throttle rates; + # when they recover, they re-enter automatically (no hardcoded providers). + # Tighter thresholds + blockHeadLag to exclude degraded upstreams faster. + selectionPolicy: + evalInterval: 3s + evalPerMethod: true + evalFunction: | + (upstreams, method) => { + if (method !== 'eth_getBlockByNumber') return upstreams + const maxErrorRate = 0.10 + const maxThrottledRate = 0.08 + const maxBlockHeadLag = 8 + const healthy = upstreams.filter(u => { + const err = u.metrics.errorRate ?? 0 + const thr = u.metrics.throttledRate ?? 0 + const lag = u.metrics.blockHeadLag ?? 999 + return err <= maxErrorRate && thr <= maxThrottledRate && lag <= maxBlockHeadLag + }) + return healthy.length > 0 ? healthy : upstreams + } + resampleExcluded: true + resampleInterval: 20s + resampleCount: 3 + failsafe: + - matchMethod: "eth_blockNumber" + timeout: + duration: 7s + retry: + maxAttempts: 4 + delay: 100ms + jitter: 50ms + + - matchMethod: "eth_getBlockByNumber" + timeout: + duration: 10s + hedge: + quantile: 0.95 + minDelay: 1500ms + maxDelay: 4s + maxCount: 1 + retry: + maxAttempts: 10 + delay: 120ms + jitter: 80ms + backoffMaxDelay: 2s + backoffFactor: 2.0 + + - matchMethod: "eth_getBlockByHash" + timeout: + duration: 10s + hedge: + quantile: 0.98 + minDelay: 400ms + maxDelay: 3s + maxCount: 1 + retry: + maxAttempts: 4 + delay: 150ms + jitter: 50ms + + - matchMethod: "eth_getBlockReceipts" + timeout: + duration: 10s + hedge: + quantile: 0.98 + minDelay: 800ms + maxDelay: 5s + maxCount: 1 + retry: + maxAttempts: 5 + delay: 500ms + backoffMaxDelay: 5s + backoffFactor: 1.5 + + - matchMethod: "eth_getTransactionReceipt" + timeout: + duration: 10s + hedge: + quantile: 0.98 + minDelay: 500ms + maxDelay: 3s + maxCount: 1 + retry: + maxAttempts: 4 + delay: 0ms + + - matchMethod: "*" + timeout: + duration: 10s + hedge: + quantile: 0.99 + minDelay: 500ms + maxDelay: 3s + maxCount: 1 + retry: + maxAttempts: 4 + delay: 500ms + + - architecture: evm + evm: + chainId: 43113 + fallbackFinalityDepth: 1024 + integrity: + enforceHighestBlock: true + enforceGetLogsBlockRange: true + enforceNonNullTaggedBlocks: true + alias: fuji + directiveDefaults: + retryEmpty: true + + - architecture: evm + evm: + chainId: 97 + fallbackFinalityDepth: 1024 + integrity: + enforceHighestBlock: true + enforceGetLogsBlockRange: true + enforceNonNullTaggedBlocks: true + alias: bsc-testnet + directiveDefaults: + retryEmpty: true + # BSC testnet-specific network strategy: + # - keep retries short to avoid long tails on public endpoints + # - keep hedging conservative for heavy receipts calls + failsafe: + - matchMethod: "eth_blockNumber" + timeout: + duration: 7s + retry: + maxAttempts: 3 + delay: 150ms + jitter: 75ms + + - matchMethod: "eth_getBlockByNumber" + timeout: + duration: 8s + retry: + maxAttempts: 3 + delay: 250ms + jitter: 100ms + backoffMaxDelay: 2s + backoffFactor: 1.5 + + - matchMethod: "eth_getBlockByHash" + timeout: + duration: 10s + hedge: + quantile: 0.98 + minDelay: 400ms + maxDelay: 3s + maxCount: 1 + retry: + maxAttempts: 4 + delay: 150ms + jitter: 50ms + + - matchMethod: "eth_getBlockReceipts" + timeout: + duration: 8s + hedge: + quantile: 0.995 + minDelay: 1200ms + maxDelay: 3s + maxCount: 1 + retry: + maxAttempts: 2 + delay: 200ms + backoffMaxDelay: 1200ms + backoffFactor: 1.3 + blockUnavailableDelay: 800ms + + - matchMethod: "eth_getTransactionReceipt" + timeout: + duration: 10s + hedge: + quantile: 0.98 + minDelay: 500ms + maxDelay: 3s + maxCount: 1 + retry: + maxAttempts: 4 + delay: 0ms + + - matchMethod: "*" + timeout: + duration: 10s + hedge: + quantile: 0.99 + minDelay: 500ms + maxDelay: 3s + maxCount: 1 + retry: + maxAttempts: 4 + delay: 500ms + selectionPolicy: + evalInterval: 10s + evalPerMethod: true + evalFunction: | + (upstreams, method) => { + const hotMethods = ['eth_blockNumber', 'eth_getBlockByNumber', 'eth_getBlockReceipts'] + const primaryIdHint = 'bsc-testnet-rpc.publicnode.com' + const getId = (u) => (u.config.id || '').toLowerCase() + const isPrimary = (u) => getId(u).includes(primaryIdHint) + const isZan = (u) => getId(u).includes('zan.top') + const isOnfinality = (u) => getId(u).includes('onfinality.io') + const isBnbSeed = (u) => getId(u).includes('prebsc') + const isHealthy = (u, maxErr, maxLag) => ( + u.metrics.errorRate < maxErr && u.metrics.blockHeadLag <= maxLag + ) + const byReliability = (a, b) => { + if (a.metrics.errorRate === b.metrics.errorRate) { + return a.metrics.p90ResponseSeconds - b.metrics.p90ResponseSeconds + } + return a.metrics.errorRate - b.metrics.errorRate + } + + const sorted = [...upstreams].sort(byReliability) + const strictHealthy = sorted.filter(u => isHealthy(u, 0.15, 2)) + const relaxedHealthy = sorted.filter(u => isHealthy(u, 0.30, 8)) + const primaryStrict = strictHealthy.filter(isPrimary) + const primaryRelaxed = relaxedHealthy.filter(isPrimary) + + // Throttled providers observed in BSC tests for hot methods. + const hotPool = sorted.filter(u => !isZan(u) && !isOnfinality(u)) + const hotStrict = hotPool.filter(u => isHealthy(u, 0.20, 4)) + const hotRelaxed = hotPool.filter(u => isHealthy(u, 0.35, 10)) + const hotPrimaryStrict = hotStrict.filter(isPrimary) + const hotPrimaryRelaxed = hotRelaxed.filter(isPrimary) + + // Receipt calls often return "block unavailable" on some public seed endpoints. + const receiptsPool = sorted.filter( + u => !isZan(u) && !isOnfinality(u) && !isBnbSeed(u) + ) + const receiptsStrict = receiptsPool.filter(u => isHealthy(u, 0.25, 6)) + const receiptsRelaxed = receiptsPool.filter(u => isHealthy(u, 0.40, 12)) + const receiptsPrimaryStrict = receiptsStrict.filter(isPrimary) + const receiptsPrimaryRelaxed = receiptsRelaxed.filter(isPrimary) + + if (hotMethods.includes(method)) { + if (method === 'eth_getBlockReceipts') { + const result = [] + if (receiptsPrimaryStrict.length > 0) { + result.push(...receiptsPrimaryStrict) + } else if (receiptsPrimaryRelaxed.length > 0) { + result.push(...receiptsPrimaryRelaxed) + } + for (const u of receiptsStrict) { + if (!result.includes(u)) result.push(u) + } + for (const u of receiptsRelaxed) { + if (!result.includes(u)) result.push(u) + } + if (result.length > 0) return result + if (receiptsPool.length > 0) return receiptsPool + return upstreams + } + + const result = [] + if (hotPrimaryStrict.length > 0) { + result.push(...hotPrimaryStrict) + } else if (hotPrimaryRelaxed.length > 0) { + result.push(...hotPrimaryRelaxed) + } + for (const u of hotStrict) { + if (!result.includes(u)) result.push(u) + } + for (const u of hotRelaxed) { + if (!result.includes(u)) result.push(u) + } + if (result.length > 0) return result + if (hotPool.length > 0) return hotPool + return upstreams + } + + if (strictHealthy.length > 0) return strictHealthy + if (relaxedHealthy.length > 0) return relaxedHealthy + return upstreams + } + resampleExcluded: true + resampleInterval: 1m + resampleCount: 3 + + - architecture: evm + evm: + chainId: 80002 + fallbackFinalityDepth: 128 + integrity: + enforceHighestBlock: true + enforceGetLogsBlockRange: true + enforceNonNullTaggedBlocks: true + alias: amoy + directiveDefaults: + retryEmpty: true + + - architecture: evm + evm: + chainId: 84532 + fallbackFinalityDepth: 64 + integrity: + enforceHighestBlock: true + enforceGetLogsBlockRange: true + enforceNonNullTaggedBlocks: true + alias: base-sepolia + directiveDefaults: + retryEmpty: true + # Base Sepolia: generic metrics-based strategy (no provider names). + # - selection policy filters by errorRate, throttledRate, blockHeadLag for hot methods + # - hedge + more retries for eth_blockNumber and eth_getBlockByNumber + failsafe: + - matchMethod: "eth_blockNumber" + timeout: + duration: 7s + hedge: + quantile: 0.95 + minDelay: 1500ms + maxDelay: 4s + maxCount: 1 + retry: + maxAttempts: 6 + delay: 120ms + jitter: 80ms + + - matchMethod: "eth_getBlockByNumber" + timeout: + duration: 10s + hedge: + quantile: 0.95 + minDelay: 1000ms + maxDelay: 4s + maxCount: 1 + retry: + maxAttempts: 6 + delay: 80ms + jitter: 40ms + backoffMaxDelay: 1s + backoffFactor: 1.5 + + - matchMethod: "eth_getBlockByHash" + timeout: + duration: 10s + hedge: + quantile: 0.98 + minDelay: 400ms + maxDelay: 3s + maxCount: 1 + retry: + maxAttempts: 4 + delay: 150ms + jitter: 50ms + + - matchMethod: "eth_getBlockReceipts" + timeout: + duration: 10s + hedge: + quantile: 0.95 + minDelay: 1000ms + maxDelay: 4s + maxCount: 1 + retry: + maxAttempts: 7 + delay: 200ms + jitter: 100ms + backoffMaxDelay: 2s + backoffFactor: 1.5 + blockUnavailableDelay: 800ms + + - matchMethod: "eth_getTransactionReceipt" + timeout: + duration: 10s + hedge: + quantile: 0.98 + minDelay: 500ms + maxDelay: 3s + maxCount: 1 + retry: + maxAttempts: 4 + delay: 0ms + + - matchMethod: "*" + timeout: + duration: 10s + hedge: + quantile: 0.99 + minDelay: 500ms + maxDelay: 3s + maxCount: 1 + retry: + maxAttempts: 4 + delay: 500ms + selectionPolicy: + evalInterval: 3s + evalPerMethod: true + evalFunction: | + (upstreams, method) => { + const hotMethods = ['eth_blockNumber', 'eth_getBlockByNumber', 'eth_getBlockReceipts'] + if (!hotMethods.includes(method)) return upstreams + + const maxBlockHeadLag = 10 + const maxErrorRate = method === 'eth_getBlockByNumber' ? 0.12 : (method === 'eth_getBlockReceipts' ? 0.10 : 0.15) + const maxThrottledRate = method === 'eth_getBlockByNumber' ? 0.10 : (method === 'eth_getBlockReceipts' ? 0.08 : 0.12) + + const healthy = upstreams.filter(u => { + const err = u.metrics.errorRate ?? 0 + const thr = u.metrics.throttledRate ?? 0 + const lag = u.metrics.blockHeadLag ?? 999 + return err <= maxErrorRate && thr <= maxThrottledRate && lag <= maxBlockHeadLag + }) + + return healthy.length > 0 ? healthy : upstreams + } + resampleExcluded: true + resampleInterval: 20s + resampleCount: 3 + + # ========================================================================= + # PROVIDERS — Repository only (public nodes from evm-public-endpoints.erpc.cloud) + # ========================================================================= + providers: + - id: public + vendor: repository + settings: + repositoryUrl: https://evm-public-endpoints.erpc.cloud + recheckInterval: 2h # doubled from 1h default + onlyNetworks: + # evm:1 removed — repository discovers 28+ garbage endpoints that + # cause request amplification. Ethereum uses explicit upstreams below. + - evm:56 + - evm:137 + - evm:8453 + - evm:43114 + - evm:11155111 + - evm:43113 + - evm:97 + - evm:80002 + - evm:84532 + overrides: + "evm:*": + rateLimitBudget: very-high-limits + evm: + statePollerInterval: 60s # doubled from 30s + statePollerDebounce: 10s # doubled (5s for Ethereum, 2s for others; use max) + routing: + scoreLatencyQuantile: 0.10 # Use almost ALL upstreams (was 0.50) + scoreMultipliers: + - overall: 0.1 # Minimal overall weight = near round-robin + - errorRate: 20.0 # Heavily penalize errors + - throttledRate: 30.0 # MAXIMUM penalty for rate-limited nodes + - respLatency: 0.5 # Almost ignore latency + - blockHeadLag: 5.0 + - totalRequests: 10.0 # STRONGLY prefer less-used nodes (round-robin effect) + rateLimitAutoTune: + enabled: true + adjustmentPeriod: 1m # Reduce oscillation in auto-tuning + errorRateThreshold: 0.10 # Avoid overreacting to short 429 bursts + increaseFactor: 1.03 # Recover budget faster when healthy + decreaseFactor: 0.90 # Smoother backoff to reduce hard clamping + minBudget: 50 # Preserve baseline throughput during spikes + maxBudget: 200 # Avoid local budget becoming bottleneck + # Sepolia-specific upstream routing: aggressively penalize throttled/errored nodes. + "evm:11155111": + rateLimitBudget: very-high-limits + evm: + statePollerInterval: 30s + statePollerDebounce: 5s + failsafe: + - matchMethod: "eth_getBlockByNumber" + timeout: + duration: 1500ms + retry: + maxAttempts: 1 + circuitBreaker: + failureThresholdCount: 20 + failureThresholdCapacity: 40 + halfOpenAfter: 15s + successThresholdCount: 3 + successThresholdCapacity: 10 + routing: + scoreLatencyQuantile: 0.75 + scoreMultipliers: + - overall: 1.0 + errorRate: 50.0 + throttledRate: 100.0 + respLatency: 4.0 + blockHeadLag: 20.0 + finalizationLag: 10.0 + totalRequests: 1.5 + - method: "eth_getBlockByNumber" + errorRate: 120.0 + throttledRate: 200.0 + respLatency: 2.0 + blockHeadLag: 30.0 + totalRequests: 10.0 + rateLimitAutoTune: + enabled: true + adjustmentPeriod: 1m + errorRateThreshold: 0.08 + increaseFactor: 1.02 + decreaseFactor: 0.75 + minBudget: 20 + maxBudget: 200 + # BSC testnet-specific upstream routing: reliability-first over load spreading. + "evm:97": + rateLimitBudget: very-high-limits + evm: + statePollerInterval: 30s + statePollerDebounce: 2s + routing: + scoreLatencyQuantile: 0.75 + scoreMultipliers: + - overall: 1.0 + errorRate: 30.0 + throttledRate: 45.0 + respLatency: 4.0 + blockHeadLag: 15.0 + finalizationLag: 10.0 + totalRequests: 0.2 + rateLimitAutoTune: + enabled: true + adjustmentPeriod: 1m + errorRateThreshold: 0.10 + increaseFactor: 1.03 + decreaseFactor: 0.90 + minBudget: 50 + maxBudget: 200 + # Base Sepolia: generic strategy — circuit breaker for hot methods. + "evm:84532": + rateLimitBudget: very-high-limits + evm: + statePollerInterval: 30s + statePollerDebounce: 2s + failsafe: + - matchMethod: "eth_blockNumber" + timeout: + duration: 1500ms + retry: + maxAttempts: 1 + circuitBreaker: + failureThresholdCount: 20 + failureThresholdCapacity: 40 + halfOpenAfter: 15s + successThresholdCount: 3 + successThresholdCapacity: 10 + - matchMethod: "eth_getBlockByNumber" + timeout: + duration: 1200ms + retry: + maxAttempts: 1 + circuitBreaker: + failureThresholdCount: 15 + failureThresholdCapacity: 30 + halfOpenAfter: 12s + successThresholdCount: 3 + successThresholdCapacity: 10 + - matchMethod: "eth_getBlockReceipts" + timeout: + duration: 8s + retry: + maxAttempts: 1 + circuitBreaker: + failureThresholdCount: 15 + failureThresholdCapacity: 30 + halfOpenAfter: 20s + successThresholdCount: 3 + successThresholdCapacity: 10 + routing: + scoreLatencyQuantile: 0.75 + scoreMultipliers: + - overall: 1.0 + errorRate: 40.0 + throttledRate: 60.0 + respLatency: 5.0 + blockHeadLag: 12.0 + finalizationLag: 10.0 + totalRequests: 1.5 + - method: "eth_blockNumber" + errorRate: 80.0 + throttledRate: 120.0 + totalRequests: 8.0 + - method: "eth_getBlockByNumber" + errorRate: 120.0 + throttledRate: 200.0 + respLatency: 2.0 + blockHeadLag: 30.0 + totalRequests: 10.0 + - method: "eth_getBlockReceipts" + errorRate: 100.0 + throttledRate: 150.0 + totalRequests: 10.0 + rateLimitAutoTune: + enabled: true + adjustmentPeriod: 1m + errorRateThreshold: 0.08 + increaseFactor: 1.02 + decreaseFactor: 0.85 + minBudget: 20 + maxBudget: 200 + # Amoy-specific upstream behavior: pin upstream defaults for tested chain. + "evm:80002": + jsonRpc: + supportsBatch: true + batchMaxSize: 100 + batchMaxWait: 50ms + routing: + scoreLatencyQuantile: 0.75 + scoreMultipliers: + - respLatency: 10.0 + errorRate: 6.0 + throttledRate: 5.0 + blockHeadLag: 8.0 + totalRequests: 0.5 + finalizationLag: 3.0 + failsafe: + - matchMethod: "eth_blockNumber" + timeout: + duration: 1500ms + retry: + maxAttempts: 1 + + - matchMethod: "eth_getBlockByNumber" + timeout: + duration: 1500ms + retry: + maxAttempts: 1 + circuitBreaker: + failureThresholdCount: 40 + failureThresholdCapacity: 80 + halfOpenAfter: 30s + successThresholdCount: 3 + successThresholdCapacity: 10 + + - matchMethod: "eth_getBlockByHash" + timeout: + duration: 10s + retry: + maxAttempts: 3 + delay: 300ms + jitter: 100ms + circuitBreaker: + failureThresholdCount: 80 + failureThresholdCapacity: 100 + halfOpenAfter: 3m + successThresholdCount: 5 + successThresholdCapacity: 10 + + - matchMethod: "eth_getBlockReceipts" + timeout: + duration: 10s + retry: + maxAttempts: 3 + delay: 500ms + jitter: 200ms + circuitBreaker: + failureThresholdCount: 80 + failureThresholdCapacity: 100 + halfOpenAfter: 5m + successThresholdCount: 3 + successThresholdCapacity: 10 + + - matchMethod: "eth_getTransactionReceipt" + timeout: + duration: 10s + retry: + maxAttempts: 3 + delay: 200ms + jitter: 100ms + circuitBreaker: + failureThresholdCount: 80 + failureThresholdCapacity: 100 + halfOpenAfter: 3m + successThresholdCount: 5 + successThresholdCapacity: 10 + + - matchMethod: "*" + timeout: + duration: 10s + retry: + maxAttempts: 3 + delay: 1000ms + backoffMaxDelay: 10s + backoffFactor: 1.3 + jitter: 500ms + circuitBreaker: + failureThresholdCount: 80 + failureThresholdCapacity: 100 + halfOpenAfter: 5m + successThresholdCount: 3 + successThresholdCapacity: 10 + + # ========================================================================= + # UPSTREAMS — Ethereum mainnet + # ========================================================================= + # All verified with curl: eth_getBlockReceipts returns 153 receipts for + # block 0x13B6740 with <330ms latency. No API keys needed. + # + # Score determines try order. publicnode (100) always first. + # Fallbacks scored 1-7 — eRPC tries them sequentially if publicnode fails. + # With network retry=1, publicnode is tried ONCE then fallbacks take over. + # + # Upstream failsafe from upstreamDefaults: 5s timeout, 1 attempt each. + # ========================================================================= + # >>>>>>>>>> BEGIN UPSTREAM CLEANUP (2026-03-20) >>>>>>>>>> + # Removed 4 dead-weight upstreams based on metrics analysis: + # - drpc-eth: 100% ServerSideException (86/86 receipts, 81/81 blocks) + # - gatewayfm-eth: 79% CapacityExceeded (355/449 receipts, 80/159 blocks) + # - meowrpc-eth: 100% ServerSideException (86/86 receipts, 62/62 blocks) + # - blockpi-eth: 100% ServerSideException (86/86 receipts) + # Kept 4 upstreams with proven success rates: + # - publicnode (primary), nodies (1.8% err), blockrazor (1.4% err), 0xrpc (24% err) + # Rollback: re-add the removed upstreams from git history. + # ────────────────────────────────────────────────────────── + upstreams: + # ── Primary ───────────────────────────────────────────────────────── + - endpoint: https://ethereum-rpc.publicnode.com + id: publicnode-eth + evm: + chainId: 1 + nodeType: archive + rateLimitBudget: very-high-limits + routing: + scoreMultipliers: + - overall: 100.0 # Always tried first + + # ── Fallbacks (tried in descending score order after publicnode) ── + - endpoint: https://ethereum-public.nodies.app + id: nodies-eth + evm: + chainId: 1 + rateLimitBudget: very-high-limits + routing: + scoreMultipliers: + - overall: 3.0 # Best fallback: 1.8% error rate, 94ms latency + + - endpoint: https://eth.blockrazor.xyz + id: blockrazor-eth + evm: + chainId: 1 + rateLimitBudget: very-high-limits + routing: + scoreMultipliers: + - overall: 2.0 # 1.4% error rate, 262ms latency + + - endpoint: https://0xrpc.io/eth + id: 0xrpc-eth + evm: + chainId: 1 + rateLimitBudget: very-high-limits + routing: + scoreMultipliers: + - overall: 1.0 # 24% error rate (heavy blocks), 271ms latency + # <<<<<<<<<< END UPSTREAM CLEANUP (2026-03-20) <<<<<<<<<< diff --git a/listener/config/listener-1.yaml b/listener/config/listener-1.yaml new file mode 100644 index 0000000000..6dbebaa519 --- /dev/null +++ b/listener/config/listener-1.yaml @@ -0,0 +1,28 @@ +name: listener +http_port: 8080 + +database: + db_url: postgres://postgres:postgres@listener-postgres:5432/listener + migration_max_attempts: 5 + pool: + max_connections: 10 + min_connections: 2 + +broker: + broker_type: redis + broker_url: redis://listener-redis:6379 + +blockchain: + type: evm + chain_id: 1 + rpc_url: http://listener-erpc:4000/listener-indexer/evm/1 + network: ethereum-mainnet + strategy: + automatic_startup: true + block_start_on_first_start: 24572795 + range_size: 100 + loop_delay_ms: 1000 + max_parallel_requests: 50 + block_fetcher: block_receipts + # batch_receipts_size_range: 10 + compute_block: true diff --git a/listener/config/listener-2.yaml b/listener/config/listener-2.yaml new file mode 100644 index 0000000000..d01006c65f --- /dev/null +++ b/listener/config/listener-2.yaml @@ -0,0 +1,46 @@ +# Example configuration template. +# +# For ready-to-use configs, see: +# config/listener-amqp.yaml — RabbitMQ backend +# config/listener-redis.yaml — Redis Streams backend +# +# Values can be overridden via environment variables: +# APP_BROKER__BROKER_TYPE=redis APP_BROKER__BROKER_URL=redis://localhost:6379 cargo run + +name: listener +http_port: 8081 +database: + db_url: postgres://postgres:postgres@listener-postgres:5432/listener + migration_max_attempts: 5 + pool: + max_connections: 10 + min_connections: 2 + # acquire_timeout_secs: 30 + # idle_timeout_secs: 600 + # max_lifetime_secs: 1800 +broker: + broker_type: amqp + broker_url: amqp://user:pass@listener-rabbitmq:5672/%2f + +blockchain: + type: evm + chain_id: 84532 + rpc_url: http://listener-erpc:4000/listener-indexer/evm/84532 + network: base-sepolia + finality_depth: 64 + # cleaner: + # enabled: true + # blocks_to_keep: 1000 + strategy: + automatic_startup: true + block_start_on_first_start: 38342520 + range_size: 100 + loop_delay_ms: 1000 + max_parallel_requests: 50 + # block_receipts | batch_receipts_full | batch_receipts_range | transaction_receipts_parallel | transaction_receipts_seqential + block_fetcher: block_receipts + # taken into account only for batch_receipts_range + batch_receipts_size_range: 10 + compute_block: false + # Maximum backoff in ms for rate-limit retries (exponential backoff cap, default: 20000) + max_exponential_backoff_ms: 20000 diff --git a/listener/config/listener-default.yaml b/listener/config/listener-default.yaml new file mode 100644 index 0000000000..c227b9f37d --- /dev/null +++ b/listener/config/listener-default.yaml @@ -0,0 +1,66 @@ +# Canonical listener configuration defaults. +# +# This file is the single source of truth for all listener config defaults. +# It is symlinked into charts/listener/configs/ and loaded by the Helm chart +# as the base layer (before commonConfig and per-listener overrides). +# +# When adding new config fields to the Rust Settings struct, add them here +# with their default values. This ensures the Helm chart stays in sync. +# +# Merge order (last wins): +# 1. This file (application defaults) +# 2. values.yaml → commonConfig (operator-shared overrides) +# 3. values.yaml → listeners[].config (chain-specific overrides) + +name: listener +http_port: 8080 + +database: + db_url: "placeholder-overridden-by-env" + migration_max_attempts: 5 + # IAM auth configuration. When absent or enabled=false, uses db_url. + iam_auth: + enabled: false + ssl_ca_path: "placeholder-overridden-by-env" + pool: + max_connections: 10 + min_connections: 2 + acquire_timeout_secs: 30 + idle_timeout_secs: 600 + max_lifetime_secs: 1800 + +broker: + broker_type: amqp + broker_url: "placeholder-overridden-by-env" + ensure_publish: false + +blockchain: + type: evm + chain_id: 1 + rpc_url: "http://placeholder" + network: placeholder + finality_depth: 64 + strategy: + automatic_startup: true + block_start_on_first_start: "current" + range_size: 100 + loop_delay_ms: 1000 + max_parallel_requests: 50 + block_fetcher: block_receipts + batch_receipts_size_range: 10 + compute_block: false + compute_block_allow_skipping: true + max_exponential_backoff_ms: 20000 + +telemetry: + enabled: true + metrics_port: 9090 + +log: + format: json + show_file_line: false + show_thread_ids: true + show_timestamp: true + show_target: true + show_constants: true + level: info diff --git a/listener/config/listener-e2e-test.yaml b/listener/config/listener-e2e-test.yaml new file mode 100644 index 0000000000..b213e6fdaa --- /dev/null +++ b/listener/config/listener-e2e-test.yaml @@ -0,0 +1,27 @@ +name: listener + +database: + db_url: postgres://postgres:postgres@coprocessor-and-kms-db:5432/listener + migration_max_attempts: 5 + pool: + max_connections: 10 + min_connections: 1 + +broker: + broker_type: redis + broker_url: redis://listener-redis:6379 + +blockchain: + type: evm + chain_id: 1 + rpc_url: http://listener-erpc:4000/listener-indexer/evm/1 + network: ethereum-mainnet + strategy: + automatic_startup: true + # block_start: 24572795 + range_size: 100 + loop_delay_ms: 1000 + max_parallel_requests: 50 + block_fetcher: block_receipts + # batch_receipts_size_range: 10 + compute_block: true \ No newline at end of file diff --git a/listener/crates/consumer/Cargo.toml b/listener/crates/consumer/Cargo.toml new file mode 100644 index 0000000000..b3aab8f988 --- /dev/null +++ b/listener/crates/consumer/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "consumer" +version = "0.1.0" +edition.workspace = true + +[dependencies] +alloy.workspace = true +async-trait.workspace = true +broker = { path = "../shared/broker" } +primitives = { path = "../shared/primitives" } +serde = { workspace = true } +serde_json.workspace = true +tracing.workspace = true +thiserror.workspace = true + +[dev-dependencies] +redis = { version = "0.29", features = ["tokio-comp", "aio"] } +test-support = { path = "../test-support" } +tokio = { workspace = true, features = ["rt-multi-thread", "sync", "time", "macros"] } diff --git a/listener/crates/consumer/src/client.rs b/listener/crates/consumer/src/client.rs new file mode 100644 index 0000000000..d85f2a0277 --- /dev/null +++ b/listener/crates/consumer/src/client.rs @@ -0,0 +1,264 @@ +use alloy::primitives::Address; +use async_trait::async_trait; +pub use broker::{AckDecision, Broker, HandlerError}; +use broker::{BrokerError, CancellationToken, Consumer, Handler, Message, Topic}; +use primitives::event::{BlockPayload, FilterCommand, FilterCommandValidationError}; +use primitives::routing; +use primitives::utils::chain_id_to_namespace; +use std::future::Future; +use std::sync::Arc; +use tracing::warn; + +pub use crate::error::ConsumerError; + +/// Chain & Consumer-scoped client for the consumer library. +/// +/// Downstream services instantiate this once with their broker connection and +/// target chain, then call instance methods for watch/unwatch operations. +#[derive(Clone)] +pub struct ListenerConsumer { + broker: Broker, + chain_id: u64, + consumer_id: String, + cancel: CancellationToken, +} + +impl ListenerConsumer { + /// Create a new consumer bound to a broker and chain ID. + pub fn new(broker: &Broker, chain_id: u64, consumer_id: &str) -> Self { + let consumer_id_trimmed = consumer_id.trim(); + if consumer_id != consumer_id_trimmed { + warn!( + "Consumer ID has leading or trailing whitespace, which may cause issues with routing. Consider trimming it before passing to ListenerConsumer::new." + ); + } + Self { + broker: broker.clone(), + chain_id, + consumer_id: consumer_id_trimmed.into(), + cancel: CancellationToken::new(), + } + } + + /// Cancel via the cancel token passed to new + /// This is useful for stopping the consumer from another task. + pub fn cancel(&self) { + self.cancel.cancel(); + } + + /// Return the chain ID this client publishes into. + pub fn chain_id(&self) -> u64 { + self.chain_id + } + + /// Return the consumer ID this client publishes into. + pub fn consumer_id(&self) -> &str { + &self.consumer_id + } + + pub fn create_filter_on_log_address(&self, contract: Address) -> FilterCommand { + FilterCommand { + consumer_id: self.consumer_id.clone(), + from: None, + to: None, + log_address: Some(contract), + } + } + + /// Publish a filter removal command to the unwatch topic. + pub async fn unregister_filter(&self, command: &FilterCommand) -> Result<(), ConsumerError> { + self.publish_filter_command(command, routing::UNWATCH).await + } + + /// Publish a filter registration command to the watch topic. + pub async fn register_filter(&self, command: &FilterCommand) -> Result<(), ConsumerError> { + self.publish_filter_command(command, routing::WATCH).await + } + + async fn publish_filter_command( + &self, + command: &FilterCommand, + routing_key: &'static str, + ) -> Result<(), ConsumerError> { + let mut command = command.clone(); + if command.consumer_id != self.consumer_id { + return Err(ConsumerError::InconsistentConsumerId( + command.consumer_id.clone(), + self.consumer_id.clone(), + )); + } + command.validate()?; + + let namespace = chain_id_to_namespace(self.chain_id); + let publisher = self.broker.publisher(&namespace).await?; + publisher.publish(routing_key, &command).await?; + Ok(()) + } + + pub fn consumer_topic(&self) -> Topic { + let routing = routing::consumer_new_event_routing(self.consumer_id.clone()); + Topic::new(routing) + } + + fn broker_consumer(&self) -> Result { + let topic = self.consumer_topic(); + let cancel = self.cancel.clone(); + self.broker + .consumer(&topic) + .group(topic.to_string()) + .prefetch(10) + .with_cancellation(cancel) + .build() + } + + /// Publish filters registration command to watch contracts. + pub async fn register_contracts(&self, contracts: &[Address]) -> Result<(), ConsumerError> { + if contracts.is_empty() { + return Err(ConsumerError::InvalidParameter( + "contracts array cannot be empty".into(), + )); + } + for contract in contracts { + self.register_filter(&self.create_filter_on_log_address(*contract)) + .await?; + } + Ok(()) + } + + /// Publish filters removal command to unwatch contracts. + pub async fn unregister_contracts(&self, contracts: &[Address]) -> Result<(), ConsumerError> { + if contracts.is_empty() { + return Err(ConsumerError::InvalidFilterCommand( + FilterCommandValidationError::MissingContractAddresses, + )); + } + for contract in contracts { + self.unregister_filter(&self.create_filter_on_log_address(*contract)) + .await?; + } + Ok(()) + } + + /// Ensure the consumer topology is set up in the broker. + pub async fn ensure_consumer(&self) -> Result<(), BrokerError> { + // TODO: start core listener + self.broker_consumer()?.ensure_topology().await + } + + /// Start consuming messages with the provided handler function. + /// + /// The returned future owns an internal clone of the client, so it can be + /// spawned without forcing the caller to clone `ListenerConsumer` first. + pub fn consume( + &self, + f: F, + ) -> impl Future> + Send + 'static + where + F: Fn(BlockPayload, CancellationToken) -> Fut + Send + Sync + 'static, + Fut: Future> + Send + 'static, + { + let client = self.clone(); + async move { + let consumer = client.broker_consumer()?; + let handler = ConsumerHandler { + call: Arc::new(f), + cancel: client.cancel.clone(), + }; + consumer.run(handler).await?; + Ok(()) + } + } +} + +struct ConsumerHandler { + call: Arc, + cancel: CancellationToken, +} + +impl Clone for ConsumerHandler { + fn clone(&self) -> Self { + Self { + call: Arc::clone(&self.call), + cancel: self.cancel.clone(), + } + } +} + +#[async_trait] +impl Handler for ConsumerHandler +where + F: Fn(BlockPayload, CancellationToken) -> Fut + Send + Sync + 'static, + Fut: Future> + Send + 'static, +{ + async fn call(&self, msg: &Message) -> Result { + let payload: BlockPayload = serde_json::from_slice(&msg.payload)?; + (self.call)(payload, self.cancel.clone()).await + } +} + +#[cfg(test)] +mod tests { + use alloy::primitives::B256; + use broker::{amqp::RmqPublisher, traits::Publisher}; + use primitives::event::BlockFlow; + + use super::*; + use std::sync::atomic::{AtomicU64, Ordering}; + static TEST_ID: AtomicU64 = AtomicU64::new(0); + + fn unique_name(prefix: &str) -> String { + format!("{prefix}-{}", TEST_ID.fetch_add(1, Ordering::Relaxed)) + } + + #[tokio::test] + #[ignore = "requires Docker"] + async fn test_consumer_happy_path() { + let broker_url = "amqp://user:pass@localhost:5672"; + let broker = Broker::amqp(broker_url).build().await.unwrap(); + let chain_id = 1; + let consumer_id = unique_name("copro-1-host-eth"); + let consumer = ListenerConsumer::new(&broker, chain_id, &consumer_id); + let contracts = vec![Address::ZERO]; + consumer.register_contracts(&contracts).await.unwrap(); + consumer.ensure_consumer().await.unwrap(); + static COUNTER: AtomicU64 = AtomicU64::new(0); + let consumer_task = consumer.consume(|payload, cancel| async move { + let v = COUNTER.fetch_add(1, Ordering::Relaxed); + eprintln!("Received payload: {:?} {v}", payload); + if v + 1 >= 2 { + cancel.cancel(); + println!("Cancel after receiving 2 payloads"); + } + Ok(AckDecision::Ack) + }); + let consumer_run = tokio::spawn(consumer_task); + eprintln!("Consumer task spawned, waiting for messages or timeout..."); + let routing_key = consumer.consumer_topic(); + let publisher = RmqPublisher::connect(broker_url, "main").await; + let fake_block = BlockPayload { + flow: BlockFlow::Live, + chain_id, + block_number: 0, + block_hash: B256::ZERO, + parent_hash: B256::ZERO, + timestamp: 0, + transactions: vec![], + }; + for _ in 1..=2 { + publisher + .publish(&routing_key.to_string(), &fake_block) + .await + .unwrap(); + } + let with_timeout = + tokio::time::timeout(std::time::Duration::from_secs(5), consumer_run).await; + eprintln!("Consumer task completed or timed out: {with_timeout:?}"); + consumer.cancel(); + consumer.unregister_contracts(&contracts).await.unwrap(); + assert!( + with_timeout.is_ok(), + "Consumer should have cancel and not timeout" + ); + assert_eq!(COUNTER.fetch_add(0, Ordering::Relaxed), 2); + } +} diff --git a/listener/crates/consumer/src/client.rs.orig b/listener/crates/consumer/src/client.rs.orig new file mode 100644 index 0000000000..2b95c4d531 --- /dev/null +++ b/listener/crates/consumer/src/client.rs.orig @@ -0,0 +1,272 @@ +use alloy::primitives::Address; +use async_trait::async_trait; +<<<<<<< HEAD +pub use broker::{AckDecision, Broker, HandlerError}; +use broker::{BrokerError, CancellationToken, Consumer, Handler, Message, Topic}; +======= +use broker::{ + AckDecision, Broker, BrokerError, CancellationToken, Consumer, Handler, HandlerError, Message, + Topic, +}; +>>>>>>> 4123c78 (feat(consumer): provide consume & register_contracts) +use primitives::event::{BlockPayload, FilterCommand, FilterCommandValidationError}; +use primitives::routing; +use primitives::utils::chain_id_to_namespace; +use std::future::Future; +use std::sync::Arc; +use tracing::warn; + +pub use crate::error::ConsumerError; + +/// Chain & Consumer-scoped client for the consumer library. +/// +/// Downstream services instantiate this once with their broker connection and +/// target chain, then call instance methods for watch/unwatch operations. +#[derive(Clone)] +pub struct ListenerConsumer { + broker: Broker, + chain_id: u64, + consumer_id: String, + cancel: CancellationToken, +} + +impl ListenerConsumer { + /// Create a new consumer bound to a broker and chain ID. + pub fn new(broker: &Broker, chain_id: u64, consumer_id: &str) -> Self { + let consumer_id_trimmed = consumer_id.trim(); + if consumer_id != consumer_id_trimmed { + warn!( + "Consumer ID has leading or trailing whitespace, which may cause issues with routing. Consider trimming it before passing to ListenerConsumer::new." + ); + } + Self { + broker: broker.clone(), + chain_id, + consumer_id: consumer_id_trimmed.into(), + cancel: CancellationToken::new(), + } + } + + /// Cancel via the cancel token passed to new + /// This is useful for stopping the consumer from another task. + pub fn cancel(&self) { + self.cancel.cancel(); + } + + /// Return the chain ID this client publishes into. + pub fn chain_id(&self) -> u64 { + self.chain_id + } + + /// Return the consumer ID this client publishes into. + pub fn consumer_id(&self) -> &str { + &self.consumer_id + } + + pub fn create_filter_on_log_address(&self, contract: Address) -> FilterCommand { + FilterCommand { + consumer_id: self.consumer_id.clone(), + from: None, + to: None, + log_address: Some(contract), + } + } + + /// Publish a filter removal command to the unwatch topic. + pub async fn unregister_filter(&self, command: &FilterCommand) -> Result<(), ConsumerError> { + self.publish_filter_command(command, routing::UNWATCH).await + } + + /// Publish a filter registration command to the watch topic. + pub async fn register_filter(&self, command: &FilterCommand) -> Result<(), ConsumerError> { + self.publish_filter_command(command, routing::WATCH).await + } + + async fn publish_filter_command( + &self, + command: &FilterCommand, + routing_key: &'static str, + ) -> Result<(), ConsumerError> { + let mut command = command.clone(); + if command.consumer_id != self.consumer_id { + return Err(ConsumerError::InconsistentConsumerId( + command.consumer_id.clone(), + self.consumer_id.clone(), + )); + } + command.validate()?; + + let namespace = chain_id_to_namespace(self.chain_id); + let publisher = self.broker.publisher(&namespace).await?; + publisher.publish(routing_key, &command).await?; + Ok(()) + } + + pub fn consumer_topic(&self) -> Topic { + let routing = routing::consumer_new_event_routing(self.consumer_id.clone()); + Topic::new(routing) + } + + fn broker_consumer(&self) -> Result { + let topic = self.consumer_topic(); + let consumer_group = self.consumer_id.clone(); + let cancel = self.cancel.clone(); + self.broker + .consumer(&topic) + .group(&consumer_group) + .prefetch(10) + .with_cancellation(cancel) + .build() + } + + /// Publish filters registration command to watch contracts. + pub async fn register_contracts(&self, contracts: &[Address]) -> Result<(), ConsumerError> { + if contracts.is_empty() { + return Err(ConsumerError::InvalidParameter( + "contracts array cannot be empty".into(), + )); + } + for contract in contracts { + self.register_filter(&self.create_filter_on_log_address(*contract)) + .await?; + } + Ok(()) + } + + /// Publish filters removal command to unwatch contracts. + pub async fn unregister_contracts(&self, contracts: &[Address]) -> Result<(), ConsumerError> { + if contracts.is_empty() { + return Err(ConsumerError::InvalidFilterCommand( + FilterCommandValidationError::MissingContractAddresses, + )); + } + for contract in contracts { + self.unregister_filter(&self.create_filter_on_log_address(*contract)) + .await?; + } + Ok(()) + } + + /// Ensure the consumer topology is set up in the broker. + pub async fn ensure_consumer(&self) -> Result<(), BrokerError> { + // TODO: start core listener + self.broker_consumer()?.ensure_topology().await + } + + /// Start consuming messages with the provided handler function. + /// + /// The returned future owns an internal clone of the client, so it can be + /// spawned without forcing the caller to clone `ListenerConsumer` first. + pub fn consume( + &self, + f: F, + ) -> impl Future> + Send + 'static + where + F: Fn(BlockPayload, CancellationToken) -> Fut + Send + Sync + 'static, + Fut: Future> + Send + 'static, + { + let client = self.clone(); + async move { + let consumer = client.broker_consumer()?; + let handler = ConsumerHandler { + call: Arc::new(f), + cancel: client.cancel.clone(), + }; + consumer.run(handler).await?; + Ok(()) + } + } +} + +struct ConsumerHandler { + call: Arc, + cancel: CancellationToken, +} + +impl Clone for ConsumerHandler { + fn clone(&self) -> Self { + Self { + call: Arc::clone(&self.call), + cancel: self.cancel.clone(), + } + } +} + +#[async_trait] +impl Handler for ConsumerHandler +where + F: Fn(BlockPayload, CancellationToken) -> Fut + Send + Sync + 'static, + Fut: Future> + Send + 'static, +{ + async fn call(&self, msg: &Message) -> Result { + let payload: BlockPayload = serde_json::from_slice(&msg.payload)?; + (self.call)(payload, self.cancel.clone()).await + } +} + +#[cfg(test)] +mod tests { + use alloy::primitives::B256; + use broker::{amqp::RmqPublisher, traits::Publisher}; + use primitives::event::BlockFlow; + + use super::*; + use std::sync::atomic::{AtomicU64, Ordering}; + static TEST_ID: AtomicU64 = AtomicU64::new(0); + + fn unique_name(prefix: &str) -> String { + format!("{prefix}-{}", TEST_ID.fetch_add(1, Ordering::Relaxed)) + } + + #[tokio::test] + #[ignore = "requires Docker"] + async fn test_consumer_happy_path() { + let broker_url = "amqp://user:pass@localhost:5672"; + let broker = Broker::amqp(broker_url).build().await.unwrap(); + let chain_id = 1; + let consumer_id = unique_name("copro-1-host-eth"); + let consumer = ListenerConsumer::new(&broker, chain_id, &consumer_id); + let contracts = vec![Address::ZERO]; + consumer.register_contracts(&contracts).await.unwrap(); + consumer.ensure_consumer().await.unwrap(); + static COUNTER: AtomicU64 = AtomicU64::new(0); + let consumer_task = consumer.consume(|payload, cancel| async move { + let v = COUNTER.fetch_add(1, Ordering::Relaxed); + eprintln!("Received payload: {:?} {v}", payload); + if v + 1 >= 2 { + cancel.cancel(); + println!("Cancel after receiving 2 payloads"); + } + Ok(AckDecision::Ack) + }); + let consumer_run = tokio::spawn(consumer_task); + eprintln!("Consumer task spawned, waiting for messages or timeout..."); + let routing_key = consumer.consumer_topic(); + let publisher = RmqPublisher::connect(broker_url, "main").await; + let fake_block = BlockPayload { + flow: BlockFlow::Live, + chain_id, + block_number: 0, + block_hash: B256::ZERO, + parent_hash: B256::ZERO, + timestamp: 0, + transactions: vec![], + }; + for _ in 1..=2 { + publisher + .publish(&routing_key.to_string(), &fake_block) + .await + .unwrap(); + } + let with_timeout = + tokio::time::timeout(std::time::Duration::from_secs(5), consumer_run).await; + eprintln!("Consumer task completed or timed out: {with_timeout:?}"); + consumer.cancel(); + consumer.unregister_contracts(&contracts).await.unwrap(); + assert!( + with_timeout.is_ok(), + "Consumer should have cancel and not timeout" + ); + assert_eq!(COUNTER.fetch_add(0, Ordering::Relaxed), 2); + } +} diff --git a/listener/crates/consumer/src/error.rs b/listener/crates/consumer/src/error.rs new file mode 100644 index 0000000000..b430b6844f --- /dev/null +++ b/listener/crates/consumer/src/error.rs @@ -0,0 +1,16 @@ +use broker::BrokerError; +use primitives::event::FilterCommandValidationError; +use thiserror::Error; + +#[derive(Debug, Error)] +#[non_exhaustive] +pub enum ConsumerError { + #[error(transparent)] + Broker(#[from] BrokerError), + #[error(transparent)] + InvalidFilterCommand(#[from] FilterCommandValidationError), + #[error("FilterCommand consumer_id '{}' does not match ListenerConsumer consumer_id '{}'", .0, .1)] + InconsistentConsumerId(String, String), + #[error("Invalid parameter when configuring the consumer {}", .0)] + InvalidParameter(String), +} diff --git a/listener/crates/consumer/src/lib.rs b/listener/crates/consumer/src/lib.rs new file mode 100644 index 0000000000..cdcd3fc273 --- /dev/null +++ b/listener/crates/consumer/src/lib.rs @@ -0,0 +1,6 @@ +mod client; +mod error; + +pub use client::{AckDecision, Broker, HandlerError, ListenerConsumer}; +pub use error::ConsumerError; +pub use primitives::event::FilterCommand; diff --git a/listener/crates/consumer/tests/watch_e2e.rs b/listener/crates/consumer/tests/watch_e2e.rs new file mode 100644 index 0000000000..88d9f10e9f --- /dev/null +++ b/listener/crates/consumer/tests/watch_e2e.rs @@ -0,0 +1,195 @@ +//! End-to-end tests for `ListenerConsumer::watch_contract` / +//! `ListenerConsumer::unwatch_contract`. +//! +//! Spins up a throwaway Redis via testcontainers, publishes through the +//! consumer-lib API, and verifies the messages arrive on the expected +//! routing keys with the correct payload. +//! +//! Run via: +//! +//! ```bash +//! make test-e2e-consumer +//! ``` + +use std::{ + sync::{ + Arc, Mutex, + atomic::{AtomicU64, Ordering}, + }, + time::Duration, +}; + +use broker::{AsyncHandlerPayloadOnly, Broker, CancellationToken, Topic}; +use consumer::{FilterCommand, ListenerConsumer}; +use primitives::routing; +use primitives::utils::chain_id_to_namespace; +use test_support::shared_redis_url; +use tokio::sync::Mutex as AsyncMutex; + +// ── Shared container ──────────────────────────────────────────────────────── + +static REDIS_TEST_LOCK: AsyncMutex<()> = AsyncMutex::const_new(()); +static TEST_ID: AtomicU64 = AtomicU64::new(0); + +async fn consumer_redis_url() -> String { + // Use a dedicated logical DB in the shared Redis container so this suite can + // reset state with FLUSHDB without disturbing other tests. + format!("{}/15", shared_redis_url().await.trim_end_matches('/')) +} + +fn unique_name(prefix: &str) -> String { + format!( + "{prefix}-{}-{}", + std::process::id(), + TEST_ID.fetch_add(1, Ordering::Relaxed) + ) +} + +async fn reset_redis(url: &str) { + let client = redis::Client::open(url).expect("invalid Redis URL in reset_redis"); + let mut conn = client + .get_multiplexed_async_connection() + .await + .expect("failed to connect to Redis in reset_redis"); + redis::cmd("FLUSHDB") + .query_async::<()>(&mut conn) + .await + .expect("FLUSHDB failed in reset_redis"); +} + +async fn wait_for_consumer_ack(broker: &Broker, topic: &Topic, group: &str) { + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + while tokio::time::Instant::now() < deadline { + if broker + .is_empty(topic, group) + .await + .expect("broker.is_empty failed during wait_for_consumer_ack") + { + return; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + + panic!("consumer group {group} did not drain and ACK within 5 seconds"); +} + +/// Publish a FilterCommand through the given routing key, subscribe on the +/// other side, and return the deserialized message that arrived. +async fn assert_filter_command_roundtrip( + routing_key: &str, + group_prefix: &str, + command: &FilterCommand, +) -> FilterCommand { + let _guard = REDIS_TEST_LOCK.lock().await; + let url = consumer_redis_url().await; + reset_redis(&url).await; + let broker = Broker::redis(&url).await.unwrap(); + + let chain_id = 1; + let consumer = ListenerConsumer::new(&broker, chain_id, &command.consumer_id); + let topic = Topic::new(routing_key).with_namespace(chain_id_to_namespace(chain_id)); + let group = unique_name(group_prefix); + let consumer_name = unique_name("consumer"); + let cancel = CancellationToken::new(); + + let received = Arc::new(Mutex::new(None::)); + let received_clone = received.clone(); + let handler = AsyncHandlerPayloadOnly::new(move |msg: FilterCommand| { + let received = received_clone.clone(); + async move { + *received.lock().unwrap() = Some(msg); + Ok::<(), std::convert::Infallible>(()) + } + }); + + let consumer_broker = broker.clone(); + let consumer_topic = topic.clone(); + let consumer_group = group.clone(); + let consumer_cancel = cancel.clone(); + let consumer_handle = tokio::spawn(async move { + consumer_broker + .consumer(&consumer_topic) + .group(&consumer_group) + .consumer_name(&consumer_name) + .prefetch(10) + .redis_block_ms(100) + .with_cancellation(consumer_cancel) + .run(handler) + .await + }); + + tokio::time::sleep(Duration::from_millis(400)).await; + if consumer_handle.is_finished() { + let result = consumer_handle + .await + .expect("consumer task should not panic"); + panic!("{routing_key} consumer exited before publish: {result:?}"); + } + + match routing_key { + routing::WATCH => consumer.register_filter(command).await.unwrap(), + routing::UNWATCH => consumer.unregister_filter(command).await.unwrap(), + other => panic!("unexpected routing key: {other}"), + }; + + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + while received.lock().unwrap().is_none() && tokio::time::Instant::now() < deadline { + tokio::time::sleep(Duration::from_millis(100)).await; + } + + wait_for_consumer_ack(&broker, &topic, &group).await; + + cancel.cancel(); + let run_result = tokio::time::timeout(Duration::from_secs(5), consumer_handle) + .await + .expect("consumer should stop after cancellation") + .expect("consumer task should not panic"); + run_result.expect("consumer should not return an error"); + + received + .lock() + .unwrap() + .clone() + .unwrap_or_else(|| panic!("should receive {routing_key} message")) +} + +#[tokio::test] +#[ignore = "requires Docker"] +async fn watch_contract_publishes_register_filter() { + let command = FilterCommand { + consumer_id: "gateway".into(), + from: Some( + "0x00000000000000000000000000000000deadbeef" + .parse() + .unwrap(), + ), + to: Some( + "0x00000000000000000000000000000000cafebabe" + .parse() + .unwrap(), + ), + log_address: None, + }; + + let msg = assert_filter_command_roundtrip(routing::WATCH, "watch-e2e-register", &command).await; + assert_eq!(msg, command); +} + +#[tokio::test] +#[ignore = "requires Docker"] +async fn unwatch_contract_publishes_unregister_filter() { + let command = FilterCommand { + consumer_id: "gateway".into(), + from: None, + to: Some( + "0x00000000000000000000000000000000deadbeef" + .parse() + .unwrap(), + ), + log_address: None, + }; + + let msg = + assert_filter_command_roundtrip(routing::UNWATCH, "watch-e2e-unregister", &command).await; + assert_eq!(msg, command); +} diff --git a/listener/crates/listener_core/.gitignore b/listener/crates/listener_core/.gitignore new file mode 100644 index 0000000000..48c013c4c4 --- /dev/null +++ b/listener/crates/listener_core/.gitignore @@ -0,0 +1,3 @@ +target +.claude +.env \ No newline at end of file diff --git a/listener/crates/listener_core/.sqlx/query-092b769e5c02d75f91ce14350318ea448e20cb13538ade4759777457c7eb2000.json b/listener/crates/listener_core/.sqlx/query-092b769e5c02d75f91ce14350318ea448e20cb13538ade4759777457c7eb2000.json new file mode 100644 index 0000000000..4dd60f77a3 --- /dev/null +++ b/listener/crates/listener_core/.sqlx/query-092b769e5c02d75f91ce14350318ea448e20cb13538ade4759777457c7eb2000.json @@ -0,0 +1,69 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT id, chain_id, block_number, block_hash, parent_hash, status as \"status: BlockStatus\", created_at\n FROM blocks\n WHERE chain_id = $1 AND status = 'CANONICAL'::block_status\n ORDER BY block_number DESC\n LIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "chain_id", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "block_number", + "type_info": "Int8" + }, + { + "ordinal": 3, + "name": "block_hash", + "type_info": "Bytea" + }, + { + "ordinal": 4, + "name": "parent_hash", + "type_info": "Bytea" + }, + { + "ordinal": 5, + "name": "status: BlockStatus", + "type_info": { + "Custom": { + "name": "block_status", + "kind": { + "Enum": [ + "CANONICAL", + "FINALIZED", + "UNCLE" + ] + } + } + } + }, + { + "ordinal": 6, + "name": "created_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false + ] + }, + "hash": "092b769e5c02d75f91ce14350318ea448e20cb13538ade4759777457c7eb2000" +} diff --git a/listener/crates/listener_core/.sqlx/query-13863e97044e6a4b779477d74bdc391e60e15e489392e8259d88c0857b97c145.json b/listener/crates/listener_core/.sqlx/query-13863e97044e6a4b779477d74bdc391e60e15e489392e8259d88c0857b97c145.json new file mode 100644 index 0000000000..122b8aec54 --- /dev/null +++ b/listener/crates/listener_core/.sqlx/query-13863e97044e6a4b779477d74bdc391e60e15e489392e8259d88c0857b97c145.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH ranked_blocks AS (\n SELECT id,\n ROW_NUMBER() OVER (\n ORDER BY block_number DESC, created_at DESC\n ) as rn\n FROM blocks\n WHERE chain_id = $1\n )\n DELETE FROM blocks\n WHERE chain_id = $1 AND id IN (SELECT id FROM ranked_blocks WHERE rn > $2)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "13863e97044e6a4b779477d74bdc391e60e15e489392e8259d88c0857b97c145" +} diff --git a/listener/crates/listener_core/.sqlx/query-1d76cf198b152774dbd8b1896db893670d3a1beddf21342b188763dcb39d2ed6.json b/listener/crates/listener_core/.sqlx/query-1d76cf198b152774dbd8b1896db893670d3a1beddf21342b188763dcb39d2ed6.json new file mode 100644 index 0000000000..513be27b43 --- /dev/null +++ b/listener/crates/listener_core/.sqlx/query-1d76cf198b152774dbd8b1896db893670d3a1beddf21342b188763dcb39d2ed6.json @@ -0,0 +1,62 @@ +{ + "db_name": "PostgreSQL", + "query": "\n DELETE FROM filters\n WHERE chain_id = $1\n AND consumer_id = $2\n AND COALESCE(\"from\", '') = COALESCE($3, '')\n AND COALESCE(\"to\", '') = COALESCE($4, '')\n AND COALESCE(\"log_address\", '') = COALESCE($5, '')\n RETURNING id, chain_id, consumer_id, \"from\", \"to\", \"log_address\", created_at\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "chain_id", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "consumer_id", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "from", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "to", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "log_address", + "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "created_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Int8", + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + true, + true, + true, + false + ] + }, + "hash": "1d76cf198b152774dbd8b1896db893670d3a1beddf21342b188763dcb39d2ed6" +} diff --git a/listener/crates/listener_core/.sqlx/query-2122e3c5ad5aeec320d1e61a3b3a1b6f3d6ab84e951e5ea2b721df3ba315b38c.json b/listener/crates/listener_core/.sqlx/query-2122e3c5ad5aeec320d1e61a3b3a1b6f3d6ab84e951e5ea2b721df3ba315b38c.json new file mode 100644 index 0000000000..cd075d7875 --- /dev/null +++ b/listener/crates/listener_core/.sqlx/query-2122e3c5ad5aeec320d1e61a3b3a1b6f3d6ab84e951e5ea2b721df3ba315b38c.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "\n DELETE FROM blocks\n WHERE chain_id = $1 AND created_at < NOW() - make_interval(secs => $2)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Float8" + ] + }, + "nullable": [] + }, + "hash": "2122e3c5ad5aeec320d1e61a3b3a1b6f3d6ab84e951e5ea2b721df3ba315b38c" +} diff --git a/listener/crates/listener_core/.sqlx/query-32e737d4319646733dfa5fae0b24e47c8570cac4210a66fd73ad6f98427ba0e6.json b/listener/crates/listener_core/.sqlx/query-32e737d4319646733dfa5fae0b24e47c8570cac4210a66fd73ad6f98427ba0e6.json new file mode 100644 index 0000000000..4f8a6cfe6a --- /dev/null +++ b/listener/crates/listener_core/.sqlx/query-32e737d4319646733dfa5fae0b24e47c8570cac4210a66fd73ad6f98427ba0e6.json @@ -0,0 +1,70 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT id, chain_id, block_number, block_hash, parent_hash, status as \"status: BlockStatus\", created_at\n FROM blocks\n WHERE chain_id = $1 AND block_number = $2 AND status = 'CANONICAL'::block_status\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "chain_id", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "block_number", + "type_info": "Int8" + }, + { + "ordinal": 3, + "name": "block_hash", + "type_info": "Bytea" + }, + { + "ordinal": 4, + "name": "parent_hash", + "type_info": "Bytea" + }, + { + "ordinal": 5, + "name": "status: BlockStatus", + "type_info": { + "Custom": { + "name": "block_status", + "kind": { + "Enum": [ + "CANONICAL", + "FINALIZED", + "UNCLE" + ] + } + } + } + }, + { + "ordinal": 6, + "name": "created_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false + ] + }, + "hash": "32e737d4319646733dfa5fae0b24e47c8570cac4210a66fd73ad6f98427ba0e6" +} diff --git a/listener/crates/listener_core/.sqlx/query-382bf466e9a920c6b300ee479c381d8695effa6baf5fadb5799208c6d3306390.json b/listener/crates/listener_core/.sqlx/query-382bf466e9a920c6b300ee479c381d8695effa6baf5fadb5799208c6d3306390.json new file mode 100644 index 0000000000..c8c1ce582c --- /dev/null +++ b/listener/crates/listener_core/.sqlx/query-382bf466e9a920c6b300ee479c381d8695effa6baf5fadb5799208c6d3306390.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE blocks SET status = 'CANONICAL'::block_status WHERE chain_id = $1 AND block_hash = $2\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Bytea" + ] + }, + "nullable": [] + }, + "hash": "382bf466e9a920c6b300ee479c381d8695effa6baf5fadb5799208c6d3306390" +} diff --git a/listener/crates/listener_core/.sqlx/query-54b15de246e6f15ac8582a6e9c75cdec505156a57e7b2fe8daab47cb21ebf414.json b/listener/crates/listener_core/.sqlx/query-54b15de246e6f15ac8582a6e9c75cdec505156a57e7b2fe8daab47cb21ebf414.json new file mode 100644 index 0000000000..41b827b067 --- /dev/null +++ b/listener/crates/listener_core/.sqlx/query-54b15de246e6f15ac8582a6e9c75cdec505156a57e7b2fe8daab47cb21ebf414.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT MIN(block_number) as \"min_block_number: i64\"\n FROM blocks\n WHERE chain_id = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "min_block_number: i64", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "54b15de246e6f15ac8582a6e9c75cdec505156a57e7b2fe8daab47cb21ebf414" +} diff --git a/listener/crates/listener_core/.sqlx/query-552995e0af0f85bc527dce06390f1934204ee12a9885404f60a1a6eaef197f04.json b/listener/crates/listener_core/.sqlx/query-552995e0af0f85bc527dce06390f1934204ee12a9885404f60a1a6eaef197f04.json new file mode 100644 index 0000000000..4ebb97987a --- /dev/null +++ b/listener/crates/listener_core/.sqlx/query-552995e0af0f85bc527dce06390f1934204ee12a9885404f60a1a6eaef197f04.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT status as \"status: BlockStatus\" FROM blocks WHERE chain_id = $1 AND block_hash = $2\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "status: BlockStatus", + "type_info": { + "Custom": { + "name": "block_status", + "kind": { + "Enum": [ + "CANONICAL", + "FINALIZED", + "UNCLE" + ] + } + } + } + } + ], + "parameters": { + "Left": [ + "Int8", + "Bytea" + ] + }, + "nullable": [ + false + ] + }, + "hash": "552995e0af0f85bc527dce06390f1934204ee12a9885404f60a1a6eaef197f04" +} diff --git a/listener/crates/listener_core/.sqlx/query-974132b7cfb7b7f498ed01e327d764aa2b8a7667699e76c7b16097722427378f.json b/listener/crates/listener_core/.sqlx/query-974132b7cfb7b7f498ed01e327d764aa2b8a7667699e76c7b16097722427378f.json new file mode 100644 index 0000000000..0102af86bf --- /dev/null +++ b/listener/crates/listener_core/.sqlx/query-974132b7cfb7b7f498ed01e327d764aa2b8a7667699e76c7b16097722427378f.json @@ -0,0 +1,63 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO filters (id, chain_id, consumer_id, \"from\", \"to\", \"log_address\")\n VALUES ($1, $2, $3, $4, $5, $6)\n ON CONFLICT (chain_id, consumer_id, COALESCE(\"from\", ''), COALESCE(\"to\", ''), COALESCE(\"log_address\", ''))\n DO NOTHING\n RETURNING id, chain_id, consumer_id, \"from\", \"to\", \"log_address\", created_at\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "chain_id", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "consumer_id", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "from", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "to", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "log_address", + "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "created_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Int8", + "Varchar", + "Varchar", + "Varchar", + "Varchar" + ] + }, + "nullable": [ + false, + false, + false, + true, + true, + true, + false + ] + }, + "hash": "974132b7cfb7b7f498ed01e327d764aa2b8a7667699e76c7b16097722427378f" +} diff --git a/listener/crates/listener_core/.sqlx/query-a18b47e762618babf483fd3ce24b29416397ae5551b1b6a8d649b6f0cdf6f02c.json b/listener/crates/listener_core/.sqlx/query-a18b47e762618babf483fd3ce24b29416397ae5551b1b6a8d649b6f0cdf6f02c.json new file mode 100644 index 0000000000..b92a61f318 --- /dev/null +++ b/listener/crates/listener_core/.sqlx/query-a18b47e762618babf483fd3ce24b29416397ae5551b1b6a8d649b6f0cdf6f02c.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE blocks SET status = 'UNCLE'::block_status\n WHERE chain_id = $1 AND block_number = $2 AND block_hash != $3 AND status = 'CANONICAL'::block_status\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Int8", + "Bytea" + ] + }, + "nullable": [] + }, + "hash": "a18b47e762618babf483fd3ce24b29416397ae5551b1b6a8d649b6f0cdf6f02c" +} diff --git a/listener/crates/listener_core/.sqlx/query-a3dcb603ea257638e0e5bd3891af907c4bdb821215ba9e200bf532550492225d.json b/listener/crates/listener_core/.sqlx/query-a3dcb603ea257638e0e5bd3891af907c4bdb821215ba9e200bf532550492225d.json new file mode 100644 index 0000000000..a10178bfde --- /dev/null +++ b/listener/crates/listener_core/.sqlx/query-a3dcb603ea257638e0e5bd3891af907c4bdb821215ba9e200bf532550492225d.json @@ -0,0 +1,85 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO blocks (id, chain_id, block_number, block_hash, parent_hash, status)\n VALUES ($1, $2, $3, $4, $5, $6)\n RETURNING id, chain_id, block_number, block_hash, parent_hash, status as \"status: BlockStatus\", created_at\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "chain_id", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "block_number", + "type_info": "Int8" + }, + { + "ordinal": 3, + "name": "block_hash", + "type_info": "Bytea" + }, + { + "ordinal": 4, + "name": "parent_hash", + "type_info": "Bytea" + }, + { + "ordinal": 5, + "name": "status: BlockStatus", + "type_info": { + "Custom": { + "name": "block_status", + "kind": { + "Enum": [ + "CANONICAL", + "FINALIZED", + "UNCLE" + ] + } + } + } + }, + { + "ordinal": 6, + "name": "created_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Int8", + "Int8", + "Bytea", + "Bytea", + { + "Custom": { + "name": "block_status", + "kind": { + "Enum": [ + "CANONICAL", + "FINALIZED", + "UNCLE" + ] + } + } + } + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false + ] + }, + "hash": "a3dcb603ea257638e0e5bd3891af907c4bdb821215ba9e200bf532550492225d" +} diff --git a/listener/crates/listener_core/.sqlx/query-e5076f6f048cc74e1aafa5758f568c698e2d83eb84ebaf25961121ee1343e145.json b/listener/crates/listener_core/.sqlx/query-e5076f6f048cc74e1aafa5758f568c698e2d83eb84ebaf25961121ee1343e145.json new file mode 100644 index 0000000000..4f5984a520 --- /dev/null +++ b/listener/crates/listener_core/.sqlx/query-e5076f6f048cc74e1aafa5758f568c698e2d83eb84ebaf25961121ee1343e145.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO blocks (id, chain_id, block_number, block_hash, parent_hash, status)\n VALUES ($1, $2, $3, $4, $5, 'CANONICAL'::block_status)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Int8", + "Int8", + "Bytea", + "Bytea" + ] + }, + "nullable": [] + }, + "hash": "e5076f6f048cc74e1aafa5758f568c698e2d83eb84ebaf25961121ee1343e145" +} diff --git a/listener/crates/listener_core/.sqlx/query-eef4e0688f08727be89e1cb0bb62114f0a861d1ea05d8d7fde2d17a07cca9634.json b/listener/crates/listener_core/.sqlx/query-eef4e0688f08727be89e1cb0bb62114f0a861d1ea05d8d7fde2d17a07cca9634.json new file mode 100644 index 0000000000..e21c32a5ef --- /dev/null +++ b/listener/crates/listener_core/.sqlx/query-eef4e0688f08727be89e1cb0bb62114f0a861d1ea05d8d7fde2d17a07cca9634.json @@ -0,0 +1,58 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT id, chain_id, consumer_id, \"from\", \"to\", \"log_address\", created_at\n FROM filters\n WHERE chain_id = $1\n ORDER BY consumer_id\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "chain_id", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "consumer_id", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "from", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "to", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "log_address", + "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "created_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + false, + false, + false, + true, + true, + true, + false + ] + }, + "hash": "eef4e0688f08727be89e1cb0bb62114f0a861d1ea05d8d7fde2d17a07cca9634" +} diff --git a/listener/crates/listener_core/AGENTS.md b/listener/crates/listener_core/AGENTS.md new file mode 100644 index 0000000000..5e6f0a8310 --- /dev/null +++ b/listener/crates/listener_core/AGENTS.md @@ -0,0 +1,5 @@ +# AGENTS.md + +@CLAUDE.md + +This repository keeps agent guidance in CLAUDE.md to avoid duplication. Please refer to CLAUDE.md for the full instructions. diff --git a/listener/crates/listener_core/CLAUDE.md b/listener/crates/listener_core/CLAUDE.md new file mode 100644 index 0000000000..5fd14d92fe --- /dev/null +++ b/listener/crates/listener_core/CLAUDE.md @@ -0,0 +1,106 @@ +# listener_core — Claude Code Context + +## What This Crate Does + +Production EVM block listener. Fetches blocks from RPC nodes, validates chain continuity, detects reorgs, filters transactions per consumer, and publishes events to a message broker (Redis Streams or RabbitMQ). + +## Pipeline + +``` +RPC Node + → SemEvmRpcProvider (semaphore rate-limited) + → EvmBlockFetcher (5 strategies, parallel) + → AsyncSlotBuffer (out-of-order insert → in-order read) + → Cursor (parent-hash chain validation, sequential) + → Publisher (FilterIndex matching → per-consumer payloads) + → Broker (Redis | AMQP, with ensure_publish) + → DB insert (PostgreSQL, canonical block) +``` + +## Core Algorithms + +### Cursor (`core/evm_listener.rs`) +Producer-consumer over `AsyncSlotBuffer`. Producer spawns one tokio task per block in range, fetching in parallel. Consumer reads sequentially, validates `block.parent_hash == expected_hash`. On mismatch → `CursorResult::ReorgDetected`, cancels producer. On success → publish events → insert block as CANONICAL. + +Dedup guard: Lock flow is handling the dedup by design, it will be able to discard message if there is some processing and duplicated tasks. + +### Reorg Backtrack (`core/evm_listener.rs`) +Three-phase crash-safe algorithm: +1. **Walk + Publish (read-only)**: Fetch reorged blocks by parent_hash, publish events (`BlockFlow::Reorged`), collect lightweight metadata. DB untouched. +2. **Batch Commit (atomic)**: Single DB transaction — upsert all walked blocks as CANONICAL, previous ones become UNCLE. Rollback on any failure. +3. **Resume**: Publish `FETCH_NEW_BLOCKS` to restart cursor. + +Crash at any phase → DB untouched or fully committed → retry re-walks safely (at-least-once). + +### Slot Buffer (`core/slot_buffer.rs`) +`AsyncSlotBuffer`: fixed-size vec of `Mutex>` + `Notify`. Producers call `set_once(index, value)` in any order. Consumer calls `get(index)` sequentially, awaits `Notify` if slot empty. Cancel-safe. + +### Block Fetcher (`blockchain/evm/evm_block_fetcher.rs`) +Five strategies, all with retry + cancellation: +1. **BlockReceipts** — 2 parallel tasks (block + `eth_getBlockReceipts`). Most efficient. +2. **BatchReceiptsFull** — Single batch JSON-RPC for all receipts. +3. **BatchReceiptsRange** — Chunked parallel batches (`batch_receipt_size_range`). +4. **TransactionReceiptsParallel** — One task per receipt. +5. **TransactionReceiptsSequential** — One at a time, rate-limit friendly. + +Retry classification: Unrecoverable (fail) | RateLimited (exponential backoff) | Recoverable (fixed interval). + +### Block Computer (`blockchain/evm/evm_block_computer.rs`) +Optional cryptographic verification: transaction root, receipt root, block hash. Handles L2-specific encodings (Optimism deposit tx type 126, Arbitrum internal type 106). + +### Publisher (`core/publisher.rs`) +`FilterIndex` — inverted index for O(1) transaction matching: +- `by_from`, `by_to`, `by_pair`, `by_log`, `unfiltered` (wildcard consumers) +- Builds per-consumer `BlockPayload` with filtered transactions/logs +- **Publish-before-commit**: events sent before DB insert. If publish fails → no DB mutation → cursor retries same block. Guarantees zero missed events. +- Per-consumer infinite retry until broker ACK. + +### Consumer Routing +- Internal: `FETCH_NEW_BLOCKS`, `BACKTRACK_REORG`, `WATCH`, `UNWATCH`, `CLEAN_BLOCKS` +- External: `{consumer_id}.new-event` — dynamic from filters DB +- All topics namespaced by `chain_id_to_namespace(chain_id)` + +## File Map + +| File | What | +|------|------| +| `src/main.rs` | Entry point, wiring | +| `src/lib.rs` | Public API exports | +| `src/core/evm_listener.rs` | Cursor + reorg algorithms | +| `src/core/slot_buffer.rs` | AsyncSlotBuffer | +| `src/core/publisher.rs` | FilterIndex + event publishing | +| `src/core/workers.rs` | Broker message handlers (FetchHandler, ReorgHandler, etc.) | +| `src/core/filters.rs` | Filter lifecycle (add/remove) | +| `src/core/cleaner.rs` | Old block deletion | +| `src/blockchain/evm/evm_block_fetcher.rs` | 5 fetching strategies | +| `src/blockchain/evm/evm_block_computer.rs` | Block verification | +| `src/blockchain/evm/sem_evm_rpc_provider.rs` | Semaphore-based RPC provider | +| `src/config/config.rs` | YAML + env config schema | +| `src/store/repositories/block_repo.rs` | Block DB operations (upsert, batch upsert) | +| `src/store/repositories/filter_repo.rs` | Filter DB operations | +| `src/store/models/block_model.rs` | Block, BlockStatus, UpsertResult | + +## Hard Invariants + +1. **ZERO EVENTS MISSED.** Duplication acceptable (at-least-once), but every matching event must reach its consumer. Publish-before-commit enforces this. +2. **Atomicity & crash resilience.** DB failures, Redis failures, broker failures must not corrupt state. Reorg uses read-only walk then atomic batch commit. Cursor publishes before inserting. +3. **HPA compatible.** No duplicate processing of main flow messages. `is_empty_or_pending()` dedup guard + `prefetch=1` prevents concurrent cursor runs per chain. +4. **Broker parity.** Redis Streams and RabbitMQ must produce identical behavior. `ensure_publish` maps to `WAIT 1 500` (Redis) / `confirm_select` (AMQP). Same handler interface for both. +5. **Correct consumer routing.** Events published only to consumers with matching filters. FilterIndex must be rebuilt from DB on each cursor iteration. +6. **Chain ID segregation.** All topics namespaced by chain_id. DB queries filtered by chain_id. One canonical block per (chain_id, block_number). +7. **Keep up with fastest chains.** Parallel fetching via SlotBuffer, configurable batch sizes, 5 RPC strategies, semaphore-controlled concurrency. + +## Error Handling + +Handlers classify errors for the broker: +- **Transient** (`HandlerError::Transient`): DB errors, RPC failures, broker publish failures, payload build errors → broker retries (max 5). +- **Permanent** (`HandlerError::Permanent`): Invariant violations, deserialization failures → dead-letter queue, no retry. + +## Known Accepted Limitations + +- RPC stall blocks cursor until timeout (accepted, exponential backoff mitigates). + +## Skills to load + +- if available locally, load the skill /karpathy-guidelines +- for planning, or when benchmarking, if available locally, load the skill /brainstorming diff --git a/listener/crates/listener_core/Cargo.toml b/listener/crates/listener_core/Cargo.toml new file mode 100644 index 0000000000..23ea6a3450 --- /dev/null +++ b/listener/crates/listener_core/Cargo.toml @@ -0,0 +1,41 @@ +[package] +name = "listener_core" +version = "0.1.0" +edition.workspace = true + +[dependencies] +async-trait.workspace = true +axum = { workspace = true, features = ["json", "tokio", "http1"] } +broker = { path = "../shared/broker" } +telemetry = { path = "../shared/telemetry" } +primitives = { path = "../shared/primitives" } +alloy.workspace = true +alloy-json-rpc.workspace = true +alloy-transport-ipc.workspace = true +rand = "0.9.2" +reqwest = { version = "0.13.2", features = ["json"] } +serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true +thiserror.workspace = true +tokio = { workspace = true, features = ["rt-multi-thread", "signal"] } +tokio-util.workspace = true +tracing.workspace = true +tracing-subscriber = { version = "0.3.22", features = ["json", "env-filter"] } +url = "2.5.8" +uuid = { version = "1", features = ["v4", "serde"] } +chrono = { version = "0.4", features = ["serde"] } +config = { version = "0.14", features = ["yaml"] } +derivative = "2.2" +futures.workspace = true +metrics.workspace = true +sqlx = { version = "0.8", features = ["runtime-tokio", "postgres", "migrate", "uuid", "chrono"] } +anyhow = "1.0" + +# IAM RDS auth (optional) +aws-config = { version = "1", optional = true } +aws-credential-types = { version = "1", optional = true } +aws-sigv4 = { version = "1", optional = true } + +[features] +default = [] +iam-auth = ["aws-config", "aws-credential-types", "aws-sigv4"] diff --git a/listener/crates/listener_core/config-avalanche.yaml b/listener/crates/listener_core/config-avalanche.yaml new file mode 100644 index 0000000000..83a17b1e3a --- /dev/null +++ b/listener/crates/listener_core/config-avalanche.yaml @@ -0,0 +1,39 @@ +name: avalanche-mainnet-listener + +http_port: 8082 + +database: + db_url: postgres://postgres:postgres@localhost:5432/listener + +broker: + broker_type: redis + broker_url: redis://localhost:6379 + +log: + format: compact + +telemetry: + enabled: true + metrics_port: 9093 + +blockchain: + type: evm + chain_id: 43114 + rpc_url: https://avalanche-c-chain-rpc.publicnode.com + network: avalanche-mainnet + finality_depth: 10 + cleaner: + active: true + blocks_to_keep: 1000 + cron_secs: 3600 + strategy: + automatic_startup: true + # Only for the first startup. + block_start_on_first_start: current + range_size: 200 + loop_delay_ms: 1000 + max_parallel_requests: 50 + block_fetcher: block_receipts + compute_block: true + compute_block_allow_skipping: true + max_exponential_backoff_ms: 2000 diff --git a/listener/crates/listener_core/config-minimal.yaml b/listener/crates/listener_core/config-minimal.yaml new file mode 100644 index 0000000000..f3ffbac3ad --- /dev/null +++ b/listener/crates/listener_core/config-minimal.yaml @@ -0,0 +1,36 @@ +name: polygon-mainnet-listener + +http_port: 8081 + +database: + db_url: postgres://postgres:postgres@localhost:5432/listener + +broker: + broker_type: redis + broker_url: redis://localhost:6379 + +telemetry: + enabled: true + metrics_port: 9092 + +blockchain: + type: evm + chain_id: 137 + rpc_url: https://polygon-bor-rpc.publicnode.com + network: polygon-mainnet + finality_depth: 10 + cleaner: + active: true + blocks_to_keep: 1000 + cron_secs: 3600 + strategy: + automatic_startup: true + # Only for the first startup. + block_start_on_first_start: current + range_size: 200 + loop_delay_ms: 200 + max_parallel_requests: 50 + block_fetcher: block_receipts + compute_block: true + compute_block_allow_skipping: true + max_exponential_backoff_ms: 2000 diff --git a/listener/crates/listener_core/config-polygon-compute.yaml b/listener/crates/listener_core/config-polygon-compute.yaml new file mode 100644 index 0000000000..f6c308d76c --- /dev/null +++ b/listener/crates/listener_core/config-polygon-compute.yaml @@ -0,0 +1,39 @@ +name: polygon-mainnet-listener + +http_port: 8080 + +database: + db_url: postgres://postgres:postgres@localhost:5432/listener + +broker: + broker_type: redis + broker_url: redis://localhost:6379 + +log: + format: compact + +telemetry: + enabled: true + metrics_port: 9092 + +blockchain: + type: evm + chain_id: 137 + rpc_url: https://polygon-bor-rpc.publicnode.com + network: polygon-mainnet + finality_depth: 10 + cleaner: + active: true + blocks_to_keep: 1000 + cron_secs: 3600 + strategy: + automatic_startup: true + # Only for the first startup. + block_start_on_first_start: 85523125 + range_size: 200 + loop_delay_ms: 1000 + max_parallel_requests: 50 + block_fetcher: block_receipts + compute_block: true + compute_block_allow_skipping: true + max_exponential_backoff_ms: 2000 diff --git a/listener/crates/listener_core/config.yaml b/listener/crates/listener_core/config.yaml new file mode 100644 index 0000000000..678373f049 --- /dev/null +++ b/listener/crates/listener_core/config.yaml @@ -0,0 +1,93 @@ +# Example configuration template. +# +# For ready-to-use configs, see: +# config/listener-amqp.yaml — RabbitMQ backend +# config/listener-redis.yaml — Redis Streams backend +# +# Values can be overridden via environment variables: +# APP_BROKER__BROKER_TYPE=redis APP_BROKER__BROKER_URL=redis://localhost:6379 cargo run + +name: listener + +# Shared HTTP server port. Required — no default. +# Today: /livez, /readyz. Designed to host future operational routes +# (metrics, admin, ...) on the same port. +http_port: 8080 + +database: + db_url: postgres://postgres:postgres@localhost:5432/listener + migration_max_attempts: 5 + pool: + max_connections: 10 + min_connections: 2 + # acquire_timeout_secs: 5 + # idle_timeout_secs: 600 + # max_lifetime_secs: 1800 +broker: + broker_type: redis + broker_url: redis://localhost:6379 + # broker_type: amqp + # broker_url: amqp://user:pass@localhost:5672/%2f + # ensure_publish: true # Enable replication-aware publish durability (WAIT for Redis, confirms for AMQP) + # circuit_breaker_cooldown_secs: 10 # Seconds to wait before retrying a failed publish (circuit breaker) + # circuit_breaker_threshold: 3 # Number of consecutive publish failures to trigger the circuit breaker + # claim_min_idle: 30 # Minimum idle time in seconds before a claim is eligible for processing optionnal + +telemetry: + enabled: true + metrics_port: 9091 + +log: + format: pretty # json | compact | pretty +# show_file_line: false # show source file:line in output +# show_thread_ids: true # show thread IDs +# show_timestamp: true # show RFC 3339 timestamps +# show_target: true # show module path target +# show_constants: true # inject name, network, chain_id in every log line +# level: info # default level (overridden by RUST_LOG env var) + +blockchain: + type: evm + # Ethereum mainnet + chain_id: 1 + rpc_url: https://ethereum-rpc.publicnode.com + network: ethereum-mainnet + # Monad + # chain_id: 143 + # rpc_url: https://rpc.monad.xyz/ + # network: monad-mainnet + # Arbitrum One + # chain_id: 42161 + # rpc_url: https://arbitrum.drpc.org + # network: arbitrum-one + # Bnb Smart Chain + # chain_id: 56 + # rpc_url: https://bsc-dataseed.binance.org/ + # network: bsc-mainnet + + finality_depth: 64 + cleaner: + active: true + blocks_to_keep: 1000 + cron_secs: 3600 + strategy: + automatic_startup: true + # Only used on first startup when the database is empty. + # Set to "current" to start from the latest chain height - 1, or a specific block number. + # block_start_on_first_start: 24281832 + block_start_on_first_start: "current" + range_size: 200 + loop_delay_ms: 2000 + max_parallel_requests: 50 + # block_receipts | batch_receipts_full | batch_receipts_range | transaction_receipts_parallel | transaction_receipts_seqential + block_fetcher: block_receipts + # taken into account only for batch_receipts_range + batch_receipts_size_range: 10 + compute_block: true + # Maximum backoff in ms for rate-limit retries (exponential backoff cap, default: 20000) + max_exponential_backoff_ms: 500 + publish: + publish_retry_secs: 1 + publish_stale: true + # Active if publish_stale is false. Number of retries for stale publish attempts before giving up (default: 5) + publish_no_stale_retries: 5 diff --git a/listener/crates/listener_core/migrations/20260224175428_init.sql b/listener/crates/listener_core/migrations/20260224175428_init.sql new file mode 100644 index 0000000000..eab38e490c --- /dev/null +++ b/listener/crates/listener_core/migrations/20260224175428_init.sql @@ -0,0 +1,56 @@ +-- Create enum type for block status +CREATE TYPE block_status AS ENUM ('CANONICAL', 'FINALIZED', 'UNCLE'); + +-- Create blocks table +CREATE TABLE IF NOT EXISTS blocks ( + id UUID PRIMARY KEY, + chain_id BIGINT NOT NULL, + block_number BIGINT NOT NULL, + block_hash BYTEA NOT NULL, + parent_hash BYTEA NOT NULL, + status block_status NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT uq_blocks_chain_block_hash UNIQUE (chain_id, block_hash) +); + +-- Indices for common query patterns (all include chain_id as first column) +CREATE INDEX IF NOT EXISTS idx_blocks_chain_created_at ON blocks(chain_id, created_at); +CREATE INDEX IF NOT EXISTS idx_blocks_chain_number_status ON blocks(chain_id, block_number, status); +CREATE INDEX IF NOT EXISTS idx_blocks_chain_number_created ON blocks(chain_id, block_number DESC, created_at DESC); +CREATE INDEX IF NOT EXISTS idx_blocks_chain_status_number_desc ON blocks(chain_id, status, block_number DESC); + +-- Enforce exactly one CANONICAL block per (chain_id, block_number). +-- This is critical for the cursor algorithm integrity: +-- get_latest_canonical_block() must return a single unambiguous tip. +-- Other statuses (UNCLE, FINALIZED) are not constrained — multiple can coexist at the same height. +CREATE UNIQUE INDEX IF NOT EXISTS idx_blocks_unique_canonical_per_number + ON blocks(chain_id, block_number) WHERE status = 'CANONICAL'; + +-- Trigger function for auto-updating updated_at +CREATE OR REPLACE FUNCTION set_updated_at() +RETURNS TRIGGER AS $$ +BEGIN + NEW.updated_at = NOW(); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +-- Create filters table +CREATE TABLE IF NOT EXISTS filters ( + id UUID PRIMARY KEY, + chain_id BIGINT NOT NULL, + consumer_id VARCHAR(128) NOT NULL, + "from" VARCHAR(42), + "to" VARCHAR(42), + "log_address" VARCHAR(42), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +-- Unique index: one filter per (chain, consumer, from, to, log_address) combination. +-- COALESCE maps NULL → '' so that NULLs are treated as equal for uniqueness +-- (compatible with Postgres versions before 15 which lack NULLS NOT DISTINCT). +CREATE UNIQUE INDEX IF NOT EXISTS idx_filters_unique_chain_consumer_from_to + ON filters(chain_id, consumer_id, COALESCE("from", ''), COALESCE("to", ''), COALESCE("log_address", '')); + +-- Index for fast retrieval of all filters for a given chain_id. +CREATE INDEX IF NOT EXISTS idx_filters_chain_id ON filters(chain_id); diff --git a/listener/crates/listener_core/src/blockchain/evm/evm_block_computer.rs b/listener/crates/listener_core/src/blockchain/evm/evm_block_computer.rs new file mode 100644 index 0000000000..f05a4ba6f3 --- /dev/null +++ b/listener/crates/listener_core/src/blockchain/evm/evm_block_computer.rs @@ -0,0 +1,547 @@ +use alloy::{ + consensus::{Header, ReceiptWithBloom, TxReceipt, Typed2718}, + eips::eip2718::Encodable2718, + network::{ + AnyReceiptEnvelope, AnyRpcBlock, AnyTransactionReceipt, AnyTxEnvelope, UnknownTxEnvelope, + }, + primitives::{Address, B256, Bytes, Log, U256}, + rlp::Encodable, + rpc::types::BlockTransactions, + serde::OtherFields, + trie::root::ordered_trie_root, +}; +use serde::de::DeserializeOwned; +use thiserror::Error; +use tracing::error; + +#[derive(Error, Debug)] +pub enum BlockVerificationError { + #[error("Receipt Root Mismatch: expected {expected}, calculated {calculated}")] + ReceiptRootMismatch { expected: B256, calculated: B256 }, + + #[error("Transaction Root Mismatch: expected {expected}, calculated {calculated}")] + TransactionRootMismatch { expected: B256, calculated: B256 }, + + #[error("Block Hash Mismatch: expected {expected}, calculated {calculated}")] + BlockHashMismatch { expected: B256, calculated: B256 }, + + #[error("Missing full transactions in block")] + MissingFullTransactions, + + #[error("Receipt count mismatch: expected {expected}, actual {actual}")] + ReceiptCountMismatch { expected: usize, actual: usize }, + + #[error("Unsupported transaction type {tx_type} at index {index}")] + UnsupportedTransactionType { tx_type: u8, index: usize }, + + #[error("Transaction encoding failed at index {index}: {reason}")] + TransactionEncodingFailed { index: usize, reason: String }, + + #[error("Transaction encoding failed at index {index}: missing field {field}")] + TransactionFieldMissing { index: usize, field: String }, + + #[error("Receipt encoding failed at index {index}: {reason}")] + ReceiptEncodingFailed { index: usize, reason: String }, + + #[error("Block hash verification exhausted all strategies")] + BlockHashVerificationExhausted { + expected: B256, + standard_calculated: B256, + avalanche_calculated: Option, + }, +} + +/// A helper to pass pre-encoded RLP bytes to the trie root +/// without the compiler adding extra length prefixes. +struct RawBytes(Vec); + +impl alloy::rlp::Encodable for RawBytes { + fn encode(&self, out: &mut dyn alloy::rlp::BufMut) { + out.put_slice(&self.0); + } + fn length(&self) -> usize { + self.0.len() + } +} + +/// Extract a deserializable field from OtherFields. +fn extract_field( + fields: &OtherFields, + key: &str, + index: usize, +) -> Result { + fields + .get_deserialized(key) + .ok_or_else(|| BlockVerificationError::TransactionFieldMissing { + index, + field: key.to_string(), + })? + .map_err(|e| BlockVerificationError::TransactionEncodingFailed { + index, + reason: format!("Failed to deserialize field '{}': {}", key, e), + }) +} + +/// Extract a u64 field, handling both hex strings and numbers via OtherFields. +fn extract_u64( + fields: &OtherFields, + key: &str, + index: usize, +) -> Result { + // Use get_with to access the raw JSON value and handle both hex string and number formats + fields + .get_with(key, |v| { + // Try as hex string first + if let Some(s) = v.as_str() { + u64::from_str_radix(s.trim_start_matches("0x"), 16).ok() + } else { + // Try as number + v.as_u64() + } + }) + .flatten() + .ok_or_else(|| BlockVerificationError::TransactionFieldMissing { + index, + field: key.to_string(), + }) +} + +/// Extract an optional address field. +fn extract_optional_address(fields: &OtherFields, key: &str) -> Option
{ + fields.get_deserialized::
(key).and_then(|r| r.ok()) +} + +/// RLP encode an optional address (None becomes empty bytes). +fn encode_optional_address(addr: Option
, out: &mut Vec) { + match addr { + Some(a) => a.encode(out), + None => { + // Encode empty bytes for contract creation + let empty: &[u8] = &[]; + empty.encode(out); + } + } +} + +/// Encode an Optimism deposit transaction (type 126 / 0x7E). +/// +/// RLP Field Order: +/// 0x7E + RLP([sourceHash, from, to, mint, value, gas, isSystemTx, data]) +/// +/// Note: The `from` address comes from the recovered signer in the RPC response, +/// not from the inner fields, since deposit transactions don't have signatures. +fn encode_deposit_transaction( + unknown: &UnknownTxEnvelope, + from: Address, + index: usize, +) -> Result, BlockVerificationError> { + let fields = &unknown.inner.fields; + + // Extract required fields + let source_hash: B256 = extract_field(fields, "sourceHash", index)?; + // Note: `from` is passed as parameter since it comes from the RPC response's signer field + let to: Option
= extract_optional_address(fields, "to"); + let mint: U256 = extract_field(fields, "mint", index).unwrap_or_default(); + let value: U256 = extract_field(fields, "value", index)?; + let gas: u64 = extract_u64(fields, "gas", index)?; + let is_system_tx: bool = extract_field(fields, "isSystemTx", index).unwrap_or(false); + let data: Bytes = extract_field(fields, "input", index)?; // Note: "input" not "data" + + // RLP encode fields in order + let mut payload = Vec::new(); + source_hash.encode(&mut payload); + from.encode(&mut payload); + encode_optional_address(to, &mut payload); + mint.encode(&mut payload); + value.encode(&mut payload); + gas.encode(&mut payload); + is_system_tx.encode(&mut payload); + data.encode(&mut payload); + + // Build final: type_byte + RLP list + let mut result = vec![0x7e]; + let header = alloy::rlp::Header { + list: true, + payload_length: payload.len(), + }; + header.encode(&mut result); + result.extend_from_slice(&payload); + + Ok(result) +} + +/// Encode an Arbitrum internal transaction (type 106 / 0x6A). +/// +/// RLP Field Order: +/// 0x6A + RLP([chainId, data]) +fn encode_arbitrum_internal_transaction( + unknown: &UnknownTxEnvelope, + index: usize, +) -> Result, BlockVerificationError> { + let fields = &unknown.inner.fields; + + let chain_id: u64 = extract_u64(fields, "chainId", index)?; + let data: Bytes = extract_field(fields, "input", index)?; + + let mut payload = Vec::new(); + chain_id.encode(&mut payload); + data.encode(&mut payload); + + let mut result = vec![0x6a]; + let header = alloy::rlp::Header { + list: true, + payload_length: payload.len(), + }; + header.encode(&mut result); + result.extend_from_slice(&payload); + + Ok(result) +} + +/// Encode a deposit receipt with L2-specific fields (type 126). +/// +/// Standard encoding: `type + RLP([status, cumulative_gas_used, bloom, logs])` +/// L2 encoding: `type + RLP([status, cumulative_gas_used, bloom, logs, depositNonce, depositReceiptVersion])` +fn encode_deposit_receipt( + receipt: &ReceiptWithBloom>, + deposit_nonce: Option, + deposit_receipt_version: Option, +) -> Result, BlockVerificationError> { + let mut payload = Vec::new(); + + // Standard receipt fields (use TxReceipt trait methods) + receipt.status().encode(&mut payload); + receipt.cumulative_gas_used().encode(&mut payload); + receipt.bloom().encode(&mut payload); + + // Encode logs as a list - convert slice to Vec for Encodable trait + let logs: Vec<_> = receipt.logs().to_vec(); + logs.encode(&mut payload); + + // L2 deposit fields (Canyon upgrade) + if let Some(nonce) = deposit_nonce { + nonce.encode(&mut payload); + } + if let Some(version) = deposit_receipt_version { + version.encode(&mut payload); + } + + // Build final: type_byte + RLP list + let mut result = vec![0x7e]; + let header = alloy::rlp::Header { + list: true, + payload_length: payload.len(), + }; + header.encode(&mut result); + result.extend_from_slice(&payload); + + Ok(result) +} + +/// Encode a receipt, handling L2-specific formats. +fn encode_receipt( + receipt: &AnyTransactionReceipt, + _index: usize, +) -> Result, BlockVerificationError> { + let any_envelope = &receipt.inner.inner; + let receipt_type = any_envelope.r#type; + + // Transform logs to consensus format + let consensus_receipt = any_envelope.inner.clone().map_logs(|rpc_log| rpc_log.inner); + + // Check for L2 deposit receipt (type 126) + if receipt_type == 0x7e { + // Extract L2-specific fields from "other" + let deposit_nonce: Option = receipt + .other + .get("depositNonce") + .and_then(|v| v.as_str()) + .and_then(|s| u64::from_str_radix(s.trim_start_matches("0x"), 16).ok()); + + let deposit_receipt_version: Option = receipt + .other + .get("depositReceiptVersion") + .and_then(|v| v.as_str()) + .and_then(|s| u8::from_str_radix(s.trim_start_matches("0x"), 16).ok()); + + // If L2 fields present, encode with them + if deposit_nonce.is_some() || deposit_receipt_version.is_some() { + return encode_deposit_receipt( + &consensus_receipt, + deposit_nonce, + deposit_receipt_version, + ); + } + } + + // Standard encoding for all other receipts + let consensus_envelope = AnyReceiptEnvelope { + inner: consensus_receipt, + r#type: receipt_type, + }; + + Ok(consensus_envelope.encoded_2718()) +} + +/// EVM block computer that verifies block data consistency. +/// All EVM chains (Ethereum, Optimism, Arbitrum, Base, etc.) use the same block hash algorithm. +pub struct EvmBlockComputer; + +impl EvmBlockComputer { + /// Verify the entire block: transaction root, receipt root, and block hash. + /// + /// When `allow_skipping` is true, unsupported transaction types cause the + /// transaction root check to be skipped with an ERROR log instead of failing. + /// Receipt root and block hash are always verified regardless. + pub fn verify_block( + block: &AnyRpcBlock, + receipts: &[AnyTransactionReceipt], + allow_skipping: bool, + chain_id: u64, + ) -> Result<(), BlockVerificationError> { + let chain_id_str = chain_id.to_string(); + + if allow_skipping { + match Self::verify_transaction_root(block) { + Ok(()) => {} + Err(e) => { + metrics::counter!( + "listener_compute_transaction_failure_total", + "chain_id" => chain_id_str.clone(), + "stalling" => "false" + ) + .increment(1); + error!( + block_number = block.header.number, + chain_id = chain_id, + error = %e, + "Transaction root verification failed, skipping." + ); + } + }; + match Self::verify_receipt_root(block, receipts) { + Ok(()) => {} + Err(e) => { + metrics::counter!( + "listener_compute_receipt_failure_total", + "chain_id" => chain_id_str.clone(), + "stalling" => "false" + ) + .increment(1); + error!( + block_number = block.header.number, + chain_id = chain_id, + error = %e, + "Receipt root verification failed, skipping." + ); + } + }; + match Self::verify_block_hash(block) { + Ok(()) => {} + Err(e) => { + metrics::counter!( + "listener_compute_block_failure_total", + "chain_id" => chain_id_str, + "stalling" => "false" + ) + .increment(1); + error!( + block_number = block.header.number, + chain_id = chain_id, + error = %e, + "Block hash verification failed, skipping." + ); + } + }; + } else { + if let Err(e) = Self::verify_transaction_root(block) { + metrics::counter!( + "listener_compute_transaction_failure_total", + "chain_id" => chain_id_str.clone(), + "stalling" => "true" + ) + .increment(1); + return Err(e); + } + if let Err(e) = Self::verify_receipt_root(block, receipts) { + metrics::counter!( + "listener_compute_receipt_failure_total", + "chain_id" => chain_id_str.clone(), + "stalling" => "true" + ) + .increment(1); + return Err(e); + } + if let Err(e) = Self::verify_block_hash(block) { + metrics::counter!( + "listener_compute_block_failure_total", + "chain_id" => chain_id_str, + "stalling" => "true" + ) + .increment(1); + return Err(e); + } + } + Ok(()) + } + + /// Safely encode a transaction, handling unknown types gracefully. + /// Implements encoding for L2-specific transaction types: + /// - Type 126 (0x7E): Optimism/Base deposit transactions + /// - Type 106 (0x6A): Arbitrum internal transactions + /// + /// The `from` address is needed for deposit transactions which don't have signatures. + fn safe_encode_transaction( + tx: &AnyTxEnvelope, + from: Address, + index: usize, + ) -> Result, BlockVerificationError> { + match tx { + AnyTxEnvelope::Ethereum(eth_tx) => Ok(eth_tx.encoded_2718()), + AnyTxEnvelope::Unknown(unknown) => { + let tx_type = unknown.ty(); + + // Try type-specific encoding based on transaction type + match tx_type { + // Optimism, Base + 126 => encode_deposit_transaction(unknown, from, index), + 106 => encode_arbitrum_internal_transaction(unknown, index), + // TODO: Arbitrum specific encoding for transactions. + // 100 => encode_arbitrum_deposit_transaction(unknown, index), + // 101 => encode_arbitrum_unsigned_transaction(unknown, from, index), + // 102 => encode_arbitrum_contract_transaction(unknown, index), + // 104 => encode_arbitrum_retry_transaction(unknown, from, index), + // 105 => encode_arbitrum_submit_retryable_transaction(unknown, index), + // TODO: Specific polygon 127 transaction. + // 127 => encore transaction 127 type from polygon, source: raw json. + _ => { + error!( + tx_type = tx_type, + index = index, + "Could not encode the transaction skipping for this specific type. Skipping to avoid stalling." + ); + Err(BlockVerificationError::UnsupportedTransactionType { tx_type, index }) + } + } + } + } + } + + /// Verify the transaction trie root matches the header's transactions_root. + pub fn verify_transaction_root(block: &AnyRpcBlock) -> Result<(), BlockVerificationError> { + let header = &block.header; + + let BlockTransactions::Full(txs) = &block.transactions else { + return Err(BlockVerificationError::MissingFullTransactions); + }; + + let mut encoded_txs = Vec::with_capacity(txs.len()); + for (index, tx) in txs.iter().enumerate() { + // Get the sender (from) address from the Recovered wrapper + // tx.inner is Transaction, tx.inner.inner is Recovered + let recovered = &tx.inner.inner; + let from = recovered.signer(); + let tx_envelope = recovered.inner(); + let encoded = Self::safe_encode_transaction(tx_envelope, from, index)?; + encoded_txs.push(RawBytes(encoded)); + } + + let calculated = ordered_trie_root(&encoded_txs); + + if calculated != header.transactions_root { + return Err(BlockVerificationError::TransactionRootMismatch { + expected: header.transactions_root, + calculated, + }); + } + + Ok(()) + } + + /// Verify the receipt trie root matches the header's receipts_root. + /// Handles L2-specific receipt formats (e.g., Optimism deposit receipts with depositNonce). + pub fn verify_receipt_root( + block: &AnyRpcBlock, + receipts: &[AnyTransactionReceipt], + ) -> Result<(), BlockVerificationError> { + let header = &block.header; + + // Verify receipt count matches transaction count + let tx_count = match &block.transactions { + BlockTransactions::Full(txs) => txs.len(), + BlockTransactions::Hashes(hashes) => hashes.len(), + BlockTransactions::Uncle => 0, + }; + + if receipts.len() != tx_count { + return Err(BlockVerificationError::ReceiptCountMismatch { + expected: tx_count, + actual: receipts.len(), + }); + } + + let mut encoded_receipts = Vec::with_capacity(receipts.len()); + + for (index, r) in receipts.iter().enumerate() { + let encoded = encode_receipt(r, index)?; + encoded_receipts.push(RawBytes(encoded)); + } + + let calculated = ordered_trie_root(&encoded_receipts); + + if calculated != header.receipts_root { + return Err(BlockVerificationError::ReceiptRootMismatch { + expected: header.receipts_root, + calculated, + }); + } + + Ok(()) + } + + /// Verify the block hash matches the header's hash. + pub fn verify_block_hash(block: &AnyRpcBlock) -> Result<(), BlockVerificationError> { + let rpc_header = &block.header; + let expected = rpc_header.hash; + + // Build standard Ethereum consensus header + let consensus_header = Header { + parent_hash: rpc_header.parent_hash, + ommers_hash: rpc_header.ommers_hash, + beneficiary: rpc_header.beneficiary, + state_root: rpc_header.state_root, + transactions_root: rpc_header.transactions_root, + receipts_root: rpc_header.receipts_root, + logs_bloom: rpc_header.logs_bloom, + difficulty: rpc_header.difficulty, + number: rpc_header.number, + gas_limit: rpc_header.gas_limit, + gas_used: rpc_header.gas_used, + timestamp: rpc_header.timestamp, + extra_data: rpc_header.extra_data.clone(), + mix_hash: rpc_header.mix_hash.unwrap_or_default(), + nonce: rpc_header.nonce.unwrap_or_default(), + base_fee_per_gas: rpc_header.base_fee_per_gas, + withdrawals_root: rpc_header.withdrawals_root, + blob_gas_used: rpc_header.blob_gas_used, + excess_blob_gas: rpc_header.excess_blob_gas, + parent_beacon_block_root: rpc_header.parent_beacon_block_root, + requests_hash: rpc_header.requests_hash, + }; + + let calculated = consensus_header.hash_slow(); + + if calculated == expected { + return Ok(()); + } + + Err(BlockVerificationError::BlockHashMismatch { + expected, + calculated, + }) + } +} + +#[cfg(test)] +#[path = "./tests/evm_block_computer_tests.rs"] +mod tests; diff --git a/listener/crates/listener_core/src/blockchain/evm/evm_block_fetcher.rs b/listener/crates/listener_core/src/blockchain/evm/evm_block_fetcher.rs new file mode 100644 index 0000000000..057a0fcae1 --- /dev/null +++ b/listener/crates/listener_core/src/blockchain/evm/evm_block_fetcher.rs @@ -0,0 +1,893 @@ +//! EVM Block Fetcher - Production-ready module for fetching EVM blocks with receipts. +//! +//! Provides 5 different fetching strategies with parallel tokio tasks, cancellation support, +//! smart retry on RPC errors, and optional block verification. +//! +//! # Strategies +//! +//! 1. **Block + eth_getBlockReceipts**: Most efficient when supported. 2 parallel tasks. +//! 2. **Block + Batch Receipts**: Single HTTP request with batched JSON-RPC. +//! 3. **Block + Chunked Batch Receipts**: Batches in configurable chunk sizes. +//! 4. **Block + Individual Receipts**: One task per receipt, maximum parallelism. +//! 5. **Block + Sequential Receipts**: One receipt at a time, rate-limit friendly. +//! +//! # Error Handling +//! +//! The fetcher classifies RPC errors into three categories: +//! - **Unrecoverable**: `UnsupportedMethod`, `BatchUnsupported`, `DeserializationError` - fail immediately +//! - **RateLimited**: `RateLimited` (HTTP 429) - retry with exponential backoff (500ms -> 1s -> 2s -> ... -> max) +//! - **Recoverable**: `TransportError`, `NotFound`, etc. - retry with fixed interval +//! +//! # Example +//! +//! ```ignore +//! use listener_core::blockchain::evm_block_fetcher::{EvmBlockFetcher, FetchConfig}; +//! use tokio_util::sync::CancellationToken; +//! +//! let fetcher = EvmBlockFetcher::new(provider) +//! .with_verify_block(true) +//! .with_retry_interval(Duration::from_millis(500)); +//! +//! let block = fetcher.fetch_block_with_block_receipts_by_number(12345).await?; +//! ``` + +use std::collections::HashMap; +use std::future::Future; +use std::time::Duration; + +use alloy::network::{AnyRpcBlock, AnyTransactionReceipt}; +use alloy::primitives::B256; +use thiserror::Error; +use tokio::task::JoinSet; +use tokio_util::sync::CancellationToken; +use tracing::{error, warn}; +use uuid::Uuid; + +use super::evm_block_computer::{BlockVerificationError, EvmBlockComputer}; +use super::sem_evm_rpc_provider::{RpcProviderError, SemEvmRpcProvider}; + +/// Errors that can occur during block fetching. +/// +/// Includes both operational errors and unrecoverable RPC errors. +/// Recoverable RPC errors (transport issues, rate limits) trigger retry internally. +#[derive(Error, Debug)] +pub enum BlockFetchError { + #[error("Fetch cancelled")] + Cancelled, + + #[error("Block verification failed: {0}")] + VerificationFailed(#[from] BlockVerificationError), + + #[error( + "Receipt count mismatch: block has {block_tx_count} transactions but got {receipt_count} receipts" + )] + ReceiptCountMismatch { + block_tx_count: usize, + receipt_count: usize, + }, + + #[error("Block not found after fetch")] + BlockNotFound, + + #[error("Missing receipt for transaction {tx_hash}")] + MissingReceipt { tx_hash: B256 }, + + #[error("RPC method not supported: {method} - {details}")] + UnsupportedMethod { method: String, details: String }, + + #[error("Batch requests not supported by RPC node")] + BatchUnsupported, + + #[error("RPC deserialization error in {method}: {details}")] + DeserializationError { method: String, details: String }, +} + +/// A successfully fetched block with all its receipts. +#[derive(Debug, Clone)] +pub struct FetchedBlock { + /// Unique identifier for this fetch operation (useful for distributed tracing/logging). + pub fetch_id: Uuid, + /// The fetched block with full transaction details. + pub block: AnyRpcBlock, + /// Receipts indexed by transaction hash for O(1) lookup. + pub receipts: HashMap, +} + +impl FetchedBlock { + /// Get a receipt by transaction hash. + pub fn get_receipt(&self, tx_hash: &B256) -> Option<&AnyTransactionReceipt> { + self.receipts.get(tx_hash) + } + + /// Get receipts ordered by transaction index in the block. + pub fn receipts_ordered(&self) -> Vec<&AnyTransactionReceipt> { + let tx_hashes: Vec = self.block.transactions.hashes().collect(); + tx_hashes + .iter() + .filter_map(|hash| self.receipts.get(hash)) + .collect() + } + + /// Get the number of transactions in the block. + pub fn transaction_count(&self) -> usize { + self.block.transactions.len() + } +} + +/// Configuration for block fetching operations. +#[derive(Clone)] +pub struct FetchConfig { + /// Interval between retry attempts on RPC failure. + pub retry_interval: Duration, + /// Whether to verify block integrity after fetching. + pub verify_block: bool, + /// When true, unsupported transaction types are skipped with an ERROR log + /// instead of causing a hard verification failure. Defaults to true. + pub verify_block_allow_skipping: bool, + /// Token for cancelling the fetch operation. + pub cancellation_token: CancellationToken, + /// Maximum backoff duration (ms) for rate-limit retries (exponential backoff cap). + pub max_exponential_backoff_ms: u64, + /// Chain ID for metric labels. + pub chain_id: u64, +} + +impl Default for FetchConfig { + fn default() -> Self { + Self { + retry_interval: Duration::from_millis(500), + verify_block: false, + verify_block_allow_skipping: true, + cancellation_token: CancellationToken::new(), + max_exponential_backoff_ms: 20_000, + chain_id: 0, + } + } +} + +/// EVM block fetcher with multiple fetching strategies. +/// +/// Provides production-ready block fetching with: +/// - Parallel task execution +/// - Infinite retry on RPC errors with triage with critical errors +/// - Cancellation support via CancellationToken +/// - Optional block verification +#[derive(Clone)] +pub struct EvmBlockFetcher { + provider: SemEvmRpcProvider, + config: FetchConfig, +} + +impl EvmBlockFetcher { + /// Create a new block fetcher with default configuration. + pub fn new(provider: SemEvmRpcProvider) -> Self { + Self { + provider, + config: FetchConfig::default(), + } + } + + /// Set the retry interval for RPC failures. + pub fn with_retry_interval(mut self, interval: Duration) -> Self { + self.config.retry_interval = interval; + self + } + + /// Enable or disable block verification after fetching. + pub fn with_verify_block(mut self, verify: bool) -> Self { + self.config.verify_block = verify; + self + } + + /// When true, unsupported transaction types are skipped during verification + /// with an ERROR log instead of causing a hard failure. Defaults to true. + pub fn with_verify_block_allow_skipping(mut self, allow_skipping: bool) -> Self { + self.config.verify_block_allow_skipping = allow_skipping; + self + } + + /// Set the chain ID for metric labels. + pub fn with_chain_id(mut self, chain_id: u64) -> Self { + self.config.chain_id = chain_id; + self + } + + /// Set the cancellation token for this fetcher. + pub fn with_cancellation_token(mut self, token: CancellationToken) -> Self { + self.config.cancellation_token = token; + self + } + + /// Set the maximum exponential backoff duration (ms) for rate-limit retries. + pub fn with_max_exponential_backoff_ms(mut self, max_ms: u64) -> Self { + self.config.max_exponential_backoff_ms = max_ms; + self + } + + /// Fetch a block and its receipts using eth_getBlockReceipts (by block number). + /// + /// This is the most efficient strategy when the RPC node supports eth_getBlockReceipts. + /// Spawns 2 parallel tasks: one for the block, one for all receipts. + pub async fn fetch_block_with_block_receipts_by_number( + &self, + block_number: u64, + ) -> Result { + let fetch_id = Uuid::new_v4(); + let child_token = self.config.cancellation_token.child_token(); + let mut join_set: JoinSet> = JoinSet::new(); + + // Spawn block fetch task + let provider = self.provider.clone(); + let retry_interval = self.config.retry_interval; + let max_backoff = self.config.max_exponential_backoff_ms; + let token = child_token.clone(); + join_set.spawn(async move { + let block = retry_with_cancel( + || provider.get_block_by_number(block_number), + retry_interval, + max_backoff, + &token, + ) + .await?; + Ok(FetchTaskResult::Block(Box::new(block))) + }); + + // Spawn receipts fetch task + let provider = self.provider.clone(); + let token = child_token.clone(); + join_set.spawn(async move { + let receipts = retry_with_cancel( + || provider.get_block_receipts(block_number), + retry_interval, + max_backoff, + &token, + ) + .await?; + Ok(FetchTaskResult::Receipts(receipts)) + }); + + self.collect_results(fetch_id, join_set, child_token).await + } + + /// Fetch a block and its receipts using eth_getBlockReceipts (by block hash). + pub async fn fetch_block_with_block_receipts_by_hash( + &self, + block_hash: B256, + ) -> Result { + let fetch_id = Uuid::new_v4(); + let child_token = self.config.cancellation_token.child_token(); + let mut join_set: JoinSet> = JoinSet::new(); + + // Spawn block fetch task + let provider = self.provider.clone(); + let retry_interval = self.config.retry_interval; + let max_backoff = self.config.max_exponential_backoff_ms; + let token = child_token.clone(); + let hash_str = format!("{:?}", block_hash); + join_set.spawn(async move { + let block = retry_with_cancel( + || provider.get_block_by_hash(hash_str.clone()), + retry_interval, + max_backoff, + &token, + ) + .await?; + Ok(FetchTaskResult::Block(Box::new(block))) + }); + + // For receipts by hash, we need to first get block number from the block + // So we fetch block first, then receipts + let provider = self.provider.clone(); + let token = child_token.clone(); + let hash_str = format!("{:?}", block_hash); + join_set.spawn(async move { + // First get the block to find its number + let block = retry_with_cancel( + || provider.get_block_by_hash(hash_str.clone()), + retry_interval, + max_backoff, + &token, + ) + .await?; + let block_number = block.header.number; + + // Then get receipts by block number + let receipts = retry_with_cancel( + || provider.get_block_receipts(block_number), + retry_interval, + max_backoff, + &token, + ) + .await?; + Ok(FetchTaskResult::Receipts(receipts)) + }); + + self.collect_results(fetch_id, join_set, child_token).await + } + + /// Fetch a block and its receipts using batched JSON-RPC (by block number). + /// + /// Single HTTP request with all receipt requests batched together. + pub async fn fetch_block_with_batch_receipts_by_number( + &self, + block_number: u64, + ) -> Result { + let fetch_id = Uuid::new_v4(); + let child_token = self.config.cancellation_token.child_token(); + + // First fetch the block to get transaction hashes + let block = retry_with_cancel( + || self.provider.get_block_by_number(block_number), + self.config.retry_interval, + self.config.max_exponential_backoff_ms, + &child_token, + ) + .await?; + + let tx_hashes: Vec = block.transactions.hashes().collect(); + + if tx_hashes.is_empty() { + return self.build_fetched_block(fetch_id, block, vec![]); + } + + // Fetch all receipts in a single batch + let receipts = retry_with_cancel( + || self.provider.get_transaction_receipts_batch(&tx_hashes), + self.config.retry_interval, + self.config.max_exponential_backoff_ms, + &child_token, + ) + .await?; + + self.build_fetched_block(fetch_id, block, receipts) + } + + /// Fetch a block and its receipts using batched JSON-RPC (by block hash). + pub async fn fetch_block_with_batch_receipts_by_hash( + &self, + block_hash: B256, + ) -> Result { + let fetch_id = Uuid::new_v4(); + let child_token = self.config.cancellation_token.child_token(); + let hash_str = format!("{:?}", block_hash); + + // First fetch the block to get transaction hashes + let block = retry_with_cancel( + || self.provider.get_block_by_hash(hash_str.clone()), + self.config.retry_interval, + self.config.max_exponential_backoff_ms, + &child_token, + ) + .await?; + + let tx_hashes: Vec = block.transactions.hashes().collect(); + + if tx_hashes.is_empty() { + return self.build_fetched_block(fetch_id, block, vec![]); + } + + // Fetch all receipts in a single batch + let receipts = retry_with_cancel( + || self.provider.get_transaction_receipts_batch(&tx_hashes), + self.config.retry_interval, + self.config.max_exponential_backoff_ms, + &child_token, + ) + .await?; + + self.build_fetched_block(fetch_id, block, receipts) + } + + /// Fetch a block and its receipts using chunked batched JSON-RPC (by block number). + /// + /// Receipts are fetched in parallel chunks of `batch_size`. + pub async fn fetch_block_by_number_with_parallel_batched_receipts( + &self, + block_number: u64, + batch_receipt_size_range: usize, + ) -> Result { + let fetch_id = Uuid::new_v4(); + let child_token = self.config.cancellation_token.child_token(); + + // First fetch the block to get transaction hashes + let block = retry_with_cancel( + || self.provider.get_block_by_number(block_number), + self.config.retry_interval, + self.config.max_exponential_backoff_ms, + &child_token, + ) + .await?; + + let receipts = self + .fetch_receipts_chunked(&block, batch_receipt_size_range, &child_token) + .await?; + self.build_fetched_block(fetch_id, block, receipts) + } + + /// Fetch a block and its receipts using chunked batched JSON-RPC (by block hash). + pub async fn fetch_block_by_hash_with_parallel_batched_receipts( + &self, + block_hash: B256, + batch_size: usize, + ) -> Result { + let fetch_id = Uuid::new_v4(); + let child_token = self.config.cancellation_token.child_token(); + let hash_str = format!("{:?}", block_hash); + + // First fetch the block to get transaction hashes + let block = retry_with_cancel( + || self.provider.get_block_by_hash(hash_str.clone()), + self.config.retry_interval, + self.config.max_exponential_backoff_ms, + &child_token, + ) + .await?; + + let receipts = self + .fetch_receipts_chunked(&block, batch_size, &child_token) + .await?; + self.build_fetched_block(fetch_id, block, receipts) + } + + /// Helper to fetch receipts in chunks. + async fn fetch_receipts_chunked( + &self, + block: &AnyRpcBlock, + batch_size: usize, + cancel_token: &CancellationToken, + ) -> Result, BlockFetchError> { + let tx_hashes: Vec = block.transactions.hashes().collect(); + + if tx_hashes.is_empty() { + return Ok(vec![]); + } + + let chunks: Vec> = tx_hashes + .chunks(batch_size) + .map(|chunk| chunk.to_vec()) + .collect(); + + let mut join_set: JoinSet, BlockFetchError>> = + JoinSet::new(); + + for chunk in chunks { + let provider = self.provider.clone(); + let retry_interval = self.config.retry_interval; + let max_backoff = self.config.max_exponential_backoff_ms; + let token = cancel_token.clone(); + + join_set.spawn(async move { + retry_with_cancel( + || provider.get_transaction_receipts_batch(&chunk), + retry_interval, + max_backoff, + &token, + ) + .await + }); + } + + let mut all_receipts = Vec::with_capacity(tx_hashes.len()); + + while let Some(result) = join_set.join_next().await { + match result { + Ok(Ok(chunk_receipts)) => { + all_receipts.extend(chunk_receipts); + } + Ok(Err(e)) => { + cancel_token.cancel(); + // Drain remaining tasks + while join_set.join_next().await.is_some() {} + return Err(e); + } + Err(join_err) => { + cancel_token.cancel(); + while join_set.join_next().await.is_some() {} + // JoinError means task panicked - treat as cancelled + warn!(error = %join_err, "Task panicked during chunked receipt fetch"); + return Err(BlockFetchError::Cancelled); + } + } + } + + Ok(all_receipts) + } + + /// Fetch a block and its receipts individually (by block number). + /// + /// One task per receipt - maximum parallelism but more RPC calls. + pub async fn fetch_block_with_individual_receipts_by_number( + &self, + block_number: u64, + ) -> Result { + let fetch_id = Uuid::new_v4(); + let child_token = self.config.cancellation_token.child_token(); + + // First fetch the block to get transaction hashes + let block = retry_with_cancel( + || self.provider.get_block_by_number(block_number), + self.config.retry_interval, + self.config.max_exponential_backoff_ms, + &child_token, + ) + .await?; + + let receipts = self.fetch_receipts_individual(&block, &child_token).await?; + self.build_fetched_block(fetch_id, block, receipts) + } + + /// Fetch a block and its receipts individually (by block hash). + pub async fn fetch_block_with_individual_receipts_by_hash( + &self, + block_hash: B256, + ) -> Result { + let fetch_id = Uuid::new_v4(); + let child_token = self.config.cancellation_token.child_token(); + let hash_str = format!("{:?}", block_hash); + + // First fetch the block to get transaction hashes + let block = retry_with_cancel( + || self.provider.get_block_by_hash(hash_str.clone()), + self.config.retry_interval, + self.config.max_exponential_backoff_ms, + &child_token, + ) + .await?; + + let receipts = self.fetch_receipts_individual(&block, &child_token).await?; + self.build_fetched_block(fetch_id, block, receipts) + } + + /// Helper to fetch receipts individually (one task per receipt). + async fn fetch_receipts_individual( + &self, + block: &AnyRpcBlock, + cancel_token: &CancellationToken, + ) -> Result, BlockFetchError> { + let tx_hashes: Vec = block.transactions.hashes().collect(); + + if tx_hashes.is_empty() { + return Ok(vec![]); + } + + let mut join_set: JoinSet> = + JoinSet::new(); + + for (index, tx_hash) in tx_hashes.iter().enumerate() { + let provider = self.provider.clone(); + let retry_interval = self.config.retry_interval; + let max_backoff = self.config.max_exponential_backoff_ms; + let token = cancel_token.clone(); + let hash_str = tx_hash.to_string(); + + join_set.spawn(async move { + let receipt = retry_with_cancel( + || provider.get_transaction_receipt(hash_str.clone()), + retry_interval, + max_backoff, + &token, + ) + .await?; + Ok((index, receipt)) + }); + } + + let mut indexed_receipts: Vec<(usize, AnyTransactionReceipt)> = + Vec::with_capacity(tx_hashes.len()); + + while let Some(result) = join_set.join_next().await { + match result { + Ok(Ok(indexed_receipt)) => { + indexed_receipts.push(indexed_receipt); + } + Ok(Err(e)) => { + cancel_token.cancel(); + while join_set.join_next().await.is_some() {} + return Err(e); + } + Err(join_err) => { + cancel_token.cancel(); + while join_set.join_next().await.is_some() {} + warn!(error = %join_err, "Task panicked during individual receipt fetch"); + return Err(BlockFetchError::Cancelled); + } + } + } + + // Sort by index to maintain transaction order + indexed_receipts.sort_by_key(|(idx, _)| *idx); + Ok(indexed_receipts.into_iter().map(|(_, r)| r).collect()) + } + + /// Fetch a block and its receipts sequentially (by block number). + /// + /// Receipts are fetched one at a time - slowest but most RPC-friendly. + /// Use this strategy for rate-limited RPC providers. + pub async fn fetch_block_with_sequential_receipts_by_number( + &self, + block_number: u64, + ) -> Result { + let fetch_id = Uuid::new_v4(); + let child_token = self.config.cancellation_token.child_token(); + + // First fetch the block to get transaction hashes + let block = retry_with_cancel( + || self.provider.get_block_by_number(block_number), + self.config.retry_interval, + self.config.max_exponential_backoff_ms, + &child_token, + ) + .await?; + + let receipts = self.fetch_receipts_sequential(&block, &child_token).await?; + self.build_fetched_block(fetch_id, block, receipts) + } + + /// Fetch a block and its receipts sequentially (by block hash). + pub async fn fetch_block_with_sequential_receipts_by_hash( + &self, + block_hash: B256, + ) -> Result { + let fetch_id = Uuid::new_v4(); + let child_token = self.config.cancellation_token.child_token(); + let hash_str = format!("{:?}", block_hash); + + // First fetch the block to get transaction hashes + let block = retry_with_cancel( + || self.provider.get_block_by_hash(hash_str.clone()), + self.config.retry_interval, + self.config.max_exponential_backoff_ms, + &child_token, + ) + .await?; + + let receipts = self.fetch_receipts_sequential(&block, &child_token).await?; + self.build_fetched_block(fetch_id, block, receipts) + } + + /// Helper to fetch receipts sequentially (one at a time, no parallelism). + async fn fetch_receipts_sequential( + &self, + block: &AnyRpcBlock, + cancel_token: &CancellationToken, + ) -> Result, BlockFetchError> { + let tx_hashes: Vec = block.transactions.hashes().collect(); + + if tx_hashes.is_empty() { + return Ok(vec![]); + } + + let mut receipts = Vec::with_capacity(tx_hashes.len()); + + for tx_hash in tx_hashes { + let hash_str = tx_hash.to_string(); + let receipt = retry_with_cancel( + || self.provider.get_transaction_receipt(hash_str.clone()), + self.config.retry_interval, + self.config.max_exponential_backoff_ms, + cancel_token, + ) + .await?; + receipts.push(receipt); + } + + Ok(receipts) + } + + /// Collect results from parallel block and receipts fetch tasks. + async fn collect_results( + &self, + fetch_id: Uuid, + mut join_set: JoinSet>, + child_token: CancellationToken, + ) -> Result { + let mut block: Option = None; + let mut receipts: Option> = None; + + while let Some(result) = join_set.join_next().await { + match result { + Ok(Ok(FetchTaskResult::Block(b))) => { + block = Some(*b); + } + Ok(Ok(FetchTaskResult::Receipts(r))) => { + receipts = Some(r); + } + Ok(Err(e)) => { + child_token.cancel(); + while join_set.join_next().await.is_some() {} + return Err(e); + } + Err(join_err) => { + child_token.cancel(); + while join_set.join_next().await.is_some() {} + warn!(error = %join_err, "Task panicked during fetch"); + return Err(BlockFetchError::Cancelled); + } + } + } + + let block = block.ok_or(BlockFetchError::BlockNotFound)?; + let receipts = receipts.unwrap_or_default(); + + self.build_fetched_block(fetch_id, block, receipts) + } + + /// Build a FetchedBlock from block and receipts, optionally verifying. + fn build_fetched_block( + &self, + fetch_id: Uuid, + block: AnyRpcBlock, + receipts: Vec, + ) -> Result { + // Verify receipt count matches transaction count + let tx_count = block.transactions.len(); + if receipts.len() != tx_count { + return Err(BlockFetchError::ReceiptCountMismatch { + block_tx_count: tx_count, + receipt_count: receipts.len(), + }); + } + + // Build receipts HashMap + let receipts_map: HashMap = receipts + .into_iter() + .map(|r| (r.transaction_hash, r)) + .collect(); + + // Verify all transactions have receipts + for tx_hash in block.transactions.hashes() { + if !receipts_map.contains_key(&tx_hash) { + return Err(BlockFetchError::MissingReceipt { tx_hash }); + } + } + + // Optional block verification + if self.config.verify_block { + let receipts_ordered: Vec = block + .transactions + .hashes() + .filter_map(|h| receipts_map.get(&h).cloned()) + .collect(); + + EvmBlockComputer::verify_block( + &block, + &receipts_ordered, + self.config.verify_block_allow_skipping, + self.config.chain_id, + )?; + } + + Ok(FetchedBlock { + fetch_id, + block, + receipts: receipts_map, + }) + } +} + +/// Internal enum for distinguishing task results. +enum FetchTaskResult { + Block(Box), + Receipts(Vec), +} + +/// Retry an RPC operation until it succeeds, is cancelled, or hits an unrecoverable error. +/// +/// # Retry strategy (via `ErrorClassification`) +/// - **Unrecoverable**: fail immediately, no retry. +/// - **RateLimited**: exponential backoff starting at 500ms, doubling each attempt, +/// capped at `max_exponential_backoff_ms`. Sequence: 500ms -> 1s -> 2s -> 4s -> ... -> max. +/// The backoff counter resets when a non-rate-limit attempt succeeds or errors differently. +/// - **Recoverable**: fixed `retry_interval` sleep, then retry. +/// +/// # Cancellation +/// Checked (via `biased` select) at two points per iteration: +/// 1. Before the operation is attempted. +/// 2. During the backoff/retry sleep. +/// This guarantees prompt exit when the token is cancelled, even mid-backoff. +async fn retry_with_cancel( + operation: F, + retry_interval: Duration, + max_exponential_backoff_ms: u64, + cancel_token: &CancellationToken, +) -> Result +where + F: Fn() -> Fut, + Fut: Future>, +{ + let mut rate_limit_attempt: u32 = 0; + + loop { + tokio::select! { + biased; + _ = cancel_token.cancelled() => { + return Err(BlockFetchError::Cancelled); + } + result = operation() => { + match result { + Ok(value) => return Ok(value), + Err(e) => { + match classify_error(&e) { + ErrorClassification::Unrecoverable(fatal) => { + error!(error = %e, "Unrecoverable RPC error, not retrying"); + return Err(fatal); + } + ErrorClassification::RateLimited => { + let backoff_ms = (500u64 * 2u64.saturating_pow(rate_limit_attempt)) + .min(max_exponential_backoff_ms); + rate_limit_attempt = rate_limit_attempt.saturating_add(1); + warn!(error = %e, backoff_ms, "Rate limited, backing off..."); + tokio::select! { + _ = cancel_token.cancelled() => return Err(BlockFetchError::Cancelled), + _ = tokio::time::sleep(Duration::from_millis(backoff_ms)) => {} + } + } + ErrorClassification::Recoverable => { + rate_limit_attempt = 0; + warn!(error = %e, "RPC failed, retrying..."); + tokio::select! { + _ = cancel_token.cancelled() => return Err(BlockFetchError::Cancelled), + _ = tokio::time::sleep(retry_interval) => {} + } + } + } + } + } + } + } + } +} + +/// Classification of RPC errors for retry strategy selection. +#[derive(Debug)] +enum ErrorClassification { + /// Fatal — stop retrying immediately. + Unrecoverable(BlockFetchError), + /// Transient — retry with fixed interval. + Recoverable, + /// Rate limited — retry with exponential backoff. + RateLimited, +} + +/// Classify RPC errors into unrecoverable, recoverable, or rate-limited. +/// +/// # Unrecoverable Errors +/// - `UnsupportedMethod` - RPC node doesn't support the method (e.g., eth_getBlockReceipts) +/// - `BatchUnsupported` - RPC node doesn't support batch requests +/// - `DeserializationError` - Response format issue (unlikely to self-heal) +/// +/// # Rate Limited +/// - `RateLimited` - HTTP 429 or rate limit message detected +/// +/// # Recoverable Errors +/// - `TransportError` - Network issues, timeouts +/// - `BatchError` - Batch request failed (could be transient) +/// - `SerdeError` - Serialization error (transient) +/// - `SemaphoreClosed` - Internal semaphore issue +/// - `NotFound` - Block/receipt not found YET (node may not have propagated data) +fn classify_error(error: &RpcProviderError) -> ErrorClassification { + match error { + RpcProviderError::UnsupportedMethod { method, details } => { + ErrorClassification::Unrecoverable(BlockFetchError::UnsupportedMethod { + method: method.clone(), + details: details.clone(), + }) + } + RpcProviderError::BatchUnsupported => { + ErrorClassification::Unrecoverable(BlockFetchError::BatchUnsupported) + } + RpcProviderError::DeserializationError { method, details } => { + ErrorClassification::Unrecoverable(BlockFetchError::DeserializationError { + method: method.clone(), + details: details.clone(), + }) + } + RpcProviderError::RateLimited(_) => ErrorClassification::RateLimited, + // All other errors are recoverable (transient) + _ => ErrorClassification::Recoverable, + } +} + +#[cfg(test)] +#[path = "./tests/evm_block_fetcher_tests.rs"] +mod tests; diff --git a/listener/crates/listener_core/src/blockchain/evm/mod.rs b/listener/crates/listener_core/src/blockchain/evm/mod.rs new file mode 100644 index 0000000000..165a9df531 --- /dev/null +++ b/listener/crates/listener_core/src/blockchain/evm/mod.rs @@ -0,0 +1,3 @@ +pub mod evm_block_computer; +pub mod evm_block_fetcher; +pub mod sem_evm_rpc_provider; diff --git a/listener/crates/listener_core/src/blockchain/evm/sem_evm_rpc_provider.rs b/listener/crates/listener_core/src/blockchain/evm/sem_evm_rpc_provider.rs new file mode 100644 index 0000000000..7dbd48793c --- /dev/null +++ b/listener/crates/listener_core/src/blockchain/evm/sem_evm_rpc_provider.rs @@ -0,0 +1,544 @@ +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use alloy::network::AnyRpcBlock; +use alloy::network::AnyTransactionReceipt; +use alloy::primitives::B256; +use alloy::providers::fillers::{ + BlobGasFiller, ChainIdFiller, FillProvider, GasFiller, JoinFill, NonceFiller, +}; +use alloy::providers::{Identity, Provider, ProviderBuilder, RootProvider}; +use alloy::rpc::client::RpcClient; +use alloy::transports::http::Client; +// use alloy::rpc::types::{Block, Transaction, TransactionReceipt}; +use alloy::transports::http::Http; +use serde::de::DeserializeOwned; +use serde_json::{Value, json}; +use thiserror::Error; +use tokio::sync::{Semaphore, SemaphorePermit}; +use tracing::{error, instrument, trace, warn}; +use url::Url; + +#[derive(Error, Debug)] +pub enum RpcProviderError { + #[error("Invalid RPC URL: {0}")] + UrlParseError(#[from] url::ParseError), + + #[error("Semaphore closed")] + SemaphoreClosed, + + #[error("RPC Deserialization Error in method {method}: {details}")] + DeserializationError { method: String, details: String }, + + #[error("RPC UnsupportedMethod Error in method {method}: {details}")] + UnsupportedMethod { method: String, details: String }, + + #[error("RPC Batch Error: {0}")] + BatchError(String), + + #[error("RPC Batch Unsupported by node")] + BatchUnsupported, + + #[error("RPC Rate Limited: {0}")] + RateLimited(String), + + #[error("RPC Transport Error: {0}")] + TransportError(String), + + #[error("JSON Serialization Error: {0}")] + SerdeError(#[from] serde_json::Error), + + #[error("Node returned null for expected data (Block/Receipt not found)")] + NotFound, +} + +type ProviderForSemRpc = FillProvider< + JoinFill< + Identity, + JoinFill>>, + >, + RootProvider, +>; + +#[derive(Clone)] +pub struct SemEvmRpcProvider { + /// RootProvider in Alloy 1.x takes exactly one generic: + provider: ProviderForSemRpc, + /// Semaphore to limit concurrent requests to the RPC node + semaphore: Arc, + /// RPC URL for batch requests (reqwest needs the raw URL) + rpc_url: String, + /// Sanitized host extracted from `rpc_url` (via `Url::host_str()`) used as + /// the `endpoint` label on Prometheus metrics. Only the host is emitted — + /// the URL path (which may contain API keys) is deliberately NOT exposed. + endpoint: Arc, + /// Shared HTTP client for batch requests, built with the same production policy + // batch_client: reqwest::Client, + batch_client: Client, +} + +impl SemEvmRpcProvider { + /// Create a new provider with a concurrency limit and a tuned HTTP client. + pub fn new(rpc_url: String, max_concurrent_requests: usize) -> Result { + let url = Url::parse(&rpc_url)?; + + // Extract the host only (no path, no query) to avoid leaking API keys via Prometheus. + let endpoint = Arc::new(url.host_str().unwrap_or("unknown").to_owned()); + + // Build a tuned Alloy HTTP client for heavy-duty indexing + // Slow providers (30 s timeout) + // Docker/Kubernetes stale connections (10 s idle eviction + 15 s keepalive) + // Pod rollouts and scaling events (short idle timeout forces connection recycling) + // Network policy blackholes (TCP keepalive detects silent drops) + // Load balancer idle timeout alignment (10 s < any cloud LB timeout) + let http_client = alloy::transports::http::Client::builder() + .timeout(Duration::from_secs(25)) // Must accommodate slow providers with high response time + .connect_timeout(Duration::from_secs(3)) // fast failure if the host is unreachable + .pool_idle_timeout(Duration::from_secs(10)) // aggressive eviction of unused request. + .pool_max_idle_per_host(max_concurrent_requests) // Match the semaphore size setting + .tcp_keepalive(Duration::from_secs(15)) // OS-level dead connection detection. (the pool with self heal within 10 (idle timeout) - 15 sec) + .build() + .map_err(|e| RpcProviderError::TransportError(e.to_string()))?; + + // 1. Create the Http transport with Alloy's client + let transport = Http::with_client(http_client.clone(), url); + + // 2. Create the RpcClient (The low-level engine) + // RpcClient does not take generic arguments in this version. + let rpc_client = RpcClient::new(transport, true); + + // 3. Build the Provider using the RpcClient + let provider = ProviderBuilder::new().connect_client(rpc_client); + + // Build a dedicated reqwest client for batch requests with the same production policy + let batch_client = http_client.clone(); + + Ok(Self { + provider, + semaphore: Arc::new(Semaphore::new(max_concurrent_requests)), + rpc_url, + endpoint, + batch_client, + }) + } + + /// Internal helper to acquire permit and execute a raw JSON-RPC request. + /// This ensures every single call respects the concurrency limit. + async fn raw_request( + &self, + method: &str, + params: Value, + ) -> Result { + let start = Instant::now(); + let method_owned = method.to_owned(); + let endpoint = self.endpoint.clone(); + + let _permit: SemaphorePermit = self + .semaphore + .acquire() + .await + .map_err(|_| RpcProviderError::SemaphoreClosed)?; + + metrics::gauge!( + "listener_rpc_semaphore_available", + "endpoint" => endpoint.as_str().to_owned(), + ) + .set(self.semaphore.available_permits() as f64); + + trace!(method = method, "Acquired semaphore, sending request"); + + let client = self.provider.root().client(); + + let result: T = client + .request(method.to_string(), params) + .await + .map_err(|e| { + let err_msg = e.to_string(); + let elapsed = start.elapsed().as_secs_f64(); + + if err_msg.contains("deserialization error") + || err_msg.contains("data did not match") + { + error!( + method = method, + error = %err_msg, + "CRITICAL: rpc deserialization failure." + ); + + record_rpc_error(&method_owned, endpoint.as_str(), "deserialization", elapsed); + + return RpcProviderError::DeserializationError { + method: method.to_string(), + details: err_msg, + }; + } + if err_msg.contains("does not exist") + || err_msg.contains("is not whitelisted") + || err_msg.contains("is unsupported") + { + error!( + method = method, + error = %err_msg, + "CRITICAL: method is not supported." + ); + + record_rpc_error( + &method_owned, + endpoint.as_str(), + "unsupported_method", + elapsed, + ); + + return RpcProviderError::UnsupportedMethod { + method: method.to_string(), + details: err_msg, + }; + } + if err_msg.contains("rate limit") + || err_msg.contains("too many requests") + || err_msg.contains("429") + { + warn!(method = method, error = %err_msg, "Rate limited by RPC provider"); + + record_rpc_error(&method_owned, endpoint.as_str(), "rate_limited", elapsed); + + return RpcProviderError::RateLimited(err_msg); + } + + record_rpc_error(&method_owned, endpoint.as_str(), "transport", elapsed); + + RpcProviderError::TransportError(err_msg) + })?; + + // Success path + let elapsed = start.elapsed().as_secs_f64(); + metrics::counter!( + "listener_rpc_requests_total", + "method" => method_owned, + "endpoint" => endpoint.as_str().to_owned(), + "status" => "success", + ) + .increment(1); + metrics::histogram!( + "listener_rpc_request_duration_seconds", + "method" => method.to_owned(), + "endpoint" => endpoint.as_str().to_owned(), + ) + .record(elapsed); + + Ok(result) + } + + #[instrument(skip(self), level = "debug")] + pub async fn get_block_number(&self) -> Result { + let hex: String = self.raw_request("eth_blockNumber", json!([])).await?; + + u64::from_str_radix(hex.trim_start_matches("0x"), 16) + .map_err(|e| RpcProviderError::SerdeError(serde::ser::Error::custom(e))) + } + + #[instrument(skip(self), level = "debug")] + pub async fn get_chain_id(&self) -> Result { + let hex: String = self.raw_request("eth_chainId", json!([])).await?; + + u64::from_str_radix(hex.trim_start_matches("0x"), 16) + .map_err(|e| RpcProviderError::SerdeError(serde::ser::Error::custom(e))) + } + + #[instrument(skip(self), level = "debug")] + pub async fn get_block_by_number(&self, number: u64) -> Result { + let hex_number = format!("0x{:x}", number); + self.get_block_by_number_hex(hex_number).await + } + + pub async fn get_block_by_number_hex( + &self, + number_hex: String, + ) -> Result { + let block: Option = self + .raw_request("eth_getBlockByNumber", json!([number_hex, true])) + .await?; + + block.ok_or(RpcProviderError::NotFound) + } + + #[instrument(skip(self), level = "debug")] + pub async fn get_block_by_hash(&self, hash: String) -> Result { + let block: Option = self + .raw_request("eth_getBlockByHash", json!([hash, true])) + .await?; + + block.ok_or(RpcProviderError::NotFound) + } + + #[instrument(skip(self), level = "debug")] + pub async fn get_transaction_receipt( + &self, + tx_hash: String, + ) -> Result { + let receipt: Option = self + .raw_request("eth_getTransactionReceipt", json!([tx_hash])) + .await?; + + receipt.ok_or(RpcProviderError::NotFound) + } + + #[instrument(skip(self), level = "debug")] + pub async fn get_block_receipts( + &self, + number: u64, + ) -> Result, RpcProviderError> { + let hex_number = format!("0x{:x}", number); + self.get_block_receipts_hex(hex_number).await + } + + pub async fn get_block_receipts_hex( + &self, + number_hex: String, + ) -> Result, RpcProviderError> { + let receipts: Option> = self + .raw_request("eth_getBlockReceipts", json!([number_hex])) + .await?; + + receipts.ok_or(RpcProviderError::NotFound) + } + + /// Fetch multiple transaction receipts in a single JSON-RPC batch request. + /// More efficient than individual calls when eth_getBlockReceipts is unavailable. + #[instrument(skip(self, tx_hashes), level = "debug")] + pub async fn get_transaction_receipts_batch( + &self, + tx_hashes: &[B256], + ) -> Result, RpcProviderError> { + const BATCH_METHOD: &str = "eth_getTransactionReceipt_batch"; + + if tx_hashes.is_empty() { + return Ok(vec![]); + } + + let start = Instant::now(); + let endpoint = self.endpoint.clone(); + + let _permit = self + .semaphore + .acquire() + .await + .map_err(|_| RpcProviderError::SemaphoreClosed)?; + + metrics::gauge!( + "listener_rpc_semaphore_available", + "endpoint" => endpoint.as_str().to_owned(), + ) + .set(self.semaphore.available_permits() as f64); + + trace!(count = tx_hashes.len(), "Sending batch receipt request"); + + // Build batch request array + let batch_request: Vec = tx_hashes + .iter() + .enumerate() + .map(|(id, hash)| { + json!({ + "jsonrpc": "2.0", + "id": id, + "method": "eth_getTransactionReceipt", + "params": [format!("{:?}", hash)] + }) + }) + .collect(); + + // Send batch request via the shared, production-tuned HTTP client + let response = self + .batch_client + .post(&self.rpc_url) + .header("Content-Type", "application/json") + .json(&batch_request) + .send() + .await + .map_err(|e| { + record_rpc_error( + BATCH_METHOD, + endpoint.as_str(), + "transport", + start.elapsed().as_secs_f64(), + ); + RpcProviderError::BatchError(format!("HTTP request failed: {}", e)) + })?; + + if response.status() == reqwest::StatusCode::TOO_MANY_REQUESTS { + record_rpc_error( + BATCH_METHOD, + endpoint.as_str(), + "rate_limited", + start.elapsed().as_secs_f64(), + ); + return Err(RpcProviderError::RateLimited( + "HTTP 429: rate limited".to_string(), + )); + } + + if !response.status().is_success() { + record_rpc_error( + BATCH_METHOD, + endpoint.as_str(), + "batch_error", + start.elapsed().as_secs_f64(), + ); + return Err(RpcProviderError::BatchError(format!( + "HTTP error: {}", + response.status() + ))); + } + + let response_text = response.text().await.map_err(|e| { + record_rpc_error( + BATCH_METHOD, + endpoint.as_str(), + "batch_error", + start.elapsed().as_secs_f64(), + ); + RpcProviderError::BatchError(format!("Failed to read response: {}", e)) + })?; + + // Parse the batch response + let batch_response: Vec = serde_json::from_str(&response_text).map_err(|e| { + // Check if the node doesn't support batch requests (returns single error object) + if response_text.contains("error") && !response_text.starts_with('[') { + record_rpc_error( + BATCH_METHOD, + endpoint.as_str(), + "batch_unsupported", + start.elapsed().as_secs_f64(), + ); + return RpcProviderError::BatchUnsupported; + } + record_rpc_error( + BATCH_METHOD, + endpoint.as_str(), + "batch_error", + start.elapsed().as_secs_f64(), + ); + RpcProviderError::BatchError(format!("Failed to parse batch response: {}", e)) + })?; + + // Sort by ID to maintain order + let mut sorted_responses: Vec<(usize, Value)> = batch_response + .into_iter() + .filter_map(|v| { + let id = v.get("id")?.as_u64()? as usize; + Some((id, v)) + }) + .collect(); + sorted_responses.sort_by_key(|(id, _)| *id); + + // Extract and deserialize results + let mut receipts = Vec::with_capacity(tx_hashes.len()); + for (id, response) in sorted_responses { + // Check for error in individual response + if let Some(err) = response.get("error") { + let err_msg = err + .get("message") + .and_then(|m| m.as_str()) + .unwrap_or("Unknown error"); + record_rpc_error( + BATCH_METHOD, + endpoint.as_str(), + "batch_error", + start.elapsed().as_secs_f64(), + ); + return Err(RpcProviderError::BatchError(format!( + "Error for tx {}: {}", + id, err_msg + ))); + } + + let result = response.get("result").ok_or_else(|| { + record_rpc_error( + BATCH_METHOD, + endpoint.as_str(), + "batch_error", + start.elapsed().as_secs_f64(), + ); + RpcProviderError::BatchError(format!("Missing result for tx {}", id)) + })?; + + if result.is_null() { + record_rpc_error( + BATCH_METHOD, + endpoint.as_str(), + "not_found", + start.elapsed().as_secs_f64(), + ); + return Err(RpcProviderError::NotFound); + } + + let receipt: AnyTransactionReceipt = + serde_json::from_value(result.clone()).map_err(|e| { + record_rpc_error( + BATCH_METHOD, + endpoint.as_str(), + "deserialization", + start.elapsed().as_secs_f64(), + ); + RpcProviderError::DeserializationError { + method: "eth_getTransactionReceipt (batch)".to_string(), + details: format!("Failed to deserialize receipt {}: {}", id, e), + } + })?; + + receipts.push(receipt); + } + + // Success path + let elapsed = start.elapsed().as_secs_f64(); + metrics::counter!( + "listener_rpc_requests_total", + "method" => BATCH_METHOD, + "endpoint" => endpoint.as_str().to_owned(), + "status" => "success", + ) + .increment(1); + metrics::histogram!( + "listener_rpc_request_duration_seconds", + "method" => BATCH_METHOD, + "endpoint" => endpoint.as_str().to_owned(), + ) + .record(elapsed); + + Ok(receipts) + } +} + +/// Record RPC error metrics (shared by `raw_request` and `get_transaction_receipts_batch`). +fn record_rpc_error(method: &str, endpoint: &str, error_kind: &'static str, elapsed_secs: f64) { + metrics::counter!( + "listener_rpc_requests_total", + "method" => method.to_owned(), + "endpoint" => endpoint.to_owned(), + "status" => "error", + ) + .increment(1); + metrics::counter!( + "listener_rpc_errors_total", + "method" => method.to_owned(), + "endpoint" => endpoint.to_owned(), + "error_kind" => error_kind, + ) + .increment(1); + metrics::histogram!( + "listener_rpc_request_duration_seconds", + "method" => method.to_owned(), + "endpoint" => endpoint.to_owned(), + ) + .record(elapsed_secs); +} + +/// Extract transaction hashes from an AnyRpcBlock. +/// Convenience utility for batch operations. +pub fn extract_tx_hashes(block: &AnyRpcBlock) -> Vec { + block.transactions.hashes().collect() +} + +#[cfg(test)] +#[path = "./tests/sem_evm_rpc_provider_tests.rs"] +mod tests; diff --git a/listener/crates/listener_core/src/blockchain/evm/tests/evm_block_computer_tests.rs b/listener/crates/listener_core/src/blockchain/evm/tests/evm_block_computer_tests.rs new file mode 100644 index 0000000000..2fe321b2ef --- /dev/null +++ b/listener/crates/listener_core/src/blockchain/evm/tests/evm_block_computer_tests.rs @@ -0,0 +1,284 @@ +#[cfg(test)] +mod evm_block_computer_tests { + use crate::blockchain::evm::{ + evm_block_computer::EvmBlockComputer, + sem_evm_rpc_provider::{RpcProviderError, SemEvmRpcProvider, extract_tx_hashes}, + }; + use alloy::network::{AnyRpcBlock, AnyTransactionReceipt}; + use tracing::{Level, error, info, trace}; + use tracing_subscriber::FmtSubscriber; + + struct RpcTarget { + name: &'static str, + url: &'static str, + } + + /// Fetch receipts with 3-tier fallback strategy. + /// 1. eth_getBlockReceipts (most efficient, single call) + /// 2. Batched eth_getTransactionReceipt (single HTTP call, multiple JSON-RPC requests) + /// 3. Individual eth_getTransactionReceipt calls (least efficient) + async fn fetch_receipts_with_fallback( + provider: &SemEvmRpcProvider, + block: &AnyRpcBlock, + ) -> Result, String> { + // 1. Try eth_getBlockReceipts first (most efficient) + match provider.get_block_receipts(block.header.number).await { + Ok(receipts) => return Ok(receipts), + Err(RpcProviderError::UnsupportedMethod { .. }) => { + info!("eth_getBlockReceipts unsupported, trying batch..."); + } + Err(e) => return Err(e.to_string()), + } + + // 2. Try batched eth_getTransactionReceipt (single HTTP call) + let tx_hashes = extract_tx_hashes(block); + match provider.get_transaction_receipts_batch(&tx_hashes).await { + Ok(receipts) => return Ok(receipts), + Err(RpcProviderError::UnsupportedMethod { .. }) + | Err(RpcProviderError::BatchError(_)) + | Err(RpcProviderError::BatchUnsupported) => { + info!("Batch receipts unsupported, falling back to individual calls..."); + } + Err(e) => return Err(e.to_string()), + } + + // 3. Final fallback: individual eth_getTransactionReceipt calls + let mut receipts = Vec::new(); + for hash in block.transactions.hashes() { + let receipt = provider + .get_transaction_receipt(hash.to_string()) + .await + .map_err(|e| format!("Receipt fetch failed for {}: {}", hash, e))?; + receipts.push(receipt); + } + Ok(receipts) + } + + struct VerificationReport { + name: String, + url: String, + block_number: Result, + tx_count: Result, + receipt_count: Result, + tx_root_verification: Result<(), String>, + receipt_root_verification: Result<(), String>, + block_hash_verification: Result<(), String>, + } + + impl VerificationReport { + fn fatal(target: &RpcTarget, reason: String) -> Self { + Self { + name: target.name.to_string(), + url: target.url.to_string(), + block_number: Err(reason), + tx_count: Err("Skipped".to_string()), + receipt_count: Err("Skipped".to_string()), + tx_root_verification: Err("Skipped".to_string()), + receipt_root_verification: Err("Skipped".to_string()), + block_hash_verification: Err("Skipped".to_string()), + } + } + } + + #[tokio::test] + async fn report_block_verification() { + // Initialize tracing + let subscriber = FmtSubscriber::builder() + .with_max_level(Level::INFO) + .finish(); + let _ = tracing::subscriber::set_global_default(subscriber); + + let targets = vec![ + RpcTarget { + name: "Ethereum Mainnet", + url: "https://ethereum-rpc.publicnode.com", + }, + // RpcTarget { + // name: "Binance Smart Chain Mainnet", + // url: "https://bsc-dataseed.bnbchain.org", + // }, + // RpcTarget { + // name: "Polygon PoS", + // // url: "https://polygon-rpc.com", + // url: "https://polygon.drpc.org", + // }, + // RpcTarget { + // name: "Avalanche", + // url: "https://api.avax.network/ext/bc/C/rpc", + // }, + // RpcTarget { + // name: "Monad", + // url: "https://rpc.monad.xyz/", + // }, + // RpcTarget { + // name: "Optimism", + // url: "https://mainnet.optimism.io", + // }, + // RpcTarget { + // name: "Arbitrum One", + // url: "https://arb1.arbitrum.io/rpc", + // }, + // RpcTarget { + // name: "Base", + // url: "https://mainnet.base.org", + // }, + // RpcTarget { + // name: "Zama Gateway Testnet", + // url: "https://rpc-zama-testnet-0.t.conduit.xyz", + // }, + // RpcTarget { + // name: "Polkadot mainnet", + // url: "https://eth-rpc.polkadot.io/", + // }, + // RpcTarget { + // name: "Polkadot testnet", + // url: "https://services.polkadothub-rpc.com/testnet", + // }, + ]; + + println!("\n\n{}", "=".repeat(80)); + println!("🔍 STARTING BLOCK VERIFICATION REPORT"); + println!("{}\n", "=".repeat(80)); + + for target in targets { + info!("Probing {}...", target.name); + + let report = probe_chain_verification(&target).await; + + print_verification_report(&report); + } + + println!("{}", "=".repeat(80)); + println!("✅ VERIFICATION REPORT COMPLETE"); + println!("{}\n", "=".repeat(80)); + } + + async fn probe_chain_verification(target: &RpcTarget) -> VerificationReport { + // 1. Create provider + let provider = match SemEvmRpcProvider::new(target.url.to_string(), 5) { + Ok(p) => p, + Err(e) => { + error!("Failed to create provider: {}", e); + return VerificationReport::fatal( + target, + format!("Provider creation failed: {}", e), + ); + } + }; + + // 2. Get latest block number + let height = match provider.get_block_number().await { + Ok(h) => h, + Err(e) => { + error!("Failed to get block number: {}", e); + return VerificationReport::fatal( + target, + format!("Block number call failed: {}", e), + ); + } + }; + + let target_block = height - 10; + info!("Block #{} fetched (height: {})", target_block, height); + + // 3. Fetch block + trace!("Fetching block..."); + let block = match provider.get_block_by_number(target_block).await { + Ok(b) => b, + Err(e) => { + error!("Failed to fetch block: {}", e); + return VerificationReport::fatal(target, format!("Block fetch failed: {}", e)); + } + }; + + let tx_count = block.transactions.len(); + info!("Block #{} fetched ({} txs)", target_block, tx_count); + + // 4. Fetch all receipts for that block with fallback + trace!("Fetching receipts..."); + let receipts_result = fetch_receipts_with_fallback(&provider, &block).await; + let (receipt_count, receipts) = match receipts_result { + Ok(r) => { + info!( + "Block #{} receipts fetched ({} receipts)", + target_block, + r.len() + ); + (Ok(r.len()), Some(r)) + } + Err(e) => { + error!("Failed to fetch receipts: {}", e); + (Err(format!("Receipt fetch failed: {}", e)), None) + } + }; + + // 5. Run each verification separately to report granular results + // Panic handling is now done in the main code, so we just call directly + let tx_root = EvmBlockComputer::verify_transaction_root(&block).map_err(|e| { + error!("Transaction root verification failed: {}", e); + e.to_string() + }); + + let receipt_root = match &receipts { + Some(r) => EvmBlockComputer::verify_receipt_root(&block, r).map_err(|e| { + error!("Receipt root verification failed: {}", e); + e.to_string() + }), + None => Err("Skipped (no receipts)".to_string()), + }; + + let block_hash = EvmBlockComputer::verify_block_hash(&block).map_err(|e| { + error!("Block hash verification failed: {}", e); + e.to_string() + }); + + // 6. Return report + VerificationReport { + name: target.name.to_string(), + url: target.url.to_string(), + block_number: Ok(target_block), + tx_count: Ok(tx_count), + receipt_count, + tx_root_verification: tx_root, + receipt_root_verification: receipt_root, + block_hash_verification: block_hash, + } + } + + fn print_verification_report(r: &VerificationReport) { + info!("CHAIN: {}", r.name); + info!("URL: {}", r.url); + + // Print block info + match (&r.block_number, &r.tx_count, &r.receipt_count) { + (Ok(block), Ok(txs), Ok(receipts)) => { + info!(" Block #{} ({} txs, {} receipts)", block, txs, receipts); + } + (Ok(block), Ok(txs), Err(_)) => { + info!(" Block #{} ({} txs, receipts unavailable)", block, txs); + } + (Err(e), _, _) => { + info!(" [FAIL] Block fetch: {}", e); + } + _ => {} + } + + // Print verification results + match &r.tx_root_verification { + Ok(()) => info!(" [OK] Transaction Root Verified ✓"), + Err(e) => info!(" [FAIL] Transaction Root {}", e), + } + + match &r.receipt_root_verification { + Ok(()) => info!(" [OK] Receipt Root Verified ✓"), + Err(e) => info!(" [FAIL] Receipt Root {}", e), + } + + match &r.block_hash_verification { + Ok(()) => info!(" [OK] Block Hash Verified ✓"), + Err(e) => info!(" [FAIL] Block Hash {}", e), + } + + info!("{}\n", "-".repeat(40)); + } +} diff --git a/listener/crates/listener_core/src/blockchain/evm/tests/evm_block_fetcher_tests.rs b/listener/crates/listener_core/src/blockchain/evm/tests/evm_block_fetcher_tests.rs new file mode 100644 index 0000000000..8cf436268f --- /dev/null +++ b/listener/crates/listener_core/src/blockchain/evm/tests/evm_block_fetcher_tests.rs @@ -0,0 +1,1442 @@ +//! Tests for the EVM Block Fetcher module. +//! +//! Test categories: +//! 1. Strategy tests (all 5 strategies, all chains) +//! 2. Cancellation tests +//! 3. Verification tests +//! 4. Retry tests +//! +//! Note: Some tests are commented out due to rate limiting from free RPC providers. +//! These tests should be re-enabled when using private/paid RPC nodes. + +use super::*; +use std::time::Duration; +use tokio_util::sync::CancellationToken; +use tracing::{Level, info}; +use tracing_subscriber::FmtSubscriber; + +struct RpcTarget { + name: &'static str, + url: &'static str, +} + +/// Returns a minimal set of test chains to avoid rate limiting from free RPC providers. +/// The full list is commented below for reference when using private/paid nodes. +fn get_test_chains() -> Vec { + vec![ + // Using only chains with generous rate limits for free tiers + RpcTarget { + name: "Zama Gateway Testnet", + url: "https://rpc-zama-testnet-0.t.conduit.xyz", + }, + RpcTarget { + name: "Optimism", + url: "https://mainnet.optimism.io", + }, + RpcTarget { + name: "Base", + url: "https://mainnet.base.org", + }, + ] +} + +// Full chain list - uncomment when using private/paid RPC nodes: +// fn get_test_chains_full() -> Vec { +// vec![ +// RpcTarget { +// name: "Ethereum Mainnet", +// url: "https://ethereum-rpc.publicnode.com", +// }, +// RpcTarget { +// name: "Binance Smart Chain Mainnet", +// url: "https://bsc-dataseed.bnbchain.org", +// }, +// RpcTarget { +// name: "Polygon PoS", +// url: "https://polygon-rpc.com", +// }, +// RpcTarget { +// name: "Avalanche", +// url: "https://api.avax.network/ext/bc/C/rpc", +// }, +// RpcTarget { +// name: "Monad", +// url: "https://rpc.monad.xyz/", +// }, +// RpcTarget { +// name: "Optimism", +// url: "https://mainnet.optimism.io", +// }, +// RpcTarget { +// name: "Arbitrum One", +// url: "https://arb1.arbitrum.io/rpc", +// }, +// RpcTarget { +// name: "Base", +// url: "https://mainnet.base.org", +// }, +// RpcTarget { +// name: "Zama Gateway Testnet", +// url: "https://rpc-zama-testnet-0.t.conduit.xyz", +// }, +// RpcTarget { +// name: "Polkadot mainnet", +// url: "https://eth-rpc.polkadot.io/", +// }, +// RpcTarget { +// name: "Polkadot testnet", +// url: "https://services.polkadothub-rpc.com/testnet", +// }, +// ] +// } + +fn init_tracing() { + let subscriber = FmtSubscriber::builder() + .with_max_level(Level::INFO) + .finish(); + let _ = tracing::subscriber::set_global_default(subscriber); +} + +#[tokio::test] +async fn test_fetcher_clone() { + init_tracing(); + + println!("\n{}", "=".repeat(80)); + println!("CLONE: EvmBlockFetcher Clone Functionality"); + println!("{}\n", "=".repeat(80)); + + let target = RpcTarget { + name: "Zama Gateway Testnet", + url: "https://rpc-zama-testnet-0.t.conduit.xyz", + }; + + let provider = SemEvmRpcProvider::new(target.url.to_string(), 5).expect("Provider creation"); + let master_token = CancellationToken::new(); + + // Create one fetcher with shared cancellation token + let fetcher = EvmBlockFetcher::new(provider) + .with_cancellation_token(master_token.clone()) + .with_retry_interval(Duration::from_millis(100)); + + // Clone the fetcher (this is what we're testing) + let fetcher_clone = fetcher.clone(); + + // Both fetchers should work independently + let result1 = fetcher.fetch_block_with_block_receipts_by_number(1).await; + let result2 = fetcher_clone + .fetch_block_with_block_receipts_by_number(1) + .await; + + match (result1, result2) { + (Ok(r1), Ok(r2)) => { + info!( + " [OK] Original fetcher: Block #1 ({} txs)", + r1.transaction_count() + ); + info!( + " [OK] Cloned fetcher: Block #1 ({} txs)", + r2.transaction_count() + ); + assert_eq!(r1.block.header.hash, r2.block.header.hash); + info!(" [OK] Both fetchers returned identical block hashes"); + } + (Err(e1), _) => { + info!(" [FAIL] Original fetcher error: {}", e1); + } + (_, Err(e2)) => { + info!(" [FAIL] Cloned fetcher error: {}", e2); + } + } +} + +#[tokio::test] +async fn test_fetcher_clone_shared_cancellation() { + init_tracing(); + + println!("\n{}", "=".repeat(80)); + println!("CLONE: Shared Cancellation Token"); + println!("{}\n", "=".repeat(80)); + + let target = RpcTarget { + name: "Zama Gateway Testnet", + url: "https://rpc-zama-testnet-0.t.conduit.xyz", + }; + + let provider = SemEvmRpcProvider::new(target.url.to_string(), 5).expect("Provider creation"); + let master_token = CancellationToken::new(); + + // Create fetcher with shared cancellation token + let fetcher = EvmBlockFetcher::new(provider) + .with_cancellation_token(master_token.clone()) + .with_retry_interval(Duration::from_millis(100)); + + // Clone the fetcher + let fetcher_clone = fetcher.clone(); + + // Cancel the master token + master_token.cancel(); + + // Both fetchers should be cancelled + let result1 = fetcher.fetch_block_with_block_receipts_by_number(1).await; + let result2 = fetcher_clone + .fetch_block_with_block_receipts_by_number(1) + .await; + + match (&result1, &result2) { + (Err(BlockFetchError::Cancelled), Err(BlockFetchError::Cancelled)) => { + info!(" [OK] Both fetchers were cancelled by master token"); + } + _ => { + info!( + " [WARN] Expected both cancelled, got: {:?}, {:?}", + result1.is_ok(), + result2.is_ok() + ); + } + } +} + +#[tokio::test] +async fn test_strategy1_block_receipts_by_number_all_chains() { + init_tracing(); + + println!("\n{}", "=".repeat(80)); + println!("STRATEGY 1: eth_getBlockReceipts by Number"); + println!("{}\n", "=".repeat(80)); + + for target in get_test_chains() { + info!("Testing {}...", target.name); + + let provider = match SemEvmRpcProvider::new(target.url.to_string(), 5) { + Ok(p) => p, + Err(e) => { + info!(" [SKIP] {}: Provider creation failed: {}", target.name, e); + continue; + } + }; + + let height = match provider.get_block_number().await { + Ok(h) => h, + Err(e) => { + info!(" [SKIP] {}: Block number failed: {}", target.name, e); + continue; + } + }; + + let fetcher = + EvmBlockFetcher::new(provider).with_retry_interval(Duration::from_millis(100)); + + let target_block = height.saturating_sub(10); + + match fetcher + .fetch_block_with_block_receipts_by_number(target_block) + .await + { + Ok(result) => { + info!( + " [OK] {}: Block #{} ({} txs, {} receipts)", + target.name, + target_block, + result.transaction_count(), + result.receipts.len() + ); + assert_eq!(result.transaction_count(), result.receipts.len()); + } + Err(e) => { + info!(" [FAIL] {}: {}", target.name, e); + } + } + } +} + +#[tokio::test] +async fn test_strategy1_block_receipts_by_hash_all_chains() { + init_tracing(); + + println!("\n{}", "=".repeat(80)); + println!("STRATEGY 1: eth_getBlockReceipts by Hash"); + println!("{}\n", "=".repeat(80)); + + for target in get_test_chains() { + info!("Testing {}...", target.name); + + let provider = match SemEvmRpcProvider::new(target.url.to_string(), 5) { + Ok(p) => p, + Err(e) => { + info!(" [SKIP] {}: Provider creation failed: {}", target.name, e); + continue; + } + }; + + let height = match provider.get_block_number().await { + Ok(h) => h, + Err(e) => { + info!(" [SKIP] {}: Block number failed: {}", target.name, e); + continue; + } + }; + + // First get the block to get its hash + let target_block_num = height.saturating_sub(10); + let block = match provider.get_block_by_number(target_block_num).await { + Ok(b) => b, + Err(e) => { + info!(" [SKIP] {}: Block fetch failed: {}", target.name, e); + continue; + } + }; + + let block_hash = block.header.hash; + let fetcher = + EvmBlockFetcher::new(provider).with_retry_interval(Duration::from_millis(100)); + + match fetcher + .fetch_block_with_block_receipts_by_hash(block_hash) + .await + { + Ok(result) => { + info!( + " [OK] {}: Block {} ({} txs, {} receipts)", + target.name, + block_hash, + result.transaction_count(), + result.receipts.len() + ); + assert_eq!(result.transaction_count(), result.receipts.len()); + assert_eq!(result.block.header.hash, block_hash); + } + Err(e) => { + info!(" [FAIL] {}: {}", target.name, e); + } + } + } +} + +#[tokio::test] +async fn test_strategy2_batch_receipts_by_number_all_chains() { + init_tracing(); + + println!("\n{}", "=".repeat(80)); + println!("STRATEGY 2: Batch Receipts by Number"); + println!("{}\n", "=".repeat(80)); + + for target in get_test_chains() { + info!("Testing {}...", target.name); + + let provider = match SemEvmRpcProvider::new(target.url.to_string(), 5) { + Ok(p) => p, + Err(e) => { + info!(" [SKIP] {}: Provider creation failed: {}", target.name, e); + continue; + } + }; + + let height = match provider.get_block_number().await { + Ok(h) => h, + Err(e) => { + info!(" [SKIP] {}: Block number failed: {}", target.name, e); + continue; + } + }; + + let fetcher = + EvmBlockFetcher::new(provider).with_retry_interval(Duration::from_millis(100)); + + let target_block = height.saturating_sub(10); + + match fetcher + .fetch_block_with_batch_receipts_by_number(target_block) + .await + { + Ok(result) => { + info!( + " [OK] {}: Block #{} ({} txs, {} receipts)", + target.name, + target_block, + result.transaction_count(), + result.receipts.len() + ); + assert_eq!(result.transaction_count(), result.receipts.len()); + } + Err(e) => { + info!(" [FAIL] {}: {}", target.name, e); + } + } + } +} + +#[tokio::test] +async fn test_strategy2_batch_receipts_by_hash_all_chains() { + init_tracing(); + + println!("\n{}", "=".repeat(80)); + println!("STRATEGY 2: Batch Receipts by Hash"); + println!("{}\n", "=".repeat(80)); + + for target in get_test_chains() { + info!("Testing {}...", target.name); + + let provider = match SemEvmRpcProvider::new(target.url.to_string(), 5) { + Ok(p) => p, + Err(e) => { + info!(" [SKIP] {}: Provider creation failed: {}", target.name, e); + continue; + } + }; + + let height = match provider.get_block_number().await { + Ok(h) => h, + Err(e) => { + info!(" [SKIP] {}: Block number failed: {}", target.name, e); + continue; + } + }; + + // First get the block to get its hash + let target_block_num = height.saturating_sub(10); + let block = match provider.get_block_by_number(target_block_num).await { + Ok(b) => b, + Err(e) => { + info!(" [SKIP] {}: Block fetch failed: {}", target.name, e); + continue; + } + }; + + let block_hash = block.header.hash; + let fetcher = + EvmBlockFetcher::new(provider).with_retry_interval(Duration::from_millis(100)); + + match fetcher + .fetch_block_with_batch_receipts_by_hash(block_hash) + .await + { + Ok(result) => { + info!( + " [OK] {}: Block {} ({} txs, {} receipts)", + target.name, + block_hash, + result.transaction_count(), + result.receipts.len() + ); + assert_eq!(result.transaction_count(), result.receipts.len()); + } + Err(e) => { + info!(" [FAIL] {}: {}", target.name, e); + } + } + } +} + +// NOTE: This test is commented out due to rate limiting from free RPC providers. +// Running multiple chunk sizes sequentially on high-tx blocks exhausts rate limits. +// Re-enable when using private/paid RPC nodes with higher rate limits. + +// #[tokio::test] +// async fn test_strategy3_chunked_batch_various_sizes() { +// init_tracing(); +// +// println!("\n{}", "=".repeat(80)); +// println!("STRATEGY 3: Chunked Batch Receipts (Various Sizes)"); +// println!("{}\n", "=".repeat(80)); +// +// let chunk_sizes = [5, 10, 50]; +// +// // Use Ethereum Mainnet for this test +// let target = RpcTarget { +// name: "Ethereum Mainnet", +// url: "https://ethereum-rpc.publicnode.com", +// }; +// +// let provider = match SemEvmRpcProvider::new(target.url.to_string(), 10) { +// Ok(p) => p, +// Err(e) => { +// info!(" [SKIP] Provider creation failed: {}", e); +// return; +// } +// }; +// +// let height = match provider.get_block_number().await { +// Ok(h) => h, +// Err(e) => { +// info!(" [SKIP] Block number failed: {}", e); +// return; +// } +// }; +// +// let target_block = height.saturating_sub(10); +// +// for chunk_size in chunk_sizes { +// info!("Testing chunk_size={}...", chunk_size); +// +// let fetcher = EvmBlockFetcher::new(provider.clone()) +// .with_retry_interval(Duration::from_millis(100)); +// +// match fetcher +// .fetch_block_by_number_with_parallel_batched_receipts(target_block, chunk_size) +// .await +// { +// Ok(result) => { +// info!( +// " [OK] chunk_size={}: Block #{} ({} txs, {} receipts)", +// chunk_size, +// target_block, +// result.transaction_count(), +// result.receipts.len() +// ); +// assert_eq!(result.transaction_count(), result.receipts.len()); +// } +// Err(e) => { +// info!(" [FAIL] chunk_size={}: {}", chunk_size, e); +// } +// } +// } +// } + +#[tokio::test] +async fn test_strategy3_chunked_batch_by_hash() { + init_tracing(); + + println!("\n{}", "=".repeat(80)); + println!("STRATEGY 3: Chunked Batch Receipts by Hash"); + println!("{}\n", "=".repeat(80)); + + let target = RpcTarget { + name: "Ethereum Mainnet", + url: "https://ethereum-rpc.publicnode.com", + }; + + let provider = match SemEvmRpcProvider::new(target.url.to_string(), 10) { + Ok(p) => p, + Err(e) => { + info!(" [SKIP] Provider creation failed: {}", e); + return; + } + }; + + let height = match provider.get_block_number().await { + Ok(h) => h, + Err(e) => { + info!(" [SKIP] Block number failed: {}", e); + return; + } + }; + + let target_block_num = height.saturating_sub(10); + let block = match provider.get_block_by_number(target_block_num).await { + Ok(b) => b, + Err(e) => { + info!(" [SKIP] Block fetch failed: {}", e); + return; + } + }; + + let block_hash = block.header.hash; + let fetcher = EvmBlockFetcher::new(provider).with_retry_interval(Duration::from_millis(100)); + + match fetcher + .fetch_block_by_hash_with_parallel_batched_receipts(block_hash, 10) + .await + { + Ok(result) => { + info!( + " [OK] Block {} ({} txs, {} receipts)", + block_hash, + result.transaction_count(), + result.receipts.len() + ); + assert_eq!(result.transaction_count(), result.receipts.len()); + } + Err(e) => { + info!(" [FAIL] {}", e); + } + } +} + +// NOTE: Strategy 4 tests are commented out due to rate limiting from free RPC providers. +// These tests spawn many parallel requests which quickly exhaust rate limits. +// Re-enable when using private/paid RPC nodes with higher rate limits. + +// #[tokio::test] +// async fn test_strategy4_individual_receipts_by_number_all_chains() { +// init_tracing(); +// +// println!("\n{}", "=".repeat(80)); +// println!("STRATEGY 4: Individual Receipts by Number"); +// println!("{}\n", "=".repeat(80)); +// +// for target in get_test_chains() { +// info!("Testing {}...", target.name); +// +// let provider = match SemEvmRpcProvider::new(target.url.to_string(), 20) { +// Ok(p) => p, +// Err(e) => { +// info!(" [SKIP] {}: Provider creation failed: {}", target.name, e); +// continue; +// } +// }; +// +// let height = match provider.get_block_number().await { +// Ok(h) => h, +// Err(e) => { +// info!(" [SKIP] {}: Block number failed: {}", target.name, e); +// continue; +// } +// }; +// +// let fetcher = EvmBlockFetcher::new(provider) +// .with_retry_interval(Duration::from_millis(100)); +// +// let target_block = height.saturating_sub(10); +// +// match fetcher.fetch_block_with_individual_receipts_by_number(target_block).await { +// Ok(result) => { +// info!( +// " [OK] {}: Block #{} ({} txs, {} receipts)", +// target.name, +// target_block, +// result.transaction_count(), +// result.receipts.len() +// ); +// assert_eq!(result.transaction_count(), result.receipts.len()); +// } +// Err(e) => { +// info!(" [FAIL] {}: {}", target.name, e); +// } +// } +// } +// } + +// #[tokio::test] +// async fn test_strategy4_individual_receipts_by_hash_all_chains() { +// init_tracing(); +// +// println!("\n{}", "=".repeat(80)); +// println!("STRATEGY 4: Individual Receipts by Hash"); +// println!("{}\n", "=".repeat(80)); +// +// for target in get_test_chains() { +// info!("Testing {}...", target.name); +// +// let provider = match SemEvmRpcProvider::new(target.url.to_string(), 20) { +// Ok(p) => p, +// Err(e) => { +// info!(" [SKIP] {}: Provider creation failed: {}", target.name, e); +// continue; +// } +// }; +// +// let height = match provider.get_block_number().await { +// Ok(h) => h, +// Err(e) => { +// info!(" [SKIP] {}: Block number failed: {}", target.name, e); +// continue; +// } +// }; +// +// // First get the block to get its hash +// let target_block_num = height.saturating_sub(10); +// let block = match provider.get_block_by_number(target_block_num).await { +// Ok(b) => b, +// Err(e) => { +// info!(" [SKIP] {}: Block fetch failed: {}", target.name, e); +// continue; +// } +// }; +// +// let block_hash = block.header.hash; +// let fetcher = EvmBlockFetcher::new(provider) +// .with_retry_interval(Duration::from_millis(100)); +// +// match fetcher.fetch_block_with_individual_receipts_by_hash(block_hash).await { +// Ok(result) => { +// info!( +// " [OK] {}: Block {} ({} txs, {} receipts)", +// target.name, +// block_hash, +// result.transaction_count(), +// result.receipts.len() +// ); +// assert_eq!(result.transaction_count(), result.receipts.len()); +// } +// Err(e) => { +// info!(" [FAIL] {}: {}", target.name, e); +// } +// } +// } +// } + +#[tokio::test] +async fn test_strategy5_sequential_receipts_by_number() { + init_tracing(); + + println!("\n{}", "=".repeat(80)); + println!("STRATEGY 5: Sequential Receipts by Number"); + println!("{}\n", "=".repeat(80)); + + // Test on a single chain with a low-tx block to avoid long test times + let target = RpcTarget { + name: "Ethereum Mainnet", + url: "https://ethereum-rpc.publicnode.com", + }; + + info!("Testing {}...", target.name); + + let provider = match SemEvmRpcProvider::new(target.url.to_string(), 5) { + Ok(p) => p, + Err(e) => { + info!(" [SKIP] {}: Provider creation failed: {}", target.name, e); + return; + } + }; + + // Use block 1 which has 0 transactions (fast test) + let fetcher = EvmBlockFetcher::new(provider).with_retry_interval(Duration::from_millis(100)); + + match fetcher + .fetch_block_with_sequential_receipts_by_number(1) + .await + { + Ok(result) => { + info!( + " [OK] {}: Block #1 ({} txs, {} receipts)", + target.name, + result.transaction_count(), + result.receipts.len() + ); + assert_eq!(result.transaction_count(), result.receipts.len()); + } + Err(e) => { + info!(" [FAIL] {}: {}", target.name, e); + } + } +} + +#[tokio::test] +async fn test_strategy5_sequential_receipts_by_hash() { + init_tracing(); + + println!("\n{}", "=".repeat(80)); + println!("STRATEGY 5: Sequential Receipts by Hash"); + println!("{}\n", "=".repeat(80)); + + let target = RpcTarget { + name: "Ethereum Mainnet", + url: "https://ethereum-rpc.publicnode.com", + }; + + info!("Testing {}...", target.name); + + let provider = match SemEvmRpcProvider::new(target.url.to_string(), 5) { + Ok(p) => p, + Err(e) => { + info!(" [SKIP] {}: Provider creation failed: {}", target.name, e); + return; + } + }; + + // Get block 1's hash + let block = match provider.get_block_by_number(1).await { + Ok(b) => b, + Err(e) => { + info!(" [SKIP] {}: Block fetch failed: {}", target.name, e); + return; + } + }; + + let block_hash = block.header.hash; + let fetcher = EvmBlockFetcher::new(provider).with_retry_interval(Duration::from_millis(100)); + + match fetcher + .fetch_block_with_sequential_receipts_by_hash(block_hash) + .await + { + Ok(result) => { + info!( + " [OK] {}: Block {} ({} txs, {} receipts)", + target.name, + block_hash, + result.transaction_count(), + result.receipts.len() + ); + assert_eq!(result.transaction_count(), result.receipts.len()); + assert_eq!(result.block.header.hash, block_hash); + } + Err(e) => { + info!(" [FAIL] {}: {}", target.name, e); + } + } +} + +#[tokio::test] +async fn test_strategy5_sequential_with_transactions() { + init_tracing(); + + println!("\n{}", "=".repeat(80)); + println!("STRATEGY 5: Sequential Receipts with Transactions"); + println!("{}\n", "=".repeat(80)); + + // Test on Zama Gateway Testnet which has fewer transactions and is less rate-limited + let target = RpcTarget { + name: "Zama Gateway Testnet", + url: "https://rpc-zama-testnet-0.t.conduit.xyz", + }; + + info!("Testing {}...", target.name); + + let provider = match SemEvmRpcProvider::new(target.url.to_string(), 5) { + Ok(p) => p, + Err(e) => { + info!(" [SKIP] {}: Provider creation failed: {}", target.name, e); + return; + } + }; + + let height = match provider.get_block_number().await { + Ok(h) => h, + Err(e) => { + info!(" [SKIP] {}: Block number failed: {}", target.name, e); + return; + } + }; + + let fetcher = EvmBlockFetcher::new(provider).with_retry_interval(Duration::from_millis(100)); + + let target_block = height.saturating_sub(10); + + match fetcher + .fetch_block_with_sequential_receipts_by_number(target_block) + .await + { + Ok(result) => { + info!( + " [OK] {}: Block #{} ({} txs, {} receipts)", + target.name, + target_block, + result.transaction_count(), + result.receipts.len() + ); + assert_eq!(result.transaction_count(), result.receipts.len()); + } + Err(e) => { + info!(" [FAIL] {}: {}", target.name, e); + } + } +} + +#[tokio::test] +async fn test_cancellation_immediate() { + init_tracing(); + + println!("\n{}", "=".repeat(80)); + println!("CANCELLATION: Immediate Cancel"); + println!("{}\n", "=".repeat(80)); + + let target = RpcTarget { + name: "Ethereum Mainnet", + url: "https://ethereum-rpc.publicnode.com", + }; + + let provider = SemEvmRpcProvider::new(target.url.to_string(), 5).expect("Provider creation"); + + let cancel_token = CancellationToken::new(); + + // Cancel immediately before starting + cancel_token.cancel(); + + let fetcher = EvmBlockFetcher::new(provider) + .with_cancellation_token(cancel_token) + .with_retry_interval(Duration::from_millis(100)); + + let result = fetcher + .fetch_block_with_block_receipts_by_number(12345) + .await; + + match result { + Err(BlockFetchError::Cancelled) => { + info!(" [OK] Immediate cancellation worked correctly"); + } + Ok(_) => { + panic!("Expected Cancelled error, got success"); + } + Err(e) => { + panic!("Expected Cancelled error, got: {}", e); + } + } +} + +#[tokio::test] +async fn test_cancellation_during_fetch() { + init_tracing(); + + println!("\n{}", "=".repeat(80)); + println!("CANCELLATION: During Fetch"); + println!("{}\n", "=".repeat(80)); + + let target = RpcTarget { + name: "Ethereum Mainnet", + url: "https://ethereum-rpc.publicnode.com", + }; + + let provider = SemEvmRpcProvider::new(target.url.to_string(), 5).expect("Provider creation"); + + let cancel_token = CancellationToken::new(); + let cancel_token_clone = cancel_token.clone(); + + let fetcher = EvmBlockFetcher::new(provider) + .with_cancellation_token(cancel_token) + .with_retry_interval(Duration::from_millis(100)); + + // Spawn task to cancel after a short delay + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(50)).await; + cancel_token_clone.cancel(); + }); + + // Try to fetch a block - should be cancelled + let result = fetcher + .fetch_block_with_individual_receipts_by_number(1) + .await; + + // Result could be success (if fast enough) or cancelled + match result { + Ok(_) => { + info!(" [OK] Fetch completed before cancellation (acceptable)"); + } + Err(BlockFetchError::Cancelled) => { + info!(" [OK] Cancellation during fetch worked correctly"); + } + Err(e) => { + info!(" [WARN] Unexpected error: {}", e); + } + } +} + +// NOTE: This test is commented out due to rate limiting from free RPC providers. +// Uses individual receipts strategy which spawns many parallel requests. +// Re-enable when using private/paid RPC nodes with higher rate limits. + +// #[tokio::test] +// async fn test_cancellation_cleans_up_all_tasks() { +// init_tracing(); +// +// println!("\n{}", "=".repeat(80)); +// println!("CANCELLATION: Task Cleanup Verification"); +// println!("{}\n", "=".repeat(80)); +// +// let target = RpcTarget { +// name: "Ethereum Mainnet", +// url: "https://ethereum-rpc.publicnode.com", +// }; +// +// let provider = SemEvmRpcProvider::new(target.url.to_string(), 5).expect("Provider creation"); +// +// let height = provider.get_block_number().await.expect("Block number"); +// let target_block = height.saturating_sub(10); +// +// // First fetch the block to get transaction count +// let block = provider +// .get_block_by_number(target_block) +// .await +// .expect("Block fetch"); +// let tx_count = block.transactions.len(); +// +// info!("Block #{} has {} transactions", target_block, tx_count); +// +// if tx_count == 0 { +// info!(" [SKIP] Block has no transactions, cannot test task cleanup"); +// return; +// } +// +// let cancel_token = CancellationToken::new(); +// let cancel_token_clone = cancel_token.clone(); +// +// let fetcher = EvmBlockFetcher::new(provider) +// .with_cancellation_token(cancel_token) +// .with_retry_interval(Duration::from_millis(100)); +// +// // Spawn task to cancel quickly - this should interrupt the individual receipt fetches +// tokio::spawn(async move { +// tokio::time::sleep(Duration::from_millis(10)).await; +// cancel_token_clone.cancel(); +// }); +// +// // Use individual receipts strategy which spawns many tasks +// let result = fetcher.fetch_block_with_individual_receipts_by_number(target_block).await; +// +// match result { +// Ok(_) => { +// info!(" [OK] Fetch completed before cancellation"); +// } +// Err(BlockFetchError::Cancelled) => { +// info!(" [OK] Cancellation occurred and tasks were cleaned up"); +// } +// Err(e) => { +// info!(" [WARN] Unexpected error: {}", e); +// } +// } +// +// // Give time for any leaked tasks to complete/fail +// tokio::time::sleep(Duration::from_millis(100)).await; +// info!(" [OK] No task leaks detected (test completed without hanging)"); +// } + +#[tokio::test] +async fn test_verification_enabled() { + init_tracing(); + + println!("\n{}", "=".repeat(80)); + println!("VERIFICATION: Block Verification Enabled"); + println!("{}\n", "=".repeat(80)); + + // Test on chains known to pass verification + let verification_chains = vec![ + RpcTarget { + name: "Ethereum Mainnet", + url: "https://ethereum-rpc.publicnode.com", + }, + RpcTarget { + name: "Optimism", + url: "https://mainnet.optimism.io", + }, + RpcTarget { + name: "Base", + url: "https://mainnet.base.org", + }, + ]; + + for target in verification_chains { + info!("Testing {}...", target.name); + + let provider = match SemEvmRpcProvider::new(target.url.to_string(), 5) { + Ok(p) => p, + Err(e) => { + info!(" [SKIP] {}: Provider creation failed: {}", target.name, e); + continue; + } + }; + + let height = match provider.get_block_number().await { + Ok(h) => h, + Err(e) => { + info!(" [SKIP] {}: Block number failed: {}", target.name, e); + continue; + } + }; + + let fetcher = EvmBlockFetcher::new(provider) + .with_verify_block(true) + .with_retry_interval(Duration::from_millis(100)); + + let target_block = height.saturating_sub(10); + + // Use block receipts (eth_getBlockReceipts) to get all receipts + match fetcher + .fetch_block_with_block_receipts_by_number(target_block) + .await + { + Ok(result) => { + info!( + " [OK] {}: Block #{} verified ({} txs)", + target.name, + target_block, + result.transaction_count() + ); + } + Err(BlockFetchError::VerificationFailed(e)) => { + info!(" [FAIL] {}: Verification failed: {}", target.name, e); + } + Err(e) => { + info!(" [FAIL] {}: {}", target.name, e); + } + } + } +} + +#[tokio::test] +async fn test_retry_eventually_succeeds() { + init_tracing(); + + println!("\n{}", "=".repeat(80)); + println!("RETRY: Eventually Succeeds"); + println!("{}\n", "=".repeat(80)); + + // This test verifies that the retry mechanism works by using a real RPC + // that should succeed on the first try + + let target = RpcTarget { + name: "Ethereum Mainnet", + url: "https://ethereum-rpc.publicnode.com", + }; + + let provider = SemEvmRpcProvider::new(target.url.to_string(), 5).expect("Provider creation"); + + let fetcher = EvmBlockFetcher::new(provider).with_retry_interval(Duration::from_millis(100)); + + // Fetch a known block + let result = fetcher.fetch_block_with_block_receipts_by_number(1).await; + + match result { + Ok(result) => { + info!( + " [OK] Block #1 fetched ({} txs)", + result.transaction_count() + ); + // Block 1 should have 0 transactions (genesis has no txs) + assert!( + result.transaction_count() <= 1, + "Block 1 should have 0-1 transactions" + ); + } + Err(e) => { + info!(" [FAIL] {}", e); + } + } +} + +#[tokio::test] +#[ignore] // Long-running test +async fn test_retry_with_timeout() { + init_tracing(); + + println!("\n{}", "=".repeat(80)); + println!("RETRY: With Timeout (ignored by default)"); + println!("{}\n", "=".repeat(80)); + + // This test verifies retry behavior with a short timeout + // Uses a non-existent RPC to ensure retries happen + + let provider = SemEvmRpcProvider::new( + "http://localhost:9999".to_string(), // Non-existent RPC + 5, + ) + .expect("Provider creation"); + + let cancel_token = CancellationToken::new(); + let cancel_token_clone = cancel_token.clone(); + + // Cancel after 2 seconds + tokio::spawn(async move { + tokio::time::sleep(Duration::from_secs(2)).await; + cancel_token_clone.cancel(); + }); + + let fetcher = EvmBlockFetcher::new(provider) + .with_cancellation_token(cancel_token) + .with_retry_interval(Duration::from_millis(200)); + + let result = fetcher.fetch_block_with_batch_receipts_by_number(1).await; + + match result { + Err(BlockFetchError::Cancelled) => { + info!(" [OK] Retry was cancelled after timeout"); + } + Ok(_) => { + panic!("Expected cancellation, got success"); + } + Err(e) => { + info!(" [WARN] Got error: {}", e); + } + } +} + +#[tokio::test] +async fn test_fetched_block_methods() { + init_tracing(); + + println!("\n{}", "=".repeat(80)); + println!("FETCHED_BLOCK: Helper Methods"); + println!("{}\n", "=".repeat(80)); + + let target = RpcTarget { + name: "Ethereum Mainnet", + url: "https://ethereum-rpc.publicnode.com", + }; + + let provider = SemEvmRpcProvider::new(target.url.to_string(), 5).expect("Provider creation"); + + let height = provider.get_block_number().await.expect("Block number"); + let target_block = height.saturating_sub(10); + + let fetcher = EvmBlockFetcher::new(provider).with_retry_interval(Duration::from_millis(100)); + + let result = fetcher + .fetch_block_with_block_receipts_by_number(target_block) + .await + .expect("Fetch should succeed"); + + // Test transaction_count() + let tx_count = result.transaction_count(); + info!(" transaction_count(): {}", tx_count); + + // Test receipts_ordered() + let ordered = result.receipts_ordered(); + assert_eq!(ordered.len(), tx_count); + info!(" receipts_ordered(): {} receipts", ordered.len()); + + // Test get_receipt() for each transaction + let tx_hashes: Vec = result.block.transactions.hashes().collect(); + for hash in &tx_hashes { + let receipt = result.get_receipt(hash); + assert!(receipt.is_some(), "Receipt should exist for tx {}", hash); + } + info!( + " get_receipt(): All {} receipts accessible", + tx_hashes.len() + ); + + // Test fetch_id is valid UUID + assert!(!result.fetch_id.is_nil()); + info!(" fetch_id: {}", result.fetch_id); + + info!(" [OK] All FetchedBlock methods work correctly"); +} + +#[tokio::test] +async fn test_empty_block() { + init_tracing(); + + println!("\n{}", "=".repeat(80)); + println!("EDGE CASE: Empty Block (No Transactions)"); + println!("{}\n", "=".repeat(80)); + + // Block 1 on Ethereum has 0 transactions + let target = RpcTarget { + name: "Ethereum Mainnet", + url: "https://ethereum-rpc.publicnode.com", + }; + + let provider = SemEvmRpcProvider::new(target.url.to_string(), 5).expect("Provider creation"); + + let fetcher = EvmBlockFetcher::new(provider).with_retry_interval(Duration::from_millis(100)); + + // Fetch block 1 which has no transactions + let result = fetcher.fetch_block_with_batch_receipts_by_number(1).await; + + match result { + Ok(result) => { + info!( + " [OK] Block #1: {} transactions, {} receipts", + result.transaction_count(), + result.receipts.len() + ); + assert_eq!(result.transaction_count(), result.receipts.len()); + } + Err(e) => { + info!(" [FAIL] {}", e); + } + } +} + +// NOTE: This test is commented out due to rate limiting from free RPC providers. +// Running all strategies on the same block with many transactions exhausts rate limits. +// Re-enable when using private/paid RPC nodes with higher rate limits. + +// #[tokio::test] +// async fn test_all_strategies_same_result() { +// init_tracing(); +// +// println!("\n{}", "=".repeat(80)); +// println!("COMPARISON: All Strategies Return Same Data"); +// println!("{}\n", "=".repeat(80)); +// +// let target = RpcTarget { +// name: "Ethereum Mainnet", +// url: "https://ethereum-rpc.publicnode.com", +// }; +// +// let provider = SemEvmRpcProvider::new(target.url.to_string(), 20).expect("Provider creation"); +// +// let height = provider.get_block_number().await.expect("Block number"); +// let target_block = height.saturating_sub(10); +// +// info!("Fetching block #{} with all 5 strategies...", target_block); +// +// // Strategy 1: Block Receipts +// let fetcher1 = EvmBlockFetcher::new(provider.clone()) +// .with_retry_interval(Duration::from_millis(100)); +// let result1 = fetcher1 +// .fetch_block_with_block_receipts_by_number(target_block) +// .await; +// +// // Strategy 2: Batch Receipts +// let fetcher2 = EvmBlockFetcher::new(provider.clone()) +// .with_retry_interval(Duration::from_millis(100)); +// let result2 = fetcher2 +// .fetch_block_with_batch_receipts_by_number(target_block) +// .await; +// +// // Strategy 3: Chunked Batch +// let fetcher3 = EvmBlockFetcher::new(provider.clone()) +// .with_retry_interval(Duration::from_millis(100)); +// let result3 = fetcher3 +// .fetch_block_by_number_with_parallel_batched_receipts(target_block, 10) +// .await; +// +// // Strategy 4: Individual +// let fetcher4 = EvmBlockFetcher::new(provider.clone()) +// .with_retry_interval(Duration::from_millis(100)); +// let result4 = fetcher4 +// .fetch_block_with_individual_receipts_by_number(target_block) +// .await; +// +// // Strategy 5: Sequential +// let fetcher5 = EvmBlockFetcher::new(provider) +// .with_retry_interval(Duration::from_millis(100)); +// let result5 = fetcher5 +// .fetch_block_with_sequential_receipts_by_number(target_block) +// .await; +// +// // Compare results +// let results = vec![ +// ("Strategy 1 (BlockReceipts)", result1), +// ("Strategy 2 (Batch)", result2), +// ("Strategy 3 (Chunked)", result3), +// ("Strategy 4 (Individual)", result4), +// ("Strategy 5 (Sequential)", result5), +// ]; +// +// let mut successful_results: Vec<(&str, FetchedBlock)> = vec![]; +// +// for (name, result) in results { +// match result { +// Ok(r) => { +// info!(" [OK] {}: {} txs", name, r.transaction_count()); +// successful_results.push((name, r)); +// } +// Err(e) => { +// info!(" [FAIL] {}: {}", name, e); +// } +// } +// } +// +// // Verify all successful results have the same data +// if successful_results.len() >= 2 { +// let (first_name, first) = &successful_results[0]; +// let first_tx_count = first.transaction_count(); +// let first_block_hash = first.block.header.hash; +// +// for (name, result) in &successful_results[1..] { +// assert_eq!( +// result.transaction_count(), +// first_tx_count, +// "{} tx count differs from {}", +// name, +// first_name +// ); +// assert_eq!( +// result.block.header.hash, first_block_hash, +// "{} block hash differs from {}", +// name, first_name +// ); +// +// // Verify all receipts match +// for (hash, receipt) in &first.receipts { +// let other_receipt = result.get_receipt(hash); +// assert!( +// other_receipt.is_some(), +// "{} missing receipt for {}", +// name, +// hash +// ); +// assert_eq!( +// other_receipt.unwrap().transaction_hash, +// receipt.transaction_hash, +// "{} receipt hash mismatch", +// name +// ); +// } +// } +// +// info!( +// " [OK] All {} successful strategies returned identical data", +// successful_results.len() +// ); +// } else { +// info!( +// " [WARN] Only {} strategies succeeded, cannot compare", +// successful_results.len() +// ); +// } +// } + +// ============================================================ +// classify_error unit tests +// ============================================================ + +#[test] +fn test_classify_error_rate_limited() { + let err = RpcProviderError::RateLimited("429 Too Many Requests".into()); + assert!(matches!( + classify_error(&err), + ErrorClassification::RateLimited + )); +} + +#[test] +fn test_classify_error_transport_is_recoverable() { + let err = RpcProviderError::TransportError("connection timeout".into()); + assert!(matches!( + classify_error(&err), + ErrorClassification::Recoverable + )); +} + +#[test] +fn test_classify_error_unsupported_is_unrecoverable() { + let err = RpcProviderError::UnsupportedMethod { + method: "eth_getBlockReceipts".into(), + details: "method does not exist".into(), + }; + assert!(matches!( + classify_error(&err), + ErrorClassification::Unrecoverable(BlockFetchError::UnsupportedMethod { .. }) + )); +} + +#[test] +fn test_classify_error_batch_unsupported_is_unrecoverable() { + let err = RpcProviderError::BatchUnsupported; + assert!(matches!( + classify_error(&err), + ErrorClassification::Unrecoverable(BlockFetchError::BatchUnsupported) + )); +} + +#[test] +fn test_classify_error_deserialization_is_unrecoverable() { + let err = RpcProviderError::DeserializationError { + method: "eth_getBlockByNumber".into(), + details: "data did not match".into(), + }; + assert!(matches!( + classify_error(&err), + ErrorClassification::Unrecoverable(BlockFetchError::DeserializationError { .. }) + )); +} + +#[test] +fn test_classify_error_batch_error_is_recoverable() { + let err = RpcProviderError::BatchError("HTTP request failed".into()); + assert!(matches!( + classify_error(&err), + ErrorClassification::Recoverable + )); +} + +#[test] +fn test_classify_error_not_found_is_recoverable() { + let err = RpcProviderError::NotFound; + assert!(matches!( + classify_error(&err), + ErrorClassification::Recoverable + )); +} + +#[test] +fn test_classify_error_semaphore_closed_is_recoverable() { + let err = RpcProviderError::SemaphoreClosed; + assert!(matches!( + classify_error(&err), + ErrorClassification::Recoverable + )); +} diff --git a/listener/crates/listener_core/src/blockchain/evm/tests/sem_evm_rpc_provider_tests.rs b/listener/crates/listener_core/src/blockchain/evm/tests/sem_evm_rpc_provider_tests.rs new file mode 100644 index 0000000000..ce95581e7c --- /dev/null +++ b/listener/crates/listener_core/src/blockchain/evm/tests/sem_evm_rpc_provider_tests.rs @@ -0,0 +1,253 @@ +#[cfg(test)] +mod sem_evm_rpc_provider_tests { + use tracing::{Level, info}; + use tracing_subscriber::FmtSubscriber; + + use crate::blockchain::evm::sem_evm_rpc_provider::{SemEvmRpcProvider, extract_tx_hashes}; + + struct RpcTarget { + name: &'static str, + url: &'static str, + } + + #[tokio::test] + async fn report_provider_workability() { + // 1. Force initialize tracing to stay informed during async calls + let subscriber = FmtSubscriber::builder() + .with_max_level(Level::INFO) + .finish(); + let _ = tracing::subscriber::set_global_default(subscriber); + + // 2. Hardcoded List + let targets = vec![ + RpcTarget { + name: "Ethereum Mainnet", + url: "https://ethereum-rpc.publicnode.com", + }, + // RpcTarget { + // name: "Binance Smart Chain Mainnet", + // url: "https://bsc-dataseed.bnbchain.org", + // }, + // RpcTarget { + // name: "Polygon PoS", + // // url: "https://polygon-rpc.com", + // url: "https://polygon.drpc.org", + // }, + // RpcTarget { + // name: "Avalanche", + // url: "https://api.avax.network/ext/bc/C/rpc", + // }, + // RpcTarget { + // name: "Monad", + // url: "https://rpc.monad.xyz/", + // }, + // RpcTarget { + // name: "Optimism", + // url: "https://mainnet.optimism.io", + // }, + // RpcTarget { + // name: "Arbitrum One", + // url: "https://arb1.arbitrum.io/rpc", + // }, + // RpcTarget { + // name: "Base", + // url: "https://mainnet.base.org", + // }, + // RpcTarget { + // name: "Zama Gateway Testnet", + // url: "https://rpc-zama-testnet-0.t.conduit.xyz", + // }, + // RpcTarget { + // name: "Polkadot mainnet", + // url: "https://eth-rpc.polkadot.io/", + // }, + // RpcTarget { + // name: "Polkadot testnet", + // url: "https://services.polkadothub-rpc.com/testnet", + // }, + ]; + + println!("\n\n{}", "=".repeat(80)); + println!("🚀 STARTING RPC WORKABILITY REPORT"); + println!("{}\n", "=".repeat(80)); + + for target in targets { + info!("Probing {}...", target.name); + + let report = probe_chain(target).await; + + // PRINT IMMEDIATELY + print_report_entry(report); + } + + println!("{}", "=".repeat(80)); + println!("✅ REPORT COMPLETE"); + println!("{}\n", "=".repeat(80)); + } + + async fn probe_chain(target: RpcTarget) -> ChainReport { + let provider = match SemEvmRpcProvider::new(target.url.to_string(), 5) { + Ok(p) => p, + Err(e) => return ChainReport::fatal(target, format!("Creation failed: {}", e)), + }; + + let chain_id = provider.get_chain_id().await.map_err(|e| e.to_string()); + + let height = match provider.get_block_number().await { + Ok(h) => h, + Err(e) => return ChainReport::fatal(target, format!("Height call failed: {}", e)), + }; + + // Get block by number (Verify Header Deserialization) + let block_res = provider.get_block_by_number(height - 10).await; + + // NEW: Get block by hash (Verify Hash-based lookup and Deserialization) + let mut block_hash_info = Err("No Hash available".to_string()); + if let Ok(ref block) = block_res { + let hash = block.header.hash; + block_hash_info = provider + .get_block_by_hash(hash.to_string()) + .await + .map(|b| b.header.hash == hash) // Verify we got the same block back + .map_err(|e| e.to_string()); + } + + // Get receipt (Verify Log Deserialization) + let mut nb_of_logs_for_tx_1 = Err("No Tx".to_string()); + if let Ok(ref block) = block_res { + // info!( + // "Transaction hashes {:?}", + // block.transactions.hashes().collect::>() + // ); + if let Some(tx) = block.transactions.hashes().next() { + // info!("Tested tx hash: {}", tx.clone().to_string()); + let receipt_info = provider + .get_transaction_receipt(tx.to_string()) + .await + .map_err(|e| e.to_string()); + // info!( + // "Receipt for hash {}: {:?}", + // tx, + // receipt_info.clone().unwrap() + // ); + + let log = receipt_info + .map(|r| r.logs().len()) + .map_err(|e| e.to_string()); + // info!("Number of logs for hash {}: {}", tx, log.clone().unwrap()); + nb_of_logs_for_tx_1 = log; + } + } + + // Get Block Receipts (Efficiency check) + let block_receipts_info = provider + .get_block_receipts(height - 10) + .await + .map(|r| r.len()) + .map_err(|e| e.to_string()); + + // Test batch receipts - need to re-fetch block for tx hashes + let batch_receipts_info = match provider.get_block_by_number(height - 10).await { + Ok(block) => { + let tx_hashes = extract_tx_hashes(&block); + if tx_hashes.is_empty() { + Ok(0) + } else { + // Only test with first 3 transactions to avoid overwhelming the node + // let len = tx_hashes.len(); + // let test_hashes: Vec<_> = + // tx_hashes.into_iter().take(len).collect(); + let test_hashes: Vec<_> = tx_hashes.into_iter().take(3).collect(); + provider + .get_transaction_receipts_batch(&test_hashes) + .await + .map(|r| r.len()) + .map_err(|e| e.to_string()) + } + } + Err(e) => Err(format!("Block fetch failed: {}", e)), + }; + + ChainReport { + name: target.name.to_string(), + url: target.url.to_string(), + chain_id, + block_number: Ok(height), + get_block: block_res + .map(|b| b.transactions.len()) + .map_err(|e| e.to_string()), + get_block_hash: block_hash_info, + get_receipt: nb_of_logs_for_tx_1, + block_receipts: block_receipts_info, + batch_receipts: batch_receipts_info, + } + } + + struct ChainReport { + name: String, + url: String, + chain_id: Result, + block_number: Result, + get_block: Result, + get_block_hash: Result, + get_receipt: Result, + block_receipts: Result, + batch_receipts: Result, + } + + impl ChainReport { + fn fatal(target: RpcTarget, reason: String) -> Self { + Self { + name: target.name.to_string(), + url: target.url.to_string(), + chain_id: Err("Skipped".to_string()), + block_number: Err(reason), + get_block: Err("Skipped".to_string()), + get_block_hash: Err("Skipped".to_string()), + get_receipt: Err("Skipped".to_string()), + block_receipts: Err("Skipped".to_string()), + batch_receipts: Err("Skipped".to_string()), + } + } + } + + fn print_report_entry(r: ChainReport) { + info!("CHAIN: {}", r.name); + info!("URL: {}", r.url); + + let fmt_res = |res: &Result, label: &str| match res { + Ok(val) => info!(" [OK] {:<25} Count: {}", label, val), + Err(e) => info!(" [FAIL] {:<25} Error: {}", label, e), + }; + + let fmt_res_for_log = |res: &Result, label: &str| match res { + Ok(val) => info!(" [OK] {:<25} Log Count for tx 1: {}", label, val), + Err(e) => info!(" [FAIL] {:<25} Error: {}", label, e), + }; + + // Custom formatter for the boolean hash check + let fmt_bool = |res: &Result, label: &str| match res { + Ok(true) => info!(" [OK] {:<25} Verified", label), + Ok(false) => info!(" [FAIL] {:<25} Hash Mismatch", label), + Err(e) => info!(" [FAIL] {:<25} Error: {}", label, e), + }; + + match r.chain_id { + Ok(id) => info!(" [OK] eth_chainId Chain ID: {}", id), + Err(e) => info!(" [FAIL] eth_chainId Error: {}", e), + } + + match r.block_number { + Ok(h) => info!(" [OK] eth_blockNumber Height: {}", h), + Err(e) => info!(" [FAIL] eth_blockNumber Error: {}", e), + } + + fmt_res(&r.get_block, "eth_getBlockByNumber"); + fmt_bool(&r.get_block_hash, "eth_getBlockByHash"); + fmt_res_for_log(&r.get_receipt, "eth_getTransactionReceipt"); + fmt_res(&r.block_receipts, "eth_getBlockReceipts"); + fmt_res(&r.batch_receipts, "batch receipts"); + + info!("{}\n", "-".repeat(40)); + } +} diff --git a/listener/crates/listener_core/src/blockchain/mod.rs b/listener/crates/listener_core/src/blockchain/mod.rs new file mode 100644 index 0000000000..c469d0c8ed --- /dev/null +++ b/listener/crates/listener_core/src/blockchain/mod.rs @@ -0,0 +1 @@ +pub mod evm; diff --git a/listener/crates/listener_core/src/config/config.rs b/listener/crates/listener_core/src/config/config.rs new file mode 100644 index 0000000000..5142a36966 --- /dev/null +++ b/listener/crates/listener_core/src/config/config.rs @@ -0,0 +1,1124 @@ +//! Configuration module for listener_core. +//! +//! Loads settings from YAML files and/or environment variables. +//! Environment variables override file values using format: APP_SECTION__FIELD +//! +//! # Example +//! ```bash +//! APP_DATABASE__DB_URL="postgres://..." cargo run +//! ``` +//! +//! # Broker Configuration +//! +//! The broker section supports both AMQP (RabbitMQ) and Redis Streams backends. +//! Set `broker_type` to switch between them: +//! +//! ```yaml +//! broker: +//! broker_type: redis # or 'amqp' +//! broker_url: redis://localhost:6379 +//! ``` + +use config::{Config, Environment, File}; +use derivative::Derivative; +use serde::de::{self, Visitor}; +use serde::{Deserialize, Deserializer}; +use std::fmt; +use thiserror::Error; + +const MIN_RANGE_SIZE: usize = 1; +const MAX_RANGE_SIZE: usize = 10000; +const MIN_PARALLEL_REQUESTS: usize = 1; +const MAX_PARALLEL_REQUESTS: usize = 200; +const MIN_BATCH_RANGE: usize = 1; + +/// 1 advisory-lock session + 1 concurrent query at least. +const MIN_POOL_MIN_CONNECTIONS: u32 = 2; +/// 1 advisory-lock + 4 concurrent handler queries (fetch/reorg, watch, unwatch, cleaner). +const MIN_POOL_MAX_CONNECTIONS: u32 = 5; +const MAX_BATCH_RANGE: usize = 100; + +#[derive(Error, Debug)] +pub enum ConfigError { + #[error("Configuration error: {0}")] + Parse(String), + #[error("Validation error: {0}")] + Validation(String), +} + +fn redact(_: &T, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "[REDACTED]") +} + +/// Broker backend type. +/// +/// Determines which message broker to use for publishing and consuming events. +#[derive(Debug, Deserialize, Clone, Default, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum BrokerType { + /// RabbitMQ/AMQP backend + Amqp, + #[default] + /// Redis Streams backend (default) + Redis, +} + +#[derive(Debug, Deserialize, Clone, Default, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum BlockFetcherStrategy { + #[default] + BlockReceipts, + BatchReceiptsFull, + BatchReceiptsRange, + TransactionReceiptsParallel, + TransactionReceiptsSequential, +} + +/// Configuration for the starting block when the database is empty. +/// +/// Accepts either `"current"` (resolves to chain tip - 1 at init) or a specific block number. +/// Defaults to `Current`. +#[derive(Debug, Clone, PartialEq, Default)] +pub enum BlockStartConfig { + /// Start from the current chain height minus 1 (for reorg safety at first block). + #[default] + Current, + /// Start from a specific block number. + Number(u64), +} + +impl fmt::Display for BlockStartConfig { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + BlockStartConfig::Current => write!(f, "current"), + BlockStartConfig::Number(n) => write!(f, "{}", n), + } + } +} + +impl<'de> Deserialize<'de> for BlockStartConfig { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct BlockStartVisitor; + + impl<'de> Visitor<'de> for BlockStartVisitor { + type Value = BlockStartConfig; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("\"current\" or a block number (u64)") + } + + fn visit_u64(self, v: u64) -> Result + where + E: de::Error, + { + Ok(BlockStartConfig::Number(v)) + } + + fn visit_i64(self, v: i64) -> Result + where + E: de::Error, + { + if v < 0 { + return Err(de::Error::custom(format!( + "invalid block_start_on_first_start: block number cannot be negative, got {}", + v + ))); + } + Ok(BlockStartConfig::Number(v as u64)) + } + + fn visit_str(self, v: &str) -> Result + where + E: de::Error, + { + if v.eq_ignore_ascii_case("current") { + return Ok(BlockStartConfig::Current); + } + match v.parse::() { + Ok(n) => Ok(BlockStartConfig::Number(n)), + Err(_) => Err(de::Error::custom(format!( + "invalid block_start_on_first_start: expected 'current' or a block number, got '{}'", + v + ))), + } + } + } + + deserializer.deserialize_any(BlockStartVisitor) + } +} + +#[derive(Debug, Deserialize, Clone)] +pub struct StrategyConfig { + #[serde(default)] + pub block_start_on_first_start: BlockStartConfig, + + #[serde(default = "default_range_size")] + pub range_size: usize, + + #[serde(default = "default_max_parallel_requests")] + pub max_parallel_requests: usize, + + #[serde(default)] + pub block_fetcher: BlockFetcherStrategy, + + #[serde(default = "default_batch_receipts_size_range")] + pub batch_receipts_size_range: usize, + + #[serde(default)] + pub compute_block: Option, + + /// Active only if compute_block is active, other wise, computation is fully skipped. + /// When true, block verification will skip computation. + /// (e.g. Polygon type 0x7F for transaction root) with an ERROR log instead of failing. + /// When false, unsupported types cause a hard verification failure. + /// Defaults to true. + #[serde(default = "default_compute_block_allow_skipping")] + pub compute_block_allow_skipping: bool, + + #[serde(default)] + pub automatic_startup: bool, + + #[serde(default = "default_loop_delay_ms")] + pub loop_delay_ms: u64, + + #[serde(default = "default_max_exponential_backoff_ms")] + pub max_exponential_backoff_ms: u64, + + #[serde(default)] + pub publish: PublishConfig, +} + +fn default_range_size() -> usize { + 100 +} +fn default_max_parallel_requests() -> usize { + 50 +} +fn default_batch_receipts_size_range() -> usize { + 10 +} +fn default_loop_delay_ms() -> u64 { + 1000 +} +fn default_max_exponential_backoff_ms() -> u64 { + 20_000 +} +fn default_compute_block_allow_skipping() -> bool { + true +} +fn default_publish_retry_secs() -> u64 { + 1 +} +fn default_publish_stale() -> bool { + true +} +fn default_publish_no_stale_retries() -> u32 { + 5 +} + +#[derive(Debug, Deserialize, Clone)] +pub struct PublishConfig { + #[serde(default = "default_publish_retry_secs")] + pub publish_retry_secs: u64, + + #[serde(default = "default_publish_stale")] + pub publish_stale: bool, + + #[serde(default = "default_publish_no_stale_retries")] + pub publish_no_stale_retries: u32, +} + +impl Default for PublishConfig { + fn default() -> Self { + Self { + publish_retry_secs: default_publish_retry_secs(), + publish_stale: default_publish_stale(), + publish_no_stale_retries: default_publish_no_stale_retries(), + } + } +} + +impl StrategyConfig { + pub fn validate(&self) -> Result<(), ConfigError> { + if self.range_size < MIN_RANGE_SIZE || self.range_size > MAX_RANGE_SIZE { + return Err(ConfigError::Validation(format!( + "range size must be between {} and {}, got: {}", + MIN_RANGE_SIZE, MAX_RANGE_SIZE, self.range_size + ))); + } + if self.max_parallel_requests < MIN_PARALLEL_REQUESTS + || self.max_parallel_requests > MAX_PARALLEL_REQUESTS + { + return Err(ConfigError::Validation(format!( + "max_parallel_requests must be between {} and {}, got: {}", + MIN_PARALLEL_REQUESTS, MAX_PARALLEL_REQUESTS, self.max_parallel_requests + ))); + } + if self.batch_receipts_size_range < MIN_BATCH_RANGE + || self.batch_receipts_size_range > MAX_BATCH_RANGE + { + return Err(ConfigError::Validation(format!( + "batch_receipts_size_range must be between {} and {}, got: {}", + MIN_BATCH_RANGE, MAX_BATCH_RANGE, self.batch_receipts_size_range + ))); + } + Ok(()) + } +} + +impl Default for StrategyConfig { + fn default() -> Self { + Self { + block_start_on_first_start: BlockStartConfig::default(), + range_size: default_range_size(), + max_parallel_requests: default_max_parallel_requests(), + block_fetcher: BlockFetcherStrategy::default(), + batch_receipts_size_range: default_batch_receipts_size_range(), + compute_block: None, + compute_block_allow_skipping: default_compute_block_allow_skipping(), + automatic_startup: true, + loop_delay_ms: default_loop_delay_ms(), + max_exponential_backoff_ms: default_max_exponential_backoff_ms(), + publish: PublishConfig::default(), + } + } +} + +#[derive(Debug, Deserialize, Clone)] +pub struct CleanerConfig { + #[serde(default = "default_cleaner_active")] + pub active: bool, + + #[serde(default = "default_blocks_to_keep")] + pub blocks_to_keep: u64, + + #[serde(default = "default_cron_secs")] + pub cron_secs: u64, +} + +fn default_cleaner_active() -> bool { + true +} +fn default_blocks_to_keep() -> u64 { + 1000 +} +fn default_cron_secs() -> u64 { + 3600 +} + +impl Default for CleanerConfig { + fn default() -> Self { + Self { + active: default_cleaner_active(), + blocks_to_keep: default_blocks_to_keep(), + cron_secs: default_cron_secs(), + } + } +} + +#[derive(Deserialize, Clone, Derivative)] +#[derivative(Debug)] +pub struct BlockchainConfig { + #[serde(default = "default_blockchain_type")] + pub r#type: String, + + pub chain_id: u64, // REQUIRED + + #[derivative(Debug(format_with = "redact"))] + pub rpc_url: String, // REQUIRED, REDACTED + + pub network: String, // REQUIRED + + #[serde(default = "default_finality_depth")] + pub finality_depth: u64, + + #[serde(default)] + pub cleaner: CleanerConfig, + + #[serde(default)] + pub strategy: StrategyConfig, +} + +fn default_finality_depth() -> u64 { + 64 +} + +fn default_blockchain_type() -> String { + "evm".to_string() +} + +impl BlockchainConfig { + pub fn validate(&self) -> Result<(), ConfigError> { + if !self.rpc_url.starts_with("http://") && !self.rpc_url.starts_with("https://") { + return Err(ConfigError::Validation( + "rpc_url must start with http:// or https://".to_string(), + )); + } + if self.chain_id == 0 { + return Err(ConfigError::Validation( + "chain_id must be strictly positive".to_string(), + )); + } + if self.cleaner.blocks_to_keep < 2 { + return Err(ConfigError::Validation(format!( + "cleaner.blocks_to_keep ({}) must be >= 2 for reorg checking", + self.cleaner.blocks_to_keep + ))); + } + if self.cleaner.blocks_to_keep <= self.finality_depth { + return Err(ConfigError::Validation(format!( + "cleaner.blocks_to_keep ({}) must be greater than finality_depth ({})", + self.cleaner.blocks_to_keep, self.finality_depth + ))); + } + self.strategy.validate()?; + Ok(()) + } +} + +#[derive(Debug, Deserialize, Clone)] +pub struct PoolConfig { + #[serde(default = "default_max_connections")] + pub max_connections: u32, + #[serde(default = "default_min_connections")] + pub min_connections: u32, + #[serde(default = "default_acquire_timeout_secs")] + pub acquire_timeout_secs: u64, + #[serde(default = "default_idle_timeout_secs")] + pub idle_timeout_secs: u64, + #[serde(default = "default_max_lifetime_secs")] + pub max_lifetime_secs: u64, +} + +fn default_max_connections() -> u32 { + 10 +} +fn default_min_connections() -> u32 { + 2 +} +fn default_acquire_timeout_secs() -> u64 { + 5 +} +fn default_idle_timeout_secs() -> u64 { + 600 +} +fn default_max_lifetime_secs() -> u64 { + 1800 +} + +impl Default for PoolConfig { + fn default() -> Self { + Self { + max_connections: default_max_connections(), + min_connections: default_min_connections(), + acquire_timeout_secs: default_acquire_timeout_secs(), + idle_timeout_secs: default_idle_timeout_secs(), + max_lifetime_secs: default_max_lifetime_secs(), + } + } +} + +impl PoolConfig { + pub fn validate(&self) -> Result<(), ConfigError> { + if self.min_connections < MIN_POOL_MIN_CONNECTIONS { + return Err(ConfigError::Validation(format!( + "pool.min_connections ({}) must be >= {} (1 advisory-lock session + 1 query connection)", + self.min_connections, MIN_POOL_MIN_CONNECTIONS + ))); + } + if self.max_connections < MIN_POOL_MAX_CONNECTIONS { + return Err(ConfigError::Validation(format!( + "pool.max_connections ({}) must be >= {} (1 advisory-lock + 4 concurrent handler queries)", + self.max_connections, MIN_POOL_MAX_CONNECTIONS + ))); + } + if self.max_connections < self.min_connections { + return Err(ConfigError::Validation(format!( + "pool.max_connections ({}) must be >= pool.min_connections ({})", + self.max_connections, self.min_connections + ))); + } + Ok(()) + } +} + +#[derive(Deserialize, Clone, Derivative)] +#[derivative(Debug)] +pub struct DatabaseConfig { + #[derivative(Debug(format_with = "redact"))] + pub db_url: String, // REQUIRED, REDACTED — used by both password and IAM modes + /// IAM auth configuration. When absent or enabled=false, password from db_url is used. + /// When enabled=true, host/port/user/dbname are parsed from db_url and the password + /// is replaced with a short-lived IAM token. + /// Always deserialized (even without iam-auth feature) so config errors are visible. + #[serde(default)] + pub iam_auth: Option, + #[serde(default = "default_migration_max_attempts")] + pub migration_max_attempts: u32, + #[serde(default)] + pub pool: PoolConfig, +} + +fn default_migration_max_attempts() -> u32 { + 5 +} + +/// IAM role-based RDS authentication configuration. +/// Connection details (host, port, user, dbname) are parsed from `db_url`. +/// This struct is always deserialized regardless of the `iam-auth` feature flag, +/// so that misconfiguration is detected at startup rather than silently ignored. +#[derive(Debug, Deserialize, Clone, Default)] +pub struct IamAuthConfig { + /// Opt-in flag. When false (or section absent), falls back to password in db_url. + #[serde(default)] + pub enabled: bool, + /// Optional path to a custom RDS CA bundle PEM file. + /// If not set, relies on the system/rustls trust store (supports modern RDS CAs). + pub ssl_ca_path: Option, +} + +impl DatabaseConfig { + /// Returns true if IAM auth is configured and enabled. + pub fn use_iam_auth(&self) -> bool { + self.iam_auth.as_ref().is_some_and(|c| c.enabled) + } + + pub fn validate(&self) -> Result<(), ConfigError> { + if !self.db_url.starts_with("postgres://") && !self.db_url.starts_with("postgresql://") { + return Err(ConfigError::Validation( + "db_url must start with postgres:// or postgresql://".to_string(), + )); + } + + // Catch misconfiguration: IAM auth requested but feature not compiled in + #[cfg(not(feature = "iam-auth"))] + if self.use_iam_auth() { + return Err(ConfigError::Validation( + "iam_auth.enabled=true but binary was built without the iam-auth feature" + .to_string(), + )); + } + + self.pool.validate()?; + Ok(()) + } +} + +/// Unified broker configuration supporting both AMQP and Redis backends. +/// +/// # Backend Selection +/// +/// Set `broker_type` to choose the backend: +/// - `amqp` (default): Uses RabbitMQ with exchanges and queues +/// - `redis`: Uses Redis Streams with consumer groups +/// +/// # URL Requirements +/// +/// The `broker_url` must match the `broker_type`: +/// - For AMQP: URL starting with `amqp://` or `amqps://` +/// - For Redis: URL starting with `redis://` or `rediss://` +/// +/// # Durability +/// +/// Set `ensure_publish` to enable replication-aware publish durability: +/// - Redis: issues `WAIT 1 500` after every `XADD` +/// - AMQP: enables publisher confirms (`confirm_select`) +/// +/// # Example +/// +/// ```yaml +/// broker: +/// broker_type: redis +/// broker_url: redis://localhost:6379 +/// ensure_publish: true +/// ``` +#[derive(Deserialize, Clone, Derivative)] +#[derivative(Debug)] +pub struct BrokerConfig { + /// Backend type: `amqp` or `redis` + #[serde(default)] + pub broker_type: BrokerType, + + /// Broker connection URL + #[derivative(Debug(format_with = "redact"))] + pub broker_url: String, + + /// Enable replication-aware publish durability. + /// + /// - Redis: issues `WAIT` after every `XADD` to confirm replication + /// - AMQP: enables publisher confirms (`confirm_select`) + /// + /// Default: `false` (backward compatible) + #[serde(default)] + pub ensure_publish: bool, + + #[serde(default = "default_circuit_breaker_cooldown_secs")] + pub circuit_breaker_cooldown_secs: u64, + + #[serde(default = "default_circuit_breaker_threshold")] + pub circuit_breaker_threshold: u32, + + #[serde(default = "default_claim_min_idle")] + pub claim_min_idle: u64, +} + +fn default_claim_min_idle() -> u64 { + // Defaulted to 30, + // to add buffer the value from semaphore evm rpc provider http client timeout. + 30 +} + +fn default_circuit_breaker_cooldown_secs() -> u64 { + 10 +} + +fn default_circuit_breaker_threshold() -> u32 { + 3 +} + +impl BrokerConfig { + /// Get the broker URL. + pub fn url(&self) -> &str { + &self.broker_url + } + + pub fn validate(&self) -> Result<(), ConfigError> { + match self.broker_type { + BrokerType::Amqp => { + if !self.broker_url.starts_with("amqp://") + && !self.broker_url.starts_with("amqps://") + { + return Err(ConfigError::Validation( + "broker_url must start with amqp:// or amqps:// when broker_type is 'amqp'" + .to_string(), + )); + } + } + BrokerType::Redis => { + if !self.broker_url.starts_with("redis://") + && !self.broker_url.starts_with("rediss://") + { + return Err(ConfigError::Validation( + "broker_url must start with redis:// or rediss:// when broker_type is 'redis'" + .to_string(), + )); + } + } + } + Ok(()) + } +} + +/// Telemetry / metrics configuration. +/// +/// When `enabled` is `true` (default), the binary starts a Prometheus HTTP +/// server on `metrics_port` that serves `/metrics`. +#[derive(Debug, Deserialize, Clone)] +pub struct TelemetrySettings { + /// Enable the Prometheus metrics endpoint. Default: `true`. + #[serde(default = "default_telemetry_enabled")] + pub enabled: bool, + /// Port for the Prometheus `/metrics` endpoint. Default: `9090`. + #[serde(default = "default_metrics_port")] + pub metrics_port: u16, +} + +fn default_telemetry_enabled() -> bool { + true +} +fn default_metrics_port() -> u16 { + 9090 +} + +impl Default for TelemetrySettings { + fn default() -> Self { + Self { + enabled: default_telemetry_enabled(), + metrics_port: default_metrics_port(), + } + } +} + +/// Log format selection. +#[derive(Debug, Deserialize, Clone, Default, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum LogFormat { + /// JSON output — Loki-ready for production ingestion. + #[default] + Json, + /// Compact single-line human-readable format. + Compact, + /// Pretty multi-line format for local development. + Pretty, +} + +/// Logging configuration. +/// +/// Controls tracing-subscriber initialization: output format, verbosity, +/// and which metadata fields are included in each log line. +/// +/// All fields have sensible production defaults and can be overridden via +/// YAML config or environment variables (`APP_LOG__FORMAT`, `APP_LOG__LEVEL`, etc.). +#[derive(Debug, Deserialize, Clone)] +pub struct LogConfig { + /// Output format: `json`, `compact`, or `pretty`. + #[serde(default)] + pub format: LogFormat, + + /// Show source file and line number in log output. + #[serde(default)] + pub show_file_line: bool, + + /// Show thread IDs in log output. + #[serde(default = "default_true")] + pub show_thread_ids: bool, + + /// Show RFC 3339 timestamps in log output. + #[serde(default = "default_true")] + pub show_timestamp: bool, + + /// Show the tracing target (module path) in log output. + #[serde(default = "default_true")] + pub show_target: bool, + + /// Inject `name`, `network`, and `chain_id` as constant fields in every log line. + #[serde(default = "default_true")] + pub show_constants: bool, + + /// Default log level. Overridden entirely by `RUST_LOG` env var when set. + #[serde(default = "default_log_level")] + pub level: String, +} + +fn default_true() -> bool { + true +} + +fn default_log_level() -> String { + "info".to_string() +} + +impl Default for LogConfig { + fn default() -> Self { + Self { + format: LogFormat::default(), + show_file_line: false, + show_thread_ids: true, + show_timestamp: true, + show_target: true, + show_constants: true, + level: default_log_level(), + } + } +} + +#[derive(Debug, Deserialize, Clone)] +pub struct Settings { + #[serde(default = "default_name")] + pub name: String, + + /// Port of the shared application HTTP server. + /// + /// **Required — no default.** Hosts the Kubernetes health probes + /// (`/livez`, `/readyz`) today; designed as the single mount point + /// for future operational routes (metrics, admin endpoints, ...). + pub http_port: u16, + + pub database: DatabaseConfig, + pub broker: BrokerConfig, + pub blockchain: BlockchainConfig, + #[serde(default)] + pub telemetry: TelemetrySettings, + #[serde(default)] + pub log: LogConfig, +} + +fn default_name() -> String { + "listener".to_string() +} + +impl Settings { + /// Load config from file and/or environment variables. + /// + /// Environment variables override file values. + /// Format: `APP_SECTION__FIELD` (double underscore separator) + /// + /// # Example + /// ```bash + /// APP_DATABASE__DB_URL="postgres://..." cargo run + /// ``` + pub fn new(config_file: Option<&str>) -> Result { + let mut builder = Config::builder(); + + // 1. Load from file if provided + if let Some(file) = config_file { + builder = builder.add_source(File::with_name(file).required(true)); + } + + // 2. Load from environment (APP_SECTION__FIELD format) + builder = builder.add_source( + Environment::with_prefix("APP") + .separator("__") + .prefix_separator("_"), + ); + + // 3. Build and deserialize + let settings: Settings = builder + .build() + .map_err(|e| ConfigError::Parse(e.to_string()))? + .try_deserialize() + .map_err(|e| ConfigError::Parse(e.to_string()))?; + + // 4. Validate all sections + settings.validate()?; + + Ok(settings) + } + + pub fn validate(&self) -> Result<(), ConfigError> { + if self.http_port == 0 { + return Err(ConfigError::Validation( + "http_port must be a non-zero port number (no default is provided; configure it explicitly)".to_string(), + )); + } + self.database.validate()?; + self.broker.validate()?; + self.blockchain.validate()?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_strategy_config_defaults() { + let config = StrategyConfig::default(); + assert_eq!(config.block_start_on_first_start, BlockStartConfig::Current); + assert_eq!(config.range_size, 100); + assert_eq!(config.max_parallel_requests, 50); + assert_eq!(config.block_fetcher, BlockFetcherStrategy::BlockReceipts); + assert_eq!(config.batch_receipts_size_range, 10); + assert!(config.automatic_startup); + } + + #[test] + fn test_block_start_config_from_u64() { + let value: BlockStartConfig = serde_json::from_str("12345").unwrap(); + assert_eq!(value, BlockStartConfig::Number(12345)); + } + + #[test] + fn test_block_start_config_from_string_current() { + let value: BlockStartConfig = serde_json::from_str("\"current\"").unwrap(); + assert_eq!(value, BlockStartConfig::Current); + } + + #[test] + fn test_block_start_config_from_string_current_case_insensitive() { + let value: BlockStartConfig = serde_json::from_str("\"Current\"").unwrap(); + assert_eq!(value, BlockStartConfig::Current); + let value: BlockStartConfig = serde_json::from_str("\"CURRENT\"").unwrap(); + assert_eq!(value, BlockStartConfig::Current); + } + + #[test] + fn test_block_start_config_from_numeric_string() { + // Env vars arrive as strings via the config crate + let value: BlockStartConfig = serde_json::from_str("\"99999\"").unwrap(); + assert_eq!(value, BlockStartConfig::Number(99999)); + } + + #[test] + fn test_block_start_config_invalid_string_rejected() { + let result = serde_json::from_str::("\"foo\""); + assert!(result.is_err()); + let err = result.unwrap_err().to_string(); + assert!(err.contains("invalid block_start_on_first_start")); + assert!(err.contains("foo")); + } + + #[test] + fn test_block_start_config_display() { + assert_eq!(BlockStartConfig::Current.to_string(), "current"); + assert_eq!(BlockStartConfig::Number(42).to_string(), "42"); + } + + #[test] + fn test_blocks_to_keep_minimum_validation() { + let config = BlockchainConfig { + r#type: "evm".to_string(), + chain_id: 1, + rpc_url: "https://rpc.example.com".to_string(), + network: "test".to_string(), + finality_depth: 0, + cleaner: CleanerConfig { + blocks_to_keep: 1, + ..Default::default() + }, + strategy: StrategyConfig::default(), + }; + let result = config.validate(); + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("must be >= 2")); + } + + #[test] + fn test_publish_config_defaults() { + let config = PublishConfig::default(); + assert_eq!(config.publish_retry_secs, 1); + assert!(config.publish_stale); + assert_eq!(config.publish_no_stale_retries, 5); + } + + #[test] + fn test_strategy_config_with_publish_defaults() { + let config = StrategyConfig::default(); + assert_eq!(config.publish.publish_retry_secs, 1); + assert!(config.publish.publish_stale); + assert_eq!(config.publish.publish_no_stale_retries, 5); + } + + #[test] + fn test_strategy_validation_batch_size_too_large() { + let config = StrategyConfig { + range_size: 20000, + ..Default::default() + }; + let result = config.validate(); + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("range size")); + } + + #[test] + fn test_strategy_validation_parallel_requests_too_large() { + let config = StrategyConfig { + max_parallel_requests: 500, + ..Default::default() + }; + let result = config.validate(); + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .to_string() + .contains("max_parallel_requests") + ); + } + + #[test] + fn zero_chain_id() { + let config = BlockchainConfig { + network: "fake-chain".to_string(), + chain_id: 0, + r#type: default_blockchain_type(), + rpc_url: "https://ethereum-rpc.publicnode.com".to_string(), // REQUIRED, REDACTED + finality_depth: default_finality_depth(), + cleaner: CleanerConfig::default(), + strategy: StrategyConfig::default(), + }; + let result = config.validate(); + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .to_string() + .contains("strictly positive") + ); + } + + #[test] + fn test_database_url_validation() { + let config = DatabaseConfig { + db_url: "mysql://invalid".to_string(), + iam_auth: None, + migration_max_attempts: 5, + pool: PoolConfig::default(), + }; + let result = config.validate(); + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("postgres://")); + } + + #[test] + fn test_broker_amqp_url_validation() { + let config = BrokerConfig { + broker_type: BrokerType::Amqp, + broker_url: "http://invalid".to_string(), + ensure_publish: false, + circuit_breaker_cooldown_secs: 10, + circuit_breaker_threshold: 3, + claim_min_idle: 30, + }; + let result = config.validate(); + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("amqp://")); + } + + #[test] + fn test_broker_redis_url_validation() { + let config = BrokerConfig { + broker_type: BrokerType::Redis, + broker_url: "http://invalid".to_string(), + ensure_publish: false, + circuit_breaker_cooldown_secs: 10, + circuit_breaker_threshold: 3, + claim_min_idle: 30, + }; + let result = config.validate(); + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("redis://")); + } + + #[test] + fn test_broker_type_default_is_redis() { + assert_eq!(BrokerType::default(), BrokerType::Redis); + } + + #[test] + fn test_broker_url_method() { + let amqp_config = BrokerConfig { + broker_type: BrokerType::Amqp, + broker_url: "amqp://localhost:5672".to_string(), + ensure_publish: false, + circuit_breaker_cooldown_secs: 10, + circuit_breaker_threshold: 3, + claim_min_idle: 30, + }; + assert_eq!(amqp_config.url(), "amqp://localhost:5672"); + + let redis_config = BrokerConfig { + broker_type: BrokerType::Redis, + broker_url: "redis://localhost:6379".to_string(), + ensure_publish: false, + circuit_breaker_cooldown_secs: 10, + circuit_breaker_threshold: 3, + claim_min_idle: 30, + }; + assert_eq!(redis_config.url(), "redis://localhost:6379"); + } + + #[test] + fn test_broker_valid_amqp_config() { + let config = BrokerConfig { + broker_type: BrokerType::Amqp, + broker_url: "amqp://localhost:5672".to_string(), + ensure_publish: false, + circuit_breaker_cooldown_secs: 10, + circuit_breaker_threshold: 3, + claim_min_idle: 30, + }; + assert!(config.validate().is_ok()); + } + + #[test] + fn test_broker_valid_redis_config() { + let config = BrokerConfig { + broker_type: BrokerType::Redis, + broker_url: "redis://localhost:6379".to_string(), + ensure_publish: false, + circuit_breaker_cooldown_secs: 10, + circuit_breaker_threshold: 3, + claim_min_idle: 30, + }; + assert!(config.validate().is_ok()); + } + + #[test] + fn test_blockchain_rpc_url_validation() { + let config = BlockchainConfig { + r#type: "evm".to_string(), + chain_id: 1, + rpc_url: "ws://invalid".to_string(), + network: "ethereum-mainnet".to_string(), + cleaner: CleanerConfig::default(), + strategy: StrategyConfig::default(), + finality_depth: 15, + }; + let result = config.validate(); + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("http://")); + } + + #[test] + fn test_secrets_redacted_in_debug() { + let db_config = DatabaseConfig { + db_url: "postgres://user:secret_password@localhost/db".to_string(), + iam_auth: None, + migration_max_attempts: 5, + pool: PoolConfig::default(), + }; + let debug_output = format!("{:?}", db_config); + assert!(!debug_output.contains("secret_password")); + assert!(debug_output.contains("[REDACTED]")); + + let broker_config = BrokerConfig { + broker_type: BrokerType::Amqp, + broker_url: "amqp://user:secret_password@localhost/vhost".to_string(), + ensure_publish: false, + circuit_breaker_cooldown_secs: 10, + circuit_breaker_threshold: 3, + claim_min_idle: 30, + }; + let debug_output = format!("{:?}", broker_config); + assert!(!debug_output.contains("secret_password")); + assert!(debug_output.contains("[REDACTED]")); + + let blockchain_config = BlockchainConfig { + r#type: "evm".to_string(), + chain_id: 1, + rpc_url: "https://secret-api-key@rpc.example.com".to_string(), + network: "ethereum-mainnet".to_string(), + cleaner: CleanerConfig::default(), + strategy: StrategyConfig::default(), + finality_depth: 15, + }; + let debug_output = format!("{:?}", blockchain_config); + assert!(!debug_output.contains("secret-api-key")); + assert!(debug_output.contains("[REDACTED]")); + } + + #[test] + fn test_pool_default_passes_validation() { + assert!(PoolConfig::default().validate().is_ok()); + } + + #[test] + fn test_pool_min_connections_too_low() { + let config = PoolConfig { + min_connections: 1, + ..Default::default() + }; + let result = config.validate(); + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("min_connections")); + } + + #[test] + fn test_pool_max_connections_too_low() { + let config = PoolConfig { + max_connections: 3, + ..Default::default() + }; + let result = config.validate(); + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("max_connections")); + } + + #[test] + fn test_pool_max_less_than_min() { + let config = PoolConfig { + min_connections: 6, + max_connections: 5, + ..Default::default() + }; + let result = config.validate(); + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("max_connections")); + } +} diff --git a/listener/crates/listener_core/src/config/mod.rs b/listener/crates/listener_core/src/config/mod.rs new file mode 100644 index 0000000000..aa2d7e974a --- /dev/null +++ b/listener/crates/listener_core/src/config/mod.rs @@ -0,0 +1,9 @@ +#[allow(clippy::module_inception)] +pub mod config; + +// Re-export configuration types for easier access +pub use config::{ + BlockFetcherStrategy, BlockStartConfig, BlockchainConfig, BrokerConfig, BrokerType, + ConfigError, DatabaseConfig, LogConfig, LogFormat, PoolConfig, PublishConfig, Settings, + StrategyConfig, +}; diff --git a/listener/crates/listener_core/src/core/cleaner.rs b/listener/crates/listener_core/src/core/cleaner.rs new file mode 100644 index 0000000000..785936a6ff --- /dev/null +++ b/listener/crates/listener_core/src/core/cleaner.rs @@ -0,0 +1,99 @@ +use std::time::Duration; + +use broker::Publisher; +use primitives::routing; +use thiserror::Error; +use tracing::{error, info}; + +use primitives::utils::saturating_u64_to_i64; + +use crate::config::config::CleanerConfig; +use crate::store::repositories::BlockRepository; + +#[derive(Error, Debug)] +pub enum CleanerError { + #[error("Broker publish error: {message}")] + BrokerPublishError { message: String }, +} + +#[derive(Clone)] +pub struct Cleaner { + blocks: BlockRepository, + publisher: Publisher, + active: bool, + blocks_to_keep: u64, + cron_secs: u64, +} + +impl Cleaner { + pub fn new(blocks: BlockRepository, publisher: Publisher, config: &CleanerConfig) -> Self { + Self { + blocks, + publisher, + active: config.active, + blocks_to_keep: config.blocks_to_keep, + cron_secs: config.cron_secs, + } + } + + pub async fn run(&self) -> Result<(), CleanerError> { + if !self.active { + info!("Cleaner: inactive — skipping cleanup and not re-triggering"); + return Ok(()); + } + + match self + .blocks + .delete_blocks_keeping_latest(saturating_u64_to_i64(self.blocks_to_keep)) + .await + { + Ok(deleted) => { + if deleted > 0 { + match self.blocks.get_min_block_number().await { + Ok(Some(min_block)) => { + info!( + deleted, + min_block_kept = min_block, + blocks_to_keep = self.blocks_to_keep, + "Cleaner: removed {deleted} blocks, blocks below {min_block} were deleted" + ); + } + _ => { + info!( + deleted, + blocks_to_keep = self.blocks_to_keep, + "Cleaner: removed {deleted} blocks" + ); + } + } + } else { + info!( + blocks_to_keep = self.blocks_to_keep, + "Cleaner: no blocks to clean up" + ); + } + } + Err(e) => { + error!( + error = %e, + blocks_to_keep = self.blocks_to_keep, + "Cleaner: failed to delete old blocks, skipping this iteration" + ); + } + } + + tokio::time::sleep(Duration::from_secs(self.cron_secs)).await; + + self.publisher + .publish(routing::CLEAN_BLOCKS, &serde_json::Value::Null) + .await + .map_err(|e| { + error!(error = %e, "Cleaner: failed to publish next iteration"); + CleanerError::BrokerPublishError { + message: format!("Broker publish failed: {}", e), + } + })?; + + Ok(()) + } +} diff --git a/listener/crates/listener_core/src/core/evm_listener.rs b/listener/crates/listener_core/src/core/evm_listener.rs new file mode 100644 index 0000000000..29091a3902 --- /dev/null +++ b/listener/crates/listener_core/src/core/evm_listener.rs @@ -0,0 +1,1160 @@ +use std::time::{Duration, Instant}; + +use alloy::primitives::B256; +use thiserror::Error; +use tokio::task::JoinSet; +use tokio_util::sync::CancellationToken; +use tracing::{debug, error, info, warn}; + +use broker::{Broker, Publisher}; + +use primitives::event::{BlockFlow, ReorgBacktrackEvent}; + +use super::publisher::{self, PublisherError}; +use crate::config::{BlockStartConfig, BlockchainConfig, PublishConfig}; +use crate::store::SqlError; +use crate::{ + blockchain::evm::{ + evm_block_fetcher::{BlockFetchError, EvmBlockFetcher, FetchedBlock}, + sem_evm_rpc_provider::{RpcProviderError, SemEvmRpcProvider}, + }, + config::BlockFetcherStrategy, + core::slot_buffer::{AsyncSlotBuffer, BufferError}, + store::models::{BlockStatus, NewDatabaseBlock, UpsertResult}, + store::repositories::Repositories, +}; + +/// Errors that can occur during EvmListener operations. +#[derive(Error, Debug)] +pub enum EvmListenerError { + #[error("Could not fetch block: {source}")] + CouldNotFetchBlock { + #[source] + source: BlockFetchError, + }, + + #[error("Could not compute block: {source}")] + CouldNotComputeBlock { + #[source] + source: BlockFetchError, + }, + + #[error("Database error: {source}")] + DatabaseError { + #[source] + source: SqlError, + }, + + #[error("Could not fetch chain height: {source}")] + ChainHeightError { + #[source] + source: RpcProviderError, + }, + + #[error("Buffer error: {source}")] + SlotBufferError { + #[source] + source: BufferError, + }, + + #[error("Broker publish error: {message}")] + BrokerPublishError { message: String }, + + #[error("Payload build error: {source}")] + PayloadBuildError { + #[source] + source: PublisherError, + }, + + #[error("Invariant violation: {message}")] + InvariantViolation { message: String }, + + #[error("Message processing error: {message}")] + MessageProcessingError { message: String }, +} + +/// Outcome of a single cursor iteration. +/// +/// Reorgs are a normal operational event on blockchains (not an error), +/// so they are represented as a distinct result variant rather than an error. +#[derive(Debug)] +pub enum CursorResult { + /// All blocks in the batch were validated and inserted as canonical. + Complete, + /// Chain hasn't advanced beyond the DB tip. Nothing to fetch. + UpToDate, + /// Reorg detected: the parent hash of the block at this number + /// did not match the expected hash from the previous canonical block. + ReorgDetected { + block_number: u64, + block_hash: B256, + parent_hash: B256, + }, +} + +#[derive(Clone)] +pub struct EvmListener { + provider: SemEvmRpcProvider, + repositories: Repositories, + broker: Broker, + event_publisher: Publisher, + publish_config: PublishConfig, + chain_id: u64, + range_size: usize, + fetcher_strategy: BlockFetcherStrategy, + batch_receipts_size_range: Option, + compute_block: bool, + compute_block_allow_skipping: bool, + loop_delay_ms: u64, + finality_depth: u64, + max_exponential_backoff_ms: u64, +} + +impl EvmListener { + pub fn new( + provider: SemEvmRpcProvider, + repositories: Repositories, + broker: Broker, + event_publisher: Publisher, + blockchain_settings: &BlockchainConfig, + ) -> Self { + let compute_block = blockchain_settings + .strategy + .compute_block + .unwrap_or_default(); + + Self { + provider, + repositories, + broker, + event_publisher, + publish_config: blockchain_settings.strategy.publish.clone(), + chain_id: blockchain_settings.chain_id, + range_size: blockchain_settings.strategy.range_size, + fetcher_strategy: blockchain_settings.strategy.block_fetcher.clone(), + batch_receipts_size_range: Some(blockchain_settings.strategy.batch_receipts_size_range), + compute_block, + compute_block_allow_skipping: blockchain_settings.strategy.compute_block_allow_skipping, + loop_delay_ms: blockchain_settings.strategy.loop_delay_ms, + finality_depth: blockchain_settings.finality_depth, + max_exponential_backoff_ms: blockchain_settings.strategy.max_exponential_backoff_ms, + } + } + + /// Returns the chain ID this listener is configured for. + pub fn chain_id(&self) -> u64 { + self.chain_id + } + + /// Create a fetcher with the given cancellation token and compute_block flag. + fn create_fetcher( + &self, + cancel_token: CancellationToken, + compute_block: bool, + ) -> EvmBlockFetcher { + EvmBlockFetcher::new(self.provider.clone()) + .with_chain_id(self.chain_id) + .with_verify_block(compute_block) + .with_verify_block_allow_skipping(self.compute_block_allow_skipping) + .with_cancellation_token(cancel_token) + .with_max_exponential_backoff_ms(self.max_exponential_backoff_ms) + } + + /// Map a BlockFetchError to an EvmListenerError. + /// + /// VerificationFailed errors map to CouldNotComputeBlock, + /// all other errors map to CouldNotFetchBlock. + fn map_fetch_error(error: BlockFetchError) -> EvmListenerError { + match error { + BlockFetchError::VerificationFailed(_) => { + EvmListenerError::CouldNotComputeBlock { source: error } + } + _ => EvmListenerError::CouldNotFetchBlock { source: error }, + } + } + + /// Fetch a block by its number using the configured strategy. + /// + /// # Arguments + /// * `block_number` - The block number to fetch + /// * `cancel_token` - Cancellation token for this operation (share across batch for batch cancellation) + /// * `compute_block` - Whether to verify block integrity (transaction root, receipt root, block hash) + /// + /// # Cancellation + /// When `cancel_token.cancel()` is called, this operation will return `BlockFetchError::Cancelled`. + /// All parallel operations sharing the same token will be cancelled together. + pub async fn get_block_by_number( + &self, + block_number: u64, + cancel_token: CancellationToken, + compute_block: bool, + ) -> Result { + let fetcher = self.create_fetcher(cancel_token, compute_block); + + let result = match self.fetcher_strategy { + BlockFetcherStrategy::BlockReceipts => { + fetcher + .fetch_block_with_block_receipts_by_number(block_number) + .await + } + BlockFetcherStrategy::BatchReceiptsFull => { + fetcher + .fetch_block_with_batch_receipts_by_number(block_number) + .await + } + BlockFetcherStrategy::BatchReceiptsRange => { + let batch_size = self.batch_receipts_size_range.unwrap_or(1); + fetcher + .fetch_block_by_number_with_parallel_batched_receipts(block_number, batch_size) + .await + } + BlockFetcherStrategy::TransactionReceiptsParallel => { + fetcher + .fetch_block_with_individual_receipts_by_number(block_number) + .await + } + BlockFetcherStrategy::TransactionReceiptsSequential => { + fetcher + .fetch_block_with_sequential_receipts_by_number(block_number) + .await + } + }; + + result.map_err(Self::map_fetch_error) + } + + /// Fetch a block by its hash using the configured strategy. + /// + /// # Arguments + /// * `hash` - The block hash to fetch + /// * `cancel_token` - Cancellation token for this operation + /// * `compute_block` - Whether to verify block integrity (transaction root, receipt root, block hash) + /// + /// # Cancellation + /// When `cancel_token.cancel()` is called, this operation will return `BlockFetchError::Cancelled`. + pub async fn get_block_by_hash( + &self, + hash: B256, + cancel_token: CancellationToken, + compute_block: bool, + ) -> Result { + let fetcher = self.create_fetcher(cancel_token, compute_block); + + let result = match self.fetcher_strategy { + BlockFetcherStrategy::BlockReceipts => { + fetcher.fetch_block_with_block_receipts_by_hash(hash).await + } + BlockFetcherStrategy::BatchReceiptsFull => { + fetcher.fetch_block_with_batch_receipts_by_hash(hash).await + } + BlockFetcherStrategy::BatchReceiptsRange => { + let batch_size = self.batch_receipts_size_range.unwrap_or(1); + fetcher + .fetch_block_by_hash_with_parallel_batched_receipts(hash, batch_size) + .await + } + BlockFetcherStrategy::TransactionReceiptsParallel => { + fetcher + .fetch_block_with_individual_receipts_by_hash(hash) + .await + } + BlockFetcherStrategy::TransactionReceiptsSequential => { + fetcher + .fetch_block_with_sequential_receipts_by_hash(hash) + .await + } + }; + + result.map_err(Self::map_fetch_error) + } + + /// Validates the block fetching strategy and initializes the database with a starting block. + /// + /// This function is called during service initialization and will panic on any failure. + /// The rationale is that if initialization fails, the process cannot operate correctly + /// and should crash to trigger a restart (crash-loop-backoff pattern). + /// + /// # Panics + /// - If the RPC node is unreachable or returns an error + /// - If the configured strategy is not compatible with the RPC node + /// - If the database is unreachable or returns an error + /// - If the starting block cannot be fetched or inserted + pub async fn validate_strategy_and_init_block(&self, block_start_config: BlockStartConfig) { + // Fetch latest block number from blockchain using SemEvmRpcProvider + let latest_block_number = match self.provider.get_block_number().await { + Ok(block_number) => block_number, + Err(e) => { + tracing::error!( + error = %e, + "CRITICAL: Could not fetch latest block number from RPC" + ); + panic!("Could not fetch latest block number from RPC: {}", e); + } + }; + + tracing::info!( + block_number = latest_block_number, + "Fetched latest block number from blockchain" + ); + + // Resolve block_start_on_first_start config to a concrete block number. + // "current" resolves to height - 1 for reorg safety at the first block. + let block_start: u64 = match block_start_config { + BlockStartConfig::Number(n) => n, + BlockStartConfig::Current => { + let resolved = latest_block_number.saturating_sub(1); + tracing::info!( + latest_block_number, + resolved_block_start = resolved, + "Resolved block_start_on_first_start='current' to height - 1" + ); + resolved + } + }; + + // Validate strategy by attempting to fetch the latest block + let cancel_token = CancellationToken::new(); + match self + .get_block_by_number(latest_block_number, cancel_token, self.compute_block) + .await + { + Ok(fetched_block) => { + tracing::info!( + block_number = latest_block_number, + block_hash = %fetched_block.block.header.hash, + tx_count = fetched_block.transaction_count(), + strategy = ?self.fetcher_strategy, + "Strategy validation successful" + ); + } + Err(e) => { + tracing::error!( + error = %e, + strategy = ?self.fetcher_strategy, + "CRITICAL: Strategy validation failed - binary should not start" + ); + panic!( + "Strategy validation failed: {}. The configured strategy {:?} is not compatible with the RPC node.", + e, self.fetcher_strategy + ); + } + } + + // Check for existing canonical block in the database + let latest_db_block = match self.repositories.blocks.get_latest_canonical_block().await { + Ok(block) => block, + Err(e) => { + tracing::error!( + error = %e, + "CRITICAL: Could not query database for latest canonical block" + ); + panic!("Could not query database for latest canonical block: {}", e); + } + }; + + if let Some(block) = latest_db_block { + tracing::info!( + block_number = block.block_number, + block_hash = %block.block_hash, + "Found existing canonical block in database, no initialization needed" + ); + return; + } + + tracing::info!( + block_start = block_start, + "No canonical block in database, initializing with block_start" + ); + + // Fetch the starting block from the blockchain + let cancel_token = CancellationToken::new(); + let fetched_block = match self + .get_block_by_number(block_start, cancel_token, self.compute_block) + .await + { + Ok(block) => block, + Err(e) => { + tracing::error!( + error = %e, + block_start = block_start, + "CRITICAL: Could not fetch starting block from blockchain" + ); + panic!( + "Could not fetch starting block {} from blockchain: {}", + block_start, e + ); + } + }; + + // Convert FetchedBlock to NewDatabaseBlock and insert into database + let new_db_block = + NewDatabaseBlock::from_rpc_block(&fetched_block.block, BlockStatus::Canonical); + + if let Err(e) = self.repositories.blocks.insert_block(&new_db_block).await { + if e.is_unique_violation() { + tracing::info!( + block_number = new_db_block.block_number, + block_hash = %new_db_block.block_hash, + "Starting block already exists in database, skipping insert" + ); + } else { + tracing::error!( + error = %e, + block_number = new_db_block.block_number, + block_hash = %new_db_block.block_hash, + "CRITICAL: Could not insert starting block into database" + ); + panic!("Could not insert starting block into database: {}", e); + } + } else { + tracing::info!( + block_number = new_db_block.block_number, + block_hash = %new_db_block.block_hash, + "Successfully initialized database with starting block" + ); + } + } + + /// Orchestrates one iteration of the V2 cursor algorithm. + /// + /// This is the main entry point for block fetching. It: + /// 1. Reads the DB tip (latest canonical block) + /// 2. Gets the current chain height from the RPC node + /// 3. Calculates the range of blocks to fetch (capped by `batch_size`) + /// 4. Spawns two concurrent tasks via `tokio::spawn`: + /// - **Producer** (`fetch_blocks_in_parallel`): fetches blocks via parallel RPC calls, + /// fills an `AsyncSlotBuffer` out-of-order + /// - **Consumer** (`cursor_processing`): reads slots sequentially, validates the + /// parent_hash chain, inserts validated blocks into the DB + /// 5. Awaits both tasks and analyzes outcomes + /// + /// # Returns + /// - `Ok(CursorResult::Complete)` — all blocks validated and inserted; sleeps `loop_delay_ms` before returning + /// - `Ok(CursorResult::UpToDate)` — chain hasn't advanced, nothing to fetch + /// - `Ok(CursorResult::ReorgDetected { block_number })` — reorg detected at this height (no sleep) + /// - `Err(...)` — unrecoverable or transient error (no sleep, propagate fast for retry logic) + /// + /// # Cancellation + /// A fresh `CancellationToken` is created per iteration. It is shared between the producer + /// and consumer. Either side can cancel the other: + /// - Cursor cancels on reorg detection or DB failure + /// - Fetcher cancellation propagates to cursor via the shared token + /// + /// # Panics + /// This function does not panic. Task panics are caught via `JoinError` and converted + /// to `EvmListenerError::InvariantViolation`. + pub async fn fetch_blocks_and_run_cursor(&self) -> Result { + metrics::counter!( + "listener_cursor_iterations_total", + "chain_id" => self.chain_id.to_string() + ) + .increment(1); + + // We don't need to dedup the message, the flow lock + ack will handle the deduplication by itself. + + // Step 1: Get the latest canonical block from DB (the "tip" of our chain view). + // validate_strategy_and_init_block guarantees at least one block exists. + let db_block = self + .repositories + .blocks + .get_latest_canonical_block() + .await + .map_err(|e| EvmListenerError::DatabaseError { source: e })? + .ok_or_else(|| EvmListenerError::InvariantViolation { + message: "No canonical block in database. \ + validate_strategy_and_init_block must run before the cursor." + .to_string(), + })?; + + let db_block_number = db_block.block_number; + let db_block_hash = db_block.block_hash; + + metrics::gauge!( + "listener_db_tip_block_number", + "chain_id" => self.chain_id.to_string() + ) + .set(db_block_number as f64); + + debug!( + db_block_number = db_block_number, + db_block_hash = %db_block_hash, + "Retrieved DB tip for cursor iteration" + ); + + // Step 2: Get the current chain height from the RPC node + let chain_height = self + .provider + .get_block_number() + .await + .map_err(|e| EvmListenerError::ChainHeightError { source: e })?; + + metrics::gauge!( + "listener_chain_height_block_number", + "chain_id" => self.chain_id.to_string() + ) + .set(chain_height as f64); + + // Step 3: If the chain hasn't advanced beyond our DB tip, there's nothing to do + if chain_height <= db_block_number { + debug!( + chain_height = chain_height, + db_block_number = db_block_number, + "Chain has not advanced beyond DB tip" + ); + tokio::time::sleep(Duration::from_millis(self.loop_delay_ms)).await; + return Ok(CursorResult::UpToDate); + } + + // Step 4: Calculate the inclusive range [range_start, range_end] + // Example: db=100, batch=10, chain=500 -> start=101, end=110, length=10 (blocks 101..=110) + // Example: db=100, batch=10, chain=103 -> start=101, end=103, length=3 (only 3 available) + let range_start = db_block_number + 1; + let range_end = std::cmp::min(chain_height, db_block_number + self.range_size as u64); + let range_length = (range_end - range_start + 1) as usize; // +1 because both bounds are inclusive + + info!( + range_start = range_start, + range_end = range_end, + range_length = range_length, + chain_height = chain_height, + "Starting cursor iteration" + ); + + // Step 5: Create shared state for producer-consumer coordination + let range_start_time = Instant::now(); + let buffer = AsyncSlotBuffer::::new(range_length); + let cancel_token = CancellationToken::new(); + + // Step 6: Spawn both tasks concurrently + // - cursor_processing (consumer): reads slots sequentially, validates hash chain, inserts to DB + // - fetch_blocks_in_parallel (producer): fetches blocks via parallel RPC calls, fills slots + let cursor_handle = tokio::spawn(cursor_processing( + self.clone(), + buffer.clone(), + cancel_token.clone(), + db_block_hash, + range_start, + range_length, + )); + + let fetcher_handle = tokio::spawn(fetch_blocks_in_parallel( + self.clone(), + buffer, + cancel_token.clone(), + range_start, + range_length, + )); + + // Step 7: Await both tasks to completion. + // tokio::join! ensures neither task is abandoned even if the other completes/fails first. + // On cancellation, in-flight HTTP requests may take up to 10s (RPC timeout) before + // tasks finish. This is acceptable. + let (cursor_join_result, fetcher_join_result) = tokio::join!(cursor_handle, fetcher_handle); + + metrics::histogram!( + "listener_range_fetch_duration_seconds", + "chain_id" => self.chain_id.to_string() + ) + .record(range_start_time.elapsed().as_secs_f64()); + + // Step 8: Unwrap JoinHandle results — a JoinError means the task panicked, + // which is a critical bug (we never panic in our code). + let cursor_outcome = cursor_join_result.map_err(|join_err| { + cancel_token.cancel(); + error!(error = %join_err, "Cursor task panicked — this is a critical bug"); + EvmListenerError::InvariantViolation { + message: format!("Cursor task panicked: {}", join_err), + } + })?; + + let fetcher_outcome = fetcher_join_result.map_err(|join_err| { + cancel_token.cancel(); + error!(error = %join_err, "Fetcher task panicked — this is a critical bug"); + EvmListenerError::InvariantViolation { + message: format!("Fetcher task panicked: {}", join_err), + } + })?; + + // NOTE: redo this, with clear error path and skips (Due to message broker mostly) + // Step 9: Analyze outcomes — cursor takes priority since it's the authority + // on what actually made it into the DB. + // Arms are ordered: reorg first (no sleep), then success (sleep), then errors (no sleep). + match (cursor_outcome, fetcher_outcome) { + // === REORG PATH === + // Fetcher error (Cancelled) is expected here since cursor cancelled the token. + // No sleep — reorg should be handled immediately. + ( + Ok(CursorResult::ReorgDetected { + block_number, + block_hash, + parent_hash, + }), + _, + ) => { + info!( + block_number = block_number, + block_hash = %block_hash, + "Reorg detected" + ); + + metrics::counter!( + "listener_reorgs_total", + "chain_id" => self.chain_id.to_string() + ) + .increment(1); + + Ok(CursorResult::ReorgDetected { + block_number, + block_hash, + parent_hash, + }) + } + + // === SUCCESS PATH === + // All blocks validated and inserted. Sleep before returning to avoid hammering RPC. + // Also handles the theoretically unreachable CursorResult::UpToDate from cursor_processing + // (cursor_processing only returns Complete or ReorgDetected, but Rust requires exhaustive match). + (Ok(cursor_result), Ok(())) => { + info!( + range_start = range_start, + range_end = range_end, + "Cursor iteration complete — all blocks validated and inserted" + ); + tokio::time::sleep(Duration::from_millis(self.loop_delay_ms)).await; + Ok(cursor_result) + } + + // === ERROR PATHS === + + // Cursor was cancelled because fetcher failed — return the ROOT CAUSE (fetcher's error). + // The cursor saw cancellation via tokio::select!, but the real problem is in the fetcher. + ( + Err(EvmListenerError::CouldNotFetchBlock { + source: BlockFetchError::Cancelled, + }), + Err(fetcher_err), + ) => { + // TODO(retry): caller decides retry strategy: + // - Unrecoverable (UnsupportedMethod, Deserialization): escalate/alert, do NOT retry blindly + // - Recoverable (transport, timeout): retry the whole iteration. + + // TODO: Emit event for next loop iteration. + error!(error = %fetcher_err, "Fetcher error caused cursor cancellation"); + Err(fetcher_err) + } + + // Cursor had a real error (DB failure, invariant violation, etc.) — return it. + // No sleep — propagate fast for retry logic. + (Err(cursor_err), _) => { + // TODO(retry): caller decides retry strategy: + // - DatabaseError: may retry after delay (DB might recover) + // - InvariantViolation: critical, needs investigation + error!(error = %cursor_err, "Cursor error during iteration"); + Err(cursor_err) + } + + // Cursor succeeded but fetcher errored — unexpected but cursor's result is authoritative. + // This shouldn't happen: if cursor completed, all slots were filled successfully. + (Ok(result), Err(fetcher_err)) => { + warn!( + error = %fetcher_err, + "Fetcher errored despite cursor succeeding — unexpected" + ); + Ok(result) + } + } + } + + /// Walks backwards from the reorg point to reconstruct the canonical chain. + /// + /// # Algorithm + /// + /// ## Phase 1 — Walk + Publish (DB read-only) + /// 1a. Fetch block N by `event.block_hash` from RPC. Publish block N's + /// events with `BlockFlow::Reorged`. Collect `NewDatabaseBlock` metadata. + /// (v1 never published block N's events — this closes the R2 gap.) + /// 1b. Walk backwards from N-1: for each height, fetch by parent_hash, + /// publish events immediately (on-the-go), collect lightweight metadata. + /// The `FetchedBlock` is dropped after publishing — only `NewDatabaseBlock` + /// (72 bytes) is retained. DB is NEVER modified. + /// Stop when fork-point found, genesis reached, or indexing boundary + + /// finality_depth exhausted. + /// + /// ## Phase 2 — Commit (single DB transaction) + /// Reverse collected blocks to ascending order. Batch-upsert all blocks + /// via `batch_upsert_blocks_canonical`. On error: transaction rolled back, + /// DB untouched, broker retries the message → clean restart. + /// + /// ## Phase 3 — Resume + /// Publish `FETCH_NEW_BLOCKS` to resume the cursor. + /// + /// # Crash safety + /// + /// - Crash during Phase 1: DB untouched → retry re-walks from scratch, + /// re-publishes (at-least-once). No false fork-point possible. + /// - Crash during Phase 2: transaction rolled back → DB untouched → same. + /// - Crash after Phase 2 commit: DB correct → retry finds DB already updated, + /// walk terminates in ~1 block, re-publishes ~1 event (at-least-once). + /// + /// # Reorg depth definition + /// + /// `reorg_depth` = number of blocks replaced, INCLUDING block N. + /// Example: reorg at height 100 with fork-point at 97 → depth = 3 + /// (blocks 100, 99, 98 replaced; block 97 is the common ancestor). + pub async fn reorg_backtrack( + &self, + event: ReorgBacktrackEvent, + ) -> Result<(), EvmListenerError> { + let block_number = event.block_number; + let chain_id_u64 = self.repositories.chain_id() as u64; + + // We don't need to dedup the message, the flow lock + ack will handle the deduplication by itself. + + let mut collected_blocks: Vec = Vec::new(); + + info!( + block_number, + block_hash = %event.block_hash, + parent_hash = %event.parent_hash, + "reorg_backtrack_v2: starting (DB read-only during walk)" + ); + + // ══════════════════════════════════════════════════════════════ + // Phase 1a: Fetch block N by hash, publish its events. + // Comes from the live flow, marked as live block. + // ══════════════════════════════════════════════════════════════ + + let cancel_token = CancellationToken::new(); + let block_n = self + .get_block_by_hash(event.block_hash, cancel_token, self.compute_block) + .await?; + + publisher::publish_block_events( + &self.repositories, + &block_n, + chain_id_u64, + // This block is coming from the live flow, and detect the reorg processing, but was issued from the live flow. + BlockFlow::Live, + &self.broker, + &self.event_publisher, + &self.publish_config, + ) + .await + .map_err(|source| EvmListenerError::PayloadBuildError { source })?; + + collected_blocks.push(NewDatabaseBlock::from_rpc_block( + &block_n.block, + BlockStatus::Canonical, + )); + + info!( + block_number, + block_hash = %event.block_hash, + "reorg_backtrack_v2: block N published, walking backwards" + ); + + // ══════════════════════════════════════════════════════════════ + // Phase 1b: Walk backwards from N-1. + // + // For each block: fetch by parent hash → publish events → collect + // lightweight metadata (NewDatabaseBlock, 72 bytes). Then check + // fork-point against DB canonical block at prev_height. + // + // DB is NEVER modified. If the process crashes here, the DB is + // in its original state and retry starts from scratch. + // ══════════════════════════════════════════════════════════════ + + let mut current_block = block_n; + let mut current_height = block_number; + let mut steps_past_db: u64 = 0; + + loop { + // Genesis guard — cannot compare parent hash below block 0. + if current_height == 0 { + warn!("reorg_backtrack_v2: reached genesis block, cannot walk further back"); + break; + } + + let prev_height = current_height - 1; + + // Fork-point detection: read-only DB check. + // Since the DB is NEVER modified during the walk, this comparison is + // ALWAYS against the original state. No false positives possible. + let db_prev_block = self + .repositories + .blocks + .get_canonical_block_by_number(prev_height) + .await + .map_err(|e| EvmListenerError::DatabaseError { source: e })?; + + match db_prev_block { + Some(prev_block) + if current_block.block.header.parent_hash == prev_block.block_hash => + { + // Fork-point found — DB block below matches our chain. + info!( + fork_point = prev_height, + reorg_depth = collected_blocks.len(), + "reorg_backtrack_v2: fork-point found" + ); + break; + } + Some(prev_block) => { + // Mismatch — reorg goes deeper. + info!( + height = prev_height, + db_hash = %prev_block.block_hash, + new_parent = %current_block.block.header.parent_hash, + "reorg_backtrack_v2: parent mismatch, walking back" + ); + } + None => { + // No DB block at previous height — indexing boundary. + steps_past_db += 1; + // We are passing db blocks only if we are under finality, with already collected blocks. + // Indeed passing collected_blocks will also considers the matching blocks into the database. + if collected_blocks.len() as u64 >= self.finality_depth { + warn!( + steps_past_db, + finality_depth = self.finality_depth, + prev_height, + "reorg_backtrack_v2: finality limit reached past DB boundary, stopping" + ); + break; + } + info!( + prev_height, + steps_past_db, + finality_depth = self.finality_depth, + "reorg_backtrack_v2: no DB block at prev height, continuing" + ); + } + } + + // Fetch parent block by hash from RPC. + let cancel_token = CancellationToken::new(); + current_block = self + .get_block_by_hash( + current_block.block.header.parent_hash, + cancel_token, + self.compute_block, + ) + .await?; + current_height = prev_height; + + // Publish events on-the-go. The FetchedBlock is dropped after this + // loop iteration — only the NewDatabaseBlock (72 bytes) is retained. + publisher::publish_block_events( + &self.repositories, + ¤t_block, + chain_id_u64, + BlockFlow::Reorged, + &self.broker, + &self.event_publisher, + &self.publish_config, + ) + .await + .map_err(|source| EvmListenerError::PayloadBuildError { source })?; + + collected_blocks.push(NewDatabaseBlock::from_rpc_block( + ¤t_block.block, + BlockStatus::Canonical, + )); + + info!( + block_number = current_height, + block_hash = %current_block.block.header.hash, + blocks_collected = collected_blocks.len(), + "reorg_backtrack_v2: published events, collected metadata" + ); + } + + // ══════════════════════════════════════════════════════════════ + // Phase 2: Batch commit — single DB transaction. + // + // Reverse to ascending order (fork+1, ..., N-1, N) and upsert all + // blocks atomically. If the transaction fails, the entire batch is + // rolled back and the DB remains in its original state. + // + // All events have already been published (Phase 1). This satisfies + // R4: publish-before-commit for every block. + // ══════════════════════════════════════════════════════════════ + + // collected_blocks is in descending order [N, N-1, ..., fork+1]. + // Reverse to ascending for conventional batch upsert ordering. + collected_blocks.reverse(); + + let reorg_depth = collected_blocks.len() as u64; + info!( + reorg_depth, + "reorg_backtrack_v2: walk complete, committing batch to DB" + ); + + let upsert_results = self + .repositories + .blocks + .batch_upsert_blocks_canonical(&collected_blocks) + .await + .map_err(|e| EvmListenerError::DatabaseError { source: e })?; + + let inserted_count = upsert_results + .iter() + .filter(|r| **r == UpsertResult::Inserted) + .count(); + let updated_count = upsert_results + .iter() + .filter(|r| **r == UpsertResult::Updated) + .count(); + let noop_count = upsert_results + .iter() + .filter(|r| **r == UpsertResult::NoOp) + .count(); + let known_branch = updated_count > 0 || noop_count > 0; + + info!( + block_number, + reorg_depth, + inserted_count, + updated_count, + noop_count, + known_branch, + "reorg_backtrack_v2: batch commit complete. {}", + if known_branch { + "Some blocks were from a previously known branch." + } else { + "All blocks were new to this node." + } + ); + + // Phase 3 (publish FETCH_NEW_BLOCKS) moved to ReorgHandler — after lock release. + + Ok(()) + } +} + +/// Sequential block validator and DB inserter (the "consumer" in the producer-consumer pattern). +/// +/// Reads blocks from the `AsyncSlotBuffer` in order (slot 0, 1, 2, ...), validates the +/// parent_hash chain against the previous block's hash, and inserts validated blocks into +/// the database as CANONICAL. +/// +/// # Parameters +/// - `listener`: Cloned `EvmListener` (owns repositories for DB access). Passed by value +/// because this runs in `tokio::spawn` which requires `'static`. +/// - `buffer`: Shared slot buffer where the producer writes fetched blocks. +/// - `cancel_token`: Shared cancellation token. Cursor checks this before each slot read +/// (via `tokio::select!`) and cancels it on reorg detection or DB failure. +/// - `expected_parent_hash`: The hash of the DB tip block. The first fetched block's +/// `parent_hash` must match this value. +/// - `range_start`: The block number of slot 0 in the buffer. +/// - `range_length`: Total number of slots to process. +/// +/// # Returns +/// - `Ok(CursorResult::Complete)` — all blocks validated and inserted +/// - `Ok(CursorResult::ReorgDetected { block_number })` — parent hash mismatch detected +/// - `Err(...)` — DB failure, buffer error, or cancellation from the fetcher side +/// +/// # Cancellation Safety +/// `buffer.get()` is cancel-safe: if `tokio::select!` drops it while awaiting `Mutex::lock`, +/// the guard drops correctly. If dropped during `Notify::notified()`, the waiter is deregistered. +async fn cursor_processing( + listener: EvmListener, + buffer: AsyncSlotBuffer, + cancel_token: CancellationToken, + expected_parent_hash: B256, + range_start: u64, + range_length: usize, +) -> Result { + let mut current_expected_hash = expected_parent_hash; + + for i in 0..range_length { + let block_number = range_start + i as u64; + + // Wait for the block with cancellation guard. + // buffer.get() blocks forever if the slot is never filled (e.g., fetcher cancelled), + // so we race it against the cancellation token. + // `biased` ensures cancellation is always checked first to prevent processing + // stale data after cancellation is signaled. + let fetched_block = tokio::select! { + biased; + _ = cancel_token.cancelled() => { + return Err(EvmListenerError::CouldNotFetchBlock { + source: BlockFetchError::Cancelled, + }); + } + block_opt = buffer.get(i) => { + block_opt.ok_or(EvmListenerError::SlotBufferError { + source: BufferError::IndexOutOfBounds, + })? + } + }; + + let block_hash = fetched_block.block.header.hash; + let parent_hash = fetched_block.block.header.parent_hash; + + // Validate the parent hash chain: this block's parent must be the previous block's hash. + // For slot 0, current_expected_hash is the DB tip's hash. + // For subsequent slots, it's the hash of the block we just inserted. + if parent_hash != current_expected_hash { + // REORG DETECTED: the chain has diverged from our canonical view. + // Cancel the fetcher to stop wasting RPC calls on a now-invalid range. + cancel_token.cancel(); + warn!( + block_number = block_number, + expected_parent = %current_expected_hash, + actual_parent = %parent_hash, + block_hash = %block_hash, + slot = i, + "Reorg detected: parent hash mismatch" + ); + return Ok(CursorResult::ReorgDetected { + block_number, + block_hash, + parent_hash, + }); + } + + // Hash chain is valid — publish events BEFORE inserting block. + // Events MUST be delivered before the block is registered in DB. + // If publish fails: block is NOT inserted, DB tip unchanged, + // cursor retries this exact block on next iteration. Zero missed events. + let chain_id_u64 = listener.repositories.chain_id() as u64; + publisher::publish_block_events( + &listener.repositories, + &fetched_block, + chain_id_u64, + BlockFlow::Live, + &listener.broker, + &listener.event_publisher, + &listener.publish_config, + ) + .await + .map_err(|source| { + // Stop fetcher on publish failure — no point fetching more blocks + cancel_token.cancel(); + EvmListenerError::PayloadBuildError { source } + })?; + + // Safe to insert now — events have been delivered to all consumers. + let new_db_block = + NewDatabaseBlock::from_rpc_block(&fetched_block.block, BlockStatus::Canonical); + + listener + .repositories + .blocks + .insert_block(&new_db_block) + .await + .map_err(|e| { + // Stop fetcher on DB failure — no point fetching more blocks if we can't store them + cancel_token.cancel(); + EvmListenerError::DatabaseError { source: e } + })?; + + info!( + block_number = block_number, + block_hash = %block_hash, + tx_count = fetched_block.transaction_count(), + slot = i + 1, + total = range_length, + "Block validated and inserted as canonical" + ); + + // Update expected hash for next iteration: the NEXT block's parent must be THIS block's hash + current_expected_hash = block_hash; + } + + Ok(CursorResult::Complete) +} + +/// Parallel block fetcher (the "producer" in the producer-consumer pattern). +/// +/// Spawns one `tokio::spawn` task per block in the range. Each task fetches a block via RPC +/// (using the listener's configured strategy with infinite retry on recoverable errors) and +/// stores the result in the corresponding slot of the `AsyncSlotBuffer`. +/// +/// # Parameters +/// - `listener`: Cloned `EvmListener` (owns provider, strategy, compute_block config). +/// Passed by value because this runs in `tokio::spawn`. +/// - `buffer`: Shared slot buffer. Each task writes to exactly one slot (index = position in range). +/// - `cancel_token`: Shared cancellation token. On error, this function cancels the token to +/// stop both the cursor and any remaining fetch tasks. +/// - `range_start`: The block number corresponding to slot 0. +/// - `range_length`: Total number of blocks to fetch. +/// +/// # Error Handling +/// - `get_block_by_number` has infinite retry built-in for recoverable errors (transport, timeout). +/// Only unrecoverable errors (UnsupportedMethod, DeserializationError) or cancellation bubble up. +/// - On the first error from any task, `cancel_token` is cancelled and remaining tasks are drained. +/// - Task panics (JoinError) are treated as cancellation errors. +async fn fetch_blocks_in_parallel( + listener: EvmListener, + buffer: AsyncSlotBuffer, + cancel_token: CancellationToken, + range_start: u64, + range_length: usize, +) -> Result<(), EvmListenerError> { + let compute_block = listener.compute_block; + let mut join_set: JoinSet> = JoinSet::new(); + + for i in 0..range_length { + let block_number = range_start + i as u64; + let listener = listener.clone(); + let buffer = buffer.clone(); + // Child token: cancelled when parent cancel_token is cancelled + let child_token = cancel_token.child_token(); + + join_set.spawn(async move { + let fetch_start = Instant::now(); + let fetched_block = listener + .get_block_by_number(block_number, child_token, compute_block) + .await?; + + metrics::histogram!( + "listener_block_fetch_duration_seconds", + "chain_id" => listener.chain_id.to_string() + ) + .record(fetch_start.elapsed().as_secs_f64()); + + // Store the fetched block in the corresponding slot. + // set_once ensures each slot is written exactly once — AlreadyFilled indicates a logic bug. + buffer.set_once(i, fetched_block).await.map_err(|e| { + error!( + slot = i, + block_number = block_number, + error = %e, + "Buffer slot already filled — this is a logic bug" + ); + EvmListenerError::SlotBufferError { source: e } + }) + }); + } + + // Drain JoinSet: propagate first error, cancel remaining tasks. + // This follows the established pattern from evm_block_fetcher.rs. + while let Some(result) = join_set.join_next().await { + match result { + // Task completed successfully — slot was filled + Ok(Ok(())) => continue, + + // Task returned an error — cancel all remaining and propagate + Ok(Err(e)) => { + cancel_token.cancel(); + // Drain remaining tasks to avoid abandoned futures + while join_set.join_next().await.is_some() {} + return Err(e); + } + + // Task panicked (JoinError) — cancel all remaining, treat as cancellation + Err(join_err) => { + cancel_token.cancel(); + while join_set.join_next().await.is_some() {} + error!(error = %join_err, "Fetch task panicked"); + return Err(EvmListenerError::CouldNotFetchBlock { + source: BlockFetchError::Cancelled, + }); + } + } + } + + Ok(()) +} diff --git a/listener/crates/listener_core/src/core/evm_listener.rs.orig b/listener/crates/listener_core/src/core/evm_listener.rs.orig new file mode 100644 index 0000000000..dd3ff56c90 --- /dev/null +++ b/listener/crates/listener_core/src/core/evm_listener.rs.orig @@ -0,0 +1,1281 @@ +use std::time::Duration; + +use alloy::primitives::B256; +use primitives::utils::chain_id_to_namespace; +use thiserror::Error; +use tokio::task::JoinSet; +use tokio_util::sync::CancellationToken; +use tracing::{debug, error, info, warn}; + +<<<<<<< HEAD +use broker::{Broker, Publisher, Topic}; +======= +use broker::{Broker, Publisher}; +>>>>>>> c4fb34a (chore(core): publisher, with is exists strategy, and republish staled process) + +use primitives::routing; + +use primitives::event::{BlockFlow, ReorgBacktrackEvent}; + +use super::publisher::{self, PublisherError}; +use crate::config::{BlockchainConfig, PublishConfig}; +use crate::store::SqlError; +use crate::{ + blockchain::evm::{ + evm_block_fetcher::{BlockFetchError, EvmBlockFetcher, FetchedBlock}, + sem_evm_rpc_provider::{RpcProviderError, SemEvmRpcProvider}, + }, + config::BlockFetcherStrategy, + core::slot_buffer::{AsyncSlotBuffer, BufferError}, + store::models::{BlockStatus, NewDatabaseBlock, UpsertResult}, + store::repositories::Repositories, +}; + +/// Errors that can occur during EvmListener operations. +#[derive(Error, Debug)] +pub enum EvmListenerError { + #[error("Could not fetch block: {source}")] + CouldNotFetchBlock { + #[source] + source: BlockFetchError, + }, + + #[error("Could not compute block: {source}")] + CouldNotComputeBlock { + #[source] + source: BlockFetchError, + }, + + #[error("Database error: {source}")] + DatabaseError { + #[source] + source: SqlError, + }, + + #[error("Could not fetch chain height: {source}")] + ChainHeightError { + #[source] + source: RpcProviderError, + }, + + #[error("Buffer error: {source}")] + SlotBufferError { + #[source] + source: BufferError, + }, + + #[error("Broker publish error: {message}")] + BrokerPublishError { message: String }, + + #[error("Payload build error: {source}")] + PayloadBuildError { + #[source] + source: PublisherError, + }, + + #[error("Invariant violation: {message}")] + InvariantViolation { message: String }, + + #[error("Message processing error: {message}")] + MessageProcessingError { message: String }, +} + +/// Outcome of a single cursor iteration. +/// +/// Reorgs are a normal operational event on blockchains (not an error), +/// so they are represented as a distinct result variant rather than an error. +#[derive(Debug)] +pub enum CursorResult { + /// All blocks in the batch were validated and inserted as canonical. + Complete, + /// Chain hasn't advanced beyond the DB tip. Nothing to fetch. + UpToDate, + /// Reorg detected: the parent hash of the block at this number + /// did not match the expected hash from the previous canonical block. + ReorgDetected { + block_number: u64, + block_hash: B256, + parent_hash: B256, + }, + /// Skip task to avoid deduplication + Skip, +} + +#[derive(Clone)] +pub struct EvmListener { + provider: SemEvmRpcProvider, + repositories: Repositories, + publisher: Publisher, + broker: Broker, + event_publisher: Publisher, +<<<<<<< HEAD + publish_config: PublishConfig, + chain_id: u64, +======= +>>>>>>> c4fb34a (chore(core): publisher, with is exists strategy, and republish staled process) + range_size: usize, + fetcher_strategy: BlockFetcherStrategy, + batch_receipts_size_range: Option, + compute_block: bool, + loop_delay_ms: u64, + finality_depth: u64, + max_exponential_backoff_ms: u64, +} + +impl EvmListener { + pub fn new( + provider: SemEvmRpcProvider, + repositories: Repositories, + publisher: Publisher, + broker: Broker, + event_publisher: Publisher, +<<<<<<< HEAD + blockchain_settings: &BlockchainConfig, +======= + strategy: &StrategyConfig, + finality_depth: u64, +>>>>>>> c4fb34a (chore(core): publisher, with is exists strategy, and republish staled process) + ) -> Self { + let compute_block = blockchain_settings + .strategy + .compute_block + .unwrap_or_default(); + + Self { + provider, + repositories, + publisher, + broker, + event_publisher, +<<<<<<< HEAD + publish_config: blockchain_settings.strategy.publish.clone(), + chain_id: blockchain_settings.chain_id, + range_size: blockchain_settings.strategy.range_size, + fetcher_strategy: blockchain_settings.strategy.block_fetcher.clone(), + batch_receipts_size_range: Some(blockchain_settings.strategy.batch_receipts_size_range), +======= + range_size: strategy.range_size, + fetcher_strategy: strategy.block_fetcher.clone(), + batch_receipts_size_range: Some(strategy.batch_receipts_size_range), +>>>>>>> c4fb34a (chore(core): publisher, with is exists strategy, and republish staled process) + compute_block, + loop_delay_ms: blockchain_settings.strategy.loop_delay_ms, + finality_depth: blockchain_settings.finality_depth, + max_exponential_backoff_ms: blockchain_settings.strategy.max_exponential_backoff_ms, + } + } + + /// Create a fetcher with the given cancellation token and compute_block flag. + fn create_fetcher( + &self, + cancel_token: CancellationToken, + compute_block: bool, + ) -> EvmBlockFetcher { + EvmBlockFetcher::new(self.provider.clone()) + .with_verify_block(compute_block) + .with_cancellation_token(cancel_token) + .with_max_exponential_backoff_ms(self.max_exponential_backoff_ms) + } + + /// Map a BlockFetchError to an EvmListenerError. + /// + /// VerificationFailed errors map to CouldNotComputeBlock, + /// all other errors map to CouldNotFetchBlock. + fn map_fetch_error(error: BlockFetchError) -> EvmListenerError { + match error { + BlockFetchError::VerificationFailed(_) => { + EvmListenerError::CouldNotComputeBlock { source: error } + } + _ => EvmListenerError::CouldNotFetchBlock { source: error }, + } + } + + /// Fetch a block by its number using the configured strategy. + /// + /// # Arguments + /// * `block_number` - The block number to fetch + /// * `cancel_token` - Cancellation token for this operation (share across batch for batch cancellation) + /// * `compute_block` - Whether to verify block integrity (transaction root, receipt root, block hash) + /// + /// # Cancellation + /// When `cancel_token.cancel()` is called, this operation will return `BlockFetchError::Cancelled`. + /// All parallel operations sharing the same token will be cancelled together. + pub async fn get_block_by_number( + &self, + block_number: u64, + cancel_token: CancellationToken, + compute_block: bool, + ) -> Result { + let fetcher = self.create_fetcher(cancel_token, compute_block); + + let result = match self.fetcher_strategy { + BlockFetcherStrategy::BlockReceipts => { + fetcher + .fetch_block_with_block_receipts_by_number(block_number) + .await + } + BlockFetcherStrategy::BatchReceiptsFull => { + fetcher + .fetch_block_with_batch_receipts_by_number(block_number) + .await + } + BlockFetcherStrategy::BatchReceiptsRange => { + let batch_size = self.batch_receipts_size_range.unwrap_or(1); + fetcher + .fetch_block_by_number_with_parallel_batched_receipts(block_number, batch_size) + .await + } + BlockFetcherStrategy::TransactionReceiptsParallel => { + fetcher + .fetch_block_with_individual_receipts_by_number(block_number) + .await + } + BlockFetcherStrategy::TransactionReceiptsSequential => { + fetcher + .fetch_block_with_sequential_receipts_by_number(block_number) + .await + } + }; + + result.map_err(Self::map_fetch_error) + } + + /// Fetch a block by its hash using the configured strategy. + /// + /// # Arguments + /// * `hash` - The block hash to fetch + /// * `cancel_token` - Cancellation token for this operation + /// * `compute_block` - Whether to verify block integrity (transaction root, receipt root, block hash) + /// + /// # Cancellation + /// When `cancel_token.cancel()` is called, this operation will return `BlockFetchError::Cancelled`. + pub async fn get_block_by_hash( + &self, + hash: B256, + cancel_token: CancellationToken, + compute_block: bool, + ) -> Result { + let fetcher = self.create_fetcher(cancel_token, compute_block); + + let result = match self.fetcher_strategy { + BlockFetcherStrategy::BlockReceipts => { + fetcher.fetch_block_with_block_receipts_by_hash(hash).await + } + BlockFetcherStrategy::BatchReceiptsFull => { + fetcher.fetch_block_with_batch_receipts_by_hash(hash).await + } + BlockFetcherStrategy::BatchReceiptsRange => { + let batch_size = self.batch_receipts_size_range.unwrap_or(1); + fetcher + .fetch_block_by_hash_with_parallel_batched_receipts(hash, batch_size) + .await + } + BlockFetcherStrategy::TransactionReceiptsParallel => { + fetcher + .fetch_block_with_individual_receipts_by_hash(hash) + .await + } + BlockFetcherStrategy::TransactionReceiptsSequential => { + fetcher + .fetch_block_with_sequential_receipts_by_hash(hash) + .await + } + }; + + result.map_err(Self::map_fetch_error) + } + + /// Validates the block fetching strategy and initializes the database with a starting block. + /// + /// This function is called during service initialization and will panic on any failure. + /// The rationale is that if initialization fails, the process cannot operate correctly + /// and should crash to trigger a restart (crash-loop-backoff pattern). + /// + /// # Panics + /// - If the RPC node is unreachable or returns an error + /// - If the configured strategy is not compatible with the RPC node + /// - If the database is unreachable or returns an error + /// - If the starting block cannot be fetched or inserted + pub async fn validate_strategy_and_init_block(&self, block_start: u64) { + // Fetch latest block number from blockchain using SemEvmRpcProvider + let latest_block_number = match self.provider.get_block_number().await { + Ok(block_number) => block_number, + Err(e) => { + tracing::error!( + error = %e, + "CRITICAL: Could not fetch latest block number from RPC" + ); + panic!("Could not fetch latest block number from RPC: {}", e); + } + }; + + tracing::info!( + block_number = latest_block_number, + "Fetched latest block number from blockchain" + ); + + // Validate strategy by attempting to fetch the latest block + let cancel_token = CancellationToken::new(); + match self + .get_block_by_number(latest_block_number, cancel_token, self.compute_block) + .await + { + Ok(fetched_block) => { + tracing::info!( + block_number = latest_block_number, + block_hash = %fetched_block.block.header.hash, + tx_count = fetched_block.transaction_count(), + strategy = ?self.fetcher_strategy, + "Strategy validation successful" + ); + } + Err(e) => { + tracing::error!( + error = %e, + strategy = ?self.fetcher_strategy, + "CRITICAL: Strategy validation failed - binary should not start" + ); + panic!( + "Strategy validation failed: {}. The configured strategy {:?} is not compatible with the RPC node.", + e, self.fetcher_strategy + ); + } + } + + // Check for existing canonical block in the database + let latest_db_block = match self.repositories.blocks.get_latest_canonical_block().await { + Ok(block) => block, + Err(e) => { + tracing::error!( + error = %e, + "CRITICAL: Could not query database for latest canonical block" + ); + panic!("Could not query database for latest canonical block: {}", e); + } + }; + + if let Some(block) = latest_db_block { + tracing::info!( + block_number = block.block_number, + block_hash = %block.block_hash, + "Found existing canonical block in database, no initialization needed" + ); + return; + } + + tracing::info!( + block_start = block_start, + "No canonical block in database, initializing with block_start" + ); + + // Fetch the starting block from the blockchain + let cancel_token = CancellationToken::new(); + let fetched_block = match self + .get_block_by_number(block_start, cancel_token, self.compute_block) + .await + { + Ok(block) => block, + Err(e) => { + tracing::error!( + error = %e, + block_start = block_start, + "CRITICAL: Could not fetch starting block from blockchain" + ); + panic!( + "Could not fetch starting block {} from blockchain: {}", + block_start, e + ); + } + }; + + // Convert FetchedBlock to NewDatabaseBlock and insert into database + let new_db_block = + NewDatabaseBlock::from_rpc_block(&fetched_block.block, BlockStatus::Canonical); + + if let Err(e) = self.repositories.blocks.insert_block(&new_db_block).await { + tracing::error!( + error = %e, + block_number = new_db_block.block_number, + block_hash = %new_db_block.block_hash, + "CRITICAL: Could not insert starting block into database" + ); + panic!("Could not insert starting block into database: {}", e); + } + + tracing::info!( + block_number = new_db_block.block_number, + block_hash = %new_db_block.block_hash, + "Successfully initialized database with starting block" + ); + } + + /// Orchestrates one iteration of the V2 cursor algorithm. + /// + /// This is the main entry point for block fetching. It: + /// 1. Reads the DB tip (latest canonical block) + /// 2. Gets the current chain height from the RPC node + /// 3. Calculates the range of blocks to fetch (capped by `batch_size`) + /// 4. Spawns two concurrent tasks via `tokio::spawn`: + /// - **Producer** (`fetch_blocks_in_parallel`): fetches blocks via parallel RPC calls, + /// fills an `AsyncSlotBuffer` out-of-order + /// - **Consumer** (`cursor_processing`): reads slots sequentially, validates the + /// parent_hash chain, inserts validated blocks into the DB + /// 5. Awaits both tasks and analyzes outcomes + /// + /// # Returns + /// - `Ok(CursorResult::Complete)` — all blocks validated and inserted; sleeps `loop_delay_ms` before returning + /// - `Ok(CursorResult::UpToDate)` — chain hasn't advanced, nothing to fetch + /// - `Ok(CursorResult::ReorgDetected { block_number })` — reorg detected at this height (no sleep) + /// - `Err(...)` — unrecoverable or transient error (no sleep, propagate fast for retry logic) + /// + /// # Cancellation + /// A fresh `CancellationToken` is created per iteration. It is shared between the producer + /// and consumer. Either side can cancel the other: + /// - Cursor cancels on reorg detection or DB failure + /// - Fetcher cancellation propagates to cursor via the shared token + /// + /// # Panics + /// This function does not panic. Task panics are caught via `JoinError` and converted + /// to `EvmListenerError::InvariantViolation`. + pub async fn fetch_blocks_and_run_cursor(&self) -> Result { + // Pre verification: + // The prefetch count set up to 1 allows us to simply verify emptiness of the queue/stream, to ensure there is no possibility of overlapping + // messages. + // If there is another message sitting in the queue, we can basically skip the message and avoid consuming it, by directly acknowledging. + // TODO: This is a first security, but not compliant with horizontal scaling and needs to be locked. + let fetch_topic = Topic::new(routing::FETCH_NEW_BLOCKS) + .with_namespace(chain_id_to_namespace(self.chain_id)); + let task_lock = self + .broker + .is_empty_or_pending(&fetch_topic, routing::FETCH_NEW_BLOCKS) + .await + .map_err(|e| { + error!(error = %e, "Failed to check message processing lock on cursor process."); + EvmListenerError::MessageProcessingError { + message: "Couldn't check message before processing".to_string(), + } + })?; + + let backtrack_topic = Topic::new(routing::BACKTRACK_REORG) + .with_namespace(chain_id_to_namespace(self.chain_id)); + let backtrack_task_lock = self + .broker + .is_empty(&backtrack_topic, routing::BACKTRACK_REORG) + .await + .map_err(|e| { + error!(error = %e, "Failed to check message processing lock on cursor process."); + EvmListenerError::MessageProcessingError { + message: "Couldn't check message before processing".to_string(), + } + })?; + + if !task_lock || !backtrack_task_lock { + warn!("Cursor: Duplicate message in the queue, skipping"); + return Ok(CursorResult::Skip); + } + + // Step 1: Get the latest canonical block from DB (the "tip" of our chain view). + // validate_strategy_and_init_block guarantees at least one block exists. + let db_block = self + .repositories + .blocks + .get_latest_canonical_block() + .await + .map_err(|e| EvmListenerError::DatabaseError { source: e })? + .ok_or_else(|| EvmListenerError::InvariantViolation { + message: "No canonical block in database. \ + validate_strategy_and_init_block must run before the cursor." + .to_string(), + })?; + + let db_block_number = db_block.block_number; + let db_block_hash = db_block.block_hash; + + debug!( + db_block_number = db_block_number, + db_block_hash = %db_block_hash, + "Retrieved DB tip for cursor iteration" + ); + + // Step 2: Get the current chain height from the RPC node + let chain_height = self + .provider + .get_block_number() + .await + .map_err(|e| EvmListenerError::ChainHeightError { source: e })?; + + // Step 3: If the chain hasn't advanced beyond our DB tip, there's nothing to do + if chain_height <= db_block_number { + debug!( + chain_height = chain_height, + db_block_number = db_block_number, + "Chain has not advanced beyond DB tip" + ); + tokio::time::sleep(Duration::from_millis(self.loop_delay_ms)).await; + // Publish wake-up signal to trigger next fetch iteration + self.publisher + .publish(routing::FETCH_NEW_BLOCKS, &serde_json::Value::Null) + .await + .map_err(|e| { + error!(error = %e, "Failed to publish fetch trigger"); + EvmListenerError::BrokerPublishError { + message: format!("Broker publish failed: {}", e), + } + })?; + return Ok(CursorResult::UpToDate); + } + + // Step 4: Calculate the inclusive range [range_start, range_end] + // Example: db=100, batch=10, chain=500 -> start=101, end=110, length=10 (blocks 101..=110) + // Example: db=100, batch=10, chain=103 -> start=101, end=103, length=3 (only 3 available) + let range_start = db_block_number + 1; + let range_end = std::cmp::min(chain_height, db_block_number + self.range_size as u64); + let range_length = (range_end - range_start + 1) as usize; // +1 because both bounds are inclusive + + info!( + range_start = range_start, + range_end = range_end, + range_length = range_length, + chain_height = chain_height, + "Starting cursor iteration" + ); + + // Step 5: Create shared state for producer-consumer coordination + let buffer = AsyncSlotBuffer::::new(range_length); + let cancel_token = CancellationToken::new(); + + // Step 6: Spawn both tasks concurrently + // - cursor_processing (consumer): reads slots sequentially, validates hash chain, inserts to DB + // - fetch_blocks_in_parallel (producer): fetches blocks via parallel RPC calls, fills slots + let cursor_handle = tokio::spawn(cursor_processing( + self.clone(), + buffer.clone(), + cancel_token.clone(), + db_block_hash, + range_start, + range_length, + )); + + let fetcher_handle = tokio::spawn(fetch_blocks_in_parallel( + self.clone(), + buffer, + cancel_token.clone(), + range_start, + range_length, + )); + + // Step 7: Await both tasks to completion. + // tokio::join! ensures neither task is abandoned even if the other completes/fails first. + // On cancellation, in-flight HTTP requests may take up to 10s (RPC timeout) before + // tasks finish. This is acceptable. + let (cursor_join_result, fetcher_join_result) = tokio::join!(cursor_handle, fetcher_handle); + + // Step 8: Unwrap JoinHandle results — a JoinError means the task panicked, + // which is a critical bug (we never panic in our code). + let cursor_outcome = cursor_join_result.map_err(|join_err| { + cancel_token.cancel(); + error!(error = %join_err, "Cursor task panicked — this is a critical bug"); + EvmListenerError::InvariantViolation { + message: format!("Cursor task panicked: {}", join_err), + } + })?; + + let fetcher_outcome = fetcher_join_result.map_err(|join_err| { + cancel_token.cancel(); + error!(error = %join_err, "Fetcher task panicked — this is a critical bug"); + EvmListenerError::InvariantViolation { + message: format!("Fetcher task panicked: {}", join_err), + } + })?; + + // NOTE: redo this, with clear error path and skips (Due to message broker mostly) + // Step 9: Analyze outcomes — cursor takes priority since it's the authority + // on what actually made it into the DB. + // Arms are ordered: reorg first (no sleep), then success (sleep), then errors (no sleep). + match (cursor_outcome, fetcher_outcome) { + // === REORG PATH === + // Fetcher error (Cancelled) is expected here since cursor cancelled the token. + // No sleep — reorg should be handled immediately. + ( + Ok(CursorResult::ReorgDetected { + block_number, + block_hash, + parent_hash, + }), + _, + ) => { + // Publish the backtrack event — no DB mutation here. + // Handled in reorg_backtrack function + let event = ReorgBacktrackEvent { + block_number, + block_hash, + parent_hash, + }; + self.publisher + .publish(routing::BACKTRACK_REORG, &event) + .await + .map_err(|e| { + error!(error = %e, "Failed to publish backtrack event"); + EvmListenerError::BrokerPublishError { + message: format!("Broker publish failed: {}", e), + } + })?; + + info!( + block_number = block_number, + block_hash = %block_hash, + "Reorg detected — backtrack event published" + ); + + Ok(CursorResult::ReorgDetected { + block_number, + block_hash, + parent_hash, + }) + } + + // === SUCCESS PATH === + // All blocks validated and inserted. Sleep before returning to avoid hammering RPC. + // Also handles the theoretically unreachable CursorResult::UpToDate from cursor_processing + // (cursor_processing only returns Complete or ReorgDetected, but Rust requires exhaustive match). + (Ok(cursor_result), Ok(())) => { + info!( + range_start = range_start, + range_end = range_end, + "Cursor iteration complete — all blocks validated and inserted" + ); + tokio::time::sleep(Duration::from_millis(self.loop_delay_ms)).await; + // Publish wake-up signal to trigger next fetch iteration + self.publisher + .publish(routing::FETCH_NEW_BLOCKS, &serde_json::Value::Null) + .await + .map_err(|e| { + error!(error = %e, "Failed to publish fetch trigger"); + EvmListenerError::BrokerPublishError { + message: format!("Broker publish failed: {}", e), + } + })?; + Ok(cursor_result) + } + + // === ERROR PATHS === + + // TODO: redo all this error path. + // Cursor was cancelled because fetcher failed — return the ROOT CAUSE (fetcher's error). + // The cursor saw cancellation via tokio::select!, but the real problem is in the fetcher. + ( + Err(EvmListenerError::CouldNotFetchBlock { + source: BlockFetchError::Cancelled, + }), + Err(fetcher_err), + ) => { + // TODO(retry): caller decides retry strategy: + // - Unrecoverable (UnsupportedMethod, Deserialization): escalate/alert, do NOT retry blindly + // - Recoverable (transport, timeout): retry the whole iteration. + + // TODO: Emit event for next loop iteration. + error!(error = %fetcher_err, "Fetcher error caused cursor cancellation"); + Err(fetcher_err) + } + + // Cursor had a real error (DB failure, invariant violation, etc.) — return it. + // No sleep — propagate fast for retry logic. + (Err(cursor_err), _) => { + // TODO(retry): caller decides retry strategy: + // - DatabaseError: may retry after delay (DB might recover) + // - InvariantViolation: critical, needs investigation + error!(error = %cursor_err, "Cursor error during iteration"); + Err(cursor_err) + } + + // Cursor succeeded but fetcher errored — unexpected but cursor's result is authoritative. + // This shouldn't happen: if cursor completed, all slots were filled successfully. + (Ok(result), Err(fetcher_err)) => { + warn!( + error = %fetcher_err, + "Fetcher errored despite cursor succeeding — unexpected" + ); + Ok(result) + } + } + } + + /// Walks backwards from the reorg point to reconstruct the canonical chain. + /// + /// # Algorithm + /// + /// ## Phase 1 — Walk + Publish (DB read-only) + /// 1a. Fetch block N by `event.block_hash` from RPC. Publish block N's + /// events with `BlockFlow::Reorged`. Collect `NewDatabaseBlock` metadata. + /// (v1 never published block N's events — this closes the R2 gap.) + /// 1b. Walk backwards from N-1: for each height, fetch by parent_hash, + /// publish events immediately (on-the-go), collect lightweight metadata. + /// The `FetchedBlock` is dropped after publishing — only `NewDatabaseBlock` + /// (72 bytes) is retained. DB is NEVER modified. + /// Stop when fork-point found, genesis reached, or indexing boundary + + /// finality_depth exhausted. + /// + /// ## Phase 2 — Commit (single DB transaction) + /// Reverse collected blocks to ascending order. Batch-upsert all blocks + /// via `batch_upsert_blocks_canonical`. On error: transaction rolled back, + /// DB untouched, broker retries the message → clean restart. + /// + /// ## Phase 3 — Resume + /// Publish `FETCH_NEW_BLOCKS` to resume the cursor. + /// + /// # Crash safety + /// + /// - Crash during Phase 1: DB untouched → retry re-walks from scratch, + /// re-publishes (at-least-once). No false fork-point possible. + /// - Crash during Phase 2: transaction rolled back → DB untouched → same. + /// - Crash after Phase 2 commit: DB correct → retry finds DB already updated, + /// walk terminates in ~1 block, re-publishes ~1 event (at-least-once). + /// + /// # Reorg depth definition + /// + /// `reorg_depth` = number of blocks replaced, INCLUDING block N. + /// Example: reorg at height 100 with fork-point at 97 → depth = 3 + /// (blocks 100, 99, 98 replaced; block 97 is the common ancestor). + pub async fn reorg_backtrack( + &self, + event: ReorgBacktrackEvent, + ) -> Result<(), EvmListenerError> { + let block_number = event.block_number; + let chain_id_u64 = self.repositories.chain_id() as u64; + + // pre-check: Skipping duplicate messages. + // TODO: This is a first security, but not compliant with horizontal scaling and needs to be locked. + let reorg_topic = Topic::new(routing::BACKTRACK_REORG) + .with_namespace(chain_id_to_namespace(self.chain_id)); + let task_lock = self + .broker + .is_empty_or_pending(&reorg_topic, routing::BACKTRACK_REORG) + .await + .map_err(|e| { + error!(error = %e, "Failed to check message processing lock on reorg process."); + EvmListenerError::MessageProcessingError { + message: "Couldn't check message before processing".to_string(), + } + })?; + + if !task_lock { + warn!("Reorg: Duplicate message in the queue, skipping"); + return Ok(()); + } + + let mut collected_blocks: Vec = Vec::new(); + + info!( + block_number, + block_hash = %event.block_hash, + parent_hash = %event.parent_hash, + "reorg_backtrack_v2: starting (DB read-only during walk)" + ); + + // ══════════════════════════════════════════════════════════════ + // Phase 1a: Fetch block N by hash, publish its events. + // Comes from the live flow, marked as live block. + // ══════════════════════════════════════════════════════════════ + + let cancel_token = CancellationToken::new(); + let block_n = self + .get_block_by_hash(event.block_hash, cancel_token, self.compute_block) + .await?; + + publisher::publish_block_events( + &self.repositories, + &block_n, + chain_id_u64, + // This block is coming from the live flow, and detect the reorg processing, but was issued from the live flow. + BlockFlow::Live, + &self.broker, + &self.event_publisher, + &self.publish_config, + ) + .await + .map_err(|source| EvmListenerError::PayloadBuildError { source })?; + + collected_blocks.push(NewDatabaseBlock::from_rpc_block( + &block_n.block, + BlockStatus::Canonical, + )); + + info!( + block_number, + block_hash = %event.block_hash, + "reorg_backtrack_v2: block N published, walking backwards" + ); + + // ══════════════════════════════════════════════════════════════ + // Phase 1b: Walk backwards from N-1. + // + // For each block: fetch by parent hash → publish events → collect + // lightweight metadata (NewDatabaseBlock, 72 bytes). Then check + // fork-point against DB canonical block at prev_height. + // + // DB is NEVER modified. If the process crashes here, the DB is + // in its original state and retry starts from scratch. + // ══════════════════════════════════════════════════════════════ + + let mut current_block = block_n; + let mut current_height = block_number; + let mut steps_past_db: u64 = 0; + + loop { +<<<<<<< HEAD + // Genesis guard — cannot compare parent hash below block 0. +======= + // Publish events BEFORE upserting block — same atomicity guarantee as cursor path. + // On retry: reorg_backtrack restarts from block N, re-walks, + // re-publishes (at-least-once), re-upserts (idempotent NoOp). + let chain_id_u64 = self.repositories.chain_id() as u64; + publisher::publish_block_events( + &self.repositories, + ¤t_block, + chain_id_u64, + BlockFlow::Reorged, + &self.broker, + &self.event_publisher, + ) + .await + .map_err(|e| EvmListenerError::PublishError { source: e })?; + + // Now safe to upsert — events delivered. + // Upsert as canonical → marks old block at same height as UNCLE. + let new_db_block = + NewDatabaseBlock::from_rpc_block(¤t_block.block, BlockStatus::Canonical); + + let upsert_result = self + .repositories + .blocks + .upsert_block_canonical(&new_db_block) + .await + .map_err(|e| EvmListenerError::DatabaseError { source: e })?; + + match upsert_result { + UpsertResult::Inserted => { + inserted_count += 1; + info!( + block_number = current_height, + block_hash = %current_block.block.header.hash, + "Backtrack: inserted new canonical block" + ); + } + UpsertResult::Updated => { + updated_count += 1; + info!( + block_number = current_height, + block_hash = %current_block.block.header.hash, + "Backtrack: promoted existing block to canonical (known branch)" + ); + } + UpsertResult::NoOp => { + // This operation should never happen. + noop_count += 1; + error!( + block_number = current_height, + "Backtrack: block already canonical, no-op" + ); + } + } + + // NOTE: Genesis guard — cannot compare parent hash below block 0. + // This should never happen in practice (finality depth and indexing + // boundary checks will stop the walk long before genesis). +>>>>>>> c4fb34a (chore(core): publisher, with is exists strategy, and republish staled process) + if current_height == 0 { + warn!("reorg_backtrack_v2: reached genesis block, cannot walk further back"); + break; + } + + let prev_height = current_height - 1; + + // Fork-point detection: read-only DB check. + // Since the DB is NEVER modified during the walk, this comparison is + // ALWAYS against the original state. No false positives possible. + let db_prev_block = self + .repositories + .blocks + .get_canonical_block_by_number(prev_height) + .await + .map_err(|e| EvmListenerError::DatabaseError { source: e })?; + + match db_prev_block { + Some(prev_block) + if current_block.block.header.parent_hash == prev_block.block_hash => + { + // Fork-point found — DB block below matches our chain. + info!( + fork_point = prev_height, + reorg_depth = collected_blocks.len(), + "reorg_backtrack_v2: fork-point found" + ); + break; + } + Some(prev_block) => { + // Mismatch — reorg goes deeper. + info!( + height = prev_height, + db_hash = %prev_block.block_hash, + new_parent = %current_block.block.header.parent_hash, + "reorg_backtrack_v2: parent mismatch, walking back" + ); + } + None => { + // No DB block at previous height — indexing boundary. + steps_past_db += 1; + // We are passing db blocks only if we are under finality, with already collected blocks. + // Indeed passing collected_blocks will also considers the matching blocks into the database. + if collected_blocks.len() as u64 >= self.finality_depth { + warn!( + steps_past_db, + finality_depth = self.finality_depth, + prev_height, + "reorg_backtrack_v2: finality limit reached past DB boundary, stopping" + ); + break; + } + info!( + prev_height, + steps_past_db, + finality_depth = self.finality_depth, + "reorg_backtrack_v2: no DB block at prev height, continuing" + ); + } + } + + // Fetch parent block by hash from RPC. + let cancel_token = CancellationToken::new(); + current_block = self + .get_block_by_hash( + current_block.block.header.parent_hash, + cancel_token, + self.compute_block, + ) + .await?; + current_height = prev_height; + + // Publish events on-the-go. The FetchedBlock is dropped after this + // loop iteration — only the NewDatabaseBlock (72 bytes) is retained. + publisher::publish_block_events( + &self.repositories, + ¤t_block, + chain_id_u64, + BlockFlow::Reorged, + &self.broker, + &self.event_publisher, + &self.publish_config, + ) + .await + .map_err(|source| EvmListenerError::PayloadBuildError { source })?; + + collected_blocks.push(NewDatabaseBlock::from_rpc_block( + ¤t_block.block, + BlockStatus::Canonical, + )); + + info!( + block_number = current_height, + block_hash = %current_block.block.header.hash, + blocks_collected = collected_blocks.len(), + "reorg_backtrack_v2: published events, collected metadata" + ); + } + + // ══════════════════════════════════════════════════════════════ + // Phase 2: Batch commit — single DB transaction. + // + // Reverse to ascending order (fork+1, ..., N-1, N) and upsert all + // blocks atomically. If the transaction fails, the entire batch is + // rolled back and the DB remains in its original state. + // + // All events have already been published (Phase 1). This satisfies + // R4: publish-before-commit for every block. + // ══════════════════════════════════════════════════════════════ + + // collected_blocks is in descending order [N, N-1, ..., fork+1]. + // Reverse to ascending for conventional batch upsert ordering. + collected_blocks.reverse(); + + let reorg_depth = collected_blocks.len() as u64; + info!( + reorg_depth, + "reorg_backtrack_v2: walk complete, committing batch to DB" + ); + + let upsert_results = self + .repositories + .blocks + .batch_upsert_blocks_canonical(&collected_blocks) + .await + .map_err(|e| EvmListenerError::DatabaseError { source: e })?; + + let inserted_count = upsert_results + .iter() + .filter(|r| **r == UpsertResult::Inserted) + .count(); + let updated_count = upsert_results + .iter() + .filter(|r| **r == UpsertResult::Updated) + .count(); + let noop_count = upsert_results + .iter() + .filter(|r| **r == UpsertResult::NoOp) + .count(); + let known_branch = updated_count > 0 || noop_count > 0; + + info!( + block_number, + reorg_depth, + inserted_count, + updated_count, + noop_count, + known_branch, + "reorg_backtrack_v2: batch commit complete. {}", + if known_branch { + "Some blocks were from a previously known branch." + } else { + "All blocks were new to this node." + } + ); + + // ══════════════════════════════════════════════════════════════ + // Phase 3: Resume cursor via FETCH_NEW_BLOCKS. + // ══════════════════════════════════════════════════════════════ + + self.publisher + .publish(routing::FETCH_NEW_BLOCKS, &serde_json::Value::Null) + .await + .map_err(|e| { + error!(error = %e, "Failed to publish fetch trigger after backtrack v2"); + EvmListenerError::BrokerPublishError { + message: format!("Broker publish failed: {}", e), + } + })?; + + Ok(()) + } +} + +/// Sequential block validator and DB inserter (the "consumer" in the producer-consumer pattern). +/// +/// Reads blocks from the `AsyncSlotBuffer` in order (slot 0, 1, 2, ...), validates the +/// parent_hash chain against the previous block's hash, and inserts validated blocks into +/// the database as CANONICAL. +/// +/// # Parameters +/// - `listener`: Cloned `EvmListener` (owns repositories for DB access). Passed by value +/// because this runs in `tokio::spawn` which requires `'static`. +/// - `buffer`: Shared slot buffer where the producer writes fetched blocks. +/// - `cancel_token`: Shared cancellation token. Cursor checks this before each slot read +/// (via `tokio::select!`) and cancels it on reorg detection or DB failure. +/// - `expected_parent_hash`: The hash of the DB tip block. The first fetched block's +/// `parent_hash` must match this value. +/// - `range_start`: The block number of slot 0 in the buffer. +/// - `range_length`: Total number of slots to process. +/// +/// # Returns +/// - `Ok(CursorResult::Complete)` — all blocks validated and inserted +/// - `Ok(CursorResult::ReorgDetected { block_number })` — parent hash mismatch detected +/// - `Err(...)` — DB failure, buffer error, or cancellation from the fetcher side +/// +/// # Cancellation Safety +/// `buffer.get()` is cancel-safe: if `tokio::select!` drops it while awaiting `Mutex::lock`, +/// the guard drops correctly. If dropped during `Notify::notified()`, the waiter is deregistered. +async fn cursor_processing( + listener: EvmListener, + buffer: AsyncSlotBuffer, + cancel_token: CancellationToken, + expected_parent_hash: B256, + range_start: u64, + range_length: usize, +) -> Result { + let mut current_expected_hash = expected_parent_hash; + + for i in 0..range_length { + let block_number = range_start + i as u64; + + // Wait for the block with cancellation guard. + // buffer.get() blocks forever if the slot is never filled (e.g., fetcher cancelled), + // so we race it against the cancellation token. + // `biased` ensures cancellation is always checked first to prevent processing + // stale data after cancellation is signaled. + let fetched_block = tokio::select! { + biased; + _ = cancel_token.cancelled() => { + return Err(EvmListenerError::CouldNotFetchBlock { + source: BlockFetchError::Cancelled, + }); + } + block_opt = buffer.get(i) => { + block_opt.ok_or(EvmListenerError::SlotBufferError { + source: BufferError::IndexOutOfBounds, + })? + } + }; + + let block_hash = fetched_block.block.header.hash; + let parent_hash = fetched_block.block.header.parent_hash; + + // Validate the parent hash chain: this block's parent must be the previous block's hash. + // For slot 0, current_expected_hash is the DB tip's hash. + // For subsequent slots, it's the hash of the block we just inserted. + if parent_hash != current_expected_hash { + // REORG DETECTED: the chain has diverged from our canonical view. + // Cancel the fetcher to stop wasting RPC calls on a now-invalid range. + cancel_token.cancel(); + warn!( + block_number = block_number, + expected_parent = %current_expected_hash, + actual_parent = %parent_hash, + block_hash = %block_hash, + slot = i, + "Reorg detected: parent hash mismatch" + ); + return Ok(CursorResult::ReorgDetected { + block_number, + block_hash, + parent_hash, + }); + } + + // Hash chain is valid — publish events BEFORE inserting block. + // Events MUST be delivered before the block is registered in DB. + // If publish fails: block is NOT inserted, DB tip unchanged, + // cursor retries this exact block on next iteration. Zero missed events. + let chain_id_u64 = listener.repositories.chain_id() as u64; + publisher::publish_block_events( + &listener.repositories, + &fetched_block, + chain_id_u64, + BlockFlow::Live, + &listener.broker, + &listener.event_publisher, +<<<<<<< HEAD + &listener.publish_config, +======= +>>>>>>> c4fb34a (chore(core): publisher, with is exists strategy, and republish staled process) + ) + .await + .map_err(|source| { + // Stop fetcher on publish failure — no point fetching more blocks + cancel_token.cancel(); + EvmListenerError::PayloadBuildError { source } + })?; + + // Safe to insert now — events have been delivered to all consumers. + let new_db_block = + NewDatabaseBlock::from_rpc_block(&fetched_block.block, BlockStatus::Canonical); + + listener + .repositories + .blocks + .insert_block(&new_db_block) + .await + .map_err(|e| { + // Stop fetcher on DB failure — no point fetching more blocks if we can't store them + cancel_token.cancel(); + EvmListenerError::DatabaseError { source: e } + })?; + + info!( + block_number = block_number, + block_hash = %block_hash, + tx_count = fetched_block.transaction_count(), + slot = i + 1, + total = range_length, + "Block validated and inserted as canonical" + ); + + // Update expected hash for next iteration: the NEXT block's parent must be THIS block's hash + current_expected_hash = block_hash; + } + + Ok(CursorResult::Complete) +} + +/// Parallel block fetcher (the "producer" in the producer-consumer pattern). +/// +/// Spawns one `tokio::spawn` task per block in the range. Each task fetches a block via RPC +/// (using the listener's configured strategy with infinite retry on recoverable errors) and +/// stores the result in the corresponding slot of the `AsyncSlotBuffer`. +/// +/// # Parameters +/// - `listener`: Cloned `EvmListener` (owns provider, strategy, compute_block config). +/// Passed by value because this runs in `tokio::spawn`. +/// - `buffer`: Shared slot buffer. Each task writes to exactly one slot (index = position in range). +/// - `cancel_token`: Shared cancellation token. On error, this function cancels the token to +/// stop both the cursor and any remaining fetch tasks. +/// - `range_start`: The block number corresponding to slot 0. +/// - `range_length`: Total number of blocks to fetch. +/// +/// # Error Handling +/// - `get_block_by_number` has infinite retry built-in for recoverable errors (transport, timeout). +/// Only unrecoverable errors (UnsupportedMethod, DeserializationError) or cancellation bubble up. +/// - On the first error from any task, `cancel_token` is cancelled and remaining tasks are drained. +/// - Task panics (JoinError) are treated as cancellation errors. +async fn fetch_blocks_in_parallel( + listener: EvmListener, + buffer: AsyncSlotBuffer, + cancel_token: CancellationToken, + range_start: u64, + range_length: usize, +) -> Result<(), EvmListenerError> { + let compute_block = listener.compute_block; + let mut join_set: JoinSet> = JoinSet::new(); + + for i in 0..range_length { + let block_number = range_start + i as u64; + let listener = listener.clone(); + let buffer = buffer.clone(); + // Child token: cancelled when parent cancel_token is cancelled + let child_token = cancel_token.child_token(); + + join_set.spawn(async move { + let fetched_block = listener + .get_block_by_number(block_number, child_token, compute_block) + .await?; + + // Store the fetched block in the corresponding slot. + // set_once ensures each slot is written exactly once — AlreadyFilled indicates a logic bug. + buffer.set_once(i, fetched_block).await.map_err(|e| { + error!( + slot = i, + block_number = block_number, + error = %e, + "Buffer slot already filled — this is a logic bug" + ); + EvmListenerError::SlotBufferError { source: e } + }) + }); + } + + // Drain JoinSet: propagate first error, cancel remaining tasks. + // This follows the established pattern from evm_block_fetcher.rs. + while let Some(result) = join_set.join_next().await { + match result { + // Task completed successfully — slot was filled + Ok(Ok(())) => continue, + + // Task returned an error — cancel all remaining and propagate + Ok(Err(e)) => { + cancel_token.cancel(); + // Drain remaining tasks to avoid abandoned futures + while join_set.join_next().await.is_some() {} + return Err(e); + } + + // Task panicked (JoinError) — cancel all remaining, treat as cancellation + Err(join_err) => { + cancel_token.cancel(); + while join_set.join_next().await.is_some() {} + error!(error = %join_err, "Fetch task panicked"); + return Err(EvmListenerError::CouldNotFetchBlock { + source: BlockFetchError::Cancelled, + }); + } + } + } + + Ok(()) +} diff --git a/listener/crates/listener_core/src/core/filters.rs b/listener/crates/listener_core/src/core/filters.rs new file mode 100644 index 0000000000..7b0144e708 --- /dev/null +++ b/listener/crates/listener_core/src/core/filters.rs @@ -0,0 +1,70 @@ +use thiserror::Error; +use tracing::info; + +use crate::store::SqlError; +use crate::store::models::Filter; +use crate::store::repositories::Repositories; + +#[derive(Error, Debug)] +pub enum FilterError { + #[error("Database error: {source}")] + DatabaseError { + #[source] + source: SqlError, + }, +} + +/// Manages filter lifecycle for a specific chain. +/// Wrap in `Arc` for sharing between handlers and evm_listener. +#[derive(Clone)] +pub struct Filters { + repositories: Repositories, + chain_id: u64, +} + +impl Filters { + pub fn new(repositories: Repositories, chain_id: u64) -> Self { + Self { + repositories, + chain_id, + } + } + + /// Add a filter. + pub async fn add_filter( + &self, + consumer_id: &str, + from: Option<&str>, + to: Option<&str>, + log_address: Option<&str>, + ) -> Result, FilterError> { + info!( + chain_id = self.chain_id, + consumer_id, from, to, log_address, "Adding filter" + ); + self.repositories + .filters + .add_filter(consumer_id, from, to, log_address) + .await + .map_err(|source| FilterError::DatabaseError { source }) + } + + /// Remove a filter. + pub async fn remove_filter( + &self, + consumer_id: &str, + from: Option<&str>, + to: Option<&str>, + log_address: Option<&str>, + ) -> Result, FilterError> { + info!( + chain_id = self.chain_id, + consumer_id, from, to, log_address, "Removing filter" + ); + self.repositories + .filters + .remove_filter(consumer_id, from, to, log_address) + .await + .map_err(|source| FilterError::DatabaseError { source }) + } +} diff --git a/listener/crates/listener_core/src/core/mod.rs b/listener/crates/listener_core/src/core/mod.rs new file mode 100644 index 0000000000..599fb23d01 --- /dev/null +++ b/listener/crates/listener_core/src/core/mod.rs @@ -0,0 +1,11 @@ +pub mod cleaner; +pub mod evm_listener; +pub mod filters; +pub mod publisher; +pub mod slot_buffer; +pub mod workers; + +pub use cleaner::Cleaner; +pub use evm_listener::{CursorResult, EvmListener, EvmListenerError}; +pub use filters::Filters; +pub use workers::{CleanerHandler, FetchHandler, ReorgHandler, UnwatchHandler, WatchHandler}; diff --git a/listener/crates/listener_core/src/core/publisher.rs b/listener/crates/listener_core/src/core/publisher.rs new file mode 100644 index 0000000000..95ebca0976 --- /dev/null +++ b/listener/crates/listener_core/src/core/publisher.rs @@ -0,0 +1,908 @@ +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::time::Duration; + +use alloy::consensus::Transaction as _; +use alloy::network::AnyTransactionReceipt; +use alloy::primitives::Address; +use alloy::rpc::types::BlockTransactions; +use broker::{Broker, Publisher, Topic}; +use thiserror::Error; +use tokio::time::sleep; +use tracing::{error, info, warn}; + +use primitives::event::{BlockFlow, BlockPayload, IndexedLog, TransactionPayload}; +use primitives::routing::consumer_new_event_routing; + +use crate::blockchain::evm::evm_block_fetcher::FetchedBlock; +use crate::config::PublishConfig; +use crate::store::models::Filter; +use crate::store::repositories::Repositories; + +#[derive(Error, Debug)] +pub enum PublisherError { + #[error("Failed to build payload for block {block_number}: {reason}")] + PayloadBuildError { block_number: u64, reason: String }, + + #[error("Broker error during publish: {message}")] + BrokerError { message: String }, + + #[error("Filter fetch failed: {message}")] + FilterFetchError { message: String }, +} + +/// An entry in the inverted index carrying optional log-level filtering. +#[derive(Debug, Clone)] +struct FilterEntry { + consumer_id: String, + log_address: Option
, +} + +/// Inverted index built from all filters for a chain_id. +/// Enables O(1) per-transaction matching instead of O(filters) scanning. +/// +/// - `consumers`: every consumer that registered at least one filter. +/// Ensures all get a payload (even empty) so downstream can track block progress. +/// - `unfiltered`: subset of `consumers` with wildcard (None, None, None) filters. +/// These receive ALL transactions with ALL logs — matching is skipped for them. +pub struct FilterIndex { + /// Every consumer that registered at least one filter. + consumers: HashSet, + /// Consumers with (None, None, None) wildcard — receive ALL transactions, ALL logs. + unfiltered: HashSet, + /// from_address → entries that filter on this `from` (with optional log_address). + by_from: HashMap>, + /// to_address → entries that filter on this `to` (with optional log_address). + by_to: HashMap>, + /// (from, to) → entries that filter on this exact pair (with optional log_address). + by_pair: HashMap<(Address, Address), Vec>, + /// log_address → consumer_ids for (None, None, Some(log)) filters. + by_log: HashMap>, +} + +impl FilterIndex { + /// Build the inverted index from a list of filters. O(F) time and space. + pub fn from_filters(filters: Vec) -> Self { + let mut consumers = HashSet::new(); + let mut unfiltered = HashSet::new(); + let mut by_from: HashMap> = HashMap::new(); + let mut by_to: HashMap> = HashMap::new(); + let mut by_pair: HashMap<(Address, Address), Vec> = HashMap::new(); + let mut by_log: HashMap> = HashMap::new(); + + for filter in filters { + consumers.insert(filter.consumer_id.clone()); + + let from_addr = filter.from.as_deref().and_then(|s| { + s.parse::
() + .inspect_err(|e| { + warn!( + consumer_id = %filter.consumer_id, + raw_from = %s, + error = %e, + "Invalid 'from' address in filter, treating as None" + ); + }) + .ok() + }); + let to_addr = filter.to.as_deref().and_then(|s| { + s.parse::
() + .inspect_err(|e| { + warn!( + consumer_id = %filter.consumer_id, + raw_to = %s, + error = %e, + "Invalid 'to' address in filter, treating as None" + ); + }) + .ok() + }); + let log_addr = filter.log_address.as_deref().and_then(|s| { + s.parse::
() + .inspect_err(|e| { + warn!( + consumer_id = %filter.consumer_id, + raw_log_address = %s, + error = %e, + "Invalid 'log_address' in filter, treating as None" + ); + }) + .ok() + }); + + match (from_addr, to_addr, log_addr) { + (None, None, None) => { + unfiltered.insert(filter.consumer_id); + } + (None, None, Some(log)) => { + by_log.entry(log).or_default().push(filter.consumer_id); + } + (Some(from), Some(to), _) => { + by_pair.entry((from, to)).or_default().push(FilterEntry { + consumer_id: filter.consumer_id, + log_address: log_addr, + }); + } + (Some(from), None, _) => { + by_from.entry(from).or_default().push(FilterEntry { + consumer_id: filter.consumer_id, + log_address: log_addr, + }); + } + (None, Some(to), _) => { + by_to.entry(to).or_default().push(FilterEntry { + consumer_id: filter.consumer_id, + log_address: log_addr, + }); + } + } + } + + Self { + consumers, + unfiltered, + by_from, + by_to, + by_pair, + by_log, + } + } + + /// Match transactions and filter their logs according to filter rules. + /// + /// Returns a Vec of (consumer_id, filtered_transactions) pairs. + /// + /// Log filtering uses `Option>` per matched (consumer, tx) pair: + /// - `None` = include all logs (no restriction). + /// - `Some(set)` = include only logs from addresses in the set. + pub fn match_and_filter_transactions( + &self, + all_tx_payloads: &[TransactionPayload], + ) -> Vec<(String, Vec)> { + // consumer_id → BTreeMap (BTreeMap preserves tx order). + // log_filter: None = all logs, Some(addrs) = only these log addresses. + let mut matched: HashMap>>> = + HashMap::new(); + + /// Record a match: merge log filters for the same (consumer, tx) pair. + /// - None absorbs anything (all logs). + /// - Some(a) ∪ Some(b) = Some(a ∪ b). + fn record_match( + matched: &mut HashMap>>>, + consumer_id: &str, + tx_idx: usize, + log_addr: Option
, + ) { + matched + .entry(consumer_id.to_string()) + .or_default() + .entry(tx_idx) + .and_modify(|existing| { + if let Some(set) = existing { + match log_addr { + None => *existing = None, + Some(addr) => { + set.insert(addr); + } + } + } + // else: already None (all logs), no-op + }) + .or_insert_with(|| log_addr.map(|addr| HashSet::from([addr]))); + } + + for (i, tx_payload) in all_tx_payloads.iter().enumerate() { + let from = tx_payload.from; + let to = tx_payload.to; + + // by_from lookup + if let Some(entries) = self.by_from.get(&from) { + for entry in entries { + record_match(&mut matched, &entry.consumer_id, i, entry.log_address); + } + } + + // by_to and by_pair lookup + if let Some(to_addr) = to { + if let Some(entries) = self.by_to.get(&to_addr) { + for entry in entries { + record_match(&mut matched, &entry.consumer_id, i, entry.log_address); + } + } + if let Some(entries) = self.by_pair.get(&(from, to_addr)) { + for entry in entries { + record_match(&mut matched, &entry.consumer_id, i, entry.log_address); + } + } + } + + // by_log: scan logs for matching addresses + if !self.by_log.is_empty() { + for log in &tx_payload.logs { + if let Some(consumer_ids) = self.by_log.get(&log.address) { + for consumer_id in consumer_ids { + record_match(&mut matched, consumer_id, i, Some(log.address)); + } + } + } + } + } + + // Assemble results — one entry per consumer. + let mut results = Vec::with_capacity(self.consumers.len()); + + for consumer_id in &self.consumers { + let transactions = if self.unfiltered.contains(consumer_id) { + // Unfiltered consumer: all transactions, all logs, no filtering. + all_tx_payloads.to_vec() + } else { + matched + .get(consumer_id.as_str()) + .map(|tx_map| { + tx_map + .iter() + .map(|(&idx, log_filter)| { + let mut tx = all_tx_payloads[idx].clone(); + if let Some(addrs) = log_filter { + tx.logs.retain(|log| addrs.contains(&log.address)); + } + // None = all logs, no filtering needed. + tx + }) + .collect() + }) + .unwrap_or_default() + }; + + results.push((consumer_id.clone(), transactions)); + } + + results + } + + /// Build one BlockPayload per consumer_id from a fetched block. + /// + /// Returns a Vec of (consumer_id, BlockPayload) pairs. + /// Complexity: O(T × A + C) where T = transactions, A = avg fan-out, C = consumers. + pub fn build_block_payloads( + &self, + fetched_block: &FetchedBlock, + chain_id: u64, + flow: BlockFlow, + ) -> Result, PublisherError> { + let block = &fetched_block.block; + let block_number = block.header.number; + let block_hash = block.header.hash; + let parent_hash = block.header.parent_hash; + let timestamp = block.header.timestamp; + + // Extract full transactions from the block. + // Guaranteed by our RPC calls (full=true hardcoded in sem_evm_rpc_provider.rs), + // but checked defensively because the alloy type allows Hashes and Uncle variants. + let txs = match &block.transactions { + BlockTransactions::Full(txs) => txs, + _ => { + return Err(PublisherError::PayloadBuildError { + block_number, + reason: "block does not contain full transactions".to_string(), + }); + } + }; + + // Pre-build all TransactionPayloads once (shared across consumers). + let all_tx_payloads: Vec = txs + .iter() + .enumerate() + .map(|(i, tx)| { + let recovered = &tx.inner.inner; + let from = recovered.signer(); + let tx_envelope = recovered.inner(); + + let tx_hash = block.transactions.hashes().nth(i).ok_or_else(|| { + PublisherError::PayloadBuildError { + block_number, + reason: format!("missing hash for tx index {i}"), + } + })?; + let to = tx_envelope.to(); + let value = tx_envelope.value(); + let data = tx_envelope.input().clone(); + + // Safety: build_fetched_block() validates receipt completeness for all strategies. + // A missing receipt here indicates corrupted data — stale and retry to self-heal. + let logs = fetched_block + .get_receipt(&tx_hash) + .map(build_indexed_logs) + .ok_or_else(|| PublisherError::PayloadBuildError { + block_number, + reason: format!("missing receipt for tx {tx_hash}"), + })?; + + Ok(TransactionPayload { + from, + to, + hash: tx_hash, + transaction_index: i as u64, + value, + data, + logs, + }) + }) + .collect::, _>>()?; + + // Match and filter transactions per consumer. + let consumer_txs = self.match_and_filter_transactions(&all_tx_payloads); + + // Wrap in BlockPayload. + Ok(consumer_txs + .into_iter() + .map(|(consumer_id, transactions)| { + ( + consumer_id, + BlockPayload { + flow, + chain_id, + block_number, + block_hash, + parent_hash, + timestamp, + transactions, + }, + ) + }) + .collect()) + } +} + +/// Build IndexedLog entries from a transaction receipt. +fn build_indexed_logs(receipt: &AnyTransactionReceipt) -> Vec { + receipt + .inner + .logs() + .iter() + .map(|log| IndexedLog { + log_index: log.log_index.unwrap_or(0), + address: log.address(), + topics: log.topics().to_vec(), + data: log.data().data.clone(), + }) + .collect() +} + +/// Orchestration function: fetch filters, build index, match, and publish. +/// +/// Called from evm_listener.rs before each block is persisted to DB. +/// Returns errors on broker failures, filter fetch failures, or payload +/// construction failures — callers MUST treat any error as a signal to NOT +/// advance DB state. The handler framework retries the entire block, which +/// guarantees at-least-once delivery to all consumers. +/// +/// Broker-level retries (3 for Redis, 10 for AMQP with exponential backoff) +/// are already exhausted before the error reaches this function. Retrying +/// at this layer would be redundant and masks infrastructure failures from +/// the circuit breaker. +pub async fn publish_block_events( + repositories: &Repositories, + fetched_block: &FetchedBlock, + chain_id: u64, + flow: BlockFlow, + broker: &Broker, + event_publisher: &Publisher, + publish_config: &PublishConfig, +) -> Result<(), PublisherError> { + let publish_retry_delay = Duration::from_secs(publish_config.publish_retry_secs); + + // 1. Fetch filters — propagate DB errors to caller for handler-level retry. + let filters = repositories + .filters + .get_filters_by_chain_id() + .await + .map_err(|e| { + error!(error = %e, "Failed to fetch filters"); + PublisherError::FilterFetchError { + message: e.to_string(), + } + })?; + + if filters.is_empty() { + return Ok(()); + } + + // 2. Build inverted index from filters — O(F). + let filter_index = FilterIndex::from_filters(filters); + + // 3. Match transactions and build per-consumer payloads. + // PayloadBuildError propagated to caller — data may be stale, re-fetch can self-heal. + let payloads = filter_index.build_block_payloads(fetched_block, chain_id, flow)?; + + // 4. For each consumer: verify queue exists, then publish. + // When publish_stale=true: retry queue existence until queue appears (bounded by queue-not-found retries). + // When publish_stale=false: bounded retry via publish_no_stale_retries, then skip consumer. + // Broker errors (connection/publish failures) propagate immediately — the handler framework retries. + for (consumer_id, payload) in &payloads { + let routing_key = consumer_new_event_routing(consumer_id.clone()); + let topic = Topic::new(&routing_key); + let mut queue_not_found_attempts: u32 = 0; + + loop { + // Check queue existence (prevents AMQP silent drops). + let queue_exists = match broker.exists(&topic).await { + Ok(exists) => exists, + Err(e) => { + error!( + consumer_id = %consumer_id, + block_number = payload.block_number, + error = %e, + "Failed to check queue existence, propagating error" + ); + return Err(PublisherError::BrokerError { + message: format!( + "Queue existence check failed for consumer {consumer_id}, block {}: {e}", + payload.block_number + ), + }); + } + }; + + if !queue_exists { + queue_not_found_attempts += 1; + if !publish_config.publish_stale + && queue_not_found_attempts >= publish_config.publish_no_stale_retries + { + error!( + consumer_id = %consumer_id, + block_number = payload.block_number, + routing_key = %routing_key, + attempts = queue_not_found_attempts, + "Consumer queue not found after max retries, skipping consumer" + ); + break; // Skip this consumer — move to next. + } + if publish_config.publish_stale { + error!( + consumer_id = %consumer_id, + block_number = payload.block_number, + routing_key = %routing_key, + attempt = queue_not_found_attempts, + "Consumer queue not found, retrying indefinitely (publish_stale=true)" + ); + } else { + error!( + consumer_id = %consumer_id, + block_number = payload.block_number, + routing_key = %routing_key, + attempt = queue_not_found_attempts, + max_attempts = publish_config.publish_no_stale_retries, + "Consumer queue not found, retrying" + ); + } + sleep(publish_retry_delay).await; + continue; + } + + // Publish. + match event_publisher.publish(&routing_key, payload).await { + Ok(()) => { + info!( + consumer_id = %consumer_id, + block_number = payload.block_number, + tx_count = payload.transactions.len(), + routing_key = %routing_key, + "Published block event to consumer" + ); + break; // Success — move to next consumer. + } + Err(e) => { + metrics::counter!( + "listener_publish_errors_total", + "chain_id" => chain_id.to_string() + ) + .increment(1); + + error!( + consumer_id = %consumer_id, + block_number = payload.block_number, + error = %e, + "Publish failed, propagating error for handler retry" + ); + return Err(PublisherError::BrokerError { + message: format!( + "Publish failed for consumer {consumer_id}, block {}: {e}", + payload.block_number + ), + }); + } + } + } + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use alloy::primitives::{B256, Bytes, U256}; + use chrono::Utc; + use uuid::Uuid; + + // Address constants for readability. + const ADDR_1: &str = "0x0000000000000000000000000000000000000001"; + const ADDR_2: &str = "0x0000000000000000000000000000000000000002"; + const ADDR_3: &str = "0x0000000000000000000000000000000000000003"; + const ADDR_4: &str = "0x0000000000000000000000000000000000000004"; + const ADDR_5: &str = "0x0000000000000000000000000000000000000005"; + + fn addr(s: &str) -> Address { + s.parse().unwrap() + } + + fn make_filter( + consumer_id: &str, + from: Option<&str>, + to: Option<&str>, + log_address: Option<&str>, + ) -> Filter { + Filter { + id: Uuid::new_v4(), + chain_id: 1, + consumer_id: consumer_id.to_string(), + from: from.map(|s| s.to_string()), + to: to.map(|s| s.to_string()), + log_address: log_address.map(|s| s.to_string()), + created_at: Utc::now(), + } + } + + /// Build a TransactionPayload with specified from, to, and logs. + /// `logs` is a slice of (address_str, log_index) tuples. + fn make_tx(from: &str, to: &str, logs: &[(&str, u64)]) -> TransactionPayload { + TransactionPayload { + from: addr(from), + to: Some(addr(to)), + hash: B256::ZERO, + transaction_index: 0, + value: U256::ZERO, + data: Bytes::new(), + logs: logs + .iter() + .map(|(a, idx)| IndexedLog { + log_index: *idx, + address: addr(a), + topics: vec![], + data: Bytes::new(), + }) + .collect(), + } + } + + /// Find the transaction list for a given consumer in match results. + fn find_consumer_txs<'a>( + results: &'a [(String, Vec)], + consumer_id: &str, + ) -> &'a Vec { + &results + .iter() + .find(|(id, _)| id == consumer_id) + .unwrap_or_else(|| panic!("consumer {consumer_id} not found in results")) + .1 + } + + // ---- Index construction tests ---- + + #[test] + fn wildcard_filter_is_classified_correctly() { + let filters = vec![make_filter("consumer_a", None, None, None)]; + let index = FilterIndex::from_filters(filters); + + assert!(index.unfiltered.contains("consumer_a")); + assert_eq!(index.consumers.len(), 1); + assert!(index.by_from.is_empty()); + assert!(index.by_to.is_empty()); + assert!(index.by_pair.is_empty()); + assert!(index.by_log.is_empty()); + } + + #[test] + fn from_only_filter_is_indexed() { + let filters = vec![make_filter("consumer_a", Some(ADDR_1), None, None)]; + let index = FilterIndex::from_filters(filters); + + let parsed = addr(ADDR_1); + assert!(index.by_from.contains_key(&parsed)); + assert_eq!(index.by_from[&parsed].len(), 1); + assert_eq!(index.by_from[&parsed][0].consumer_id, "consumer_a"); + assert!(index.by_from[&parsed][0].log_address.is_none()); + assert!(index.unfiltered.is_empty()); + } + + #[test] + fn to_only_filter_is_indexed() { + let filters = vec![make_filter("consumer_b", None, Some(ADDR_2), None)]; + let index = FilterIndex::from_filters(filters); + + let parsed = addr(ADDR_2); + assert!(index.by_to.contains_key(&parsed)); + assert_eq!(index.by_to[&parsed].len(), 1); + assert_eq!(index.by_to[&parsed][0].consumer_id, "consumer_b"); + assert!(index.by_to[&parsed][0].log_address.is_none()); + } + + #[test] + fn pair_filter_is_indexed() { + let filters = vec![make_filter("consumer_c", Some(ADDR_1), Some(ADDR_2), None)]; + let index = FilterIndex::from_filters(filters); + + let from_parsed = addr(ADDR_1); + let to_parsed = addr(ADDR_2); + assert!(index.by_pair.contains_key(&(from_parsed, to_parsed))); + assert_eq!(index.by_pair[&(from_parsed, to_parsed)].len(), 1); + assert_eq!( + index.by_pair[&(from_parsed, to_parsed)][0].consumer_id, + "consumer_c" + ); + assert!(index.by_from.is_empty()); + assert!(index.by_to.is_empty()); + } + + #[test] + fn multiple_consumers_multiple_filter_types() { + let filters = vec![ + make_filter("wildcard_consumer", None, None, None), + make_filter("from_consumer", Some(ADDR_1), None, None), + make_filter("to_consumer", None, Some(ADDR_2), None), + make_filter("pair_consumer", Some(ADDR_1), Some(ADDR_2), None), + make_filter("log_consumer", None, None, Some(ADDR_3)), + ]; + let index = FilterIndex::from_filters(filters); + + assert_eq!(index.consumers.len(), 5); + assert!(index.unfiltered.contains("wildcard_consumer")); + assert_eq!(index.by_from.len(), 1); + assert_eq!(index.by_to.len(), 1); + assert_eq!(index.by_pair.len(), 1); + assert_eq!(index.by_log.len(), 1); + } + + #[test] + fn invalid_address_in_filter_treated_as_none() { + let filters = vec![make_filter( + "consumer_bad", + Some("not_an_address"), + None, + None, + )]; + let index = FilterIndex::from_filters(filters); + + assert!(index.unfiltered.contains("consumer_bad")); + assert!(index.by_from.is_empty()); + } + + #[test] + fn empty_filters_produces_empty_index() { + let index = FilterIndex::from_filters(vec![]); + + assert!(index.consumers.is_empty()); + assert!(index.unfiltered.is_empty()); + assert!(index.by_from.is_empty()); + assert!(index.by_to.is_empty()); + assert!(index.by_pair.is_empty()); + assert!(index.by_log.is_empty()); + } + + #[test] + fn duplicate_consumer_across_filter_types() { + let filters = vec![ + make_filter("consumer_x", None, None, None), + make_filter("consumer_x", Some(ADDR_1), Some(ADDR_2), None), + ]; + let index = FilterIndex::from_filters(filters); + + assert_eq!(index.consumers.len(), 1); + assert!(index.unfiltered.contains("consumer_x")); + assert_eq!(index.by_pair.len(), 1); + } + + #[test] + fn log_only_filter_is_indexed() { + let filters = vec![make_filter("log_consumer", None, None, Some(ADDR_3))]; + let index = FilterIndex::from_filters(filters); + + let parsed = addr(ADDR_3); + assert!(index.by_log.contains_key(&parsed)); + assert_eq!(index.by_log[&parsed], vec!["log_consumer"]); + assert!(index.unfiltered.is_empty()); + assert!(index.by_from.is_empty()); + assert!(index.by_to.is_empty()); + assert!(index.by_pair.is_empty()); + } + + #[test] + fn from_with_log_address_is_indexed_in_from_index() { + let filters = vec![make_filter("consumer_d", Some(ADDR_1), None, Some(ADDR_3))]; + let index = FilterIndex::from_filters(filters); + + let parsed = addr(ADDR_1); + assert!(index.by_from.contains_key(&parsed)); + assert_eq!(index.by_from[&parsed].len(), 1); + assert_eq!(index.by_from[&parsed][0].consumer_id, "consumer_d"); + assert_eq!(index.by_from[&parsed][0].log_address, Some(addr(ADDR_3))); + assert!(index.by_log.is_empty()); + } + + #[test] + fn invalid_log_address_treated_as_none() { + let filters = vec![make_filter( + "consumer_e", + Some(ADDR_1), + None, + Some("not_valid"), + )]; + let index = FilterIndex::from_filters(filters); + + let parsed = addr(ADDR_1); + assert!(index.by_from.contains_key(&parsed)); + assert!(index.by_from[&parsed][0].log_address.is_none()); + } + + // ---- Matching behavior tests ---- + + #[test] + fn wildcard_receives_all_txs_all_logs() { + let filters = vec![make_filter("wc", None, None, None)]; + let index = FilterIndex::from_filters(filters); + + let txs = vec![ + make_tx(ADDR_1, ADDR_2, &[(ADDR_3, 0)]), + make_tx(ADDR_4, ADDR_5, &[(ADDR_3, 1)]), + ]; + let results = index.match_and_filter_transactions(&txs); + + let consumer_txs = find_consumer_txs(&results, "wc"); + assert_eq!(consumer_txs.len(), 2); + assert_eq!(consumer_txs[0].logs.len(), 1); + assert_eq!(consumer_txs[1].logs.len(), 1); + } + + #[test] + fn from_filter_matches_and_includes_all_logs() { + let filters = vec![make_filter("fc", Some(ADDR_1), None, None)]; + let index = FilterIndex::from_filters(filters); + + let txs = vec![ + make_tx(ADDR_1, ADDR_2, &[(ADDR_3, 0), (ADDR_4, 1)]), + make_tx(ADDR_5, ADDR_2, &[(ADDR_3, 2)]), + ]; + let results = index.match_and_filter_transactions(&txs); + + let consumer_txs = find_consumer_txs(&results, "fc"); + assert_eq!(consumer_txs.len(), 1); + // All logs included (no log_address filter). + assert_eq!(consumer_txs[0].logs.len(), 2); + } + + #[test] + fn log_only_filter_matches_tx_with_matching_log_and_filters_logs() { + let filters = vec![make_filter("lc", None, None, Some(ADDR_3))]; + let index = FilterIndex::from_filters(filters); + + let txs = vec![make_tx(ADDR_1, ADDR_2, &[(ADDR_3, 0), (ADDR_4, 1)])]; + let results = index.match_and_filter_transactions(&txs); + + let consumer_txs = find_consumer_txs(&results, "lc"); + assert_eq!(consumer_txs.len(), 1); + // Only the matching log is included. + assert_eq!(consumer_txs[0].logs.len(), 1); + assert_eq!(consumer_txs[0].logs[0].address, addr(ADDR_3)); + } + + #[test] + fn log_only_does_not_match_tx_without_matching_log() { + let filters = vec![make_filter("lc", None, None, Some(ADDR_3))]; + let index = FilterIndex::from_filters(filters); + + let txs = vec![make_tx(ADDR_1, ADDR_2, &[(ADDR_4, 0), (ADDR_5, 1)])]; + let results = index.match_and_filter_transactions(&txs); + + let consumer_txs = find_consumer_txs(&results, "lc"); + assert_eq!(consumer_txs.len(), 0); + } + + #[test] + fn from_with_log_address_matches_tx_and_filters_logs() { + let filters = vec![make_filter("fc_log", Some(ADDR_1), None, Some(ADDR_3))]; + let index = FilterIndex::from_filters(filters); + + let txs = vec![make_tx( + ADDR_1, + ADDR_2, + &[(ADDR_3, 0), (ADDR_4, 1), (ADDR_5, 2)], + )]; + let results = index.match_and_filter_transactions(&txs); + + let consumer_txs = find_consumer_txs(&results, "fc_log"); + assert_eq!(consumer_txs.len(), 1); + assert_eq!(consumer_txs[0].logs.len(), 1); + assert_eq!(consumer_txs[0].logs[0].address, addr(ADDR_3)); + } + + #[test] + fn broad_filter_supersedes_log_filter_for_same_consumer() { + // Same consumer has a broad from filter AND a log-only filter. + // The broad match (All) should supersede the narrow one (Specific). + let filters = vec![ + make_filter("c", Some(ADDR_1), None, None), // broad: all logs + make_filter("c", None, None, Some(ADDR_3)), // narrow: only ADDR_3 logs + ]; + let index = FilterIndex::from_filters(filters); + + let txs = vec![make_tx(ADDR_1, ADDR_2, &[(ADDR_3, 0), (ADDR_4, 1)])]; + let results = index.match_and_filter_transactions(&txs); + + let consumer_txs = find_consumer_txs(&results, "c"); + assert_eq!(consumer_txs.len(), 1); + // Broad subsumes narrow → all logs. + assert_eq!(consumer_txs[0].logs.len(), 2); + } + + #[test] + fn multiple_log_only_filters_merge_addresses() { + let filters = vec![ + make_filter("lc", None, None, Some(ADDR_3)), + make_filter("lc", None, None, Some(ADDR_4)), + ]; + let index = FilterIndex::from_filters(filters); + + let txs = vec![make_tx( + ADDR_1, + ADDR_2, + &[(ADDR_3, 0), (ADDR_4, 1), (ADDR_5, 2)], + )]; + let results = index.match_and_filter_transactions(&txs); + + let consumer_txs = find_consumer_txs(&results, "lc"); + assert_eq!(consumer_txs.len(), 1); + // Both ADDR_3 and ADDR_4 logs included, but not ADDR_5. + assert_eq!(consumer_txs[0].logs.len(), 2); + let log_addrs: HashSet
= consumer_txs[0].logs.iter().map(|l| l.address).collect(); + assert!(log_addrs.contains(&addr(ADDR_3))); + assert!(log_addrs.contains(&addr(ADDR_4))); + } + + #[test] + fn from_filter_and_log_only_filter_merge_for_same_tx() { + // from_filter with log_address + log_only filter for different address, same consumer. + let filters = vec![ + make_filter("c", Some(ADDR_1), None, Some(ADDR_3)), + make_filter("c", None, None, Some(ADDR_4)), + ]; + let index = FilterIndex::from_filters(filters); + + let txs = vec![make_tx( + ADDR_1, + ADDR_2, + &[(ADDR_3, 0), (ADDR_4, 1), (ADDR_5, 2)], + )]; + let results = index.match_and_filter_transactions(&txs); + + let consumer_txs = find_consumer_txs(&results, "c"); + assert_eq!(consumer_txs.len(), 1); + // ADDR_3 (from from_filter) + ADDR_4 (from log_only) merged. + assert_eq!(consumer_txs[0].logs.len(), 2); + let log_addrs: HashSet
= consumer_txs[0].logs.iter().map(|l| l.address).collect(); + assert!(log_addrs.contains(&addr(ADDR_3))); + assert!(log_addrs.contains(&addr(ADDR_4))); + } + + #[test] + fn no_filter_match_produces_empty_tx_list() { + let filters = vec![make_filter("fc", Some(ADDR_1), None, None)]; + let index = FilterIndex::from_filters(filters); + + // Transaction from ADDR_5, not ADDR_1. + let txs = vec![make_tx(ADDR_5, ADDR_2, &[(ADDR_3, 0)])]; + let results = index.match_and_filter_transactions(&txs); + + let consumer_txs = find_consumer_txs(&results, "fc"); + assert_eq!(consumer_txs.len(), 0); + } +} diff --git a/listener/crates/listener_core/src/core/publisher.rs.orig b/listener/crates/listener_core/src/core/publisher.rs.orig new file mode 100644 index 0000000000..c863d92872 --- /dev/null +++ b/listener/crates/listener_core/src/core/publisher.rs.orig @@ -0,0 +1,938 @@ +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::time::Duration; + +use alloy::consensus::Transaction as _; +use alloy::network::AnyTransactionReceipt; +use alloy::primitives::Address; +use alloy::rpc::types::BlockTransactions; +use broker::{Broker, Publisher, Topic}; +use thiserror::Error; +use tokio::time::sleep; +use tracing::{error, info, warn}; + +use primitives::event::{BlockFlow, BlockPayload, IndexedLog, TransactionPayload}; +<<<<<<< HEAD +use primitives::routing::consumer_new_event_routing; +======= +use primitives::routing; +>>>>>>> c4fb34a (chore(core): publisher, with is exists strategy, and republish staled process) + +use crate::blockchain::evm::evm_block_fetcher::FetchedBlock; +use crate::config::PublishConfig; +use crate::store::models::Filter; +use crate::store::repositories::Repositories; + +#[derive(Error, Debug)] +pub enum PublisherError { + #[error("Failed to build payload for block {block_number}: {reason}")] + PayloadBuildError { block_number: u64, reason: String }, + + #[error("Consumer queue not found for consumer {consumer_id} at block {block_number}")] + ConsumerQueueNotFound { + consumer_id: String, + block_number: u64, + }, +} + +/// An entry in the inverted index carrying optional log-level filtering. +#[derive(Debug, Clone)] +struct FilterEntry { + consumer_id: String, + log_address: Option
, +} + +/// Inverted index built from all filters for a chain_id. +/// Enables O(1) per-transaction matching instead of O(filters) scanning. +/// +/// - `consumers`: every consumer that registered at least one filter. +/// Ensures all get a payload (even empty) so downstream can track block progress. +/// - `unfiltered`: subset of `consumers` with wildcard (None, None, None) filters. +/// These receive ALL transactions with ALL logs — matching is skipped for them. +pub struct FilterIndex { + /// Every consumer that registered at least one filter. + consumers: HashSet, + /// Consumers with (None, None, None) wildcard — receive ALL transactions, ALL logs. + unfiltered: HashSet, + /// from_address → entries that filter on this `from` (with optional log_address). + by_from: HashMap>, + /// to_address → entries that filter on this `to` (with optional log_address). + by_to: HashMap>, + /// (from, to) → entries that filter on this exact pair (with optional log_address). + by_pair: HashMap<(Address, Address), Vec>, + /// log_address → consumer_ids for (None, None, Some(log)) filters. + by_log: HashMap>, +} + +impl FilterIndex { + /// Build the inverted index from a list of filters. O(F) time and space. + pub fn from_filters(filters: Vec) -> Self { + let mut consumers = HashSet::new(); + let mut unfiltered = HashSet::new(); + let mut by_from: HashMap> = HashMap::new(); + let mut by_to: HashMap> = HashMap::new(); + let mut by_pair: HashMap<(Address, Address), Vec> = HashMap::new(); + let mut by_log: HashMap> = HashMap::new(); + + for filter in filters { + consumers.insert(filter.consumer_id.clone()); + + let from_addr = filter.from.as_deref().and_then(|s| { + s.parse::
() + .inspect_err(|e| { + warn!( + consumer_id = %filter.consumer_id, + raw_from = %s, + error = %e, + "Invalid 'from' address in filter, treating as None" + ); + }) + .ok() + }); + let to_addr = filter.to.as_deref().and_then(|s| { + s.parse::
() + .inspect_err(|e| { + warn!( + consumer_id = %filter.consumer_id, + raw_to = %s, + error = %e, + "Invalid 'to' address in filter, treating as None" + ); + }) + .ok() + }); + let log_addr = filter.log_address.as_deref().and_then(|s| { + s.parse::
() + .inspect_err(|e| { + warn!( + consumer_id = %filter.consumer_id, + raw_log_address = %s, + error = %e, + "Invalid 'log_address' in filter, treating as None" + ); + }) + .ok() + }); + + match (from_addr, to_addr, log_addr) { + (None, None, None) => { + unfiltered.insert(filter.consumer_id); + } + (None, None, Some(log)) => { + by_log.entry(log).or_default().push(filter.consumer_id); + } + (Some(from), Some(to), _) => { + by_pair.entry((from, to)).or_default().push(FilterEntry { + consumer_id: filter.consumer_id, + log_address: log_addr, + }); + } + (Some(from), None, _) => { + by_from.entry(from).or_default().push(FilterEntry { + consumer_id: filter.consumer_id, + log_address: log_addr, + }); + } + (None, Some(to), _) => { + by_to.entry(to).or_default().push(FilterEntry { + consumer_id: filter.consumer_id, + log_address: log_addr, + }); + } + } + } + + Self { + consumers, + unfiltered, + by_from, + by_to, + by_pair, + by_log, + } + } + + /// Match transactions and filter their logs according to filter rules. + /// + /// Returns a Vec of (consumer_id, filtered_transactions) pairs. + /// + /// Log filtering uses `Option>` per matched (consumer, tx) pair: + /// - `None` = include all logs (no restriction). + /// - `Some(set)` = include only logs from addresses in the set. + pub fn match_and_filter_transactions( + &self, + all_tx_payloads: &[TransactionPayload], + ) -> Vec<(String, Vec)> { + // consumer_id → BTreeMap (BTreeMap preserves tx order). + // log_filter: None = all logs, Some(addrs) = only these log addresses. + let mut matched: HashMap>>> = + HashMap::new(); + + /// Record a match: merge log filters for the same (consumer, tx) pair. + /// - None absorbs anything (all logs). + /// - Some(a) ∪ Some(b) = Some(a ∪ b). + fn record_match( + matched: &mut HashMap>>>, + consumer_id: &str, + tx_idx: usize, + log_addr: Option
, + ) { + matched + .entry(consumer_id.to_string()) + .or_default() + .entry(tx_idx) + .and_modify(|existing| { + if let Some(set) = existing { + match log_addr { + None => *existing = None, + Some(addr) => { + set.insert(addr); + } + } + } + // else: already None (all logs), no-op + }) + .or_insert_with(|| log_addr.map(|addr| HashSet::from([addr]))); + } + + for (i, tx_payload) in all_tx_payloads.iter().enumerate() { + let from = tx_payload.from; + let to = tx_payload.to; + + // by_from lookup + if let Some(entries) = self.by_from.get(&from) { + for entry in entries { + record_match(&mut matched, &entry.consumer_id, i, entry.log_address); + } + } + + // by_to and by_pair lookup + if let Some(to_addr) = to { + if let Some(entries) = self.by_to.get(&to_addr) { + for entry in entries { + record_match(&mut matched, &entry.consumer_id, i, entry.log_address); + } + } + if let Some(entries) = self.by_pair.get(&(from, to_addr)) { + for entry in entries { + record_match(&mut matched, &entry.consumer_id, i, entry.log_address); + } + } + } + + // by_log: scan logs for matching addresses + if !self.by_log.is_empty() { + for log in &tx_payload.logs { + if let Some(consumer_ids) = self.by_log.get(&log.address) { + for consumer_id in consumer_ids { + record_match(&mut matched, consumer_id, i, Some(log.address)); + } + } + } + } + } + + // Assemble results — one entry per consumer. + let mut results = Vec::with_capacity(self.consumers.len()); + + for consumer_id in &self.consumers { + let transactions = if self.unfiltered.contains(consumer_id) { + // Unfiltered consumer: all transactions, all logs, no filtering. + all_tx_payloads.to_vec() + } else { + matched + .get(consumer_id.as_str()) + .map(|tx_map| { + tx_map + .iter() + .map(|(&idx, log_filter)| { + let mut tx = all_tx_payloads[idx].clone(); + if let Some(addrs) = log_filter { + tx.logs.retain(|log| addrs.contains(&log.address)); + } + // None = all logs, no filtering needed. + tx + }) + .collect() + }) + .unwrap_or_default() + }; + + results.push((consumer_id.clone(), transactions)); + } + + results + } + + /// Build one BlockPayload per consumer_id from a fetched block. + /// + /// Returns a Vec of (consumer_id, BlockPayload) pairs. + /// Complexity: O(T × A + C) where T = transactions, A = avg fan-out, C = consumers. + pub fn build_block_payloads( + &self, + fetched_block: &FetchedBlock, + chain_id: u64, + flow: BlockFlow, + ) -> Result, PublisherError> { + let block = &fetched_block.block; + let block_number = block.header.number; + let block_hash = block.header.hash; + let parent_hash = block.header.parent_hash; + let timestamp = block.header.timestamp; + + // Extract full transactions from the block. + // Guaranteed by our RPC calls (full=true hardcoded in sem_evm_rpc_provider.rs), + // but checked defensively because the alloy type allows Hashes and Uncle variants. + let txs = match &block.transactions { + BlockTransactions::Full(txs) => txs, + _ => { + return Err(PublisherError::PayloadBuildError { + block_number, + reason: "block does not contain full transactions".to_string(), + }); + } + }; + + // Pre-build all TransactionPayloads once (shared across consumers). + let all_tx_payloads: Vec = txs + .iter() + .enumerate() + .map(|(i, tx)| { + let recovered = &tx.inner.inner; + let from = recovered.signer(); + let tx_envelope = recovered.inner(); + + let tx_hash = block.transactions.hashes().nth(i).ok_or_else(|| { + PublisherError::PayloadBuildError { + block_number, + reason: format!("missing hash for tx index {i}"), + } + })?; + let to = tx_envelope.to(); + let value = tx_envelope.value(); + let data = tx_envelope.input().clone(); + + // Safety: build_fetched_block() validates receipt completeness for all strategies. + // A missing receipt here indicates corrupted data — stale and retry to self-heal. + let logs = fetched_block + .get_receipt(&tx_hash) + .map(build_indexed_logs) + .ok_or_else(|| PublisherError::PayloadBuildError { + block_number, + reason: format!("missing receipt for tx {tx_hash}"), + })?; + + Ok(TransactionPayload { + from, + to, + hash: tx_hash, + transaction_index: i as u64, + value, + data, + logs, + }) + }) + .collect::, _>>()?; + + // Match and filter transactions per consumer. + let consumer_txs = self.match_and_filter_transactions(&all_tx_payloads); + + // Wrap in BlockPayload. + Ok(consumer_txs + .into_iter() + .map(|(consumer_id, transactions)| { + ( + consumer_id, + BlockPayload { + flow, + chain_id, + block_number, + block_hash, + parent_hash, + timestamp, + transactions, + }, + ) + }) + .collect()) + } +} + +/// Build IndexedLog entries from a transaction receipt. +fn build_indexed_logs(receipt: &AnyTransactionReceipt) -> Vec { + receipt + .inner + .logs() + .iter() + .map(|log| IndexedLog { + log_index: log.log_index.unwrap_or(0), + address: log.address(), + topics: log.topics().to_vec(), + data: log.data().data.clone(), + }) + .collect() +} + +/// Orchestration function: fetch filters, build index, match, and publish. +/// +/// Called from evm_listener.rs before each block is persisted to DB. +/// Uses infinite per-consumer retry to guarantee all consumers receive the event +/// before the function returns — prevents duplicate publishing on outer retry. +/// +/// Returns `Err(PublisherError)` if payload construction fails (missing full txs, +/// missing receipt, etc.). Callers MUST treat this as a signal to NOT advance DB +/// state — the error is transient and a re-fetch from RPC can self-heal. +pub async fn publish_block_events( + repositories: &Repositories, + fetched_block: &FetchedBlock, + chain_id: u64, + flow: BlockFlow, + broker: &Broker, + event_publisher: &Publisher, +<<<<<<< HEAD + publish_config: &PublishConfig, +======= +>>>>>>> c4fb34a (chore(core): publisher, with is exists strategy, and republish staled process) +) -> Result<(), PublisherError> { + let publish_retry_delay = Duration::from_secs(publish_config.publish_retry_secs); + + // 1. Fetch filters — retry infinitely on DB error. + let filters = loop { + match repositories.filters.get_filters_by_chain_id().await { + Ok(f) => break f, + Err(e) => { + error!(error = %e, "Failed to fetch filters, retrying"); + sleep(publish_retry_delay).await; + } + } + }; + + if filters.is_empty() { + return Ok(()); + } + + // 2. Build inverted index from filters — O(F). + let filter_index = FilterIndex::from_filters(filters); + + // 3. Match transactions and build per-consumer payloads. + // PayloadBuildError propagated to caller — data may be stale, re-fetch can self-heal. + let payloads = filter_index.build_block_payloads(fetched_block, chain_id, flow)?; + + // 4. For each consumer: verify queue exists, then publish. +<<<<<<< HEAD + // When publish_stale=true: retry queue existence forever (stall until queue appears). + // When publish_stale=false: bounded retry via publish_no_stale_retries, then skip consumer. + // Broker/publish errors still retry infinitely (infrastructure-level, affects all consumers equally). + for (consumer_id, payload) in &payloads { + let routing_key = consumer_new_event_routing(consumer_id.clone()); + let topic = Topic::new(&routing_key); + let mut queue_not_found_attempts: u32 = 0; + + loop { + // Check queue existence (prevents AMQP silent drops). + let queue_exists = match broker.exists(&topic).await { + Ok(exists) => exists, + Err(e) => { + error!( + consumer_id = %consumer_id, + block_number = payload.block_number, + error = %e, + "Failed to check queue existence, retrying" + ); + sleep(publish_retry_delay).await; + continue; + } + }; + + if !queue_exists { + queue_not_found_attempts += 1; + if !publish_config.publish_stale + && queue_not_found_attempts >= publish_config.publish_no_stale_retries + { + error!( + consumer_id = %consumer_id, + block_number = payload.block_number, + routing_key = %routing_key, + attempts = queue_not_found_attempts, + "Consumer queue not found after max retries, skipping consumer" + ); + break; // Skip this consumer — move to next. + } + if publish_config.publish_stale { + error!( + consumer_id = %consumer_id, + block_number = payload.block_number, + routing_key = %routing_key, + attempt = queue_not_found_attempts, + "Consumer queue not found, retrying indefinitely (publish_stale=true)" + ); + } else { + error!( + consumer_id = %consumer_id, + block_number = payload.block_number, + routing_key = %routing_key, + attempt = queue_not_found_attempts, + max_attempts = publish_config.publish_no_stale_retries, + "Consumer queue not found, retrying" + ); + } + sleep(publish_retry_delay).await; + continue; + } + + // Publish. + match event_publisher.publish(&routing_key, payload).await { + Ok(()) => { + info!( + consumer_id = %consumer_id, + block_number = payload.block_number, + tx_count = payload.transactions.len(), + routing_key = %routing_key, + "Published block event to consumer" + ); + break; // Success — move to next consumer. + } + Err(e) => { + error!( + consumer_id = %consumer_id, + block_number = payload.block_number, + error = %e, + "Publish failed, retrying" + ); + sleep(publish_retry_delay).await; + } + } + } +======= + for (consumer_id, payload) in &payloads { + let routing_key = format!("{}.{}", consumer_id, routing::NEW_EVENT); + let topic = Topic::new(&routing_key); + + // Guard: ensure the consumer's queue exists before publishing. + // Without this, AMQP silently drops the message (no error propagation). + let queue_exists = + broker + .exists(&topic) + .await + .map_err(|e| PublisherError::PublishFailed { + consumer_id: consumer_id.clone(), + block_number: payload.block_number, + message: format!("Failed to check queue existence: {}", e), + })?; + + if !queue_exists { + return Err(PublisherError::ConsumerQueueNotFound { + consumer_id: consumer_id.clone(), + block_number: payload.block_number, + }); + } + + info!( + consumer_id = %consumer_id, + block_number = payload.block_number, + tx_count = payload.transactions.len(), + routing_key = %routing_key, + "Publishing block event to consumer" + ); + + event_publisher + .publish(&routing_key, payload) + .await + .map_err(|e| PublisherError::PublishFailed { + consumer_id: consumer_id.clone(), + block_number: payload.block_number, + message: e.to_string(), + })?; +>>>>>>> c4fb34a (chore(core): publisher, with is exists strategy, and republish staled process) + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use alloy::primitives::{B256, Bytes, U256}; + use chrono::Utc; + use uuid::Uuid; + + // Address constants for readability. + const ADDR_1: &str = "0x0000000000000000000000000000000000000001"; + const ADDR_2: &str = "0x0000000000000000000000000000000000000002"; + const ADDR_3: &str = "0x0000000000000000000000000000000000000003"; + const ADDR_4: &str = "0x0000000000000000000000000000000000000004"; + const ADDR_5: &str = "0x0000000000000000000000000000000000000005"; + + fn addr(s: &str) -> Address { + s.parse().unwrap() + } + + fn make_filter( + consumer_id: &str, + from: Option<&str>, + to: Option<&str>, + log_address: Option<&str>, + ) -> Filter { + Filter { + id: Uuid::new_v4(), + chain_id: 1, + consumer_id: consumer_id.to_string(), + from: from.map(|s| s.to_string()), + to: to.map(|s| s.to_string()), + log_address: log_address.map(|s| s.to_string()), + created_at: Utc::now(), + } + } + + /// Build a TransactionPayload with specified from, to, and logs. + /// `logs` is a slice of (address_str, log_index) tuples. + fn make_tx(from: &str, to: &str, logs: &[(&str, u64)]) -> TransactionPayload { + TransactionPayload { + from: addr(from), + to: Some(addr(to)), + hash: B256::ZERO, + transaction_index: 0, + value: U256::ZERO, + data: Bytes::new(), + logs: logs + .iter() + .map(|(a, idx)| IndexedLog { + log_index: *idx, + address: addr(a), + topics: vec![], + data: Bytes::new(), + }) + .collect(), + } + } + + /// Find the transaction list for a given consumer in match results. + fn find_consumer_txs<'a>( + results: &'a [(String, Vec)], + consumer_id: &str, + ) -> &'a Vec { + &results + .iter() + .find(|(id, _)| id == consumer_id) + .unwrap_or_else(|| panic!("consumer {consumer_id} not found in results")) + .1 + } + + // ---- Index construction tests ---- + + #[test] + fn wildcard_filter_is_classified_correctly() { + let filters = vec![make_filter("consumer_a", None, None, None)]; + let index = FilterIndex::from_filters(filters); + + assert!(index.unfiltered.contains("consumer_a")); + assert_eq!(index.consumers.len(), 1); + assert!(index.by_from.is_empty()); + assert!(index.by_to.is_empty()); + assert!(index.by_pair.is_empty()); + assert!(index.by_log.is_empty()); + } + + #[test] + fn from_only_filter_is_indexed() { + let filters = vec![make_filter("consumer_a", Some(ADDR_1), None, None)]; + let index = FilterIndex::from_filters(filters); + + let parsed = addr(ADDR_1); + assert!(index.by_from.contains_key(&parsed)); + assert_eq!(index.by_from[&parsed].len(), 1); + assert_eq!(index.by_from[&parsed][0].consumer_id, "consumer_a"); + assert!(index.by_from[&parsed][0].log_address.is_none()); + assert!(index.unfiltered.is_empty()); + } + + #[test] + fn to_only_filter_is_indexed() { + let filters = vec![make_filter("consumer_b", None, Some(ADDR_2), None)]; + let index = FilterIndex::from_filters(filters); + + let parsed = addr(ADDR_2); + assert!(index.by_to.contains_key(&parsed)); + assert_eq!(index.by_to[&parsed].len(), 1); + assert_eq!(index.by_to[&parsed][0].consumer_id, "consumer_b"); + assert!(index.by_to[&parsed][0].log_address.is_none()); + } + + #[test] + fn pair_filter_is_indexed() { + let filters = vec![make_filter("consumer_c", Some(ADDR_1), Some(ADDR_2), None)]; + let index = FilterIndex::from_filters(filters); + + let from_parsed = addr(ADDR_1); + let to_parsed = addr(ADDR_2); + assert!(index.by_pair.contains_key(&(from_parsed, to_parsed))); + assert_eq!(index.by_pair[&(from_parsed, to_parsed)].len(), 1); + assert_eq!( + index.by_pair[&(from_parsed, to_parsed)][0].consumer_id, + "consumer_c" + ); + assert!(index.by_from.is_empty()); + assert!(index.by_to.is_empty()); + } + + #[test] + fn multiple_consumers_multiple_filter_types() { + let filters = vec![ + make_filter("wildcard_consumer", None, None, None), + make_filter("from_consumer", Some(ADDR_1), None, None), + make_filter("to_consumer", None, Some(ADDR_2), None), + make_filter("pair_consumer", Some(ADDR_1), Some(ADDR_2), None), + make_filter("log_consumer", None, None, Some(ADDR_3)), + ]; + let index = FilterIndex::from_filters(filters); + + assert_eq!(index.consumers.len(), 5); + assert!(index.unfiltered.contains("wildcard_consumer")); + assert_eq!(index.by_from.len(), 1); + assert_eq!(index.by_to.len(), 1); + assert_eq!(index.by_pair.len(), 1); + assert_eq!(index.by_log.len(), 1); + } + + #[test] + fn invalid_address_in_filter_treated_as_none() { + let filters = vec![make_filter( + "consumer_bad", + Some("not_an_address"), + None, + None, + )]; + let index = FilterIndex::from_filters(filters); + + assert!(index.unfiltered.contains("consumer_bad")); + assert!(index.by_from.is_empty()); + } + + #[test] + fn empty_filters_produces_empty_index() { + let index = FilterIndex::from_filters(vec![]); + + assert!(index.consumers.is_empty()); + assert!(index.unfiltered.is_empty()); + assert!(index.by_from.is_empty()); + assert!(index.by_to.is_empty()); + assert!(index.by_pair.is_empty()); + assert!(index.by_log.is_empty()); + } + + #[test] + fn duplicate_consumer_across_filter_types() { + let filters = vec![ + make_filter("consumer_x", None, None, None), + make_filter("consumer_x", Some(ADDR_1), Some(ADDR_2), None), + ]; + let index = FilterIndex::from_filters(filters); + + assert_eq!(index.consumers.len(), 1); + assert!(index.unfiltered.contains("consumer_x")); + assert_eq!(index.by_pair.len(), 1); + } + + #[test] + fn log_only_filter_is_indexed() { + let filters = vec![make_filter("log_consumer", None, None, Some(ADDR_3))]; + let index = FilterIndex::from_filters(filters); + + let parsed = addr(ADDR_3); + assert!(index.by_log.contains_key(&parsed)); + assert_eq!(index.by_log[&parsed], vec!["log_consumer"]); + assert!(index.unfiltered.is_empty()); + assert!(index.by_from.is_empty()); + assert!(index.by_to.is_empty()); + assert!(index.by_pair.is_empty()); + } + + #[test] + fn from_with_log_address_is_indexed_in_from_index() { + let filters = vec![make_filter("consumer_d", Some(ADDR_1), None, Some(ADDR_3))]; + let index = FilterIndex::from_filters(filters); + + let parsed = addr(ADDR_1); + assert!(index.by_from.contains_key(&parsed)); + assert_eq!(index.by_from[&parsed].len(), 1); + assert_eq!(index.by_from[&parsed][0].consumer_id, "consumer_d"); + assert_eq!(index.by_from[&parsed][0].log_address, Some(addr(ADDR_3))); + assert!(index.by_log.is_empty()); + } + + #[test] + fn invalid_log_address_treated_as_none() { + let filters = vec![make_filter( + "consumer_e", + Some(ADDR_1), + None, + Some("not_valid"), + )]; + let index = FilterIndex::from_filters(filters); + + let parsed = addr(ADDR_1); + assert!(index.by_from.contains_key(&parsed)); + assert!(index.by_from[&parsed][0].log_address.is_none()); + } + + // ---- Matching behavior tests ---- + + #[test] + fn wildcard_receives_all_txs_all_logs() { + let filters = vec![make_filter("wc", None, None, None)]; + let index = FilterIndex::from_filters(filters); + + let txs = vec![ + make_tx(ADDR_1, ADDR_2, &[(ADDR_3, 0)]), + make_tx(ADDR_4, ADDR_5, &[(ADDR_3, 1)]), + ]; + let results = index.match_and_filter_transactions(&txs); + + let consumer_txs = find_consumer_txs(&results, "wc"); + assert_eq!(consumer_txs.len(), 2); + assert_eq!(consumer_txs[0].logs.len(), 1); + assert_eq!(consumer_txs[1].logs.len(), 1); + } + + #[test] + fn from_filter_matches_and_includes_all_logs() { + let filters = vec![make_filter("fc", Some(ADDR_1), None, None)]; + let index = FilterIndex::from_filters(filters); + + let txs = vec![ + make_tx(ADDR_1, ADDR_2, &[(ADDR_3, 0), (ADDR_4, 1)]), + make_tx(ADDR_5, ADDR_2, &[(ADDR_3, 2)]), + ]; + let results = index.match_and_filter_transactions(&txs); + + let consumer_txs = find_consumer_txs(&results, "fc"); + assert_eq!(consumer_txs.len(), 1); + // All logs included (no log_address filter). + assert_eq!(consumer_txs[0].logs.len(), 2); + } + + #[test] + fn log_only_filter_matches_tx_with_matching_log_and_filters_logs() { + let filters = vec![make_filter("lc", None, None, Some(ADDR_3))]; + let index = FilterIndex::from_filters(filters); + + let txs = vec![make_tx(ADDR_1, ADDR_2, &[(ADDR_3, 0), (ADDR_4, 1)])]; + let results = index.match_and_filter_transactions(&txs); + + let consumer_txs = find_consumer_txs(&results, "lc"); + assert_eq!(consumer_txs.len(), 1); + // Only the matching log is included. + assert_eq!(consumer_txs[0].logs.len(), 1); + assert_eq!(consumer_txs[0].logs[0].address, addr(ADDR_3)); + } + + #[test] + fn log_only_does_not_match_tx_without_matching_log() { + let filters = vec![make_filter("lc", None, None, Some(ADDR_3))]; + let index = FilterIndex::from_filters(filters); + + let txs = vec![make_tx(ADDR_1, ADDR_2, &[(ADDR_4, 0), (ADDR_5, 1)])]; + let results = index.match_and_filter_transactions(&txs); + + let consumer_txs = find_consumer_txs(&results, "lc"); + assert_eq!(consumer_txs.len(), 0); + } + + #[test] + fn from_with_log_address_matches_tx_and_filters_logs() { + let filters = vec![make_filter("fc_log", Some(ADDR_1), None, Some(ADDR_3))]; + let index = FilterIndex::from_filters(filters); + + let txs = vec![make_tx( + ADDR_1, + ADDR_2, + &[(ADDR_3, 0), (ADDR_4, 1), (ADDR_5, 2)], + )]; + let results = index.match_and_filter_transactions(&txs); + + let consumer_txs = find_consumer_txs(&results, "fc_log"); + assert_eq!(consumer_txs.len(), 1); + assert_eq!(consumer_txs[0].logs.len(), 1); + assert_eq!(consumer_txs[0].logs[0].address, addr(ADDR_3)); + } + + #[test] + fn broad_filter_supersedes_log_filter_for_same_consumer() { + // Same consumer has a broad from filter AND a log-only filter. + // The broad match (All) should supersede the narrow one (Specific). + let filters = vec![ + make_filter("c", Some(ADDR_1), None, None), // broad: all logs + make_filter("c", None, None, Some(ADDR_3)), // narrow: only ADDR_3 logs + ]; + let index = FilterIndex::from_filters(filters); + + let txs = vec![make_tx(ADDR_1, ADDR_2, &[(ADDR_3, 0), (ADDR_4, 1)])]; + let results = index.match_and_filter_transactions(&txs); + + let consumer_txs = find_consumer_txs(&results, "c"); + assert_eq!(consumer_txs.len(), 1); + // Broad subsumes narrow → all logs. + assert_eq!(consumer_txs[0].logs.len(), 2); + } + + #[test] + fn multiple_log_only_filters_merge_addresses() { + let filters = vec![ + make_filter("lc", None, None, Some(ADDR_3)), + make_filter("lc", None, None, Some(ADDR_4)), + ]; + let index = FilterIndex::from_filters(filters); + + let txs = vec![make_tx( + ADDR_1, + ADDR_2, + &[(ADDR_3, 0), (ADDR_4, 1), (ADDR_5, 2)], + )]; + let results = index.match_and_filter_transactions(&txs); + + let consumer_txs = find_consumer_txs(&results, "lc"); + assert_eq!(consumer_txs.len(), 1); + // Both ADDR_3 and ADDR_4 logs included, but not ADDR_5. + assert_eq!(consumer_txs[0].logs.len(), 2); + let log_addrs: HashSet
= consumer_txs[0].logs.iter().map(|l| l.address).collect(); + assert!(log_addrs.contains(&addr(ADDR_3))); + assert!(log_addrs.contains(&addr(ADDR_4))); + } + + #[test] + fn from_filter_and_log_only_filter_merge_for_same_tx() { + // from_filter with log_address + log_only filter for different address, same consumer. + let filters = vec![ + make_filter("c", Some(ADDR_1), None, Some(ADDR_3)), + make_filter("c", None, None, Some(ADDR_4)), + ]; + let index = FilterIndex::from_filters(filters); + + let txs = vec![make_tx( + ADDR_1, + ADDR_2, + &[(ADDR_3, 0), (ADDR_4, 1), (ADDR_5, 2)], + )]; + let results = index.match_and_filter_transactions(&txs); + + let consumer_txs = find_consumer_txs(&results, "c"); + assert_eq!(consumer_txs.len(), 1); + // ADDR_3 (from from_filter) + ADDR_4 (from log_only) merged. + assert_eq!(consumer_txs[0].logs.len(), 2); + let log_addrs: HashSet
= consumer_txs[0].logs.iter().map(|l| l.address).collect(); + assert!(log_addrs.contains(&addr(ADDR_3))); + assert!(log_addrs.contains(&addr(ADDR_4))); + } + + #[test] + fn no_filter_match_produces_empty_tx_list() { + let filters = vec![make_filter("fc", Some(ADDR_1), None, None)]; + let index = FilterIndex::from_filters(filters); + + // Transaction from ADDR_5, not ADDR_1. + let txs = vec![make_tx(ADDR_5, ADDR_2, &[(ADDR_3, 0)])]; + let results = index.match_and_filter_transactions(&txs); + + let consumer_txs = find_consumer_txs(&results, "fc"); + assert_eq!(consumer_txs.len(), 0); + } +} diff --git a/listener/crates/listener_core/src/core/slot_buffer.rs b/listener/crates/listener_core/src/core/slot_buffer.rs new file mode 100644 index 0000000000..6b5d07bef7 --- /dev/null +++ b/listener/crates/listener_core/src/core/slot_buffer.rs @@ -0,0 +1,270 @@ +use std::sync::Arc; +use thiserror::Error; +use tokio::sync::{Mutex, Notify}; + +/// A thread-safe, generic buffer for async slotting. +/// Optimized for: Parallel random-order insertion -> Sequential ordered reading. +#[derive(Clone)] +pub struct AsyncSlotBuffer { + slots: Arc>>, +} + +struct Slot { + // The data container. Mutex ensures visibility across threads. + value: Mutex>, + // The signaling mechanism for the cursor. + notify: Notify, +} + +#[derive(Error, Debug, PartialEq)] +pub enum BufferError { + #[error("Index out of bounds")] + IndexOutOfBounds, + + #[error("Index already filled")] + AlreadyFilled, +} + +impl AsyncSlotBuffer { + /// Initialize with fixed size. All slots start empty. + pub fn new(size: usize) -> Self { + let mut slots = Vec::with_capacity(size); + for _ in 0..size { + slots.push(Slot { + value: Mutex::new(None), + notify: Notify::new(), + }); + } + Self { + slots: Arc::new(slots), + } + } + + /// PRODUCER (Strict): Sets the value ONLY if the slot is empty. + /// Returns `Err(AlreadyFilled)` if data exists. + pub async fn set_once(&self, index: usize, item: T) -> Result<(), BufferError> { + let slot = self.slots.get(index).ok_or(BufferError::IndexOutOfBounds)?; + + { + let mut guard = slot.value.lock().await; + if guard.is_some() { + return Err(BufferError::AlreadyFilled); + } + *guard = Some(item); + } + + // Notify any waiting cursors + slot.notify.notify_waiters(); + Ok(()) + } + + /// PRODUCER (Overwrite): Sets the value, replacing any existing data. + /// Could be useful if a better version of a block is fetched or for correction. + pub async fn set(&self, index: usize, item: T) -> Result<(), BufferError> { + let slot = self.slots.get(index).ok_or(BufferError::IndexOutOfBounds)?; + + { + let mut guard = slot.value.lock().await; + // We unconditionally overwrite the data + *guard = Some(item); + } + + // We notify waiters. + slot.notify.notify_waiters(); + Ok(()) + } + + /// CONSUMER: Gets the value at index. + /// If empty: WAITS (Sleeps) until filled. + /// If full: Returns immediately. + /// Returns None only if index is out of bounds. + pub async fn get(&self, index: usize) -> Option { + let slot = self.slots.get(index)?; + + // The Loop pattern ensures we don't miss notifications + loop { + let wait_for_fill = slot.notify.notified(); + { + let guard = slot.value.lock().await; + if let Some(val) = &*guard { + return Some(val.clone()); + } + } + wait_for_fill.await; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Duration; + use tokio::time::sleep; + + // Mock structure + #[derive(Clone, Debug, PartialEq)] + struct MockBlock { + height: u64, + hash: String, + parent_hash: String, + } + + impl MockBlock { + fn new(height: u64, seed: &str, parent_seed: &str) -> Self { + Self { + height, + hash: format!("hash_{}", seed), + parent_hash: format!("hash_{}", parent_seed), + } + } + } + + // SCENARIO 1: Simple Sequential Fill & Check + #[tokio::test] + async fn test_sequential_fill_and_read() { + let buffer = AsyncSlotBuffer::::new(3); + let b0 = MockBlock::new(0, "0", "gen"); + let b1 = MockBlock::new(1, "1", "0"); + let b2 = MockBlock::new(2, "2", "1"); + + buffer.set_once(1, b1.clone()).await.unwrap(); + buffer.set_once(2, b2.clone()).await.unwrap(); + buffer.set_once(0, b0.clone()).await.unwrap(); + + let mut prev_hash = "hash_gen".to_string(); + for i in 0..3 { + let block = buffer.get(i).await.expect("Bounds check"); + assert_eq!(block.parent_hash, prev_hash); + prev_hash = block.hash; + } + } + + // SCENARIO 2: Parallel Producers (Random Order) + #[tokio::test] + async fn test_parallel_fill_random_latency() { + let size = 10; + let buffer = AsyncSlotBuffer::::new(size); + let mut previous_known_hash = "hash_old".to_string(); + + for i in 0..size { + let buf_clone = buffer.clone(); + let prev_seed = if i == 0 { + "old".to_string() + } else { + format!("{}", i - 1) + }; + + tokio::spawn(async move { + let delay = if i == 0 { 50 } else { 5 }; + sleep(Duration::from_millis(delay)).await; + let block = MockBlock::new(i as u64, &i.to_string(), &prev_seed); + buf_clone.set_once(i, block).await.unwrap(); + }); + } + + for i in 0..size { + let block = buffer.get(i).await.unwrap(); + assert_eq!(block.parent_hash, previous_known_hash); + previous_known_hash = block.hash; + } + } + + // SCENARIO 3: Bounds & strict Set + #[tokio::test] + async fn test_bounds_and_strict_set() { + let buffer = AsyncSlotBuffer::::new(1); + + // Fill once + assert_eq!(buffer.set_once(0, 100).await, Ok(())); + + // Fill again (Should Fail) + assert_eq!( + buffer.set_once(0, 200).await, + Err(BufferError::AlreadyFilled) + ); + + // Check value is still original + assert_eq!(buffer.get(0).await, Some(100)); + + // Bounds + assert_eq!( + buffer.set_once(5, 10).await, + Err(BufferError::IndexOutOfBounds) + ); + assert_eq!(buffer.set(5, 10).await, Err(BufferError::IndexOutOfBounds)); + } + + // SCENARIO 4: Overwrite / Replace capability + #[tokio::test] + async fn test_overwrite_scenario() { + let buffer = AsyncSlotBuffer::::new(1); + + // 1. Set initial value + buffer + .set_once(0, "Original Data".to_string()) + .await + .unwrap(); + + // Verify + let val = buffer.get(0).await.unwrap(); + assert_eq!(val, "Original Data"); + + // 2. Replace value + buffer + .set(0, "New Corrected Data".to_string()) + .await + .unwrap(); + + // Verify the data inside is updated + let val_new = buffer.get(0).await.unwrap(); + assert_eq!(val_new, "New Corrected Data"); + } + + // SCENARIO 5: External Predecessor Handover (The "Chain Tip" Scenario) + // This tests using a data point OUTSIDE the struct to validate element 0. + #[tokio::test] + async fn test_external_predecessor_handover() { + // 1. Context: We have a "Tip" from the database or previous iteration + let chain_tip = MockBlock::new(99, "99", "98"); + let tip_hash = chain_tip.hash.clone(); // "hash_99" + + // 2. Setup Buffer for the NEXT range (100-102) + let buffer = AsyncSlotBuffer::::new(3); + + // 3. Fill the buffer (mimic parallel fetch) + // Note: Block 100's parent MUST match the chain_tip hash ("hash_99") + let b100 = MockBlock::new(100, "100", "99"); + let b101 = MockBlock::new(101, "101", "100"); + let b102 = MockBlock::new(102, "102", "101"); + + let producer_buf = buffer.clone(); + tokio::spawn(async move { + producer_buf.set_once(0, b100).await.unwrap(); + producer_buf.set_once(1, b101).await.unwrap(); + producer_buf.set_once(2, b102).await.unwrap(); + }); + + // 4. Cursor Logic + // Initialize the comparator with the EXTERNAL data + let mut last_valid_hash = tip_hash; + + for i in 0..3 { + let current_block = buffer.get(i).await.expect("Block exists"); + + println!( + "Checking Block {}: Expect Parent {} == {}", + current_block.height, current_block.parent_hash, last_valid_hash + ); + + // This assertion works for Index 0 because `last_valid_hash` + // was seeded with the external `chain_tip` + assert_eq!(current_block.parent_hash, last_valid_hash); + + // Update for next loop + last_valid_hash = current_block.hash.clone(); + } + + // Ensure we reached the end of this batch + assert_eq!(last_valid_hash, "hash_102"); + } +} diff --git a/listener/crates/listener_core/src/core/workers.rs b/listener/crates/listener_core/src/core/workers.rs new file mode 100644 index 0000000000..e8996ff4c6 --- /dev/null +++ b/listener/crates/listener_core/src/core/workers.rs @@ -0,0 +1,403 @@ +use std::sync::Arc; + +use async_trait::async_trait; +use broker::{AckDecision, Handler, HandlerError, Message, Publisher}; +use tracing::{error, info, warn}; + +use primitives::event::{FilterCommand, ReorgBacktrackEvent}; +use primitives::routing; +use primitives::utils::checksum_optional_address; + +use crate::store::FlowLock; + +use super::cleaner::{Cleaner, CleanerError}; +use super::evm_listener::{CursorResult, EvmListener, EvmListenerError}; +use super::filters::{FilterError, Filters}; +use crate::metrics::error_kind_label; + +/// Classify an [`EvmListenerError`] as transient (infrastructure) or permanent (logic bug). +/// +/// Explicit match arms — no wildcard — so that adding a new `EvmListenerError` +/// variant forces a conscious classification decision at compile time. +fn classify(err: EvmListenerError, chain_id: u64) -> HandlerError { + let chain_id_str = chain_id.to_string(); + let kind = error_kind_label(&err); + + match &err { + EvmListenerError::CouldNotFetchBlock { .. } + | EvmListenerError::CouldNotComputeBlock { .. } + | EvmListenerError::DatabaseError { .. } + | EvmListenerError::ChainHeightError { .. } + | EvmListenerError::SlotBufferError { .. } + | EvmListenerError::BrokerPublishError { .. } + | EvmListenerError::MessageProcessingError { .. } + | EvmListenerError::PayloadBuildError { .. } => { + metrics::counter!( + "listener_transient_errors_total", + "chain_id" => chain_id_str, + "error_kind" => kind, + ) + .increment(1); + HandlerError::transient(err) + } + EvmListenerError::InvariantViolation { .. } => { + metrics::counter!( + "listener_permanent_errors_total", + "chain_id" => chain_id_str, + "error_kind" => kind, + ) + .increment(1); + HandlerError::permanent(err) + } + } +} + +/// Classify a [`FilterError`] as transient or permanent. +fn classify_filter(err: FilterError) -> HandlerError { + match &err { + FilterError::DatabaseError { .. } => HandlerError::transient(err), + } +} + +// ── CleanerHandler ────────────────────────────────────────────────────── + +/// Classify a [`CleanerError`] as transient or permanent. +fn classify_cleaner(err: CleanerError) -> HandlerError { + match &err { + CleanerError::BrokerPublishError { .. } => HandlerError::transient(err), + } +} + +/// Manual [`Handler`] impl for the clean-blocks consumer. +/// +/// Ignores the message payload (the message is just a wake-up signal) and +/// calls [`Cleaner::run`]. DB errors are caught and skipped internally; +/// only broker publish failures bubble up as transient errors. +#[derive(Clone)] +pub struct CleanerHandler { + cleaner: Arc, +} + +impl CleanerHandler { + pub fn new(cleaner: Arc) -> Self { + Self { cleaner } + } +} + +#[async_trait] +impl Handler for CleanerHandler { + async fn call(&self, _msg: &Message) -> Result { + self.cleaner + .run() + .await + .map(|_| AckDecision::Ack) + .map_err(classify_cleaner) + } +} + +// ── FetchHandler ───────────────────────────────────────────────────────── + +/// Manual [`Handler`] impl for the fetch-new-blocks consumer. +/// +/// Ignores the message payload (the message is just a wake-up signal) and +/// calls [`EvmListener::fetch_blocks_and_run_cursor`]. Errors are routed +/// through [`classify`] so that infrastructure failures (DB, RPC) produce +/// `HandlerError::Transient` — enabling the circuit breaker. +/// Acquires a PostgreSQL advisory lock (per chain_id) before processing. +/// If the lock is held by another pod, the message is Acked (not requeued). +/// Avoids infinite message requeuing over message duplication. +/// This provides HPA-safe mutual exclusion for the fetch flow. +#[derive(Clone)] +pub struct FetchHandler { + listener: Arc, + flow_lock: FlowLock, + publisher: Publisher, +} + +impl FetchHandler { + pub fn new(listener: Arc, flow_lock: FlowLock, publisher: Publisher) -> Self { + Self { + listener, + flow_lock, + publisher, + } + } +} + +#[async_trait] +impl Handler for FetchHandler { + async fn call(&self, _msg: &Message) -> Result { + // Step 1: Try to acquire the distributed lock (non-blocking). + let guard = match self.flow_lock.try_acquire().await { + Ok(Some(guard)) => guard, + Ok(None) => { + warn!( + "Fetch: advisory lock held by another processor, Acking and skipping this process, mostly duplicate." + ); + return Ok(AckDecision::Ack); + } + Err(e) => { + return Err(HandlerError::transient( + EvmListenerError::MessageProcessingError { + message: format!("Failed to acquire advisory lock: {e}"), + }, + )); + } + }; + + // Step 2: Process under lock. + let result = self.listener.fetch_blocks_and_run_cursor().await; + + // Step 3: Release lock BEFORE publishing (eliminates race with other handlers). + if let Err(unlock_err) = guard.release().await { + warn!(error = %unlock_err, "Failed to explicitly release advisory lock"); + } + + // Step 4: Publish continuation message AFTER lock release, then Ack. + match result { + Ok(CursorResult::ReorgDetected { + block_number, + block_hash, + parent_hash, + }) => { + let event = ReorgBacktrackEvent { + block_number, + block_hash, + parent_hash, + }; + self.publisher + .publish(routing::BACKTRACK_REORG, &event) + .await + .map_err(|e| { + error!(error = %e, "Failed to publish backtrack event"); + HandlerError::transient(EvmListenerError::BrokerPublishError { + message: format!("Broker publish failed: {e}"), + }) + })?; + info!( + block_number = block_number, + block_hash = %block_hash, + "Backtrack event published" + ); + Ok(AckDecision::Ack) + } + Ok(_) => { + // Complete or UpToDate — schedule next fetch iteration. + self.publisher + .publish(routing::FETCH_NEW_BLOCKS, &serde_json::Value::Null) + .await + .map_err(|e| { + error!(error = %e, "Failed to publish fetch trigger"); + HandlerError::transient(EvmListenerError::BrokerPublishError { + message: format!("Broker publish failed: {e}"), + }) + })?; + Ok(AckDecision::Ack) + } + Err(e) => Err(classify(e, self.listener.chain_id())), + } + } +} + +// ── ReorgHandlerV2 ────────────────────────────────────────────────────── + +/// Handler for the backtrack-reorg consumer using the state-atomic v2 algorithm. +/// +/// Identical wiring to [`ReorgHandler`] but calls [`EvmListener::reorg_backtrack_v2`]. +/// Errors go through [`classify`] unchanged — the handler preserves all existing +/// error semantics (transient for infra, permanent for invariants). +/// +/// Acquires a PostgreSQL advisory lock (per chain_id) before processing. +/// Shares the same lock key as [`FetchHandler`], guaranteeing fetch and +/// reorg never run in parallel for the same chain. +#[derive(Clone)] +pub struct ReorgHandler { + listener: Arc, + flow_lock: FlowLock, + publisher: Publisher, +} + +impl ReorgHandler { + pub fn new(listener: Arc, flow_lock: FlowLock, publisher: Publisher) -> Self { + Self { + listener, + flow_lock, + publisher, + } + } +} + +#[async_trait] +impl Handler for ReorgHandler { + async fn call(&self, msg: &Message) -> Result { + // Deserialize before lock — dead-letter garbage early. + let event: ReorgBacktrackEvent = serde_json::from_slice(&msg.payload)?; + + // Step 1: Try to acquire the distributed lock (non-blocking). + let guard = match self.flow_lock.try_acquire().await { + Ok(Some(guard)) => guard, + Ok(None) => { + warn!("Reorg: advisory lock held by another processor, Acking, mostly duplicate."); + return Ok(AckDecision::Ack); + } + Err(e) => { + return Err(HandlerError::transient( + EvmListenerError::MessageProcessingError { + message: format!("Failed to acquire advisory lock: {e}"), + }, + )); + } + }; + + // Step 2: Process under lock. + let result = self.listener.reorg_backtrack(event).await; + + // Step 3: Release lock BEFORE publishing (eliminates race with other handlers). + if let Err(unlock_err) = guard.release().await { + warn!(error = %unlock_err, "Failed to explicitly release advisory lock"); + } + + // Step 4: Publish cursor resume AFTER lock release, then Ack. + match result { + Ok(()) => { + self.publisher + .publish(routing::FETCH_NEW_BLOCKS, &serde_json::Value::Null) + .await + .map_err(|e| { + error!(error = %e, "Failed to publish fetch trigger after reorg backtrack"); + HandlerError::transient(EvmListenerError::BrokerPublishError { + message: format!("Broker publish failed: {e}"), + }) + })?; + Ok(AckDecision::Ack) + } + Err(e) => Err(classify(e, self.listener.chain_id())), + } + } +} + +// ── WatchHandler ──────────────────────────────────────────────────────── + +/// Handler for the control.watch consumer. +/// +/// Deserializes `msg.payload` into [`FilterCommand`], validates and checksums +/// it, then calls [`Filters::add_filter`]. Deserialization and validation +/// errors are dead-lettered immediately (deterministic, will never succeed on +/// retry). Database errors are transient via [`classify_filter`]. +#[derive(Clone)] +pub struct WatchHandler { + filters: Arc, +} + +impl WatchHandler { + pub fn new(filters: Arc) -> Self { + Self { filters } + } +} + +#[async_trait] +impl Handler for WatchHandler { + async fn call(&self, msg: &Message) -> Result { + let mut event: FilterCommand = match serde_json::from_slice(&msg.payload) { + Ok(e) => e, + Err(err) => { + error!( + %err, + msg_id = %msg.metadata.id, + topic = %msg.metadata.topic, + payload_len = msg.payload.len(), + "Dead-lettering watch FilterCommand: deserialization failed", + ); + return Ok(AckDecision::Dead); + } + }; + + if let Err(err) = event.validate() { + error!( + %err, + msg_id = %msg.metadata.id, + topic = %msg.metadata.topic, + "Dead-lettering watch FilterCommand: validation failed", + ); + return Ok(AckDecision::Dead); + } + + let from = checksum_optional_address(&event.from); + let to = checksum_optional_address(&event.to); + let log_address = checksum_optional_address(&event.log_address); + + self.filters + .add_filter( + &event.consumer_id, + from.as_deref(), + to.as_deref(), + log_address.as_deref(), + ) + .await + .map(|_| AckDecision::Ack) + .map_err(classify_filter) + } +} + +// ── UnwatchHandler ────────────────────────────────────────────────────── + +/// Handler for the control.unwatch consumer. +/// +/// Deserializes `msg.payload` into [`FilterCommand`], validates and checksums +/// it, then calls [`Filters::remove_filter`]. Deserialization and validation +/// errors are dead-lettered immediately (deterministic, will never succeed on +/// retry). Database errors are transient via [`classify_filter`]. +#[derive(Clone)] +pub struct UnwatchHandler { + filters: Arc, +} + +impl UnwatchHandler { + pub fn new(filters: Arc) -> Self { + Self { filters } + } +} + +#[async_trait] +impl Handler for UnwatchHandler { + async fn call(&self, msg: &Message) -> Result { + let mut event: FilterCommand = match serde_json::from_slice(&msg.payload) { + Ok(e) => e, + Err(err) => { + error!( + %err, + msg_id = %msg.metadata.id, + topic = %msg.metadata.topic, + payload_len = msg.payload.len(), + "Dead-lettering unwatch FilterCommand: deserialization failed", + ); + return Ok(AckDecision::Dead); + } + }; + + if let Err(err) = event.validate() { + error!( + %err, + msg_id = %msg.metadata.id, + topic = %msg.metadata.topic, + "Dead-lettering unwatch FilterCommand: validation failed", + ); + return Ok(AckDecision::Dead); + } + + let from = checksum_optional_address(&event.from); + let to = checksum_optional_address(&event.to); + let log_address = checksum_optional_address(&event.log_address); + + self.filters + .remove_filter( + &event.consumer_id, + from.as_deref(), + to.as_deref(), + log_address.as_deref(), + ) + .await + .map(|_| AckDecision::Ack) + .map_err(classify_filter) + } +} diff --git a/listener/crates/listener_core/src/health.rs b/listener/crates/listener_core/src/health.rs new file mode 100644 index 0000000000..d76c2a65ca --- /dev/null +++ b/listener/crates/listener_core/src/health.rs @@ -0,0 +1,146 @@ +//! HTTP health endpoints: `/livez` and `/readyz`. +//! +//! # Design +//! +//! - **`/livez`** — stateless liveness beacon. Always returns +//! `200 {"status":"ok"}`. The only condition that can make it fail is the +//! process being hard-down (no TCP listener, no axum task) — in which case +//! Kubernetes already restarts the pod via the probe's TCP error. We +//! deliberately do NOT infer logical stalls here: a stall is not fixed by +//! a restart, and false-positive 503s during transient broker/DB blips +//! would cause pointless restart loops. Stall detection belongs in +//! Prometheus-backed alerting (queue depth, cursor progress, etc.). +//! +//! - **`/readyz`** — one-shot readiness probe. Runs a single DB ping +//! (`SELECT 1`) plus a broker connectivity check. Returns +//! `200 {"status":"ok"}` when both respond; `503 {"status":"error","reason":"not ready"}` +//! otherwise. K8s readiness failures only **gate traffic** — they never +//! restart the pod — so it is safe to 503 during an upstream outage. The +//! probe has no app-level retry: K8s `readinessProbe.periodSeconds` × +//! `failureThreshold` is the tolerance budget. + +use std::net::SocketAddr; +use std::sync::Arc; + +use axum::Json; +use axum::Router; +use axum::extract::State; +use axum::http::StatusCode; +use axum::response::IntoResponse; +use axum::routing::get; +use broker::Broker; +use serde::Serialize; +use sqlx::PgPool; +use tracing::{info, warn}; + +/// JSON body returned by `/livez` and `/readyz`. +#[derive(Debug, Serialize)] +pub struct HealthResponse { + pub status: &'static str, + #[serde(skip_serializing_if = "Option::is_none")] + pub reason: Option, +} + +impl HealthResponse { + fn ok() -> Self { + Self { + status: "ok", + reason: None, + } + } + + fn error(reason: &'static str) -> Self { + Self { + status: "error", + reason: Some(reason.to_string()), + } + } +} + +/// Readiness probe for upstream dependencies (DB + broker). +#[derive(Clone)] +pub struct ReadinessChecker { + broker: Broker, + pool: PgPool, +} + +impl ReadinessChecker { + pub fn new(broker: Broker, pool: PgPool) -> Self { + Self { broker, pool } + } + + async fn check_once(&self) -> Result<(), String> { + sqlx::query("SELECT 1") + .execute(&self.pool) + .await + .map_err(|e| format!("database: {e}"))?; + self.broker + .health_check() + .await + .map_err(|e| format!("broker: {e}"))?; + Ok(()) + } + + async fn check(&self) -> (StatusCode, Json) { + match self.check_once().await { + Ok(()) => (StatusCode::OK, Json(HealthResponse::ok())), + Err(e) => { + warn!(error = %e, "readyz: dependency probe failed"); + ( + StatusCode::SERVICE_UNAVAILABLE, + Json(HealthResponse::error("not ready")), + ) + } + } + } +} + +async fn livez_handler() -> impl IntoResponse { + (StatusCode::OK, Json(HealthResponse::ok())) +} + +async fn readyz_handler(State(readyz): State>) -> impl IntoResponse { + readyz.check().await +} + +/// Build the axum `Router` exposing `/livez` and `/readyz`. +/// +/// `/livez` is stateless; only `/readyz` needs the `ReadinessChecker`. +pub fn router(readyz: ReadinessChecker) -> Router { + Router::new() + .route("/livez", get(livez_handler)) + .route("/readyz", get(readyz_handler)) + .with_state(Arc::new(readyz)) +} + +/// Bind `addr` and spawn the HTTP server as a background tokio task. +/// +/// Returns once the TCP listener is bound, so the caller can proceed to start +/// the consumer loop. Errors only on bind failure. +pub async fn serve(addr: SocketAddr, app: Router) -> std::io::Result<()> { + let listener = tokio::net::TcpListener::bind(addr).await?; + info!(addr = %addr, "HTTP endpoints listening: /livez /readyz"); + tokio::spawn(async move { + if let Err(e) = axum::serve(listener, app).await { + tracing::error!(error = %e, "HTTP server exited"); + } + }); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn health_response_ok_shape() { + let body = serde_json::to_string(&HealthResponse::ok()).unwrap(); + assert_eq!(body, r#"{"status":"ok"}"#); + } + + #[test] + fn health_response_error_shape() { + let body = serde_json::to_string(&HealthResponse::error("not ready")).unwrap(); + assert_eq!(body, r#"{"status":"error","reason":"not ready"}"#); + } +} diff --git a/listener/crates/listener_core/src/lib.rs b/listener/crates/listener_core/src/lib.rs new file mode 100644 index 0000000000..e826bed032 --- /dev/null +++ b/listener/crates/listener_core/src/lib.rs @@ -0,0 +1,11 @@ +pub mod blockchain; +pub mod config; +pub mod core; +pub mod health; +pub mod logging; +pub mod metrics; +pub mod store; + +// Re-export broker crate for convenience +pub use broker; +pub use primitives; diff --git a/listener/crates/listener_core/src/logging.rs b/listener/crates/listener_core/src/logging.rs new file mode 100644 index 0000000000..3f90915ad0 --- /dev/null +++ b/listener/crates/listener_core/src/logging.rs @@ -0,0 +1,131 @@ +use crate::config::{LogConfig, LogFormat}; +use tracing_subscriber::layer::SubscriberExt; +use tracing_subscriber::util::SubscriberInitExt; +use tracing_subscriber::{EnvFilter, fmt}; + +/// Initialize the tracing subscriber and optionally enter a root span +/// that injects constant fields (`name`, `network`, `chain_id`) into every log line. +/// +/// # EnvFilter behaviour +/// - If `RUST_LOG` is set, it takes full precedence over `config.level`. +/// - Otherwise, the default filter is `warn,listener_core={level}` where +/// `{level}` comes from `config.level`. +/// +/// # Returns +/// An `Option` that **must** be held alive for the +/// lifetime of `main()`. Dropping it removes the constant fields from subsequent logs. +pub fn init_logging( + config: &LogConfig, + name: &str, + network: &str, + chain_id: u64, +) -> Option { + let filter = build_env_filter(config); + + // Each format × timestamp combination produces a different concrete type, + // so we call .init() in each branch rather than boxing. + match (&config.format, config.show_timestamp) { + (LogFormat::Json, true) => { + tracing_subscriber::registry() + .with(filter) + .with( + fmt::layer() + .with_file(config.show_file_line) + .with_line_number(config.show_file_line) + .with_thread_ids(config.show_thread_ids) + .with_target(config.show_target) + .json() + .flatten_event(true), + ) + .init(); + } + (LogFormat::Json, false) => { + tracing_subscriber::registry() + .with(filter) + .with( + fmt::layer() + .with_file(config.show_file_line) + .with_line_number(config.show_file_line) + .with_thread_ids(config.show_thread_ids) + .with_target(config.show_target) + .json() + .flatten_event(true) + .without_time(), + ) + .init(); + } + (LogFormat::Compact, true) => { + tracing_subscriber::registry() + .with(filter) + .with( + fmt::layer() + .with_file(config.show_file_line) + .with_line_number(config.show_file_line) + .with_thread_ids(config.show_thread_ids) + .with_target(config.show_target) + .compact(), + ) + .init(); + } + (LogFormat::Compact, false) => { + tracing_subscriber::registry() + .with(filter) + .with( + fmt::layer() + .with_file(config.show_file_line) + .with_line_number(config.show_file_line) + .with_thread_ids(config.show_thread_ids) + .with_target(config.show_target) + .compact() + .without_time(), + ) + .init(); + } + (LogFormat::Pretty, true) => { + tracing_subscriber::registry() + .with(filter) + .with( + fmt::layer() + .with_file(config.show_file_line) + .with_line_number(config.show_file_line) + .with_thread_ids(config.show_thread_ids) + .with_target(config.show_target) + .pretty(), + ) + .init(); + } + (LogFormat::Pretty, false) => { + tracing_subscriber::registry() + .with(filter) + .with( + fmt::layer() + .with_file(config.show_file_line) + .with_line_number(config.show_file_line) + .with_thread_ids(config.show_thread_ids) + .with_target(config.show_target) + .pretty() + .without_time(), + ) + .init(); + } + } + + if config.show_constants { + let span = tracing::info_span!( + "listener", + name = %name, + network = %network, + chain_id = chain_id, + ); + Some(span.entered()) + } else { + None + } +} + +fn build_env_filter(config: &LogConfig) -> EnvFilter { + match std::env::var("RUST_LOG") { + Ok(_) => EnvFilter::from_default_env(), + Err(_) => EnvFilter::new(format!("warn,listener_core={}", config.level)), + } +} diff --git a/listener/crates/listener_core/src/main.rs b/listener/crates/listener_core/src/main.rs new file mode 100644 index 0000000000..2a91375c78 --- /dev/null +++ b/listener/crates/listener_core/src/main.rs @@ -0,0 +1,549 @@ +use anyhow::{Result, anyhow}; +use std::env; +use std::net::SocketAddr; +use std::process; +use std::sync::Arc; +use std::time::Duration; + +use broker::{Broker, Topic}; +use listener_core::blockchain::evm::sem_evm_rpc_provider::SemEvmRpcProvider; +use listener_core::config::BrokerType; +use listener_core::config::config::Settings; +use listener_core::core::{ + Cleaner, CleanerHandler, EvmListener, FetchHandler, Filters, ReorgHandler, UnwatchHandler, + WatchHandler, +}; +use listener_core::logging; +use listener_core::store::repositories::Repositories; +use listener_core::store::{FlowLock, PgClient, run_migrations}; +use primitives::routing; +use primitives::utils::chain_id_to_namespace; +use tracing::{error, info}; + +const DEFAULT_CONFIG_PATH: &str = "./config.yaml"; + +/// Parse command line arguments and return the config file path. +/// Uses --config or defaults to ./config.yaml +fn parse_config_path() -> String { + let args: Vec = env::args().collect(); + + let mut i = 1; + while i < args.len() { + if args[i] == "--config" { + if i + 1 < args.len() { + return args[i + 1].clone(); + } else { + eprintln!("FATAL: --config requires a path argument"); + process::exit(1); + } + } + i += 1; + } + + DEFAULT_CONFIG_PATH.to_string() +} + +/// Load settings from config file and environment variables. +/// Uses `eprintln!` because tracing is not yet initialized at this point. +fn load_settings() -> Settings { + let config_path = parse_config_path(); + + match Settings::new(Some(&config_path)) { + Ok(settings) => settings, + Err(e) => { + eprintln!("FATAL: Failed to load configuration from {config_path}: {e}"); + process::exit(1); + } + } +} + +fn checked_chain_id(chain_id: u64) -> Result { + i64::try_from(chain_id) + .map_err(|_| anyhow!("Configured chain_id exceeds the supported i64 database range")) +} + +#[tokio::main] +async fn main() { + // Load configuration first (before tracing — uses eprintln for errors) + let settings = load_settings(); + + // Initialize structured logging from config + let _root_guard = logging::init_logging( + &settings.log, + &settings.name, + &settings.blockchain.network, + settings.blockchain.chain_id, + ); + + info!("Configuration loaded: {:?}", settings); + + // ── Telemetry: Prometheus metrics endpoint ──────────────────────────── + if settings.telemetry.enabled { + let metrics_config = telemetry::MetricsConfig { + listen_addr: SocketAddr::from(([0, 0, 0, 0], settings.telemetry.metrics_port)), + }; + if let Err(e) = telemetry::init_metrics(metrics_config) { + error!("Failed to start metrics endpoint: {}", e); + process::exit(1); + } + broker::metrics::describe_metrics(); + listener_core::metrics::describe_metrics(); + listener_core::metrics::init_gauges(settings.blockchain.chain_id); + listener_core::metrics::init_counters(settings.blockchain.chain_id); + } + + // Initialize database connection + let pg_client = if settings.database.use_iam_auth() { + #[cfg(feature = "iam-auth")] + { + let cancel = tokio_util::sync::CancellationToken::new(); + let pool = listener_core::store::connect_iam(&settings.database, cancel) + .await + .unwrap_or_else(|e| { + error!("Failed to initialize IAM database connection: {e}"); + process::exit(1); + }); + PgClient::from_pool(pool) + } + #[cfg(not(feature = "iam-auth"))] + { + // validate() already catches this, but belt-and-suspenders + error!("IAM auth requested but iam-auth feature is not enabled"); + process::exit(1); + } + } else { + match PgClient::new(&settings.database).await { + Ok(client) => client, + Err(e) => { + error!("Failed to initialize database connection: {}", e); + process::exit(1); + } + } + }; + + // Run database migrations + run_migrations(&pg_client, settings.database.migration_max_attempts).await; + + let arc_pg_client = Arc::new(pg_client); + + let configured_chain_id = match checked_chain_id(settings.blockchain.chain_id) { + Ok(chain_id) => chain_id, + Err(err) => { + error!(chain_id = settings.blockchain.chain_id, %err); + process::exit(1); + } + }; + let repositories = Repositories::new(Arc::clone(&arc_pg_client), configured_chain_id); + let provider = match SemEvmRpcProvider::new( + settings.blockchain.rpc_url.clone(), + settings.blockchain.strategy.max_parallel_requests, + ) { + Ok(provider) => provider, + Err(e) => { + error!("Could not instantiate the semaphore provider: {}", e); + process::exit(1); + } + }; + + // Declaring broker: + let broker = match settings.broker.broker_type { + BrokerType::Redis => { + match Broker::redis_with_ensure_publish( + &settings.broker.broker_url, + settings.broker.ensure_publish, + ) + .await + { + Ok(b) => b, + Err(e) => { + error!("Failed to initialize Redis broker: {}", e); + process::exit(1); + } + } + } + BrokerType::Amqp => { + match Broker::amqp(&settings.broker.broker_url) + .with_ensure_publish(settings.broker.ensure_publish) + .build() + .await + { + Ok(b) => b, + Err(e) => { + error!("Failed to initialize AMQP broker: {}", e); + process::exit(1); + } + } + } + }; + + // checking matching chain_id with rpc chain_id. + match provider.get_chain_id().await { + Ok(chain_id) => { + if settings.blockchain.chain_id == chain_id { + info!("Chain id verified"); + } else { + error!("Chain id doesn't match with rpc chain_id"); + process::exit(1); + } + } + Err(e) => { + error!("Couldn't call chain id from rpc at instantiation: {}", e); + process::exit(1); + } + }; + + let chain_id = &chain_id_to_namespace(settings.blockchain.chain_id); + let publisher = match broker.publisher(chain_id).await { + Ok(p) => p, + Err(e) => { + error!("Failed to create publisher: {}", e); + process::exit(1); + } + }; + + let event_publisher = match broker.publisher_unscoped().await { + Ok(p) => p, + Err(e) => { + error!("Failed to create event publisher: {}", e); + process::exit(1); + } + }; + + let seed_publisher = publisher.clone(); + let cleaner_publisher = publisher.clone(); + let handler_publisher = publisher; + + let repositories_for_filters = repositories.clone(); + let repositories_for_cleaner = repositories.clone(); + + let evm_listener = EvmListener::new( + provider, + repositories, + broker.clone(), + event_publisher, + &settings.blockchain, + ); + // Validate strategy (This function panics if there is a problem regarding the strategy). + // NOTE: maybe send error, and propagate error here, then process::exit(1). + evm_listener + .validate_strategy_and_init_block( + settings + .blockchain + .strategy + .block_start_on_first_start + .clone(), + ) + .await; + + let evm_listener = Arc::new(evm_listener); + + // let network = &settings.blockchain.network; + + // Here the routing key, is basically the same than the queue name (e.g group) + let fetch_consumer = match broker + .consumer(&Topic::new(routing::FETCH_NEW_BLOCKS).with_namespace(chain_id)) + // .group("fetch-new-blocks") + .group(routing::FETCH_NEW_BLOCKS) + .prefetch(1) + .max_retries(5) + .redis_claim_min_idle(settings.broker.claim_min_idle) + .redis_claim_interval(1) + .redis_block_ms(200) + .circuit_breaker( + settings.broker.circuit_breaker_threshold, + Duration::from_secs(settings.broker.circuit_breaker_cooldown_secs), + ) + .build() + { + Ok(c) => c, + Err(e) => { + error!("Failed to build fetch consumer: {}", e); + process::exit(1); + } + }; + + let reorg_consumer = match broker + .consumer(&Topic::new(routing::BACKTRACK_REORG).with_namespace(chain_id)) + .group(routing::BACKTRACK_REORG) + .prefetch(1) + .max_retries(5) + .redis_claim_min_idle(settings.broker.claim_min_idle) + .redis_claim_interval(1) + .redis_block_ms(200) + .circuit_breaker( + settings.broker.circuit_breaker_threshold, + Duration::from_secs(settings.broker.circuit_breaker_cooldown_secs), + ) + .build() + { + Ok(c) => c, + Err(e) => { + error!("Failed to build reorg consumer: {}", e); + process::exit(1); + } + }; + + let watch_consumer = match broker + .consumer(&Topic::new(routing::WATCH).with_namespace(chain_id)) + .group(routing::WATCH) + .prefetch(1) + .max_retries(5) + .redis_claim_min_idle(settings.broker.claim_min_idle) + .redis_claim_interval(1) + .redis_block_ms(200) + .circuit_breaker( + settings.broker.circuit_breaker_threshold, + Duration::from_secs(settings.broker.circuit_breaker_cooldown_secs), + ) + .build() + { + Ok(c) => c, + Err(e) => { + error!("Failed to build watch consumer: {}", e); + process::exit(1); + } + }; + + let unwatch_consumer = match broker + .consumer(&Topic::new(routing::UNWATCH).with_namespace(chain_id)) + .group(routing::UNWATCH) + .prefetch(1) + .max_retries(5) + .redis_claim_min_idle(settings.broker.claim_min_idle) + .redis_claim_interval(1) + .redis_block_ms(200) + .circuit_breaker( + settings.broker.circuit_breaker_threshold, + Duration::from_secs(settings.broker.circuit_breaker_cooldown_secs), + ) + .build() + { + Ok(c) => c, + Err(e) => { + error!("Failed to build unwatch consumer: {}", e); + process::exit(1); + } + }; + + let cleaner_consumer = match broker + .consumer(&Topic::new(routing::CLEAN_BLOCKS).with_namespace(chain_id)) + .group(routing::CLEAN_BLOCKS) + .prefetch(1) + .max_retries(5) + .redis_claim_min_idle(settings.blockchain.cleaner.cron_secs + 25) + .redis_claim_interval(1) + .redis_block_ms(200) + .circuit_breaker(3, Duration::from_secs(30)) + .build() + { + Ok(c) => c, + Err(e) => { + error!("Failed to build cleaner consumer: {}", e); + process::exit(1); + } + }; + + // ── Define handlers ───────────────────────────────────────────────── + let flow_lock = FlowLock::new(Arc::clone(&arc_pg_client), configured_chain_id); + let fetch_handler = FetchHandler::new( + Arc::clone(&evm_listener), + flow_lock.clone(), + handler_publisher.clone(), + ); + let reorg_handler = ReorgHandler::new(Arc::clone(&evm_listener), flow_lock, handler_publisher); + + let filters = Filters::new(repositories_for_filters, settings.blockchain.chain_id); + let filters = Arc::new(filters); + + let watch_handler = WatchHandler::new(Arc::clone(&filters)); + let unwatch_handler = UnwatchHandler::new(Arc::clone(&filters)); + + let cleaner = Arc::new(Cleaner::new( + repositories_for_cleaner.blocks, + cleaner_publisher, + &settings.blockchain.cleaner, + )); + let cleaner_handler = CleanerHandler::new(Arc::clone(&cleaner)); + + // ── Ensure AMQP queues/bindings exist before checking depth ──────── + // Without this, AMQP silently drops the seed message because no queue + // is bound to the exchange yet (queues are normally created by consumer.run()). + if let Err(e) = fetch_consumer.ensure_topology().await { + error!(error = %e, "Failed to set up fetch consumer topology"); + process::exit(1); + } + + if let Err(e) = reorg_consumer.ensure_topology().await { + error!(error = %e, "Failed to set up reorg consumer topology"); + process::exit(1); + } + + if let Err(e) = watch_consumer.ensure_topology().await { + error!(error = %e, "Failed to set up control watch consumer topology"); + process::exit(1); + } + + if let Err(e) = unwatch_consumer.ensure_topology().await { + error!(error = %e, "Failed to set up control unwatch consumer topology"); + process::exit(1); + } + + // ── Seed the cursor loop if no pending work exists ───────────────── + // TODO: replicate this to the consumer library for non auto startup. + let fetch_topic = Topic::new(routing::FETCH_NEW_BLOCKS).with_namespace(chain_id); + let reorg_topic = Topic::new(routing::BACKTRACK_REORG).with_namespace(chain_id); + + let should_seed = settings.blockchain.strategy.automatic_startup + && match broker + .is_empty(&fetch_topic, routing::FETCH_NEW_BLOCKS) + .await + { + Ok(empty) => empty, + Err(e) => { + error!(error = %e, "Failed to check fetch queue depth"); + process::exit(1); + } + } + && match broker + .is_empty(&reorg_topic, routing::BACKTRACK_REORG) + .await + { + Ok(empty) => empty, + Err(e) => { + error!(error = %e, "Failed to check reorg backtrack queue depth"); + process::exit(1); + } + }; + + if should_seed { + info!("Fetch queue empty — publishing seed to bootstrap cursor loop"); + if let Err(e) = seed_publisher + .publish(routing::FETCH_NEW_BLOCKS, &serde_json::Value::Null) + .await + { + error!(error = %e, "Failed to publish fetch seed"); + process::exit(1); + } + } + + // ── Ensure cleaner topology and seed (always, not gated by automatic_startup) ── + if let Err(e) = cleaner_consumer.ensure_topology().await { + error!(error = %e, "Failed to set up cleaner consumer topology"); + process::exit(1); + } + + let cleaner_topic = Topic::new(routing::CLEAN_BLOCKS).with_namespace(chain_id); + let cleaner_is_empty = match broker.is_empty(&cleaner_topic, routing::CLEAN_BLOCKS).await { + Ok(empty) => empty, + Err(e) => { + error!(error = %e, "Failed to check cleaner queue depth"); + process::exit(1); + } + }; + + if cleaner_is_empty && settings.blockchain.cleaner.active { + info!("Cleaner queue empty — publishing seed to bootstrap cleaner loop"); + if let Err(e) = seed_publisher + .publish(routing::CLEAN_BLOCKS, &serde_json::Value::Null) + .await + { + error!(error = %e, "Failed to publish cleaner seed"); + process::exit(1); + } + } + // Remove this, its only for testing purposes. + // TODO: Remove, only for testing purposes. + // It simulates consumer lib to test if we are properly revcieving the events. + // let dyn_routing = format!("coprocessor.{}", routing::NEW_EVENT); + // let test_consumer_lib = match broker + // .consumer(&Topic::new(dyn_routing.clone()).with_namespace(namespace::EMPTY_NAMESPACE)) + // .group(dyn_routing.clone()) + // .prefetch(20) + // .max_retries(5) + // .redis_claim_min_idle(10) + // .redis_claim_interval(1) + // .redis_block_ms(200) + // .circuit_breaker(3, Duration::from_secs(30)) + // .build() + // { + // Ok(c) => c, + // Err(e) => { + // error!("Failed to build test consumer consumer: {}", e); + // process::exit(1); + // } + // }; + + // let consumer_lib_handler_test = + // AsyncHandlerPayloadOnly::new(move |event: BlockPayload| async move { + // // println!("Consumer received event: {}", event); + // info!("GETTING EVENT FROM BLOCK: {}", event.block_number); + // Ok::<(), EvmListenerError>(()) + // }); + + // ── Periodic queue depth poller ────────────────────────────────────── + // Polls broker.queue_depths() every 15s for each listener topic and emits + // the values as broker_queue_depth_* Prometheus gauges. + let queue_depth_cancel = tokio_util::sync::CancellationToken::new(); + let _queue_depth_task = broker::metrics::spawn_queue_depth_poller( + broker.clone(), + vec![ + broker::metrics::QueueDepthPollTarget::new( + fetch_topic.clone(), + routing::FETCH_NEW_BLOCKS, + ), + broker::metrics::QueueDepthPollTarget::new( + reorg_topic.clone(), + routing::BACKTRACK_REORG, + ), + broker::metrics::QueueDepthPollTarget::new( + Topic::new(routing::WATCH).with_namespace(chain_id), + routing::WATCH, + ), + broker::metrics::QueueDepthPollTarget::new( + Topic::new(routing::UNWATCH).with_namespace(chain_id), + routing::UNWATCH, + ), + broker::metrics::QueueDepthPollTarget::new( + cleaner_topic.clone(), + routing::CLEAN_BLOCKS, + ), + ], + Duration::from_secs(15), + queue_depth_cancel, + ); + + // ── Start the shared HTTP server (hosts /livez + /readyz today) ───── + // Bound to `settings.http_port` — a single application port designed to + // host all operational routes (health now, metrics/admin/... later). + // `/livez` is a stateless OK beacon; `/readyz` probes DB + broker. + { + let readyz = listener_core::health::ReadinessChecker::new( + broker.clone(), + arc_pg_client.pool().clone(), + ); + let app = listener_core::health::router(readyz); + let addr = SocketAddr::from(([0, 0, 0, 0], settings.http_port)); + if let Err(e) = listener_core::health::serve(addr, app).await { + error!(addr = %addr, error = %e, "Failed to bind HTTP server"); + process::exit(1); + } + } + + // ── Run both consumers concurrently ───────────────────────────────── + info!("Starting consumers"); + + let (consumer_name, result) = tokio::select! { + r = fetch_consumer.run(fetch_handler) => ("Fetch", r), + r = reorg_consumer.run(reorg_handler) => ("Reorg", r), + r = watch_consumer.run(watch_handler) => ("Watch", r), + r = unwatch_consumer.run(unwatch_handler) => ("Unwatch", r), + r = cleaner_consumer.run(cleaner_handler) => ("Cleaner", r), + + // TEST CONSUMER: TO REMOVE. + // r = test_consumer_lib.run(consumer_lib_handler_test) => ("test copro lib consumer", r), + }; + + error!("{consumer_name} consumer exited: {result:?}"); + error!("Shutting down all consumers and exiting"); + process::exit(1); +} diff --git a/listener/crates/listener_core/src/main.rs.orig b/listener/crates/listener_core/src/main.rs.orig new file mode 100644 index 0000000000..542b1f52ef --- /dev/null +++ b/listener/crates/listener_core/src/main.rs.orig @@ -0,0 +1,470 @@ +use anyhow::{Result, anyhow}; +<<<<<<< HEAD +======= +// use broker::AsyncHandlerPayloadOnly; +// use listener_core::core::EvmListenerError; +// use primitives::event::BlockPayload; +// use primitives::namespace; +use std::env; +use std::process; +use std::sync::Arc; +use std::time::Duration; + +>>>>>>> c4fb34a (chore(core): publisher, with is exists strategy, and republish staled process) +use broker::{Broker, Topic}; +use listener_core::blockchain::evm::sem_evm_rpc_provider::SemEvmRpcProvider; +use listener_core::config::BrokerType; +use listener_core::config::config::Settings; +use listener_core::core::{ + Cleaner, CleanerHandler, EvmListener, FetchHandler, Filters, ReorgHandler, UnwatchHandler, + WatchHandler, +}; +use listener_core::store::repositories::Repositories; +use listener_core::store::{PgClient, run_migrations}; +use primitives::routing; +use primitives::utils::chain_id_to_namespace; +use std::env; +use std::process; +use std::sync::Arc; +use std::time::Duration; +use tracing::{error, info}; + +const DEFAULT_CONFIG_PATH: &str = "./config.yaml"; + +/// Parse command line arguments and return the config file path. +/// Uses --config or defaults to ./config.yaml +fn parse_config_path() -> String { + let args: Vec = env::args().collect(); + + let mut i = 1; + while i < args.len() { + if args[i] == "--config" { + if i + 1 < args.len() { + return args[i + 1].clone(); + } else { + error!("--config requires a path argument"); + process::exit(1); + } + } + i += 1; + } + + DEFAULT_CONFIG_PATH.to_string() +} + +/// Load settings from config file and environment variables. +/// Exits the process if configuration fails to load. +fn load_settings() -> Settings { + let config_path = parse_config_path(); + info!("Loading configuration from: {}", config_path); + + match Settings::new(Some(&config_path)) { + Ok(settings) => { + info!("Configuration loaded: {:?}", settings); + settings + } + Err(e) => { + error!("Failed to load configuration: {}", e); + process::exit(1); + } + } +} + +fn checked_chain_id(chain_id: u64) -> Result { + i64::try_from(chain_id) + .map_err(|_| anyhow!("Configured chain_id exceeds the supported i64 database range")) +} + +#[tokio::main] +async fn main() { + // Initialize logging + tracing_subscriber::fmt().with_target(false).init(); + + // TODO: add metrics initialization + health api + // NOTE: profiling could be added as well. + + // Load configuration + let settings = load_settings(); + + // Initialize database connection + let pg_client = match PgClient::new(&settings.database).await { + Ok(client) => client, + Err(e) => { + error!("Failed to initialize database connection: {}", e); + process::exit(1); + } + }; + + // Run database migrations + run_migrations(&pg_client, settings.database.migration_max_attempts).await; + + let arc_pg_client = Arc::new(pg_client); + + let configured_chain_id = match checked_chain_id(settings.blockchain.chain_id) { + Ok(chain_id) => chain_id, + Err(err) => { + error!(chain_id = settings.blockchain.chain_id, %err); + process::exit(1); + } + }; + let repositories = Repositories::new(arc_pg_client, configured_chain_id); + let provider = match SemEvmRpcProvider::new( + settings.blockchain.rpc_url.clone(), + settings.blockchain.strategy.max_parallel_requests, + ) { + Ok(provider) => provider, + Err(e) => { + error!("Could not instantiate the semaphore provider: {}", e); + process::exit(1); + } + }; + + // Declaring broker: + let broker = match settings.broker.broker_type { + BrokerType::Redis => { + match Broker::redis_with_ensure_publish( + &settings.broker.broker_url, + settings.broker.ensure_publish, + ) + .await + { + Ok(b) => b, + Err(e) => { + error!("Failed to initialize Redis broker: {}", e); + process::exit(1); + } + } + } + BrokerType::Amqp => { + match Broker::amqp(&settings.broker.broker_url) + .with_ensure_publish(settings.broker.ensure_publish) + .build() + .await + { + Ok(b) => b, + Err(e) => { + error!("Failed to initialize AMQP broker: {}", e); + process::exit(1); + } + } + } + }; + + // checking matching chain_id with rpc chain_id. + match provider.get_chain_id().await { + Ok(chain_id) => { + if settings.blockchain.chain_id == chain_id { + info!("Chain id verified"); + } else { + error!("Chain id doesn't match with rpc chain_id"); + process::exit(1); + } + } + Err(e) => { + error!("Couldn't call chain id from rpc at instantiation: {}", e); + process::exit(1); + } + }; + + let chain_id = &chain_id_to_namespace(settings.blockchain.chain_id); + let publisher = match broker.publisher(chain_id).await { + Ok(p) => p, + Err(e) => { + error!("Failed to create publisher: {}", e); + process::exit(1); + } + }; + + let event_publisher = match broker.publisher_unscoped().await { + Ok(p) => p, + Err(e) => { + error!("Failed to create event publisher: {}", e); + process::exit(1); + } + }; + + let seed_publisher = publisher.clone(); + let cleaner_publisher = publisher.clone(); + + let repositories_for_filters = repositories.clone(); + let repositories_for_cleaner = repositories.clone(); + + let evm_listener = EvmListener::new( + provider, + repositories, + publisher, + broker.clone(), + event_publisher, +<<<<<<< HEAD + &settings.blockchain, +======= + &settings.blockchain.strategy, + settings.blockchain.finality_depth, +>>>>>>> c4fb34a (chore(core): publisher, with is exists strategy, and republish staled process) + ); + // Validate strategy (This function panics if there is a problem regarding the strategy). + // NOTE: maybe send error, and propagate error here, then process::exit(1). + evm_listener + .validate_strategy_and_init_block(settings.blockchain.strategy.block_start) + .await; + + let evm_listener = Arc::new(evm_listener); + + // let network = &settings.blockchain.network; + + // Here the routing key, is basically the same than the queue name (e.g group) + let fetch_consumer = match broker + .consumer(&Topic::new(routing::FETCH_NEW_BLOCKS).with_namespace(chain_id)) + // .group("fetch-new-blocks") + .group(routing::FETCH_NEW_BLOCKS) + .prefetch(1) + .max_retries(5) + .redis_claim_min_idle(settings.broker.claim_min_idle) + .redis_claim_interval(1) + .redis_block_ms(200) + .circuit_breaker( + settings.broker.circuit_breaker_threshold, + Duration::from_secs(settings.broker.circuit_breaker_cooldown_secs), + ) + .build() + { + Ok(c) => c, + Err(e) => { + error!("Failed to build fetch consumer: {}", e); + process::exit(1); + } + }; + + let reorg_consumer = match broker + .consumer(&Topic::new(routing::BACKTRACK_REORG).with_namespace(chain_id)) + .group(routing::BACKTRACK_REORG) + .prefetch(1) + .max_retries(5) + .redis_claim_min_idle(settings.broker.claim_min_idle) + .redis_claim_interval(1) + .redis_block_ms(200) + .circuit_breaker( + settings.broker.circuit_breaker_threshold, + Duration::from_secs(settings.broker.circuit_breaker_cooldown_secs), + ) + .build() + { + Ok(c) => c, + Err(e) => { + error!("Failed to build reorg consumer: {}", e); + process::exit(1); + } + }; + + let watch_consumer = match broker + .consumer(&Topic::new(routing::WATCH).with_namespace(chain_id)) + .group(routing::WATCH) + .prefetch(1) + .max_retries(5) + .redis_claim_min_idle(settings.broker.claim_min_idle) + .redis_claim_interval(1) + .redis_block_ms(200) + .circuit_breaker( + settings.broker.circuit_breaker_threshold, + Duration::from_secs(settings.broker.circuit_breaker_cooldown_secs), + ) + .build() + { + Ok(c) => c, + Err(e) => { + error!("Failed to build watch consumer: {}", e); + process::exit(1); + } + }; + + let unwatch_consumer = match broker + .consumer(&Topic::new(routing::UNWATCH).with_namespace(chain_id)) + .group(routing::UNWATCH) + .prefetch(1) + .max_retries(5) + .redis_claim_min_idle(settings.broker.claim_min_idle) + .redis_claim_interval(1) + .redis_block_ms(200) + .circuit_breaker( + settings.broker.circuit_breaker_threshold, + Duration::from_secs(settings.broker.circuit_breaker_cooldown_secs), + ) + .build() + { + Ok(c) => c, + Err(e) => { + error!("Failed to build unwatch consumer: {}", e); + process::exit(1); + } + }; + + let cleaner_consumer = match broker + .consumer(&Topic::new(routing::CLEAN_BLOCKS).with_namespace(chain_id)) + .group(routing::CLEAN_BLOCKS) + .prefetch(1) + .max_retries(5) + .redis_claim_min_idle(settings.blockchain.cleaner.cron_secs + 25) + .redis_claim_interval(1) + .redis_block_ms(200) + .circuit_breaker(3, Duration::from_secs(30)) + .build() + { + Ok(c) => c, + Err(e) => { + error!("Failed to build cleaner consumer: {}", e); + process::exit(1); + } + }; + + // ── Define handlers ───────────────────────────────────────────────── + let fetch_handler = FetchHandler::new(Arc::clone(&evm_listener)); + let reorg_handler = ReorgHandler::new(Arc::clone(&evm_listener)); + + let filters = Filters::new(repositories_for_filters, settings.blockchain.chain_id); + let filters = Arc::new(filters); + + let watch_handler = WatchHandler::new(Arc::clone(&filters)); + let unwatch_handler = UnwatchHandler::new(Arc::clone(&filters)); + + let cleaner = Arc::new(Cleaner::new( + repositories_for_cleaner.blocks, + cleaner_publisher, + &settings.blockchain.cleaner, + )); + let cleaner_handler = CleanerHandler::new(Arc::clone(&cleaner)); + + // ── Ensure AMQP queues/bindings exist before checking depth ──────── + // Without this, AMQP silently drops the seed message because no queue + // is bound to the exchange yet (queues are normally created by consumer.run()). + if let Err(e) = fetch_consumer.ensure_topology().await { + error!(error = %e, "Failed to set up fetch consumer topology"); + process::exit(1); + } + + if let Err(e) = reorg_consumer.ensure_topology().await { + error!(error = %e, "Failed to set up reorg consumer topology"); + process::exit(1); + } + + if let Err(e) = watch_consumer.ensure_topology().await { + error!(error = %e, "Failed to set up control watch consumer topology"); + process::exit(1); + } + + if let Err(e) = unwatch_consumer.ensure_topology().await { + error!(error = %e, "Failed to set up control unwatch consumer topology"); + process::exit(1); + } + + // ── Seed the cursor loop if no pending work exists ───────────────── + // TODO: replicate this to the consumer library for non auto startup. + let fetch_topic = Topic::new(routing::FETCH_NEW_BLOCKS).with_namespace(chain_id); + let reorg_topic = Topic::new(routing::BACKTRACK_REORG).with_namespace(chain_id); + + let should_seed = settings.blockchain.strategy.automatic_startup + && match broker + .is_empty(&fetch_topic, routing::FETCH_NEW_BLOCKS) + .await + { + Ok(empty) => empty, + Err(e) => { + error!(error = %e, "Failed to check fetch queue depth"); + process::exit(1); + } + } + && match broker + .is_empty(&reorg_topic, routing::BACKTRACK_REORG) + .await + { + Ok(empty) => empty, + Err(e) => { + error!(error = %e, "Failed to check reorg backtrack queue depth"); + process::exit(1); + } + }; + + if should_seed { + info!("Fetch queue empty — publishing seed to bootstrap cursor loop"); + if let Err(e) = seed_publisher + .publish(routing::FETCH_NEW_BLOCKS, &serde_json::Value::Null) + .await + { + error!(error = %e, "Failed to publish fetch seed"); + process::exit(1); + } + } + + // ── Ensure cleaner topology and seed (always, not gated by automatic_startup) ── + if let Err(e) = cleaner_consumer.ensure_topology().await { + error!(error = %e, "Failed to set up cleaner consumer topology"); + process::exit(1); + } + + let cleaner_topic = Topic::new(routing::CLEAN_BLOCKS).with_namespace(chain_id); + let cleaner_is_empty = match broker.is_empty(&cleaner_topic, routing::CLEAN_BLOCKS).await { + Ok(empty) => empty, + Err(e) => { + error!(error = %e, "Failed to check cleaner queue depth"); + process::exit(1); + } + }; + + if cleaner_is_empty && settings.blockchain.cleaner.active { + info!("Cleaner queue empty — publishing seed to bootstrap cleaner loop"); + if let Err(e) = seed_publisher + .publish(routing::CLEAN_BLOCKS, &serde_json::Value::Null) + .await + { + error!(error = %e, "Failed to publish cleaner seed"); + process::exit(1); + } + } + // Remove this, its only for testing purposes. +<<<<<<< HEAD + // TODO: Remove, only for testing purposes. + // It simulates consumer lib to test if we are properly revcieving the events. +======= +>>>>>>> c4fb34a (chore(core): publisher, with is exists strategy, and republish staled process) + // let dyn_routing = format!("coprocessor.{}", routing::NEW_EVENT); + // let test_consumer_lib = match broker + // .consumer(&Topic::new(dyn_routing.clone()).with_namespace(namespace::EMPTY_NAMESPACE)) + // .group(dyn_routing.clone()) + // .prefetch(20) + // .max_retries(5) + // .redis_claim_min_idle(10) + // .redis_claim_interval(1) + // .redis_block_ms(200) + // .circuit_breaker(3, Duration::from_secs(30)) + // .build() + // { + // Ok(c) => c, + // Err(e) => { + // error!("Failed to build test consumer consumer: {}", e); + // process::exit(1); + // } + // }; + + // let consumer_lib_handler_test = + // AsyncHandlerPayloadOnly::new(move |event: BlockPayload| async move { + // println!("Consumer received event: {}", event); + // Ok::<(), EvmListenerError>(()) + // }); + + // ── Run both consumers concurrently ───────────────────────────────── + info!("Starting consumers"); + + let (consumer_name, result) = tokio::select! { + r = fetch_consumer.run(fetch_handler) => ("Fetch", r), + r = reorg_consumer.run(reorg_handler) => ("Reorg", r), + r = watch_consumer.run(watch_handler) => ("Watch", r), + r = unwatch_consumer.run(unwatch_handler) => ("Unwatch", r), + r = cleaner_consumer.run(cleaner_handler) => ("Cleaner", r), + + // TEST CONSUMER: TO REMOVE. + // r = test_consumer_lib.run(consumer_lib_handler_test) => ("test copro lib consumer", r), + }; + + error!("{consumer_name} consumer exited: {result:?}"); + error!("Shutting down all consumers and exiting"); + process::exit(1); +} diff --git a/listener/crates/listener_core/src/metrics.rs b/listener/crates/listener_core/src/metrics.rs new file mode 100644 index 0000000000..fba7255610 --- /dev/null +++ b/listener/crates/listener_core/src/metrics.rs @@ -0,0 +1,202 @@ +//! Listener metrics registration and helpers. +//! +//! This module provides: +//! - [`describe_metrics()`]: registers Prometheus HELP strings for all listener metrics +//! - [`init_gauges()`]: initializes gauge values to zero for Grafana discoverability +//! - [`error_kind_label()`]: maps [`EvmListenerError`] variants to static label strings + +use crate::core::evm_listener::EvmListenerError; + +/// Register metric descriptions with the global recorder. +/// +/// Call once at application startup, after installing the metrics exporter. +/// Safe to call multiple times (describe is idempotent). +pub fn describe_metrics() { + use metrics::{Unit, describe_counter, describe_gauge, describe_histogram}; + + // ── Cursor liveness ───────────────────────────────────────────────── + describe_counter!( + "listener_cursor_iterations_total", + Unit::Count, + "Total main cursor loop iterations (stall detection: rate should be > 0)" + ); + + // ── Reorgs ────────────────────────────────────────────────────────── + describe_counter!( + "listener_reorgs_total", + Unit::Count, + "Total chain reorganizations detected" + ); + + // ── Block heights ─────────────────────────────────────────────────── + describe_gauge!( + "listener_db_tip_block_number", + Unit::Count, + "Latest canonical block number in the database" + ); + describe_gauge!( + "listener_chain_height_block_number", + Unit::Count, + "Latest block number reported by the RPC node" + ); + + // ── Fetch timing ──────────────────────────────────────────────────── + describe_histogram!( + "listener_block_fetch_duration_seconds", + Unit::Seconds, + "Wall-clock time to fetch a single block with receipts" + ); + describe_histogram!( + "listener_range_fetch_duration_seconds", + Unit::Seconds, + "Wall-clock time to fetch and process an entire block range" + ); + + // ── Publish errors ────────────────────────────────────────────────── + describe_counter!( + "listener_publish_errors_total", + Unit::Count, + "Failures during event publishing to broker" + ); + + // ── Error classification ──────────────────────────────────────────── + describe_counter!( + "listener_transient_errors_total", + Unit::Count, + "Transient (infrastructure) errors from handler error classification" + ); + describe_counter!( + "listener_permanent_errors_total", + Unit::Count, + "Permanent (logic) errors from handler error classification" + ); + + // ── Block compute verification ───────────────────────────────────── + describe_counter!( + "listener_compute_block_failure_total", + Unit::Count, + "Block hash verification failures during block compute" + ); + describe_counter!( + "listener_compute_transaction_failure_total", + Unit::Count, + "Transaction root verification failures during block compute" + ); + describe_counter!( + "listener_compute_receipt_failure_total", + Unit::Count, + "Receipt root verification failures during block compute" + ); + + // ── RPC provider ──────────────────────────────────────────────────── + describe_histogram!( + "listener_rpc_request_duration_seconds", + Unit::Seconds, + "Wall-clock time per JSON-RPC call (includes semaphore wait)" + ); + describe_counter!( + "listener_rpc_requests_total", + Unit::Count, + "Total RPC requests partitioned by outcome" + ); + describe_counter!( + "listener_rpc_errors_total", + Unit::Count, + "RPC errors by method and error type" + ); + describe_gauge!( + "listener_rpc_semaphore_available", + Unit::Count, + "Available permits in the RPC concurrency semaphore" + ); +} + +/// Initialize gauges to zero so Grafana discovers the time series on the first scrape, +/// even before the first cursor iteration completes. +/// +/// Call once at startup, after [`describe_metrics()`]. +pub fn init_gauges(chain_id: u64) { + let chain_id_str = chain_id.to_string(); + + metrics::gauge!( + "listener_db_tip_block_number", + "chain_id" => chain_id_str.clone() + ) + .set(0.0); + + metrics::gauge!( + "listener_chain_height_block_number", + "chain_id" => chain_id_str + ) + .set(0.0); +} + +/// Initialize block-compute failure counters to zero for every `stalling` label +/// combination, so the time series exist from startup. +/// +/// Why: `increase()` / `rate()` need at least two samples in the lookback window +/// to compute a delta. If a counter goes from "absent" to `1` on the first +/// failure, a Grafana stat panel using `increase(...[24h])` will report `0` +/// because there is no baseline to compare against. Seeding the series at `0` +/// makes the first real failure show up immediately as `1`. +/// +/// Call once at startup, after [`describe_metrics()`]. +pub fn init_counters(chain_id: u64) { + let chain_id_str = chain_id.to_string(); + + for stalling in ["true", "false"] { + metrics::counter!( + "listener_compute_block_failure_total", + "chain_id" => chain_id_str.clone(), + "stalling" => stalling + ) + .increment(0); + metrics::counter!( + "listener_compute_transaction_failure_total", + "chain_id" => chain_id_str.clone(), + "stalling" => stalling + ) + .increment(0); + metrics::counter!( + "listener_compute_receipt_failure_total", + "chain_id" => chain_id_str.clone(), + "stalling" => stalling + ) + .increment(0); + } +} + +/// Map an [`EvmListenerError`] variant to a static label string for the `error_kind` label. +pub(crate) fn error_kind_label(err: &EvmListenerError) -> &'static str { + match err { + EvmListenerError::CouldNotFetchBlock { .. } => "block_fetch", + EvmListenerError::CouldNotComputeBlock { .. } => "block_compute", + EvmListenerError::DatabaseError { .. } => "database", + EvmListenerError::ChainHeightError { .. } => "chain_height", + EvmListenerError::SlotBufferError { .. } => "slot_buffer", + EvmListenerError::BrokerPublishError { .. } => "broker_publish", + EvmListenerError::PayloadBuildError { .. } => "payload_build", + EvmListenerError::InvariantViolation { .. } => "invariant_violation", + EvmListenerError::MessageProcessingError { .. } => "message_processing", + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn describe_metrics_does_not_panic() { + describe_metrics(); + } + + #[test] + fn init_gauges_does_not_panic() { + init_gauges(1); + } + + #[test] + fn init_counters_does_not_panic() { + init_counters(1); + } +} diff --git a/listener/crates/listener_core/src/store/client.rs b/listener/crates/listener_core/src/store/client.rs new file mode 100644 index 0000000000..d98ac64f0d --- /dev/null +++ b/listener/crates/listener_core/src/store/client.rs @@ -0,0 +1,61 @@ +use crate::config::config::DatabaseConfig; +use sqlx::Postgres; +use sqlx::pool::PoolConnection; +use sqlx::postgres::{PgPool, PgPoolOptions}; +use std::time::Duration; +use tracing::info; + +pub struct PgClient { + pool: PgPool, +} + +impl PgClient { + /// Create from an existing pool (used by IAM auth path). + #[cfg(feature = "iam-auth")] + pub fn from_pool(pool: PgPool) -> Self { + Self { pool } + } + + pub async fn new(config: &DatabaseConfig) -> Result { + info!("Initializing database connection..."); + + let pool = PgPoolOptions::new() + .max_connections(config.pool.max_connections) + .min_connections(config.pool.min_connections) + .acquire_timeout(Duration::from_secs(config.pool.acquire_timeout_secs)) + .idle_timeout(Duration::from_secs(config.pool.idle_timeout_secs)) + .max_lifetime(Duration::from_secs(config.pool.max_lifetime_secs)) + .connect(&config.db_url) + .await?; + + // Validate connection + sqlx::query("SELECT 1").execute(&pool).await?; + + info!( + "Database pool initialized: min={}, max={}", + config.pool.min_connections, config.pool.max_connections + ); + + Ok(Self { pool }) + } + + pub fn pool(&self) -> &PgPool { + &self.pool + } + + pub fn get_pool(&self) -> PgPool { + self.pool.clone() + } + + pub async fn acquire(&self) -> Result, sqlx::Error> { + self.pool.acquire().await + } + + pub async fn get_app_connection(&self) -> Result, sqlx::Error> { + self.pool.acquire().await + } + + pub async fn close(&self) { + self.pool.close().await; + } +} diff --git a/listener/crates/listener_core/src/store/error.rs b/listener/crates/listener_core/src/store/error.rs new file mode 100644 index 0000000000..44e4d68b90 --- /dev/null +++ b/listener/crates/listener_core/src/store/error.rs @@ -0,0 +1,22 @@ +use thiserror::Error; + +pub type SqlResult = Result; + +#[derive(Error, Debug)] +pub enum SqlError { + #[error("Database error: {0}")] + Database(#[from] sqlx::Error), + + #[error("Connection pool error: {0}")] + Pool(String), +} + +impl SqlError { + /// Returns `true` if the underlying error is a PostgreSQL unique constraint violation (SQLSTATE 23505). + pub fn is_unique_violation(&self) -> bool { + matches!( + self, + SqlError::Database(sqlx::Error::Database(db_err)) if db_err.code().as_deref() == Some("23505") + ) + } +} diff --git a/listener/crates/listener_core/src/store/flow_lock.rs b/listener/crates/listener_core/src/store/flow_lock.rs new file mode 100644 index 0000000000..5bacae1cde --- /dev/null +++ b/listener/crates/listener_core/src/store/flow_lock.rs @@ -0,0 +1,98 @@ +use sqlx::Postgres; +use sqlx::pool::PoolConnection; +use std::sync::Arc; +use tracing::{debug, warn}; + +use super::client::PgClient; +use super::error::SqlResult; + +/// RAII guard for a PostgreSQL session-level advisory lock. +/// +/// Holds a dedicated [`PoolConnection`] for the duration of the lock. +/// The lock is released when [`release()`](Self::release) is called or +/// when the guard is dropped (connection close triggers PG auto-release). +pub struct FlowLockGuard { + conn: Option>, + lock_key: i64, +} + +impl FlowLockGuard { + /// Explicitly release the advisory lock and return the connection to the pool. + pub async fn release(mut self) -> SqlResult<()> { + if let Some(mut conn) = self.conn.take() { + let released: bool = sqlx::query_scalar::<_, bool>("SELECT pg_advisory_unlock($1)") + .bind(self.lock_key) + .fetch_one(&mut *conn) + .await?; + if !released { + warn!( + lock_key = self.lock_key, + "pg_advisory_unlock returned false — lock was not held by this session" + ); + } + debug!(lock_key = self.lock_key, "Advisory lock released"); + } + Ok(()) + } +} + +impl Drop for FlowLockGuard { + fn drop(&mut self) { + if self.conn.is_some() { + warn!( + lock_key = self.lock_key, + "FlowLockGuard dropped without explicit release — PG will auto-release on session close" + ); + // Connection is dropped here, PG auto-releases the advisory lock. + } + } +} + +/// Non-blocking distributed lock backed by `pg_try_advisory_lock`. +/// +/// Provides mutual exclusion per `chain_id` across all pods sharing the +/// same PostgreSQL database. The lock key IS the `chain_id`, so different +/// chains on the same database are completely independent. +/// +/// Used to prevent concurrent execution of fetch and reorg flows for the +/// same chain under HPA (Horizontal Pod Autoscaling). +#[derive(Clone)] +pub struct FlowLock { + client: Arc, + chain_id: i64, +} + +impl FlowLock { + pub fn new(client: Arc, chain_id: i64) -> Self { + Self { client, chain_id } + } + + /// Attempt to acquire the advisory lock (non-blocking). + /// + /// Returns `Some(guard)` if the lock was acquired, `None` if another + /// session already holds it. The guard holds a [`PoolConnection`] — the + /// lock remains held until [`FlowLockGuard::release()`] is called or + /// the guard is dropped. + pub async fn try_acquire(&self) -> SqlResult> { + let mut conn = self.client.acquire().await?; + + let acquired: bool = sqlx::query_scalar("SELECT pg_try_advisory_lock($1)") + .bind(self.chain_id) + .fetch_one(&mut *conn) + .await?; + + if acquired { + debug!(chain_id = self.chain_id, "Advisory lock acquired"); + Ok(Some(FlowLockGuard { + conn: Some(conn), + lock_key: self.chain_id, + })) + } else { + debug!( + chain_id = self.chain_id, + "Advisory lock held by another session" + ); + Ok(None) + } + } +} diff --git a/listener/crates/listener_core/src/store/iam_auth.rs b/listener/crates/listener_core/src/store/iam_auth.rs new file mode 100644 index 0000000000..1904f64683 --- /dev/null +++ b/listener/crates/listener_core/src/store/iam_auth.rs @@ -0,0 +1,319 @@ +//! IAM role-based RDS authentication. +//! +//! Generates short-lived SigV4 tokens used as PostgreSQL passwords, +//! and refreshes them in a background task before they expire. +//! Connection details (host, port, user, dbname) are parsed from `db_url`. + +use crate::config::config::DatabaseConfig; +use anyhow::Context; +use aws_credential_types::provider::ProvideCredentials; +use aws_sigv4::http_request::{SignableBody, SignableRequest, SigningSettings, sign}; +use aws_sigv4::sign::v4; +use sqlx::postgres::{PgConnectOptions, PgPool, PgPoolOptions, PgSslMode}; +use std::time::{Duration, SystemTime}; +use tokio_util::sync::CancellationToken; +use tracing::{error, info}; + +/// Refresh every 10 min (token valid for 15 min, 5 min safety buffer). +const TOKEN_REFRESH_INTERVAL: Duration = Duration::from_secs(600); + +/// Force connection recycle at 14 min (before 15-min token expiry). +const IAM_MAX_LIFETIME: Duration = Duration::from_secs(840); + +/// Validated parameters for IAM-authenticated RDS connections. +/// +/// Parsed from `db_url` at startup (`api-parse-dont-validate`). +/// Invalid config fails immediately, not at first token refresh. +#[derive(Debug, Clone)] +struct ConnectParameters { + host: String, + port: u16, + username: String, + database: String, + /// `None` = rely on system/rustls trust store, `Some(path)` = custom CA PEM. + ssl_ca_path: Option, +} + +impl ConnectParameters { + /// Parse connection parameters from a PostgreSQL URL. + /// + /// # Errors + /// + /// Returns error if the URL is malformed or missing required fields + /// (host, username, database). + fn from_db_url(db_url: &str, ssl_ca_path: Option) -> anyhow::Result { + let url = url::Url::parse(db_url).context("invalid db_url")?; + + let host = url + .host_str() + .filter(|h| !h.is_empty()) + .context("db_url missing host")? + .to_string(); + + let username = { + let u = url.username(); + anyhow::ensure!(!u.is_empty(), "db_url missing username"); + u.to_string() + }; + + let database = { + let path = url.path().trim_start_matches('/'); + anyhow::ensure!(!path.is_empty(), "db_url missing database name"); + path.to_string() + }; + + let port = url.port().unwrap_or(5432); + + Ok(Self { + host, + port, + username, + database, + ssl_ca_path, + }) + } + + /// Generate a fresh IAM token and build `PgConnectOptions` with SSL. + /// + /// By default, relies on the system/rustls trust store (which includes + /// Amazon Root CAs). If `ssl_ca_path` is set, uses that PEM file instead + /// (needed for legacy `rds-ca-2019` or custom CAs). + /// + /// # Errors + /// + /// Returns error if AWS credentials are unavailable or SigV4 signing fails. + async fn connect_options(&self) -> anyhow::Result { + let token = generate_rds_iam_token(&self.host, self.port, &self.username) + .await + .context("IAM token generation failed")?; + + let mut opts = PgConnectOptions::new() + .host(&self.host) + .port(self.port) + .username(&self.username) + .password(&token) + .database(&self.database) + .ssl_mode(PgSslMode::VerifyFull); + + if let Some(path) = &self.ssl_ca_path { + opts = opts.ssl_root_cert(path); + } + + Ok(opts) + } +} + +/// Create an IAM-authenticated connection pool and spawn the background +/// token refresh task. Returns the pool directly — caller wraps in `PgClient`. +/// +/// Connection details (host, port, user, dbname) are parsed from `db_config.db_url`. +/// The password in `db_url` is ignored — replaced by a short-lived IAM token. +/// +/// # Errors +/// +/// Returns error if URL parsing, initial token generation, or connection fails. +pub async fn connect_iam( + db_config: &DatabaseConfig, + cancel: CancellationToken, +) -> anyhow::Result { + let ssl_ca_path = db_config + .iam_auth + .as_ref() + .and_then(|c| c.ssl_ca_path.clone()); // clone: own the path before ConnectParameters takes ownership + + let params = ConnectParameters::from_db_url(&db_config.db_url, ssl_ca_path)?; + + let opts = params + .connect_options() + .await + .context("initial IAM connection failed")?; + + let pool = PgPoolOptions::new() + .max_connections(db_config.pool.max_connections) + .min_connections(db_config.pool.min_connections) + .acquire_timeout(Duration::from_secs(db_config.pool.acquire_timeout_secs)) + .idle_timeout(Duration::from_secs(db_config.pool.idle_timeout_secs)) + // Override pool.max_lifetime_secs: IAM tokens expire at 15 min, + // so connections must recycle before that. 14 min is the hard ceiling. + .max_lifetime(IAM_MAX_LIFETIME) + .connect_with(opts) + .await + .context("failed to connect to RDS with IAM token")?; + + sqlx::query("SELECT 1") + .execute(&pool) + .await + .context("IAM-authenticated connection validation failed")?; + + info!( + host = %params.host, + "Database pool initialized (IAM): min={}, max={}, max_lifetime={}s", + db_config.pool.min_connections, + db_config.pool.max_connections, + IAM_MAX_LIFETIME.as_secs() + ); + + let refresh_pool = pool.clone(); // clone: PgPool is Arc-backed; refresh task needs its own handle + tokio::spawn(token_refresh_loop(params, refresh_pool, cancel)); + + Ok(pool) +} + +async fn token_refresh_loop(params: ConnectParameters, pool: PgPool, cancel: CancellationToken) { + loop { + tokio::select! { + biased; + _ = cancel.cancelled() => { + info!("IAM token refresh loop shutting down"); + break; + } + _ = tokio::time::sleep(TOKEN_REFRESH_INTERVAL) => { + match params.connect_options().await { + Ok(opts) => { + pool.set_connect_options(opts); + info!("IAM RDS auth token refreshed for new connections"); + } + Err(e) => { + error!( + error = %e, + "IAM token refresh failed — existing connections still valid" + ); + } + } + } + } + } +} + +/// Generate an IAM RDS auth token (SigV4 pre-signed URL). +/// +/// The token is used as the PostgreSQL password. It expires in 15 minutes +/// but existing connections authenticated with it remain valid. +/// +/// # Errors +/// +/// Returns error if AWS credentials are unavailable (e.g., IRSA not configured, +/// metadata service unreachable) or if SigV4 signing fails. +async fn generate_rds_iam_token( + db_hostname: &str, + port: u16, + db_username: &str, +) -> anyhow::Result { + let config = aws_config::load_defaults(aws_config::BehaviorVersion::latest()).await; + + let credentials = config + .credentials_provider() + .context("no AWS credentials provider found — is IRSA configured?")? + .provide_credentials() + .await + .context("failed to load AWS credentials")?; + + let identity = credentials.into(); + let region = config + .region() + .context("AWS region not configured")? + .to_string(); + + let mut signing_settings = SigningSettings::default(); + signing_settings.expires_in = Some(Duration::from_secs(900)); + signing_settings.signature_location = aws_sigv4::http_request::SignatureLocation::QueryParams; + + let signing_params = v4::SigningParams::builder() + .identity(&identity) + .region(®ion) + .name("rds-db") + .time(SystemTime::now()) + .settings(signing_settings) + .build() + .context("failed to build SigV4 signing params")?; + + let url = format!("https://{db_hostname}:{port}/?Action=connect&DBUser={db_username}"); + + let signable_request = + SignableRequest::new("GET", &url, std::iter::empty(), SignableBody::Bytes(&[])) + .context("failed to create signable request")?; + + let (signing_instructions, _signature) = sign(signable_request, &signing_params.into()) + .context("SigV4 signing failed")? + .into_parts(); + + let mut url = url::Url::parse(&url).context("failed to parse token URL")?; + for (name, value) in signing_instructions.params() { + url.query_pairs_mut().append_pair(name, value); + } + + // Strip "https://" — the token is the signed URL without scheme + Ok(url.to_string().split_off("https://".len())) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_standard_postgres_url() { + let params = ConnectParameters::from_db_url( + "postgres://listener_iam@my-db.rds.amazonaws.com:5432/listener", + None, + ) + .unwrap(); + + assert_eq!(params.host, "my-db.rds.amazonaws.com"); + assert_eq!(params.port, 5432); + assert_eq!(params.username, "listener_iam"); + assert_eq!(params.database, "listener"); + assert!(params.ssl_ca_path.is_none()); + } + + #[test] + fn parses_url_with_password_ignores_it() { + let params = + ConnectParameters::from_db_url("postgres://user:secret@host:5432/mydb", None).unwrap(); + + // Password is ignored — IAM token replaces it + assert_eq!(params.username, "user"); + assert_eq!(params.host, "host"); + assert_eq!(params.database, "mydb"); + } + + #[test] + fn default_port_when_omitted() { + let params = ConnectParameters::from_db_url("postgres://user@host/db", None).unwrap(); + assert_eq!(params.port, 5432); + } + + #[test] + fn missing_host_rejected() { + let result = ConnectParameters::from_db_url("postgres:///db", None); + assert!(result.is_err()); + } + + #[test] + fn missing_username_rejected() { + let result = ConnectParameters::from_db_url("postgres://host/db", None); + assert!(result.is_err()); + } + + #[test] + fn missing_database_rejected() { + let result = ConnectParameters::from_db_url("postgres://user@host", None); + assert!(result.is_err()); + } + + #[test] + fn ssl_ca_path_preserved() { + let params = ConnectParameters::from_db_url( + "postgres://user@host/db", + Some("/etc/ssl/rds/ca.pem".into()), + ) + .unwrap(); + + assert_eq!(params.ssl_ca_path.as_deref(), Some("/etc/ssl/rds/ca.pem")); + } + + #[test] + fn invalid_url_rejected() { + let result = ConnectParameters::from_db_url("not-a-url", None); + assert!(result.is_err()); + } +} diff --git a/listener/crates/listener_core/src/store/migration.rs b/listener/crates/listener_core/src/store/migration.rs new file mode 100644 index 0000000000..a73f42c40b --- /dev/null +++ b/listener/crates/listener_core/src/store/migration.rs @@ -0,0 +1,49 @@ +use super::PgClient; +use sqlx::migrate::Migrator; +use std::path::Path; +use std::time::Duration; +use tracing::{error, info}; + +pub async fn run_migrations(client: &PgClient, max_attempts: u32) { + let migrations_path = Path::new("./migrations"); + + for attempt in 1..=max_attempts { + info!( + "Running database migrations (attempt {}/{})", + attempt, max_attempts + ); + + let migrator = match Migrator::new(migrations_path).await { + Ok(m) => m, + Err(e) => { + error!("Failed to load migrations: {}", e); + if attempt < max_attempts { + tokio::time::sleep(Duration::from_secs(2)).await; + continue; + } + panic!( + "Failed to load migrations after {} attempts: {}", + max_attempts, e + ); + } + }; + + match migrator.run(client.pool()).await { + Ok(_) => { + info!("Database migrations completed successfully"); + return; + } + Err(e) => { + error!("Migration attempt {} failed: {}", attempt, e); + if attempt < max_attempts { + tokio::time::sleep(Duration::from_secs(2)).await; + } else { + panic!( + "Database migrations failed after {} attempts: {}", + max_attempts, e + ); + } + } + } + } +} diff --git a/listener/crates/listener_core/src/store/mod.rs b/listener/crates/listener_core/src/store/mod.rs new file mode 100644 index 0000000000..40f7b83a27 --- /dev/null +++ b/listener/crates/listener_core/src/store/mod.rs @@ -0,0 +1,17 @@ +mod client; +mod error; +#[cfg(feature = "iam-auth")] +mod iam_auth; +#[cfg(feature = "iam-auth")] +pub use iam_auth::connect_iam; +pub mod flow_lock; +mod migration; +pub mod models; +pub mod repositories; + +pub use client::PgClient; +pub use error::{SqlError, SqlResult}; +pub use flow_lock::{FlowLock, FlowLockGuard}; +pub use migration::run_migrations; +pub use models::{Block, BlockStatus, Filter, NewDatabaseBlock, UpsertResult}; +pub use repositories::{BlockRepository, FilterRepository}; diff --git a/listener/crates/listener_core/src/store/models/block_model.rs b/listener/crates/listener_core/src/store/models/block_model.rs new file mode 100644 index 0000000000..475a9d0e0f --- /dev/null +++ b/listener/crates/listener_core/src/store/models/block_model.rs @@ -0,0 +1,69 @@ +use alloy::network::AnyRpcBlock; +use alloy::primitives::B256; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +/// Maps to PostgreSQL enum type `block_status` +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, sqlx::Type)] +#[sqlx(type_name = "block_status", rename_all = "SCREAMING_SNAKE_CASE")] +pub enum BlockStatus { + Canonical, + Finalized, + Uncle, +} + +/// Represents a row in the `blocks` table +#[derive(Debug, Clone)] +pub struct Block { + pub id: Uuid, + pub chain_id: i64, + pub block_number: u64, + pub block_hash: B256, + pub parent_hash: B256, + pub status: BlockStatus, + pub created_at: DateTime, +} + +/// Input for creating a new block +#[derive(Debug, Clone)] +pub struct NewDatabaseBlock { + pub block_number: u64, + pub block_hash: B256, + pub parent_hash: B256, + pub status: BlockStatus, +} + +impl NewDatabaseBlock { + pub fn from_rpc_block(block: &AnyRpcBlock, status: BlockStatus) -> Self { + Self { + block_number: block.header.number, + block_hash: block.header.hash, + parent_hash: block.header.parent_hash, + status, + } + } +} + +/// Result of an upsert_block_canonical operation. +/// +/// # Status Codes +/// - `Inserted (0)` - A new block was inserted into the database +/// - `Updated (1)` - An existing block was updated (was UNCLE, now CANONICAL) +/// - `NoOp (2)` - Block was already CANONICAL, no changes made +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u8)] +pub enum UpsertResult { + /// A new block was inserted (status code: 0) + Inserted = 0, + /// An existing block was updated to CANONICAL (status code: 1) + Updated = 1, + /// Block was already CANONICAL, no operation performed (status code: 2) + NoOp = 2, +} + +impl UpsertResult { + pub fn as_code(&self) -> u8 { + *self as u8 + } +} diff --git a/listener/crates/listener_core/src/store/models/filter_model.rs b/listener/crates/listener_core/src/store/models/filter_model.rs new file mode 100644 index 0000000000..e8d9609e6a --- /dev/null +++ b/listener/crates/listener_core/src/store/models/filter_model.rs @@ -0,0 +1,14 @@ +use chrono::{DateTime, Utc}; +use uuid::Uuid; + +/// Represents a row in the `filters` table +#[derive(Debug, Clone)] +pub struct Filter { + pub id: Uuid, + pub chain_id: i64, + pub consumer_id: String, + pub from: Option, + pub to: Option, + pub log_address: Option, + pub created_at: DateTime, +} diff --git a/listener/crates/listener_core/src/store/models/mod.rs b/listener/crates/listener_core/src/store/models/mod.rs new file mode 100644 index 0000000000..dbfdd3acd2 --- /dev/null +++ b/listener/crates/listener_core/src/store/models/mod.rs @@ -0,0 +1,5 @@ +pub mod block_model; +pub mod filter_model; + +pub use block_model::{Block, BlockStatus, NewDatabaseBlock, UpsertResult}; +pub use filter_model::Filter; diff --git a/listener/crates/listener_core/src/store/repositories/block_repo.rs b/listener/crates/listener_core/src/store/repositories/block_repo.rs new file mode 100644 index 0000000000..f71960de29 --- /dev/null +++ b/listener/crates/listener_core/src/store/repositories/block_repo.rs @@ -0,0 +1,372 @@ +use alloy::primitives::B256; +use sqlx::Acquire; +use std::sync::Arc; +use uuid::Uuid; + +use crate::store::client::PgClient; +use crate::store::error::SqlResult; +use crate::store::models::{Block, BlockStatus, NewDatabaseBlock, UpsertResult}; + +#[derive(Clone)] +pub struct BlockRepository { + client: Arc, + chain_id: i64, +} + +impl BlockRepository { + pub fn new(client: Arc, chain_id: i64) -> Self { + Self { client, chain_id } + } + + /// Insert a new block into the database. + pub async fn insert_block(&self, block: &NewDatabaseBlock) -> SqlResult { + let mut conn = self.client.get_app_connection().await?; + let id = Uuid::new_v4(); + + let row = sqlx::query!( + r#" + INSERT INTO blocks (id, chain_id, block_number, block_hash, parent_hash, status) + VALUES ($1, $2, $3, $4, $5, $6) + RETURNING id, chain_id, block_number, block_hash, parent_hash, status as "status: BlockStatus", created_at + "#, + id, + self.chain_id, + block.block_number as i64, + block.block_hash.as_slice(), + block.parent_hash.as_slice(), + block.status as BlockStatus + ) + .fetch_one(&mut *conn) + .await?; + + Ok(Block { + id: row.id, + chain_id: row.chain_id, + block_number: row.block_number as u64, + block_hash: B256::from_slice(&row.block_hash), + parent_hash: B256::from_slice(&row.parent_hash), + status: row.status, + created_at: row.created_at, + }) + } + + /// Upsert a block as CANONICAL and mark other blocks at the same height as UNCLE. + /// + /// # Returns + /// - `UpsertResult::Inserted` - A new block was inserted + /// - `UpsertResult::Updated` - An existing block was updated (was UNCLE, now CANONICAL) + /// - `UpsertResult::NoOp` - Block was already CANONICAL or FINALIZED, no changes made + pub async fn upsert_block_canonical( + &self, + block: &NewDatabaseBlock, + ) -> SqlResult { + let mut conn = self.client.get_app_connection().await?; + let mut tx = conn.begin().await?; + + // Step 1: Check if block exists and its current status + let existing = sqlx::query!( + r#" + SELECT status as "status: BlockStatus" FROM blocks WHERE chain_id = $1 AND block_hash = $2 + "#, + self.chain_id, + block.block_hash.as_slice() + ) + .fetch_optional(&mut *tx) + .await?; + + // Early return: no status change needed for CANONICAL or FINALIZED + if let Some(ref row) = existing + && (row.status == BlockStatus::Canonical || row.status == BlockStatus::Finalized) + { + tx.commit().await?; + return Ok(UpsertResult::NoOp); + } + + // Step 2: Mark other canonical blocks at same height as UNCLE (before INSERT to + // avoid violating the partial unique index on (chain_id, block_number) WHERE status = 'CANONICAL') + sqlx::query!( + r#" + UPDATE blocks SET status = 'UNCLE'::block_status + WHERE chain_id = $1 AND block_number = $2 AND block_hash != $3 AND status = 'CANONICAL'::block_status + "#, + self.chain_id, + block.block_number as i64, + block.block_hash.as_slice() + ) + .execute(&mut *tx) + .await?; + + // Step 3: Insert or update the block as CANONICAL + let result = match existing { + None => { + let id = Uuid::new_v4(); + sqlx::query!( + r#" + INSERT INTO blocks (id, chain_id, block_number, block_hash, parent_hash, status) + VALUES ($1, $2, $3, $4, $5, 'CANONICAL'::block_status) + "#, + id, + self.chain_id, + block.block_number as i64, + block.block_hash.as_slice(), + block.parent_hash.as_slice() + ) + .execute(&mut *tx) + .await?; + + UpsertResult::Inserted + } + Some(_) => { + sqlx::query!( + r#" + UPDATE blocks SET status = 'CANONICAL'::block_status WHERE chain_id = $1 AND block_hash = $2 + "#, + self.chain_id, + block.block_hash.as_slice() + ) + .execute(&mut *tx) + .await?; + + UpsertResult::Updated + } + }; + + tx.commit().await?; + Ok(result) + } + + /// Atomically upsert a batch of blocks as CANONICAL within a single DB transaction. + /// + /// Mirrors [`upsert_block_canonical`] semantics exactly, per block: + /// 1. If the block already exists as CANONICAL or FINALIZED → `NoOp` (FINALIZED is never demoted). + /// 2. Demote any other CANONICAL block at the same height to UNCLE. + /// 3. Insert the new block or promote an existing UNCLE to CANONICAL. + /// + /// All-or-nothing: if any SQL operation fails mid-batch, the entire transaction + /// is rolled back and no DB state has changed. + /// + /// # Arguments + /// * `blocks` - Slice of blocks to upsert. Order matters only for logging; + /// ascending height order is conventional. + /// + /// # Returns + /// A `Vec` parallel to the input, one entry per block. + pub async fn batch_upsert_blocks_canonical( + &self, + blocks: &[NewDatabaseBlock], + ) -> SqlResult> { + if blocks.is_empty() { + return Ok(vec![]); + } + + let mut conn = self.client.get_app_connection().await?; + let mut tx = conn.begin().await?; + let mut results = Vec::with_capacity(blocks.len()); + + for block in blocks { + // Step 1: Check if block exists and its current status + // NOTE: SQL string whitespace must match upsert_block_canonical exactly + // for sqlx offline cache to reuse the same query hash. + let existing = sqlx::query!( + r#" + SELECT status as "status: BlockStatus" FROM blocks WHERE chain_id = $1 AND block_hash = $2 + "#, + self.chain_id, + block.block_hash.as_slice() + ) + .fetch_optional(&mut *tx) + .await?; + + // Early continue: no status change needed for CANONICAL or FINALIZED + // (mirrors upsert_block_canonical lines 77-83 exactly) + if let Some(ref row) = existing + && (row.status == BlockStatus::Canonical || row.status == BlockStatus::Finalized) + { + results.push(UpsertResult::NoOp); + continue; + } + + // Step 2: Mark other canonical blocks at same height as UNCLE + sqlx::query!( + r#" + UPDATE blocks SET status = 'UNCLE'::block_status + WHERE chain_id = $1 AND block_number = $2 AND block_hash != $3 AND status = 'CANONICAL'::block_status + "#, + self.chain_id, + block.block_number as i64, + block.block_hash.as_slice() + ) + .execute(&mut *tx) + .await?; + + // Step 3: Insert or update the block as CANONICAL + match existing { + None => { + let id = Uuid::new_v4(); + sqlx::query!( + r#" + INSERT INTO blocks (id, chain_id, block_number, block_hash, parent_hash, status) + VALUES ($1, $2, $3, $4, $5, 'CANONICAL'::block_status) + "#, + id, + self.chain_id, + block.block_number as i64, + block.block_hash.as_slice(), + block.parent_hash.as_slice() + ) + .execute(&mut *tx) + .await?; + + results.push(UpsertResult::Inserted); + } + Some(_) => { + // Block exists as UNCLE → promote to CANONICAL + sqlx::query!( + r#" + UPDATE blocks SET status = 'CANONICAL'::block_status WHERE chain_id = $1 AND block_hash = $2 + "#, + self.chain_id, + block.block_hash.as_slice() + ) + .execute(&mut *tx) + .await?; + + results.push(UpsertResult::Updated); + } + } + } + + tx.commit().await?; + Ok(results) + } + + /// Get the canonical block at a specific block number. + /// Returns None if no canonical block exists at that height. + pub async fn get_canonical_block_by_number( + &self, + block_number: u64, + ) -> SqlResult> { + let mut conn = self.client.get_app_connection().await?; + + let row = sqlx::query!( + r#" + SELECT id, chain_id, block_number, block_hash, parent_hash, status as "status: BlockStatus", created_at + FROM blocks + WHERE chain_id = $1 AND block_number = $2 AND status = 'CANONICAL'::block_status + "#, + self.chain_id, + block_number as i64 + ) + .fetch_optional(&mut *conn) + .await?; + + Ok(row.map(|r| Block { + id: r.id, + chain_id: r.chain_id, + block_number: r.block_number as u64, + block_hash: B256::from_slice(&r.block_hash), + parent_hash: B256::from_slice(&r.parent_hash), + status: r.status, + created_at: r.created_at, + })) + } + + /// Delete blocks older than the specified number of seconds (finality cleanup). + /// + /// # Returns + /// Number of blocks deleted + pub async fn delete_blocks_before_timestamp(&self, seconds: i64) -> SqlResult { + let mut conn = self.client.get_app_connection().await?; + + let result = sqlx::query!( + r#" + DELETE FROM blocks + WHERE chain_id = $1 AND created_at < NOW() - make_interval(secs => $2) + "#, + self.chain_id, + seconds as f64 + ) + .execute(&mut *conn) + .await?; + + Ok(result.rows_affected()) + } + + /// Keep only the N most recent blocks, delete the rest. + /// + /// # Returns + /// Number of blocks deleted + pub async fn delete_blocks_keeping_latest(&self, keep_count: i64) -> SqlResult { + let mut conn = self.client.get_app_connection().await?; + + let result = sqlx::query!( + r#" + WITH ranked_blocks AS ( + SELECT id, + ROW_NUMBER() OVER ( + ORDER BY block_number DESC, created_at DESC + ) as rn + FROM blocks + WHERE chain_id = $1 + ) + DELETE FROM blocks + WHERE chain_id = $1 AND id IN (SELECT id FROM ranked_blocks WHERE rn > $2) + "#, + self.chain_id, + keep_count + ) + .execute(&mut *conn) + .await?; + + Ok(result.rows_affected()) + } + + /// Get the lowest block number stored for this chain. + /// + /// Returns `None` if no blocks exist. + pub async fn get_min_block_number(&self) -> SqlResult> { + let mut conn = self.client.get_app_connection().await?; + + let row = sqlx::query!( + r#" + SELECT MIN(block_number) as "min_block_number: i64" + FROM blocks + WHERE chain_id = $1 + "#, + self.chain_id + ) + .fetch_one(&mut *conn) + .await?; + + Ok(row.min_block_number) + } + + /// Get the latest canonical block (highest block_number with status CANONICAL). + /// Returns None if no canonical block exists in the database. + pub async fn get_latest_canonical_block(&self) -> SqlResult> { + let mut conn = self.client.get_app_connection().await?; + + let row = sqlx::query!( + r#" + SELECT id, chain_id, block_number, block_hash, parent_hash, status as "status: BlockStatus", created_at + FROM blocks + WHERE chain_id = $1 AND status = 'CANONICAL'::block_status + ORDER BY block_number DESC + LIMIT 1 + "#, + self.chain_id + ) + .fetch_optional(&mut *conn) + .await?; + + Ok(row.map(|r| Block { + id: r.id, + chain_id: r.chain_id, + block_number: r.block_number as u64, + block_hash: B256::from_slice(&r.block_hash), + parent_hash: B256::from_slice(&r.parent_hash), + status: r.status, + created_at: r.created_at, + })) + } +} diff --git a/listener/crates/listener_core/src/store/repositories/filter_repo.rs b/listener/crates/listener_core/src/store/repositories/filter_repo.rs new file mode 100644 index 0000000000..d3edaf8330 --- /dev/null +++ b/listener/crates/listener_core/src/store/repositories/filter_repo.rs @@ -0,0 +1,108 @@ +use std::sync::Arc; +use uuid::Uuid; + +use crate::store::client::PgClient; +use crate::store::error::SqlResult; +use crate::store::models::Filter; + +#[derive(Clone)] +pub struct FilterRepository { + client: Arc, + chain_id: i64, +} + +impl FilterRepository { + pub fn new(client: Arc, chain_id: i64) -> Self { + Self { client, chain_id } + } + + /// Insert a new filter. + /// + /// - If filter doesn't exist → inserts new row, returns `Some(Filter)` + /// - If filter already exists (conflict) → no-op, returns `None` + pub async fn add_filter( + &self, + consumer_id: &str, + from: Option<&str>, + to: Option<&str>, + log_address: Option<&str>, + ) -> SqlResult> { + let mut conn = self.client.get_app_connection().await?; + let id = Uuid::new_v4(); + + let row = sqlx::query_as!( + Filter, + r#" + INSERT INTO filters (id, chain_id, consumer_id, "from", "to", "log_address") + VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT (chain_id, consumer_id, COALESCE("from", ''), COALESCE("to", ''), COALESCE("log_address", '')) + DO NOTHING + RETURNING id, chain_id, consumer_id, "from", "to", "log_address", created_at + "#, + id, + self.chain_id, + consumer_id, + from as Option<&str>, + to as Option<&str>, + log_address as Option<&str>, + ) + .fetch_optional(&mut *conn) + .await?; + + Ok(row) + } + + /// Fetch all active filters for this chain_id. + /// Results are ordered by consumer_id for efficient grouping. + pub async fn get_filters_by_chain_id(&self) -> SqlResult> { + let mut conn = self.client.get_app_connection().await?; + let rows = sqlx::query_as!( + Filter, + r#" + SELECT id, chain_id, consumer_id, "from", "to", "log_address", created_at + FROM filters + WHERE chain_id = $1 + ORDER BY consumer_id + "#, + self.chain_id, + ) + .fetch_all(&mut *conn) + .await?; + Ok(rows) + } + + /// Remove a filter matching the given (chain_id, consumer_id, from, to). + /// + /// Returns `Some(Filter)` if a filter was removed, `None` if no matching filter found. + pub async fn remove_filter( + &self, + consumer_id: &str, + from: Option<&str>, + to: Option<&str>, + log_address: Option<&str>, + ) -> SqlResult> { + let mut conn = self.client.get_app_connection().await?; + + let row = sqlx::query_as!( + Filter, + r#" + DELETE FROM filters + WHERE chain_id = $1 + AND consumer_id = $2 + AND COALESCE("from", '') = COALESCE($3, '') + AND COALESCE("to", '') = COALESCE($4, '') + AND COALESCE("log_address", '') = COALESCE($5, '') + RETURNING id, chain_id, consumer_id, "from", "to", "log_address", created_at + "#, + self.chain_id, + consumer_id, + from as Option<&str>, + to as Option<&str>, + log_address as Option<&str>, + ) + .fetch_optional(&mut *conn) + .await?; + + Ok(row) + } +} diff --git a/listener/crates/listener_core/src/store/repositories/mod.rs b/listener/crates/listener_core/src/store/repositories/mod.rs new file mode 100644 index 0000000000..05d4a8705d --- /dev/null +++ b/listener/crates/listener_core/src/store/repositories/mod.rs @@ -0,0 +1,32 @@ +pub mod block_repo; +pub mod filter_repo; +use crate::store::client::PgClient; +pub use block_repo::BlockRepository; +pub use filter_repo::FilterRepository; +use std::sync::Arc; + +/// Centralized container for all SQL repositories. +/// +/// Provides a single initialization point for all repositories from database config, +/// reducing parameter passing and simplifying dependency management. +#[derive(Clone)] +pub struct Repositories { + pub blocks: BlockRepository, + pub filters: FilterRepository, + chain_id: i64, +} + +impl Repositories { + /// Create all repositories from database configuration. + pub fn new(client: Arc, chain_id: i64) -> Self { + Self { + blocks: BlockRepository::new(client.clone(), chain_id), + filters: FilterRepository::new(client, chain_id), + chain_id, + } + } + + pub fn chain_id(&self) -> i64 { + self.chain_id + } +} diff --git a/listener/crates/shared/broker/Cargo.toml b/listener/crates/shared/broker/Cargo.toml new file mode 100644 index 0000000000..034fcad206 --- /dev/null +++ b/listener/crates/shared/broker/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "broker" +version = "0.1.0" +edition.workspace = true + +[features] +default = ["redis", "amqp"] +redis = ["dep:redis"] +amqp = ["dep:lapin"] + +[dependencies] +async-trait.workspace = true +futures.workspace = true +metrics.workspace = true +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true, features = ["raw_value"] } +thiserror.workspace = true +tokio = { workspace = true, features = ["rt-multi-thread", "sync", "time", "macros"] } +tokio-util.workspace = true +tracing.workspace = true +uuid = { version = "1", features = ["v7"] } + +arc-swap = "1" + +# Backend-specific (optional) +redis = { version = "0.29", features = ["tokio-comp", "aio", "streams", "connection-manager"], optional = true } +lapin = { version = "4.0.0", optional = true } + +[dev-dependencies] +testcontainers = { version = "0.23", features = ["reusable-containers"] } +testcontainers-modules = { version = "0.11", features = ["redis", "rabbitmq"] } +test-support = { path = "../../test-support" } +metrics-util = "0.19" +ordered-float = "4" +serde = { workspace = true, features = ["derive"] } diff --git a/listener/crates/shared/broker/README.md b/listener/crates/shared/broker/README.md new file mode 100644 index 0000000000..190515f1ba --- /dev/null +++ b/listener/crates/shared/broker/README.md @@ -0,0 +1,491 @@ +# broker + +A simplified broker abstraction for Redis Streams and RabbitMQ. + +## Features + +- **Unified API** - Same code works on Redis or RabbitMQ +- **Direct Routing** - Messages to specific consumers +- **Fanout** - Broadcast to multiple consumer groups +- **Competing Consumers** - Load balancing within a group +- **Fluent Builder** - Easy configuration with sensible defaults +- **Graceful Shutdown** - CancellationToken support for clean consumer stop + +## Quick Start + +```rust +use broker::{Broker, Topic, routing, AsyncHandlerPayloadOnly}; + +#[tokio::main] +async fn main() -> Result<(), Box> { + // Connect to broker + let broker = Broker::redis("redis://localhost:6379").await?; + // Or: Broker::amqp("amqp://localhost:5672").build().await?; + + // Publisher + let publisher = broker.publisher("ethereum").await?; + publisher.publish("blocks", &serde_json::json!({"number": 12345})).await?; + + // Consumer + let topic = Topic::new(routing::BLOCKS).with_namespace("ethereum"); + let handler = AsyncHandlerPayloadOnly::new(|block: serde_json::Value| async move { + println!("Processing block: {:?}", block); + Ok::<(), std::convert::Infallible>(()) + }); + + broker.consumer(&topic) + .group("indexer") + .consumer_name("pod-1") + .prefetch(100) + .run(handler).await?; + + Ok(()) +} +``` + +### RabbitMQ Topology Default + +`Broker::amqp(...)` uses shared global exchanges by default: + +- `main` +- `retry` +- `dlx` + +Broker auto-declares this topology during publish/consume setup, so application +code does not need explicit AMQP topology calls. + +## How Topic Works + +`Topic` is the backend-agnostic route identifier. + +```rust +let topic = Topic::new("blocks").with_namespace("ethereum"); + +assert_eq!(topic.key(), "ethereum.blocks"); // qualified key +assert_eq!(topic.dead_key(), "ethereum.blocks:dead"); // dead-letter key +assert_eq!(topic.routing_segment(), "blocks"); // unqualified segment +``` + +At runtime, broker maps this topic differently per backend: + +- Redis: keys map to stream/dead-stream names. +- AMQP: key becomes the routing key on broker-managed shared exchanges. + +So your app code always uses `Topic`, and only infra wiring differs by backend. + +## Routing Model + +For `Topic::new("blocks").with_namespace("ethereum")` and `.group("indexer")`, routing is +shown separately per backend. + +### Redis Streams Model (Publish + Consume) + +```mermaid +flowchart LR + publisher["Publisher namespace=ethereum"] + consumer["Consumer"] + group["Consumer group: indexer"] + broker["Broker (Redis mode)"] + main["Main stream: ethereum.blocks"] + dead["Dead stream: ethereum.blocks:dead"] + + publisher --> broker -->|XADD| main + group --> consumer --> broker -->|XREADGROUP| main + main -->|dead-letter writes| dead +``` + +### RabbitMQ Model (Publish + Consume) + +```mermaid +flowchart LR + publisher["Publisher namespace=ethereum"] + consumer["Consumer"] + group["Group: indexer"] + broker["Broker (AMQP mode)"] + exmain["Exchange: main"] + exretry["Exchange: retry"] + exdlx["Exchange: dlx"] + qmain["Queue: ethereum.indexer"] + qretry["Queue: ethereum.indexer.retry"] + qerror["Queue: ethereum.indexer.error"] + + publisher --> broker -->|publish routing_key ethereum.blocks| exmain + broker -->|bind key ethereum.blocks| exmain + exmain --> qmain + + group -->|derive namespace.group| qmain + consumer --> broker -->|consume| qmain + + qmain -->|retry path| exretry --> qretry -->|TTL return| qmain + qmain -->|dead-letter path| exdlx --> qerror +``` + +### Key Concepts + +| Term | RabbitMQ | Redis | +|------|----------|-------| +| **Global Topology** | Exchanges `main/retry/dlx` + derived queues | Main + dead streams | +| **Exchange** | Fixed shared exchange (`main`) | N/A | +| **Namespace** | Routing key prefix | Stream prefix | +| **Routing** | Routing key suffix | Stream suffix | +| **Topic** | Full routing key (`namespace.routing`) | Full stream name (`namespace.routing`) | +| **Group** | Logical group name (queue derived as `namespace.group`) | Consumer group | +| **Queue Name** | Derived from `group` as `namespace.group` | N/A | +| **Consumer Name** | Consumer tag | Consumer name | + +## Generic Agnostic Usage + +The same publish/consume code path works for both Redis and AMQP. +Only broker construction differs. +This example runs **multiple consumers** (`blocks` + `forks`) with the same +application code on both backends. + +```rust +use broker::{routing, Broker, Topic, AsyncHandlerPayloadOnly}; + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +struct BlockEvent { + number: u64, +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +struct ForkEvent { + at_block: u64, +} + +async fn build_broker( + broker_type: &str, + broker_url: &str, +) -> Result> { + let broker = match broker_type { + "redis" => Broker::redis(broker_url).await?, + "amqp" => Broker::amqp(broker_url).build().await?, + other => return Err(format!("unsupported broker type: {other}").into()), + }; + + Ok(broker) +} + +async fn run_pipeline(broker: Broker) -> Result<(), Box> { + let blocks_topic = Topic::new(routing::BLOCKS).with_namespace("ethereum"); + let forks_topic = Topic::new(routing::FORKS).with_namespace("ethereum"); + + // Publisher scoped by namespace + let publisher = broker.publisher("ethereum").await?; + publisher + .publish_to_topic(&blocks_topic, &BlockEvent { number: 12345 }) + .await?; + publisher + .publish_to_topic(&forks_topic, &ForkEvent { at_block: 12340 }) + .await?; + + // Consumer setup is identical across backends. + // Only group names and topics differ. + let block_handler = AsyncHandlerPayloadOnly::new(|evt: BlockEvent| async move { + println!("[blocks] processing block {}", evt.number); + Ok::<(), std::convert::Infallible>(()) + }); + + let fork_handler = AsyncHandlerPayloadOnly::new(|evt: ForkEvent| async move { + println!("[forks] processing fork at {}", evt.at_block); + Ok::<(), std::convert::Infallible>(()) + }); + + let broker_blocks = broker.clone(); + let broker_forks = broker.clone(); + + tokio::select! { + result = broker_blocks + .consumer(&blocks_topic) + .group("indexer") + .consumer_name("pod-1") + .prefetch(100) + .run(block_handler) => { + eprintln!("blocks consumer exited: {:?}", result); + } + result = broker_forks + .consumer(&forks_topic) + .group("fork-handler") + .consumer_name("pod-1") + .prefetch(50) + .run(fork_handler) => { + eprintln!("forks consumer exited: {:?}", result); + } + } + + Ok(()) +} +``` + +## Error Classification & Handler Variants + +The broker distinguishes **transient** errors (infrastructure failures) from **permanent** +errors (bad message payload). This classification is backend-agnostic and drives two behaviors: + +- **Retry budget**: Transient errors get infinite retries (message is fine, infra is broken). + Permanent errors count against `max_retries` and route to dead-letter on exhaustion. +- **Circuit breaker**: When consecutive transient errors exceed a threshold, the consumer + **pauses consumption** entirely — preventing DLQ pollution during outages. After a cooldown, + one probe message is processed. If it succeeds, the circuit closes and consumption resumes. + +| Handler | Closure signature | Error classification | +|---------|-------------------|---------------------| +| `AsyncHandlerPayloadOnly` | `Fn(T) -> impl Future>` | All closure errors wrapped as `HandlerError::Execution` (permanent). Use when your handler only needs a success/fail distinction. | +| `AsyncHandlerPayloadClassified` | `Fn(T) -> impl Future>` | Error classification **preserved** as returned by the closure. Use when your handler needs to distinguish transient from permanent failures. | +| `AsyncHandlerNoArgs` | `Fn() -> impl Future>` | All errors wrapped as permanent. Use for side-effect-only handlers (heartbeats, timers). | + +### Choosing a handler + +Use `AsyncHandlerPayloadOnly` when all errors are equivalent, the consumer retries up to +`max_retries` and then dead-letters. This is the simplest option. + +Use **`AsyncHandlerPayloadClassified`** when your handler calls external infrastructure +(database, RPC, API) and you need transient failures to trip the circuit breaker while +permanent failures (bad payload) go straight to the retry/DLQ path: + +```rust +use broker::{AsyncHandlerPayloadClassified, HandlerError}; + +let handler = AsyncHandlerPayloadClassified::new(|block: BlockEvent| async move { + // Transient: infrastructure is broken, not the message. + // → Infinite retries, trips circuit breaker. + db.save(&block).await.map_err(HandlerError::transient)?; + + // Permanent: the message itself is invalid. + // → Counts against max_retries, resets CB transient counter. + verify_block(&block).map_err(HandlerError::permanent)?; + + Ok(()) +}); +``` + +> **Key difference**: `AsyncHandlerPayloadOnly` wraps *all* closure errors as +> `HandlerError::Execution` (permanent), so transient infra failures never trip the +> circuit breaker. `AsyncHandlerPayloadClassified` lets the closure return +> `HandlerError::Transient` or `HandlerError::Execution` directly, preserving the +> classification the consumer loop relies on. + +### Circuit breaker state machine + +```text + ┌──────────┐ threshold consecutive ┌──────────┐ cooldown ┌───────────┐ + │ Closed │ transient failures ──► │ Open │ expires ─► │ Half-Open │ + │ (normal) │ │ (paused) │ │ (probe) │ + └──────────┘ └──────────┘ └───────────┘ + ▲ │ + │ probe succeeds │ + └───────────────────────────────────────────────────────────────┘ + probe fails → back to Open +``` + +| State | Behavior | +|-------|----------| +| **Closed** | Normal consumption. Transient failures increment counter; successes/permanent failures reset it. | +| **Open** | No messages consumed. Waits for cooldown to expire. | +| **Half-Open** | One probe message processed. Success → Closed. Failure → Open (restart cooldown). | + +### Full example with circuit breaker + +```rust +use std::time::Duration; +use broker::{routing, Broker, Topic, AsyncHandlerPayloadClassified, HandlerError}; + +async fn run(broker: Broker) -> Result<(), Box> { + let topic = Topic::new(routing::BLOCKS).with_namespace("ethereum"); + + let handler = AsyncHandlerPayloadClassified::new(|block: BlockEvent| async move { + db.save(&block).await.map_err(HandlerError::transient)?; + verify_block(&block).map_err(HandlerError::permanent)?; + Ok(()) + }); + + broker.consumer(&topic) + .group("indexer") + .prefetch(100) + .max_retries(5) // permanent errors: 5 attempts then DLQ + .circuit_breaker(3, Duration::from_secs(30)) // 3 transient errors → pause 30s + .run(handler) + .await?; + + Ok(()) +} +``` + +Works identically on Redis Streams and RabbitMQ — the circuit breaker state persists +across reconnections on both backends. + +## Queue Inspection + +`Broker::exists()` checks whether the underlying queue or stream for a topic has been +created, without requiring a consumer group to exist. + +```rust +use broker::{Broker, Topic, routing}; + +let topic = Topic::new(routing::BLOCKS).with_namespace("ethereum"); + +if !broker.exists(&topic).await? { + // Queue/stream hasn't been created yet — skip depth check +} +``` + +| Backend | Mechanism | Returns `true` when | +|---------|-----------|---------------------| +| **Redis** | `TYPE` command on the key | Key exists and is a `stream` | +| **AMQP** | Passive `queue_declare` | Broker acknowledges the queue (404 → `false`) | + +## Consumer Patterns + +### Direct Routing + +Publisher sends to routing key, only matching queue/group receives: + +```rust +// Only receives "blocks" messages +broker.consumer(&Topic::new("blocks").with_namespace("ethereum")) + .group("block-processor") + .run(handler).await?; +``` + +### Fanout (Multiple Groups) + +Multiple groups bind to same routing key, all receive a copy: + +```rust +let blocks_topic = Topic::new("blocks").with_namespace("ethereum"); + +// Both groups receive ALL block messages independently +broker.consumer(&blocks_topic).group("indexer").run(handler1).await?; +broker.consumer(&blocks_topic).group("analytics").run(handler2).await?; +``` + +### Competing Consumers (Load Balancing) + +Same group, different consumer names, messages load-balanced: + +```rust +// Pod 1 +broker.consumer(&topic) + .group("indexer") + .consumer_name("pod-1") + .run(handler).await?; + +// Pod 2 - shares the load with pod-1 +broker.consumer(&topic) + .group("indexer") + .consumer_name("pod-2") + .run(handler).await?; +``` + +## Configuration Options + +### Common Options + +```rust +broker.consumer(&topic) + .group("worker") // Required: group name + .consumer_name("pod-1") // Optional: instance name + .prefetch(100) // Messages to buffer (default: 10) + .max_retries(5) // Before dead-letter (default: 3) + .circuit_breaker(3, Duration::from_secs(30)) + .run(handler).await?; +``` + +### Redis-Specific + +```rust +broker.consumer(&topic) + .group("worker") + .redis_block_ms(10000) // Block for 10s on empty stream + .redis_claim_min_idle(60) // Reclaim stuck messages after 60s + .redis_claim_interval(10) // Check every 10s + .run(handler).await?; +``` + +### AMQP-Specific + +```rust +broker.consumer(&topic) + .group("worker") + .amqp_retry_delay(10) // 10s between retries + .amqp_routing_pattern("ethereum.blocks.#") // Wildcard subscription + .run(handler).await?; +``` + +## Building and Running Consumers + +### Option 1: Build and Run Immediately + +```rust +broker.consumer(&topic) + .group("worker") + .run(handler).await?; +``` + +### Option 2: Build First, Run Later + +```rust +// Build consumers +let blocks_consumer = broker.consumer(&Topic::new("blocks").with_namespace("ethereum")) + .group("block-processor") + .prefetch(100) + .build()?; + +let forks_consumer = broker.consumer(&Topic::new("forks").with_namespace("ethereum")) + .group("fork-handler") + .build()?; + +// Run with tokio::spawn +let h1 = tokio::spawn(blocks_consumer.run(block_handler)); +let h2 = tokio::spawn(forks_consumer.run(fork_handler)); + +futures::future::join_all(vec![h1, h2]).await; +``` + +## Multi-Namespace Setup + +```rust +let broker = Broker::redis("redis://localhost:6379").await?; + +for ns in ["ethereum", "polygon", "arbitrum"] { + let topic = Topic::new("blocks").with_namespace(ns); + let broker = broker.clone(); + + tokio::spawn(async move { + broker.consumer(&topic) + .group("indexer") // AMQP queue becomes "{ns}.indexer" + .consumer_name("pod-1") + .run(handler).await + }); +} +``` + +## Graceful Shutdown + +Use `CancellationToken` to stop consumers cleanly: + +```rust +use broker::{Broker, Topic, routing, CancellationToken}; + +let token = CancellationToken::new(); + +let consumer = broker.consumer(&topic) + .group("indexer") + .with_cancellation(token.clone()) + .build()?; + +// Run in a task +let handle = tokio::spawn(consumer.run(handler)); + +// Later, trigger graceful shutdown +token.cancel(); +handle.await??; +``` + +## Routing Keys + +Use custom routing keys: + +```rust +let topic = Topic::new("erc20-transfers").with_namespace("ethereum"); +``` diff --git a/listener/crates/shared/broker/src/amqp/config.rs b/listener/crates/shared/broker/src/amqp/config.rs new file mode 100644 index 0000000000..009ebdc38d --- /dev/null +++ b/listener/crates/shared/broker/src/amqp/config.rs @@ -0,0 +1,411 @@ +use std::time::Duration; + +use crate::traits::circuit_breaker::CircuitBreakerConfig; +use crate::traits::consumer::RetryPolicy; + +use super::error::ConsumerError; + +/// Exchange topology for a specific chain. +/// Provides consistent naming for main, retry, and dead-letter exchanges. +#[derive(Debug, Clone)] +pub struct ExchangeTopology { + /// Main exchange name (e.g., "ethereum.events") + pub main: String, + /// Retry exchange name (e.g., "ethereum.events.retry") + pub retry: String, + /// Dead-letter exchange name (e.g., "ethereum.events.dlx") + pub dlx: String, +} + +impl ExchangeTopology { + /// Explicit constructor for full control over exchange names. + pub fn new(main: impl Into, retry: impl Into, dlx: impl Into) -> Self { + Self { + main: main.into(), + retry: retry.into(), + dlx: dlx.into(), + } + } + + /// Derive topology from a base prefix. + /// Example: `"orders.events"` -> retry=`"orders.events.retry"`, dlx=`"orders.events.dlx"` + /// + /// # Example + /// ``` + /// use broker::amqp::ExchangeTopology; + /// let topology = ExchangeTopology::from_prefix("ethereum.events"); + /// assert_eq!(topology.main, "ethereum.events"); + /// assert_eq!(topology.retry, "ethereum.events.retry"); + /// assert_eq!(topology.dlx, "ethereum.events.dlx"); + /// ``` + pub fn from_prefix(prefix: impl AsRef) -> Self { + let prefix = prefix.as_ref(); + Self::new(prefix, format!("{prefix}.retry"), format!("{prefix}.dlx")) + } +} + +/// Base consumer configuration. +#[derive(Debug, Clone)] +pub struct ConsumerConfig { + /// Exchange to consume from + pub exchange: String, + /// Queue name + pub queue: String, + /// Routing key for binding + pub routing_key: String, + /// Unique consumer tag + pub consumer_tag: String, +} + +/// Consumer configuration with retry support. +#[derive(Debug, Clone)] +pub struct RetryConfig { + /// Base consumer configuration + pub base: ConsumerConfig, + /// Retry exchange name + pub retry_exchange: String, + /// Dead-letter exchange name + pub dead_exchange: String, + /// Maximum retry attempts for permanent failures before DLQ routing. + /// + /// `HandlerError::Transient` retries are infinite and do not consume this budget. + pub max_retries: u32, + /// Delay between retries + pub retry_delay: Duration, + /// Optional circuit breaker — pauses consumption on consecutive `Transient` + /// handler errors, preventing DLQ pollution during downstream outages + /// (DB down, API timeout, etc.). When `None`, all errors go through the + /// normal retry/DLQ path. + pub circuit_breaker: Option, +} + +impl RetryConfig { + const RETRY_ROUTING_PREFIX: &'static str = "__mq.retry"; + const DEAD_ROUTING_PREFIX: &'static str = "__mq.dead"; + + /// Build a queue-scoped retry routing key for internal retry plumbing. + pub(crate) fn retry_routing_key_for_queue(queue: &str) -> String { + format!("{}.{}", Self::RETRY_ROUTING_PREFIX, queue) + } + + /// Build a queue-scoped dead routing key for internal dead-letter plumbing. + pub(crate) fn dead_routing_key_for_queue(queue: &str) -> String { + format!("{}.{}", Self::DEAD_ROUTING_PREFIX, queue) + } + + /// Queue-scoped retry routing key used for internal retry plumbing. + /// + /// This isolates retries per queue even when multiple queues are bound to + /// the same retry exchange. + pub(crate) fn retry_routing_key(&self) -> String { + Self::retry_routing_key_for_queue(&self.base.queue) + } + + /// Queue-scoped dead-letter routing key used for internal DLQ plumbing. + /// + /// This isolates DLQ routing per queue even when multiple queues share + /// the same dead-letter exchange. + pub(crate) fn dead_routing_key(&self) -> String { + Self::dead_routing_key_for_queue(&self.base.queue) + } +} + +/// Consumer configuration with retry and prefetch support for high-throughput. +#[derive(Debug, Clone)] +pub struct PrefetchConfig { + /// Retry configuration + pub retry: RetryConfig, + /// Number of messages to prefetch (QoS) + pub prefetch_count: u16, +} + +impl RetryPolicy for RetryConfig { + fn max_retries(&self) -> u32 { + self.max_retries + } + + fn retry_delay(&self) -> Duration { + self.retry_delay + } +} + +/// Consumer configuration for cron-style scheduled jobs. +#[derive(Debug, Clone)] +pub struct CronConfig { + /// Base consumer configuration + pub base: ConsumerConfig, + /// Retry/delay exchange name + pub retry_exchange: String, + /// Interval between job executions + pub interval: Duration, +} + +/// Builder for constructing consumer configurations with validation. +#[must_use] +#[derive(Debug, Default)] +pub struct ConsumerConfigBuilder { + exchange: Option, + queue: Option, + routing_key: Option, + consumer_tag: Option, + retry_exchange: Option, + dead_exchange: Option, + max_retries: Option, + retry_delay: Option, + prefetch_count: Option, + cron_interval: Option, + cb_failure_threshold: Option, + cb_cooldown_duration: Option, +} + +impl ConsumerConfigBuilder { + pub fn new() -> Self { + Self::default() + } + + pub fn exchange(mut self, exchange: impl Into) -> Self { + self.exchange = Some(exchange.into()); + self + } + + pub fn queue(mut self, queue: impl Into) -> Self { + self.queue = Some(queue.into()); + self + } + + pub fn routing_key(mut self, routing_key: impl Into) -> Self { + self.routing_key = Some(routing_key.into()); + self + } + + pub fn consumer_tag(mut self, consumer_tag: impl Into) -> Self { + self.consumer_tag = Some(consumer_tag.into()); + self + } + + pub fn retry_exchange(mut self, retry_exchange: impl Into) -> Self { + self.retry_exchange = Some(retry_exchange.into()); + self + } + + pub fn dead_exchange(mut self, dead_exchange: impl Into) -> Self { + self.dead_exchange = Some(dead_exchange.into()); + self + } + + /// Set the maximum retry attempts for permanent failures. + /// + /// This limit is only applied to `Execution`/`Deserialization` failures. + /// `Transient` failures always retry indefinitely. + pub fn max_retries(mut self, max_retries: u32) -> Self { + self.max_retries = Some(max_retries); + self + } + + pub fn retry_delay(mut self, retry_delay: Duration) -> Self { + self.retry_delay = Some(retry_delay); + self + } + + pub fn prefetch_count(mut self, prefetch_count: u16) -> Self { + self.prefetch_count = Some(prefetch_count); + self + } + + pub fn cron_interval(mut self, interval: Duration) -> Self { + self.cron_interval = Some(interval); + self + } + + /// Set the circuit breaker failure threshold (consecutive `Transient` errors to trip). + /// + /// When both `circuit_breaker_threshold` and `circuit_breaker_cooldown` are set, + /// the consumer will pause consumption after this many consecutive transient failures. + /// When not set, no circuit breaker is used. + pub fn circuit_breaker_threshold(mut self, threshold: u32) -> Self { + self.cb_failure_threshold = Some(threshold); + self + } + + /// Set the circuit breaker cooldown duration (how long to pause before probing). + /// + /// After the circuit trips, the consumer pauses for this duration, then allows + /// one test message through (half-open). If it succeeds, consumption resumes. + pub fn circuit_breaker_cooldown(mut self, cooldown: Duration) -> Self { + self.cb_cooldown_duration = Some(cooldown); + self + } + + /// Apply exchange topology to configure exchanges automatically. + pub fn with_topology(mut self, topology: &ExchangeTopology) -> Self { + self.exchange = Some(topology.main.clone()); + self.retry_exchange = Some(topology.retry.clone()); + self.dead_exchange = Some(topology.dlx.clone()); + self + } + + fn build_base(&self) -> Result { + Ok(ConsumerConfig { + exchange: self + .exchange + .clone() + .ok_or_else(|| ConsumerError::Configuration("exchange is required".into()))?, + queue: self + .queue + .clone() + .ok_or_else(|| ConsumerError::Configuration("queue is required".into()))?, + routing_key: self + .routing_key + .clone() + .ok_or_else(|| ConsumerError::Configuration("routing_key is required".into()))?, + consumer_tag: self + .consumer_tag + .clone() + .ok_or_else(|| ConsumerError::Configuration("consumer_tag is required".into()))?, + }) + } + + fn build_retry(&self) -> Result { + let base = self.build_base()?; + + let circuit_breaker = match (self.cb_failure_threshold, self.cb_cooldown_duration) { + (Some(threshold), Some(cooldown)) => Some(CircuitBreakerConfig { + failure_threshold: threshold, + cooldown_duration: cooldown, + }), + (Some(threshold), None) => Some(CircuitBreakerConfig { + failure_threshold: threshold, + ..CircuitBreakerConfig::default() + }), + (None, Some(cooldown)) => Some(CircuitBreakerConfig { + cooldown_duration: cooldown, + ..CircuitBreakerConfig::default() + }), + (None, None) => None, + }; + + Ok(RetryConfig { + base, + retry_exchange: self + .retry_exchange + .clone() + .ok_or_else(|| ConsumerError::Configuration("retry_exchange is required".into()))?, + dead_exchange: self + .dead_exchange + .clone() + .ok_or_else(|| ConsumerError::Configuration("dead_exchange is required".into()))?, + max_retries: self.max_retries.unwrap_or(3), + retry_delay: self.retry_delay.unwrap_or(Duration::from_secs(5)), + circuit_breaker, + }) + } + + /// Build a prefetch consumer configuration. + pub fn build_prefetch(self) -> Result { + let prefetch_count = self.prefetch_count.unwrap_or(10); + let retry = self.build_retry()?; + + Ok(PrefetchConfig { + retry, + prefetch_count, + }) + } + + /// Build a cron consumer configuration. + pub fn build_cron(self) -> Result { + let base = self.build_base()?; + + Ok(CronConfig { + base, + retry_exchange: self + .retry_exchange + .ok_or_else(|| ConsumerError::Configuration("retry_exchange is required".into()))?, + interval: self + .cron_interval + .ok_or_else(|| ConsumerError::Configuration("cron_interval is required".into()))?, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_exchange_topology_from_prefix() { + let topology = ExchangeTopology::from_prefix("ethereum.events"); + + assert_eq!(topology.main, "ethereum.events"); + assert_eq!(topology.retry, "ethereum.events.retry"); + assert_eq!(topology.dlx, "ethereum.events.dlx"); + } + + #[test] + fn test_consumer_config_builder_validation_fails() { + let result = ConsumerConfigBuilder::new() + .exchange("test.exchange") + .retry_exchange("test.exchange.retry") + .dead_exchange("test.exchange.dlx") + // Missing queue, routing_key, consumer_tag + .build_prefetch(); + + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!(matches!(err, ConsumerError::Configuration(_))); + } + + #[test] + fn test_prefetch_config_builder() { + let config = ConsumerConfigBuilder::new() + .exchange("test.exchange") + .queue("test.queue") + .routing_key("test.key") + .consumer_tag("test-consumer") + .retry_exchange("test.exchange.retry") + .dead_exchange("test.exchange.dlx") + .prefetch_count(50) + .build_prefetch() + .unwrap(); + + assert_eq!(config.prefetch_count, 50); + assert_eq!(config.retry.base.exchange, "test.exchange"); + assert_eq!(config.retry.max_retries, 3); + assert_eq!(config.retry.retry_delay, Duration::from_secs(5)); + } + + #[test] + fn test_cron_config_builder() { + let config = ConsumerConfigBuilder::new() + .exchange("test.exchange") + .queue("test.queue") + .routing_key("test.key") + .consumer_tag("test-consumer") + .retry_exchange("test.exchange.retry") + .cron_interval(Duration::from_secs(60)) + .build_cron() + .unwrap(); + + assert_eq!(config.base.exchange, "test.exchange"); + assert_eq!(config.retry_exchange, "test.exchange.retry"); + assert_eq!(config.interval, Duration::from_secs(60)); + } + + #[test] + fn test_builder_with_topology() { + let topology = ExchangeTopology::from_prefix("polygon.events"); + + let config = ConsumerConfigBuilder::new() + .with_topology(&topology) + .queue("my.queue") + .routing_key("my.key") + .consumer_tag("my-consumer") + .build_prefetch() + .unwrap(); + + assert_eq!(config.retry.base.exchange, "polygon.events"); + assert_eq!(config.retry.retry_exchange, "polygon.events.retry"); + assert_eq!(config.retry.dead_exchange, "polygon.events.dlx"); + assert_eq!(config.retry.retry_routing_key(), "__mq.retry.my.queue"); + assert_eq!(config.retry.dead_routing_key(), "__mq.dead.my.queue"); + } +} diff --git a/listener/crates/shared/broker/src/amqp/connection.rs b/listener/crates/shared/broker/src/amqp/connection.rs new file mode 100644 index 0000000000..d059d696ab --- /dev/null +++ b/listener/crates/shared/broker/src/amqp/connection.rs @@ -0,0 +1,144 @@ +use lapin::{Channel, Connection, ConnectionProperties}; +use std::{sync::Arc, time::Duration}; +use tokio::{sync::RwLock, time::sleep}; +use tracing::{error, info}; + +use super::error::ConnectionError; + +/// Manages RabbitMQ connections with automatic reconnection. +#[derive(Clone)] +pub struct ConnectionManager { + addr: String, + connection: Arc>>>, + pub(crate) reconnect_delay: Duration, +} + +impl ConnectionManager { + /// Create a new ConnectionManager with the given address. + pub fn new(addr: impl Into) -> Self { + Self { + addr: addr.into(), + connection: Arc::new(RwLock::new(None)), + reconnect_delay: Duration::from_secs(5), + } + } + + /// Create a new ConnectionManager with custom reconnect delay. + pub fn with_reconnect_delay(addr: impl Into, reconnect_delay: Duration) -> Self { + Self { + addr: addr.into(), + connection: Arc::new(RwLock::new(None)), + reconnect_delay, + } + } + + /// Get the RabbitMQ address. + pub fn addr(&self) -> &str { + &self.addr + } + + /// Get or create a connection. + async fn get_or_create_connection(&self) -> Result, ConnectionError> { + // Check if we have a healthy connection + { + let guard = self.connection.read().await; + if let Some(ref conn) = *guard + && conn.status().connected() + { + return Ok(Arc::clone(conn)); + } + } + + // Need to create a new connection + let mut guard = self.connection.write().await; + + // Double-check after acquiring write lock + if let Some(ref conn) = *guard + && conn.status().connected() + { + return Ok(Arc::clone(conn)); + } + + // Create new connection + let conn = Connection::connect(&self.addr, ConnectionProperties::default()).await?; + let conn = Arc::new(conn); + *guard = Some(Arc::clone(&conn)); + info!("ConnectionManager: Connected to RabbitMQ at {}", self.addr); + + Ok(conn) + } + + /// Create a new channel from the shared connection. + pub async fn create_channel(&self) -> Result { + let conn = self.get_or_create_connection().await?; + conn.create_channel() + .await + .map_err(|e| ConnectionError::Channel(e.to_string())) + } + + /// Create a channel with infinite retry on failure. + /// Use this when the consumer must eventually succeed. + pub async fn create_channel_with_retry(&self) -> Channel { + loop { + match self.create_channel().await { + Ok(channel) => return channel, + Err(e) => { + error!( + "Failed to create channel: {}. Retrying in {:?}...", + e, self.reconnect_delay + ); + // Clear the connection so next attempt creates a fresh one + { + let mut guard = self.connection.write().await; + *guard = None; + } + sleep(self.reconnect_delay).await; + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_connection_manager_new() { + let manager = ConnectionManager::new("amqp://localhost:5672"); + assert_eq!(manager.addr(), "amqp://localhost:5672"); + assert_eq!(manager.reconnect_delay, Duration::from_secs(5)); + } + + #[test] + fn test_connection_manager_with_custom_delay() { + let manager = ConnectionManager::with_reconnect_delay( + "amqp://localhost:5672", + Duration::from_secs(10), + ); + assert_eq!(manager.reconnect_delay, Duration::from_secs(10)); + } + + // Integration tests requiring RabbitMQ + #[tokio::test] + #[ignore] + async fn test_connection_manager_create_channel() { + let manager = ConnectionManager::new("amqp://localhost:5672"); + let channel = manager.create_channel().await; + assert!(channel.is_ok()); + } + + #[tokio::test] + #[ignore] + async fn test_connection_manager_reconnect() { + let manager = ConnectionManager::new("amqp://localhost:5672"); + + // Create first channel + let channel1 = manager.create_channel().await.unwrap(); + assert!(channel1.status().connected()); + + // Create second channel (should reuse connection) + let channel2 = manager.create_channel().await.unwrap(); + assert!(channel2.status().connected()); + } +} diff --git a/listener/crates/shared/broker/src/amqp/consumer.rs b/listener/crates/shared/broker/src/amqp/consumer.rs new file mode 100644 index 0000000000..4783fcbfcd --- /dev/null +++ b/listener/crates/shared/broker/src/amqp/consumer.rs @@ -0,0 +1,977 @@ +use futures::stream::StreamExt; +use lapin::{ + Channel, ExchangeKind, + message::Delivery, + options::{ + BasicAckOptions, BasicConsumeOptions, BasicNackOptions, BasicPublishOptions, + BasicQosOptions, ExchangeDeclareOptions, QueueBindOptions, QueueDeclareOptions, + }, + types::{AMQPValue, FieldTable, ShortString}, +}; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::mpsc; +use tokio::time::sleep; +use tracing::{debug, error, info, warn}; + +use tokio_util::sync::CancellationToken; + +use crate::traits::circuit_breaker::CircuitBreaker; +use crate::traits::handler::{Handler, HandlerOutcome}; +use crate::traits::message::{Message, MessageMetadata}; + +use super::{ + config::{CronConfig, PrefetchConfig, RetryConfig}, + connection::ConnectionManager, + error::ConsumerError, +}; + +/// Result of processing a message in a worker task. +/// Used by `run()` to communicate results back to the main loop. +struct ProcessingResult { + /// The original delivery, needed for ACK/NACK + delivery: Delivery, + /// Outcome classification — preserves the Transient/Permanent distinction + /// so the main loop can call the correct circuit breaker method. + outcome: HandlerOutcome, +} + +/// RabbitMQ consumer service. +pub struct RmqConsumer { + connection: Arc, + cancel_token: CancellationToken, +} + +impl RmqConsumer { + const PERMANENT_RETRY_COUNT_HEADER: &'static str = "x-mq-permanent-retry-count"; + + /// Create a new consumer with the given connection manager. + pub fn new(connection: Arc) -> Self { + Self { + connection, + cancel_token: CancellationToken::new(), + } + } + + /// Create a new consumer with a fresh connection manager. + pub fn with_addr(addr: impl Into) -> Self { + Self { + connection: Arc::new(ConnectionManager::new(addr)), + cancel_token: CancellationToken::new(), + } + } + + /// Set a cancellation token for graceful shutdown. + /// + /// When cancelled, the consumer finishes its current batch, drains + /// in-flight results, and exits cleanly. + pub fn with_cancellation(mut self, token: CancellationToken) -> Self { + self.cancel_token = token; + self + } + + /// Get the underlying connection manager. + pub fn connection(&self) -> &Arc { + &self.connection + } + + /// Ensure the AMQP topology (exchanges, queues, bindings) exists without + /// starting to consume. + /// + /// Call this before checking queue depth or publishing seed messages to + /// guarantee that queues are bound to exchanges and messages won't be + /// silently dropped. + pub async fn ensure_topology(&self, config: &PrefetchConfig) -> Result<(), ConsumerError> { + let channel = self.connection.create_channel_with_retry().await; + self.setup_retry_queues(&channel, &config.retry).await + } + + // ═══════════════════════════════════════════════════════════════════════════ + // Public consumer methods — each wraps a reconnection loop around an inner + // consume loop. On reconnectable errors (connection drop, channel death, + // stream end), the outer loop creates a fresh channel and re-subscribes. + // Only `Configuration` errors propagate to the caller. + // ═══════════════════════════════════════════════════════════════════════════ + + /// Run a high-throughput consumer with STRONG message loss guarantees. + /// + /// This consumer: + /// - ACKs/NACKs only from the main loop (not in spawned tasks) + /// - Guarantees no message loss even on worker crash + /// - Uses bounded channel to prevent memory exhaustion + /// + /// Automatically reconnects on connection loss. Circuit breaker state + /// persists across reconnections. On reconnection the mpsc channel and + /// spawned tasks are dropped; unacked messages are redelivered by RabbitMQ. + pub async fn run( + &self, + config: PrefetchConfig, + handler: impl Handler + 'static, + ) -> Result<(), ConsumerError> { + let handler: Arc = Arc::new(handler); + let mut cb = config.retry.circuit_breaker.as_ref().map(|cfg| { + CircuitBreaker::new(cfg.clone()).with_labels("amqp", config.retry.base.queue.clone()) + }); + + loop { + let channel = self.connection.create_channel_with_retry().await; + + // Signal "connected" as soon as the channel is established. + metrics::gauge!( + "broker_consumer_connected", + "backend" => "amqp", + "topic" => config.retry.base.queue.clone(), + ) + .set(1.0); + + let result = self + .consume_loop_prefetch_safe(&channel, &config, Arc::clone(&handler), &mut cb) + .await; + + match result { + Err(e) if e.is_reconnectable() => { + if self.cancel_token.is_cancelled() { + info!(queue = %config.retry.base.queue, "Cancellation requested, skipping reconnect"); + return Ok(()); + } + metrics::counter!("broker_consumer_reconnections_total", + "backend" => "amqp", + "topic" => config.retry.base.queue.clone(), + ) + .increment(1); + metrics::gauge!( + "broker_consumer_connected", + "backend" => "amqp", + "topic" => config.retry.base.queue.clone(), + ) + .set(0.0); + warn!( + error = %e, + delay = ?self.connection.reconnect_delay, + queue = %config.retry.base.queue, + "Prefetch-safe consumer disconnected, reconnecting..." + ); + sleep(self.connection.reconnect_delay).await; + } + other => return other, + } + } + } + + /// Run a cron-style scheduled job consumer. + /// + /// Automatically reconnects on connection loss. + pub async fn run_cron( + &self, + config: CronConfig, + handler: impl Handler, + ) -> Result<(), ConsumerError> { + loop { + let channel = self.connection.create_channel_with_retry().await; + + let result = self.consume_loop_cron(&channel, &config, &handler).await; + + match result { + Err(e) if e.is_reconnectable() => { + warn!( + error = %e, + delay = ?self.connection.reconnect_delay, + queue = %config.base.queue, + "Cron consumer disconnected, reconnecting..." + ); + sleep(self.connection.reconnect_delay).await; + } + other => return other, + } + } + } + + async fn consume_loop_prefetch_safe( + &self, + channel: &Channel, + config: &PrefetchConfig, + handler: Arc, + cb: &mut Option, + ) -> Result<(), ConsumerError> { + channel + .basic_qos(config.prefetch_count, BasicQosOptions::default()) + .await + .map_err(ConsumerError::Connection)?; + + self.setup_retry_queues(channel, &config.retry).await?; + + let mut consumer = channel + .basic_consume( + ShortString::from(config.retry.base.queue.as_str()), + ShortString::from(config.retry.base.consumer_tag.as_str()), + BasicConsumeOptions::default(), + FieldTable::default(), + ) + .await + .map_err(|e| ConsumerError::ConsumerRegistration { + consumer_tag: config.retry.base.consumer_tag.clone(), + source: e, + })?; + + info!( + exchange = %config.retry.base.exchange, + queue = %config.retry.base.queue, + prefetch_count = %config.prefetch_count, + circuit_breaker = %config.retry.circuit_breaker.is_some(), + "Safe high-throughput consumer started (ACK in main loop)" + ); + + let channel = Arc::new(channel.clone()); + + let (result_tx, mut result_rx) = + mpsc::channel::(config.prefetch_count as usize); + + loop { + if let Some(breaker) = cb + && !breaker.should_allow_request() + { + let cooldown = breaker.remaining_cooldown(); + info!( + cooldown = ?cooldown, + queue = %config.retry.base.queue, + "Circuit breaker OPEN — pausing consumption" + ); + sleep(cooldown).await; + continue; + } + + tokio::select! { + maybe_delivery = consumer.next() => { + let delivery = match maybe_delivery { + Some(Ok(d)) => d, + Some(Err(_)) => { + info!("Channel error detected, breaking for reconnection"); + break; + } + None => { + info!("Consumer stream ended"); + break; + } + }; + + let handler = Arc::clone(&handler); + let tx = result_tx.clone(); + let msg = Self::delivery_to_message(&delivery, &config.retry.base.queue); + + tokio::spawn(async move { + let handler_start = std::time::Instant::now(); + let outcome = HandlerOutcome::from(handler.call(&msg).await); + metrics::histogram!("broker_handler_duration_seconds", + "backend" => "amqp", + "topic" => msg.metadata.topic.clone(), + ).record(handler_start.elapsed().as_secs_f64()); + if let Err(e) = tx.send(ProcessingResult { delivery, outcome }).await { + error!(?e, "Failed to send processing result - message will be requeued"); + } + }); + } + + Some(result) = result_rx.recv() => { + // ── metrics: outcome counter ── + metrics::counter!("broker_messages_consumed_total", + "backend" => "amqp", + "topic" => config.retry.base.queue.clone(), + "outcome" => crate::metrics::outcome_label(&result.outcome), + ).increment(1); + + match result.outcome { + HandlerOutcome::Ack => { + if let Some(b) = cb { b.record_success(); } + debug!("Handler succeeded, acknowledging message"); + result.delivery + .ack(BasicAckOptions::default()) + .await + .map_err(ConsumerError::Ack)?; + } + HandlerOutcome::Nack => { + if let Some(b) = cb { b.record_success(); } + debug!("Handler voluntarily yielded, requeuing at tail of main queue"); + result.delivery + .nack(BasicNackOptions { requeue: true, ..Default::default() }) + .await + .map_err(ConsumerError::Nack)?; + } + HandlerOutcome::Dead => { + if let Some(b) = cb { b.record_permanent_failure(); } + metrics::counter!("broker_messages_dead_lettered_total", + "backend" => "amqp", + "topic" => config.retry.base.queue.clone(), + "reason" => "handler_requested", + ).increment(1); + warn!("Handler requested immediate dead-letter, bypassing retry"); + Self::handle_dead_letter_static(&channel, &result.delivery, &config.retry).await?; + } + HandlerOutcome::Delay(duration) => { + if let Some(b) = cb { b.record_success(); } + debug!(delay_ms = %duration.as_millis(), "Handler requested delay requeue"); + Self::handle_delay_static(&channel, &result.delivery, &config.retry, duration).await?; + } + HandlerOutcome::Transient => { + if let Some(b) = cb { b.record_transient_failure(); } + warn!("Transient failure, scheduling infinite retry"); + Self::handle_transient_retry_static(&result.delivery).await?; + } + HandlerOutcome::Permanent => { + if let Some(b) = cb { b.record_permanent_failure(); } + error!("Handler failed, applying bounded retry policy"); + Self::handle_retry_static(&channel, &result.delivery, &config.retry).await?; + } + } + } + + // Graceful shutdown — finish current batch, then drain. + _ = self.cancel_token.cancelled() => { + info!( + queue = %config.retry.base.queue, + "Cancellation requested, stopping AMQP consumer gracefully" + ); + break; + } + } + } + + // Drain remaining results before returning — ACK/NACK failures are + // ignored since the channel is dead; RabbitMQ will redeliver unacked messages. + info!("Draining remaining results before reconnection"); + while let Ok(result) = result_rx.try_recv() { + match result.outcome { + HandlerOutcome::Ack => { + let _ = result.delivery.ack(BasicAckOptions::default()).await; + } + HandlerOutcome::Nack => { + let _ = result + .delivery + .nack(BasicNackOptions { + requeue: true, + ..Default::default() + }) + .await; + } + HandlerOutcome::Transient => { + let _ = Self::handle_transient_retry_static(&result.delivery).await; + } + HandlerOutcome::Permanent => { + let _ = + Self::handle_retry_static(&channel, &result.delivery, &config.retry).await; + } + HandlerOutcome::Dead => { + let _ = + Self::handle_dead_letter_static(&channel, &result.delivery, &config.retry) + .await; + } + HandlerOutcome::Delay(duration) => { + let _ = Self::handle_delay_static( + &channel, + &result.delivery, + &config.retry, + duration, + ) + .await; + } + } + } + info!("Safe consumer drain complete"); + + if self.cancel_token.is_cancelled() { + Ok(()) + } else { + Err(ConsumerError::StreamEnded) + } + } + + async fn consume_loop_cron( + &self, + channel: &Channel, + config: &CronConfig, + handler: &impl Handler, + ) -> Result<(), ConsumerError> { + self.setup_cron_queues(channel, config).await?; + + let mut consumer = channel + .basic_consume( + ShortString::from(config.base.queue.as_str()), + ShortString::from(config.base.consumer_tag.as_str()), + BasicConsumeOptions::default(), + FieldTable::default(), + ) + .await + .map_err(|e| ConsumerError::ConsumerRegistration { + consumer_tag: config.base.consumer_tag.clone(), + source: e, + })?; + + info!( + exchange = %config.base.exchange, + queue = %config.base.queue, + interval_ms = %config.interval.as_millis(), + "Cron consumer started (publish a message to start the cron job)" + ); + + while let Some(delivery_result) = consumer.next().await { + let delivery = delivery_result.map_err(ConsumerError::Connection)?; + + let msg = Self::delivery_to_message(&delivery, &config.base.queue); + match handler.call(&msg).await { + Ok(_) => { + debug!("Cron handler executed successfully"); + } + Err(err) => { + error!(?err, "Cron handler execution failed"); + } + } + + delivery + .nack(BasicNackOptions { + requeue: false, + ..Default::default() + }) + .await + .map_err(ConsumerError::Nack)?; + } + + Err(ConsumerError::StreamEnded) + } + + // ═══════════════════════════════════════════════════════════════════════════ + // Private helpers — topology setup, retry/DLQ routing, message conversion + // ═══════════════════════════════════════════════════════════════════════════ + + async fn declare_exchange_if_needed( + &self, + channel: &Channel, + exchange: &str, + ) -> Result<(), ConsumerError> { + channel + .exchange_declare( + ShortString::from(exchange), + ExchangeKind::Topic, + ExchangeDeclareOptions { + durable: true, + ..Default::default() + }, + FieldTable::default(), + ) + .await + .map_err(|e| ConsumerError::ExchangeDeclaration { + exchange: exchange.to_string(), + source: e, + })?; + Ok(()) + } + + async fn ensure_retry_exchanges( + &self, + channel: &Channel, + config: &RetryConfig, + ) -> Result<(), ConsumerError> { + self.declare_exchange_if_needed(channel, &config.base.exchange) + .await?; + self.declare_exchange_if_needed(channel, &config.retry_exchange) + .await?; + self.declare_exchange_if_needed(channel, &config.dead_exchange) + .await?; + Ok(()) + } + + async fn setup_retry_queues( + &self, + channel: &Channel, + config: &RetryConfig, + ) -> Result<(), ConsumerError> { + self.ensure_retry_exchanges(channel, config).await?; + let retry_routing_key = config.retry_routing_key(); + let dead_routing_key = config.dead_routing_key(); + + let mut queue_args = FieldTable::default(); + queue_args.insert( + "x-dead-letter-exchange".into(), + AMQPValue::LongString(config.retry_exchange.clone().into()), + ); + queue_args.insert( + "x-dead-letter-routing-key".into(), + AMQPValue::LongString(retry_routing_key.clone().into()), + ); + + channel + .queue_declare( + ShortString::from(config.base.queue.as_str()), + QueueDeclareOptions { + durable: true, + ..Default::default() + }, + queue_args, + ) + .await + .map_err(|e| ConsumerError::QueueDeclaration { + queue: config.base.queue.clone(), + source: e, + })?; + + channel + .queue_bind( + ShortString::from(config.base.queue.as_str()), + ShortString::from(config.base.exchange.as_str()), + ShortString::from(config.base.routing_key.as_str()), + QueueBindOptions::default(), + FieldTable::default(), + ) + .await + .map_err(|e| ConsumerError::QueueBinding { + queue: config.base.queue.clone(), + exchange: config.base.exchange.clone(), + source: e, + })?; + + let mut retry_args = FieldTable::default(); + retry_args.insert( + "x-message-ttl".into(), + AMQPValue::LongUInt(config.retry_delay.as_millis() as u32), + ); + // Retry queue TTL should return strictly to the originating queue. + // Using the default exchange avoids fanout through the shared main exchange. + retry_args.insert( + "x-dead-letter-exchange".into(), + AMQPValue::LongString("".into()), + ); + retry_args.insert( + "x-dead-letter-routing-key".into(), + AMQPValue::LongString(config.base.queue.clone().into()), + ); + + let retry_queue = format!("{}.retry", &config.base.queue); + channel + .queue_declare( + ShortString::from(retry_queue.as_str()), + QueueDeclareOptions { + durable: true, + ..Default::default() + }, + retry_args, + ) + .await + .map_err(|e| ConsumerError::QueueDeclaration { + queue: retry_queue.clone(), + source: e, + })?; + + channel + .queue_bind( + ShortString::from(retry_queue.as_str()), + ShortString::from(config.retry_exchange.as_str()), + ShortString::from(retry_routing_key.as_str()), + QueueBindOptions::default(), + FieldTable::default(), + ) + .await + .map_err(|e| ConsumerError::QueueBinding { + queue: retry_queue.clone(), + exchange: config.retry_exchange.clone(), + source: e, + })?; + + let error_queue = format!("{}.error", &config.base.queue); + channel + .queue_declare( + ShortString::from(error_queue.as_str()), + QueueDeclareOptions { + durable: true, + ..Default::default() + }, + FieldTable::default(), + ) + .await + .map_err(|e| ConsumerError::QueueDeclaration { + queue: error_queue.clone(), + source: e, + })?; + + channel + .queue_bind( + ShortString::from(error_queue.as_str()), + ShortString::from(config.dead_exchange.as_str()), + ShortString::from(dead_routing_key.as_str()), + QueueBindOptions::default(), + FieldTable::default(), + ) + .await + .map_err(|e| ConsumerError::QueueBinding { + queue: error_queue.clone(), + exchange: config.dead_exchange.clone(), + source: e, + })?; + + Ok(()) + } + + async fn setup_cron_queues( + &self, + channel: &Channel, + config: &CronConfig, + ) -> Result<(), ConsumerError> { + self.declare_exchange_if_needed(channel, &config.base.exchange) + .await?; + self.declare_exchange_if_needed(channel, &config.retry_exchange) + .await?; + let retry_routing_key = RetryConfig::retry_routing_key_for_queue(&config.base.queue); + + let mut queue_args = FieldTable::default(); + queue_args.insert( + "x-dead-letter-exchange".into(), + AMQPValue::LongString(config.retry_exchange.clone().into()), + ); + queue_args.insert( + "x-dead-letter-routing-key".into(), + AMQPValue::LongString(retry_routing_key.clone().into()), + ); + + channel + .queue_declare( + ShortString::from(config.base.queue.as_str()), + QueueDeclareOptions { + durable: true, + ..Default::default() + }, + queue_args, + ) + .await + .map_err(|e| ConsumerError::QueueDeclaration { + queue: config.base.queue.clone(), + source: e, + })?; + + channel + .queue_bind( + ShortString::from(config.base.queue.as_str()), + ShortString::from(config.base.exchange.as_str()), + ShortString::from(config.base.routing_key.as_str()), + QueueBindOptions::default(), + FieldTable::default(), + ) + .await + .map_err(|e| ConsumerError::QueueBinding { + queue: config.base.queue.clone(), + exchange: config.base.exchange.clone(), + source: e, + })?; + + let mut cron_args = FieldTable::default(); + cron_args.insert( + "x-message-ttl".into(), + AMQPValue::LongUInt(config.interval.as_millis() as u32), + ); + // Cron delay queue should route back only to the originating queue. + cron_args.insert( + "x-dead-letter-exchange".into(), + AMQPValue::LongString("".into()), + ); + cron_args.insert( + "x-dead-letter-routing-key".into(), + AMQPValue::LongString(config.base.queue.clone().into()), + ); + + let cron_queue = format!("{}.cron-job", &config.base.queue); + channel + .queue_declare( + ShortString::from(cron_queue.as_str()), + QueueDeclareOptions { + durable: true, + ..Default::default() + }, + cron_args, + ) + .await + .map_err(|e| ConsumerError::QueueDeclaration { + queue: cron_queue.clone(), + source: e, + })?; + + channel + .queue_bind( + ShortString::from(cron_queue.as_str()), + ShortString::from(config.retry_exchange.as_str()), + ShortString::from(retry_routing_key.as_str()), + QueueBindOptions::default(), + FieldTable::default(), + ) + .await + .map_err(|e| ConsumerError::QueueBinding { + queue: cron_queue.clone(), + exchange: config.retry_exchange.clone(), + source: e, + })?; + + Ok(()) + } + + async fn handle_retry_static( + channel: &Channel, + delivery: &lapin::message::Delivery, + config: &RetryConfig, + ) -> Result<(), ConsumerError> { + // Permanent retry budget is tracked via a dedicated header so transient + // dead-letter cycles do not consume max_retries. + let retry_count = + Self::extract_permanent_retry_count(delivery.properties.headers().as_ref()); + + if retry_count >= config.max_retries { + error!( + retry_count = %retry_count, + max_retries = %config.max_retries, + "Moving permanent failure to DLX after max retries" + ); + + let dead_routing_key = config.dead_routing_key(); + channel + .basic_publish( + ShortString::from(config.dead_exchange.as_str()), + ShortString::from(dead_routing_key.as_str()), + BasicPublishOptions::default(), + &delivery.data, + delivery.properties.clone(), + ) + .await + .map_err(ConsumerError::DeadLetter)?; + + delivery + .ack(BasicAckOptions::default()) + .await + .map_err(ConsumerError::Ack)?; + } else { + let next_retry_count = retry_count.saturating_add(1); + warn!( + retry_count = %retry_count, + next_retry_count = %next_retry_count, + "Sending permanent failure to retry queue" + ); + + let retry_routing_key = config.retry_routing_key(); + let mut headers = delivery + .properties + .headers() + .as_ref() + .cloned() + .unwrap_or_default(); + headers.insert( + ShortString::from(Self::PERMANENT_RETRY_COUNT_HEADER), + AMQPValue::LongUInt(next_retry_count), + ); + + channel + .basic_publish( + ShortString::from(config.retry_exchange.as_str()), + ShortString::from(retry_routing_key.as_str()), + BasicPublishOptions::default(), + &delivery.data, + delivery.properties.clone().with_headers(headers), + ) + .await + .map_err(ConsumerError::Retry)?; + + delivery + .ack(BasicAckOptions::default()) + .await + .map_err(ConsumerError::Ack)?; + } + + Ok(()) + } + + async fn handle_transient_retry_static( + delivery: &lapin::message::Delivery, + ) -> Result<(), ConsumerError> { + delivery + .nack(BasicNackOptions { + requeue: false, + ..Default::default() + }) + .await + .map_err(ConsumerError::Nack)?; + Ok(()) + } + + async fn handle_dead_letter_static( + channel: &Channel, + delivery: &lapin::message::Delivery, + config: &RetryConfig, + ) -> Result<(), ConsumerError> { + let dead_routing_key = config.dead_routing_key(); + channel + .basic_publish( + ShortString::from(config.dead_exchange.as_str()), + ShortString::from(dead_routing_key.as_str()), + BasicPublishOptions::default(), + &delivery.data, + delivery.properties.clone(), + ) + .await + .map_err(ConsumerError::DeadLetter)?; + + delivery + .ack(BasicAckOptions::default()) + .await + .map_err(ConsumerError::Ack)?; + Ok(()) + } + + async fn handle_delay_static( + channel: &Channel, + delivery: &lapin::message::Delivery, + config: &RetryConfig, + duration: Duration, + ) -> Result<(), ConsumerError> { + let retry_routing_key = config.retry_routing_key(); + let expiration_ms = duration.as_millis().to_string(); + channel + .basic_publish( + ShortString::from(config.retry_exchange.as_str()), + ShortString::from(retry_routing_key.as_str()), + BasicPublishOptions::default(), + &delivery.data, + delivery + .properties + .clone() + .with_expiration(expiration_ms.into()), + ) + .await + .map_err(ConsumerError::Retry)?; + + delivery + .ack(BasicAckOptions::default()) + .await + .map_err(ConsumerError::Ack)?; + Ok(()) + } + + fn delivery_to_message(delivery: &Delivery, queue: &str) -> Message { + let retry_count = Self::extract_retry_count(delivery.properties.headers().as_ref()); + Message { + payload: delivery.data.clone(), + metadata: MessageMetadata::new( + delivery.delivery_tag.to_string(), + queue, + retry_count as u64 + 1, + ), + } + } + + fn extract_retry_count(headers: Option<&FieldTable>) -> u32 { + headers + .and_then(|hdrs| hdrs.inner().get("x-death")) + .and_then(|x_death| match x_death { + AMQPValue::FieldArray(array) => array.as_slice().first(), + _ => None, + }) + .and_then(|first_entry| match first_entry { + AMQPValue::FieldTable(table) => table + .inner() + .get(&ShortString::from("count")) + .and_then(Self::parse_u32_header_value), + _ => None, + }) + .unwrap_or(0) + } + + fn extract_permanent_retry_count(headers: Option<&FieldTable>) -> u32 { + headers + .and_then(|hdrs| hdrs.inner().get(Self::PERMANENT_RETRY_COUNT_HEADER)) + .and_then(Self::parse_u32_header_value) + .unwrap_or(0) + } + + fn parse_u32_header_value(value: &AMQPValue) -> Option { + match value { + AMQPValue::LongUInt(n) => Some(*n), + AMQPValue::LongLongInt(n) => Some(*n as u32), + AMQPValue::LongInt(n) => Some(*n as u32), + _ => None, + } + } +} + +#[async_trait::async_trait] +impl crate::traits::consumer::Consumer for RmqConsumer { + type PrefetchConfig = PrefetchConfig; + type Error = ConsumerError; + + async fn connect(url: &str) -> Result { + Ok(Self::with_addr(url)) + } + + async fn run( + &self, + config: Self::PrefetchConfig, + handler: impl Handler + 'static, + ) -> Result<(), Self::Error> { + self.run(config, handler).await + } +} + +#[cfg(test)] +mod tests { + use super::*; + use lapin::types::FieldArray; + + #[test] + fn test_extract_retry_count_empty() { + let count = RmqConsumer::extract_retry_count(None); + assert_eq!(count, 0); + } + + #[test] + fn test_extract_retry_count_empty_headers() { + let headers = FieldTable::default(); + let count = RmqConsumer::extract_retry_count(Some(&headers)); + assert_eq!(count, 0); + } + + #[test] + fn test_extract_retry_count_with_value() { + let mut inner_table = FieldTable::default(); + inner_table.insert(ShortString::from("count"), AMQPValue::LongUInt(3)); + + let array = FieldArray::from(vec![AMQPValue::FieldTable(inner_table)]); + + let mut headers = FieldTable::default(); + headers.insert(ShortString::from("x-death"), AMQPValue::FieldArray(array)); + + let count = RmqConsumer::extract_retry_count(Some(&headers)); + assert_eq!(count, 3); + } + + #[test] + fn test_extract_retry_count_with_long_long_int() { + let mut inner_table = FieldTable::default(); + inner_table.insert(ShortString::from("count"), AMQPValue::LongLongInt(5)); + + let array = FieldArray::from(vec![AMQPValue::FieldTable(inner_table)]); + + let mut headers = FieldTable::default(); + headers.insert(ShortString::from("x-death"), AMQPValue::FieldArray(array)); + + let count = RmqConsumer::extract_retry_count(Some(&headers)); + assert_eq!(count, 5); + } + + #[test] + fn test_extract_permanent_retry_count_empty() { + let count = RmqConsumer::extract_permanent_retry_count(None); + assert_eq!(count, 0); + } + + #[test] + fn test_extract_permanent_retry_count_with_value() { + let mut headers = FieldTable::default(); + headers.insert( + ShortString::from(RmqConsumer::PERMANENT_RETRY_COUNT_HEADER), + AMQPValue::LongUInt(4), + ); + + let count = RmqConsumer::extract_permanent_retry_count(Some(&headers)); + assert_eq!(count, 4); + } +} diff --git a/listener/crates/shared/broker/src/amqp/depth.rs b/listener/crates/shared/broker/src/amqp/depth.rs new file mode 100644 index 0000000000..642485e2cf --- /dev/null +++ b/listener/crates/shared/broker/src/amqp/depth.rs @@ -0,0 +1,162 @@ +//! AMQP queue depth introspection via passive queue_declare. + +use crate::traits::depth::{QueueDepths, QueueInspector}; +use async_trait::async_trait; +use lapin::options::QueueDeclareOptions; +use lapin::protocol::AMQPErrorKind; +use lapin::protocol::AMQPSoftError; +use lapin::types::FieldTable; + +use super::connection::ConnectionManager; +use super::error::ConsumerError; + +/// Inspects AMQP queue depth using passive `queue_declare`. +/// +/// Derives queue names from the logical name: +/// - principal: `{name}` +/// - retry: `{name}.retry` +/// - dead-letter: `{name}.error` +/// +/// Passive declare does NOT create queues — it only queries existing ones. +/// If a queue does not exist, the depth is reported as 0. +pub struct AmqpQueueInspector { + connection: ConnectionManager, +} + +impl AmqpQueueInspector { + /// Create a new inspector from an existing connection manager. + pub fn new(connection: ConnectionManager) -> Self { + Self { connection } + } + + /// Query the message count of a single queue via passive declare. + /// + /// Returns 0 if the queue does not exist (lapin returns a channel error + /// for passive declare on a non-existent queue, which we catch here). + async fn queue_message_count(&self, queue: &str) -> Result { + let channel = self.connection.create_channel().await?; + let result = channel + .queue_declare( + queue.into(), + QueueDeclareOptions { + passive: true, + ..Default::default() + }, + FieldTable::default(), + ) + .await; + + match result { + Ok(queue_state) => Ok(queue_state.message_count() as u64), + Err(e) => { + // Passive declare on a non-existent queue closes the channel + // with a NOT_FOUND error. Treat as 0 messages. + if is_not_found(&e) { + Ok(0) + } else { + Err(ConsumerError::QueueDeclaration { + queue: queue.to_string(), + source: e, + }) + } + } + } + } + + /// Returns whether a queue exists (passive declare succeeds). + async fn queue_exists(&self, queue: &str) -> Result { + let channel = self.connection.create_channel().await?; + let result = channel + .queue_declare( + queue.into(), + QueueDeclareOptions { + passive: true, + ..Default::default() + }, + FieldTable::default(), + ) + .await; + + match result { + Ok(_) => Ok(true), + Err(e) => { + if is_not_found(&e) { + Ok(false) + } else { + Err(ConsumerError::QueueDeclaration { + queue: queue.to_string(), + source: e, + }) + } + } + } + } +} + +/// Check if a lapin error indicates a queue-not-found condition (AMQP 404). +fn is_not_found(err: &lapin::Error) -> bool { + if let lapin::ErrorKind::ProtocolError(amqp_err) = err.kind() { + amqp_err.kind() == &AMQPErrorKind::Soft(AMQPSoftError::NOTFOUND) + } else { + false + } +} + +#[async_trait] +impl QueueInspector for AmqpQueueInspector { + type Error = ConsumerError; + + async fn queue_depths( + &self, + name: &str, + _group: Option<&str>, + ) -> Result { + let retry_queue = format!("{name}.retry"); + let error_queue = format!("{name}.error"); + + let (principal, retry, dead_letter) = tokio::try_join!( + self.queue_message_count(name), + self.queue_message_count(&retry_queue), + self.queue_message_count(&error_queue), + )?; + + // AMQP `message_count()` already returns ready (unconsumed) messages, + // so `principal` is effectively the consumer-visible depth. + // `pending`/`lag` are Redis-specific concepts — left as None. + Ok(QueueDepths { + principal, + retry: Some(retry), + dead_letter, + pending: None, + lag: None, + }) + } + + /// Single passive `queue_declare` round-trip: returns `true` when the + /// principal queue has zero ready messages (or does not exist). + async fn is_empty(&self, name: &str, _group: &str) -> Result { + let count = self.queue_message_count(name).await?; + Ok(count == 0) + } + + /// Equivalent to [`is_empty`](Self::is_empty) — AMQP has no pending/lag + /// distinction, so this returns `true` when `message_count == 0`. + async fn is_empty_or_pending(&self, name: &str, _group: &str) -> Result { + let count = self.queue_message_count(name).await?; + Ok(count == 0) + } + + async fn exists(&self, name: &str) -> Result { + self.queue_exists(name).await + } +} + +#[cfg(test)] +mod tests { + #[test] + fn amqp_queue_naming() { + let name = "ethereum.indexer"; + assert_eq!(format!("{name}.retry"), "ethereum.indexer.retry"); + assert_eq!(format!("{name}.error"), "ethereum.indexer.error"); + } +} diff --git a/listener/crates/shared/broker/src/amqp/error.rs b/listener/crates/shared/broker/src/amqp/error.rs new file mode 100644 index 0000000000..38f138fc06 --- /dev/null +++ b/listener/crates/shared/broker/src/amqp/error.rs @@ -0,0 +1,113 @@ +use thiserror::Error; + +#[derive(Error, Debug)] +#[non_exhaustive] +pub enum ConnectionError { + #[error("connection failed: {0}")] + Connection(#[from] lapin::Error), + #[error("channel creation failed: {0}")] + Channel(String), +} + +impl From for ConsumerError { + fn from(err: ConnectionError) -> Self { + match err { + ConnectionError::Connection(e) => ConsumerError::Connection(e), + ConnectionError::Channel(s) => ConsumerError::Channel(s), + } + } +} + +impl From for ExchangeError { + fn from(err: ConnectionError) -> Self { + match err { + ConnectionError::Connection(e) => ExchangeError::Connection(e), + ConnectionError::Channel(s) => ExchangeError::Channel(s), + } + } +} + +#[derive(Error, Debug)] +#[non_exhaustive] +pub enum ConsumerError { + #[error("connection failed: {0}")] + Connection(#[from] lapin::Error), + + #[error("channel creation failed: {0}")] + Channel(String), + + #[error("queue declaration failed for '{queue}': {source}")] + QueueDeclaration { + queue: String, + #[source] + source: lapin::Error, + }, + + #[error("queue binding failed for '{queue}' to '{exchange}': {source}")] + QueueBinding { + queue: String, + exchange: String, + #[source] + source: lapin::Error, + }, + + #[error("exchange declaration failed for '{exchange}': {source}")] + ExchangeDeclaration { + exchange: String, + #[source] + source: lapin::Error, + }, + + #[error("consumer registration failed for '{consumer_tag}': {source}")] + ConsumerRegistration { + consumer_tag: String, + #[source] + source: lapin::Error, + }, + + #[error("message acknowledgement failed: {0}")] + Ack(#[source] lapin::Error), + + #[error("message negative acknowledgement failed: {0}")] + Nack(#[source] lapin::Error), + + #[error("publish to DLX failed: {0}")] + DeadLetter(#[source] lapin::Error), + + #[error("publish to retry exchange failed: {0}")] + Retry(lapin::Error), + + #[error("consumer stream ended unexpectedly")] + StreamEnded, + + #[error("configuration error: {0}")] + Configuration(String), +} + +impl ConsumerError { + /// Returns `true` if the error is a transient connection/channel failure + /// that can be recovered by creating a new channel and consumer. + /// + /// Only `Configuration` errors are non-reconnectable (user error that + /// won't fix itself on retry). + pub fn is_reconnectable(&self) -> bool { + !matches!(self, Self::Configuration(_)) + } +} + +#[derive(Error, Debug)] +#[non_exhaustive] +pub enum ExchangeError { + #[error("exchange declaration failed for '{name}': {source}")] + Declaration { + name: String, + #[source] + source: lapin::Error, + }, + + #[error("channel creation failed: {0}")] + Channel(String), + + #[error("connection failed: {0}")] + Connection(#[from] lapin::Error), +} diff --git a/listener/crates/shared/broker/src/amqp/exchange.rs b/listener/crates/shared/broker/src/amqp/exchange.rs new file mode 100644 index 0000000000..4ab4b3367d --- /dev/null +++ b/listener/crates/shared/broker/src/amqp/exchange.rs @@ -0,0 +1,103 @@ +use std::sync::Arc; + +use lapin::options::ExchangeDeclareOptions; +use lapin::types::{FieldTable, ShortString}; +use tracing::info; + +use super::{config::ExchangeTopology, connection::ConnectionManager, error::ExchangeError}; + +/// Manages RabbitMQ exchange declarations. +pub struct ExchangeManager { + connection: Arc, +} + +impl ExchangeManager { + /// Create a new ExchangeManager with a shared connection manager. + pub fn new(connection: Arc) -> Self { + Self { connection } + } + + /// Create a new ExchangeManager with a fresh connection manager. + pub fn with_addr(addr: impl Into) -> Self { + Self { + connection: Arc::new(ConnectionManager::new(addr)), + } + } + + /// Declare a single exchange. + pub async fn declare_exchange(&self, name: &str) -> Result<(), ExchangeError> { + let channel = self.connection.create_channel().await?; + + channel + .exchange_declare( + ShortString::from(name), + lapin::ExchangeKind::Topic, + ExchangeDeclareOptions { + durable: true, + ..Default::default() + }, + FieldTable::default(), + ) + .await + .map_err(|e| ExchangeError::Declaration { + name: name.to_string(), + source: e, + })?; + + info!(exchange = %name, "Exchange declared"); + Ok(()) + } + + /// Declare all exchanges for a chain topology (main, retry, dlx). + /// + /// NOTE: This method may be deprecated in the future as exchange management + /// moves to infrastructure-as-code or separate tooling. + pub async fn declare_topology(&self, topology: &ExchangeTopology) -> Result<(), ExchangeError> { + self.declare_exchange(&topology.main).await?; + self.declare_exchange(&topology.retry).await?; + self.declare_exchange(&topology.dlx).await?; + + info!( + main = %topology.main, + retry = %topology.retry, + dlx = %topology.dlx, + "Exchange topology declared" + ); + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_exchange_manager_new() { + let connection = Arc::new(ConnectionManager::new("amqp://localhost:5672")); + let _manager = ExchangeManager::new(connection); + } + + #[test] + fn test_exchange_manager_with_addr() { + let _manager = ExchangeManager::with_addr("amqp://localhost:5672"); + } + + // Integration tests requiring RabbitMQ + #[tokio::test] + #[ignore] + async fn test_declare_exchange() { + let manager = ExchangeManager::with_addr("amqp://localhost:5672"); + let result = manager.declare_exchange("test.exchange").await; + assert!(result.is_ok()); + } + + #[tokio::test] + #[ignore] + async fn test_declare_topology() { + let manager = ExchangeManager::with_addr("amqp://localhost:5672"); + let topology = ExchangeTopology::from_prefix("test.events"); + let result = manager.declare_topology(&topology).await; + assert!(result.is_ok()); + } +} diff --git a/listener/crates/shared/broker/src/amqp/handler.rs b/listener/crates/shared/broker/src/amqp/handler.rs new file mode 100644 index 0000000000..d30dcd91a7 --- /dev/null +++ b/listener/crates/shared/broker/src/amqp/handler.rs @@ -0,0 +1,145 @@ +pub use crate::traits::handler::{AckDecision, AsyncHandlerPayloadOnly, Handler, HandlerError}; +use crate::traits::message::Message; + +use async_trait::async_trait; +use std::{future::Future, marker::PhantomData}; + +/// Backward-compatible alias: `AsyncHandlerWithArgs` is now +/// `AsyncHandlerPayloadOnly` from the common module. +pub type AsyncHandlerWithArgs = AsyncHandlerPayloadOnly; + +/// Handler wrapper that ignores the message payload and calls `F()`. +/// +/// Kept for backward compatibility with existing broker code (e.g. cron handlers). +#[derive(Clone)] +pub struct AsyncHandlerNoArgs { + f: F, + _phantom: PhantomData, +} + +impl AsyncHandlerNoArgs { + pub fn new(f: F) -> Self { + Self { + f, + _phantom: PhantomData, + } + } +} + +#[async_trait] +impl Handler for AsyncHandlerNoArgs +where + F: Fn() -> Fut + Send + Sync + 'static, + Fut: Future> + Send, + E: std::error::Error + Send + Sync + 'static, +{ + async fn call(&self, _msg: &Message) -> Result { + (self.f)() + .await + .map(|_| AckDecision::Ack) + .map_err(|e| HandlerError::Execution(Box::new(e))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::traits::message::MessageMetadata; + use std::error::Error; + use thiserror::Error as ThisError; + + #[derive(ThisError, Debug)] + #[error("Mock error: {0}")] + struct MockError(String); + + #[derive(serde::Deserialize, Debug, PartialEq)] + struct TestPayload { + value: i32, + } + + fn make_msg(data: &[u8]) -> Message { + Message { + payload: data.to_vec(), + metadata: MessageMetadata::new("tag-1", "test.queue", 0), + } + } + + #[tokio::test] + async fn async_handler_with_args_success() { + let handler = AsyncHandlerWithArgs::new(|payload: TestPayload| async move { + assert_eq!(payload.value, 42); + Ok::<(), MockError>(()) + }); + + let msg = make_msg(br#"{"value": 42}"#); + let result = handler.call(&msg).await; + assert!(matches!(result, Ok(AckDecision::Ack))); + } + + #[tokio::test] + async fn async_handler_with_args_deserialization_error() { + let handler = + AsyncHandlerWithArgs::new( + |_payload: TestPayload| async move { Ok::<(), MockError>(()) }, + ); + + let msg = make_msg(br#"{"invalid": "json"}"#); + let result = handler.call(&msg).await; + + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!(matches!(err, HandlerError::Deserialization(_))); + assert!(err.to_string().contains("deserialization failed")); + } + + #[tokio::test] + async fn async_handler_with_args_execution_error() { + let handler = AsyncHandlerWithArgs::new(|_payload: TestPayload| async move { + Err::<(), MockError>(MockError("handler failed".to_string())) + }); + + let msg = make_msg(br#"{"value": 42}"#); + let result = handler.call(&msg).await; + + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!(matches!(err, HandlerError::Execution(_))); + assert!(err.to_string().contains("handler execution failed")); + + let source = err.source(); + assert!(source.is_some()); + assert!(source.unwrap().to_string().contains("handler failed")); + } + + #[tokio::test] + async fn async_handler_no_args_success() { + let handler = AsyncHandlerNoArgs::new(|| async move { Ok::<(), MockError>(()) }); + + let msg = make_msg(&[]); + let result = handler.call(&msg).await; + assert!(matches!(result, Ok(AckDecision::Ack))); + } + + #[tokio::test] + async fn async_handler_no_args_execution_error() { + let handler = AsyncHandlerNoArgs::new(|| async move { + Err::<(), MockError>(MockError("no args handler failed".to_string())) + }); + + let msg = make_msg(&[]); + let result = handler.call(&msg).await; + + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!(matches!(err, HandlerError::Execution(_))); + + let source = err.source(); + assert!(source.is_some()); + assert!( + source + .unwrap() + .to_string() + .contains("no args handler failed") + ); + } +} diff --git a/listener/crates/shared/broker/src/amqp/mod.rs b/listener/crates/shared/broker/src/amqp/mod.rs new file mode 100644 index 0000000000..81460107ef --- /dev/null +++ b/listener/crates/shared/broker/src/amqp/mod.rs @@ -0,0 +1,27 @@ +mod config; +mod connection; +mod consumer; +mod depth; +mod error; +mod exchange; +mod handler; +pub mod rmq_publisher; + +pub use config::{ConsumerConfigBuilder, CronConfig, ExchangeTopology, PrefetchConfig}; +pub use connection::ConnectionManager; +pub use consumer::RmqConsumer; +pub use error::{ConnectionError, ConsumerError, ExchangeError}; +pub use exchange::ExchangeManager; +pub use handler::{AsyncHandlerNoArgs, AsyncHandlerWithArgs, Handler, HandlerError}; + +// Re-export from traits for convenience +pub use crate::traits::circuit_breaker::CircuitBreakerConfig; +pub use crate::traits::{ + AckDecision, AsyncHandlerPayloadOnly, AsyncHandlerWithContext, Consumer as ConsumerTrait, + Message, MessageMetadata, Publisher as PublisherTrait, RetryPolicy, +}; + +pub use rmq_publisher::{PublisherError, RmqPublisher}; + +// Re-export depth inspector +pub use depth::AmqpQueueInspector; diff --git a/listener/crates/shared/broker/src/amqp/rmq_publisher.rs b/listener/crates/shared/broker/src/amqp/rmq_publisher.rs new file mode 100644 index 0000000000..5e1f0ad7ac --- /dev/null +++ b/listener/crates/shared/broker/src/amqp/rmq_publisher.rs @@ -0,0 +1,283 @@ +use lapin::{ + BasicProperties, Channel, Confirmation, + options::{BasicPublishOptions, ConfirmSelectOptions}, +}; +use serde::Serialize; +use std::{fmt::Debug, sync::Arc, time::Duration}; +use thiserror::Error; +use tokio::{sync::RwLock, time::sleep}; +use tracing::{info, warn}; + +use super::connection::ConnectionManager; + +#[derive(Error, Debug)] +#[non_exhaustive] +pub enum PublisherError { + #[error("serialization failed: {0}")] + Serialization(#[from] serde_json::Error), + + #[error("channel error: {0}")] + Channel(#[from] lapin::Error), + + #[error("publish was nacked by broker")] + Nacked, + + #[error( + "exchange not configured: use for_exchange() or connect() before publishing via the Publisher trait" + )] + ExchangeNotConfigured, + + #[error( + "publish failed after {retries} retries (addr: {addr}, exchange: {exchange}, routing_key: {routing_key})" + )] + RetriesExhausted { + retries: u32, + addr: String, + exchange: String, + routing_key: String, + }, +} + +pub struct RmqPublisher { + connection: Arc, + channel: RwLock, + max_retries: u32, + base_retry_delay_ms: u64, + /// Exchange fixed at construction. + /// Required for the `mq::Publisher` trait: `topic` maps to the AMQP routing key + /// and the exchange is determined here. Set via `for_exchange()`. + exchange: Option, + /// When true, publisher confirms are enabled on the channel (`confirm_select`). + /// The broker sends `basic.ack`/`basic.nack` for each publish. + confirms_enabled: bool, +} + +impl RmqPublisher { + /// Create a publisher using a shared connection manager. + pub async fn new(connection: Arc) -> Self { + Self::with_config(connection, 10, 1000).await + } + + /// Create a publisher with custom retry settings using a shared connection manager. + pub async fn with_config( + connection: Arc, + max_retries: u32, + base_retry_delay_ms: u64, + ) -> Self { + let channel = connection.create_channel_with_retry().await; + info!("RmqPublisher connected successfully"); + Self { + connection, + channel: RwLock::new(channel), + max_retries, + base_retry_delay_ms, + exchange: None, + confirms_enabled: false, + } + } + + /// Create a publisher bound to a specific exchange using a shared connection manager. + /// + /// This is required for using the queue-agnostic `Publisher` trait where + /// `topic` maps to the AMQP routing key and the exchange is fixed. + pub async fn for_exchange(connection: Arc, exchange: &str) -> Self { + let mut publisher = Self::new(connection).await; + publisher.exchange = Some(exchange.to_string()); + publisher + } + + /// Create a publisher bound to an exchange with publisher confirms enabled. + /// + /// Calls `confirm_select()` on the channel so that every `basic_publish` + /// returns a meaningful `Confirmation::Ack` or `Confirmation::Nack` from + /// the broker, rather than `Confirmation::NotRequested`. + pub async fn for_exchange_with_confirms( + connection: Arc, + exchange: &str, + ) -> Result { + let channel = connection.create_channel_with_retry().await; + channel + .confirm_select(ConfirmSelectOptions::default()) + .await + .map_err(PublisherError::Channel)?; + info!("RmqPublisher connected with publisher confirms enabled"); + Ok(Self { + connection, + channel: RwLock::new(channel), + max_retries: 10, + base_retry_delay_ms: 1000, + exchange: Some(exchange.to_string()), + confirms_enabled: true, + }) + } + + /// Convenience: create a publisher with its own connection, bound to an exchange. + /// + /// Each call creates a new TCP connection. Prefer `for_exchange()` with a shared + /// `Arc` in production to multiplex channels over one connection. + pub async fn connect(addr: &str, exchange: &str) -> Self { + let connection = Arc::new(ConnectionManager::new(addr)); + Self::for_exchange(connection, exchange).await + } + + async fn reconnect(&self) { + // Re-enable confirms on the new channel — confirm mode is per-channel, not per-connection. + // Retry with fresh channels if confirm_select fails (bounded: 3 attempts). + // If all attempts fail, install the channel anyway — the publish path's + // NotRequested handler will detect the missing confirms and trigger another retry. + let mut new_channel = self.connection.create_channel_with_retry().await; + if self.confirms_enabled { + for attempt in 0..3u32 { + match new_channel + .confirm_select(ConfirmSelectOptions::default()) + .await + { + Ok(_) => break, + Err(e) => { + warn!( + error = %e, + attempt = attempt + 1, + "Failed to enable publisher confirms on new channel" + ); + if attempt < 2 { + new_channel = self.connection.create_channel_with_retry().await; + } + } + } + } + } + let mut channel = self.channel.write().await; + *channel = new_channel; + } + + fn is_channel_healthy(channel: &Channel) -> bool { + channel.status().connected() + } + + async fn publish_amqp( + &self, + exchange: &str, + routing_key: &str, + payload: &T, + ) -> Result<(), PublisherError> { + let body = serde_json::to_vec(payload)?; + let publish_start = std::time::Instant::now(); + + for attempt in 0..self.max_retries { + { + let channel = self.channel.read().await; + if !Self::is_channel_healthy(&channel) { + drop(channel); + warn!("Channel unhealthy, attempting reconnect..."); + self.reconnect().await; + } + } + + let result = { + let channel = self.channel.read().await; + channel + .basic_publish( + exchange.into(), + routing_key.into(), + BasicPublishOptions::default(), + &body, + BasicProperties::default().with_delivery_mode(2), + ) + .await + }; + + match result { + Ok(confirm) => match confirm.await { + Ok(Confirmation::Ack(_)) => { + metrics::counter!("broker_messages_published_total", + "backend" => "amqp", "topic" => routing_key.to_owned() + ) + .increment(1); + metrics::histogram!("broker_publish_duration_seconds", + "backend" => "amqp", "topic" => routing_key.to_owned() + ) + .record(publish_start.elapsed().as_secs_f64()); + return Ok(()); + } + Ok(Confirmation::Nack(_)) => { + metrics::counter!("broker_publish_errors_total", + "backend" => "amqp", "topic" => routing_key.to_owned(), "error_kind" => "nacked" + ).increment(1); + warn!("Publish nacked by broker (attempt {})", attempt + 1); + } + Ok(Confirmation::NotRequested) => { + if self.confirms_enabled { + // confirm_select was lost (likely after reconnect failure). + // Do NOT return Ok — fall through to retry with reconnect, + // which will attempt to re-enable confirms. + warn!( + "Confirmation::NotRequested despite confirms enabled \ + (attempt {}) — will retry with reconnect", + attempt + 1 + ); + } else { + metrics::counter!("broker_messages_published_total", + "backend" => "amqp", "topic" => routing_key.to_owned() + ) + .increment(1); + metrics::histogram!("broker_publish_duration_seconds", + "backend" => "amqp", "topic" => routing_key.to_owned() + ) + .record(publish_start.elapsed().as_secs_f64()); + return Ok(()); + } + } + Err(e) => { + metrics::counter!("broker_publish_errors_total", + "backend" => "amqp", "topic" => routing_key.to_owned(), "error_kind" => "connection" + ).increment(1); + warn!("Confirm error (attempt {}): {}", attempt + 1, e); + } + }, + Err(e) => { + metrics::counter!("broker_publish_errors_total", + "backend" => "amqp", "topic" => routing_key.to_owned(), "error_kind" => "connection" + ).increment(1); + warn!("Publish failed (attempt {}): {}", attempt + 1, e); + } + } + + let delay = self + .base_retry_delay_ms + .saturating_mul(2u64.pow(attempt)) + .min(30_000); + sleep(Duration::from_millis(delay)).await; + + self.reconnect().await; + } + + Err(PublisherError::RetriesExhausted { + retries: self.max_retries, + addr: self.connection.addr().to_string(), + exchange: exchange.to_string(), + routing_key: routing_key.to_string(), + }) + } +} + +#[async_trait::async_trait] +impl crate::traits::publisher::Publisher for RmqPublisher { + type Error = PublisherError; + + /// Publish to the fixed exchange — `topic` is the AMQP routing key. + /// + /// Requires the publisher to be created via `for_exchange()`. + /// The exchange is fixed at construction; the topic string becomes the routing key, + /// matching the RFC topology (`{chain}.events.blocks`, `request.{watch_id}`, etc.). + async fn publish( + &self, + topic: &str, + payload: &T, + ) -> Result<(), Self::Error> { + let exchange = self + .exchange + .as_deref() + .ok_or(PublisherError::ExchangeNotConfigured)?; + self.publish_amqp(exchange, topic, payload).await + } +} diff --git a/listener/crates/shared/broker/src/config.rs b/listener/crates/shared/broker/src/config.rs new file mode 100644 index 0000000000..90d30a0080 --- /dev/null +++ b/listener/crates/shared/broker/src/config.rs @@ -0,0 +1,139 @@ +//! Consumer configuration types. + +use std::time::Duration; + +/// Consumer configuration with sensible defaults. +/// +/// # Examples +/// +/// ``` +/// use broker::ConsumerConfig; +/// use std::time::Duration; +/// +/// let config = ConsumerConfig::new("fetch-blocks-worker") +/// .with_prefetch(100) +/// .with_retries(5) +/// .with_circuit_breaker(3, Duration::from_secs(30)); +/// ``` +#[derive(Debug, Clone)] +pub struct ConsumerConfig { + /// Logical queue/consumer-group name (e.g., "fetch-blocks-worker", "fork-handler"). + /// - RMQ: queue defaults to `{namespace}.{group}` (or uses already-qualified input) + /// - Redis: this becomes the consumer group name + pub group: String, + + /// Instance name within the group (auto-generated if None). + /// - RMQ: this becomes the consumer tag + /// - Redis: this becomes the consumer name within the group + pub consumer_name: Option, + + /// How many messages to process in parallel (default: 10). + pub prefetch: usize, + + /// Max retries before dead-letter (default: 3). + pub max_retries: u32, + + /// Circuit breaker: (failure_threshold, cooldown_duration). + pub circuit_breaker: Option<(u32, Duration)>, +} + +impl Default for ConsumerConfig { + fn default() -> Self { + Self { + group: String::new(), + consumer_name: None, + prefetch: 10, + max_retries: 3, + circuit_breaker: None, + } + } +} + +impl ConsumerConfig { + /// Create a new consumer config with the given group name. + pub fn new(group: impl Into) -> Self { + Self { + group: group.into(), + ..Default::default() + } + } + + /// Set the prefetch count. + pub fn with_prefetch(mut self, n: usize) -> Self { + self.prefetch = n; + self + } + + /// Set the max retries before dead-letter. + pub fn with_retries(mut self, n: u32) -> Self { + self.max_retries = n; + self + } + + /// Set the circuit breaker configuration. + pub fn with_circuit_breaker(mut self, threshold: u32, cooldown: Duration) -> Self { + self.circuit_breaker = Some((threshold, cooldown)); + self + } + + /// Set the consumer name within the group. + pub fn with_consumer_name(mut self, name: impl Into) -> Self { + self.consumer_name = Some(name.into()); + self + } +} + +/// Redis-specific options (optional). +#[cfg(feature = "redis")] +#[derive(Debug, Clone, Default)] +pub struct RedisOptions { + /// Block time in milliseconds for XREADGROUP (default: 5000). + pub block_ms: Option, + /// Minimum idle time in seconds before claiming a pending message (default: 30). + pub claim_min_idle_secs: Option, + /// Interval in seconds between claim sweep cycles (default: 10). + pub claim_interval_secs: Option, +} + +/// AMQP-specific options (optional). +#[cfg(feature = "amqp")] +#[derive(Debug, Clone, Default)] +pub struct AmqpOptions { + /// Retry delay in seconds (default: 5). + pub retry_delay_secs: Option, + /// Routing key pattern for wildcards like "ethereum.blocks.#". + /// + /// When not set, broker uses the namespace-qualified default: + /// `{namespace}.{routing}`. + pub routing_key_pattern: Option, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_consumer_config_defaults() { + let config = ConsumerConfig::new("my-worker"); + assert_eq!(config.group, "my-worker"); + assert_eq!(config.prefetch, 10); + assert_eq!(config.max_retries, 3); + assert!(config.consumer_name.is_none()); + assert!(config.circuit_breaker.is_none()); + } + + #[test] + fn test_consumer_config_builder() { + let config = ConsumerConfig::new("my-worker") + .with_prefetch(100) + .with_retries(5) + .with_consumer_name("pod-1") + .with_circuit_breaker(3, Duration::from_secs(30)); + + assert_eq!(config.group, "my-worker"); + assert_eq!(config.prefetch, 100); + assert_eq!(config.max_retries, 5); + assert_eq!(config.consumer_name, Some("pod-1".to_string())); + assert_eq!(config.circuit_breaker, Some((3, Duration::from_secs(30)))); + } +} diff --git a/listener/crates/shared/broker/src/consumer.rs b/listener/crates/shared/broker/src/consumer.rs new file mode 100644 index 0000000000..a93d717fbf --- /dev/null +++ b/listener/crates/shared/broker/src/consumer.rs @@ -0,0 +1,482 @@ +//! ConsumerBuilder - fluent API for configuring consumers. + +use std::sync::Arc; +use std::time::Duration; + +use tokio_util::sync::CancellationToken; + +#[cfg(feature = "amqp")] +use crate::amqp::{ConnectionManager as AmqpConnectionManager, ConsumerConfigBuilder, RmqConsumer}; +#[cfg(feature = "redis")] +use crate::redis::{ + RedisConnectionManager, RedisConsumer, RedisConsumerConfigBuilder, StreamTopology, +}; +use crate::traits::Handler; + +#[cfg(feature = "amqp")] +use crate::AmqpInfraTopology; +#[cfg(feature = "amqp")] +use crate::config::AmqpOptions; +use crate::config::ConsumerConfig; +#[cfg(feature = "redis")] +use crate::config::RedisOptions; +use crate::error::BrokerError; +use crate::topic::Topic; + +/// Consumer builder with fluent API. +/// +/// # Examples +/// +/// ```ignore +/// // Simple case +/// broker.consumer(&topic) +/// .group("indexer") +/// .run(handler).await?; +/// +/// // Full configuration +/// broker.consumer(&topic) +/// .group("indexer") +/// .consumer_name("pod-1") +/// .prefetch(100) +/// .max_retries(5) +/// .circuit_breaker(3, Duration::from_secs(30)) +/// .redis_block_ms(5000) +/// .amqp_retry_delay(10) +/// .run(handler).await?; +/// ``` +#[must_use] +pub struct ConsumerBuilder<'a> { + backend: ConsumerBackend<'a>, + topic: Topic, + config: ConsumerConfig, + cancellation_token: Option, + #[cfg(feature = "redis")] + redis_opts: RedisOptions, + #[cfg(feature = "amqp")] + amqp_opts: AmqpOptions, +} + +pub(crate) enum ConsumerBackend<'a> { + #[cfg(feature = "redis")] + Redis(&'a Arc), + #[cfg(feature = "amqp")] + Amqp(&'a Arc, AmqpInfraTopology), +} + +impl<'a> ConsumerBuilder<'a> { + /// Create a new consumer builder (called by `Broker::consumer()`). + pub(crate) fn new(backend: ConsumerBackend<'a>, topic: Topic) -> Self { + Self { + backend, + topic, + config: ConsumerConfig::default(), + cancellation_token: None, + #[cfg(feature = "redis")] + redis_opts: RedisOptions::default(), + #[cfg(feature = "amqp")] + amqp_opts: AmqpOptions::default(), + } + } + + // ═══════════════════════════════════════════════════════════════════════════ + // Required + // ═══════════════════════════════════════════════════════════════════════════ + + /// Set the consumer group name (required). + /// + /// - **Redis**: Consumer group name + /// - **AMQP**: Logical group name (queue defaults to `{namespace}.{group}` + /// when topic is namespaced, otherwise `{group}`) + pub fn group(mut self, name: impl Into) -> Self { + self.config.group = name.into(); + self + } + + // ═══════════════════════════════════════════════════════════════════════════ + // Optional common settings + // ═══════════════════════════════════════════════════════════════════════════ + + /// Set the consumer name within the group. + /// + /// - **Redis**: Consumer name (XREADGROUP GROUP {group} {consumer}) + /// - **AMQP**: Consumer tag + /// + /// If not set, a unique name is auto-generated. + pub fn consumer_name(mut self, name: impl Into) -> Self { + self.config.consumer_name = Some(name.into()); + self + } + + /// Set the prefetch count (messages to buffer). + /// + /// - **Redis**: `read_count` for XREADGROUP + /// - **AMQP**: QoS prefetch count + /// + /// Default: 10 + pub fn prefetch(mut self, n: usize) -> Self { + self.config.prefetch = n; + self + } + + /// Set max retries before dead-letter. + /// + /// Default: 3 + pub fn max_retries(mut self, n: u32) -> Self { + self.config.max_retries = n; + self + } + + /// Set circuit breaker configuration. + /// + /// The circuit breaker pauses consumption when `threshold` consecutive + /// transient errors occur. After `cooldown`, one probe message is processed. + pub fn circuit_breaker(mut self, threshold: u32, cooldown: Duration) -> Self { + self.config.circuit_breaker = Some((threshold, cooldown)); + self + } + + /// Set a cancellation token for graceful shutdown. + /// + /// When the token is cancelled, the consumer stops after finishing its + /// current message batch. + /// + /// # Examples + /// + /// ```ignore + /// use tokio_util::sync::CancellationToken; + /// + /// let token = CancellationToken::new(); + /// let consumer = broker.consumer(&topic) + /// .group("indexer") + /// .with_cancellation(token.clone()) + /// .build()?; + /// + /// // Later, to stop the consumer: + /// token.cancel(); + /// ``` + pub fn with_cancellation(mut self, token: CancellationToken) -> Self { + self.cancellation_token = Some(token); + self + } + + // ═══════════════════════════════════════════════════════════════════════════ + // Backend-specific (silently ignored on wrong backend) + // ═══════════════════════════════════════════════════════════════════════════ + + /// Redis: Block time in milliseconds for XREADGROUP. + /// + /// **Silently ignored on AMQP.** + #[cfg(feature = "redis")] + pub fn redis_block_ms(mut self, ms: usize) -> Self { + self.redis_opts.block_ms = Some(ms); + self + } + + /// Redis: Minimum idle time (seconds) before claiming a pending message. + /// + /// **Silently ignored on AMQP.** + #[cfg(feature = "redis")] + pub fn redis_claim_min_idle(mut self, secs: u64) -> Self { + self.redis_opts.claim_min_idle_secs = Some(secs); + self + } + + /// Redis: Interval (seconds) between claim sweep cycles. + /// + /// **Silently ignored on AMQP.** + #[cfg(feature = "redis")] + pub fn redis_claim_interval(mut self, secs: u64) -> Self { + self.redis_opts.claim_interval_secs = Some(secs); + self + } + + /// AMQP: Retry delay in seconds. + /// + /// **Silently ignored on Redis.** + #[cfg(feature = "amqp")] + pub fn amqp_retry_delay(mut self, secs: u64) -> Self { + self.amqp_opts.retry_delay_secs = Some(secs); + self + } + + /// AMQP: Override the routing key for queue binding (e.g., "ethereum.blocks.#"). + /// + /// **Silently ignored on Redis.** + #[cfg(feature = "amqp")] + pub fn amqp_routing_pattern(mut self, pattern: impl Into) -> Self { + self.amqp_opts.routing_key_pattern = Some(pattern.into()); + self + } + + // ═══════════════════════════════════════════════════════════════════════════ + // Terminal methods + // ═══════════════════════════════════════════════════════════════════════════ + + /// Build the consumer without running (for later execution). + pub fn build(mut self) -> Result { + if self.config.group.is_empty() { + return Err(BrokerError::MissingGroup); + } + + let consumer_name = self + .config + .consumer_name + .clone() + .unwrap_or_else(|| format!("{}-{}", self.config.group, generate_id())); + + // Take the token before borrowing self in match arms. + let cancellation_token = self.cancellation_token.take(); + + match self.backend { + #[cfg(feature = "redis")] + ConsumerBackend::Redis(conn) => { + let mut inner = RedisConsumer::new(Arc::clone(conn)); + if let Some(token) = cancellation_token { + inner = inner.with_cancellation(token); + } + let config = self.build_redis_config(&consumer_name)?; + Ok(Consumer { + inner: ConsumerInner::Redis { + consumer: inner, + config, + }, + }) + } + #[cfg(feature = "amqp")] + ConsumerBackend::Amqp(conn, ref topology) => { + let mut inner = RmqConsumer::new(Arc::clone(conn)); + if let Some(token) = cancellation_token { + inner = inner.with_cancellation(token); + } + let config = self.build_amqp_config(&consumer_name, topology.clone())?; + Ok(Consumer { + inner: ConsumerInner::Amqp { + consumer: inner, + config, + }, + }) + } + } + } + + /// Build and run immediately (convenience method). + pub async fn run(self, handler: H) -> Result<(), BrokerError> { + let consumer = self.build()?; + consumer.run(handler).await + } + + #[cfg(feature = "redis")] + fn build_redis_config( + &self, + consumer_name: &str, + ) -> Result { + let stream_topology = StreamTopology::new(self.topic.key(), self.topic.dead_key()); + let mut builder = RedisConsumerConfigBuilder::new() + .with_topology(&stream_topology) + .group_name(&self.config.group) + .consumer_name(consumer_name) + .max_retries(self.config.max_retries) + .prefetch_count(self.config.prefetch); + + if let Some((threshold, cooldown)) = self.config.circuit_breaker { + builder = builder + .circuit_breaker_threshold(threshold) + .circuit_breaker_cooldown(cooldown); + } + + if let Some(block_ms) = self.redis_opts.block_ms { + builder = builder.block_ms(block_ms); + } + if let Some(claim_min_idle) = self.redis_opts.claim_min_idle_secs { + builder = builder.claim_min_idle(Duration::from_secs(claim_min_idle)); + } + if let Some(claim_interval) = self.redis_opts.claim_interval_secs { + builder = builder.claim_interval(Duration::from_secs(claim_interval)); + } + + Ok(builder.build_prefetch()?) + } + + #[cfg(feature = "amqp")] + fn build_amqp_config( + &self, + consumer_name: &str, + topology: AmqpInfraTopology, + ) -> Result { + let queue_name = self.resolve_amqp_queue_name(); + let routing_key: String = self + .amqp_opts + .routing_key_pattern + .clone() + .unwrap_or_else(|| self.topic.key()); + + let mut builder = ConsumerConfigBuilder::new() + .with_topology(&topology) + .queue(&queue_name) + .routing_key(&routing_key) + .consumer_tag(consumer_name) + .max_retries(self.config.max_retries) + .prefetch_count(self.config.prefetch as u16); + + if let Some((threshold, cooldown)) = self.config.circuit_breaker { + builder = builder + .circuit_breaker_threshold(threshold) + .circuit_breaker_cooldown(cooldown); + } + + if let Some(retry_delay) = self.amqp_opts.retry_delay_secs { + builder = builder.retry_delay(Duration::from_secs(retry_delay)); + } + + Ok(builder.build_prefetch()?) + } + + #[cfg(feature = "amqp")] + fn resolve_amqp_queue_name(&self) -> String { + let Some(namespace) = self.topic.namespace() else { + return self.config.group.clone(); + }; + + if self.config.group.starts_with(namespace) + && self.config.group.as_bytes().get(namespace.len()) == Some(&b'.') + { + self.config.group.clone() + } else { + format!("{namespace}.{}", self.config.group) + } + } +} + +/// A configured consumer ready to run. +pub struct Consumer { + inner: ConsumerInner, +} + +enum ConsumerInner { + #[cfg(feature = "redis")] + Redis { + consumer: RedisConsumer, + config: crate::redis::RedisPrefetchConfig, + }, + #[cfg(feature = "amqp")] + Amqp { + consumer: RmqConsumer, + config: crate::amqp::PrefetchConfig, + }, +} + +impl Consumer { + /// Ensure the underlying topology (exchanges, queues, bindings) exists + /// without starting to consume. + /// + /// For AMQP: declares exchanges, queues, and bindings so that messages + /// published to the exchange are routed correctly. This **must** be called + /// before checking queue depth (`is_empty`) or publishing seed messages, + /// otherwise AMQP silently drops messages with no bound queue. + /// + /// For Redis: no-op (streams are auto-created on first XADD). + pub async fn ensure_topology(&self) -> Result<(), BrokerError> { + match &self.inner { + #[cfg(feature = "redis")] + ConsumerInner::Redis { .. } => Ok(()), + #[cfg(feature = "amqp")] + ConsumerInner::Amqp { consumer, config } => { + consumer.ensure_topology(config).await?; + Ok(()) + } + } + } + + /// Run the consumer with the given handler (blocking, long-running). + /// + /// If a cancellation token was provided via [`ConsumerBuilder::with_cancellation`], + /// the consumer stops after finishing its current batch and draining in-flight results. + pub async fn run(self, handler: H) -> Result<(), BrokerError> { + match self.inner { + #[cfg(feature = "redis")] + ConsumerInner::Redis { consumer, config } => { + consumer.run(config, handler).await?; + } + #[cfg(feature = "amqp")] + ConsumerInner::Amqp { consumer, config } => { + consumer.run(config, handler).await?; + } + } + Ok(()) + } +} + +/// Generate a simple unique ID for consumer names. +fn generate_id() -> String { + uuid::Uuid::now_v7().to_string() +} + +#[cfg(test)] +#[cfg(feature = "amqp")] +mod tests { + use super::*; + use crate::{ + AMQP_DEFAULT_DLX_EXCHANGE, AMQP_DEFAULT_MAIN_EXCHANGE, AMQP_DEFAULT_RETRY_EXCHANGE, + }; + + fn amqp_builder<'a>(conn: &'a Arc, group: &str) -> ConsumerBuilder<'a> { + ConsumerBuilder::new( + ConsumerBackend::Amqp(conn, crate::default_amqp_infra_topology()), + Topic::new("blocks").with_namespace("ethereum"), + ) + .group(group) + } + + #[test] + fn amqp_queue_name_defaults_to_namespace_group() { + let conn = Arc::new(AmqpConnectionManager::new( + "amqp://guest:guest@localhost:5672", + )); + let builder = amqp_builder(&conn, "indexer"); + assert_eq!(builder.resolve_amqp_queue_name(), "ethereum.indexer"); + } + + #[test] + fn amqp_queue_name_keeps_prequalified_group() { + let conn = Arc::new(AmqpConnectionManager::new( + "amqp://guest:guest@localhost:5672", + )); + let builder = amqp_builder(&conn, "ethereum.indexer"); + assert_eq!(builder.resolve_amqp_queue_name(), "ethereum.indexer"); + } + + #[test] + fn amqp_queue_name_without_namespace_uses_group_as_is() { + let conn = Arc::new(AmqpConnectionManager::new( + "amqp://guest:guest@localhost:5672", + )); + let builder = ConsumerBuilder::new( + ConsumerBackend::Amqp(&conn, crate::default_amqp_infra_topology()), + Topic::new("blocks"), + ) + .group("indexer"); + + assert_eq!(builder.resolve_amqp_queue_name(), "indexer"); + } + + #[test] + fn amqp_config_uses_global_default_topology_when_not_overridden() { + let conn = Arc::new(AmqpConnectionManager::new( + "amqp://guest:guest@localhost:5672", + )); + let builder = amqp_builder(&conn, "indexer"); + let topology = match &builder.backend { + ConsumerBackend::Amqp(_, t) => t.clone(), + _ => panic!("expected AMQP backend"), + }; + let config = builder + .build_amqp_config("pod-1", topology) + .expect("amqp config should build"); + + assert_eq!(config.retry.base.exchange, AMQP_DEFAULT_MAIN_EXCHANGE); + assert_eq!(config.retry.retry_exchange, AMQP_DEFAULT_RETRY_EXCHANGE); + assert_eq!(config.retry.dead_exchange, AMQP_DEFAULT_DLX_EXCHANGE); + assert_eq!(config.retry.base.queue, "ethereum.indexer"); + assert_eq!(config.retry.base.routing_key, "ethereum.blocks"); + } +} diff --git a/listener/crates/shared/broker/src/error.rs b/listener/crates/shared/broker/src/error.rs new file mode 100644 index 0000000000..3394893b38 --- /dev/null +++ b/listener/crates/shared/broker/src/error.rs @@ -0,0 +1,53 @@ +//! Error types for broker. + +use thiserror::Error; + +/// Error type for broker operations. +#[derive(Error, Debug)] +#[non_exhaustive] +pub enum BrokerError { + /// Redis consumer error. + #[cfg(feature = "redis")] + #[error("redis error: {0}")] + Redis(#[from] crate::redis::RedisConsumerError), + + /// Redis publisher error. + #[cfg(feature = "redis")] + #[error("redis publisher error: {0}")] + RedisPublisher(#[from] crate::redis::RedisPublisherError), + + /// AMQP connection error. + #[cfg(feature = "amqp")] + #[error("amqp connection error: {0}")] + AmqpConnection(#[from] crate::amqp::ConnectionError), + + /// AMQP consumer error. + #[cfg(feature = "amqp")] + #[error("amqp consumer error: {0}")] + AmqpConsumer(#[from] crate::amqp::ConsumerError), + + /// AMQP exchange error. + #[cfg(feature = "amqp")] + #[error("amqp exchange error: {0}")] + AmqpExchange(#[from] crate::amqp::ExchangeError), + + /// AMQP publisher error. + #[cfg(feature = "amqp")] + #[error("amqp publisher error: {0}")] + AmqpPublisher(#[from] crate::amqp::PublisherError), + + /// Consumer group name was not set. + #[error("consumer group name is required")] + MissingGroup, + + /// Publisher namespace does not match topic namespace. + #[error("publisher namespace {publisher:?} does not match topic namespace {topic:?}")] + NamespaceMismatch { + publisher: Option, + topic: Option, + }, + + /// Type-erased publish error (from DynPublisher). + #[error("publish error: {0}")] + Publish(#[from] crate::traits::publisher::DynPublishError), +} diff --git a/listener/crates/shared/broker/src/lib.rs b/listener/crates/shared/broker/src/lib.rs new file mode 100644 index 0000000000..9963a83e09 --- /dev/null +++ b/listener/crates/shared/broker/src/lib.rs @@ -0,0 +1,739 @@ +//! broker - A unified message broker abstraction for Redis Streams and RabbitMQ. +//! +//! This crate provides a unified interface for message brokers with support for: +//! - **Direct routing**: Messages to specific consumers +//! - **Fanout**: Broadcast to multiple consumer groups +//! - **Competing consumers**: Load balancing within a group +//! - **Circuit breaker**: Pause consumption during downstream outages (see [`CircuitBreakerConfig`]) +//! - **Transient vs permanent errors**: [`HandlerError::transient()`] / [`HandlerError::permanent()`] drive retry + circuit breaker behavior +//! +//! # Routing Model +//! +//! ```text +//! Namespace = "ethereum" +//! Routing = "blocks" +//! +//! Redis: +//! stream = "ethereum.blocks" +//! +//! RabbitMQ (shared exchange topology): +//! exchange = "main" (configured once at broker level) +//! routing = "ethereum.blocks" (namespace-qualified) +//! ``` +//! +//! # Quick Start +//! +//! ```ignore +//! use broker::{Broker, Topic, routing, AsyncHandlerPayloadOnly}; +//! +//! // Connect to broker +//! let broker = Broker::redis("redis://localhost:6379").await?; +//! // Or: Broker::amqp("amqp://localhost:5672").build().await?; +//! +//! // Publisher +//! let publisher = broker.publisher("ethereum").await?; +//! publisher.publish("blocks", &block_event).await?; +//! +//! // Consumer +//! let topic = Topic::new(routing::BLOCKS).with_namespace("ethereum"); +//! broker.consumer(&topic) +//! .group("indexer") +//! .consumer_name("pod-1") +//! .prefetch(100) +//! .run(handler).await?; +//! ``` + +#![deny(clippy::correctness)] +#![warn(clippy::suspicious, clippy::style, clippy::complexity, clippy::perf)] + +// Core trait definitions (absorbed from the `mq` crate) +pub mod traits; + +// Backend implementations (feature-gated) +#[cfg(feature = "amqp")] +pub mod amqp; +#[cfg(feature = "redis")] +pub mod redis; + +// Metrics +pub mod metrics; + +// Facade modules +mod config; +mod consumer; +mod error; +mod publisher; +mod topic; + +use std::sync::Arc; +#[cfg(feature = "redis")] +use std::time::Duration; + +#[cfg(not(any(feature = "redis", feature = "amqp")))] +compile_error!("At least one broker backend must be enabled: \"redis\" or \"amqp\""); + +#[cfg(feature = "amqp")] +use amqp::{ + AmqpQueueInspector, ConnectionManager as AmqpConnectionManager, ExchangeManager, RmqPublisher, +}; +#[cfg(feature = "redis")] +use redis::{RedisConnectionManager, RedisPublisher, RedisQueueInspector}; +use traits::depth::QueueInspector; +use traits::publisher::DynPublisher; + +/// Default shared AMQP main exchange for broker-level global topology. +#[cfg(feature = "amqp")] +const AMQP_DEFAULT_MAIN_EXCHANGE: &str = "main"; +/// Default shared AMQP retry exchange for broker-level global topology. +#[cfg(feature = "amqp")] +const AMQP_DEFAULT_RETRY_EXCHANGE: &str = "retry"; +/// Default shared AMQP dead-letter exchange for broker-level global topology. +#[cfg(feature = "amqp")] +const AMQP_DEFAULT_DLX_EXCHANGE: &str = "dlx"; + +#[cfg(feature = "amqp")] +type AmqpInfraTopology = amqp::ExchangeTopology; + +// Re-export main types +#[cfg(feature = "amqp")] +pub use amqp::ExchangeTopology; +#[cfg(feature = "amqp")] +pub use config::AmqpOptions; +pub use config::ConsumerConfig; +#[cfg(feature = "redis")] +pub use config::RedisOptions; +pub use consumer::{Consumer, ConsumerBuilder}; +pub use error::BrokerError; +pub use publisher::Publisher; +pub use tokio_util::sync::CancellationToken; +pub use topic::Topic; + +// Re-export commonly used types from traits module +pub use traits::{ + AckDecision, AsyncHandlerNoArgs, AsyncHandlerPayloadClassified, AsyncHandlerPayloadOnly, + AsyncHandlerWithContext, CircuitBreakerConfig, Handler, HandlerError, Message, MessageMetadata, + QueueDepths, +}; + +#[cfg(feature = "amqp")] +pub(crate) fn default_amqp_infra_topology() -> AmqpInfraTopology { + AmqpInfraTopology::new( + AMQP_DEFAULT_MAIN_EXCHANGE, + AMQP_DEFAULT_RETRY_EXCHANGE, + AMQP_DEFAULT_DLX_EXCHANGE, + ) +} + +/// Message broker supporting Redis Streams and RabbitMQ. +/// +/// # Examples +/// +/// ```ignore +/// use broker::Broker; +/// +/// // Create broker from environment +/// let broker = match std::env::var("BROKER_TYPE").as_deref() { +/// Ok("redis") => Broker::redis("redis://localhost:6379").await?, +/// Ok("amqp") => Broker::amqp("amqp://localhost:5672").build().await?, +/// _ => panic!("Unknown broker type"), +/// }; +/// +/// // Get publisher +/// let publisher = broker.publisher("ethereum").await?; +/// +/// // Create consumer +/// let topic = Topic::new("blocks").with_namespace("ethereum"); +/// broker.consumer(&topic).group("indexer").run(handler).await?; +/// ``` +#[derive(Clone)] +pub enum Broker { + /// Redis Streams backend. + #[cfg(feature = "redis")] + Redis { + /// Shared connection manager. + conn: Arc, + /// When true, publisher issues `WAIT` after each `XADD` for replication durability. + ensure_publish: bool, + }, + /// RabbitMQ backend. + #[cfg(feature = "amqp")] + Amqp { + /// Shared connection manager. + conn: Arc, + /// Active AMQP topology used by publisher/consumer. + /// Defaults to shared global topology (`main`, `retry`, `dlx`). + amqp_infra_topology: Arc, + /// When true, publisher enables `confirm_select` for publisher confirms. + ensure_publish: bool, + }, +} + +impl Broker { + // ═══════════════════════════════════════════════════════════════════════════ + // Constructors + // ═══════════════════════════════════════════════════════════════════════════ + + /// Create a Redis broker from URL. + /// + /// # Examples + /// + /// ```ignore + /// let broker = Broker::redis("redis://localhost:6379").await?; + /// ``` + #[cfg(feature = "redis")] + pub async fn redis(url: &str) -> Result { + let conn = RedisConnectionManager::new_with_retry(url).await; + Ok(Self::Redis { + conn: Arc::new(conn), + ensure_publish: false, + }) + } + + /// Create a Redis broker from URL with `ensure_publish` replication durability. + /// + /// When `ensure_publish` is true, the publisher issues `WAIT 1 500` after + /// each `XADD` to confirm replication to at least one replica. + #[cfg(feature = "redis")] + pub async fn redis_with_ensure_publish( + url: &str, + ensure_publish: bool, + ) -> Result { + let conn = RedisConnectionManager::new_with_retry(url).await; + Ok(Self::Redis { + conn: Arc::new(conn), + ensure_publish, + }) + } + + /// Create an AMQP broker builder. + /// + /// Returns an [`AmqpBrokerBuilder`] for fluent configuration. + /// Defaults to exchange topology `main`, `retry`, `dlx`. + /// + /// # Examples + /// + /// ```ignore + /// // Default exchanges: "main", "retry", "dlx" + /// let broker = Broker::amqp("amqp://localhost:5672").build().await?; + /// + /// // Prefixed: "listener", "listener.retry", "listener.dlx" + /// let broker = Broker::amqp("amqp://localhost:5672") + /// .with_exchange_prefix("listener") + /// .build() + /// .await?; + /// + /// // Full control over exchange names + /// let broker = Broker::amqp("amqp://localhost:5672") + /// .with_topology(ExchangeTopology::new("a", "b", "c")) + /// .build() + /// .await?; + /// ``` + #[cfg(feature = "amqp")] + pub fn amqp(url: &str) -> AmqpBrokerBuilder<'_> { + AmqpBrokerBuilder { + url, + topology: None, + ensure_publish: false, + } + } + + #[cfg(feature = "amqp")] + fn amqp_infra_topology_snapshot(topology: &Arc) -> AmqpInfraTopology { + topology.as_ref().clone() + } + + // ═══════════════════════════════════════════════════════════════════════════ + // Publishing + // ═══════════════════════════════════════════════════════════════════════════ + + /// Get a publisher scoped to a namespace. + /// + /// # Examples + /// + /// ```ignore + /// let publisher = broker.publisher("ethereum").await?; + /// // Redis stream: "ethereum.blocks" + /// // AMQP routing key: "ethereum.blocks" on configured main exchange + /// publisher.publish("blocks", &block_event).await?; + /// publisher.publish("forks", &fork_event).await?; + /// ``` + pub async fn publisher(&self, namespace: &str) -> Result { + self.publisher_with_namespace(Some(namespace)).await + } + + /// Get an unscoped publisher (routing keys are not namespace-qualified). + pub async fn publisher_unscoped(&self) -> Result { + self.publisher_with_namespace(None).await + } + + async fn publisher_with_namespace( + &self, + namespace: Option<&str>, + ) -> Result { + let namespace = namespace.and_then(|value| { + let trimmed = value.trim(); + if trimmed.is_empty() { + None + } else { + Some(trimmed.to_string()) + } + }); + + match self { + #[cfg(feature = "redis")] + Broker::Redis { + conn, + ensure_publish, + } => { + let mut builder = RedisPublisher::builder((**conn).clone()) + .max_retries(3) + .auto_trim(Duration::from_secs(60)) + .fallback_maxlen(100_000); + + if *ensure_publish { + builder = builder.replication_wait(1, Duration::from_millis(500)); + } + + let inner = builder.build(); + let dyn_pub = DynPublisher::new(inner); + Ok(Publisher::new(dyn_pub, namespace)) + } + #[cfg(feature = "amqp")] + Broker::Amqp { + conn, + amqp_infra_topology, + ensure_publish, + } => { + let manager = ExchangeManager::new(Arc::clone(conn)); + let topology = Self::amqp_infra_topology_snapshot(amqp_infra_topology); + manager.declare_topology(&topology).await?; + + let inner = if *ensure_publish { + RmqPublisher::for_exchange_with_confirms(Arc::clone(conn), &topology.main) + .await? + } else { + RmqPublisher::for_exchange(Arc::clone(conn), &topology.main).await + }; + + let dyn_pub = DynPublisher::new(inner); + Ok(Publisher::new(dyn_pub, namespace)) + } + } + } + + // ═══════════════════════════════════════════════════════════════════════════ + // Consuming + // ═══════════════════════════════════════════════════════════════════════════ + + /// Create a consumer builder for a topic. + /// + /// # Examples + /// + /// ```ignore + /// // Simple case + /// broker.consumer(&topic) + /// .group("indexer") + /// .run(handler).await?; + /// + /// // Full configuration + /// broker.consumer(&topic) + /// .group("indexer") + /// .consumer_name("pod-1") + /// .prefetch(100) + /// .max_retries(5) + /// .circuit_breaker(3, Duration::from_secs(30)) + /// .redis_block_ms(5000) + /// .amqp_retry_delay(10) + /// .run(handler).await?; + /// ``` + pub fn consumer(&self, topic: &Topic) -> ConsumerBuilder<'_> { + match self { + #[cfg(feature = "redis")] + Broker::Redis { conn, .. } => { + ConsumerBuilder::new(consumer::ConsumerBackend::Redis(conn), topic.clone()) + } + #[cfg(feature = "amqp")] + Broker::Amqp { + conn, + amqp_infra_topology, + .. + } => { + let topology = Self::amqp_infra_topology_snapshot(amqp_infra_topology); + ConsumerBuilder::new( + consumer::ConsumerBackend::Amqp(conn, topology), + topic.clone(), + ) + } + } + } + + // ═══════════════════════════════════════════════════════════════════════════ + // Introspection + // ═══════════════════════════════════════════════════════════════════════════ + + /// Get the backend name as a string. + pub fn backend_name(&self) -> &'static str { + match self { + #[cfg(feature = "redis")] + Broker::Redis { .. } => "redis", + #[cfg(feature = "amqp")] + Broker::Amqp { .. } => "amqp", + } + } + + /// Returns whether `ensure_publish` (replication durability) is enabled. + pub fn ensure_publish(&self) -> bool { + match self { + #[cfg(feature = "redis")] + Broker::Redis { ensure_publish, .. } => *ensure_publish, + #[cfg(feature = "amqp")] + Broker::Amqp { ensure_publish, .. } => *ensure_publish, + } + } + + /// Query queue/stream depth for a given topic and optional consumer group. + /// + /// Returns message counts for the principal, retry, and dead-letter queues. + /// Uses [`Topic::key()`] as the backend queue/stream name. + /// + /// When `group` is provided, Redis also queries `XINFO GROUPS` to populate + /// `pending` (PEL count) and `lag` (undelivered entries). Use + /// [`QueueDepths::has_pending_work()`] to check if a consumer will receive + /// messages — this is the primary mechanism for seed-message decisions. + /// + /// # Examples + /// + /// ```ignore + /// use broker::{Broker, Topic, routing}; + /// + /// // Stream-level depth only (no group) + /// let topic = Topic::new(routing::BLOCKS).with_namespace("ethereum"); + /// let depths = broker.queue_depths(&topic, None).await?; + /// + /// // Consumer-group-aware depth (Redis: includes pending + lag) + /// let topic = Topic::new(routing::FETCH_NEW_BLOCKS).with_namespace("ethereum"); + /// let depths = broker.queue_depths(&topic, Some("fetch-new-blocks")).await?; + /// if !depths.has_pending_work() { + /// // Publish a seed message to bootstrap the cursor loop + /// } + /// ``` + pub async fn queue_depths( + &self, + topic: &Topic, + group: Option<&str>, + ) -> Result { + let name = topic.key(); + match self { + #[cfg(feature = "redis")] + Broker::Redis { conn, .. } => { + let inspector = RedisQueueInspector::new((**conn).clone()); + Ok(inspector.queue_depths(&name, group).await?) + } + #[cfg(feature = "amqp")] + Broker::Amqp { conn, .. } => { + let inspector = AmqpQueueInspector::new((**conn).clone()); + Ok(inspector.queue_depths(&name, group).await?) + } + } + } + + /// Fast check: is the queue/stream empty for this consumer group? + /// + /// Returns `true` when the consumer will receive **no** messages — the + /// caller should publish a seed to bootstrap the processing loop. + /// + /// Single backend round-trip (no dead-letter or retry queries). + /// + /// # Examples + /// + /// ```ignore + /// use broker::{Broker, Topic, routing}; + /// + /// let topic = Topic::new(routing::FETCH_NEW_BLOCKS).with_namespace("ethereum"); + /// if broker.is_empty(&topic, routing::FETCH_NEW_BLOCKS).await? { + /// publisher.publish(routing::FETCH_NEW_BLOCKS, &Value::Null).await?; + /// } + /// ``` + pub async fn is_empty(&self, topic: &Topic, group: &str) -> Result { + let name = topic.key(); + match self { + #[cfg(feature = "redis")] + Broker::Redis { conn, .. } => { + let inspector = RedisQueueInspector::new((**conn).clone()); + Ok(inspector.is_empty(&name, group).await?) + } + #[cfg(feature = "amqp")] + Broker::Amqp { conn, .. } => { + let inspector = AmqpQueueInspector::new((**conn).clone()); + Ok(inspector.is_empty(&name, group).await?) + } + } + } + + /// Returns `true` if the consumer group is caught up — either fully idle + /// or has at most one message currently being consumed (pending <= 1, lag == 0). + /// + /// Designed for deduplication guards: when the prefetch count is 1, a single + /// pending entry means a consumer is already processing the message. Callers + /// can use this to skip duplicate work rather than enqueueing an overlapping task. + /// + /// Single backend round-trip (no dead-letter or retry queries). + /// + /// # Examples + /// + /// ```ignore + /// use broker::{Broker, Topic, routing}; + /// + /// let topic = Topic::new(routing::FETCH_NEW_BLOCKS).with_namespace("ethereum"); + /// if broker.is_empty_or_pending(&topic, routing::FETCH_NEW_BLOCKS).await? { + /// // Consumer is idle or processing its last message — safe to skip + /// } + /// ``` + pub async fn is_empty_or_pending( + &self, + topic: &Topic, + group: &str, + ) -> Result { + let name = topic.key(); + match self { + #[cfg(feature = "redis")] + Broker::Redis { conn, .. } => { + let inspector = RedisQueueInspector::new((**conn).clone()); + Ok(inspector.is_empty_or_pending(&name, group).await?) + } + #[cfg(feature = "amqp")] + Broker::Amqp { conn, .. } => { + let inspector = AmqpQueueInspector::new((**conn).clone()); + Ok(inspector.is_empty_or_pending(&name, group).await?) + } + } + } + + /// Returns `true` if the queue or stream for this topic exists. + /// + /// - **Redis**: checks key type via `TYPE` — `true` only for stream keys. + /// - **AMQP**: passive `queue_declare` — `true` if the broker knows the queue. + /// + /// Does **not** check consumer group existence — only the underlying + /// queue/stream. + /// + /// # Examples + /// + /// ```ignore + /// use broker::{Broker, Topic, routing}; + /// + /// let topic = Topic::new(routing::FETCH_NEW_BLOCKS).with_namespace("ethereum"); + /// if !broker.exists(&topic).await? { + /// // Queue/stream hasn't been created yet — skip depth check + /// } + /// ``` + pub async fn exists(&self, topic: &Topic) -> Result { + let name = topic.key(); + match self { + #[cfg(feature = "redis")] + Broker::Redis { conn, .. } => { + let inspector = RedisQueueInspector::new((**conn).clone()); + Ok(inspector.exists(&name).await?) + } + #[cfg(feature = "amqp")] + Broker::Amqp { conn, .. } => { + let inspector = AmqpQueueInspector::new((**conn).clone()); + Ok(inspector.exists(&name).await?) + } + } + } + + // ═══════════════════════════════════════════════════════════════════════════ + // Health + // ═══════════════════════════════════════════════════════════════════════════ + + /// Lightweight connectivity probe for the broker connection. + /// + /// - **Redis**: sends a `PING` via the shared connection manager. + /// - **AMQP**: opens a throwaway channel, which internally requires + /// `conn.status().connected()`. The channel is dropped immediately. + /// + /// Does not touch any queue/stream. Intended for `/readyz`-style probes + /// that must fail fast when the broker is unreachable. + pub async fn health_check(&self) -> Result<(), BrokerError> { + match self { + #[cfg(feature = "redis")] + Broker::Redis { conn, .. } => Ok(conn.health_check().await?), + #[cfg(feature = "amqp")] + Broker::Amqp { conn, .. } => { + let _ch = conn.create_channel().await?; + Ok(()) + } + } + } +} + +/// Builder for configuring an AMQP broker. +/// +/// Created via [`Broker::amqp`]. Call [`.build()`](AmqpBrokerBuilder::build) +/// to finalize. +#[cfg(feature = "amqp")] +#[must_use] +pub struct AmqpBrokerBuilder<'a> { + url: &'a str, + topology: Option, + ensure_publish: bool, +} + +#[cfg(feature = "amqp")] +impl<'a> AmqpBrokerBuilder<'a> { + /// Set exchange prefix — derives `"{prefix}"`, `"{prefix}.retry"`, `"{prefix}.dlx"`. + /// + /// ```ignore + /// Broker::amqp("amqp://localhost:5672") + /// .with_exchange_prefix("listener") + /// .build() + /// .await?; + /// ``` + pub fn with_exchange_prefix(mut self, prefix: impl AsRef) -> Self { + self.topology = Some(AmqpInfraTopology::from_prefix(prefix)); + self + } + + /// Set a fully custom exchange topology. + /// + /// ```ignore + /// use broker::ExchangeTopology; + /// + /// Broker::amqp("amqp://localhost:5672") + /// .with_topology(ExchangeTopology::new("my-main", "my-retry", "my-dlx")) + /// .build() + /// .await?; + /// ``` + pub fn with_topology(mut self, topology: AmqpInfraTopology) -> Self { + self.topology = Some(topology); + self + } + + /// Enable publisher confirms for replication-aware publish durability. + /// + /// When enabled, the publisher calls `confirm_select()` on its channel + /// and awaits `Confirmation::Ack` from the broker for every publish. + pub fn with_ensure_publish(mut self, ensure_publish: bool) -> Self { + self.ensure_publish = ensure_publish; + self + } + + /// Build the AMQP broker. + pub async fn build(self) -> Result { + let topology = self.topology.unwrap_or_else(default_amqp_infra_topology); + let conn = AmqpConnectionManager::new(self.url); + Ok(Broker::Amqp { + conn: Arc::new(conn), + amqp_infra_topology: Arc::new(topology), + ensure_publish: self.ensure_publish, + }) + } +} + +impl std::fmt::Debug for Broker { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + #[cfg(feature = "redis")] + Broker::Redis { ensure_publish, .. } => f + .debug_struct("Broker::Redis") + .field("ensure_publish", ensure_publish) + .finish_non_exhaustive(), + #[cfg(feature = "amqp")] + Broker::Amqp { ensure_publish, .. } => f + .debug_struct("Broker::Amqp") + .field("ensure_publish", ensure_publish) + .finish_non_exhaustive(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_topic_creation() { + let topic = Topic::new("blocks").with_namespace("ethereum"); + assert_eq!(topic.namespace(), Some("ethereum")); + assert_eq!(topic.routing_segment(), "blocks"); + assert_eq!(topic.key(), "ethereum.blocks"); + } + + #[test] + fn test_consumer_config() { + let config = ConsumerConfig::new("my-worker") + .with_prefetch(100) + .with_retries(5); + + assert_eq!(config.group, "my-worker"); + assert_eq!(config.prefetch, 100); + assert_eq!(config.max_retries, 5); + } + + #[cfg(feature = "amqp")] + #[tokio::test] + async fn test_amqp_default_topology() { + let broker = Broker::amqp("amqp://guest:guest@localhost:5672") + .build() + .await + .unwrap(); + match &broker { + Broker::Amqp { + amqp_infra_topology, + .. + } => { + assert_eq!(amqp_infra_topology.main, "main"); + assert_eq!(amqp_infra_topology.retry, "retry"); + assert_eq!(amqp_infra_topology.dlx, "dlx"); + } + #[allow(unreachable_patterns)] + _ => panic!("expected AMQP variant"), + } + } + + #[cfg(feature = "amqp")] + #[tokio::test] + async fn test_amqp_with_exchange_prefix() { + let broker = Broker::amqp("amqp://guest:guest@localhost:5672") + .with_exchange_prefix("listener") + .build() + .await + .unwrap(); + match &broker { + Broker::Amqp { + amqp_infra_topology, + .. + } => { + assert_eq!(amqp_infra_topology.main, "listener"); + assert_eq!(amqp_infra_topology.retry, "listener.retry"); + assert_eq!(amqp_infra_topology.dlx, "listener.dlx"); + } + #[allow(unreachable_patterns)] + _ => panic!("expected AMQP variant"), + } + } + + #[cfg(feature = "amqp")] + #[tokio::test] + async fn test_amqp_with_custom_topology() { + let topology = ExchangeTopology::new("my-app", "my-app-retry", "my-app-dead"); + let broker = Broker::amqp("amqp://guest:guest@localhost:5672") + .with_topology(topology) + .build() + .await + .unwrap(); + match &broker { + Broker::Amqp { + amqp_infra_topology, + .. + } => { + assert_eq!(amqp_infra_topology.main, "my-app"); + assert_eq!(amqp_infra_topology.retry, "my-app-retry"); + assert_eq!(amqp_infra_topology.dlx, "my-app-dead"); + } + #[allow(unreachable_patterns)] + _ => panic!("expected AMQP variant"), + } + } +} diff --git a/listener/crates/shared/broker/src/metrics.rs b/listener/crates/shared/broker/src/metrics.rs new file mode 100644 index 0000000000..a822f6dcd7 --- /dev/null +++ b/listener/crates/shared/broker/src/metrics.rs @@ -0,0 +1,293 @@ +//! Broker metrics registration and helpers. +//! +//! This module provides: +//! - [`describe_metrics()`]: registers Prometheus HELP strings for all broker metrics +//! - [`record_queue_depths()`]: converts a [`QueueDepths`] into gauge observations +//! - [`spawn_queue_depth_poller()`]: background task that polls queue depths periodically + +use std::time::Duration; + +use tokio::task::JoinHandle; +use tokio::time::{MissedTickBehavior, interval}; +use tokio_util::sync::CancellationToken; +use tracing::debug; + +use crate::traits::depth::QueueDepths; +use crate::{Broker, Topic}; + +/// Register metric descriptions with the global recorder. +/// +/// Call once at application startup, after installing the metrics exporter. +/// Safe to call multiple times (describe is idempotent). +pub fn describe_metrics() { + use metrics::{Unit, describe_counter, describe_gauge, describe_histogram}; + + // ── Publishing ──────────────────────────────────────────────────────── + describe_counter!( + "broker_messages_published_total", + Unit::Count, + "Total messages successfully published" + ); + describe_counter!( + "broker_publish_errors_total", + Unit::Count, + "Publish failures per attempt" + ); + describe_histogram!( + "broker_publish_duration_seconds", + Unit::Seconds, + "End-to-end publish latency including retries" + ); + + // ── Consuming ───────────────────────────────────────────────────────── + describe_counter!( + "broker_messages_consumed_total", + Unit::Count, + "Messages processed by handler, partitioned by outcome" + ); + describe_histogram!( + "broker_handler_duration_seconds", + Unit::Seconds, + "Handler execution wall-clock time per message" + ); + describe_counter!( + "broker_messages_dead_lettered_total", + Unit::Count, + "Messages routed to dead-letter queue" + ); + describe_histogram!( + "broker_message_delivery_count", + Unit::Count, + "Distribution of delivery counts at processing time" + ); + + // ── Circuit breaker ─────────────────────────────────────────────────── + describe_gauge!( + "broker_circuit_breaker_state", + Unit::Count, + "Circuit breaker state: 0=closed, 1=open, 2=half_open" + ); + describe_counter!( + "broker_circuit_breaker_trips_total", + Unit::Count, + "Times the circuit breaker tripped to open" + ); + describe_gauge!( + "broker_circuit_breaker_consecutive_failures", + Unit::Count, + "Current consecutive transient failure count" + ); + + // ── Queue depth ─────────────────────────────────────────────────────── + describe_gauge!( + "broker_queue_depth_principal", + Unit::Count, + "Messages in the principal queue/stream" + ); + describe_gauge!( + "broker_queue_depth_retry", + Unit::Count, + "Messages in the retry queue (AMQP only)" + ); + describe_gauge!( + "broker_queue_depth_dead_letter", + Unit::Count, + "Messages in the dead-letter queue/stream" + ); + describe_gauge!( + "broker_queue_depth_pending", + Unit::Count, + "Pending entry list count (Redis only)" + ); + describe_gauge!( + "broker_queue_depth_lag", + Unit::Count, + "Consumer group lag (Redis 7.0+ only)" + ); + + // ── Connection health ───────────────────────────────────────────────── + describe_counter!( + "broker_consumer_reconnections_total", + Unit::Count, + "Consumer reconnection cycles" + ); + describe_gauge!( + "broker_consumer_connected", + Unit::Count, + "Consumer connectivity: 1=connected, 0=reconnecting" + ); + describe_counter!( + "broker_claim_sweeper_messages_claimed_total", + Unit::Count, + "Messages reclaimed by ClaimSweeper" + ); + describe_counter!( + "broker_claim_sweeper_messages_dead_lettered_total", + Unit::Count, + "Messages moved to DLQ by ClaimSweeper" + ); +} + +/// Record [`QueueDepths`] as Prometheus gauges. +/// +/// Intended to be called periodically (e.g., every 15s) by the application +/// to keep queue depth gauges fresh for Prometheus scraping. +/// +/// `None` fields (e.g., `retry` for Redis, `pending`/`lag` for AMQP) are +/// silently skipped — no gauge is set. +pub fn record_queue_depths(depths: &QueueDepths, backend: &str, topic: &str) { + metrics::gauge!("broker_queue_depth_principal", "backend" => backend.to_owned(), "topic" => topic.to_owned()) + .set(depths.principal as f64); + if let Some(retry) = depths.retry { + metrics::gauge!("broker_queue_depth_retry", "backend" => backend.to_owned(), "topic" => topic.to_owned()) + .set(retry as f64); + } + metrics::gauge!("broker_queue_depth_dead_letter", "backend" => backend.to_owned(), "topic" => topic.to_owned()) + .set(depths.dead_letter as f64); + if let Some(pending) = depths.pending { + metrics::gauge!("broker_queue_depth_pending", "backend" => backend.to_owned(), "topic" => topic.to_owned()) + .set(pending as f64); + } + if let Some(lag) = depths.lag { + metrics::gauge!("broker_queue_depth_lag", "backend" => backend.to_owned(), "topic" => topic.to_owned()) + .set(lag as f64); + } +} + +/// Map a [`HandlerOutcome`] to a static label string for the `outcome` label. +pub(crate) fn outcome_label(outcome: &crate::traits::handler::HandlerOutcome) -> &'static str { + use crate::traits::handler::HandlerOutcome; + match outcome { + HandlerOutcome::Ack => "ack", + HandlerOutcome::Nack => "nack", + HandlerOutcome::Dead => "dead", + HandlerOutcome::Delay(_) => "delay", + HandlerOutcome::Transient => "transient", + HandlerOutcome::Permanent => "permanent", + } +} + +// ═══════════════════════════════════════════════════════════════════════════ +// Queue depth periodic poller +// ═══════════════════════════════════════════════════════════════════════════ + +/// A (topic, consumer-group) pair to poll for queue depth metrics. +/// +/// The `topic` label emitted to Prometheus is derived from [`Topic::key`] +/// (e.g. `"chain-id-1.fetch-new-blocks"`). +#[derive(Debug, Clone)] +pub struct QueueDepthPollTarget { + pub topic: Topic, + pub group: String, +} + +impl QueueDepthPollTarget { + pub fn new(topic: Topic, group: impl Into) -> Self { + Self { + topic, + group: group.into(), + } + } +} + +/// Spawn a background task that polls queue depths at `poll_interval` and +/// records them as gauges via [`record_queue_depths`]. +/// +/// The `backend` label is read from [`Broker::backend_name`]. The `topic` +/// label is [`Topic::key`]. +/// +/// Errors (e.g. a queue/stream that doesn't exist yet) are logged at DEBUG and +/// swallowed — the next tick retries. The task stops when `cancel` fires. +/// +/// Call this once at startup, after the broker and consumer topologies are +/// ready. Dropping the returned `JoinHandle` does **not** stop the task — +/// use `cancel.cancel()` for that. +/// +/// # Example +/// +/// ```ignore +/// use broker::metrics::{spawn_queue_depth_poller, QueueDepthPollTarget}; +/// use broker::Topic; +/// use std::time::Duration; +/// use tokio_util::sync::CancellationToken; +/// +/// let cancel = CancellationToken::new(); +/// let targets = vec![ +/// QueueDepthPollTarget::new( +/// Topic::new("fetch-new-blocks").with_namespace("chain-id-1"), +/// "fetch-new-blocks", +/// ), +/// ]; +/// spawn_queue_depth_poller(broker.clone(), targets, Duration::from_secs(15), cancel); +/// ``` +pub fn spawn_queue_depth_poller( + broker: Broker, + targets: Vec, + poll_interval: Duration, + cancel: CancellationToken, +) -> JoinHandle<()> { + let backend = broker.backend_name(); + tokio::spawn(async move { + let mut tick = interval(poll_interval); + tick.set_missed_tick_behavior(MissedTickBehavior::Delay); + loop { + tokio::select! { + _ = cancel.cancelled() => return, + _ = tick.tick() => { + for target in &targets { + match broker.queue_depths(&target.topic, Some(&target.group)).await { + Ok(depths) => { + let label = target.topic.key(); + record_queue_depths(&depths, backend, &label); + } + Err(e) => { + debug!( + topic = %target.topic, + error = %e, + "queue_depths poll failed (topology may not exist yet)" + ); + } + } + } + } + } + } + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + // AC-2.1 + #[test] + fn describe_metrics_does_not_panic() { + describe_metrics(); + } + + // AC-2.2 (partial — gauges are set, but asserting requires a test recorder) + #[test] + fn record_queue_depths_with_all_fields() { + let depths = QueueDepths { + principal: 42, + retry: Some(5), + dead_letter: 3, + pending: Some(10), + lag: Some(7), + }; + // No recorder installed → calls are no-ops, must not panic + record_queue_depths(&depths, "redis", "test.stream"); + } + + #[test] + fn record_queue_depths_skips_none_fields() { + let depths = QueueDepths { + principal: 10, + retry: None, + dead_letter: 0, + pending: None, + lag: None, + }; + record_queue_depths(&depths, "redis", "test.stream"); + } +} diff --git a/listener/crates/shared/broker/src/publisher.rs b/listener/crates/shared/broker/src/publisher.rs new file mode 100644 index 0000000000..ecb63d05ab --- /dev/null +++ b/listener/crates/shared/broker/src/publisher.rs @@ -0,0 +1,110 @@ +//! Publisher wrapper for optionally namespace-scoped publishing. + +use serde::Serialize; +use std::borrow::Cow; +use std::fmt::Debug; + +use crate::traits::publisher::DynPublisher; + +use crate::error::BrokerError; +use crate::topic::Topic; + +/// Publisher optionally scoped to a namespace. +/// +/// Publishes messages to routing keys: +/// - namespaced: `{namespace}.{routing}` +/// - unscoped: `{routing}` +#[derive(Clone)] +pub struct Publisher { + inner: DynPublisher, + namespace: Option, + /// Precomputed "{namespace}." prefix to avoid format! on every publish. + namespace_prefix: Option, +} + +impl Publisher { + /// Create a new Publisher (typically called by `Broker`). + pub(crate) fn new(inner: DynPublisher, namespace: Option) -> Self { + let namespace_prefix = namespace.as_ref().map(|ns| format!("{ns}.")); + Self { + inner, + namespace, + namespace_prefix, + } + } + + /// Get the namespace this publisher is scoped to. + pub fn namespace(&self) -> Option<&str> { + self.namespace.as_deref() + } + + /// Publish to a routing key (e.g., "blocks", "forks"). + pub async fn publish( + &self, + routing: &str, + payload: &T, + ) -> Result<(), BrokerError> { + let topic = self.make_topic(routing); + self.inner.publish(&topic, payload).await?; + Ok(()) + } + + /// Publish to a `Topic` directly. + /// + /// Publisher namespace must match topic namespace (including both being unscoped). + pub async fn publish_to_topic( + &self, + topic: &Topic, + payload: &T, + ) -> Result<(), BrokerError> { + if topic.namespace() != self.namespace() { + return Err(BrokerError::NamespaceMismatch { + publisher: self.namespace.clone(), + topic: topic.namespace().map(str::to_owned), + }); + } + + self.publish(topic.routing_segment(), payload).await + } + + /// Publish multiple payloads with the same routing key. + pub async fn publish_batch( + &self, + routing: &str, + payloads: &[T], + ) -> Result<(), BrokerError> { + for payload in payloads { + self.publish(routing, payload).await?; + } + Ok(()) + } + + /// Graceful shutdown - cancels background tasks. + pub async fn shutdown(&self) { + self.inner.shutdown().await; + } + + /// Build the fully qualified logical key. + /// + /// Returns `Cow::Borrowed` when unscoped (zero allocation), + /// or `Cow::Owned` with a precomputed prefix when namespaced. + fn make_topic<'a>(&'a self, routing: &'a str) -> Cow<'a, str> { + match &self.namespace_prefix { + Some(prefix) => { + let mut s = String::with_capacity(prefix.len() + routing.len()); + s.push_str(prefix); + s.push_str(routing); + Cow::Owned(s) + } + None => Cow::Borrowed(routing), + } + } +} + +impl std::fmt::Debug for Publisher { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Publisher") + .field("namespace", &self.namespace()) + .finish_non_exhaustive() + } +} diff --git a/listener/crates/shared/broker/src/redis/circuit_breaker.rs b/listener/crates/shared/broker/src/redis/circuit_breaker.rs new file mode 100644 index 0000000000..d84beea564 --- /dev/null +++ b/listener/crates/shared/broker/src/redis/circuit_breaker.rs @@ -0,0 +1 @@ +pub use crate::traits::circuit_breaker::{CircuitBreaker, CircuitBreakerConfig}; diff --git a/listener/crates/shared/broker/src/redis/claim_task.rs b/listener/crates/shared/broker/src/redis/claim_task.rs new file mode 100644 index 0000000000..610988ec05 --- /dev/null +++ b/listener/crates/shared/broker/src/redis/claim_task.rs @@ -0,0 +1,472 @@ +use redis::Value; +use std::sync::{ + Arc, + atomic::{AtomicBool, Ordering}, +}; +use std::time::Duration; +use tokio::time::sleep; +use tokio_util::sync::CancellationToken; +use tracing::{debug, error, info, warn}; + +use super::{ + config::RedisRetryConfig, connection::RedisConnectionManager, error::RedisConsumerError, +}; + +const CLASS_TRANSIENT: &str = "transient"; +const CLASS_PERMANENT: &str = "permanent"; + +/// Background task that sweeps for idle pending messages and either +/// re-claims them for reprocessing or routes them to the dead-letter stream. +/// +/// Runs as a supervised loop — restarts after panics with a delay. +pub struct ClaimSweeper { + connection: RedisConnectionManager, + config: RedisRetryConfig, + classification_paused: Option>, +} + +/// A pending entry returned by XPENDING with detail. +#[derive(Debug)] +struct PendingEntry { + id: String, + _consumer: String, + idle_ms: u64, + delivery_count: u64, +} + +impl ClaimSweeper { + pub fn new(connection: RedisConnectionManager, config: RedisRetryConfig) -> Self { + Self { + connection, + config, + classification_paused: None, + } + } + + pub fn new_with_classification_pause( + connection: RedisConnectionManager, + config: RedisRetryConfig, + classification_paused: Arc, + ) -> Self { + Self { + connection, + config, + classification_paused: Some(classification_paused), + } + } + + /// Run the claim sweeper as a supervisor loop. + /// + /// Periodically calls `sweep_once()` at `config.claim_interval`. + /// On panic/error: logs, waits, and restarts. + /// Stops when the cancellation token is cancelled. + pub async fn run(&self, cancel: CancellationToken) { + info!( + stream = %self.config.base.stream, + group = %self.config.base.group_name, + interval = ?self.config.claim_interval, + "ClaimSweeper started" + ); + + loop { + tokio::select! { + _ = cancel.cancelled() => { + info!("ClaimSweeper: cancellation requested, stopping"); + return; + } + _ = sleep(self.config.claim_interval) => { + if self.is_classification_paused() { + debug!( + stream = %self.config.base.stream, + group = %self.config.base.group_name, + "ClaimSweeper paused: waiting for classification writes to recover" + ); + continue; + } + match self.sweep_once().await { + Ok(()) => {} + Err(e) => { + if e.is_connection_error() { + self.connection.force_reconnect().await; + } + error!( + error = %e, + "ClaimSweeper: sweep_once failed, will retry next interval" + ); + // Extra delay after error to avoid tight error loops + sleep(Duration::from_secs(5)).await; + } + } + } + } + } + } + + fn is_classification_paused(&self) -> bool { + self.classification_paused + .as_ref() + .is_some_and(|paused| paused.load(Ordering::Relaxed)) + } + + /// Perform a single sweep: find idle pending messages, claim or DLQ them. + async fn sweep_once(&self) -> Result<(), RedisConsumerError> { + let pending_entries = self.get_pending_entries().await?; + + if pending_entries.is_empty() { + debug!( + stream = %self.config.base.stream, + group = %self.config.base.group_name, + "ClaimSweeper: no idle pending messages" + ); + return Ok(()); + } + + for entry in &pending_entries { + if entry.idle_ms < self.config.claim_min_idle.as_millis() as u64 { + continue; + } + + let classification = match self.get_classification(&entry.id).await { + Ok(classification) => classification, + Err(e) => { + // Unknown marker classification must never trigger unsafe DLQ. + warn!( + error = %e, + stream_id = %entry.id, + "Classification lookup failed; defaulting to transient path" + ); + None + } + }; + + if classification.as_deref() == Some(CLASS_TRANSIENT) { + // Infinite transient retry: reclaim without increasing delivery count. + self.claim_message(entry, Some(entry.delivery_count)) + .await?; + metrics::counter!( + "broker_claim_sweeper_messages_claimed_total", + "backend" => "redis", + "topic" => self.config.base.stream.clone(), + ) + .increment(1); + continue; + } + + if classification.as_deref() != Some(CLASS_PERMANENT) { + // No explicit permanent classification => fail-safe transient handling. + self.claim_message(entry, Some(entry.delivery_count)) + .await?; + metrics::counter!( + "broker_claim_sweeper_messages_claimed_total", + "backend" => "redis", + "topic" => self.config.base.stream.clone(), + ) + .increment(1); + } else if entry.delivery_count >= self.config.max_retries as u64 { + // Explicitly permanent and retry budget exhausted => move to dead stream. + self.move_to_dead_letter(entry).await?; + metrics::counter!( + "broker_claim_sweeper_messages_dead_lettered_total", + "backend" => "redis", + "topic" => self.config.base.stream.clone(), + ) + .increment(1); + } else { + // Explicitly permanent and still under budget => bounded retry path. + self.claim_message(entry, None).await?; + metrics::counter!( + "broker_claim_sweeper_messages_claimed_total", + "backend" => "redis", + "topic" => self.config.base.stream.clone(), + ) + .increment(1); + } + } + + Ok(()) + } + + /// Get pending entries via `XPENDING {stream} {group} - + 100`. + async fn get_pending_entries(&self) -> Result, RedisConsumerError> { + let mut conn = self.connection.get_connection(); + + let result: Value = redis::cmd("XPENDING") + .arg(&self.config.base.stream) + .arg(&self.config.base.group_name) + .arg("-") + .arg("+") + .arg(100) + .query_async(&mut conn) + .await + .map_err(|e| RedisConsumerError::Claim { + stream: self.config.base.stream.clone(), + source: e, + })?; + + let entries = Self::parse_pending_entries(result); + Ok(entries) + } + + /// Parse the XPENDING detail reply into structured entries. + /// + /// XPENDING returns an array of arrays: + /// `[[id, consumer, idle_ms, delivery_count], ...]` + fn parse_pending_entries(value: Value) -> Vec { + let mut entries = Vec::new(); + + if let Value::Array(items) = value { + for item in items { + if let Value::Array(fields) = item + && fields.len() >= 4 + { + let id = match &fields[0] { + Value::BulkString(b) => String::from_utf8_lossy(b).to_string(), + _ => continue, + }; + let consumer = match &fields[1] { + Value::BulkString(b) => String::from_utf8_lossy(b).to_string(), + _ => continue, + }; + let idle_ms = match &fields[2] { + Value::Int(n) => *n as u64, + _ => continue, + }; + let delivery_count = match &fields[3] { + Value::Int(n) => *n as u64, + _ => continue, + }; + + entries.push(PendingEntry { + id, + _consumer: consumer, + idle_ms, + delivery_count, + }); + } + } + } + + entries + } + + /// Claim a message for this consumer via XCLAIM. + /// + /// When `retry_count_override` is provided, the claim uses `RETRYCOUNT` to + /// preserve that delivery count (used for infinite transient retries). + async fn claim_message( + &self, + entry: &PendingEntry, + retry_count_override: Option, + ) -> Result<(), RedisConsumerError> { + let mut conn = self.connection.get_connection(); + + let mut cmd = redis::cmd("XCLAIM"); + cmd.arg(&self.config.base.stream) + .arg(&self.config.base.group_name) + .arg(&self.config.base.consumer_name) + .arg(self.config.claim_min_idle.as_millis() as u64) + .arg(&entry.id); + + if let Some(retry_count) = retry_count_override { + cmd.arg("RETRYCOUNT").arg(retry_count); + } + + let _: Value = cmd + .query_async(&mut conn) + .await + .map_err(|e| RedisConsumerError::Claim { + stream: self.config.base.stream.clone(), + source: e, + })?; + + debug!( + stream = %self.config.base.stream, + id = %entry.id, + delivery_count = %entry.delivery_count, + idle_ms = %entry.idle_ms, + "Claimed idle message for reprocessing" + ); + + Ok(()) + } + + /// Move a message to the dead-letter stream and acknowledge from the main stream. + async fn move_to_dead_letter(&self, entry: &PendingEntry) -> Result<(), RedisConsumerError> { + let mut conn = self.connection.get_connection(); + + // First, read the message data so we can copy it to DLQ + let data = self.read_message_data(&entry.id).await?; + + // XADD to dead-letter stream with metadata + let _: String = redis::cmd("XADD") + .arg(&self.config.dead_stream) + .arg("*") + .arg("original_id") + .arg(&entry.id) + .arg("original_stream") + .arg(&self.config.base.stream) + .arg("delivery_count") + .arg(entry.delivery_count) + .arg("data") + .arg(&data) + .query_async(&mut conn) + .await + .map_err(|e| RedisConsumerError::DeadLetter { + stream: self.config.dead_stream.clone(), + source: e, + })?; + + // XACK from the main stream + let _: i64 = redis::cmd("XACK") + .arg(&self.config.base.stream) + .arg(&self.config.base.group_name) + .arg(&entry.id) + .query_async(&mut conn) + .await + .map_err(|e| RedisConsumerError::Acknowledge { + stream: self.config.base.stream.clone(), + source: e, + })?; + + // Clear stale classification marker now that the message is terminal. + self.clear_classification(&entry.id).await?; + + warn!( + stream = %self.config.base.stream, + dead_stream = %self.config.dead_stream, + id = %entry.id, + delivery_count = %entry.delivery_count, + "Message moved to dead-letter stream after max retries" + ); + + Ok(()) + } + + async fn get_classification( + &self, + stream_id: &str, + ) -> Result, RedisConsumerError> { + let mut conn = self.connection.get_connection(); + let marker_key = self.config.classification_marker_key(); + redis::cmd("HGET") + .arg(&marker_key) + .arg(stream_id) + .query_async(&mut conn) + .await + .map_err(|e| RedisConsumerError::TransientMarker { + key: marker_key, + source: e, + }) + } + + async fn clear_classification(&self, stream_id: &str) -> Result<(), RedisConsumerError> { + let mut conn = self.connection.get_connection(); + let marker_key = self.config.classification_marker_key(); + let _: i64 = redis::cmd("HDEL") + .arg(&marker_key) + .arg(stream_id) + .query_async(&mut conn) + .await + .map_err(|e| RedisConsumerError::TransientMarker { + key: marker_key, + source: e, + })?; + Ok(()) + } + + /// Read message data from the stream by ID via XRANGE. + async fn read_message_data(&self, id: &str) -> Result, RedisConsumerError> { + let mut conn = self.connection.get_connection(); + + let result: Value = redis::cmd("XRANGE") + .arg(&self.config.base.stream) + .arg(id) + .arg(id) + .query_async(&mut conn) + .await + .map_err(|e| RedisConsumerError::StreamRead { + stream: self.config.base.stream.clone(), + source: e, + })?; + + // Parse XRANGE response: [[id, [field, value, field, value, ...]], ...] + if let Value::Array(entries) = result + && let Some(Value::Array(entry)) = entries.into_iter().next() + && entry.len() >= 2 + && let Value::Array(fields) = &entry[1] + { + // Find the "data" field + let mut iter = fields.iter(); + while let Some(key) = iter.next() { + if let Value::BulkString(k) = key + && k == b"data" + && let Some(Value::BulkString(v)) = iter.next() + { + return Ok(v.clone()); + } + // Skip value if key didn't match + let _ = iter.next(); + } + } + + // If we can't find the data, return empty bytes + // This can happen if the message was already deleted + warn!( + stream = %self.config.base.stream, + id = %id, + "Could not read message data for DLQ, using empty payload" + ); + Ok(Vec::new()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_pending_entries_empty() { + let value = Value::Array(vec![]); + let entries = ClaimSweeper::parse_pending_entries(value); + assert!(entries.is_empty()); + } + + #[test] + fn test_parse_pending_entries_with_data() { + let value = Value::Array(vec![Value::Array(vec![ + Value::BulkString(b"1234567890-0".to_vec()), + Value::BulkString(b"consumer-1".to_vec()), + Value::Int(60000), + Value::Int(3), + ])]); + + let entries = ClaimSweeper::parse_pending_entries(value); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].id, "1234567890-0"); + assert_eq!(entries[0]._consumer, "consumer-1"); + assert_eq!(entries[0].idle_ms, 60000); + assert_eq!(entries[0].delivery_count, 3); + } + + #[test] + fn test_parse_pending_entries_multiple() { + let value = Value::Array(vec![ + Value::Array(vec![ + Value::BulkString(b"1-0".to_vec()), + Value::BulkString(b"c1".to_vec()), + Value::Int(10000), + Value::Int(1), + ]), + Value::Array(vec![ + Value::BulkString(b"2-0".to_vec()), + Value::BulkString(b"c2".to_vec()), + Value::Int(90000), + Value::Int(5), + ]), + ]); + + let entries = ClaimSweeper::parse_pending_entries(value); + assert_eq!(entries.len(), 2); + assert_eq!(entries[1].delivery_count, 5); + } +} diff --git a/listener/crates/shared/broker/src/redis/config.rs b/listener/crates/shared/broker/src/redis/config.rs new file mode 100644 index 0000000000..30bf50a1a8 --- /dev/null +++ b/listener/crates/shared/broker/src/redis/config.rs @@ -0,0 +1,356 @@ +use std::time::Duration; + +use crate::traits::consumer::RetryPolicy; + +use super::circuit_breaker::CircuitBreakerConfig; +use super::error::RedisConsumerError; + +/// Stream topology for a specific chain. +/// Provides consistent naming for main and dead-letter streams. +#[derive(Debug, Clone)] +pub struct StreamTopology { + /// Main stream name (e.g., "ethereum.events") + pub main: String, + /// Dead-letter stream name (e.g., "ethereum.events:dead") + pub dead: String, +} + +impl StreamTopology { + /// Explicit constructor for full control over stream names. + pub fn new(main: impl Into, dead: impl Into) -> Self { + Self { + main: main.into(), + dead: dead.into(), + } + } + + /// Derive topology from a base prefix. + /// Example: `"orders.events"` -> main=`"orders.events"`, dead=`"orders.events:dead"` + /// + /// # Example + /// ``` + /// use broker::redis::StreamTopology; + /// let topology = StreamTopology::from_prefix("ethereum.events"); + /// assert_eq!(topology.main, "ethereum.events"); + /// assert_eq!(topology.dead, "ethereum.events:dead"); + /// ``` + pub fn from_prefix(prefix: impl AsRef) -> Self { + let prefix = prefix.as_ref(); + Self::new(prefix, format!("{prefix}:dead")) + } +} + +/// Base Redis consumer configuration. +#[derive(Debug, Clone)] +pub struct RedisConsumerConfig { + /// Stream to consume from + pub stream: String, + /// Consumer group name + pub group_name: String, + /// Unique consumer name within the group + pub consumer_name: String, +} + +/// Consumer configuration with retry support. +#[derive(Debug, Clone)] +pub struct RedisRetryConfig { + /// Base consumer configuration + pub base: RedisConsumerConfig, + /// Dead-letter stream name + pub dead_stream: String, + /// Maximum retry attempts for permanent failures before dead-letter routing. + /// + /// `HandlerError::Transient` retries are infinite and do not consume this budget. + pub max_retries: u32, + /// Minimum idle time before claiming a pending message + pub claim_min_idle: Duration, + /// Interval between claim sweep cycles + pub claim_interval: Duration, + /// Optional circuit breaker configuration. + /// When set, the consumer pauses consumption on consecutive `Transient` handler errors, + /// preventing DLQ pollution during downstream outages (DB down, API timeout, etc.). + /// When `None`, all handler errors go through the normal ClaimSweeper/DLQ path. + pub circuit_breaker: Option, +} + +/// Consumer configuration with retry and prefetch support for high-throughput. +#[derive(Debug, Clone)] +pub struct RedisPrefetchConfig { + /// Retry configuration + pub retry: RedisRetryConfig, + /// Number of messages to prefetch per XREADGROUP call. + pub prefetch_count: usize, + /// Milliseconds to sleep between non-blocking XREADGROUP polls for new messages. + /// Lower values = lower latency but more Redis round-trips. + pub block_ms: usize, +} + +impl RetryPolicy for RedisRetryConfig { + fn max_retries(&self) -> u32 { + self.max_retries + } + + fn retry_delay(&self) -> Duration { + self.claim_min_idle + } +} + +impl RedisRetryConfig { + /// Redis hash key used to persist failure classification. + /// + /// Hash field = stream entry ID, value = "transient" | "permanent". + pub fn classification_marker_key(&self) -> String { + format!( + "mq:classification:{}:{}", + self.base.stream, self.base.group_name + ) + } + + /// Backward-compatible alias. + /// Prefer [`Self::classification_marker_key`]. + #[allow(dead_code)] + pub fn transient_marker_key(&self) -> String { + self.classification_marker_key() + } + + /// Backward-compatible alias. + /// Prefer [`Self::classification_marker_key`]. + #[allow(dead_code)] + pub fn permanent_marker_key(&self) -> String { + self.classification_marker_key() + } +} + +/// Builder for constructing Redis consumer configurations with validation. +#[must_use] +#[derive(Debug, Default)] +pub struct RedisConsumerConfigBuilder { + stream: Option, + group_name: Option, + consumer_name: Option, + dead_stream: Option, + max_retries: Option, + claim_min_idle: Option, + claim_interval: Option, + prefetch_count: Option, + block_ms: Option, + cb_failure_threshold: Option, + cb_cooldown_duration: Option, +} + +impl RedisConsumerConfigBuilder { + pub fn new() -> Self { + Self::default() + } + + pub fn stream(mut self, stream: impl Into) -> Self { + self.stream = Some(stream.into()); + self + } + + pub fn group_name(mut self, group_name: impl Into) -> Self { + self.group_name = Some(group_name.into()); + self + } + + pub fn consumer_name(mut self, consumer_name: impl Into) -> Self { + self.consumer_name = Some(consumer_name.into()); + self + } + + pub fn dead_stream(mut self, dead_stream: impl Into) -> Self { + self.dead_stream = Some(dead_stream.into()); + self + } + + /// Set the maximum retry attempts for permanent failures. + /// + /// This limit is only applied to `Execution`/`Deserialization` failures. + /// `Transient` failures always retry indefinitely. + pub fn max_retries(mut self, max_retries: u32) -> Self { + self.max_retries = Some(max_retries); + self + } + + pub fn claim_min_idle(mut self, claim_min_idle: Duration) -> Self { + self.claim_min_idle = Some(claim_min_idle); + self + } + + pub fn claim_interval(mut self, claim_interval: Duration) -> Self { + self.claim_interval = Some(claim_interval); + self + } + + pub fn prefetch_count(mut self, prefetch_count: usize) -> Self { + self.prefetch_count = Some(prefetch_count); + self + } + + pub fn block_ms(mut self, block_ms: usize) -> Self { + self.block_ms = Some(block_ms); + self + } + + /// Set the circuit breaker failure threshold (consecutive `Transient` errors to trip). + /// + /// When both `circuit_breaker_threshold` and `circuit_breaker_cooldown` are set, + /// the consumer will pause consumption after this many consecutive transient failures, + /// preventing DLQ pollution during downstream outages. + /// When not set, no circuit breaker is used (backward compatible). + pub fn circuit_breaker_threshold(mut self, threshold: u32) -> Self { + self.cb_failure_threshold = Some(threshold); + self + } + + /// Set the circuit breaker cooldown duration (how long to pause before probing). + /// + /// After the circuit trips, the consumer pauses for this duration, then + /// allows one test message through (half-open). If it succeeds, consumption + /// resumes. If it fails, the circuit reopens for another cooldown period. + pub fn circuit_breaker_cooldown(mut self, cooldown: Duration) -> Self { + self.cb_cooldown_duration = Some(cooldown); + self + } + + /// Apply stream topology to configure streams automatically. + pub fn with_topology(mut self, topology: &StreamTopology) -> Self { + self.stream = Some(topology.main.clone()); + self.dead_stream = Some(topology.dead.clone()); + self + } + + fn build_base(&self) -> Result { + Ok(RedisConsumerConfig { + stream: self + .stream + .clone() + .ok_or_else(|| RedisConsumerError::Configuration("stream is required".into()))?, + group_name: self.group_name.clone().ok_or_else(|| { + RedisConsumerError::Configuration("group_name is required".into()) + })?, + consumer_name: self.consumer_name.clone().ok_or_else(|| { + RedisConsumerError::Configuration("consumer_name is required".into()) + })?, + }) + } + + fn build_retry(&self) -> Result { + let base = self.build_base()?; + + // Build circuit breaker config if either threshold or cooldown is set + let circuit_breaker = match (self.cb_failure_threshold, self.cb_cooldown_duration) { + (Some(threshold), Some(cooldown)) => Some(CircuitBreakerConfig { + failure_threshold: threshold, + cooldown_duration: cooldown, + }), + (Some(threshold), None) => Some(CircuitBreakerConfig { + failure_threshold: threshold, + ..CircuitBreakerConfig::default() + }), + (None, Some(cooldown)) => Some(CircuitBreakerConfig { + cooldown_duration: cooldown, + ..CircuitBreakerConfig::default() + }), + (None, None) => None, + }; + + Ok(RedisRetryConfig { + base, + dead_stream: self.dead_stream.clone().ok_or_else(|| { + RedisConsumerError::Configuration("dead_stream is required".into()) + })?, + max_retries: self.max_retries.unwrap_or(3), + claim_min_idle: self.claim_min_idle.unwrap_or(Duration::from_secs(30)), + claim_interval: self.claim_interval.unwrap_or(Duration::from_secs(10)), + circuit_breaker, + }) + } + + /// Build a prefetch consumer configuration. + pub fn build_prefetch(self) -> Result { + let prefetch_count = self.prefetch_count.unwrap_or(10); + let block_ms = self.block_ms.unwrap_or(200); + let retry = self.build_retry()?; + + Ok(RedisPrefetchConfig { + retry, + prefetch_count, + block_ms, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_stream_topology_from_prefix() { + let topology = StreamTopology::from_prefix("ethereum.events"); + + assert_eq!(topology.main, "ethereum.events"); + assert_eq!(topology.dead, "ethereum.events:dead"); + } + + #[test] + fn test_consumer_config_builder_validation_fails() { + let result = RedisConsumerConfigBuilder::new() + .stream("ethereum.events") + .dead_stream("ethereum.events:dead") + // Missing group_name, consumer_name + .build_prefetch(); + + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!(matches!(err, RedisConsumerError::Configuration(_))); + } + + #[test] + fn test_prefetch_config_builder() { + let config = RedisConsumerConfigBuilder::new() + .stream("ethereum.events") + .group_name("notifier-group") + .consumer_name("pod-1") + .dead_stream("ethereum.events:dead") + .prefetch_count(50) + .block_ms(2000) + .build_prefetch() + .unwrap(); + + assert_eq!(config.prefetch_count, 50); + assert_eq!(config.block_ms, 2000); + assert_eq!(config.retry.base.stream, "ethereum.events"); + assert_eq!(config.retry.max_retries, 3); + assert_eq!(config.retry.claim_min_idle, Duration::from_secs(30)); + } + + #[test] + fn test_prefetch_config_defaults() { + let config = RedisConsumerConfigBuilder::new() + .stream("ethereum.events") + .group_name("notifier-group") + .consumer_name("pod-1") + .dead_stream("ethereum.events:dead") + .build_prefetch() + .unwrap(); + + assert_eq!(config.prefetch_count, 10); + assert_eq!(config.block_ms, 200); + } + + #[test] + fn test_builder_with_topology() { + let topology = StreamTopology::from_prefix("polygon.events"); + + let config = RedisConsumerConfigBuilder::new() + .with_topology(&topology) + .group_name("notifier-group") + .consumer_name("pod-1") + .build_prefetch() + .unwrap(); + + assert_eq!(config.retry.base.stream, "polygon.events"); + assert_eq!(config.retry.dead_stream, "polygon.events:dead"); + } +} diff --git a/listener/crates/shared/broker/src/redis/connection.rs b/listener/crates/shared/broker/src/redis/connection.rs new file mode 100644 index 0000000000..d2ab4167f9 --- /dev/null +++ b/listener/crates/shared/broker/src/redis/connection.rs @@ -0,0 +1,191 @@ +use arc_swap::ArcSwap; +use redis::aio::{ConnectionManager, ConnectionManagerConfig}; +use std::sync::Arc; +use std::time::Duration; +use tokio::time::sleep; +use tracing::{error, info, warn}; + +use super::error::RedisConsumerError; + +/// Per-command timeout applied to every `ConnectionManager`. +/// +/// Prevents indefinite hangs on half-open TCP sockets where the OS +/// TCP retransmission timeout (~75s macOS, ~15min Linux) is the only +/// backstop. Applied at the `ConnectionManager` level so it covers +/// **all** commands (XREADGROUP, XADD, XACK, HSET, …), not just the +/// consumer's read path. +const RESPONSE_TIMEOUT: Duration = Duration::from_secs(5); + +/// Timeout for the initial TCP handshake when creating a connection. +const CONNECTION_TIMEOUT: Duration = Duration::from_secs(5); + +/// Manages Redis connections with automatic reconnection. +/// +/// Wraps `redis::aio::ConnectionManager` which provides built-in +/// reconnection semantics. Each clone shares the same underlying +/// connection, and the connection is automatically re-established +/// if it drops. +/// +/// Additionally exposes `force_reconnect()` for callers that detect +/// dead sockets via external timeouts — `ConnectionManager` only +/// triggers internal reconnection for a subset of IO error kinds +/// (not `TimedOut`), so an external nudge is needed. +#[derive(Clone)] +pub struct RedisConnectionManager { + url: String, + inner: Arc>, + reconnect_delay: Duration, +} + +impl RedisConnectionManager { + /// Create a new RedisConnectionManager with the given URL. + pub async fn new(url: &str) -> Result { + let client = redis::Client::open(url).map_err(RedisConsumerError::Connection)?; + let manager = ConnectionManager::new_with_config(client, Self::cm_config()) + .await + .map_err(RedisConsumerError::Connection)?; + + info!("RedisConnectionManager: Connected to Redis at {}", url); + + Ok(Self { + url: url.to_string(), + inner: Arc::new(ArcSwap::from_pointee(manager)), + reconnect_delay: Duration::from_secs(5), + }) + } + + /// Create a new RedisConnectionManager with custom reconnect delay. + pub async fn with_reconnect_delay( + url: &str, + reconnect_delay: Duration, + ) -> Result { + let mut mgr = Self::new(url).await?; + mgr.reconnect_delay = reconnect_delay; + Ok(mgr) + } + + /// Get the Redis URL. + pub fn url(&self) -> &str { + &self.url + } + + /// Get a cloneable connection handle. + /// + /// `redis::aio::ConnectionManager` is cheaply cloneable and + /// automatically reconnects on failure, so this simply clones + /// the internal handle. + pub fn get_connection(&self) -> ConnectionManager { + let guard = self.inner.load(); + (**guard).clone() + } + + /// Replace the inner `ConnectionManager` with a fresh one. + /// + /// Called after detecting a dead socket via command timeout. + /// `ConnectionManager` only triggers its own reconnection for + /// specific IO error kinds (`BrokenPipe`, `ConnectionReset`, etc.) + /// but NOT for `TimedOut` — so we must create a new one. + pub async fn force_reconnect(&self) { + let client = match redis::Client::open(self.url.as_str()) { + Ok(c) => c, + Err(e) => { + warn!(error = %e, "force_reconnect: failed to open Redis client"); + return; + } + }; + + match ConnectionManager::new_with_config(client, Self::cm_config()).await { + Ok(new_manager) => { + self.inner.store(Arc::new(new_manager)); + info!(url = %self.url, "force_reconnect: replaced ConnectionManager with fresh connection"); + } + Err(e) => { + warn!(error = %e, "force_reconnect: failed to create new ConnectionManager, will retry on next command"); + } + } + } + + /// Perform a health check via PING command. + pub async fn health_check(&self) -> Result<(), RedisConsumerError> { + let mut conn = self.get_connection(); + redis::cmd("PING") + .query_async::(&mut conn) + .await + .map_err(RedisConsumerError::Connection)?; + Ok(()) + } + + /// Shared `ConnectionManagerConfig` used by both `new()` and `force_reconnect()`. + /// + /// `number_of_retries(0)` disables ConnectionManager's internal reconnection + /// retry loop. Without this, a broken connection triggers a shared future that + /// retries with exponential backoff (default 6 attempts × connection_timeout), + /// blocking ALL commands for 30+ seconds. We handle reconnection ourselves + /// via `force_reconnect()` which atomically swaps in a fresh ConnectionManager. + fn cm_config() -> ConnectionManagerConfig { + ConnectionManagerConfig::new() + .set_response_timeout(RESPONSE_TIMEOUT) + .set_connection_timeout(CONNECTION_TIMEOUT) + .set_number_of_retries(0) + } + + /// Create a connection with infinite retry on failure. + /// Use this when the consumer must eventually succeed. + pub async fn new_with_retry(url: &str) -> Self { + loop { + match Self::new(url).await { + Ok(mgr) => return mgr, + Err(e) => { + error!( + "Failed to connect to Redis at {}: {}. Retrying in 5s...", + url, e + ); + sleep(Duration::from_secs(5)).await; + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_connection_manager_url() { + // We can't easily test the async constructor without a Redis server, + // so we test the basic properties after construction in integration tests. + // This test verifies the module compiles correctly. + } + + #[tokio::test] + #[ignore] + async fn test_connection_manager_new() { + let manager = RedisConnectionManager::new("redis://localhost:6379") + .await + .unwrap(); + assert_eq!(manager.url(), "redis://localhost:6379"); + } + + #[tokio::test] + #[ignore] + async fn test_connection_manager_health_check() { + let manager = RedisConnectionManager::new("redis://localhost:6379") + .await + .unwrap(); + let result = manager.health_check().await; + assert!(result.is_ok()); + } + + #[tokio::test] + #[ignore] + async fn test_connection_manager_with_custom_delay() { + let manager = RedisConnectionManager::with_reconnect_delay( + "redis://localhost:6379", + Duration::from_secs(10), + ) + .await + .unwrap(); + assert_eq!(manager.reconnect_delay, Duration::from_secs(10)); + } +} diff --git a/listener/crates/shared/broker/src/redis/consumer.rs b/listener/crates/shared/broker/src/redis/consumer.rs new file mode 100644 index 0000000000..63d4bd98ce --- /dev/null +++ b/listener/crates/shared/broker/src/redis/consumer.rs @@ -0,0 +1,1470 @@ +use redis::Value; +use std::collections::{HashMap, HashSet}; +use std::future::Future; +use std::sync::{ + Arc, + atomic::{AtomicBool, Ordering}, +}; +use std::time::Duration; +use tokio::sync::{Mutex, mpsc}; +use tokio::time::{MissedTickBehavior, interval, sleep}; +use tokio_util::sync::CancellationToken; +use tracing::{debug, error, info, warn}; + +/// Backoff duration after a connection error before retrying. +/// Gives `ConnectionManager` time to auto-reconnect internally. +const CONN_RETRY_BACKOFF: Duration = Duration::from_secs(5); +const CLASS_TRANSIENT: &str = "transient"; +const CLASS_PERMANENT: &str = "permanent"; +/// Maximum retries for classification persistence before propagating the error. +const MAX_CLASSIFICATION_RETRIES: u32 = 10; + +use crate::traits::handler::{AckDecision, Handler, HandlerOutcome}; +use crate::traits::message::{Message, MessageMetadata}; + +use super::{ + circuit_breaker::CircuitBreaker, claim_task::ClaimSweeper, config::RedisPrefetchConfig, + connection::RedisConnectionManager, error::RedisConsumerError, +}; + +/// Result of processing a message in a worker task. +/// Used by `run()` to communicate results back to the main loop. +struct ProcessingResult { + /// The stream entry ID, needed for XACK + stream_id: String, + /// Outcome classification — preserves the Transient/Permanent distinction + /// so the main loop can call the correct circuit breaker method. + outcome: HandlerOutcome, + /// Original message payload — only populated when `outcome == Dead` so the + /// main loop can write the full message content to the dead stream. + /// `None` for all other outcomes to avoid unnecessary cloning. + payload: Option>, + /// Delivery count from the message metadata — needed for dead-letter routing + /// so the DLQ entry records how many times the message was delivered. + delivery_count: u64, +} + +/// Cancels a token when dropped. +/// +/// Used to guarantee background task shutdown even when a function returns +/// early due to an error. +struct CancelOnDrop { + token: CancellationToken, +} + +impl CancelOnDrop { + fn new(token: CancellationToken) -> Self { + Self { token } + } +} + +impl Drop for CancelOnDrop { + fn drop(&mut self) { + self.token.cancel(); + } +} + +/// Sends a stream ID to the abnormal-exit channel when dropped unless disarmed. +/// +/// Used by prefetch-safe worker tasks to guarantee in-flight cleanup even when +/// a worker panics before sending `ProcessingResult` to the main loop. +struct WorkerAbnormalExitGuard { + stream_id: Option, + tx: mpsc::UnboundedSender, +} + +impl WorkerAbnormalExitGuard { + fn new(stream_id: String, tx: mpsc::UnboundedSender) -> Self { + Self { + stream_id: Some(stream_id), + tx, + } + } + + /// Disable drop-side notification after successful result handoff. + fn disarm(&mut self) { + self.stream_id = None; + } +} + +impl Drop for WorkerAbnormalExitGuard { + fn drop(&mut self) { + if let Some(stream_id) = self.stream_id.take() { + let _ = self.tx.send(stream_id); + } + } +} + +/// Redis Streams consumer service. +/// +/// Provides three consumption strategies mirroring the RMQ consumer: +/// - `run_simple` — basic XREADGROUP with XACK on every message +/// - `run_with_retry` — XREADGROUP + ClaimSweeper for idle message recovery +/// - `run` — bounded mpsc + worker pool + ACK in main loop +pub struct RedisConsumer { + connection: Arc, + cancel_token: CancellationToken, +} + +impl RedisConsumer { + /// Create a new consumer with the given connection manager. + pub fn new(connection: Arc) -> Self { + Self { + connection, + cancel_token: CancellationToken::new(), + } + } + + /// Set a cancellation token for graceful shutdown. + /// + /// When cancelled, the consumer finishes its current batch, drains + /// in-flight results, and exits cleanly. + pub fn with_cancellation(mut self, token: CancellationToken) -> Self { + self.cancel_token = token; + self + } + + /// Create a new consumer from a Redis URL (one-liner convenience). + /// + /// Mirrors `RmqConsumer::with_addr()` — handles connection setup internally + /// so the developer doesn't need to manually create a `RedisConnectionManager` + /// and wrap it in `Arc`. + pub async fn with_url(url: &str) -> Result { + let conn = RedisConnectionManager::new_with_retry(url).await; + Ok(Self { + connection: Arc::new(conn), + cancel_token: CancellationToken::new(), + }) + } + + /// Get the underlying connection manager. + pub fn connection(&self) -> &Arc { + &self.connection + } + + // ───────────────────────────────────────────────────────────── + // Strategy 1: Simple consumer — no retry, XACK on every message + // ───────────────────────────────────────────────────────────── + + // ───────────────────────────────────────────────────────────── + // Strategy 3: Prefetch safe — bounded mpsc, worker pool, ACK in main loop + // ───────────────────────────────────────────────────────────── + + /// Run a high-throughput consumer with STRONG message loss guarantees. + /// + /// Same architecture as RMQ's `run`: + /// - Bounded `mpsc::channel` matching `prefetch_count` + /// - Workers receive `Message`, call handler, send `ProcessingResult` back + /// - Main loop: XACK on success, leave in PEL on failure + /// - ClaimSweeper runs in parallel + /// - Startup drain of own pending via `XREADGROUP ... 0` + pub async fn run( + &self, + config: RedisPrefetchConfig, + handler: impl Handler + 'static, + ) -> Result<(), RedisConsumerError> { + let handler: Arc = Arc::new(handler); + // Ensure consumer group exists + self.ensure_group(&config.retry.base.stream, &config.retry.base.group_name) + .await?; + let classification_marker_key = config.retry.classification_marker_key(); + + // Spawn ClaimSweeper + let cancel = CancellationToken::new(); + let classification_paused = Arc::new(AtomicBool::new(false)); + let sweeper = ClaimSweeper::new_with_classification_pause( + (*self.connection).clone(), + config.retry.clone(), + Arc::clone(&classification_paused), + ); + let cancel_clone = cancel.clone(); + tokio::spawn(async move { + sweeper.run(cancel_clone).await; + }); + let _sweeper_cancel_guard = CancelOnDrop::new(cancel.clone()); + + info!( + stream = %config.retry.base.stream, + group = %config.retry.base.group_name, + consumer = %config.retry.base.consumer_name, + prefetch_count = config.prefetch_count, + "Prefetch safe Redis consumer started (ACK in main loop)" + ); + + // Signal "connected" — flipped to 0 on connection errors below. + metrics::gauge!( + "broker_consumer_connected", + "backend" => "redis", + "topic" => config.retry.base.stream.clone(), + ) + .set(1.0); + + // Bounded channel — size matches prefetch_count for backpressure + let (result_tx, mut result_rx) = mpsc::channel::(config.prefetch_count); + // Worker abnormal-exit side channel (panic/send-failure before result handoff). + let (abnormal_exit_tx, mut abnormal_exit_rx) = mpsc::unbounded_channel::(); + // Guards against duplicate local dispatch of the same stream entry while + // the first worker is still in-flight (new-read path racing pending-drain path). + let in_flight = Arc::new(Mutex::new(HashSet::::new())); + + // Phase 1: drain own pending messages (single-threaded for ordering) + info!("Phase 1: draining own pending messages"); + loop { + if self.cancel_token.is_cancelled() { + info!("Cancellation requested during Phase 1, stopping"); + break; + } + let entries = match self + .xreadgroup( + &config.retry.base.stream, + &config.retry.base.group_name, + &config.retry.base.consumer_name, + config.prefetch_count, + "0", + ) + .await + { + Ok(entries) => entries, + Err(e) if e.is_connection_error() => { + warn!( + error = %e, + stream = %config.retry.base.stream, + "Connection error during Phase 1 pending drain, forcing reconnect..." + ); + self.connection.force_reconnect().await; + sleep(CONN_RETRY_BACKOFF).await; + continue; + } + Err(e) => return Err(e), + }; + + if entries.is_empty() { + info!("Phase 1 complete: no more pending messages"); + break; + } + + let delivery_counts = self + .get_pending_delivery_counts( + &config.retry.base.stream, + &config.retry.base.group_name, + &entries, + ) + .await; + + for (stream_id, data) in entries { + let count = delivery_counts.get(&stream_id).copied().unwrap_or(2); + let msg = build_message( + stream_id.clone(), + config.retry.base.stream.clone(), + count, + data, + ); + + let outcome = HandlerOutcome::from(handler.call(&msg).await); + match outcome { + HandlerOutcome::Ack => { + if let Err(e) = self + .xack( + &config.retry.base.stream, + &config.retry.base.group_name, + &stream_id, + ) + .await + { + if e.is_connection_error() { + warn!(error = %e, stream_id = %stream_id, "Connection error during XACK, message stays in PEL"); + } else { + return Err(e); + } + } else { + self.clear_failure_marker_safe( + &classification_marker_key, + &stream_id, + &classification_paused, + ) + .await?; + } + } + HandlerOutcome::Nack | HandlerOutcome::Delay(_) => { + // Voluntary yield — leave in PEL so ClaimSweeper retries. + debug!(stream_id = %stream_id, "Pending message voluntarily yielded, leaving in PEL"); + self.clear_failure_marker_safe( + &classification_marker_key, + &stream_id, + &classification_paused, + ) + .await?; + } + HandlerOutcome::Dead => { + warn!(stream_id = %stream_id, "Pending message requested dead-letter, routing immediately"); + self.xadd_dead_letter( + &config.retry, + &stream_id, + &msg.payload, + msg.metadata.delivery_count, + "AckDecision::Dead", + ) + .await?; + self.clear_failure_marker_safe( + &classification_marker_key, + &stream_id, + &classification_paused, + ) + .await?; + } + HandlerOutcome::Transient => { + warn!(stream_id = %stream_id, "Pending message transient failure, preserving infinite retry classification"); + self.mark_transient_safe( + &classification_marker_key, + &stream_id, + &classification_paused, + ) + .await?; + } + HandlerOutcome::Permanent => { + if count >= config.retry.max_retries as u64 { + warn!( + stream_id = %stream_id, + delivery_count = count, + max_retries = config.retry.max_retries, + "Permanent failure exhausted retry budget, routing to DLQ" + ); + self.xadd_dead_letter( + &config.retry, + &stream_id, + &msg.payload, + count, + "max_retries_exhausted", + ) + .await?; + self.clear_failure_marker_safe( + &classification_marker_key, + &stream_id, + &classification_paused, + ) + .await?; + } else { + warn!(stream_id = %stream_id, "Pending message permanent failure, leaving in PEL for ClaimSweeper"); + self.mark_permanent_safe( + &classification_marker_key, + &stream_id, + &classification_paused, + ) + .await?; + } + } + } + } + } + + // Phase 2: main loop with worker pool + // Circuit breaker state is preserved across connection errors (handled in-loop with backoff). + info!("Phase 2: consuming new messages with worker pool"); + let mut cb = config.retry.circuit_breaker.as_ref().map(|cfg| { + CircuitBreaker::new(cfg.clone()).with_labels("redis", config.retry.base.stream.clone()) + }); + + let mut consecutive_conn_errors: u32 = 0; + + // Periodic pending drain — ClaimSweeper XCLAIMs failed messages back to our PEL; + // XREADGROUP ">" never sees them. We poll "0" periodically so reclaimed entries + // are re-processed during steady-state (same pattern as run_with_retry). + let mut pending_check_interval = interval(Duration::from_secs(1)); + pending_check_interval.set_missed_tick_behavior(MissedTickBehavior::Skip); + + // New-message poll interval. Must use `interval` (not `sleep`) inside + // select! so the timer state persists when other branches fire first. + // With `sleep`, the timer resets on every select iteration, causing + // starvation when block_ms >= pending_check_interval period. + let mut new_msg_interval = interval(Duration::from_millis(config.block_ms as u64)); + new_msg_interval.set_missed_tick_behavior(MissedTickBehavior::Skip); + + let loop_result: Result<(), RedisConsumerError> = loop { + // Circuit breaker check: if open, pause consumption + if let Some(ref mut breaker) = cb + && !breaker.should_allow_request() + { + let cooldown = breaker.remaining_cooldown(); + info!( + cooldown = ?cooldown, + stream = %config.retry.base.stream, + "Circuit breaker OPEN — pausing consumption" + ); + sleep(cooldown).await; + continue; + } + + tokio::select! { + // Periodic drain of own PEL — reclaimed messages get handler retries + _ = pending_check_interval.tick() => { + if let Some(ref breaker) = cb + && breaker.is_open() + { + continue; + } + let pending = match self.xreadgroup( + &config.retry.base.stream, + &config.retry.base.group_name, + &config.retry.base.consumer_name, + config.prefetch_count, + "0", + ).await { + Ok(entries) => entries, + Err(e) if e.is_connection_error() => { + warn!( + error = %e, + stream = %config.retry.base.stream, + "Connection error during pending drain, forcing reconnect..." + ); + self.connection.force_reconnect().await; + continue; + } + Err(e) => break Err(e), + }; + if !pending.is_empty() { + let delivery_counts = self + .get_pending_delivery_counts( + &config.retry.base.stream, + &config.retry.base.group_name, + &pending, + ) + .await; + for (stream_id, data) in pending { + if let Some(ref breaker) = cb + && breaker.is_open() + { + break; + } + let should_dispatch = { + let mut set = in_flight.lock().await; + set.insert(stream_id.clone()) + }; + if !should_dispatch { + debug!(stream_id = %stream_id, "Skipping pending entry already in-flight"); + continue; + } + let count = delivery_counts.get(&stream_id).copied().unwrap_or(2); + let msg = build_message( + stream_id.clone(), + config.retry.base.stream.clone(), + count, + data, + ); + // Create guard before spawn so if the main loop panics between + // in-flight insert and spawn, cleanup is still signaled on drop. + let dispatch_guard = WorkerAbnormalExitGuard::new( + stream_id.clone(), + abnormal_exit_tx.clone(), + ); + let handler = Arc::clone(&handler); + let tx = result_tx.clone(); + tokio::spawn(async move { + let mut exit_guard = dispatch_guard; + let delivery_count = msg.metadata.delivery_count; + let handler_start = std::time::Instant::now(); + let call_result = handler.call(&msg).await; + metrics::histogram!("broker_handler_duration_seconds", + "backend" => "redis", + "topic" => msg.metadata.topic.clone(), + ).record(handler_start.elapsed().as_secs_f64()); + // Include payload for Dead and all error outcomes so the + // main loop can route to DLQ without re-reading the stream. + let needs_payload = matches!(&call_result, Ok(AckDecision::Dead) | Err(_)); + let payload = needs_payload.then(|| msg.payload.clone()); + let outcome = HandlerOutcome::from(call_result); + match tx + .send(ProcessingResult { + stream_id: stream_id.clone(), + outcome, + payload, + delivery_count, + }) + .await + { + Ok(()) => { + // Main loop now owns cleanup via result channel. + exit_guard.disarm(); + } + Err(e) => { + error!( + ?e, + stream_id = %stream_id, + "Failed to send pending processing result - message stays in PEL" + ); + // Keep guard armed: drop notifies abnormal-exit channel. + } + } + }); + } + } + } + + // Poll for new messages on a periodic interval. + // We do NOT use Redis BLOCK — it's incompatible with + // MultiplexedConnection (redis-rs #1236) and hangs + // indefinitely on dead sockets. + _ = new_msg_interval.tick() => { + match self.xreadgroup( + &config.retry.base.stream, + &config.retry.base.group_name, + &config.retry.base.consumer_name, + config.prefetch_count, + ">", + ).await { + Ok(entries) => { + consecutive_conn_errors = 0; + for (stream_id, data) in entries { + let should_dispatch = { + let mut set = in_flight.lock().await; + set.insert(stream_id.clone()) + }; + if !should_dispatch { + debug!(stream_id = %stream_id, "Skipping new entry already in-flight"); + continue; + } + let msg = build_message( + stream_id.clone(), + config.retry.base.stream.clone(), + 1, + data, + ); + let dispatch_guard = WorkerAbnormalExitGuard::new( + stream_id.clone(), + abnormal_exit_tx.clone(), + ); + + let handler = Arc::clone(&handler); + let tx = result_tx.clone(); + + tokio::spawn(async move { + let mut exit_guard = dispatch_guard; + let delivery_count = msg.metadata.delivery_count; + let handler_start = std::time::Instant::now(); + let call_result = handler.call(&msg).await; + metrics::histogram!("broker_handler_duration_seconds", + "backend" => "redis", + "topic" => msg.metadata.topic.clone(), + ).record(handler_start.elapsed().as_secs_f64()); + // Include payload for Dead and all error outcomes so the + // main loop can route to DLQ without re-reading the stream. + let needs_payload = matches!(&call_result, Ok(AckDecision::Dead) | Err(_)); + let payload = needs_payload.then(|| msg.payload.clone()); + let outcome = HandlerOutcome::from(call_result); + match tx + .send(ProcessingResult { + stream_id: stream_id.clone(), + outcome, + payload, + delivery_count, + }) + .await + { + Ok(()) => { + exit_guard.disarm(); + } + Err(e) => { + error!( + ?e, + stream_id = %stream_id, + "Failed to send processing result - message stays in PEL" + ); + } + } + }); + } + } + Err(e) if e.is_connection_error() => { + consecutive_conn_errors += 1; + warn!( + error = %e, + consecutive = consecutive_conn_errors, + stream = %config.retry.base.stream, + "Connection error in prefetch consumer, forcing reconnect..." + ); + metrics::counter!("broker_consumer_reconnections_total", + "backend" => "redis", + "topic" => config.retry.base.stream.clone(), + ).increment(1); + metrics::gauge!( + "broker_consumer_connected", + "backend" => "redis", + "topic" => config.retry.base.stream.clone(), + ) + .set(0.0); + self.connection.force_reconnect().await; + sleep(CONN_RETRY_BACKOFF).await; + metrics::gauge!( + "broker_consumer_connected", + "backend" => "redis", + "topic" => config.retry.base.stream.clone(), + ) + .set(1.0); + } + Err(e) => break Err(e), + } + } + + // Process completed results — ACK happens here (not in spawned tasks) + Some(result) = result_rx.recv() => { + { + let mut set = in_flight.lock().await; + set.remove(&result.stream_id); + } + + // ── metrics: outcome counter + delivery count ── + metrics::counter!("broker_messages_consumed_total", + "backend" => "redis", + "topic" => config.retry.base.stream.clone(), + "outcome" => crate::metrics::outcome_label(&result.outcome), + ).increment(1); + metrics::histogram!("broker_message_delivery_count", + "backend" => "redis", + "topic" => config.retry.base.stream.clone(), + ).record(result.delivery_count as f64); + + match result.outcome { + HandlerOutcome::Ack => { + if let Some(ref mut breaker) = cb { breaker.record_success(); } + debug!(stream_id = %result.stream_id, "Handler succeeded, acknowledging"); + if let Err(e) = self.xack( + &config.retry.base.stream, + &config.retry.base.group_name, + &result.stream_id, + ).await { + if e.is_connection_error() { + warn!(error = %e, stream_id = %result.stream_id, "Connection error during XACK, message stays in PEL"); + } else { + break Err(e); + } + } else if let Err(e) = self + .clear_failure_marker_safe( + &classification_marker_key, + &result.stream_id, + &classification_paused, + ) + .await + { + break Err(e); + } + } + HandlerOutcome::Nack | HandlerOutcome::Delay(_) => { + // Voluntary yield — leave in PEL so ClaimSweeper retries after claim_min_idle. + // Circuit breaker is unaffected (not an infra failure). + if let Some(ref mut breaker) = cb { breaker.record_success(); } + debug!(stream_id = %result.stream_id, "Handler voluntarily yielded, leaving in PEL for ClaimSweeper"); + if let Err(e) = self + .clear_failure_marker_safe( + &classification_marker_key, + &result.stream_id, + &classification_paused, + ) + .await + { + break Err(e); + } + } + HandlerOutcome::Dead => { + if let Some(ref mut breaker) = cb { breaker.record_permanent_failure(); } + metrics::counter!("broker_messages_dead_lettered_total", + "backend" => "redis", + "topic" => config.retry.base.stream.clone(), + "reason" => "handler_requested", + ).increment(1); + warn!(stream_id = %result.stream_id, "Handler requested dead-letter, routing immediately"); + let payload = result.payload.as_deref().unwrap_or(&[]); + if let Err(e) = self + .xadd_dead_letter( + &config.retry, + &result.stream_id, + payload, + result.delivery_count, + "AckDecision::Dead", + ) + .await + { + break Err(e); + } + if let Err(e) = self + .clear_failure_marker_safe( + &classification_marker_key, + &result.stream_id, + &classification_paused, + ) + .await + { + break Err(e); + } + } + HandlerOutcome::Transient => { + if let Some(ref mut breaker) = cb { breaker.record_transient_failure(); } + warn!(stream_id = %result.stream_id, "Transient failure, leaving in PEL"); + if let Err(e) = self + .mark_transient_safe( + &classification_marker_key, + &result.stream_id, + &classification_paused, + ) + .await + { + break Err(e); + } + } + HandlerOutcome::Permanent => { + if let Some(ref mut breaker) = cb { breaker.record_permanent_failure(); } + if result.delivery_count >= config.retry.max_retries as u64 { + // Retry budget exhausted — DLQ immediately instead of + // waiting for ClaimSweeper to win the idle-time race + // against the periodic pending drain. + metrics::counter!("broker_messages_dead_lettered_total", + "backend" => "redis", + "topic" => config.retry.base.stream.clone(), + "reason" => "max_retries_exhausted", + ).increment(1); + warn!( + stream_id = %result.stream_id, + delivery_count = result.delivery_count, + max_retries = config.retry.max_retries, + "Permanent failure exhausted retry budget, routing to DLQ" + ); + let payload = result.payload.as_deref().unwrap_or(&[]); + if let Err(e) = self + .xadd_dead_letter( + &config.retry, + &result.stream_id, + payload, + result.delivery_count, + "max_retries_exhausted", + ) + .await + { + break Err(e); + } + if let Err(e) = self + .clear_failure_marker_safe( + &classification_marker_key, + &result.stream_id, + &classification_paused, + ) + .await + { + break Err(e); + } + } else { + warn!(stream_id = %result.stream_id, "Handler failed, leaving in PEL for ClaimSweeper"); + if let Err(e) = self + .mark_permanent_safe( + &classification_marker_key, + &result.stream_id, + &classification_paused, + ) + .await + { + break Err(e); + } + } + } + } + } + + // Worker ended before delivering `ProcessingResult` (panic or send failure). + Some(stream_id) = abnormal_exit_rx.recv() => { + let removed = { + let mut set = in_flight.lock().await; + set.remove(&stream_id) + }; + if removed { + warn!( + stream_id = %stream_id, + "Worker exited before reporting result, cleared in-flight marker" + ); + } + } + + // Graceful shutdown — finish current batch, then drain. + _ = self.cancel_token.cancelled() => { + info!( + stream = %config.retry.base.stream, + "Cancellation requested, stopping consumer gracefully" + ); + break Ok(()); + } + } + }; + + // Shutdown drain path for fatal loop errors: + // - stop the claim sweeper + // - flush already-computed worker outcomes from the channel + // Classification persistence uses the same safe retry logic as steady-state + // so stale markers are not left behind on connection outages. + cancel.cancel(); + info!("Draining remaining results before shutdown"); + while let Ok(result) = result_rx.try_recv() { + { + let mut set = in_flight.lock().await; + set.remove(&result.stream_id); + } + match result.outcome { + HandlerOutcome::Ack => { + match self + .xack( + &config.retry.base.stream, + &config.retry.base.group_name, + &result.stream_id, + ) + .await + { + Err(e) => { + warn!(error = %e, stream_id = %result.stream_id, "Failed to XACK during shutdown drain, message stays in PEL"); + } + Ok(()) => { + if let Err(e) = self + .clear_failure_marker(&classification_marker_key, &result.stream_id) + .await + { + warn!(error = %e, stream_id = %result.stream_id, "Failed to clear failure markers during shutdown drain"); + } + } + } + } + HandlerOutcome::Nack | HandlerOutcome::Delay(_) => { + // leave in PEL — ClaimSweeper will retry + if let Err(e) = self + .clear_failure_marker(&classification_marker_key, &result.stream_id) + .await + { + warn!(error = %e, stream_id = %result.stream_id, "Failed to clear failure markers during shutdown drain"); + } + } + HandlerOutcome::Dead => { + let payload = result.payload.as_deref().unwrap_or(&[]); + match self + .xadd_dead_letter( + &config.retry, + &result.stream_id, + payload, + result.delivery_count, + "AckDecision::Dead", + ) + .await + { + Err(e) => { + warn!(error = %e, stream_id = %result.stream_id, "Failed to add to dead-letter during shutdown drain"); + } + Ok(()) => { + if let Err(e) = self + .clear_failure_marker(&classification_marker_key, &result.stream_id) + .await + { + warn!(error = %e, stream_id = %result.stream_id, "Failed to clear failure markers during shutdown drain"); + } + } + } + } + HandlerOutcome::Transient => { + // leave in PEL + if let Err(e) = self + .mark_transient(&classification_marker_key, &result.stream_id) + .await + { + warn!(error = %e, stream_id = %result.stream_id, "Failed to mark transient classification during shutdown drain"); + } + } + HandlerOutcome::Permanent => { + if result.delivery_count >= config.retry.max_retries as u64 { + let payload = result.payload.as_deref().unwrap_or(&[]); + match self + .xadd_dead_letter( + &config.retry, + &result.stream_id, + payload, + result.delivery_count, + "max_retries_exhausted", + ) + .await + { + Err(e) => { + warn!(error = %e, stream_id = %result.stream_id, "Failed to DLQ exhausted message during shutdown drain"); + } + Ok(()) => { + if let Err(e) = self + .clear_failure_marker( + &classification_marker_key, + &result.stream_id, + ) + .await + { + warn!(error = %e, stream_id = %result.stream_id, "Failed to clear failure markers during shutdown drain"); + } + } + } + } else if let Err(e) = self + .mark_permanent(&classification_marker_key, &result.stream_id) + .await + { + warn!(error = %e, stream_id = %result.stream_id, "Failed to mark permanent classification during shutdown drain"); + } + } + } + } + while let Ok(stream_id) = abnormal_exit_rx.try_recv() { + let mut set = in_flight.lock().await; + set.remove(&stream_id); + } + info!("Safe consumer shutdown complete"); + loop_result + } + + // ───────────────────────────────────────────────────────────── + // Private helpers + // ───────────────────────────────────────────────────────────── + + async fn classification_set( + &self, + marker_key: &str, + stream_id: &str, + class: &str, + ) -> Result<(), RedisConsumerError> { + let mut conn = self.connection.get_connection(); + let _: i64 = redis::cmd("HSET") + .arg(marker_key) + .arg(stream_id) + .arg(class) + .query_async(&mut conn) + .await + .map_err(|e| RedisConsumerError::TransientMarker { + key: marker_key.to_string(), + source: e, + })?; + Ok(()) + } + + async fn clear_failure_marker( + &self, + marker_key: &str, + stream_id: &str, + ) -> Result<(), RedisConsumerError> { + let mut conn = self.connection.get_connection(); + let _: i64 = redis::cmd("HDEL") + .arg(marker_key) + .arg(stream_id) + .query_async(&mut conn) + .await + .map_err(|e| RedisConsumerError::TransientMarker { + key: marker_key.to_string(), + source: e, + })?; + Ok(()) + } + + async fn clear_failure_marker_safe( + &self, + marker_key: &str, + stream_id: &str, + classification_paused: &AtomicBool, + ) -> Result<(), RedisConsumerError> { + self.classification_write_safe(stream_id, classification_paused, "clear", || { + self.clear_failure_marker(marker_key, stream_id) + }) + .await + } + + async fn mark_transient( + &self, + marker_key: &str, + stream_id: &str, + ) -> Result<(), RedisConsumerError> { + self.classification_set(marker_key, stream_id, CLASS_TRANSIENT) + .await + } + + async fn mark_transient_safe( + &self, + marker_key: &str, + stream_id: &str, + classification_paused: &AtomicBool, + ) -> Result<(), RedisConsumerError> { + self.classification_write_safe(stream_id, classification_paused, "mark transient", || { + self.mark_transient(marker_key, stream_id) + }) + .await + } + + async fn mark_permanent( + &self, + marker_key: &str, + stream_id: &str, + ) -> Result<(), RedisConsumerError> { + self.classification_set(marker_key, stream_id, CLASS_PERMANENT) + .await + } + + async fn mark_permanent_safe( + &self, + marker_key: &str, + stream_id: &str, + classification_paused: &AtomicBool, + ) -> Result<(), RedisConsumerError> { + self.classification_write_safe(stream_id, classification_paused, "mark permanent", || { + self.mark_permanent(marker_key, stream_id) + }) + .await + } + + async fn classification_write_safe( + &self, + stream_id: &str, + classification_paused: &AtomicBool, + action: &'static str, + mut op: F, + ) -> Result<(), RedisConsumerError> + where + F: FnMut() -> Fut, + Fut: Future>, + { + let mut attempts: u32 = 0; + loop { + match op().await { + Ok(()) => { + if classification_paused.swap(false, Ordering::Relaxed) { + info!("Classification persistence recovered, resuming ClaimSweeper"); + } + return Ok(()); + } + Err(e) if e.is_connection_error() => { + attempts += 1; + if attempts >= MAX_CLASSIFICATION_RETRIES { + warn!( + error = %e, + stream_id = %stream_id, + action = action, + attempts = attempts, + "Classification write exhausted retries, skipping — \ + ClaimSweeper defaults to transient (safe) when marker is missing" + ); + return Ok(()); + } + if !classification_paused.swap(true, Ordering::Relaxed) { + warn!( + error = %e, + stream_id = %stream_id, + action = action, + "Connection error while persisting message classification, pausing ClaimSweeper until Redis recovers" + ); + } else { + warn!( + error = %e, + stream_id = %stream_id, + action = action, + "Connection error while persisting message classification, retrying..." + ); + } + sleep(CONN_RETRY_BACKOFF).await; + } + Err(e) => return Err(e), + } + } + } + + /// Create consumer group (idempotent). + async fn ensure_group(&self, stream: &str, group: &str) -> Result<(), RedisConsumerError> { + let mut conn = self.connection.get_connection(); + + let result: Result = redis::cmd("XGROUP") + .arg("CREATE") + .arg(stream) + .arg(group) + .arg("0") + .arg("MKSTREAM") + .query_async(&mut conn) + .await; + + match result { + Ok(_) => { + info!(stream = %stream, group = %group, "Consumer group created"); + } + Err(e) if e.to_string().contains("BUSYGROUP") => { + debug!(stream = %stream, group = %group, "Consumer group already exists"); + } + Err(e) => { + return Err(RedisConsumerError::GroupCreation { + stream: stream.to_string(), + group: group.to_string(), + source: e, + }); + } + } + + Ok(()) + } + + /// XREADGROUP wrapper that returns parsed (stream_id, data) pairs. + /// + /// Non-blocking: does NOT use Redis BLOCK argument. BLOCK commands on + /// `ConnectionManager` (which wraps `MultiplexedConnection`) are + /// architecturally broken — a blocking call monopolizes the shared TCP + /// connection and hangs indefinitely on dead sockets (redis-rs #1236). + /// + /// Callers use client-side `sleep()` as the polling interval instead. + /// `response_timeout` (5s) on the `ConnectionManager` guarantees this + /// method returns within bounded time even on dead sockets. + /// + /// Connection recovery is the **caller's** responsibility: + /// callers must call `force_reconnect()` when they receive a connection + /// error, then retry. This mirrors the RMQ consumer's outer reconnection + /// loop pattern. + /// + /// `start_id` is `">"` for new messages or `"0"` for pending drain. + async fn xreadgroup( + &self, + stream: &str, + group: &str, + consumer: &str, + count: usize, + start_id: &str, + ) -> Result)>, RedisConsumerError> { + let mut conn = self.connection.get_connection(); + + let result: Value = redis::cmd("XREADGROUP") + .arg("GROUP") + .arg(group) + .arg(consumer) + .arg("COUNT") + .arg(count) + .arg("STREAMS") + .arg(stream) + .arg(start_id) + .query_async(&mut conn) + .await + .map_err(|e| RedisConsumerError::StreamRead { + stream: stream.to_string(), + source: e, + })?; + + Ok(parse_xreadgroup_response(result)) + } + + /// XACK wrapper. + async fn xack(&self, stream: &str, group: &str, id: &str) -> Result<(), RedisConsumerError> { + let mut conn = self.connection.get_connection(); + + let _: i64 = redis::cmd("XACK") + .arg(stream) + .arg(group) + .arg(id) + .query_async(&mut conn) + .await + .map_err(|e| RedisConsumerError::Acknowledge { + stream: stream.to_string(), + source: e, + })?; + + Ok(()) + } + + /// Query the real delivery count for a batch of pending entry IDs via `XPENDING`. + /// + /// Returns a map from stream ID to delivery count. On connection error, + /// returns an empty map (callers fall back to a default). + async fn get_pending_delivery_counts( + &self, + stream: &str, + group: &str, + ids: &[(String, Vec)], + ) -> HashMap { + if ids.is_empty() { + return HashMap::new(); + } + + let mut conn = self.connection.get_connection(); + + let first_id = &ids[0].0; + let last_id = &ids[ids.len() - 1].0; + + let result: Result = redis::cmd("XPENDING") + .arg(stream) + .arg(group) + .arg(first_id) + .arg(last_id) + .arg(ids.len()) + .query_async(&mut conn) + .await; + + match result { + Ok(Value::Array(items)) => { + let mut map = HashMap::with_capacity(items.len()); + for item in items { + if let Value::Array(fields) = item + && fields.len() >= 4 + { + let id = match &fields[0] { + Value::BulkString(b) => String::from_utf8_lossy(b).to_string(), + _ => continue, + }; + let count = match &fields[3] { + Value::Int(n) => *n as u64, + _ => continue, + }; + map.insert(id, count); + } + } + map + } + Ok(_) => HashMap::new(), + Err(e) => { + warn!( + error = %e, + stream = %stream, + "Failed to query delivery counts via XPENDING, falling back to defaults" + ); + HashMap::new() + } + } + } + + /// Route a message directly to the dead stream and XACK it from the main stream. + /// + /// Used when a handler returns [`AckDecision::Dead`] or when a permanent failure + /// exhausts its retry budget (`delivery_count >= max_retries`). The message is + /// published to `config.dead_stream` via `XADD` with the original payload, then + /// XACK'd so it is removed from the main stream's PEL immediately. + async fn xadd_dead_letter( + &self, + config: &super::config::RedisRetryConfig, + stream_id: &str, + payload: &[u8], + delivery_count: u64, + reason: &str, + ) -> Result<(), RedisConsumerError> { + let mut conn = self.connection.get_connection(); + + let _: String = redis::cmd("XADD") + .arg(&config.dead_stream) + .arg("*") + .arg("original_id") + .arg(stream_id) + .arg("original_stream") + .arg(&config.base.stream) + .arg("delivery_count") + .arg(delivery_count) + .arg("reason") + .arg(reason) + .arg("data") + .arg(payload) + .query_async(&mut conn) + .await + .map_err(|e| RedisConsumerError::DeadLetter { + stream: config.dead_stream.clone(), + source: e, + })?; + + // XACK from the main stream to remove from PEL + self.xack(&config.base.stream, &config.base.group_name, stream_id) + .await + } +} + +/// Build a `Message` from stream entry components. +fn build_message( + stream_id: String, + stream_name: String, + delivery_count: u64, + data: Vec, +) -> Message { + Message { + payload: data, + metadata: MessageMetadata { + id: stream_id, + topic: stream_name, + delivery_count, + headers: std::collections::HashMap::new(), + }, + } +} + +/// Parse the XREADGROUP response into (stream_id, data) pairs. +/// +/// Response format: +/// ```text +/// [[stream_name, [[id, [field, value, ...]], ...]], ...] +/// ``` +/// +/// We extract the "data" field from each entry. +fn parse_xreadgroup_response(value: Value) -> Vec<(String, Vec)> { + let mut results = Vec::new(); + + // Nil response means no messages (BLOCK timeout) + let streams = match value { + Value::Array(s) => s, + Value::Nil => return results, + _ => return results, + }; + + for stream in streams { + let parts = match stream { + Value::Array(p) if p.len() >= 2 => p, + _ => continue, + }; + + // parts[0] = stream name, parts[1] = entries array + let entries = match &parts[1] { + Value::Array(e) => e, + _ => continue, + }; + + for entry in entries { + let entry_parts = match entry { + Value::Array(ep) if ep.len() >= 2 => ep, + _ => continue, + }; + + // entry_parts[0] = stream ID + let stream_id = match &entry_parts[0] { + Value::BulkString(b) => String::from_utf8_lossy(b).to_string(), + _ => continue, + }; + + // entry_parts[1] = fields array [key, value, key, value, ...] + let fields = match &entry_parts[1] { + Value::Array(f) => f, + _ => continue, + }; + + // Find the "data" field + let mut data = Vec::new(); + let mut iter = fields.iter(); + while let Some(key) = iter.next() { + if let Value::BulkString(k) = key + && k == b"data" + && let Some(Value::BulkString(v)) = iter.next() + { + data = v.clone(); + break; + } + // Skip value + let _ = iter.next(); + } + + results.push((stream_id, data)); + } + } + + results +} + +#[async_trait::async_trait] +impl crate::traits::consumer::Consumer for RedisConsumer { + type PrefetchConfig = RedisPrefetchConfig; + type Error = RedisConsumerError; + + async fn connect(url: &str) -> Result { + Self::with_url(url).await + } + + async fn run( + &self, + config: Self::PrefetchConfig, + handler: impl Handler + 'static, + ) -> Result<(), Self::Error> { + self.run(config, handler).await + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_build_message() { + let msg = build_message( + "1234-0".to_string(), + "test.stream".to_string(), + 1, + b"hello".to_vec(), + ); + + assert_eq!(msg.payload, b"hello"); + assert_eq!(msg.metadata.id, "1234-0"); + assert_eq!(msg.metadata.topic, "test.stream"); + assert_eq!(msg.metadata.delivery_count, 1); + } + + #[test] + fn test_parse_xreadgroup_response_nil() { + let result = parse_xreadgroup_response(Value::Nil); + assert!(result.is_empty()); + } + + #[test] + fn test_parse_xreadgroup_response_empty() { + let result = parse_xreadgroup_response(Value::Array(vec![])); + assert!(result.is_empty()); + } + + #[test] + fn test_parse_xreadgroup_response_with_data() { + let value = Value::Array(vec![Value::Array(vec![ + Value::BulkString(b"test.stream".to_vec()), + Value::Array(vec![Value::Array(vec![ + Value::BulkString(b"1234-0".to_vec()), + Value::Array(vec![ + Value::BulkString(b"data".to_vec()), + Value::BulkString(b"{\"block\":42}".to_vec()), + ]), + ])]), + ])]); + + let result = parse_xreadgroup_response(value); + assert_eq!(result.len(), 1); + assert_eq!(result[0].0, "1234-0"); + assert_eq!(result[0].1, b"{\"block\":42}"); + } + + #[test] + fn test_parse_xreadgroup_response_multiple_entries() { + let value = Value::Array(vec![Value::Array(vec![ + Value::BulkString(b"test.stream".to_vec()), + Value::Array(vec![ + Value::Array(vec![ + Value::BulkString(b"1-0".to_vec()), + Value::Array(vec![ + Value::BulkString(b"data".to_vec()), + Value::BulkString(b"msg1".to_vec()), + ]), + ]), + Value::Array(vec![ + Value::BulkString(b"2-0".to_vec()), + Value::Array(vec![ + Value::BulkString(b"data".to_vec()), + Value::BulkString(b"msg2".to_vec()), + ]), + ]), + ]), + ])]); + + let result = parse_xreadgroup_response(value); + assert_eq!(result.len(), 2); + assert_eq!(result[0].0, "1-0"); + assert_eq!(result[1].0, "2-0"); + } + + #[test] + fn test_parse_xreadgroup_response_extra_fields() { + // Entry with extra fields besides "data" + let value = Value::Array(vec![Value::Array(vec![ + Value::BulkString(b"test.stream".to_vec()), + Value::Array(vec![Value::Array(vec![ + Value::BulkString(b"1-0".to_vec()), + Value::Array(vec![ + Value::BulkString(b"type".to_vec()), + Value::BulkString(b"block".to_vec()), + Value::BulkString(b"data".to_vec()), + Value::BulkString(b"payload".to_vec()), + ]), + ])]), + ])]); + + let result = parse_xreadgroup_response(value); + assert_eq!(result.len(), 1); + assert_eq!(result[0].1, b"payload"); + } +} diff --git a/listener/crates/shared/broker/src/redis/dead_letter.rs b/listener/crates/shared/broker/src/redis/dead_letter.rs new file mode 100644 index 0000000000..e8785c248d --- /dev/null +++ b/listener/crates/shared/broker/src/redis/dead_letter.rs @@ -0,0 +1,351 @@ +use redis::Value; +use tracing::{debug, info}; + +use super::{connection::RedisConnectionManager, error::RedisConsumerError}; + +/// A message in the dead-letter stream. +#[derive(Debug, Clone)] +pub struct DeadMessage { + /// The dead-letter stream entry ID + pub dead_id: String, + /// The original stream entry ID + pub original_id: String, + /// The original stream name + pub original_stream: String, + /// How many times the message was delivered before DLQ + pub delivery_count: u64, + /// The raw payload data + pub data: Vec, +} + +/// Processor for dead-letter stream operations. +/// +/// Provides list, replay, purge, and count operations on a dead-letter +/// stream, allowing operators to inspect and recover failed messages. +pub struct DeadLetterProcessor { + connection: RedisConnectionManager, +} + +impl DeadLetterProcessor { + pub fn new(connection: RedisConnectionManager) -> Self { + Self { connection } + } + + /// List dead-letter messages. + /// + /// Returns up to `count` messages from the dead-letter stream. + pub async fn list( + &self, + dead_stream: &str, + count: usize, + ) -> Result, RedisConsumerError> { + let mut conn = self.connection.get_connection(); + + let result: Value = redis::cmd("XRANGE") + .arg(dead_stream) + .arg("-") + .arg("+") + .arg("COUNT") + .arg(count) + .query_async(&mut conn) + .await + .map_err(|e| RedisConsumerError::DeadLetter { + stream: dead_stream.to_string(), + source: e, + })?; + + let messages = Self::parse_dead_messages(result); + debug!( + dead_stream = %dead_stream, + count = messages.len(), + "Listed dead-letter messages" + ); + + Ok(messages) + } + + /// Replay a dead-letter message back to the main stream. + /// + /// Reads the message from the dead-letter stream, publishes it + /// to the main stream via XADD, and deletes it from the dead-letter stream. + /// + /// Returns the new message ID in the main stream. + pub async fn replay( + &self, + dead_stream: &str, + main_stream: &str, + dead_msg_id: &str, + ) -> Result { + let mut conn = self.connection.get_connection(); + + // Read the specific message from dead-letter stream + let result: Value = redis::cmd("XRANGE") + .arg(dead_stream) + .arg(dead_msg_id) + .arg(dead_msg_id) + .query_async(&mut conn) + .await + .map_err(|e| RedisConsumerError::DeadLetter { + stream: dead_stream.to_string(), + source: e, + })?; + + let messages = Self::parse_dead_messages(result); + let msg = messages.into_iter().next().ok_or_else(|| { + RedisConsumerError::Configuration(format!( + "Dead-letter message not found: {} in {}", + dead_msg_id, dead_stream + )) + })?; + + // Publish to main stream + let new_id: String = redis::cmd("XADD") + .arg(main_stream) + .arg("*") + .arg("data") + .arg(&msg.data) + .query_async(&mut conn) + .await + .map_err(|e| RedisConsumerError::DeadLetter { + stream: main_stream.to_string(), + source: e, + })?; + + // Delete from dead-letter stream + let _: i64 = redis::cmd("XDEL") + .arg(dead_stream) + .arg(dead_msg_id) + .query_async(&mut conn) + .await + .map_err(|e| RedisConsumerError::DeadLetter { + stream: dead_stream.to_string(), + source: e, + })?; + + info!( + dead_stream = %dead_stream, + main_stream = %main_stream, + dead_msg_id = %dead_msg_id, + new_id = %new_id, + "Replayed dead-letter message" + ); + + Ok(new_id) + } + + /// Purge old dead-letter messages. + /// + /// Reads messages from the dead-letter stream and deletes those + /// whose stream ID timestamp is older than `older_than` duration. + /// Returns the number of purged messages. + pub async fn purge( + &self, + dead_stream: &str, + older_than: std::time::Duration, + ) -> Result { + let mut conn = self.connection.get_connection(); + + // Calculate the cutoff timestamp in milliseconds + let now_ms = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as u64; + let cutoff_ms = now_ms.saturating_sub(older_than.as_millis() as u64); + + // Read all entries up to the cutoff + let cutoff_id = format!("{}-0", cutoff_ms); + let result: Value = redis::cmd("XRANGE") + .arg(dead_stream) + .arg("-") + .arg(&cutoff_id) + .query_async(&mut conn) + .await + .map_err(|e| RedisConsumerError::DeadLetter { + stream: dead_stream.to_string(), + source: e, + })?; + + let ids = Self::extract_entry_ids(result); + if ids.is_empty() { + debug!(dead_stream = %dead_stream, "No messages to purge"); + return Ok(0); + } + + // Delete in batches + let mut deleted = 0usize; + for chunk in ids.chunks(100) { + let mut cmd = redis::cmd("XDEL"); + cmd.arg(dead_stream); + for id in chunk { + cmd.arg(id); + } + let count: i64 = + cmd.query_async(&mut conn) + .await + .map_err(|e| RedisConsumerError::DeadLetter { + stream: dead_stream.to_string(), + source: e, + })?; + deleted += count as usize; + } + + info!( + dead_stream = %dead_stream, + purged = deleted, + older_than = ?older_than, + "Purged dead-letter messages" + ); + + Ok(deleted) + } + + /// Get the count of messages in the dead-letter stream. + pub async fn count(&self, dead_stream: &str) -> Result { + let mut conn = self.connection.get_connection(); + + let len: u64 = redis::cmd("XLEN") + .arg(dead_stream) + .query_async(&mut conn) + .await + .map_err(|e| RedisConsumerError::DeadLetter { + stream: dead_stream.to_string(), + source: e, + })?; + + Ok(len) + } + + /// Parse XRANGE response into DeadMessage structs. + fn parse_dead_messages(value: Value) -> Vec { + let mut messages = Vec::new(); + + if let Value::Array(entries) = value { + for entry in entries { + if let Value::Array(parts) = entry { + if parts.len() < 2 { + continue; + } + + let dead_id = match &parts[0] { + Value::BulkString(b) => String::from_utf8_lossy(b).to_string(), + _ => continue, + }; + + if let Value::Array(fields) = &parts[1] { + let field_map = Self::fields_to_map(fields); + + let original_id = field_map.get("original_id").cloned().unwrap_or_default(); + let original_stream = field_map + .get("original_stream") + .cloned() + .unwrap_or_default(); + let delivery_count: u64 = field_map + .get("delivery_count") + .and_then(|s| s.parse().ok()) + .unwrap_or(0); + let data = field_map + .get("data") + .map(|s| s.as_bytes().to_vec()) + .unwrap_or_default(); + + messages.push(DeadMessage { + dead_id, + original_id, + original_stream, + delivery_count, + data, + }); + } + } + } + } + + messages + } + + /// Convert a flat field array [key, value, key, value, ...] to a map. + fn fields_to_map(fields: &[Value]) -> std::collections::HashMap { + let mut map = std::collections::HashMap::new(); + let mut iter = fields.iter(); + while let Some(key) = iter.next() { + if let (Value::BulkString(k), Some(Value::BulkString(v))) = (key, iter.next()) { + map.insert( + String::from_utf8_lossy(k).to_string(), + String::from_utf8_lossy(v).to_string(), + ); + } + } + map + } + + /// Extract entry IDs from XRANGE response. + fn extract_entry_ids(value: Value) -> Vec { + let mut ids = Vec::new(); + + if let Value::Array(entries) = value { + for entry in entries { + if let Value::Array(parts) = entry + && let Some(Value::BulkString(b)) = parts.first() + { + ids.push(String::from_utf8_lossy(b).to_string()); + } + } + } + + ids + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_dead_messages_empty() { + let value = Value::Array(vec![]); + let messages = DeadLetterProcessor::parse_dead_messages(value); + assert!(messages.is_empty()); + } + + #[test] + fn test_parse_dead_messages_with_data() { + let value = Value::Array(vec![Value::Array(vec![ + Value::BulkString(b"9999-0".to_vec()), + Value::Array(vec![ + Value::BulkString(b"original_id".to_vec()), + Value::BulkString(b"1234-0".to_vec()), + Value::BulkString(b"original_stream".to_vec()), + Value::BulkString(b"ethereum.events".to_vec()), + Value::BulkString(b"delivery_count".to_vec()), + Value::BulkString(b"3".to_vec()), + Value::BulkString(b"data".to_vec()), + Value::BulkString(b"{\"block\":42}".to_vec()), + ]), + ])]); + + let messages = DeadLetterProcessor::parse_dead_messages(value); + assert_eq!(messages.len(), 1); + assert_eq!(messages[0].dead_id, "9999-0"); + assert_eq!(messages[0].original_id, "1234-0"); + assert_eq!(messages[0].original_stream, "ethereum.events"); + assert_eq!(messages[0].delivery_count, 3); + assert_eq!(messages[0].data, b"{\"block\":42}"); + } + + #[test] + fn test_extract_entry_ids() { + let value = Value::Array(vec![ + Value::Array(vec![ + Value::BulkString(b"1-0".to_vec()), + Value::Array(vec![]), + ]), + Value::Array(vec![ + Value::BulkString(b"2-0".to_vec()), + Value::Array(vec![]), + ]), + ]); + + let ids = DeadLetterProcessor::extract_entry_ids(value); + assert_eq!(ids, vec!["1-0", "2-0"]); + } +} diff --git a/listener/crates/shared/broker/src/redis/depth.rs b/listener/crates/shared/broker/src/redis/depth.rs new file mode 100644 index 0000000000..04ae250b0d --- /dev/null +++ b/listener/crates/shared/broker/src/redis/depth.rs @@ -0,0 +1,366 @@ +//! Redis Streams depth introspection via XLEN and XINFO GROUPS. +//! +//! When a consumer `group` is specified, also queries `XINFO GROUPS` to +//! populate `pending` (PEL count) and `lag` (undelivered entries, Redis 7.0+). + +use crate::traits::depth::{QueueDepths, QueueInspector}; +use async_trait::async_trait; +use redis::{AsyncCommands, Value}; +use tracing::debug; + +use super::connection::RedisConnectionManager; +use super::error::RedisConsumerError; + +/// Inspects Redis Streams depth using XLEN and XINFO GROUPS. +/// +/// Derives stream names from the logical name: +/// - principal: `{name}` +/// - dead-letter: `{name}:dead` +/// - retry: `None` (Redis retry is PEL-based, not a separate stream) +/// +/// When a consumer group is specified, also queries: +/// - `pending`: PEL count from `XINFO GROUPS` +/// - `lag`: undelivered entries from `XINFO GROUPS` (Redis 7.0+) +pub struct RedisQueueInspector { + connection: RedisConnectionManager, +} + +impl RedisQueueInspector { + /// Create a new inspector from an existing connection manager. + pub fn new(connection: RedisConnectionManager) -> Self { + Self { connection } + } + + /// Query the length of a single stream, returning 0 if the stream does not exist. + /// + /// Redis XLEN natively returns 0 for non-existent keys, so errors are + /// propagated directly — no special-casing needed. + async fn stream_len(&self, stream: &str) -> Result { + let mut conn = self.connection.get_connection(); + let len: u64 = conn.xlen(stream).await?; + Ok(len) + } + + /// Query `XINFO GROUPS {stream}` and extract `pending` and `lag` for + /// the specified consumer group. + /// + /// Returns `(pending, lag)` where: + /// - `pending` = PEL count (always available when the group exists) + /// - `lag` = undelivered entries (Redis 7.0+; `None` on older versions) + /// + /// If the stream or group does not exist, returns `(Some(0), Some(0))`. + async fn group_info( + &self, + stream: &str, + group: &str, + ) -> Result<(Option, Option), RedisConsumerError> { + let mut conn = self.connection.get_connection(); + + let result: Result = redis::cmd("XINFO") + .arg("GROUPS") + .arg(stream) + .query_async(&mut conn) + .await; + + let value = match result { + Ok(v) => v, + Err(e) => { + // ERR no such key — stream doesn't exist → no work. + if e.kind() == redis::ErrorKind::ResponseError { + debug!( + stream = stream, + group = group, + "Stream does not exist, reporting pending=0 lag=0" + ); + return Ok((Some(0), Some(0))); + } + return Err(RedisConsumerError::StreamRead { + stream: stream.to_string(), + source: e, + }); + } + }; + + let info = parse_group_depth_info(&value, group); + match info { + Some((pending, lag)) => Ok((Some(pending), lag)), + // Group not found in XINFO response → no PEL, no lag. + None => { + debug!( + stream = stream, + group = group, + "Consumer group not found, reporting pending=0 lag=0" + ); + Ok((Some(0), Some(0))) + } + } + } +} + +#[async_trait] +impl QueueInspector for RedisQueueInspector { + type Error = RedisConsumerError; + + async fn queue_depths( + &self, + name: &str, + group: Option<&str>, + ) -> Result { + let dead_stream = format!("{name}:dead"); + + let (principal, dead_letter) = + tokio::try_join!(self.stream_len(name), self.stream_len(&dead_stream))?; + + let (pending, lag) = match group { + Some(g) => self.group_info(name, g).await?, + None => (None, None), + }; + + Ok(QueueDepths { + principal, + retry: None, // Redis retry is PEL-based, no separate stream + dead_letter, + pending, + lag, + }) + } + + async fn exists(&self, name: &str) -> Result { + let mut conn = self.connection.get_connection(); + let key_type: String = redis::cmd("TYPE") + .arg(name) + .query_async(&mut conn) + .await + .map_err(|e| RedisConsumerError::StreamRead { + stream: name.to_string(), + source: e, + })?; + Ok(key_type == "stream") + } + + /// Single `XINFO GROUPS` round-trip: returns `true` when the consumer + /// group has zero pending (PEL) and zero lag (undelivered) entries. + async fn is_empty(&self, name: &str, group: &str) -> Result { + let (pending, lag) = self.group_info(name, group).await?; + + let pending_zero = pending.unwrap_or(0) == 0; + let lag_zero = lag.unwrap_or(0) == 0; + + Ok(pending_zero && lag_zero) + } + + /// Single `XINFO GROUPS` round-trip: returns `true` when the consumer group + /// has at most one pending (PEL) entry and zero lag (undelivered) entries. + /// + /// One pending message means either an in-flight delivery or a stuck entry + /// from a transient error — in both cases the consumer is already handling it. + async fn is_empty_or_pending(&self, name: &str, group: &str) -> Result { + let (pending, lag) = self.group_info(name, group).await?; + + let pending_zero = pending.unwrap_or(0) == 0 || pending.unwrap_or(0) == 1; + let lag_zero = lag.unwrap_or(0) == 0; + + Ok(pending_zero && lag_zero) + } +} + +/// Parse the `XINFO GROUPS` response to extract `pending` and `lag` for a +/// specific group. +/// +/// The response is an array of groups, each represented as a flat +/// `[key, value, key, value, ...]` array. We scan for the group matching +/// `target_group` and extract: +/// - `pending` (integer, always present) +/// - `lag` (integer, Redis 7.0+ only; `None` on older versions) +/// +/// Returns `None` if the target group is not found. +fn parse_group_depth_info(value: &Value, target_group: &str) -> Option<(u64, Option)> { + let groups = match value { + Value::Array(items) => items, + _ => return None, + }; + + for group_value in groups { + let fields = match group_value { + Value::Array(f) => f, + _ => continue, + }; + + let map = flat_array_to_map(fields); + + let name = map.get("name")?; + if name != target_group { + continue; + } + + let pending = map + .get("pending") + .and_then(|v| v.parse::().ok()) + .unwrap_or(0); + + // `lag` is only available on Redis 7.0+. + let lag = map.get("lag").and_then(|v| v.parse::().ok()); + + return Some((pending, lag)); + } + + None +} + +/// Convert a flat Redis field array `[key, value, key, value, ...]` to a +/// `HashMap`. +/// +/// Same pattern used by `StreamTrimmer::flat_array_to_map`. +fn flat_array_to_map(fields: &[Value]) -> std::collections::HashMap { + let mut map = std::collections::HashMap::new(); + let mut iter = fields.iter(); + while let Some(key) = iter.next() { + if let Value::BulkString(k) = key { + if let Some(val) = iter.next() { + let v = match val { + Value::BulkString(b) => String::from_utf8_lossy(b).to_string(), + Value::Int(n) => n.to_string(), + _ => continue, + }; + map.insert(String::from_utf8_lossy(k).to_string(), v); + } + } else { + // Skip non-bulk-string keys (shouldn't happen in XINFO output). + let _ = iter.next(); + } + } + map +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn redis_queue_inspector_dead_stream_naming() { + let name = "ethereum.blocks"; + let dead = format!("{name}:dead"); + assert_eq!(dead, "ethereum.blocks:dead"); + } + + #[test] + fn parse_group_info_extracts_pending_and_lag() { + // Simulate XINFO GROUPS response (Redis 7.0+): + // [ [name, "mygroup", consumers, 1, pending, 5, last-delivered-id, "1-0", entries-read, 10, lag, 3] ] + let response = Value::Array(vec![Value::Array(vec![ + Value::BulkString(b"name".to_vec()), + Value::BulkString(b"mygroup".to_vec()), + Value::BulkString(b"consumers".to_vec()), + Value::Int(1), + Value::BulkString(b"pending".to_vec()), + Value::Int(5), + Value::BulkString(b"last-delivered-id".to_vec()), + Value::BulkString(b"1-0".to_vec()), + Value::BulkString(b"entries-read".to_vec()), + Value::Int(10), + Value::BulkString(b"lag".to_vec()), + Value::Int(3), + ])]); + + let result = parse_group_depth_info(&response, "mygroup"); + assert_eq!(result, Some((5, Some(3)))); + } + + #[test] + fn parse_group_info_without_lag_redis_6() { + // Simulate XINFO GROUPS response (Redis < 7.0, no lag/entries-read): + let response = Value::Array(vec![Value::Array(vec![ + Value::BulkString(b"name".to_vec()), + Value::BulkString(b"mygroup".to_vec()), + Value::BulkString(b"consumers".to_vec()), + Value::Int(1), + Value::BulkString(b"pending".to_vec()), + Value::Int(2), + Value::BulkString(b"last-delivered-id".to_vec()), + Value::BulkString(b"100-0".to_vec()), + ])]); + + let result = parse_group_depth_info(&response, "mygroup"); + assert_eq!(result, Some((2, None))); + } + + #[test] + fn parse_group_info_group_not_found() { + let response = Value::Array(vec![Value::Array(vec![ + Value::BulkString(b"name".to_vec()), + Value::BulkString(b"other-group".to_vec()), + Value::BulkString(b"pending".to_vec()), + Value::Int(0), + ])]); + + let result = parse_group_depth_info(&response, "mygroup"); + assert_eq!(result, None); + } + + #[test] + fn parse_group_info_empty_response() { + let response = Value::Array(vec![]); + let result = parse_group_depth_info(&response, "mygroup"); + assert_eq!(result, None); + } + + /// Validates the `is_empty_or_pending` logic: pending=1, lag=0 should + /// be treated as caught-up (the single entry is already in-flight). + #[test] + fn is_empty_or_pending_allows_single_pending() { + let response = Value::Array(vec![Value::Array(vec![ + Value::BulkString(b"name".to_vec()), + Value::BulkString(b"mygroup".to_vec()), + Value::BulkString(b"consumers".to_vec()), + Value::Int(1), + Value::BulkString(b"pending".to_vec()), + Value::Int(1), + Value::BulkString(b"last-delivered-id".to_vec()), + Value::BulkString(b"42-0".to_vec()), + Value::BulkString(b"entries-read".to_vec()), + Value::Int(42), + Value::BulkString(b"lag".to_vec()), + Value::Int(0), + ])]); + + let (pending, lag) = parse_group_depth_info(&response, "mygroup").unwrap(); + + // Mirror the logic from is_empty_or_pending + let pending_ok = pending == 0 || pending == 1; + let lag_ok = lag.unwrap_or(0) == 0; + + assert!( + pending_ok && lag_ok, + "pending=1 lag=0 must be treated as caught-up" + ); + } + + /// pending=2 with lag=0 should NOT be treated as caught-up. + #[test] + fn is_empty_or_pending_rejects_multiple_pending() { + let response = Value::Array(vec![Value::Array(vec![ + Value::BulkString(b"name".to_vec()), + Value::BulkString(b"mygroup".to_vec()), + Value::BulkString(b"consumers".to_vec()), + Value::Int(1), + Value::BulkString(b"pending".to_vec()), + Value::Int(2), + Value::BulkString(b"last-delivered-id".to_vec()), + Value::BulkString(b"42-0".to_vec()), + Value::BulkString(b"entries-read".to_vec()), + Value::Int(42), + Value::BulkString(b"lag".to_vec()), + Value::Int(0), + ])]); + + let (pending, lag) = parse_group_depth_info(&response, "mygroup").unwrap(); + + let pending_ok = pending == 0 || pending == 1; + let lag_ok = lag.unwrap_or(0) == 0; + + assert!( + !(pending_ok && lag_ok), + "pending=2 must NOT be treated as caught-up" + ); + } +} diff --git a/listener/crates/shared/broker/src/redis/error.rs b/listener/crates/shared/broker/src/redis/error.rs new file mode 100644 index 0000000000..1fb6500897 --- /dev/null +++ b/listener/crates/shared/broker/src/redis/error.rs @@ -0,0 +1,102 @@ +use thiserror::Error; + +#[derive(Error, Debug)] +#[non_exhaustive] +pub enum RedisConsumerError { + #[error("connection failed: {0}")] + Connection(#[from] redis::RedisError), + + #[error("consumer group creation failed for stream '{stream}' group '{group}': {source}")] + GroupCreation { + stream: String, + group: String, + #[source] + source: redis::RedisError, + }, + + #[error("stream read failed for '{stream}': {source}")] + StreamRead { + stream: String, + #[source] + source: redis::RedisError, + }, + + #[error("acknowledge failed for stream '{stream}': {source}")] + Acknowledge { + stream: String, + #[source] + source: redis::RedisError, + }, + + #[error("claim failed for stream '{stream}': {source}")] + Claim { + stream: String, + #[source] + source: redis::RedisError, + }, + + #[error("dead letter operation failed for stream '{stream}': {source}")] + DeadLetter { + stream: String, + #[source] + source: redis::RedisError, + }, + + #[error("transient marker operation failed for key '{key}': {source}")] + TransientMarker { + key: String, + #[source] + source: redis::RedisError, + }, + + #[error("consumer stream ended unexpectedly")] + StreamEnded, + + #[error("configuration error: {0}")] + Configuration(String), +} + +impl RedisConsumerError { + /// Returns `true` if the error is a transient connection issue that + /// `ConnectionManager` can recover from on the next command. + /// + /// Consumer loops use this to decide between: + /// - **connection error** → log, backoff, continue loop (ConnectionManager auto-reconnects) + /// - **other error** → propagate up + pub fn is_connection_error(&self) -> bool { + let redis_err = match self { + Self::Connection(e) => Some(e), + Self::StreamRead { source, .. } => Some(source), + Self::Acknowledge { source, .. } => Some(source), + Self::GroupCreation { source, .. } => Some(source), + Self::Claim { source, .. } => Some(source), + Self::DeadLetter { source, .. } => Some(source), + Self::TransientMarker { source, .. } => Some(source), + Self::StreamEnded | Self::Configuration(_) => None, + }; + + redis_err.is_some_and(|e| { + e.is_io_error() || e.is_connection_dropped() || e.is_connection_refusal() + }) + } +} + +#[derive(Error, Debug)] +#[non_exhaustive] +pub enum RedisPublisherError { + #[error("serialization failed: {0}")] + Serialization(#[from] serde_json::Error), + + #[error("connection failed: {0}")] + Connection(#[from] redis::RedisError), + + #[error("publish failed for stream '{stream}': {source}")] + Publish { + stream: String, + #[source] + source: redis::RedisError, + }, + + #[error("publish failed after {retries} retries (stream: {stream})")] + RetriesExhausted { retries: u32, stream: String }, +} diff --git a/listener/crates/shared/broker/src/redis/handler.rs b/listener/crates/shared/broker/src/redis/handler.rs new file mode 100644 index 0000000000..96f7f42256 --- /dev/null +++ b/listener/crates/shared/broker/src/redis/handler.rs @@ -0,0 +1,254 @@ +pub use crate::traits::handler::{ + AckDecision, AsyncHandlerPayloadOnly, AsyncHandlerWithContext, Handler, HandlerError, +}; +pub use crate::traits::message::{Message, MessageMetadata}; + +// Backward-compatible aliases for existing redis_broker callers +#[allow(dead_code)] +pub type RedisHandler = dyn Handler; +#[allow(dead_code)] +pub type RedisHandlerError = HandlerError; + +/// Backward-compatible alias — use `AsyncHandlerPayloadOnly` directly for new code. +pub type AsyncRedisHandlerPayloadOnly = AsyncHandlerPayloadOnly; + +/// Backward-compatible alias — use `AsyncHandlerWithContext` directly for new code. +/// The handler receives `MessageMetadata` instead of the old `RedisMessageContext`. +pub type AsyncRedisHandlerWithArgs = AsyncHandlerWithContext; + +/// Backward-compatible wrapper that ignores both payload and receives context. +/// +/// For new code, prefer `AsyncHandlerPayloadOnly` with an empty-payload handler. +#[derive(Clone)] +pub struct AsyncRedisHandlerNoArgs { + f: F, + _phantom: std::marker::PhantomData, +} + +impl AsyncRedisHandlerNoArgs { + pub fn new(f: F) -> Self { + Self { + f, + _phantom: std::marker::PhantomData, + } + } +} + +#[async_trait::async_trait] +impl Handler for AsyncRedisHandlerNoArgs +where + F: Fn(MessageMetadata) -> Fut + Send + Sync + 'static, + Fut: std::future::Future> + Send, + E: std::error::Error + Send + Sync + 'static, +{ + async fn call(&self, msg: &Message) -> Result { + (self.f)(msg.metadata.clone()) + .await + .map(|_| AckDecision::Ack) + .map_err(|e| HandlerError::Execution(Box::new(e))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::error::Error; + use thiserror::Error as ThisError; + + #[derive(ThisError, Debug)] + #[error("Mock error: {0}")] + struct MockError(String); + + #[derive(serde::Deserialize, Debug, PartialEq)] + struct TestPayload { + value: i32, + } + + fn make_message(data: &[u8]) -> Message { + Message { + payload: data.to_vec(), + metadata: MessageMetadata::new("1234567890-0", "test.stream", 1), + } + } + + #[tokio::test] + async fn async_handler_with_args_success() { + let handler = AsyncRedisHandlerWithArgs::new( + |payload: TestPayload, _ctx: MessageMetadata| async move { + assert_eq!(payload.value, 42); + Ok::<(), MockError>(()) + }, + ); + + let msg = make_message(br#"{"value": 42}"#); + let result = handler.call(&msg).await; + assert!(matches!(result, Ok(AckDecision::Ack))); + } + + #[tokio::test] + async fn async_handler_with_args_receives_context() { + let handler = AsyncRedisHandlerWithArgs::new( + |_payload: TestPayload, ctx: MessageMetadata| async move { + assert_eq!(ctx.id, "1234567890-0"); + assert_eq!(ctx.topic, "test.stream"); + assert_eq!(ctx.delivery_count, 1); + Ok::<(), MockError>(()) + }, + ); + + let msg = make_message(br#"{"value": 42}"#); + let result = handler.call(&msg).await; + assert!(matches!(result, Ok(AckDecision::Ack))); + } + + #[tokio::test] + async fn async_handler_with_args_deserialization_error() { + let handler = AsyncRedisHandlerWithArgs::new( + |_payload: TestPayload, _ctx: MessageMetadata| async move { Ok::<(), MockError>(()) }, + ); + + let msg = make_message(br#"{"invalid": "json"}"#); + let result = handler.call(&msg).await; + + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!(matches!(err, HandlerError::Deserialization(_))); + assert!(err.to_string().contains("deserialization failed")); + } + + #[tokio::test] + async fn async_handler_with_args_execution_error() { + let handler = AsyncRedisHandlerWithArgs::new( + |_payload: TestPayload, _ctx: MessageMetadata| async move { + Err::<(), MockError>(MockError("handler failed".to_string())) + }, + ); + + let msg = make_message(br#"{"value": 42}"#); + let result = handler.call(&msg).await; + + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!(matches!(err, HandlerError::Execution(_))); + assert!(err.to_string().contains("handler execution failed")); + + let source = err.source(); + assert!(source.is_some()); + assert!(source.unwrap().to_string().contains("handler failed")); + } + + #[tokio::test] + async fn async_handler_no_args_success() { + let handler = + AsyncRedisHandlerNoArgs::new( + |_ctx: MessageMetadata| async move { Ok::<(), MockError>(()) }, + ); + + let msg = make_message(&[]); + let result = handler.call(&msg).await; + assert!(matches!(result, Ok(AckDecision::Ack))); + } + + #[tokio::test] + async fn async_handler_no_args_receives_context() { + let handler = AsyncRedisHandlerNoArgs::new(|ctx: MessageMetadata| async move { + assert_eq!(ctx.id, "1234567890-0"); + assert_eq!(ctx.topic, "test.stream"); + Ok::<(), MockError>(()) + }); + + let msg = make_message(&[]); + let result = handler.call(&msg).await; + assert!(matches!(result, Ok(AckDecision::Ack))); + } + + #[tokio::test] + async fn async_handler_no_args_execution_error() { + let handler = AsyncRedisHandlerNoArgs::new(|_ctx: MessageMetadata| async move { + Err::<(), MockError>(MockError("no args handler failed".to_string())) + }); + + let msg = make_message(&[]); + let result = handler.call(&msg).await; + + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!(matches!(err, HandlerError::Execution(_))); + + let source = err.source(); + assert!(source.is_some()); + assert!( + source + .unwrap() + .to_string() + .contains("no args handler failed") + ); + } + + #[tokio::test] + async fn payload_only_handler_success() { + let handler = AsyncRedisHandlerPayloadOnly::new(|payload: TestPayload| async move { + assert_eq!(payload.value, 42); + Ok::<(), MockError>(()) + }); + + let msg = make_message(br#"{"value": 42}"#); + let result = handler.call(&msg).await; + assert!(matches!(result, Ok(AckDecision::Ack))); + } + + #[tokio::test] + async fn payload_only_handler_deserialization_error() { + let handler = AsyncRedisHandlerPayloadOnly::new(|_payload: TestPayload| async move { + Ok::<(), MockError>(()) + }); + + let msg = make_message(br#"{"invalid": "json"}"#); + let result = handler.call(&msg).await; + + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!(matches!(err, HandlerError::Deserialization(_))); + } + + #[tokio::test] + async fn payload_only_handler_execution_error() { + let handler = AsyncRedisHandlerPayloadOnly::new(|_payload: TestPayload| async move { + Err::<(), MockError>(MockError("payload only failed".to_string())) + }); + + let msg = make_message(br#"{"value": 42}"#); + let result = handler.call(&msg).await; + + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!(matches!(err, HandlerError::Execution(_))); + + let source = err.source(); + assert!(source.is_some()); + assert!(source.unwrap().to_string().contains("payload only failed")); + } + + #[tokio::test] + async fn transient_error_variant() { + let err = HandlerError::Transient(Box::new(MockError("db down".to_string()))); + assert!(matches!(err, HandlerError::Transient(_))); + assert!(err.to_string().contains("transient failure")); + + let source = err.source(); + assert!(source.is_some()); + assert!(source.unwrap().to_string().contains("db down")); + } + + #[tokio::test] + async fn transient_vs_execution_are_distinct() { + let transient = HandlerError::Transient(Box::new(MockError("infra".to_string()))); + let execution = HandlerError::Execution(Box::new(MockError("logic".to_string()))); + + assert!(matches!(transient, HandlerError::Transient(_))); + assert!(!matches!(transient, HandlerError::Execution(_))); + + assert!(matches!(execution, HandlerError::Execution(_))); + assert!(!matches!(execution, HandlerError::Transient(_))); + } +} diff --git a/listener/crates/shared/broker/src/redis/mod.rs b/listener/crates/shared/broker/src/redis/mod.rs new file mode 100644 index 0000000000..5b0aafdb21 --- /dev/null +++ b/listener/crates/shared/broker/src/redis/mod.rs @@ -0,0 +1,34 @@ +mod circuit_breaker; +mod claim_task; +mod config; +mod connection; +mod consumer; +mod dead_letter; +mod depth; +mod error; +mod handler; +mod publisher; +mod stream_manager; +pub mod trimmer; + +pub use claim_task::ClaimSweeper; +pub use config::{RedisConsumerConfigBuilder, RedisPrefetchConfig, StreamTopology}; +pub use connection::RedisConnectionManager; +pub use consumer::RedisConsumer; +pub use dead_letter::{DeadLetterProcessor, DeadMessage}; +pub use depth::RedisQueueInspector; +pub use error::{RedisConsumerError, RedisPublisherError}; +pub use handler::{ + AckDecision, AsyncRedisHandlerNoArgs, AsyncRedisHandlerPayloadOnly, AsyncRedisHandlerWithArgs, + Handler, HandlerError, Message, MessageMetadata, +}; +pub use publisher::{RedisPublisher, RedisPublisherBuilder, ReplicationConfig}; +pub use stream_manager::StreamManager; +pub use trimmer::{StreamTrimmer, StreamTrimmerConfig}; + +// Re-export from traits for convenience +pub use crate::traits::circuit_breaker::CircuitBreakerConfig; +pub use crate::traits::{ + AsyncHandlerPayloadOnly, AsyncHandlerWithContext, Consumer, Publisher as PublisherTrait, + RetryPolicy, +}; diff --git a/listener/crates/shared/broker/src/redis/publisher.rs b/listener/crates/shared/broker/src/redis/publisher.rs new file mode 100644 index 0000000000..a997483d97 --- /dev/null +++ b/listener/crates/shared/broker/src/redis/publisher.rs @@ -0,0 +1,499 @@ +use serde::Serialize; +use std::collections::HashMap; +use std::fmt::Debug; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::Mutex; +use tokio::time::sleep; +use tokio_util::sync::CancellationToken; +use tracing::{debug, warn}; + +use super::{ + connection::RedisConnectionManager, + error::RedisPublisherError, + trimmer::{StreamTrimmer, StreamTrimmerConfig}, +}; + +/// Optional auto-trim configuration embedded in the publisher. +/// When set, the publisher automatically spawns a background `StreamTrimmer` +/// per stream on first publish — the developer never needs to know about trimming. +#[derive(Debug, Clone)] +struct AutoTrimConfig { + /// Interval between trim cycles + interval: Duration, + /// Fallback MAXLEN if no consumer groups are found + fallback_maxlen: Option, +} + +/// Replication durability configuration for the publisher. +/// +/// When set, the publisher issues a `WAIT` command after each successful +/// `XADD` to ensure the write has been replicated before returning. +#[derive(Debug, Clone)] +pub struct ReplicationConfig { + /// Number of replicas that must acknowledge the write. + /// For a standard primary + 1 replica setup, this is `1`. + pub num_replicas: u32, + /// Maximum time to wait for replica acknowledgment. + /// Under normal cross-AZ conditions, replication takes < 1ms. + /// This timeout is a ceiling for degraded conditions. + pub timeout: Duration, +} + +/// Redis Streams publisher with retry, optional MAXLEN trimming, optional +/// automatic background stream trimming, and optional replication durability. +/// +/// Analogous to `RmqPublisher` — publishes serialized payloads to a +/// Redis stream via `XADD`, with exponential backoff retry on failure. +/// +/// When `auto_trim` is enabled via the builder, the publisher transparently +/// manages stream size using consumer-group-aware trimming. The developer +/// never needs to interact with `StreamTrimmer` directly. +/// +/// When `replication_wait` is enabled, the publisher issues `WAIT` after +/// each `XADD` to confirm replication to at least N replicas before returning. +pub struct RedisPublisher { + connection: RedisConnectionManager, + max_retries: u32, + base_retry_delay_ms: u64, + /// Optional approximate maximum stream length for MAXLEN trimming on each XADD. + max_stream_len: Option, + /// Optional auto-trim config — spawns background trimmers per stream. + auto_trim: Option, + /// Active trimmer cancellation tokens, keyed by stream name. + trimmers: Arc>>, + /// Optional replication durability — issues WAIT after each XADD. + replication_config: Option, +} + +impl RedisPublisher { + /// Create a new publisher with default settings (4 retries, 1s base delay, no MAXLEN, no auto-trim, no replication wait). + pub fn new(connection: RedisConnectionManager) -> Self { + Self { + connection, + max_retries: 4, + base_retry_delay_ms: 1000, + max_stream_len: None, + auto_trim: None, + trimmers: Arc::new(Mutex::new(HashMap::new())), + replication_config: None, + } + } + + /// Create a publisher with custom configuration (backward compatible, no auto-trim). + pub fn with_config( + connection: RedisConnectionManager, + max_retries: u32, + base_retry_delay_ms: u64, + max_stream_len: Option, + ) -> Self { + Self { + connection, + max_retries, + base_retry_delay_ms, + max_stream_len, + auto_trim: None, + trimmers: Arc::new(Mutex::new(HashMap::new())), + replication_config: None, + } + } + + /// Create a builder for fine-grained publisher configuration including auto-trim. + pub fn builder(connection: RedisConnectionManager) -> RedisPublisherBuilder { + RedisPublisherBuilder::new(connection) + } + + /// Publish a single payload to a Redis stream. + /// + /// Serializes the payload to JSON bytes and publishes via: + /// `XADD {stream} MAXLEN ~ {max_len} * data {json_bytes}` + /// + /// If auto-trim is enabled, lazily spawns a background trimmer for this stream + /// on the first publish call. + /// + /// Returns the auto-generated message ID on success. + /// Retries with exponential backoff on failure. + /// Returns `Err(RedisPublisherError::RetriesExhausted)` after max retries. + pub async fn publish( + &self, + stream: &str, + payload: &T, + ) -> Result { + // Lazily spawn trimmer for this stream if auto-trim is configured + self.ensure_trimmer(stream).await; + + let body = serde_json::to_vec(payload)?; + let publish_start = std::time::Instant::now(); + + for attempt in 0..self.max_retries { + let mut conn = self.connection.get_connection(); + + let result: Result = + if let Some(maxlen) = self.max_stream_len { + redis::cmd("XADD") + .arg(stream) + .arg("MAXLEN") + .arg("~") + .arg(maxlen) + .arg("*") + .arg("data") + .arg(&body) + .query_async(&mut conn) + .await + } else { + redis::cmd("XADD") + .arg(stream) + .arg("*") + .arg("data") + .arg(&body) + .query_async(&mut conn) + .await + }; + + match result { + Ok(msg_id) => { + // If replication is configured, ensure at least N replicas + // have acknowledged this write before returning to the caller. + if let Some(ref repl) = self.replication_config { + let acked: u32 = match redis::cmd("WAIT") + .arg(repl.num_replicas) + .arg(repl.timeout.as_millis() as u64) + .query_async(&mut conn) + .await + { + Ok(n) => n, + Err(e) => { + // XADD already succeeded — message is on the primary. + // WAIT failed (connection dropped mid-command). + // Do NOT retry XADD or we'll duplicate the message. + warn!( + error = %e, + stream_id = %msg_id, + stream = %stream, + "WAIT failed after successful XADD — \ + replication state unknown, message exists on primary" + ); + return Ok(msg_id); + } + }; + + if acked < repl.num_replicas { + warn!( + acked, + expected = repl.num_replicas, + stream_id = %msg_id, + stream = %stream, + "WAIT: insufficient replica ACKs — \ + write may not survive primary failover" + ); + } + } + metrics::counter!("broker_messages_published_total", + "backend" => "redis", "topic" => stream.to_owned() + ) + .increment(1); + metrics::histogram!("broker_publish_duration_seconds", + "backend" => "redis", "topic" => stream.to_owned() + ) + .record(publish_start.elapsed().as_secs_f64()); + return Ok(msg_id); + } + Err(e) => { + let error_kind = if e.is_io_error() + || e.is_connection_dropped() + || e.is_connection_refusal() + { + "connection" + } else if e.is_timeout() { + "timeout" + } else { + "other" + }; + metrics::counter!("broker_publish_errors_total", + "backend" => "redis", + "topic" => stream.to_owned(), + "error_kind" => error_kind, + ) + .increment(1); + + // On connection/timeout errors, replace the stale ConnectionManager + // so the next retry gets a fresh TCP connection. ConnectionManager + // does NOT auto-reconnect for TimedOut errors (redis-rs maps them + // to RetryMethod::RetryImmediately, not Reconnect). + if error_kind == "connection" { + self.connection.force_reconnect().await; + } + warn!( + attempt = attempt + 1, + max_retries = self.max_retries, + error = %e, + stream = %stream, + "Publish failed, retrying..." + ); + } + } + + // Exponential backoff + let delay = self + .base_retry_delay_ms + .saturating_mul(2u64.pow(attempt)) + .min(30_000); + sleep(Duration::from_millis(delay)).await; + } + + Err(RedisPublisherError::RetriesExhausted { + retries: self.max_retries, + stream: stream.to_string(), + }) + } + + /// Cancel all background trimmers and shut down cleanly. + pub async fn shutdown(&self) { + let trimmers = self.trimmers.lock().await; + for (stream, token) in trimmers.iter() { + debug!(stream = %stream, "Shutting down auto-trimmer"); + token.cancel(); + } + } + + /// Lazily spawn a background trimmer for the given stream, if auto-trim + /// is configured and no trimmer is already running for this stream. + async fn ensure_trimmer(&self, stream: &str) { + let Some(ref trim_config) = self.auto_trim else { + return; + }; + + let mut trimmers = self.trimmers.lock().await; + if trimmers.contains_key(stream) { + return; + } + + let cancel = CancellationToken::new(); + let trimmer = StreamTrimmer::new( + self.connection.clone(), + StreamTrimmerConfig { + stream: stream.to_string(), + interval: trim_config.interval, + fallback_maxlen: trim_config.fallback_maxlen, + }, + ); + + let cancel_clone = cancel.clone(); + tokio::spawn(async move { + trimmer.run(cancel_clone).await; + }); + + trimmers.insert(stream.to_string(), cancel); + debug!(stream = %stream, "Auto-trimmer spawned"); + } +} + +impl Drop for RedisPublisher { + fn drop(&mut self) { + // Cancel all trimmer tasks on drop. + // We can't async here, but CancellationToken::cancel() is sync. + if let Ok(trimmers) = self.trimmers.try_lock() { + for (_, token) in trimmers.iter() { + token.cancel(); + } + } + } +} + +/// Builder for constructing a `RedisPublisher` with fine-grained configuration. +/// +/// # Example +/// +/// ```rust,ignore +/// let publisher = RedisPublisher::builder(conn) +/// .max_retries(10) +/// .base_retry_delay_ms(1000) +/// .max_stream_len(100_000) +/// .auto_trim(Duration::from_secs(60)) +/// .fallback_maxlen(100_000) +/// .replication_wait(1, Duration::from_millis(500)) +/// .build(); +/// ``` +#[must_use] +pub struct RedisPublisherBuilder { + connection: RedisConnectionManager, + max_retries: u32, + base_retry_delay_ms: u64, + max_stream_len: Option, + auto_trim_interval: Option, + fallback_maxlen: Option, + replication_config: Option, +} + +impl RedisPublisherBuilder { + fn new(connection: RedisConnectionManager) -> Self { + Self { + connection, + max_retries: 10, + base_retry_delay_ms: 1000, + max_stream_len: None, + auto_trim_interval: None, + fallback_maxlen: None, + replication_config: None, + } + } + + /// Maximum number of publish retries before panicking (default: 10). + pub fn max_retries(mut self, max_retries: u32) -> Self { + self.max_retries = max_retries; + self + } + + /// Base delay in milliseconds for exponential backoff (default: 1000). + pub fn base_retry_delay_ms(mut self, ms: u64) -> Self { + self.base_retry_delay_ms = ms; + self + } + + /// Optional approximate maximum stream length applied on each XADD via `MAXLEN ~`. + pub fn max_stream_len(mut self, len: usize) -> Self { + self.max_stream_len = Some(len); + self + } + + /// Enable automatic background stream trimming at the given interval. + /// + /// When enabled, the publisher spawns a background task per stream that + /// periodically trims entries all consumer groups have fully processed. + /// This is the Redis equivalent of RabbitMQ's automatic queue cleanup + /// after message acknowledgement — the developer doesn't need to know + /// about Redis stream trimming. + pub fn auto_trim(mut self, interval: Duration) -> Self { + self.auto_trim_interval = Some(interval); + self + } + + /// Fallback maximum stream length used by auto-trim when no consumer groups exist. + /// Only relevant when `auto_trim` is enabled. + pub fn fallback_maxlen(mut self, len: usize) -> Self { + self.fallback_maxlen = Some(len); + self + } + + /// Enable replication durability via the Redis `WAIT` command. + /// + /// After each successful `XADD`, the publisher blocks until `num_replicas` + /// replicas have acknowledged the write, or `timeout` elapses. + /// + /// - On a single-node Redis (no replicas), `WAIT` returns `0` immediately. + /// This is harmless but provides no durability benefit. + /// - On ElastiCache Serverless, `WAIT` is blocked by AWS. Use node-based only. + /// + /// Recommended: `replication_wait(1, Duration::from_millis(500))` + pub fn replication_wait(mut self, num_replicas: u32, timeout: Duration) -> Self { + self.replication_config = Some(ReplicationConfig { + num_replicas, + timeout, + }); + self + } + + /// Build the `RedisPublisher`. + pub fn build(self) -> RedisPublisher { + let auto_trim = self.auto_trim_interval.map(|interval| AutoTrimConfig { + interval, + fallback_maxlen: self.fallback_maxlen, + }); + + RedisPublisher { + connection: self.connection, + max_retries: self.max_retries, + base_retry_delay_ms: self.base_retry_delay_ms, + max_stream_len: self.max_stream_len, + auto_trim, + trimmers: Arc::new(Mutex::new(HashMap::new())), + replication_config: self.replication_config, + } + } +} + +#[async_trait::async_trait] +impl crate::traits::publisher::Publisher for RedisPublisher { + type Error = RedisPublisherError; + + async fn publish( + &self, + topic: &str, + payload: &T, + ) -> Result<(), Self::Error> { + self.publish(topic, payload).await.map(|_id| ()) + } + + async fn shutdown(&self) { + self.shutdown().await; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_builder_defaults() { + // Can't fully test without Redis, but verify builder compiles and defaults + // are set correctly by inspecting the built publisher. + } + + #[test] + fn test_builder_with_auto_trim() { + // Verify that auto_trim config is set via builder + // (integration test would verify actual trimmer spawning) + } + + // Integration tests requiring Redis + #[tokio::test] + #[ignore] + async fn test_publish_single() { + let conn = RedisConnectionManager::new("redis://localhost:6379") + .await + .unwrap(); + let publisher = RedisPublisher::new(conn); + + let payload = serde_json::json!({"block_number": 42, "chain_id": 1}); + let result = publisher.publish("test.publish", &payload).await; + assert!(result.is_ok()); + let msg_id = result.unwrap(); + assert!(!msg_id.is_empty()); + } + + #[tokio::test] + #[ignore] + async fn test_publish_with_maxlen() { + let conn = RedisConnectionManager::new("redis://localhost:6379") + .await + .unwrap(); + let publisher = RedisPublisher::with_config(conn, 3, 500, Some(1000)); + + let payload = serde_json::json!({"block_number": 42}); + let result = publisher.publish("test.publish.maxlen", &payload).await; + assert!(result.is_ok()); + } + + #[tokio::test] + #[ignore] + async fn test_publish_with_auto_trim() { + let conn = RedisConnectionManager::new("redis://localhost:6379") + .await + .unwrap(); + let publisher = RedisPublisher::builder(conn) + .max_retries(3) + .auto_trim(Duration::from_secs(5)) + .fallback_maxlen(1000) + .build(); + + let payload = serde_json::json!({"block_number": 42}); + let result = publisher.publish("test.publish.autotrim", &payload).await; + assert!(result.is_ok()); + + // Verify trimmer was spawned + let trimmers = publisher.trimmers.lock().await; + assert!(trimmers.contains_key("test.publish.autotrim")); + + publisher.shutdown().await; + } +} diff --git a/listener/crates/shared/broker/src/redis/stream_manager.rs b/listener/crates/shared/broker/src/redis/stream_manager.rs new file mode 100644 index 0000000000..1e7e3d6545 --- /dev/null +++ b/listener/crates/shared/broker/src/redis/stream_manager.rs @@ -0,0 +1,307 @@ +use redis::AsyncCommands; +use tracing::{debug, info, warn}; + +use super::{ + config::StreamTopology, connection::RedisConnectionManager, error::RedisConsumerError, +}; + +/// Manages Redis stream and consumer group setup. +/// +/// Analogous to `ExchangeManager` in the RMQ broker — handles +/// stream creation, consumer group creation, and topology setup. +pub struct StreamManager { + connection: RedisConnectionManager, +} + +impl StreamManager { + /// Create a new StreamManager with the given connection manager. + pub fn new(connection: RedisConnectionManager) -> Self { + Self { connection } + } + + /// Ensure a stream exists. If it doesn't exist, creates it via + /// `XADD ... MAXLEN 0 * _init _init` then immediately trims. + /// This is a no-op if the stream already exists. + pub async fn ensure_stream(&self, stream: &str) -> Result<(), RedisConsumerError> { + let mut conn = self.connection.get_connection(); + + // Check if stream exists via XLEN (returns 0 for non-existent or empty) + let exists: bool = redis::cmd("EXISTS") + .arg(stream) + .query_async(&mut conn) + .await + .map_err(RedisConsumerError::Connection)?; + + if !exists { + // Create stream with a sentinel entry, then delete it + let id: String = redis::cmd("XADD") + .arg(stream) + .arg("*") + .arg("_init") + .arg("1") + .query_async(&mut conn) + .await + .map_err(|e| RedisConsumerError::StreamRead { + stream: stream.to_string(), + source: e, + })?; + + // Remove the sentinel entry + let _: i64 = redis::cmd("XDEL") + .arg(stream) + .arg(&id) + .query_async(&mut conn) + .await + .map_err(|e| RedisConsumerError::StreamRead { + stream: stream.to_string(), + source: e, + })?; + + info!(stream = %stream, "Stream created"); + } else { + debug!(stream = %stream, "Stream already exists"); + } + + Ok(()) + } + + /// Ensure a consumer group exists on a stream. + /// + /// Uses `XGROUP CREATE ... MKSTREAM` which is idempotent — catches + /// the BUSYGROUP error if the group already exists. + /// + /// `start_id` is typically `"0"` to read from the beginning or `"$"` to + /// read only new messages. + pub async fn ensure_consumer_group( + &self, + stream: &str, + group: &str, + start_id: &str, + ) -> Result<(), RedisConsumerError> { + let mut conn = self.connection.get_connection(); + + let result: Result = redis::cmd("XGROUP") + .arg("CREATE") + .arg(stream) + .arg(group) + .arg(start_id) + .arg("MKSTREAM") + .query_async(&mut conn) + .await; + + match result { + Ok(_) => { + info!( + stream = %stream, + group = %group, + start_id = %start_id, + "Consumer group created" + ); + Ok(()) + } + Err(e) => { + let err_msg = e.to_string(); + if err_msg.contains("BUSYGROUP") { + debug!( + stream = %stream, + group = %group, + "Consumer group already exists" + ); + Ok(()) + } else { + Err(RedisConsumerError::GroupCreation { + stream: stream.to_string(), + group: group.to_string(), + source: e, + }) + } + } + } + } + + /// Ensure the full stream topology for a chain: main stream + dead-letter stream. + /// + /// This is an **infrastructure-level** operation — it only creates the streams. + /// Consumer groups are automatically created when consumers start (via `XGROUP CREATE ... MKSTREAM`) + /// and when the dead-letter stream receives its first message (via `XADD`). + /// + /// No explicit consumer group setup is needed — the system handles it transparently. + pub async fn ensure_topology( + &self, + topology: &StreamTopology, + ) -> Result<(), RedisConsumerError> { + self.ensure_stream(&topology.main).await?; + self.ensure_stream(&topology.dead).await?; + + info!( + main = %topology.main, + dead = %topology.dead, + "Stream topology ensured (streams only)" + ); + + Ok(()) + } + + /// Delete a consumer from a group. + /// Useful for cleaning up stale consumers. + pub async fn delete_consumer( + &self, + stream: &str, + group: &str, + consumer_name: &str, + ) -> Result { + let mut conn = self.connection.get_connection(); + + let pending_count: u64 = redis::cmd("XGROUP") + .arg("DELCONSUMER") + .arg(stream) + .arg(group) + .arg(consumer_name) + .query_async(&mut conn) + .await + .map_err(|e| RedisConsumerError::GroupCreation { + stream: stream.to_string(), + group: group.to_string(), + source: e, + })?; + + if pending_count > 0 { + warn!( + stream = %stream, + group = %group, + consumer = %consumer_name, + pending_count = %pending_count, + "Deleted consumer with pending messages" + ); + } else { + info!( + stream = %stream, + group = %group, + consumer = %consumer_name, + "Consumer deleted" + ); + } + + Ok(pending_count) + } + + /// Get stream info via `XINFO STREAM`. + /// Returns the raw Redis value for flexible inspection. + pub async fn stream_info(&self, stream: &str) -> Result { + let mut conn = self.connection.get_connection(); + + let info: redis::Value = redis::cmd("XINFO") + .arg("STREAM") + .arg(stream) + .query_async(&mut conn) + .await + .map_err(|e| RedisConsumerError::StreamRead { + stream: stream.to_string(), + source: e, + })?; + + Ok(info) + } + + /// Get consumer group info via `XINFO GROUPS`. + /// Returns the raw Redis value for flexible inspection. + pub async fn group_info(&self, stream: &str) -> Result { + let mut conn = self.connection.get_connection(); + + let info: redis::Value = redis::cmd("XINFO") + .arg("GROUPS") + .arg(stream) + .query_async(&mut conn) + .await + .map_err(|e| RedisConsumerError::StreamRead { + stream: stream.to_string(), + source: e, + })?; + + Ok(info) + } + + /// Get stream length via `XLEN`. + pub async fn stream_len(&self, stream: &str) -> Result { + let mut conn = self.connection.get_connection(); + + let len: u64 = conn + .xlen(stream) + .await + .map_err(|e| RedisConsumerError::StreamRead { + stream: stream.to_string(), + source: e, + })?; + + Ok(len) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // Integration tests requiring Redis + #[tokio::test] + #[ignore] + async fn test_ensure_consumer_group() { + let conn = RedisConnectionManager::new("redis://localhost:6379") + .await + .unwrap(); + let manager = StreamManager::new(conn); + + let result = manager + .ensure_consumer_group("test.stream", "test-group", "0") + .await; + assert!(result.is_ok()); + + // Idempotent — calling again should succeed + let result = manager + .ensure_consumer_group("test.stream", "test-group", "0") + .await; + assert!(result.is_ok()); + } + + #[tokio::test] + #[ignore] + async fn test_ensure_topology() { + let conn = RedisConnectionManager::new("redis://localhost:6379") + .await + .unwrap(); + let manager = StreamManager::new(conn); + + let topology = StreamTopology::from_prefix("test.events"); + + // Topology only creates streams, consumer groups are created automatically by consumers + let result = manager.ensure_topology(&topology).await; + assert!(result.is_ok()); + + // Idempotent — calling again should succeed + let result = manager.ensure_topology(&topology).await; + assert!(result.is_ok()); + } + + #[tokio::test] + #[ignore] + async fn test_multiple_consumer_groups() { + let conn = RedisConnectionManager::new("redis://localhost:6379") + .await + .unwrap(); + let manager = StreamManager::new(conn); + + let topology = StreamTopology::from_prefix("test.events"); + manager.ensure_topology(&topology).await.unwrap(); + + // Multiple apps can create their own consumer groups on the same stream + // Each uses ensure_consumer_group directly (or it's auto-created by RedisConsumer) + let result = manager + .ensure_consumer_group(&topology.main, "app-a-group", "0") + .await; + assert!(result.is_ok()); + + let result = manager + .ensure_consumer_group(&topology.main, "app-b-group", "0") + .await; + assert!(result.is_ok()); + } +} diff --git a/listener/crates/shared/broker/src/redis/trimmer.rs b/listener/crates/shared/broker/src/redis/trimmer.rs new file mode 100644 index 0000000000..c1775286b7 --- /dev/null +++ b/listener/crates/shared/broker/src/redis/trimmer.rs @@ -0,0 +1,334 @@ +use redis::Value; +use std::time::Duration; +use tokio::time::sleep; +use tokio_util::sync::CancellationToken; +use tracing::{debug, error, info}; + +use super::{connection::RedisConnectionManager, error::RedisConsumerError}; + +/// Configuration for the stream trimmer. +#[derive(Debug, Clone)] +pub struct StreamTrimmerConfig { + /// Stream to trim + pub stream: String, + /// Interval between trim cycles + pub interval: Duration, + /// Fallback MAXLEN if no consumer groups are found + pub fallback_maxlen: Option, +} + +/// Server-side stream trimmer that safely trims a Redis stream based on +/// all consumer groups' progress. +/// +/// Runs on the listener core (publisher side), NOT on clients. +/// Uses `XTRIM ... MINID ~` to only trim what ALL groups have fully processed. +pub struct StreamTrimmer { + connection: RedisConnectionManager, + config: StreamTrimmerConfig, +} + +impl StreamTrimmer { + pub fn new(connection: RedisConnectionManager, config: StreamTrimmerConfig) -> Self { + Self { connection, config } + } + + /// Run the trimmer as a supervised loop. + /// + /// Periodically calls `trim_once()` at the configured interval. + /// On error: logs, waits, and retries. + /// Stops when the cancellation token is cancelled. + pub async fn run(&self, cancel: CancellationToken) { + info!( + stream = %self.config.stream, + interval = ?self.config.interval, + "StreamTrimmer started" + ); + + loop { + tokio::select! { + _ = cancel.cancelled() => { + info!("StreamTrimmer: cancellation requested, stopping"); + return; + } + _ = sleep(self.config.interval) => { + match self.trim_once().await { + Ok(trimmed) => { + if trimmed > 0 { + info!( + stream = %self.config.stream, + trimmed = trimmed, + "StreamTrimmer: trimmed entries" + ); + } + } + Err(e) => { + error!( + error = %e, + stream = %self.config.stream, + "StreamTrimmer: trim_once failed, will retry next interval" + ); + sleep(Duration::from_secs(5)).await; + } + } + } + } + } + } + + /// Perform a single trim operation. + /// + /// 1. `XINFO GROUPS {stream}` to get all consumer groups + /// 2. For each group: find the oldest pending message (or last-delivered-id) + /// 3. Compute `safe_min_id` = minimum across all groups + /// 4. `XTRIM {stream} MINID ~ {safe_min_id}` — trim only what's safe + /// + /// Returns the number of trimmed entries. + pub async fn trim_once(&self) -> Result { + let mut conn = self.connection.get_connection(); + + // Get all consumer groups + let groups_value: Value = redis::cmd("XINFO") + .arg("GROUPS") + .arg(&self.config.stream) + .query_async(&mut conn) + .await + .map_err(|e| RedisConsumerError::StreamRead { + stream: self.config.stream.clone(), + source: e, + })?; + + let groups = Self::parse_groups(groups_value); + + if groups.is_empty() { + // No consumer groups — use fallback MAXLEN if configured + if let Some(maxlen) = self.config.fallback_maxlen { + let trimmed: u64 = redis::cmd("XTRIM") + .arg(&self.config.stream) + .arg("MAXLEN") + .arg("~") + .arg(maxlen) + .query_async(&mut conn) + .await + .map_err(|e| RedisConsumerError::StreamRead { + stream: self.config.stream.clone(), + source: e, + })?; + return Ok(trimmed); + } + debug!( + stream = %self.config.stream, + "No consumer groups and no fallback MAXLEN, skipping trim" + ); + return Ok(0); + } + + // For each group, find the safe minimum ID + let mut safe_min_id: Option = None; + + for group in &groups { + let group_min = self.get_group_min_id(&group.name).await?; + + if let Some(ref min_id) = group_min { + safe_min_id = Some(match safe_min_id { + None => min_id.clone(), + Some(ref current) => Self::min_stream_id(current, min_id), + }); + } + // If a group has no min ID (no pending AND no delivered), skip trimming + // to be safe — don't trim anything + } + + let Some(min_id) = safe_min_id else { + debug!( + stream = %self.config.stream, + "No safe min ID found, skipping trim" + ); + return Ok(0); + }; + + // XTRIM with MINID ~ + let trimmed: u64 = redis::cmd("XTRIM") + .arg(&self.config.stream) + .arg("MINID") + .arg("~") + .arg(&min_id) + .query_async(&mut conn) + .await + .map_err(|e| RedisConsumerError::StreamRead { + stream: self.config.stream.clone(), + source: e, + })?; + + Ok(trimmed) + } + + /// Get the minimum safe ID for a consumer group. + /// + /// First checks XPENDING for the oldest pending message. + /// If no pending messages, uses the group's last-delivered-id. + async fn get_group_min_id(&self, group: &str) -> Result, RedisConsumerError> { + let mut conn = self.connection.get_connection(); + + // XPENDING summary: [count, min_id, max_id, [[consumer, count], ...]] + let result: Value = redis::cmd("XPENDING") + .arg(&self.config.stream) + .arg(group) + .query_async(&mut conn) + .await + .map_err(|e| RedisConsumerError::StreamRead { + stream: self.config.stream.clone(), + source: e, + })?; + + // Parse XPENDING summary + if let Value::Array(ref parts) = result + && parts.len() >= 2 + { + // parts[0] = count of pending + let pending_count = match &parts[0] { + Value::Int(n) => *n, + _ => 0, + }; + + if pending_count > 0 { + // parts[1] = min pending ID + if let Value::BulkString(b) = &parts[1] { + return Ok(Some(String::from_utf8_lossy(b).to_string())); + } + } + } + + // No pending messages — get last-delivered-id from XINFO GROUPS + let groups_value: Value = redis::cmd("XINFO") + .arg("GROUPS") + .arg(&self.config.stream) + .query_async(&mut conn) + .await + .map_err(|e| RedisConsumerError::StreamRead { + stream: self.config.stream.clone(), + source: e, + })?; + + let groups = Self::parse_groups(groups_value); + for g in &groups { + if g.name == group { + return Ok(Some(g.last_delivered_id.clone())); + } + } + + Ok(None) + } + + /// Compare two stream IDs and return the smaller one. + /// + /// Stream IDs are formatted as "{timestamp}-{sequence}". + fn min_stream_id(a: &str, b: &str) -> String { + let parse_id = |id: &str| -> (u64, u64) { + let parts: Vec<&str> = id.splitn(2, '-').collect(); + let ts = parts.first().and_then(|s| s.parse().ok()).unwrap_or(0); + let seq = parts.get(1).and_then(|s| s.parse().ok()).unwrap_or(0); + (ts, seq) + }; + + let (a_ts, a_seq) = parse_id(a); + let (b_ts, b_seq) = parse_id(b); + + if (a_ts, a_seq) <= (b_ts, b_seq) { + a.to_string() + } else { + b.to_string() + } + } + + /// Parse consumer group info from XINFO GROUPS response. + fn parse_groups(value: Value) -> Vec { + let mut groups = Vec::new(); + + if let Value::Array(items) = value { + for item in items { + if let Value::Array(fields) = item { + let map = Self::flat_array_to_map(&fields); + let name = map.get("name").cloned().unwrap_or_default(); + let last_delivered_id = map + .get("last-delivered-id") + .cloned() + .unwrap_or_else(|| "0-0".to_string()); + + groups.push(GroupInfo { + name, + last_delivered_id, + }); + } + } + } + + groups + } + + /// Convert a flat Redis field array [key, value, key, value, ...] to a HashMap. + fn flat_array_to_map(fields: &[Value]) -> std::collections::HashMap { + let mut map = std::collections::HashMap::new(); + let mut iter = fields.iter(); + while let Some(key) = iter.next() { + if let Value::BulkString(k) = key { + if let Some(val) = iter.next() { + let v = match val { + Value::BulkString(b) => String::from_utf8_lossy(b).to_string(), + Value::Int(n) => n.to_string(), + _ => continue, + }; + map.insert(String::from_utf8_lossy(k).to_string(), v); + } + } else { + // Skip non-bulk-string keys (shouldn't happen in XINFO output) + let _ = iter.next(); + } + } + map + } +} + +/// Internal representation of a consumer group from XINFO GROUPS. +struct GroupInfo { + name: String, + last_delivered_id: String, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_min_stream_id() { + assert_eq!(StreamTrimmer::min_stream_id("100-0", "200-0"), "100-0"); + assert_eq!(StreamTrimmer::min_stream_id("200-0", "100-0"), "100-0"); + assert_eq!(StreamTrimmer::min_stream_id("100-1", "100-2"), "100-1"); + assert_eq!(StreamTrimmer::min_stream_id("100-0", "100-0"), "100-0"); + } + + #[test] + fn test_parse_groups_empty() { + let value = Value::Array(vec![]); + let groups = StreamTrimmer::parse_groups(value); + assert!(groups.is_empty()); + } + + #[test] + fn test_parse_groups_with_data() { + let value = Value::Array(vec![Value::Array(vec![ + Value::BulkString(b"name".to_vec()), + Value::BulkString(b"my-group".to_vec()), + Value::BulkString(b"consumers".to_vec()), + Value::Int(2), + Value::BulkString(b"pending".to_vec()), + Value::Int(5), + Value::BulkString(b"last-delivered-id".to_vec()), + Value::BulkString(b"1234567890-0".to_vec()), + ])]); + + let groups = StreamTrimmer::parse_groups(value); + assert_eq!(groups.len(), 1); + assert_eq!(groups[0].name, "my-group"); + assert_eq!(groups[0].last_delivered_id, "1234567890-0"); + } +} diff --git a/listener/crates/shared/broker/src/topic.rs b/listener/crates/shared/broker/src/topic.rs new file mode 100644 index 0000000000..6f7c3d9487 --- /dev/null +++ b/listener/crates/shared/broker/src/topic.rs @@ -0,0 +1,162 @@ +//! Topic abstraction for optional namespace + routing. +//! +//! A `Topic` represents a routing key optionally scoped by a namespace. +//! This maps to: +//! - main key `{namespace}.{routing}` when namespace exists +//! - main key `{routing}` when namespace is absent +//! - dead-letter key `{main_key}:dead` + +/// A topic representing optional namespace + routing key. +/// +/// The fully qualified key and dead-letter key are derived on demand. +/// +/// # Examples +/// +/// ``` +/// use broker::{Topic}; +/// +/// // Using predefined routing keys +/// let blocks = Topic::new("blocks").with_namespace("ethereum"); +/// let forks = Topic::new("forks").with_namespace("ethereum"); +/// +/// // Custom routing keys +/// let custom = Topic::new("erc20-transfers").with_namespace("ethereum"); +/// +/// // Generic key API +/// assert_eq!(blocks.key(), "ethereum.blocks"); +/// assert_eq!(blocks.dead_key(), "ethereum.blocks:dead"); +/// assert_eq!(blocks.routing_segment(), "blocks"); +/// ``` +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct Topic { + /// The namespace (chain/service), e.g., "ethereum", "polygon", "my-service". + namespace: Option, + /// The routing key (message type), e.g., "blocks", "forks", "new-filters". + routing: String, +} + +impl Topic { + /// Create a new topic without namespace. + pub fn new(routing: impl Into) -> Self { + let routing = routing.into().trim().to_string(); + Self { + namespace: None, + routing, + } + } + + /// Fluent setter for namespace. + /// + /// Empty/blank namespace is treated as `None`. + pub fn with_namespace(mut self, namespace: impl Into) -> Self { + let ns = namespace.into(); + let ns = ns.trim(); + self.namespace = if ns.is_empty() { + None + } else { + Some(ns.to_string()) + }; + self + } + + /// Convenience constructor for namespaced topics. + pub fn namespaced(namespace: impl Into, routing: impl Into) -> Self { + Self::new(routing).with_namespace(namespace) + } + + /// Remove namespace and return an unscoped topic. + pub fn without_namespace(mut self) -> Self { + self.namespace = None; + self + } + + /// Borrow namespace as `Option<&str>`. + pub fn namespace(&self) -> Option<&str> { + self.namespace.as_deref() + } + + /// Fully qualified key: + /// - `{namespace}.{routing}` when namespace exists + /// - `{routing}` otherwise + pub fn key(&self) -> String { + match self.namespace.as_deref() { + Some(ns) => format!("{ns}.{}", self.routing), + None => self.routing.clone(), + } + } + + /// Dead-letter key: `{main_key}:dead`. + pub fn dead_key(&self) -> String { + format!("{}:dead", self.key()) + } + + /// Raw routing segment (without namespace). + pub fn routing_segment(&self) -> &str { + &self.routing + } +} + +impl std::fmt::Display for Topic { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.key()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_topic_new_without_namespace() { + let topic = Topic::new("blocks"); + assert_eq!(topic.namespace(), None); + assert_eq!(topic.routing, "blocks"); + } + + #[test] + fn test_topic_with_namespace() { + let topic = Topic::new("blocks").with_namespace("ethereum"); + assert_eq!(topic.namespace(), Some("ethereum")); + assert_eq!(topic.routing, "blocks"); + } + + #[test] + fn test_topic_namespaced_constructor() { + let topic = Topic::namespaced("ethereum", "blocks"); + assert_eq!(topic.namespace(), Some("ethereum")); + assert_eq!(topic.routing, "blocks"); + } + + #[test] + fn test_key_namespaced() { + let topic = Topic::new("blocks").with_namespace("ethereum"); + assert_eq!(topic.key(), "ethereum.blocks"); + } + + #[test] + fn test_key_unscoped() { + let topic = Topic::new("blocks"); + assert_eq!(topic.key(), "blocks"); + } + + #[test] + fn test_dead_key_unscoped() { + let topic = Topic::new("blocks"); + assert_eq!(topic.dead_key(), "blocks:dead"); + } + + #[test] + fn test_routing_segment() { + let topic = Topic::new("blocks").with_namespace("ethereum"); + assert_eq!(topic.routing_segment(), "blocks"); + } + + #[test] + fn test_display_namespaced_and_unscoped() { + assert_eq!( + format!("{}", Topic::new("blocks").with_namespace("ethereum")), + "ethereum.blocks" + ); + assert_eq!(format!("{}", Topic::new("blocks")), "blocks"); + } +} diff --git a/listener/crates/shared/broker/src/traits/circuit_breaker.rs b/listener/crates/shared/broker/src/traits/circuit_breaker.rs new file mode 100644 index 0000000000..461defc93b --- /dev/null +++ b/listener/crates/shared/broker/src/traits/circuit_breaker.rs @@ -0,0 +1,430 @@ +use std::time::{Duration, Instant}; +use tracing::{debug, info, warn}; + +/// Configuration for the consumer circuit breaker. +/// +/// When enabled on a retry or prefetch consumer, the circuit breaker pauses +/// consumption when consecutive `Transient` handler errors exceed the threshold, +/// preventing DLQ pollution during downstream outages (DB down, API timeout, etc.). +/// +/// Backend-agnostic: both RMQ and Redis consumers can use this. +#[derive(Debug, Clone)] +pub struct CircuitBreakerConfig { + /// Number of consecutive `Transient` failures required to trip the circuit (default: 5). + pub failure_threshold: u32, + /// How long to stay in the Open state before transitioning to Half-Open (default: 30s). + pub cooldown_duration: Duration, +} + +impl Default for CircuitBreakerConfig { + fn default() -> Self { + Self { + failure_threshold: 5, + cooldown_duration: Duration::from_secs(30), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum CircuitState { + Closed, + Open, + HalfOpen, +} + +/// Prometheus labels attached to a `CircuitBreaker` for metric emission. +/// +/// When absent, no `broker_circuit_breaker_*` metrics are emitted. When present, +/// every state transition and counter change produces a Prometheus update. +#[derive(Debug, Clone)] +struct MetricLabels { + backend: &'static str, + topic: String, +} + +/// Lightweight circuit breaker state machine. +/// +/// Tracks consecutive transient (infrastructure) failures and pauses +/// consumption when the threshold is exceeded. Resumes after a cooldown +/// period by allowing a single probe request. +/// +/// This struct is internal to consumer implementations — the developer +/// configures it via [`CircuitBreakerConfig`] in the consumer builder. +/// +/// Call [`Self::with_labels`] to enable Prometheus emission +/// (`broker_circuit_breaker_state` gauge, `broker_circuit_breaker_trips_total` +/// counter, and `broker_circuit_breaker_consecutive_failures` gauge). +pub struct CircuitBreaker { + state: CircuitState, + consecutive_transient_failures: u32, + config: CircuitBreakerConfig, + last_opened_at: Option, + labels: Option, +} + +impl CircuitBreaker { + pub fn new(config: CircuitBreakerConfig) -> Self { + Self { + state: CircuitState::Closed, + consecutive_transient_failures: 0, + config, + last_opened_at: None, + labels: None, + } + } + + /// Attach Prometheus labels and seed the circuit breaker metrics. + /// + /// After calling this, the breaker emits: + /// - `broker_circuit_breaker_state{backend,topic}` (0=closed, 1=open, 2=half-open) + /// - `broker_circuit_breaker_consecutive_failures{backend,topic}` + /// - `broker_circuit_breaker_trips_total{backend,topic}` (on each trip) + /// + /// Called once at startup; the initial state (Closed, 0 failures) is emitted + /// immediately so Grafana discovers the time series on the first scrape. + pub fn with_labels(mut self, backend: &'static str, topic: impl Into) -> Self { + self.labels = Some(MetricLabels { + backend, + topic: topic.into(), + }); + self.emit_state(); + self.emit_consecutive_failures(); + self + } + + fn emit_state(&self) { + if let Some(labels) = &self.labels { + let code = match self.state { + CircuitState::Closed => 0.0, + CircuitState::Open => 1.0, + CircuitState::HalfOpen => 2.0, + }; + metrics::gauge!( + "broker_circuit_breaker_state", + "backend" => labels.backend, + "topic" => labels.topic.clone(), + ) + .set(code); + } + } + + fn emit_consecutive_failures(&self) { + if let Some(labels) = &self.labels { + metrics::gauge!( + "broker_circuit_breaker_consecutive_failures", + "backend" => labels.backend, + "topic" => labels.topic.clone(), + ) + .set(self.consecutive_transient_failures as f64); + } + } + + fn emit_trip(&self) { + if let Some(labels) = &self.labels { + metrics::counter!( + "broker_circuit_breaker_trips_total", + "backend" => labels.backend, + "topic" => labels.topic.clone(), + ) + .increment(1); + } + } + + /// Returns `true` when the consumer should proceed with reading messages. + pub fn should_allow_request(&mut self) -> bool { + match self.state { + CircuitState::Closed | CircuitState::HalfOpen => true, + CircuitState::Open => { + if let Some(opened_at) = self.last_opened_at { + if opened_at.elapsed() >= self.config.cooldown_duration { + self.state = CircuitState::HalfOpen; + self.emit_state(); + info!("Circuit breaker: transitioning to Half-Open (probing)"); + true + } else { + false + } + } else { + self.state = CircuitState::HalfOpen; + self.emit_state(); + true + } + } + } + } + + /// Record a successful handler execution. + /// + /// - **Closed**: resets the consecutive-transient counter (normal operation). + /// - **Half-Open**: closes the circuit (probe succeeded). + /// - **Open**: no-op. An incidental success recorded inside the same XREADGROUP + /// batch that tripped the threshold must not bypass the cooldown. The only valid + /// `Open → Closed` transition is `cooldown expires → Half-Open probe → success`. + pub fn record_success(&mut self) { + match self.state { + CircuitState::Closed => { + self.consecutive_transient_failures = 0; + self.emit_consecutive_failures(); + } + CircuitState::HalfOpen => { + info!("Circuit breaker: probe succeeded, closing circuit"); + self.consecutive_transient_failures = 0; + self.state = CircuitState::Closed; + self.last_opened_at = None; + self.emit_state(); + self.emit_consecutive_failures(); + } + CircuitState::Open => { + // No-op: do not bypass the cooldown. + } + } + } + + /// Record a transient (infrastructure) failure. Trips the circuit at threshold. + pub fn record_transient_failure(&mut self) { + self.consecutive_transient_failures += 1; + self.emit_consecutive_failures(); + + match self.state { + CircuitState::Closed => { + if self.consecutive_transient_failures >= self.config.failure_threshold { + self.state = CircuitState::Open; + self.last_opened_at = Some(Instant::now()); + self.emit_state(); + self.emit_trip(); + warn!( + consecutive_failures = self.consecutive_transient_failures, + cooldown = ?self.config.cooldown_duration, + "Circuit breaker: OPEN — pausing consumption" + ); + } else { + debug!( + consecutive_failures = self.consecutive_transient_failures, + threshold = self.config.failure_threshold, + "Circuit breaker: transient failure recorded" + ); + } + } + CircuitState::HalfOpen => { + self.state = CircuitState::Open; + self.last_opened_at = Some(Instant::now()); + self.emit_state(); + self.emit_trip(); + warn!( + cooldown = ?self.config.cooldown_duration, + "Circuit breaker: probe failed, reopening circuit" + ); + } + CircuitState::Open => {} + } + } + + /// Record a permanent (execution/deserialization) failure. + /// Does NOT trip the circuit — the message is the problem, not the infrastructure. + pub fn record_permanent_failure(&mut self) { + self.consecutive_transient_failures = 0; + self.emit_consecutive_failures(); + debug!("Circuit breaker: permanent failure recorded, transient counter reset"); + } + + /// Remaining cooldown before the circuit transitions to Half-Open. + pub fn remaining_cooldown(&self) -> Duration { + match (self.state, self.last_opened_at) { + (CircuitState::Open, Some(opened_at)) => self + .config + .cooldown_duration + .checked_sub(opened_at.elapsed()) + .unwrap_or(Duration::ZERO), + _ => Duration::ZERO, + } + } + + #[allow(dead_code)] + pub fn is_open(&self) -> bool { + self.state == CircuitState::Open + } + + #[allow(dead_code)] + pub fn is_half_open(&self) -> bool { + self.state == CircuitState::HalfOpen + } + + #[allow(dead_code)] + pub fn is_closed(&self) -> bool { + self.state == CircuitState::Closed + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn default_config() -> CircuitBreakerConfig { + CircuitBreakerConfig { + failure_threshold: 3, + cooldown_duration: Duration::from_millis(100), + } + } + + #[test] + fn starts_closed() { + let cb = CircuitBreaker::new(default_config()); + assert!(cb.is_closed()); + assert!(!cb.is_open()); + assert!(!cb.is_half_open()); + } + + #[test] + fn allows_requests_when_closed() { + let mut cb = CircuitBreaker::new(default_config()); + assert!(cb.should_allow_request()); + } + + #[test] + fn stays_closed_below_threshold() { + let mut cb = CircuitBreaker::new(default_config()); + cb.record_transient_failure(); + assert!(cb.is_closed()); + cb.record_transient_failure(); + assert!(cb.is_closed()); + assert!(cb.should_allow_request()); + } + + #[test] + fn opens_at_threshold() { + let mut cb = CircuitBreaker::new(default_config()); + cb.record_transient_failure(); + cb.record_transient_failure(); + cb.record_transient_failure(); + assert!(cb.is_open()); + assert!(!cb.should_allow_request()); + } + + #[test] + fn blocks_requests_when_open() { + let mut cb = CircuitBreaker::new(default_config()); + for _ in 0..3 { + cb.record_transient_failure(); + } + assert!(cb.is_open()); + assert!(!cb.should_allow_request()); + } + + #[tokio::test] + async fn transitions_to_half_open_after_cooldown() { + let mut cb = CircuitBreaker::new(CircuitBreakerConfig { + failure_threshold: 1, + cooldown_duration: Duration::from_millis(50), + }); + cb.record_transient_failure(); + assert!(cb.is_open()); + tokio::time::sleep(Duration::from_millis(60)).await; + assert!(cb.should_allow_request()); + assert!(cb.is_half_open()); + } + + #[tokio::test] + async fn half_open_closes_on_success() { + let mut cb = CircuitBreaker::new(CircuitBreakerConfig { + failure_threshold: 1, + cooldown_duration: Duration::from_millis(50), + }); + cb.record_transient_failure(); + tokio::time::sleep(Duration::from_millis(60)).await; + cb.should_allow_request(); + cb.record_success(); + assert!(cb.is_closed()); + assert!(cb.should_allow_request()); + } + + #[tokio::test] + async fn half_open_reopens_on_failure() { + let mut cb = CircuitBreaker::new(CircuitBreakerConfig { + failure_threshold: 1, + cooldown_duration: Duration::from_millis(50), + }); + cb.record_transient_failure(); + tokio::time::sleep(Duration::from_millis(60)).await; + cb.should_allow_request(); + cb.record_transient_failure(); + assert!(cb.is_open()); + assert!(!cb.should_allow_request()); + } + + #[test] + fn success_resets_counter() { + let mut cb = CircuitBreaker::new(default_config()); + cb.record_transient_failure(); + cb.record_transient_failure(); + cb.record_success(); + cb.record_transient_failure(); + assert!(cb.is_closed()); + } + + #[test] + fn success_while_open_does_not_close_circuit() { + // A success recorded in the same message batch that tripped the circuit + // (e.g. an XREADGROUP batch where entry N trips and entry N+1 succeeds) + // must not bypass the cooldown by closing the circuit directly from Open. + let mut cb = CircuitBreaker::new(default_config()); + for _ in 0..3 { + cb.record_transient_failure(); + } + assert!(cb.is_open()); + + cb.record_success(); // incidental success — must be a no-op + assert!( + cb.is_open(), + "record_success from Open state must not close the circuit" + ); + assert!( + !cb.should_allow_request(), + "circuit must still block requests" + ); + } + + #[test] + fn permanent_failure_resets_transient_counter() { + let mut cb = CircuitBreaker::new(default_config()); + cb.record_transient_failure(); + cb.record_transient_failure(); + cb.record_permanent_failure(); + cb.record_transient_failure(); + assert!(cb.is_closed()); + } + + #[test] + fn permanent_failure_does_not_trip() { + let mut cb = CircuitBreaker::new(default_config()); + for _ in 0..100 { + cb.record_permanent_failure(); + } + assert!(cb.is_closed()); + } + + #[test] + fn remaining_cooldown_when_closed() { + let cb = CircuitBreaker::new(default_config()); + assert_eq!(cb.remaining_cooldown(), Duration::ZERO); + } + + #[test] + fn remaining_cooldown_when_open() { + let mut cb = CircuitBreaker::new(CircuitBreakerConfig { + failure_threshold: 1, + cooldown_duration: Duration::from_secs(30), + }); + cb.record_transient_failure(); + assert!(cb.is_open()); + let remaining = cb.remaining_cooldown(); + assert!(remaining > Duration::ZERO); + assert!(remaining <= Duration::from_secs(30)); + } + + #[test] + fn default_config_values() { + let config = CircuitBreakerConfig::default(); + assert_eq!(config.failure_threshold, 5); + assert_eq!(config.cooldown_duration, Duration::from_secs(30)); + } +} diff --git a/listener/crates/shared/broker/src/traits/consumer.rs b/listener/crates/shared/broker/src/traits/consumer.rs new file mode 100644 index 0000000000..3bc7338643 --- /dev/null +++ b/listener/crates/shared/broker/src/traits/consumer.rs @@ -0,0 +1,39 @@ +use async_trait::async_trait; +use std::time::Duration; + +use super::handler::Handler; + +/// Exposes the common retry fields that all backend-specific retry configs share. +/// +/// Useful for logging, monitoring, or generic code that inspects retry policy +/// without caring which backend is in use. +pub trait RetryPolicy { + fn max_retries(&self) -> u32; + fn retry_delay(&self) -> Duration; +} + +/// Queue-agnostic consumer trait. +/// +/// Both `RmqConsumer` and `RedisConsumer` implement this trait, allowing +/// application code to be generic over the backend. +/// +/// Each backend brings its own prefetch-safe config type via the associated +/// type — retry semantics (DLX vs XCLAIM) remain backend-specific, but the +/// run method shares the same signature. +#[async_trait] +pub trait Consumer: Send + Sync + Sized { + /// Config for `run` — retry + throughput tuning. + type PrefetchConfig: Send; + /// Backend-specific error type. + type Error: std::error::Error + Send + Sync + 'static; + + /// Connect to the backend and return a ready-to-use consumer. + async fn connect(url: &str) -> Result; + + /// Run a high-throughput consumer with strong message loss guarantees. + async fn run( + &self, + config: Self::PrefetchConfig, + handler: impl Handler + 'static, + ) -> Result<(), Self::Error>; +} diff --git a/listener/crates/shared/broker/src/traits/depth.rs b/listener/crates/shared/broker/src/traits/depth.rs new file mode 100644 index 0000000000..11542c4b1c --- /dev/null +++ b/listener/crates/shared/broker/src/traits/depth.rs @@ -0,0 +1,290 @@ +//! Queue depth introspection API. +//! +//! Provides a backend-agnostic way to query message counts across +//! the principal, retry, and dead-letter queues for a given topic. +//! +//! When a consumer `group` is specified, backends that support it (Redis) +//! also return **pending** (PEL) and **lag** (undelivered) counts, enabling +//! callers to determine whether a consumer will receive messages without +//! relying solely on the raw stream length. + +use async_trait::async_trait; + +/// Message counts for all queues in a topic's queue group. +/// +/// Covers the three queue tiers used by both Redis Streams and AMQP backends: +/// - **principal**: the main processing queue/stream (XLEN for Redis, ready count for AMQP) +/// - **retry**: messages awaiting retry (AMQP retry queue; `None` for Redis +/// since retry is PEL-based, not a separate stream) +/// - **dead_letter**: messages that exhausted their retry budget +/// +/// When queried with a consumer group, two additional fields are populated: +/// - **pending**: messages delivered but not yet ACKed (PEL count) +/// - **lag**: messages not yet delivered to the group (Redis 7.0+ `XINFO GROUPS` lag) +#[derive(Debug, Clone, PartialEq, Eq)] +#[must_use] +pub struct QueueDepths { + /// Messages in the principal queue/stream. + pub principal: u64, + /// Messages in the retry queue (AMQP only; `None` for backends + /// where retry is not a separate queue, e.g. Redis PEL-based retry). + pub retry: Option, + /// Messages in the dead-letter queue/stream. + pub dead_letter: u64, + /// Pending entries: delivered to a consumer but not yet ACKed (PEL count). + /// `None` when no group was specified or the backend doesn't track this. + pub pending: Option, + /// Consumer group lag: entries in the stream not yet delivered to the group. + /// Populated from Redis 7.0+ `XINFO GROUPS` `lag` field. + /// `None` when no group was specified, the group doesn't exist, or Redis < 7.0. + pub lag: Option, +} + +impl QueueDepths { + /// Total messages across all queues (principal + retry + dead-letter). + /// + /// Does **not** include `pending`/`lag` — those are different views of + /// entries already counted in `principal`. + #[must_use] + pub fn total(&self) -> u64 { + self.principal + self.retry.unwrap_or(0) + self.dead_letter + } + + /// Returns `true` if all queues are empty. + #[must_use] + pub fn is_empty(&self) -> bool { + self.total() == 0 + } + + /// Returns `true` if a consumer in the queried group will receive at + /// least one message — either from PEL drain or new delivery. + /// + /// Decision logic (group-level metrics available): + /// 1. If `pending > 0` → consumer's drain phase will deliver PEL entries. + /// 2. If `lag` is `Some(n)` where `n > 0` → stream has undelivered entries. + /// 3. If `pending == Some(0)` and `lag == None` (Redis < 7.0) → unknown lag, + /// conservatively returns `false` so the caller publishes a seed. An + /// unnecessary seed is harmless (duplicates self-correct on `UpToDate`). + /// + /// Fallback (no group queried, or AMQP where `pending`/`lag` are `None`): + /// Returns `principal > 0`. + #[must_use] + pub fn has_pending_work(&self) -> bool { + match (self.pending, self.lag) { + // Group was queried and both metrics are known. + (Some(pending), Some(lag)) => pending > 0 || lag > 0, + // PEL is non-empty — drain will deliver regardless of lag. + (Some(pending), None) if pending > 0 => true, + // PEL is empty, lag unknown (Redis < 7.0) — can't be sure, + // return false so the caller seeds conservatively. + (Some(0), None) => false, + // No group-level metrics at all (no group queried, or AMQP). + // Fall back to stream-level heuristic. + (None, _) => self.principal > 0, + // Shouldn't happen (lag without pending), but handle gracefully. + (Some(_), None) => true, + } + } +} + +/// Trait for inspecting queue/stream depth across backends. +/// +/// Implementations derive the full set of queue names (principal, retry, +/// dead-letter) from the given `name` using backend-specific conventions: +/// +/// - **Redis**: principal = `{name}`, dead = `{name}:dead` +/// - **AMQP**: principal = `{name}`, retry = `{name}.retry`, dead = `{name}.error` +/// +/// When `group` is `Some`, Redis backends also query `XINFO GROUPS` to +/// populate `pending` and `lag` fields on the returned [`QueueDepths`]. +#[async_trait] +pub trait QueueInspector: Send + Sync { + type Error: std::error::Error + Send + Sync + 'static; + + /// Returns the current message counts for the queue group identified by `name`. + /// + /// - `name`: logical queue/stream name (e.g., `"ethereum.blocks"`). + /// - `group`: optional consumer group name. When provided, Redis populates + /// `pending` (PEL count) and `lag` (undelivered count) on the result. + /// AMQP ignores this parameter. + async fn queue_depths( + &self, + name: &str, + group: Option<&str>, + ) -> Result; + + /// Returns `true` if the consumer group has **no** messages to receive — + /// neither pending (PEL) nor undelivered (lag). + /// + /// This is a fast, single-round-trip check designed for the seed-message + /// decision at startup. It answers: "will the consumer block forever if + /// I don't publish a seed?" + /// + /// - **Redis**: single `XINFO GROUPS` call → checks `pending` and `lag`. + /// Returns `true` if stream/group doesn't exist. + /// - **AMQP**: single passive `queue_declare` → checks `message_count`. + /// Returns `true` if queue doesn't exist. + async fn is_empty(&self, name: &str, group: &str) -> Result; + + /// Returns `true` if the consumer group is caught up — either fully idle + /// or has at most one message currently being consumed (pending <= 1, lag == 0). + /// + /// Designed for deduplication guards: when the prefetch count is 1, a single + /// pending entry means a consumer is already processing the message. Callers + /// can use this to skip duplicate work rather than enqueueing an overlapping task. + /// + /// - **Redis**: single `XINFO GROUPS` call — returns `true` when `pending` is + /// 0 or 1 **and** `lag` is 0. Returns `true` if stream/group doesn't exist. + /// - **AMQP**: equivalent to [`is_empty`](Self::is_empty) — AMQP has no + /// pending/lag distinction, so returns `true` when `message_count == 0`. + async fn is_empty_or_pending(&self, name: &str, _group: &str) -> Result; + + /// Returns `true` if the queue or stream identified by `name` exists. + /// + /// - **Redis**: checks key type via `TYPE` command — returns `true` only + /// for keys of type `stream`. + /// - **AMQP**: passive `queue_declare` — returns `true` if the broker + /// acknowledges the queue. Returns `false` on `NOT_FOUND` (404). + /// + /// Does **not** check consumer group existence — only the underlying + /// queue/stream. + async fn exists(&self, name: &str) -> Result; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn queue_depths_total() { + let depths = QueueDepths { + principal: 10, + retry: Some(5), + dead_letter: 2, + pending: None, + lag: None, + }; + assert_eq!(depths.total(), 17); + + let no_retry = QueueDepths { + principal: 10, + retry: None, + dead_letter: 2, + pending: None, + lag: None, + }; + assert_eq!(no_retry.total(), 12); + } + + #[test] + fn queue_depths_is_empty() { + let empty = QueueDepths { + principal: 0, + retry: None, + dead_letter: 0, + pending: None, + lag: None, + }; + assert!(empty.is_empty()); + + let non_empty = QueueDepths { + principal: 1, + retry: None, + dead_letter: 0, + pending: None, + lag: None, + }; + assert!(!non_empty.is_empty()); + } + + #[test] + fn has_pending_work_with_pending() { + let depths = QueueDepths { + principal: 5, + retry: None, + dead_letter: 0, + pending: Some(2), + lag: Some(0), + }; + assert!(depths.has_pending_work()); + } + + #[test] + fn has_pending_work_with_lag() { + let depths = QueueDepths { + principal: 5, + retry: None, + dead_letter: 0, + pending: Some(0), + lag: Some(3), + }; + assert!(depths.has_pending_work()); + } + + #[test] + fn has_pending_work_none_falls_back_to_principal() { + let depths = QueueDepths { + principal: 5, + retry: None, + dead_letter: 0, + pending: None, + lag: None, + }; + assert!(depths.has_pending_work()); + + let empty = QueueDepths { + principal: 0, + retry: None, + dead_letter: 0, + pending: None, + lag: None, + }; + assert!(!empty.has_pending_work()); + } + + #[test] + fn has_pending_work_no_work() { + let depths = QueueDepths { + principal: 10, + retry: None, + dead_letter: 0, + pending: Some(0), + lag: Some(0), + }; + // principal=10 but all delivered and ACKed (just not trimmed yet) + assert!(!depths.has_pending_work()); + } + + #[test] + fn has_pending_work_redis_6_unknown_lag_is_conservative() { + // Redis < 7.0: pending is known (0), but lag is not available. + // Must return false so the caller publishes a seed as precaution. + let depths = QueueDepths { + principal: 50, // stream has entries, but all may be consumed + retry: None, + dead_letter: 0, + pending: Some(0), + lag: None, // Redis < 7.0: no lag field + }; + assert!( + !depths.has_pending_work(), + "unknown lag with empty PEL must return false (conservative seed)" + ); + } + + #[test] + fn has_pending_work_redis_6_pending_nonzero_still_true() { + // Redis < 7.0: lag unknown, but PEL has entries → drain will deliver. + let depths = QueueDepths { + principal: 50, + retry: None, + dead_letter: 0, + pending: Some(3), + lag: None, + }; + assert!( + depths.has_pending_work(), + "non-zero pending means drain phase will deliver" + ); + } +} diff --git a/listener/crates/shared/broker/src/traits/handler.rs b/listener/crates/shared/broker/src/traits/handler.rs new file mode 100644 index 0000000000..a059fae0dc --- /dev/null +++ b/listener/crates/shared/broker/src/traits/handler.rs @@ -0,0 +1,654 @@ +use async_trait::async_trait; +use serde::de::DeserializeOwned; +use std::{future::Future, marker::PhantomData, time::Duration}; +use thiserror::Error; + +use super::message::{Message, MessageMetadata}; + +/// Explicit routing decision returned by a handler on the success path. +/// +/// Combined with [`HandlerError`] variants on the `Err` arm, this gives full +/// control over the message lifecycle without side effects on the [`Message`] +/// itself — the handler declares intent via the return type. +/// +/// # Per-backend semantics +/// +/// | Variant | AMQP (RabbitMQ) | Redis Streams | +/// |---|---|---| +/// | `Ack` | `basic_ack` | `XACK` | +/// | `Nack` | `basic_nack(requeue: true)` → back to main queue | Leave in PEL (ClaimSweeper handles retry) | +/// | `Dead` | Publish to DLQ directly, then `basic_ack` | `XADD` to dead stream + `XACK` | +/// | `Delay(d)` | Publish to retry exchange with per-message `expiration` TTL, then `basic_ack` | Leave in PEL (no per-message TTL in Redis Streams; ClaimSweeper `claim_min_idle` governs timing) | +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum AckDecision { + /// Remove from queue — processing succeeded. + Ack, + /// Requeue to main queue (AMQP) / leave in PEL (Redis) — voluntary yield. + /// + /// Use when you want to requeue deliberately (e.g. idempotency guard saw + /// another consumer already handling this event) without indicating an error. + /// Does NOT trip the circuit breaker. Does NOT increment the retry counter. + Nack, + /// Skip all retries — route directly to the dead-letter queue / dead stream. + /// + /// Use for permanently unprocessable messages: unknown schema version, invalid + /// ABI encoding, malformed payload that can never succeed on retry. + Dead, + /// Requeue with a custom delay before the next attempt. + /// + /// **AMQP**: publishes to the retry exchange with a per-message `expiration` + /// property, overriding the queue-level TTL. + /// + /// **Redis**: no per-message TTL support in Redis Streams — treated as `Nack` + /// (message stays in PEL; ClaimSweeper `claim_min_idle` governs when it is reclaimed). + Delay(Duration), +} + +#[derive(Error, Debug)] +#[non_exhaustive] +pub enum HandlerError { + #[error("deserialization failed: {0}")] + Deserialization(#[from] serde_json::Error), + + #[error("handler execution failed: {0}")] + Execution(#[source] Box), + + /// Transient (infrastructure) failure — not the message's fault. + /// Triggers the circuit breaker when configured on the consumer. + #[error("transient failure (infrastructure): {0}")] + Transient(#[source] Box), +} + +impl HandlerError { + /// Wrap an infrastructure error as `Transient`. + /// + /// Use this for failures that are not the message's fault — database + /// connection lost, external API timeout, network error, etc. + /// These trip the circuit breaker when configured on the consumer. + /// + /// Works with `?` via `.map_err`: + /// + /// ```rust,ignore + /// db.save(&event).await.map_err(HandlerError::transient)?; + /// api.call().await.map_err(HandlerError::transient)?; + /// ``` + pub fn transient(e: impl std::error::Error + Send + Sync + 'static) -> Self { + Self::Transient(Box::new(e)) + } + + /// Wrap a logic error as permanent (`Execution`). + /// + /// Use this for failures that are the message's fault — invalid data, + /// business rule violation, ABI decoding error, etc. + /// These reset the circuit breaker's transient counter. + /// + /// Works with `?` via `.map_err`: + /// + /// ```rust,ignore + /// validate(&event).map_err(HandlerError::permanent)?; + /// abi_decode(&log).map_err(HandlerError::permanent)?; + /// ``` + pub fn permanent(e: impl std::error::Error + Send + Sync + 'static) -> Self { + Self::Execution(Box::new(e)) + } +} + +/// Backend-agnostic classification of a handler result. +/// +/// Used by prefetch-safe consumers to carry the outcome through the mpsc +/// channel without losing the routing decision that the circuit breaker +/// and ACK/NACK logic depend on. +/// +/// Constructed via `From>` — consumers +/// never build this directly. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HandlerOutcome { + /// Handler succeeded and wants to ACK. + Ack, + /// Handler succeeded but voluntarily requeues (no circuit breaker impact). + Nack, + /// Handler wants immediate dead-letter routing, bypassing retries. + Dead, + /// Handler wants a custom delay before the next attempt. + Delay(Duration), + /// Infrastructure failure — increments the circuit breaker's transient counter. + Transient, + /// Logic or deserialization failure — resets the transient counter. + Permanent, +} + +impl From> for HandlerOutcome { + fn from(result: Result) -> Self { + match result { + Ok(AckDecision::Ack) => Self::Ack, + Ok(AckDecision::Nack) => Self::Nack, + Ok(AckDecision::Dead) => Self::Dead, + Ok(AckDecision::Delay(d)) => Self::Delay(d), + Err(HandlerError::Transient(_)) => Self::Transient, + Err(_) => Self::Permanent, + } + } +} + +/// Queue-agnostic message handler trait. +/// +/// Both RMQ and Redis consumers call `handler.call(&msg)` where `msg` is a +/// backend-constructed [`Message`]. The handler never knows which backend +/// produced the message. +/// +/// Returns `Ok(AckDecision)` to explicitly control the message routing on the +/// success path, or `Err(HandlerError)` to signal failure semantics. +#[async_trait] +pub trait Handler: Send + Sync + 'static { + async fn call(&self, msg: &Message) -> Result; +} + +/// Handler wrapper that ignores the message payload and calls `F()`. +/// +/// Used for trigger/signal consumers where the message is just a wake-up +/// signal and the handler does not need any data from the payload. +pub struct AsyncHandlerNoArgs { + f: F, + _phantom: PhantomData, +} + +impl Clone for AsyncHandlerNoArgs { + fn clone(&self) -> Self { + Self { + f: self.f.clone(), + _phantom: PhantomData, + } + } +} + +impl AsyncHandlerNoArgs { + pub fn new(f: F) -> Self { + Self { + f, + _phantom: PhantomData, + } + } +} + +#[async_trait] +impl Handler for AsyncHandlerNoArgs +where + F: Fn() -> Fut + Send + Sync + 'static, + Fut: Future> + Send, + E: std::error::Error + Send + Sync + 'static, +{ + async fn call(&self, _msg: &Message) -> Result { + (self.f)() + .await + .map(|_| AckDecision::Ack) + .map_err(|e| HandlerError::Execution(Box::new(e))) + } +} + +/// Handler wrapper that deserializes `msg.payload` to `T` and calls `F(T)`. +/// +/// This is the most common handler style — the handler only cares about the +/// deserialized payload and does not need delivery metadata. +/// +/// The user-supplied closure returns `Result<(), E>` for ergonomics. The +/// wrapper maps `Ok(())` to `Ok(AckDecision::Ack)` automatically. To return +/// a different `AckDecision`, use a closure that returns `Result` +/// and implement [`Handler`] directly, or use [`AsyncHandlerWithContext`]. +/// +/// Replaces both `broker::AsyncHandlerWithArgs` and +/// `redis_broker::AsyncRedisHandlerPayloadOnly`. +pub struct AsyncHandlerPayloadOnly { + f: F, + _phantom: PhantomData<(T, E)>, +} + +impl Clone for AsyncHandlerPayloadOnly { + fn clone(&self) -> Self { + Self { + f: self.f.clone(), + _phantom: PhantomData, + } + } +} + +impl AsyncHandlerPayloadOnly { + pub fn new(f: F) -> Self { + Self { + f, + _phantom: PhantomData, + } + } +} + +#[async_trait] +impl Handler for AsyncHandlerPayloadOnly +where + F: Fn(T) -> Fut + Send + Sync + 'static, + Fut: Future> + Send, + T: DeserializeOwned + Send + Sync + 'static, + E: std::error::Error + Send + Sync + 'static, +{ + async fn call(&self, msg: &Message) -> Result { + let payload: T = serde_json::from_slice(&msg.payload)?; + (self.f)(payload) + .await + .map(|_| AckDecision::Ack) + .map_err(|e| HandlerError::Execution(Box::new(e))) + } +} + +/// Handler wrapper that deserializes `msg.payload` to `T` and calls `F(T)`, +/// where the closure returns `Result<(), HandlerError>` directly. +/// +/// Unlike [`AsyncHandlerPayloadOnly`] (which wraps all closure errors as +/// `HandlerError::Execution`), this handler **preserves** the error classification +/// returned by the closure. Use this when your handler needs to distinguish +/// transient (infrastructure) from permanent (logic) failures: +/// +/// ```rust,ignore +/// use broker::{AsyncHandlerPayloadClassified, HandlerError}; +/// +/// let handler = AsyncHandlerPayloadClassified::new(|block: BlockEvent| async move { +/// // Transient: infrastructure is broken, not the message. +/// db.save(&block).await.map_err(HandlerError::transient)?; +/// +/// // Permanent: the message itself is invalid. +/// verify_block(&block).map_err(HandlerError::permanent)?; +/// +/// Ok(()) +/// }); +/// ``` +pub struct AsyncHandlerPayloadClassified { + f: F, + _phantom: PhantomData, +} + +impl Clone for AsyncHandlerPayloadClassified { + fn clone(&self) -> Self { + Self { + f: self.f.clone(), + _phantom: PhantomData, + } + } +} + +impl AsyncHandlerPayloadClassified { + pub fn new(f: F) -> Self { + Self { + f, + _phantom: PhantomData, + } + } +} + +#[async_trait] +impl Handler for AsyncHandlerPayloadClassified +where + F: Fn(T) -> Fut + Send + Sync + 'static, + Fut: Future> + Send, + T: DeserializeOwned + Send + Sync + 'static, +{ + async fn call(&self, msg: &Message) -> Result { + let payload: T = serde_json::from_slice(&msg.payload)?; + (self.f)(payload).await.map(|_| AckDecision::Ack) + } +} + +/// Handler wrapper that deserializes `msg.payload` to `T` and calls +/// `F(T, MessageMetadata)`. +/// +/// Use this when the handler needs delivery metadata such as the message ID, +/// topic, or delivery count. The metadata is backend-agnostic. +/// +/// Replaces `redis_broker::AsyncRedisHandlerWithArgs` (which took +/// `RedisMessageContext` — now [`MessageMetadata`]). +pub struct AsyncHandlerWithContext { + f: F, + _phantom: PhantomData<(T, E)>, +} + +impl Clone for AsyncHandlerWithContext { + fn clone(&self) -> Self { + Self { + f: self.f.clone(), + _phantom: PhantomData, + } + } +} + +impl AsyncHandlerWithContext { + pub fn new(f: F) -> Self { + Self { + f, + _phantom: PhantomData, + } + } +} + +#[async_trait] +impl Handler for AsyncHandlerWithContext +where + F: Fn(T, MessageMetadata) -> Fut + Send + Sync + 'static, + Fut: Future> + Send, + T: DeserializeOwned + Send + Sync + 'static, + E: std::error::Error + Send + Sync + 'static, +{ + async fn call(&self, msg: &Message) -> Result { + let payload: T = serde_json::from_slice(&msg.payload)?; + (self.f)(payload, msg.metadata.clone()) + .await + .map(|_| AckDecision::Ack) + .map_err(|e| HandlerError::Execution(Box::new(e))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::error::Error; + use thiserror::Error as ThisError; + + #[derive(ThisError, Debug)] + #[error("Mock error: {0}")] + struct MockError(String); + + #[derive(serde::Deserialize, Debug, PartialEq)] + struct TestPayload { + value: i32, + } + + fn make_message(data: &[u8]) -> Message { + Message { + payload: data.to_vec(), + metadata: MessageMetadata::new("test-id-123", "test.topic", 1), + } + } + + // ── AsyncHandlerPayloadOnly tests ────────────────────────── + + #[tokio::test] + async fn payload_only_success() { + let handler = AsyncHandlerPayloadOnly::new(|payload: TestPayload| async move { + assert_eq!(payload.value, 42); + Ok::<(), MockError>(()) + }); + + let msg = make_message(br#"{"value": 42}"#); + let result = handler.call(&msg).await; + assert!(matches!(result, Ok(AckDecision::Ack))); + } + + #[tokio::test] + async fn payload_only_deserialization_error() { + let handler = + AsyncHandlerPayloadOnly::new(|_p: TestPayload| async move { Ok::<(), MockError>(()) }); + + let msg = make_message(br#"{"invalid": "json"}"#); + let result = handler.call(&msg).await; + + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!(matches!(err, HandlerError::Deserialization(_))); + assert!(err.to_string().contains("deserialization failed")); + } + + #[tokio::test] + async fn payload_only_execution_error() { + let handler = AsyncHandlerPayloadOnly::new(|_p: TestPayload| async move { + Err::<(), MockError>(MockError("handler failed".to_string())) + }); + + let msg = make_message(br#"{"value": 42}"#); + let result = handler.call(&msg).await; + + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!(matches!(err, HandlerError::Execution(_))); + assert!(err.to_string().contains("handler execution failed")); + + let source = err.source(); + assert!(source.is_some()); + assert!(source.unwrap().to_string().contains("handler failed")); + } + + // ── AsyncHandlerWithContext tests ────────────────────────── + + #[tokio::test] + async fn with_context_success() { + let handler = + AsyncHandlerWithContext::new(|payload: TestPayload, ctx: MessageMetadata| async move { + assert_eq!(payload.value, 42); + assert_eq!(ctx.id, "test-id-123"); + assert_eq!(ctx.topic, "test.topic"); + assert_eq!(ctx.delivery_count, 1); + Ok::<(), MockError>(()) + }); + + let msg = make_message(br#"{"value": 42}"#); + let result = handler.call(&msg).await; + assert!(matches!(result, Ok(AckDecision::Ack))); + } + + #[tokio::test] + async fn with_context_deserialization_error() { + let handler = + AsyncHandlerWithContext::new(|_p: TestPayload, _ctx: MessageMetadata| async move { + Ok::<(), MockError>(()) + }); + + let msg = make_message(br#"{"bad": "data"}"#); + let result = handler.call(&msg).await; + + assert!(result.is_err()); + assert!(matches!( + result.unwrap_err(), + HandlerError::Deserialization(_) + )); + } + + #[tokio::test] + async fn with_context_execution_error() { + let handler = + AsyncHandlerWithContext::new(|_p: TestPayload, _ctx: MessageMetadata| async move { + Err::<(), MockError>(MockError("ctx handler failed".to_string())) + }); + + let msg = make_message(br#"{"value": 42}"#); + let result = handler.call(&msg).await; + + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!(matches!(err, HandlerError::Execution(_))); + + let source = err.source(); + assert!(source.is_some()); + assert!(source.unwrap().to_string().contains("ctx handler failed")); + } + + // ── AsyncHandlerNoArgs tests ────────────────────────── + + #[tokio::test] + async fn no_args_success() { + let handler = AsyncHandlerNoArgs::new(|| async move { Ok::<(), MockError>(()) }); + + let msg = make_message(b"anything"); + let result = handler.call(&msg).await; + assert!(matches!(result, Ok(AckDecision::Ack))); + } + + #[tokio::test] + async fn no_args_ignores_payload() { + let handler = AsyncHandlerNoArgs::new(|| async move { Ok::<(), MockError>(()) }); + + // Even invalid JSON should succeed — payload is ignored + let msg = make_message(b"not json at all!!!"); + let result = handler.call(&msg).await; + assert!(matches!(result, Ok(AckDecision::Ack))); + } + + #[tokio::test] + async fn no_args_execution_error() { + let handler = AsyncHandlerNoArgs::new(|| async move { + Err::<(), MockError>(MockError("handler failed".to_string())) + }); + + let msg = make_message(b"{}"); + let result = handler.call(&msg).await; + + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!(matches!(err, HandlerError::Execution(_))); + assert!(err.to_string().contains("handler execution failed")); + + let source = err.source(); + assert!(source.is_some()); + assert!(source.unwrap().to_string().contains("handler failed")); + } + + // ── HandlerError::Transient tests ────────────────────────── + + #[tokio::test] + async fn transient_error_variant() { + let err = HandlerError::Transient(Box::new(MockError("db down".to_string()))); + assert!(matches!(err, HandlerError::Transient(_))); + assert!(err.to_string().contains("transient failure")); + + let source = err.source(); + assert!(source.is_some()); + assert!(source.unwrap().to_string().contains("db down")); + } + + #[tokio::test] + async fn transient_vs_execution_are_distinct() { + let transient = HandlerError::Transient(Box::new(MockError("infra".to_string()))); + let execution = HandlerError::Execution(Box::new(MockError("logic".to_string()))); + + assert!(matches!(transient, HandlerError::Transient(_))); + assert!(!matches!(transient, HandlerError::Execution(_))); + + assert!(matches!(execution, HandlerError::Execution(_))); + assert!(!matches!(execution, HandlerError::Transient(_))); + } + + // ── Convenience constructor tests ────────────────────────── + + #[test] + fn transient_constructor_wraps_error() { + let err = HandlerError::transient(MockError("db down".to_string())); + assert!(matches!(err, HandlerError::Transient(_))); + assert!(err.to_string().contains("transient failure")); + } + + #[test] + fn permanent_constructor_wraps_error() { + let err = HandlerError::permanent(MockError("bad data".to_string())); + assert!(matches!(err, HandlerError::Execution(_))); + assert!(err.to_string().contains("handler execution failed")); + } + + // ── AckDecision / HandlerOutcome tests ───────────────────── + + #[test] + fn handler_outcome_from_ack() { + let outcome = HandlerOutcome::from(Ok::(AckDecision::Ack)); + assert_eq!(outcome, HandlerOutcome::Ack); + } + + #[test] + fn handler_outcome_from_nack() { + let outcome = HandlerOutcome::from(Ok::(AckDecision::Nack)); + assert_eq!(outcome, HandlerOutcome::Nack); + } + + #[test] + fn handler_outcome_from_dead() { + let outcome = HandlerOutcome::from(Ok::(AckDecision::Dead)); + assert_eq!(outcome, HandlerOutcome::Dead); + } + + #[test] + fn handler_outcome_from_delay() { + let d = Duration::from_secs(10); + let outcome = HandlerOutcome::from(Ok::(AckDecision::Delay(d))); + assert_eq!(outcome, HandlerOutcome::Delay(d)); + } + + #[test] + fn handler_outcome_from_transient_err() { + let outcome = HandlerOutcome::from(Err::( + HandlerError::Transient(Box::new(MockError("infra".to_string()))), + )); + assert_eq!(outcome, HandlerOutcome::Transient); + } + + #[test] + fn handler_outcome_from_permanent_err() { + let outcome = HandlerOutcome::from(Err::( + HandlerError::Execution(Box::new(MockError("logic".to_string()))), + )); + assert_eq!(outcome, HandlerOutcome::Permanent); + } + + // ── AsyncHandlerPayloadClassified tests ─────────────────── + + #[tokio::test] + async fn classified_success() { + let handler = AsyncHandlerPayloadClassified::new(|payload: TestPayload| async move { + assert_eq!(payload.value, 42); + Ok(()) + }); + + let msg = make_message(br#"{"value": 42}"#); + let result = handler.call(&msg).await; + assert!(matches!(result, Ok(AckDecision::Ack))); + } + + #[tokio::test] + async fn classified_deserialization_error() { + let handler = AsyncHandlerPayloadClassified::new(|_p: TestPayload| async move { Ok(()) }); + + let msg = make_message(br#"{"invalid": "json"}"#); + let result = handler.call(&msg).await; + + assert!(result.is_err()); + assert!(matches!( + result.unwrap_err(), + HandlerError::Deserialization(_) + )); + } + + #[tokio::test] + async fn classified_transient_error_preserved() { + let handler = AsyncHandlerPayloadClassified::new(|_p: TestPayload| async move { + Err(HandlerError::transient(MockError("db down".to_string()))) + }); + + let msg = make_message(br#"{"value": 42}"#); + let result = handler.call(&msg).await; + + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!( + matches!(err, HandlerError::Transient(_)), + "expected Transient, got {err:?}" + ); + } + + #[tokio::test] + async fn classified_permanent_error_preserved() { + let handler = AsyncHandlerPayloadClassified::new(|_p: TestPayload| async move { + Err(HandlerError::permanent(MockError("bad block".to_string()))) + }); + + let msg = make_message(br#"{"value": 42}"#); + let result = handler.call(&msg).await; + + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!( + matches!(err, HandlerError::Execution(_)), + "expected Execution, got {err:?}" + ); + } +} diff --git a/listener/crates/shared/broker/src/traits/message.rs b/listener/crates/shared/broker/src/traits/message.rs new file mode 100644 index 0000000000..1a2ba660f4 --- /dev/null +++ b/listener/crates/shared/broker/src/traits/message.rs @@ -0,0 +1,42 @@ +use std::collections::HashMap; + +/// Unified message envelope for all queue backends. +/// +/// Replaces `&[u8]` (RMQ) and `&RedisMessage` (Redis) with a single type that +/// both backends construct before calling the handler. +#[derive(Debug, Clone)] +pub struct Message { + /// Raw payload bytes — the serialized application data. + pub payload: Vec, + /// Backend-agnostic delivery metadata. + pub metadata: MessageMetadata, +} + +/// Delivery metadata attached to every [`Message`]. +/// +/// Each backend populates these fields from its native concepts: +/// - RMQ: `id` = delivery tag, `topic` = queue name, `delivery_count` = x-death count + 1 +/// - Redis: `id` = stream entry ID, `topic` = stream name, `delivery_count` = XPENDING delivery count +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MessageMetadata { + /// Unique message identifier within the backend. + pub id: String, + /// The topic/queue/stream this message was consumed from. + pub topic: String, + /// How many times this message has been delivered (1 = first delivery). + pub delivery_count: u64, + /// Arbitrary key-value headers (RMQ properties or Redis entry fields). + pub headers: HashMap, +} + +impl MessageMetadata { + /// Create metadata with no headers. + pub fn new(id: impl Into, topic: impl Into, delivery_count: u64) -> Self { + Self { + id: id.into(), + topic: topic.into(), + delivery_count, + headers: HashMap::new(), + } + } +} diff --git a/listener/crates/shared/broker/src/traits/mod.rs b/listener/crates/shared/broker/src/traits/mod.rs new file mode 100644 index 0000000000..37fb18845c --- /dev/null +++ b/listener/crates/shared/broker/src/traits/mod.rs @@ -0,0 +1,17 @@ +pub mod circuit_breaker; +pub mod consumer; +pub mod depth; +pub mod handler; +pub mod message; +pub mod publisher; + +// Re-exports matching the old `mq` crate's public API +pub use circuit_breaker::{CircuitBreaker, CircuitBreakerConfig}; +pub use consumer::{Consumer, RetryPolicy}; +pub use depth::{QueueDepths, QueueInspector}; +pub use handler::{ + AckDecision, AsyncHandlerNoArgs, AsyncHandlerPayloadClassified, AsyncHandlerPayloadOnly, + AsyncHandlerWithContext, Handler, HandlerError, HandlerOutcome, +}; +pub use message::{Message, MessageMetadata}; +pub use publisher::{DynPublishError, DynPublisher, Publisher}; diff --git a/listener/crates/shared/broker/src/traits/publisher.rs b/listener/crates/shared/broker/src/traits/publisher.rs new file mode 100644 index 0000000000..17312bc89a --- /dev/null +++ b/listener/crates/shared/broker/src/traits/publisher.rs @@ -0,0 +1,139 @@ +use std::{fmt::Debug, future::Future, pin::Pin, sync::Arc}; + +use async_trait::async_trait; +use serde::Serialize; + +/// Queue-agnostic publisher trait. +/// +/// Both `RmqPublisher` and `RedisPublisher` implement this trait. +/// +/// - `topic` maps to a routing key (RMQ, with the exchange fixed at construction) +/// or a stream name (Redis). +/// - `shutdown` has a default no-op; Redis overrides it to cancel background trimmers. +#[async_trait] +pub trait Publisher: Send + Sync + 'static { + type Error: std::error::Error + Send + Sync + 'static; + + /// Publish a single payload to the given topic. + async fn publish( + &self, + topic: &str, + payload: &T, + ) -> Result<(), Self::Error>; + + /// Graceful shutdown — cancel background tasks if any. + /// + /// Default is a no-op. Redis overrides to cancel background trimmers. + async fn shutdown(&self) {} +} + +// ── Type erasure internals ──────────────────────────────────────────────────── + +type BoxFuture<'a, T> = Pin + Send + 'a>>; + +/// Erased error type used by `DynPublisher`. +pub type DynPublishError = Box; + +/// Internal dyn-safe trait. Uses `&[u8]` instead of `&T: Serialize` so Rust +/// can build a vtable for it. +trait ErasedPublisher: Send + Sync + 'static { + fn publish_bytes<'a>( + &'a self, + topic: &'a str, + payload: &'a [u8], + ) -> BoxFuture<'a, Result<(), DynPublishError>>; + + fn shutdown<'a>(&'a self) -> BoxFuture<'a, ()>; +} + +/// Blanket impl: every `Publisher` is automatically an `ErasedPublisher`. +/// +/// Serialization (T → JSON bytes) happens here, before calling the inner +/// backend, so the backend only ever sees `&[u8]` — no generics cross the +/// vtable boundary. +impl ErasedPublisher for P { + fn publish_bytes<'a>( + &'a self, + topic: &'a str, + payload: &'a [u8], + ) -> BoxFuture<'a, Result<(), DynPublishError>> { + Box::pin(async move { + // Treat the bytes as an already-serialized JSON value so the + // backend re-encodes them without double-serialization. + let raw: &serde_json::value::RawValue = + serde_json::from_slice(payload).map_err(|e| Box::new(e) as DynPublishError)?; + self.publish(topic, &raw) + .await + .map_err(|e| Box::new(e) as DynPublishError) + }) + } + + fn shutdown<'a>(&'a self) -> BoxFuture<'a, ()> { + Box::pin(Publisher::shutdown(self)) + } +} + +// ── Public type-erased handle ───────────────────────────────────────────────── + +/// A type-erased, cheaply cloneable publisher that can be stored alongside +/// publishers of different backends in a `HashMap` or `Vec`. +/// +/// Wraps any `impl Publisher` and serializes payloads to JSON bytes before +/// delegating to the underlying backend. The `Error` type is erased to +/// [`DynPublishError`] (`Box`). +/// +/// # Multi-chain registry example +/// +/// ```rust,ignore +/// use mq::DynPublisher; +/// use mq_amqp::RmqPublisher; +/// use mq_redis::RedisPublisher; +/// use std::collections::HashMap; +/// +/// // Construction — backend-specific, done once at startup +/// let mut publishers: HashMap = HashMap::new(); +/// +/// publishers.insert( +/// "ethereum".into(), +/// DynPublisher::new(RmqPublisher::connect(url, "ethereum.events").await), +/// ); +/// publishers.insert( +/// "polygon".into(), +/// DynPublisher::new(RedisPublisher::connect("redis://...").await?), +/// ); +/// +/// // Usage — fully agnostic, same call for any backend +/// publishers["ethereum"].publish("blocks.new", &block).await?; +/// publishers["polygon"].publish("blocks.new", &block).await?; +/// ``` +#[derive(Clone)] +pub struct DynPublisher(Arc); + +impl DynPublisher { + /// Wrap any `impl Publisher` in a type-erased handle. + /// + /// The resulting `DynPublisher` is `Clone` and `Send + Sync`. + pub fn new(publisher: impl Publisher) -> Self { + Self(Arc::new(publisher)) + } + + /// Publish a single payload — serializes to JSON then delegates to the backend. + /// + /// `topic` maps to a routing key (AMQP) or stream name (Redis), exactly as + /// it does on the underlying `Publisher` trait. + pub async fn publish( + &self, + topic: &str, + payload: &T, + ) -> Result<(), DynPublishError> { + let bytes = serde_json::to_vec(payload).map_err(|e| Box::new(e) as DynPublishError)?; + self.0.publish_bytes(topic, &bytes).await + } + + /// Graceful shutdown — delegates to the backend. + /// + /// For Redis, this cancels background stream trimmers. For AMQP, this is a no-op. + pub async fn shutdown(&self) { + self.0.shutdown().await; + } +} diff --git a/listener/crates/shared/broker/tests/amqp_e2e.rs b/listener/crates/shared/broker/tests/amqp_e2e.rs new file mode 100644 index 0000000000..62d8c41f45 --- /dev/null +++ b/listener/crates/shared/broker/tests/amqp_e2e.rs @@ -0,0 +1,1894 @@ +#![cfg(feature = "amqp")] +//! End-to-end integration tests for `broker::amqp`. +//! +//! Each test follows the three-step pattern from the `broker` README: +//! 1. Define a payload type. +//! 2. Write a handler with `broker::AsyncHandlerPayloadOnly`. +//! 3. Pick the `broker::amqp` backend, build config, and call `consumer.run_*()`. +//! +//! All tests share a single RabbitMQ container (`e2e-rabbitmq`) using +//! `ReuseDirective::CurrentSession` so the container survives the handle drop +//! inside the `OnceCell` init closure. Each test uses unique exchange/queue +//! names to avoid cross-contamination. +//! +//! ⚠️ Always run via Make — the Makefile removes the named container before +//! and after the suite so it is not left running between runs: +//! +//! ```bash +//! make test-e2e-amqp +//! ``` +//! +//! Running `cargo test -p broker --features amqp --test amqp_e2e -- --include-ignored` directly +//! will leave `e2e-rabbitmq` running after the suite finishes. + +use std::{ + collections::{HashMap, HashSet}, + future::Future, + sync::{ + Arc, Mutex, + atomic::{AtomicBool, AtomicU32, Ordering}, + }, + time::Duration, +}; + +// Traits and shared types come from `broker` — the backend-agnostic layer. +use broker::traits::{Consumer, DynPublisher, Publisher}; +use broker::{AckDecision, AsyncHandlerPayloadOnly, AsyncHandlerWithContext, MessageMetadata}; +// Backend-specific construction (config builder, topology, concrete types) comes from `broker::amqp`. +use broker::amqp::{ + ConnectionManager, ConsumerConfigBuilder, ExchangeManager, ExchangeTopology, PublisherError, + RmqConsumer, RmqPublisher, +}; + +// ── Step 1: shared test payload ─────────────────────────────────────────────── + +#[derive(serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq)] +struct BlockEvent { + block_number: u64, +} + +#[derive(serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq)] +#[serde(tag = "kind", rename_all = "snake_case")] +enum ListenerEvent { + Block { + chain_id: u64, + block_number: u64, + }, + WatchRegister { + chain_id: u64, + consumer_id: String, + contract_addresses: Vec, + }, +} + +/// Publish a synthetic block stream at a fixed rate. +/// +/// This simulates the listener's polling loop producing canonical blocks and +/// publishing them to a chain-specific RMQ exchange through `DynPublisher`. +async fn publish_blocks_at_rps( + publisher: &DynPublisher, + routing_key: &str, + chain_id: u64, + start_block: u64, + count: u64, + rps: u64, +) -> Result<(), broker::traits::DynPublishError> { + let per_message_delay = Duration::from_millis((1000 / rps.max(1)).max(1)); + for i in 0..count { + publisher + .publish( + routing_key, + &ListenerEvent::Block { + chain_id, + block_number: start_block + i, + }, + ) + .await?; + tokio::time::sleep(per_message_delay).await; + } + Ok(()) +} + +// ── Shared container ───────────────────────────────────────────────────────── + +use tokio::sync::OnceCell; + +static RABBITMQ_URL: OnceCell = OnceCell::const_new(); + +/// Lazily start a single RabbitMQ container shared across all tests. +/// +/// `ReuseDirective::CurrentSession` prevents the `Drop` impl from stopping +/// the container when the `ContainerAsync` handle goes out of scope at the +/// end of the `OnceCell` init closure. The container therefore stays alive +/// for all tests in the suite. Cleanup is handled by `make test-e2e-amqp` +/// via `docker rm -f e2e-rabbitmq`. +async fn shared_rabbitmq_url() -> &'static str { + RABBITMQ_URL + .get_or_init(|| async { + use testcontainers::core::{ImageExt, ReuseDirective}; + use testcontainers::runners::AsyncRunner; + use testcontainers_modules::rabbitmq::RabbitMq; + + let container = RabbitMq::default() + .with_container_name("e2e-rabbitmq") + .with_reuse(ReuseDirective::CurrentSession) + .start() + .await + .unwrap(); + let port = container.get_host_port_ipv4(5672).await.unwrap(); + format!("amqp://guest:guest@127.0.0.1:{port}/%2f") + }) + .await + .as_str() +} + +// ── Generic Consumer trait helper ───────────────────────────────────────────── + +/// Assert that a single message published via `publish_fn` is received exactly once. +/// +/// Bound on `broker::traits::Consumer` — exercises the trait directly, independent of which +/// backend (`RmqConsumer`, `RedisConsumer`, …) is passed in. +async fn assert_simple_roundtrip(consumer: C, config: C::PrefetchConfig, publish_fn: F) +where + C: Consumer + 'static, + F: Future, +{ + let count = Arc::new(AtomicU32::new(0)); + let count_clone = count.clone(); + + // Step 2: handler — from broker + let handler = AsyncHandlerPayloadOnly::new(move |_: serde_json::Value| { + let c = count_clone.clone(); + async move { + c.fetch_add(1, Ordering::SeqCst); + Ok::<(), std::io::Error>(()) + } + }); + + // Step 3: run consumer in background + let handle = tokio::spawn(async move { + let _ = consumer.run(config, handler).await; + }); + + // Give the consumer time to start and register with RabbitMQ. + tokio::time::sleep(Duration::from_millis(600)).await; + + publish_fn.await; + + // Poll until the handler is called or 8 s elapse. + let deadline = tokio::time::Instant::now() + Duration::from_secs(8); + while count.load(Ordering::SeqCst) == 0 && tokio::time::Instant::now() < deadline { + tokio::time::sleep(Duration::from_millis(100)).await; + } + + handle.abort(); + assert_eq!( + count.load(Ordering::SeqCst), + 1, + "expected handler to be called exactly once" + ); +} + +// ── Test 1: run_simple roundtrip ────────────────────────────────────────────── + +/// Pre-declare topology, publish one message, consumer receives and ACKs it. +/// Exercises the generic `broker::traits::Consumer` trait via `assert_simple_roundtrip`. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_run_simple_roundtrip() { + let url = shared_rabbitmq_url().await; + + let exchange = "test.exchange"; + let queue = "test.queue"; + let routing_key = "test.key"; + + let topology = ExchangeTopology::from_prefix(exchange); + ExchangeManager::with_addr(url) + .declare_topology(&topology) + .await + .unwrap(); + + // Step 3 (backend): publisher + consumer from broker_amqp + // Exchange is fixed at construction; topic = AMQP routing key (broker::traits::Publisher trait). + let publisher = RmqPublisher::connect(url, exchange).await; + + let config = ConsumerConfigBuilder::new() + .with_topology(&topology) + .queue(queue) + .routing_key(routing_key) + .consumer_tag("e2e-consumer") + .max_retries(3) + .retry_delay(Duration::from_millis(200)) + .prefetch_count(10) + .build_prefetch() + .unwrap(); + + let consumer = RmqConsumer::connect(url).await.unwrap(); + + assert_simple_roundtrip(consumer, config, async move { + // broker::traits::Publisher trait — topic = routing_key; exchange is fixed in the publisher. + publisher + .publish(routing_key, &BlockEvent { block_number: 42 }) + .await + .unwrap(); + }) + .await; +} + +// ── Test 2: run_with_retry — handler fails then succeeds ────────────────────── + +/// `run_with_retry` declares its own retry/DLX topology. The handler fails on +/// the first two deliveries and succeeds on the third. RabbitMQ re-delivers via +/// DLX + TTL after each failure. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_run_with_retry_eventually_succeeds() { + let url = shared_rabbitmq_url().await; + + // AMQP requires exchanges to be declared before queue_bind — done here before consumer starts. + let topology = ExchangeTopology { + main: "retry.exchange".to_string(), + retry: "retry.exchange.retry".to_string(), + dlx: "retry.exchange.dlx".to_string(), + }; + ExchangeManager::with_addr(url) + .declare_topology(&topology) + .await + .unwrap(); + + // Step 3 (backend): config from broker_amqp; short retry_delay so the test finishes quickly. + let config = ConsumerConfigBuilder::new() + .exchange("retry.exchange") + .queue("retry.queue") + .routing_key("retry.key") + .consumer_tag("retry-consumer") + .retry_exchange("retry.exchange.retry") + .dead_exchange("retry.exchange.dlx") + .max_retries(5) + .retry_delay(Duration::from_millis(500)) + .prefetch_count(10) + .build_prefetch() + .unwrap(); + + // Step 2: handler — from broker; fails on the first two calls, succeeds on the third + let call_count = Arc::new(AtomicU32::new(0)); + let call_count_clone = call_count.clone(); + let handler = AsyncHandlerPayloadOnly::new(move |_: BlockEvent| { + let c = call_count_clone.clone(); + async move { + let prev = c.fetch_add(1, Ordering::SeqCst); + if prev < 2 { + Err(std::io::Error::other("simulated failure")) + } else { + Ok(()) + } + } + }); + + // Step 3 (backend): consumer from broker_amqp, run in background + let consumer = RmqConsumer::connect(url).await.unwrap(); + let consumer_handle = tokio::spawn(async move { + let _ = consumer.run(config, handler).await; + }); + + // Give the consumer time to start and declare all queues + bindings. + tokio::time::sleep(Duration::from_millis(800)).await; + + // Publish via broker::traits::Publisher — topic = routing_key; exchange fixed at construction. + let publisher = RmqPublisher::connect(url, "retry.exchange").await; + publisher + .publish("retry.key", &BlockEvent { block_number: 1 }) + .await + .unwrap(); + + // Poll for up to 20 s — each retry takes retry_delay (500 ms) + broker round-trip. + let deadline = tokio::time::Instant::now() + Duration::from_secs(20); + while call_count.load(Ordering::SeqCst) < 3 && tokio::time::Instant::now() < deadline { + tokio::time::sleep(Duration::from_millis(200)).await; + } + + consumer_handle.abort(); + assert!( + call_count.load(Ordering::SeqCst) >= 3, + "handler should be called at least 3 times (2 failures + 1 success), got {}", + call_count.load(Ordering::SeqCst) + ); +} + +// ── Test 3: multi-publish roundtrip ─────────────────────────────────────────── + +/// Publish 3 messages sequentially; consumer receives all 3. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_multi_publish_roundtrip() { + let url = shared_rabbitmq_url().await; + + let exchange = "batch.exchange"; + let queue = "batch.queue"; + let routing_key = "batch.key"; + + let topology = ExchangeTopology::from_prefix(exchange); + ExchangeManager::with_addr(url) + .declare_topology(&topology) + .await + .unwrap(); + + // Step 3 (backend): config from broker_amqp + let config = ConsumerConfigBuilder::new() + .with_topology(&topology) + .queue(queue) + .routing_key(routing_key) + .consumer_tag("batch-consumer") + .max_retries(3) + .retry_delay(Duration::from_millis(200)) + .prefetch_count(10) + .build_prefetch() + .unwrap(); + + // Step 2: handler — from broker + let count = Arc::new(AtomicU32::new(0)); + let count_clone = count.clone(); + let handler = AsyncHandlerPayloadOnly::new(move |_: BlockEvent| { + let c = count_clone.clone(); + async move { + c.fetch_add(1, Ordering::SeqCst); + Ok::<(), std::io::Error>(()) + } + }); + + // Step 3 (backend): consumer from broker_amqp, run in background + let consumer = RmqConsumer::connect(url).await.unwrap(); + let consumer_handle = tokio::spawn(async move { + let _ = consumer.run(config, handler).await; + }); + + // Give consumer time to start. + tokio::time::sleep(Duration::from_millis(600)).await; + + // Publish 3 messages via broker::traits::Publisher — topic = routing_key; exchange fixed at construction. + let publisher = RmqPublisher::connect(url, exchange).await; + let events = vec![ + BlockEvent { block_number: 1 }, + BlockEvent { block_number: 2 }, + BlockEvent { block_number: 3 }, + ]; + for event in &events { + publisher.publish(routing_key, event).await.unwrap(); + } + + // Poll until all 3 are received. + let deadline = tokio::time::Instant::now() + Duration::from_secs(8); + while count.load(Ordering::SeqCst) < 3 && tokio::time::Instant::now() < deadline { + tokio::time::sleep(Duration::from_millis(100)).await; + } + + consumer_handle.abort(); + assert_eq!( + count.load(Ordering::SeqCst), + 3, + "all 3 batch messages should be received" + ); +} + +// ── Test 4: AckDecision::Dead — bypasses all retry cycles ──────────────────── + +/// Verifies that a handler returning `AckDecision::Dead` routes the message +/// directly to the `.error` queue on the **first delivery**, without burning +/// any retry cycles. +/// +/// Without `AckDecision::Dead`, the only way to reach the error queue was to +/// exhaust `max_retries`. This test confirms the fast-path works. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_ack_decision_dead_skips_retry() { + let url = shared_rabbitmq_url().await; + + let topology = ExchangeTopology { + main: "dead.exchange".to_string(), + retry: "dead.exchange.retry".to_string(), + dlx: "dead.exchange.dlx".to_string(), + }; + ExchangeManager::with_addr(url) + .declare_topology(&topology) + .await + .unwrap(); + + let config = ConsumerConfigBuilder::new() + .exchange("dead.exchange") + .queue("dead.queue") + .routing_key("dead.key") + .consumer_tag("dead-consumer") + .retry_exchange("dead.exchange.retry") + .dead_exchange("dead.exchange.dlx") + // max_retries=5 but handler should never retry at all + .max_retries(5) + .retry_delay(Duration::from_millis(200)) + .prefetch_count(10) + .build_prefetch() + .unwrap(); + + let call_count = Arc::new(AtomicU32::new(0)); + + // Custom Handler impl that always returns AckDecision::Dead. + // AsyncHandlerPayloadOnly always maps Ok(()) → Ack, so we implement Handler directly + // to return Dead without going through the error path. + struct DeadHandler(Arc); + + #[async_trait::async_trait] + impl broker::Handler for DeadHandler { + async fn call(&self, _msg: &broker::Message) -> Result { + self.0.fetch_add(1, Ordering::SeqCst); + Ok(AckDecision::Dead) + } + } + + let dead_handler = DeadHandler(call_count.clone()); + + let consumer = RmqConsumer::connect(url).await.unwrap(); + let consumer_handle = tokio::spawn(async move { + let _ = consumer.run(config, dead_handler).await; + }); + + tokio::time::sleep(Duration::from_millis(800)).await; + + let publisher = RmqPublisher::connect(url, "dead.exchange").await; + publisher + .publish("dead.key", &BlockEvent { block_number: 99 }) + .await + .unwrap(); + + // Poll up to 8 s — the message should be handled once immediately (no retry delay) + let deadline = tokio::time::Instant::now() + Duration::from_secs(8); + while call_count.load(Ordering::SeqCst) == 0 && tokio::time::Instant::now() < deadline { + tokio::time::sleep(Duration::from_millis(100)).await; + } + + // Give a bit of extra time to confirm there are no retry deliveries + tokio::time::sleep(Duration::from_millis(1500)).await; + + consumer_handle.abort(); + + assert_eq!( + call_count.load(Ordering::SeqCst), + 1, + "handler should be called exactly once — AckDecision::Dead must not retry" + ); +} + +// ── Helper: query AMQP queue depth ─────────────────────────────────────────── + +/// Returns the number of ready messages in the named AMQP queue. +/// +/// Uses a passive `queue_declare` which inspects the queue without modifying it. +/// Returns 0 if the queue does not yet exist. +async fn amqp_queue_depth(url: &str, queue: &str) -> u32 { + use lapin::{ + Connection, ConnectionProperties, + options::QueueDeclareOptions, + types::{FieldTable, ShortString}, + }; + + let Ok(conn) = Connection::connect(url, ConnectionProperties::default()).await else { + return 0; + }; + let Ok(channel) = conn.create_channel().await else { + return 0; + }; + + match channel + .queue_declare( + ShortString::from(queue), + QueueDeclareOptions { + passive: true, + ..Default::default() + }, + FieldTable::default(), + ) + .await + { + Ok(q) => q.message_count(), + Err(_) => 0, + } +} + +// ── Test 5: Circuit Breaker — halts consumption during the cooldown window ──── +// +// Handler returns `HandlerError::Transient` for the first 3 deliveries, tripping +// the circuit (threshold=3), then `Ok(Ack)` for all subsequent deliveries. +// +// After the circuit opens we publish a "signal" message (block_number=99) and +// assert it is NOT consumed during the cooldown window, then IS consumed once +// the circuit transitions through Half-Open back to Closed. +// +// A long retry_delay (60 s) prevents NACKed messages from cycling back through +// the retry queue within the test window, keeping the handler call count clean. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_circuit_breaker_halts_consumption() { + let url = shared_rabbitmq_url().await; + + let topology = ExchangeTopology { + main: "cb-halt.exchange".to_string(), + retry: "cb-halt.exchange.retry".to_string(), + dlx: "cb-halt.exchange.dlx".to_string(), + }; + ExchangeManager::with_addr(url) + .declare_topology(&topology) + .await + .unwrap(); + + let config = ConsumerConfigBuilder::new() + .exchange("cb-halt.exchange") + .queue("cb-halt.queue") + .routing_key("cb-halt.key") + .consumer_tag("cb-halt-consumer") + .retry_exchange("cb-halt.exchange.retry") + .dead_exchange("cb-halt.exchange.dlx") + .max_retries(10) + // Long retry_delay keeps NACKed messages in the retry queue for the + // entire test duration — they cannot cycle back and inflate the count. + .retry_delay(Duration::from_secs(60)) + .circuit_breaker_threshold(3) + .circuit_breaker_cooldown(Duration::from_millis(2000)) + .prefetch_count(10) + .build_prefetch() + .unwrap(); + + let transient_count = Arc::new(AtomicU32::new(0)); + let signal_received = Arc::new(AtomicBool::new(false)); + + // Handler: + // block_number == 99 → signal message: record and ACK immediately. + // otherwise → increment transient_count; return Transient for + // the first 3 calls (trips CB), Ok for the rest + // (Half-Open probe and any PEL re-deliveries). + struct TrippingHandler { + transient_count: Arc, + signal_received: Arc, + } + + #[async_trait::async_trait] + impl broker::Handler for TrippingHandler { + async fn call(&self, msg: &broker::Message) -> Result { + if let Ok(event) = serde_json::from_slice::(&msg.payload) + && event.block_number == 99 + { + self.signal_received.store(true, Ordering::SeqCst); + return Ok(AckDecision::Ack); + } + let prev = self.transient_count.fetch_add(1, Ordering::SeqCst); + if prev < 3 { + Err(broker::HandlerError::Transient(Box::new( + std::io::Error::other("simulated infrastructure failure"), + ))) + } else { + Ok(AckDecision::Ack) + } + } + } + + let consumer = RmqConsumer::connect(url).await.unwrap(); + // Clone Arcs before the move closure so the outer test retains handles. + let transient_count_for_handler = transient_count.clone(); + let signal_received_for_handler = signal_received.clone(); + let consumer_handle = tokio::spawn(async move { + let _ = consumer + .run( + config, + TrippingHandler { + transient_count: transient_count_for_handler, + signal_received: signal_received_for_handler, + }, + ) + .await; + }); + + // Give the consumer time to start and declare all queues. + tokio::time::sleep(Duration::from_millis(800)).await; + + let publisher = RmqPublisher::connect(url, "cb-halt.exchange").await; + + // Publish 3 trip messages — each fails with Transient, opening the circuit + // after the third failure (threshold = 3). + for i in 0..3u64 { + publisher + .publish("cb-halt.key", &BlockEvent { block_number: i }) + .await + .unwrap(); + } + + // Wait until all 3 transient failures are recorded (circuit is now Open). + let deadline = tokio::time::Instant::now() + Duration::from_secs(15); + while transient_count.load(Ordering::SeqCst) < 3 && tokio::time::Instant::now() < deadline { + tokio::time::sleep(Duration::from_millis(50)).await; + } + assert_eq!( + transient_count.load(Ordering::SeqCst), + 3, + "expected exactly 3 transient failures to trip the circuit" + ); + + // ── Circuit is Open ─────────────────────────────────────────────────────── + // Publish the signal message — it must NOT be consumed during the cooldown. + publisher + .publish("cb-halt.key", &BlockEvent { block_number: 99 }) + .await + .unwrap(); + + // Observe: mid-cooldown (1 s into the 2 s window), signal must still be unprocessed. + tokio::time::sleep(Duration::from_millis(1000)).await; + assert!( + !signal_received.load(Ordering::SeqCst), + "signal message must not be consumed while the circuit is Open" + ); + + // ── After cooldown: Half-Open → probe succeeds → circuit Closed ─────────── + // The signal is the next new message; handler returns Ok → circuit closes. + let deadline = tokio::time::Instant::now() + Duration::from_secs(8); + while !signal_received.load(Ordering::SeqCst) && tokio::time::Instant::now() < deadline { + tokio::time::sleep(Duration::from_millis(100)).await; + } + + consumer_handle.abort(); + assert!( + signal_received.load(Ordering::SeqCst), + "signal message must be consumed after the CB cooldown expires" + ); +} + +// ── Test 6: Circuit Breaker — prevents DLQ pollution during a sustained outage ─ +// +// A handler that always returns `HandlerError::Transient` simulates a downstream +// outage (e.g., database is unavailable). +// +// Transient failures now retry indefinitely and never consume `max_retries`. +// Without a circuit breaker, messages keep cycling through the retry queue. +// With CB (threshold=2, cooldown=5 s) the consumer pauses after 2 failures; +// messages accumulate in the retry queue and the error queue stays empty. +// +// The long retry_delay (30 s) ensures no messages return from the retry queue +// during the 3 s observation window. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_circuit_breaker_prevents_dlq_pollution() { + let url = shared_rabbitmq_url().await; + + let topology = ExchangeTopology { + main: "cb-dlq.exchange".to_string(), + retry: "cb-dlq.exchange.retry".to_string(), + dlx: "cb-dlq.exchange.dlx".to_string(), + }; + ExchangeManager::with_addr(url) + .declare_topology(&topology) + .await + .unwrap(); + + let config = ConsumerConfigBuilder::new() + .exchange("cb-dlq.exchange") + .queue("cb-dlq.queue") + .routing_key("cb-dlq.key") + .consumer_tag("cb-dlq-consumer") + .retry_exchange("cb-dlq.exchange.retry") + .dead_exchange("cb-dlq.exchange.dlx") + .max_retries(3) + .retry_delay(Duration::from_secs(30)) + .circuit_breaker_threshold(2) + .circuit_breaker_cooldown(Duration::from_secs(5)) + .prefetch_count(10) + .build_prefetch() + .unwrap(); + + // Handler: always returns Transient — simulates a sustained infrastructure outage. + struct AlwaysTransient; + + #[async_trait::async_trait] + impl broker::Handler for AlwaysTransient { + async fn call(&self, _msg: &broker::Message) -> Result { + Err(broker::HandlerError::Transient(Box::new( + std::io::Error::other("downstream is unavailable"), + ))) + } + } + + let consumer = RmqConsumer::connect(url).await.unwrap(); + let consumer_handle = tokio::spawn(async move { + let _ = consumer.run(config, AlwaysTransient).await; + }); + + // Wait for the consumer to start and declare all queues, including the error queue. + tokio::time::sleep(Duration::from_millis(800)).await; + + let publisher = RmqPublisher::connect(url, "cb-dlq.exchange").await; + + // Publish 2 messages — exactly at the failure threshold. Both fail with Transient, + // opening the circuit after the second failure. + publisher + .publish("cb-dlq.key", &BlockEvent { block_number: 1 }) + .await + .unwrap(); + publisher + .publish("cb-dlq.key", &BlockEvent { block_number: 2 }) + .await + .unwrap(); + + // Wait for the CB to open and settle. cooldown = 5 s; we check at 3 s — + // still within the Open window, so no further messages are consumed. + tokio::time::sleep(Duration::from_secs(3)).await; + + // The error queue is named "{queue}.error" (declared by setup_retry_queues). + let error_queue_depth = amqp_queue_depth(url, "cb-dlq.queue.error").await; + consumer_handle.abort(); + + assert_eq!( + error_queue_depth, 0, + "circuit breaker must prevent messages from reaching the DLQ during a transient outage" + ); +} + +// ── Test 7: Transient failures retry indefinitely (bounded retry ignored) ─────── +// +// With max_retries=1, a transient failure should still keep retrying forever. +// We assert: +// - multiple transient handler calls happen (more than max_retries+1 attempts) +// - error queue remains empty. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_transient_failures_retry_indefinitely() { + let url = shared_rabbitmq_url().await; + + let topology = ExchangeTopology { + main: "transient-infinite.exchange".to_string(), + retry: "transient-infinite.exchange.retry".to_string(), + dlx: "transient-infinite.exchange.dlx".to_string(), + }; + ExchangeManager::with_addr(url) + .declare_topology(&topology) + .await + .unwrap(); + + let config = ConsumerConfigBuilder::new() + .exchange("transient-infinite.exchange") + .queue("transient-infinite.queue") + .routing_key("transient-infinite.key") + .consumer_tag("transient-infinite-consumer") + .retry_exchange("transient-infinite.exchange.retry") + .dead_exchange("transient-infinite.exchange.dlx") + .max_retries(1) + .retry_delay(Duration::from_millis(200)) + .prefetch_count(10) + .build_prefetch() + .unwrap(); + + struct AlwaysTransient(Arc); + + #[async_trait::async_trait] + impl broker::Handler for AlwaysTransient { + async fn call(&self, _msg: &broker::Message) -> Result { + self.0.fetch_add(1, Ordering::SeqCst); + Err(broker::HandlerError::Transient(Box::new( + std::io::Error::other("downstream unavailable"), + ))) + } + } + + let attempts = Arc::new(AtomicU32::new(0)); + let attempts_for_handler = attempts.clone(); + let consumer = RmqConsumer::connect(url).await.unwrap(); + let consumer_handle = tokio::spawn(async move { + let _ = consumer + .run(config, AlwaysTransient(attempts_for_handler)) + .await; + }); + + tokio::time::sleep(Duration::from_millis(700)).await; + + let publisher = RmqPublisher::connect(url, "transient-infinite.exchange").await; + publisher + .publish("transient-infinite.key", &BlockEvent { block_number: 1 }) + .await + .unwrap(); + + let attempts_deadline = tokio::time::Instant::now() + Duration::from_secs(12); + while tokio::time::Instant::now() < attempts_deadline { + let current = attempts.load(Ordering::SeqCst); + if current >= 4 { + break; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + + let error_depth = amqp_queue_depth(url, "transient-infinite.queue.error").await; + consumer_handle.abort(); + + assert!( + attempts.load(Ordering::SeqCst) >= 4, + "transient message should keep retrying even with max_retries=1; attempts={}", + attempts.load(Ordering::SeqCst) + ); + assert_eq!( + error_depth, 0, + "transient failures must not be moved to DLQ by max_retries" + ); +} + +// ── Test 8: Permanent failures still honor max_retries ────────────────────────── +// +// Permanent failures keep bounded retry semantics and should still dead-letter. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_permanent_failures_still_dead_letter_after_max_retries() { + let url = shared_rabbitmq_url().await; + + let topology = ExchangeTopology { + main: "permanent-bounded.exchange".to_string(), + retry: "permanent-bounded.exchange.retry".to_string(), + dlx: "permanent-bounded.exchange.dlx".to_string(), + }; + ExchangeManager::with_addr(url) + .declare_topology(&topology) + .await + .unwrap(); + + let config = ConsumerConfigBuilder::new() + .exchange("permanent-bounded.exchange") + .queue("permanent-bounded.queue") + .routing_key("permanent-bounded.key") + .consumer_tag("permanent-bounded-consumer") + .retry_exchange("permanent-bounded.exchange.retry") + .dead_exchange("permanent-bounded.exchange.dlx") + .max_retries(1) + .retry_delay(Duration::from_millis(200)) + .prefetch_count(10) + .build_prefetch() + .unwrap(); + + struct AlwaysPermanent(Arc); + + #[async_trait::async_trait] + impl broker::Handler for AlwaysPermanent { + async fn call(&self, _msg: &broker::Message) -> Result { + self.0.fetch_add(1, Ordering::SeqCst); + Err(broker::HandlerError::permanent(std::io::Error::other( + "invalid payload", + ))) + } + } + + let attempts = Arc::new(AtomicU32::new(0)); + let attempts_for_handler = attempts.clone(); + let consumer = RmqConsumer::connect(url).await.unwrap(); + let consumer_handle = tokio::spawn(async move { + let _ = consumer + .run(config, AlwaysPermanent(attempts_for_handler)) + .await; + }); + + tokio::time::sleep(Duration::from_millis(700)).await; + + let publisher = RmqPublisher::connect(url, "permanent-bounded.exchange").await; + publisher + .publish("permanent-bounded.key", &BlockEvent { block_number: 1 }) + .await + .unwrap(); + + let dlq_deadline = tokio::time::Instant::now() + Duration::from_secs(12); + let error_depth = loop { + let depth = amqp_queue_depth(url, "permanent-bounded.queue.error").await; + if depth >= 1 || tokio::time::Instant::now() >= dlq_deadline { + break depth; + } + tokio::time::sleep(Duration::from_millis(200)).await; + }; + + consumer_handle.abort(); + + assert!( + attempts.load(Ordering::SeqCst) >= 2, + "permanent failure should be attempted then retried before DLQ; attempts={}", + attempts.load(Ordering::SeqCst) + ); + assert_eq!( + error_depth, 1, + "permanent failure should reach DLQ after max_retries" + ); +} + +// ── Test 9: Transient -> Permanent transition preserves permanent retry budget ── +// +// First delivery is transient (infinite path), then failures become permanent. +// With max_retries=1, we expect one permanent retry before DLQ: +// transient -> permanent(retry) -> permanent(DLQ) +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_transient_then_permanent_eventually_dead_letters() { + let url = shared_rabbitmq_url().await; + + let topology = ExchangeTopology { + main: "transient-to-permanent.exchange".to_string(), + retry: "transient-to-permanent.exchange.retry".to_string(), + dlx: "transient-to-permanent.exchange.dlx".to_string(), + }; + ExchangeManager::with_addr(url) + .declare_topology(&topology) + .await + .unwrap(); + + let config = ConsumerConfigBuilder::new() + .exchange("transient-to-permanent.exchange") + .queue("transient-to-permanent.queue") + .routing_key("transient-to-permanent.key") + .consumer_tag("transient-to-permanent-consumer") + .retry_exchange("transient-to-permanent.exchange.retry") + .dead_exchange("transient-to-permanent.exchange.dlx") + .max_retries(1) + .retry_delay(Duration::from_millis(200)) + .prefetch_count(10) + .build_prefetch() + .unwrap(); + + struct TransientThenPermanent(Arc); + + #[async_trait::async_trait] + impl broker::Handler for TransientThenPermanent { + async fn call(&self, _msg: &broker::Message) -> Result { + let call_index = self.0.fetch_add(1, Ordering::SeqCst); + if call_index == 0 { + Err(broker::HandlerError::Transient(Box::new( + std::io::Error::other("downstream unavailable"), + ))) + } else { + Err(broker::HandlerError::permanent(std::io::Error::other( + "invalid payload", + ))) + } + } + } + + let calls = Arc::new(AtomicU32::new(0)); + let calls_for_handler = calls.clone(); + let consumer = RmqConsumer::connect(url).await.unwrap(); + let consumer_handle = tokio::spawn(async move { + let _ = consumer + .run(config, TransientThenPermanent(calls_for_handler)) + .await; + }); + + tokio::time::sleep(Duration::from_millis(700)).await; + + let publisher = RmqPublisher::connect(url, "transient-to-permanent.exchange").await; + publisher + .publish( + "transient-to-permanent.key", + &BlockEvent { block_number: 1 }, + ) + .await + .unwrap(); + + let deadline = tokio::time::Instant::now() + Duration::from_secs(15); + let error_depth = loop { + let depth = amqp_queue_depth(url, "transient-to-permanent.queue.error").await; + if (depth >= 1 && calls.load(Ordering::SeqCst) >= 3) + || tokio::time::Instant::now() >= deadline + { + break depth; + } + tokio::time::sleep(Duration::from_millis(200)).await; + }; + + consumer_handle.abort(); + + assert!( + calls.load(Ordering::SeqCst) >= 3, + "transient history must not consume permanent retry budget; calls={}", + calls.load(Ordering::SeqCst) + ); + assert_eq!( + error_depth, 1, + "message should DLQ only after bounded permanent retries" + ); +} + +// ── Test 10: AsyncHandlerWithContext receives correct MessageMetadata ───────── +// +// Verifies that `AsyncHandlerWithContext` receives a `MessageMetadata` with: +// - `delivery_count == 1` on first delivery +// - `topic` matching the queue name +// - `id` being a non-empty delivery tag +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_async_handler_with_context_metadata() { + let url = shared_rabbitmq_url().await; + + let exchange = "ctx.exchange"; + let queue = "ctx.queue"; + let routing_key = "ctx.key"; + + let topology = ExchangeTopology::from_prefix(exchange); + ExchangeManager::with_addr(url) + .declare_topology(&topology) + .await + .unwrap(); + + let config = ConsumerConfigBuilder::new() + .with_topology(&topology) + .queue(queue) + .routing_key(routing_key) + .consumer_tag("ctx-consumer") + .max_retries(3) + .retry_delay(Duration::from_millis(200)) + .prefetch_count(10) + .build_prefetch() + .unwrap(); + + let (meta_tx, mut meta_rx) = tokio::sync::mpsc::channel::<(BlockEvent, MessageMetadata)>(1); + + let handler = AsyncHandlerWithContext::new(move |event: BlockEvent, meta: MessageMetadata| { + let tx = meta_tx.clone(); + async move { + let _ = tx.send((event, meta)).await; + Ok::<(), std::io::Error>(()) + } + }); + + let consumer = RmqConsumer::connect(url).await.unwrap(); + let consumer_handle = tokio::spawn(async move { + let _ = consumer.run(config, handler).await; + }); + + tokio::time::sleep(Duration::from_millis(600)).await; + + let publisher = RmqPublisher::connect(url, exchange).await; + publisher + .publish(routing_key, &BlockEvent { block_number: 42 }) + .await + .unwrap(); + + let deadline = tokio::time::Instant::now() + Duration::from_secs(8); + let received = tokio::select! { + result = meta_rx.recv() => result, + _ = tokio::time::sleep_until(deadline) => None, + }; + + consumer_handle.abort(); + + let (event, meta) = received.expect("handler should have received the message"); + + assert_eq!( + event.block_number, 42, + "payload should be deserialized correctly" + ); + assert_eq!( + meta.delivery_count, 1, + "first delivery should have delivery_count=1" + ); + assert_eq!(meta.topic, queue, "topic should match the queue name"); + assert!(!meta.id.is_empty(), "id (delivery tag) should be non-empty"); +} + +// ── Test: RmqPublisher missing exchange returns typed error (no panic) ───────────── + +/// Publisher constructed via `new`/`with_config` has exchange `None`. Trait `publish` +/// must return `Err(ExchangeNotConfigured)` instead of panicking. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_rmq_publisher_publish_without_exchange_returns_error() { + let url = shared_rabbitmq_url().await; + + let connection = Arc::new(ConnectionManager::new(url)); + let publisher = RmqPublisher::with_config(connection, 10, 1000).await; + + let result = publisher + .publish("test.key", &BlockEvent { block_number: 1 }) + .await; + + assert!( + result.is_err(), + "publish without exchange must return Err, got {:?}", + result + ); + + let err = result.unwrap_err(); + assert!( + matches!(err, PublisherError::ExchangeNotConfigured), + "expected ExchangeNotConfigured, got {:?}", + err + ); +} + +// ── Test: DynPublisher + RMQ routing topology for multi-chain + multi-app ──── +// +// Simulates listener-like block publishing at fixed RPS and verifies: +// - fanout to multiple app queues on the same chain +// - exchange isolation across chains +// - routing-key isolation between data (`blocks.#`) and control (`control.watch.#`) +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_dynpublisher_multichain_routing_for_multiple_apps() { + let url = shared_rabbitmq_url().await; + + let eth_exchange = "dynpub.ethereum.events"; + let polygon_exchange = "dynpub.polygon.events"; + + let app_a_eth_blocks_q = "dynpub.app-a.eth.blocks"; + let app_b_eth_blocks_q = "dynpub.app-b.eth.blocks"; + let app_a_polygon_blocks_q = "dynpub.app-a.polygon.blocks"; + let listener_watch_q = "dynpub.listener.eth.watch"; + + let eth_topology = ExchangeTopology { + main: eth_exchange.to_string(), + retry: format!("{eth_exchange}.retry"), + dlx: format!("{eth_exchange}.dlx"), + }; + let polygon_topology = ExchangeTopology { + main: polygon_exchange.to_string(), + retry: format!("{polygon_exchange}.retry"), + dlx: format!("{polygon_exchange}.dlx"), + }; + + let exchange_manager = ExchangeManager::with_addr(url); + exchange_manager + .declare_topology(ð_topology) + .await + .unwrap(); + exchange_manager + .declare_topology(&polygon_topology) + .await + .unwrap(); + + let app_a_eth_blocks = Arc::new(AtomicU32::new(0)); + let app_a_eth_block_numbers = Arc::new(Mutex::new(HashSet::::new())); + let app_a_eth_unexpected = Arc::new(AtomicU32::new(0)); + let app_b_eth_blocks = Arc::new(AtomicU32::new(0)); + let app_b_eth_block_numbers = Arc::new(Mutex::new(HashSet::::new())); + let app_b_eth_unexpected = Arc::new(AtomicU32::new(0)); + let app_a_polygon_blocks = Arc::new(AtomicU32::new(0)); + let app_a_polygon_block_numbers = Arc::new(Mutex::new(HashSet::::new())); + let app_a_polygon_unexpected = Arc::new(AtomicU32::new(0)); + let watch_count = Arc::new(AtomicU32::new(0)); + let watch_received = Arc::new(Mutex::new(None::)); + let watch_unexpected = Arc::new(AtomicU32::new(0)); + + let app_a_eth_handler = { + let blocks = app_a_eth_blocks.clone(); + let block_numbers = app_a_eth_block_numbers.clone(); + let unexpected = app_a_eth_unexpected.clone(); + AsyncHandlerPayloadOnly::new(move |event: ListenerEvent| { + let blocks = blocks.clone(); + let block_numbers = block_numbers.clone(); + let unexpected = unexpected.clone(); + async move { + match event { + ListenerEvent::Block { + chain_id: 1, + block_number, + } => { + blocks.fetch_add(1, Ordering::SeqCst); + block_numbers.lock().unwrap().insert(block_number); + } + _ => { + unexpected.fetch_add(1, Ordering::SeqCst); + } + } + Ok::<(), std::convert::Infallible>(()) + } + }) + }; + + let app_b_eth_handler = { + let blocks = app_b_eth_blocks.clone(); + let block_numbers = app_b_eth_block_numbers.clone(); + let unexpected = app_b_eth_unexpected.clone(); + AsyncHandlerPayloadOnly::new(move |event: ListenerEvent| { + let blocks = blocks.clone(); + let block_numbers = block_numbers.clone(); + let unexpected = unexpected.clone(); + async move { + match event { + ListenerEvent::Block { + chain_id: 1, + block_number, + } => { + blocks.fetch_add(1, Ordering::SeqCst); + block_numbers.lock().unwrap().insert(block_number); + } + _ => { + unexpected.fetch_add(1, Ordering::SeqCst); + } + } + Ok::<(), std::convert::Infallible>(()) + } + }) + }; + + let app_a_polygon_handler = { + let blocks = app_a_polygon_blocks.clone(); + let block_numbers = app_a_polygon_block_numbers.clone(); + let unexpected = app_a_polygon_unexpected.clone(); + AsyncHandlerPayloadOnly::new(move |event: ListenerEvent| { + let blocks = blocks.clone(); + let block_numbers = block_numbers.clone(); + let unexpected = unexpected.clone(); + async move { + match event { + ListenerEvent::Block { + chain_id: 137, + block_number, + } => { + blocks.fetch_add(1, Ordering::SeqCst); + block_numbers.lock().unwrap().insert(block_number); + } + _ => { + unexpected.fetch_add(1, Ordering::SeqCst); + } + } + Ok::<(), std::convert::Infallible>(()) + } + }) + }; + + let watch_handler = { + let count = watch_count.clone(); + let received = watch_received.clone(); + let unexpected = watch_unexpected.clone(); + AsyncHandlerPayloadOnly::new(move |event: ListenerEvent| { + let count = count.clone(); + let received = received.clone(); + let unexpected = unexpected.clone(); + async move { + match &event { + ListenerEvent::WatchRegister { chain_id: 1, .. } => { + count.fetch_add(1, Ordering::SeqCst); + let mut r = received.lock().unwrap(); + if r.is_none() { + *r = Some(event.clone()); + } + } + _ => { + unexpected.fetch_add(1, Ordering::SeqCst); + } + } + Ok::<(), std::convert::Infallible>(()) + } + }) + }; + + let app_a_eth_config = ConsumerConfigBuilder::new() + .with_topology(ð_topology) + .queue(app_a_eth_blocks_q) + .routing_key("blocks.#") + .consumer_tag("dynpub-app-a-eth") + .max_retries(3) + .retry_delay(Duration::from_secs(2)) + .prefetch_count(32) + .build_prefetch() + .unwrap(); + + let app_b_eth_config = ConsumerConfigBuilder::new() + .with_topology(ð_topology) + .queue(app_b_eth_blocks_q) + .routing_key("blocks.#") + .consumer_tag("dynpub-app-b-eth") + .max_retries(3) + .retry_delay(Duration::from_secs(2)) + .prefetch_count(32) + .build_prefetch() + .unwrap(); + + let app_a_polygon_config = ConsumerConfigBuilder::new() + .with_topology(&polygon_topology) + .queue(app_a_polygon_blocks_q) + .routing_key("blocks.#") + .consumer_tag("dynpub-app-a-polygon") + .max_retries(3) + .retry_delay(Duration::from_secs(2)) + .prefetch_count(32) + .build_prefetch() + .unwrap(); + + let watch_config = ConsumerConfigBuilder::new() + .with_topology(ð_topology) + .queue(listener_watch_q) + .routing_key("control.watch.#") + .consumer_tag("dynpub-listener-watch") + .max_retries(3) + .retry_delay(Duration::from_secs(2)) + .prefetch_count(16) + .build_prefetch() + .unwrap(); + + let app_a_eth_consumer = RmqConsumer::connect(url).await.unwrap(); + let app_b_eth_consumer = RmqConsumer::connect(url).await.unwrap(); + let app_a_polygon_consumer = RmqConsumer::connect(url).await.unwrap(); + let watch_consumer = RmqConsumer::connect(url).await.unwrap(); + + let app_a_eth_handle = tokio::spawn(async move { + let _ = app_a_eth_consumer + .run(app_a_eth_config, app_a_eth_handler) + .await; + }); + let app_b_eth_handle = tokio::spawn(async move { + let _ = app_b_eth_consumer + .run(app_b_eth_config, app_b_eth_handler) + .await; + }); + let app_a_polygon_handle = tokio::spawn(async move { + let _ = app_a_polygon_consumer + .run(app_a_polygon_config, app_a_polygon_handler) + .await; + }); + let watch_handle = tokio::spawn(async move { + let _ = watch_consumer.run(watch_config, watch_handler).await; + }); + + tokio::time::sleep(Duration::from_millis(1200)).await; + + let mut publishers: HashMap = HashMap::new(); + publishers.insert( + "ethereum".to_string(), + DynPublisher::new(RmqPublisher::connect(url, eth_exchange).await), + ); + publishers.insert( + "polygon".to_string(), + DynPublisher::new(RmqPublisher::connect(url, polygon_exchange).await), + ); + + publishers["ethereum"] + .publish( + "control.watch.register", + &ListenerEvent::WatchRegister { + chain_id: 1, + consumer_id: "app-a".to_string(), + contract_addresses: vec!["0xabc0000000000000000000000000000000000001".to_string()], + }, + ) + .await + .unwrap(); + + let eth_expected: u32 = 24; + let polygon_expected: u32 = 11; + + let eth_pub = publishers + .get("ethereum") + .expect("ethereum publisher should exist") + .clone(); + let polygon_pub = publishers + .get("polygon") + .expect("polygon publisher should exist") + .clone(); + + let eth_publish_handle = tokio::spawn(async move { + publish_blocks_at_rps( + ð_pub, + "blocks.canonical", + 1, + 1_000_000, + u64::from(eth_expected), + 20, + ) + .await + .unwrap(); + }); + + let polygon_publish_handle = tokio::spawn(async move { + publish_blocks_at_rps( + &polygon_pub, + "blocks.canonical", + 137, + 2_000_000, + u64::from(polygon_expected), + 12, + ) + .await + .unwrap(); + }); + + eth_publish_handle.await.unwrap(); + polygon_publish_handle.await.unwrap(); + + let deadline = tokio::time::Instant::now() + Duration::from_secs(20); + while tokio::time::Instant::now() < deadline { + let done = app_a_eth_blocks.load(Ordering::SeqCst) == eth_expected + && app_b_eth_blocks.load(Ordering::SeqCst) == eth_expected + && app_a_polygon_blocks.load(Ordering::SeqCst) == polygon_expected + && watch_count.load(Ordering::SeqCst) == 1; + if done { + break; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + + for publisher in publishers.values() { + publisher.shutdown().await; + } + + app_a_eth_handle.abort(); + app_b_eth_handle.abort(); + app_a_polygon_handle.abort(); + watch_handle.abort(); + + assert_eq!( + app_a_eth_blocks.load(Ordering::SeqCst), + eth_expected, + "app-a should receive every ETH block" + ); + assert_eq!( + app_b_eth_blocks.load(Ordering::SeqCst), + eth_expected, + "app-b should receive every ETH block" + ); + assert_eq!( + app_a_polygon_blocks.load(Ordering::SeqCst), + polygon_expected, + "polygon queue should receive only polygon blocks" + ); + assert_eq!( + watch_count.load(Ordering::SeqCst), + 1, + "watch queue should receive exactly one control message" + ); + + assert_eq!( + app_a_eth_unexpected.load(Ordering::SeqCst), + 0, + "app-a ETH queue should not receive unexpected payloads" + ); + assert_eq!( + app_b_eth_unexpected.load(Ordering::SeqCst), + 0, + "app-b ETH queue should not receive unexpected payloads" + ); + assert_eq!( + app_a_polygon_unexpected.load(Ordering::SeqCst), + 0, + "polygon queue should not receive unexpected payloads" + ); + assert_eq!( + watch_unexpected.load(Ordering::SeqCst), + 0, + "watch queue should not receive unexpected payloads" + ); + + // ── Content validation: verify received payloads match what was published ── + let eth_expected_blocks: HashSet = + (1_000_000..1_000_000 + u64::from(eth_expected)).collect(); + let polygon_expected_blocks: HashSet = + (2_000_000..2_000_000 + u64::from(polygon_expected)).collect(); + + let app_a_eth_received = app_a_eth_block_numbers.lock().unwrap().clone(); + let app_b_eth_received = app_b_eth_block_numbers.lock().unwrap().clone(); + let app_a_polygon_received = app_a_polygon_block_numbers.lock().unwrap().clone(); + + assert_eq!( + app_a_eth_received, eth_expected_blocks, + "app-a ETH should receive exactly the published block numbers" + ); + assert_eq!( + app_b_eth_received, eth_expected_blocks, + "app-b ETH should receive exactly the published block numbers" + ); + assert_eq!( + app_a_polygon_received, polygon_expected_blocks, + "polygon consumer should receive exactly the published block numbers" + ); + + let watch_payload = watch_received.lock().unwrap().clone(); + let expected_watch = ListenerEvent::WatchRegister { + chain_id: 1, + consumer_id: "app-a".to_string(), + contract_addresses: vec!["0xabc0000000000000000000000000000000000001".to_string()], + }; + assert_eq!( + watch_payload.as_ref(), + Some(&expected_watch), + "watch queue should receive the exact WatchRegister payload" + ); +} + +// ── Test 10: Shared exchange retry path is isolated per queue ───────────────── +// +// Two queues bind to the same main exchange/routing pattern. Queue A always fails +// permanently; Queue B always ACKs. +// +// Queue A retries must not leak into Queue B. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_shared_exchange_retry_isolated_per_queue() { + let url = shared_rabbitmq_url().await; + + let topology = ExchangeTopology { + main: "isolation.retry.exchange".to_string(), + retry: "isolation.retry.exchange.retry".to_string(), + dlx: "isolation.retry.exchange.dlx".to_string(), + }; + ExchangeManager::with_addr(url) + .declare_topology(&topology) + .await + .unwrap(); + + let queue_a = "isolation.retry.queue.a"; + let queue_b = "isolation.retry.queue.b"; + let routing_key = "blocks.#"; + + let cfg_a = ConsumerConfigBuilder::new() + .with_topology(&topology) + .queue(queue_a) + .routing_key(routing_key) + .consumer_tag("isolation-retry-a") + .max_retries(1) + .retry_delay(Duration::from_millis(200)) + .prefetch_count(10) + .build_prefetch() + .unwrap(); + + let cfg_b = ConsumerConfigBuilder::new() + .with_topology(&topology) + .queue(queue_b) + .routing_key(routing_key) + .consumer_tag("isolation-retry-b") + .max_retries(1) + .retry_delay(Duration::from_millis(200)) + .prefetch_count(10) + .build_prefetch() + .unwrap(); + + let b_count = Arc::new(AtomicU32::new(0)); + + // Queue A: always permanent failure + let handler_a = AsyncHandlerPayloadOnly::new(|_: BlockEvent| async move { + Err::<(), std::io::Error>(std::io::Error::other("forced permanent failure")) + }); + + // Queue B: always ACK + let b_count_for_handler = b_count.clone(); + let handler_b = AsyncHandlerPayloadOnly::new(move |_: BlockEvent| { + let count = b_count_for_handler.clone(); + async move { + count.fetch_add(1, Ordering::SeqCst); + Ok::<(), std::io::Error>(()) + } + }); + + let consumer_a = RmqConsumer::connect(url).await.unwrap(); + let consumer_b = RmqConsumer::connect(url).await.unwrap(); + + let handle_a = tokio::spawn(async move { + let _ = consumer_a.run(cfg_a, handler_a).await; + }); + let handle_b = tokio::spawn(async move { + let _ = consumer_b.run(cfg_b, handler_b).await; + }); + + tokio::time::sleep(Duration::from_millis(900)).await; + + let publisher = RmqPublisher::connect(url, &topology.main).await; + publisher + .publish("blocks.canonical", &BlockEvent { block_number: 42 }) + .await + .unwrap(); + + let first_delivery_deadline = tokio::time::Instant::now() + Duration::from_secs(8); + while b_count.load(Ordering::SeqCst) < 1 + && tokio::time::Instant::now() < first_delivery_deadline + { + tokio::time::sleep(Duration::from_millis(100)).await; + } + + // Allow Queue A retry + DLQ flow to complete; Queue B should not see extra deliveries. + tokio::time::sleep(Duration::from_secs(2)).await; + + let a_error_queue = format!("{queue_a}.error"); + let b_error_queue = format!("{queue_b}.error"); + + let a_error_deadline = tokio::time::Instant::now() + Duration::from_secs(8); + let a_error_depth = loop { + let depth = amqp_queue_depth(url, &a_error_queue).await; + if depth >= 1 || tokio::time::Instant::now() >= a_error_deadline { + break depth; + } + tokio::time::sleep(Duration::from_millis(200)).await; + }; + let b_error_depth = amqp_queue_depth(url, &b_error_queue).await; + + handle_a.abort(); + handle_b.abort(); + + assert_eq!( + b_count.load(Ordering::SeqCst), + 1, + "Queue B must receive exactly one original delivery (no retry leakage from Queue A)" + ); + assert_eq!( + a_error_depth, 1, + "Queue A should dead-letter after max retries" + ); + assert_eq!( + b_error_depth, 0, + "Queue B must not receive dead-letter traffic from Queue A" + ); +} + +// ── Test 11: Shared exchange delay path is isolated per queue ───────────────── +// +// Queue A requests delayed requeue twice (`AckDecision::Delay`), then ACKs. +// Queue B should still see only the original message once. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_shared_exchange_delay_isolated_per_queue() { + let url = shared_rabbitmq_url().await; + + let topology = ExchangeTopology { + main: "isolation.delay.exchange".to_string(), + retry: "isolation.delay.exchange.retry".to_string(), + dlx: "isolation.delay.exchange.dlx".to_string(), + }; + ExchangeManager::with_addr(url) + .declare_topology(&topology) + .await + .unwrap(); + + let queue_a = "isolation.delay.queue.a"; + let queue_b = "isolation.delay.queue.b"; + let routing_key = "blocks.#"; + + let cfg_a = ConsumerConfigBuilder::new() + .with_topology(&topology) + .queue(queue_a) + .routing_key(routing_key) + .consumer_tag("isolation-delay-a") + .max_retries(5) + .retry_delay(Duration::from_millis(100)) + .prefetch_count(10) + .build_prefetch() + .unwrap(); + + let cfg_b = ConsumerConfigBuilder::new() + .with_topology(&topology) + .queue(queue_b) + .routing_key(routing_key) + .consumer_tag("isolation-delay-b") + .max_retries(5) + .retry_delay(Duration::from_millis(100)) + .prefetch_count(10) + .build_prefetch() + .unwrap(); + + struct DelayThenAck { + calls: Arc, + } + + #[async_trait::async_trait] + impl broker::Handler for DelayThenAck { + async fn call(&self, _msg: &broker::Message) -> Result { + let prev = self.calls.fetch_add(1, Ordering::SeqCst); + if prev < 2 { + Ok(AckDecision::Delay(Duration::from_millis(150))) + } else { + Ok(AckDecision::Ack) + } + } + } + + let a_calls = Arc::new(AtomicU32::new(0)); + let b_count = Arc::new(AtomicU32::new(0)); + + let handler_a = DelayThenAck { + calls: a_calls.clone(), + }; + let b_count_for_handler = b_count.clone(); + let handler_b = AsyncHandlerPayloadOnly::new(move |_: BlockEvent| { + let count = b_count_for_handler.clone(); + async move { + count.fetch_add(1, Ordering::SeqCst); + Ok::<(), std::io::Error>(()) + } + }); + + let consumer_a = RmqConsumer::connect(url).await.unwrap(); + let consumer_b = RmqConsumer::connect(url).await.unwrap(); + + let handle_a = tokio::spawn(async move { + let _ = consumer_a.run(cfg_a, handler_a).await; + }); + let handle_b = tokio::spawn(async move { + let _ = consumer_b.run(cfg_b, handler_b).await; + }); + + tokio::time::sleep(Duration::from_millis(900)).await; + + let publisher = RmqPublisher::connect(url, &topology.main).await; + publisher + .publish("blocks.canonical", &BlockEvent { block_number: 99 }) + .await + .unwrap(); + + // Wait until Queue A completes its 2 delay cycles and final ACK. + let a_done_deadline = tokio::time::Instant::now() + Duration::from_secs(10); + while a_calls.load(Ordering::SeqCst) < 3 && tokio::time::Instant::now() < a_done_deadline { + tokio::time::sleep(Duration::from_millis(100)).await; + } + + // Additional buffer to catch potential leakage into Queue B. + tokio::time::sleep(Duration::from_secs(1)).await; + + let b_error_queue = format!("{queue_b}.error"); + let b_error_depth = amqp_queue_depth(url, &b_error_queue).await; + + handle_a.abort(); + handle_b.abort(); + + assert!( + a_calls.load(Ordering::SeqCst) >= 3, + "Queue A should process delayed redeliveries before final ACK" + ); + assert_eq!( + b_count.load(Ordering::SeqCst), + 1, + "Queue B must receive exactly one original delivery (no delay leakage from Queue A)" + ); + assert_eq!( + b_error_depth, 0, + "Queue B must not receive dead-letter traffic from Queue A delay flow" + ); +} + +// ── Queue depth introspection tests ───────────────────────────────────────── + +/// Verify `AmqpQueueInspector::queue_depths` returns correct counts for +/// principal, retry, and dead-letter queues after publishing messages. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_queue_depth_introspection() { + use broker::amqp::AmqpQueueInspector; + use broker::traits::depth::QueueInspector; + + let url = shared_rabbitmq_url().await; + + let exchange = "depth.exchange"; + let queue = "depth.queue"; + let routing_key = "depth.key"; + + let topology = ExchangeTopology::from_prefix(exchange); + ExchangeManager::with_addr(url) + .declare_topology(&topology) + .await + .unwrap(); + + // Declare the principal queue bound to the exchange so messages land there. + let config = ConsumerConfigBuilder::new() + .with_topology(&topology) + .queue(queue) + .routing_key(routing_key) + .consumer_tag("depth-consumer") + .max_retries(3) + .retry_delay(Duration::from_millis(200)) + .prefetch_count(10) + .build_prefetch() + .unwrap(); + + // Start a consumer briefly to force queue declaration, then drop it. + let consumer = RmqConsumer::connect(url).await.unwrap(); + let noop_handler = AsyncHandlerPayloadOnly::new(move |_: serde_json::Value| async move { + Ok::<(), std::io::Error>(()) + }); + let handle = tokio::spawn({ + let config = config.clone(); + async move { + let _ = consumer.run(config, noop_handler).await; + } + }); + // Give the consumer time to declare the queue and bind it. + tokio::time::sleep(Duration::from_millis(500)).await; + handle.abort(); + + // Publish 5 messages (no consumer running → they pile up in the queue). + let publisher = RmqPublisher::connect(url, exchange).await; + for i in 0..5u64 { + publisher + .publish(routing_key, &BlockEvent { block_number: i }) + .await + .unwrap(); + } + + // Query depth via our inspector. + let conn = ConnectionManager::new(url); + let inspector = AmqpQueueInspector::new(conn); + let depths = inspector.queue_depths(queue, None).await.unwrap(); + + assert_eq!(depths.principal, 5, "5 messages published, none consumed"); + assert_eq!(depths.retry, Some(0), "no messages in retry queue"); + assert_eq!(depths.dead_letter, 0, "no messages in dead-letter queue"); + assert_eq!(depths.total(), 5); + assert!(!depths.is_empty()); + assert_eq!(depths.pending, None, "AMQP does not track pending"); + assert_eq!(depths.lag, None, "AMQP does not track lag"); +} + +/// Verify `AmqpQueueInspector::queue_depths` returns zeros for a queue that +/// does not exist (passive declare returns NOT_FOUND → treated as 0). +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_queue_depth_nonexistent_queue_returns_zeros() { + use broker::amqp::AmqpQueueInspector; + use broker::traits::depth::QueueInspector; + + let url = shared_rabbitmq_url().await; + let conn = ConnectionManager::new(url); + let inspector = AmqpQueueInspector::new(conn); + + let depths = inspector + .queue_depths("nonexistent.queue.depth.test", None) + .await + .unwrap(); + + assert_eq!(depths.principal, 0); + assert_eq!(depths.retry, Some(0)); + assert_eq!(depths.dead_letter, 0); + assert!(depths.is_empty()); +} + +/// Verify `is_empty` returns `true` for a non-existent queue. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_is_empty_nonexistent_queue() { + use broker::amqp::AmqpQueueInspector; + use broker::traits::depth::QueueInspector; + + let url = shared_rabbitmq_url().await; + let conn = ConnectionManager::new(url); + let inspector = AmqpQueueInspector::new(conn); + + let empty = inspector + .is_empty("nonexistent.is_empty.test", "ignored") + .await + .unwrap(); + + assert!(empty, "non-existent queue should be empty"); +} + +/// Verify `exists` returns `false` for a queue that does not exist. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_exists_nonexistent_queue() { + use broker::amqp::AmqpQueueInspector; + use broker::traits::depth::QueueInspector; + + let url = shared_rabbitmq_url().await; + let conn = ConnectionManager::new(url); + let inspector = AmqpQueueInspector::new(conn); + + let found = inspector.exists("nonexistent.exists.test").await.unwrap(); + + assert!(!found, "non-existent queue should return false"); +} + +/// Verify `exists` returns `true` for a queue that has been declared. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_exists_after_declare() { + use broker::amqp::AmqpQueueInspector; + use broker::traits::depth::QueueInspector; + use lapin::options::QueueDeclareOptions; + use lapin::types::FieldTable; + + let url = shared_rabbitmq_url().await; + let conn = ConnectionManager::new(url); + + // Declare a durable queue so it exists. + let channel = conn.create_channel().await.unwrap(); + channel + .queue_declare( + "exists.after_declare.test".into(), + QueueDeclareOptions { + durable: true, + ..Default::default() + }, + FieldTable::default(), + ) + .await + .unwrap(); + + let inspector = AmqpQueueInspector::new(conn); + let found = inspector.exists("exists.after_declare.test").await.unwrap(); + + assert!(found, "declared queue should exist"); +} diff --git a/listener/crates/shared/broker/tests/broker_e2e.rs b/listener/crates/shared/broker/tests/broker_e2e.rs new file mode 100644 index 0000000000..bb53eb6269 --- /dev/null +++ b/listener/crates/shared/broker/tests/broker_e2e.rs @@ -0,0 +1,1014 @@ +#![cfg(feature = "redis")] +//! End-to-end integration tests for the `Broker` **facade** API. +//! +//! Unlike `redis_e2e.rs` which constructs backend-specific types directly +//! (`RedisPublisher`, `RedisConsumer`, `RedisConsumerConfigBuilder`), these +//! tests exercise the high-level `Broker → Publisher / ConsumerBuilder` +//! workflow — the API that application code should use. +//! +//! Run via: +//! +//! ```bash +//! make test-e2e-broker +//! ``` + +use std::{ + collections::HashSet, + sync::{ + Arc, Mutex, + atomic::{AtomicBool, AtomicU32, Ordering}, + }, + time::Duration, +}; + +use broker::{ + AckDecision, AsyncHandlerPayloadClassified, AsyncHandlerPayloadOnly, Broker, HandlerError, + Message, Topic, +}; +use test_support::shared_redis_url; + +// ── Shared payload types ───────────────────────────────────────────────────── + +#[derive(serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq)] +struct BlockEvent { + block_number: u64, +} + +#[derive(serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq)] +#[serde(tag = "kind", rename_all = "snake_case")] +enum ListenerEvent { + Block { + chain_id: u64, + block_number: u64, + }, + WatchRegister { + chain_id: u64, + consumer_id: String, + contract_addresses: Vec, + }, +} + +// ── Redis assertion helpers ────────────────────────────────────────────────── + +async fn redis_xlen(url: &str, stream: &str) -> u64 { + let client = redis::Client::open(url).unwrap(); + let mut conn = client.get_multiplexed_async_connection().await.unwrap(); + redis::cmd("XLEN") + .arg(stream) + .query_async(&mut conn) + .await + .unwrap_or(0) +} + +// ── Test 1: Simple roundtrip via Broker facade ────────────────────────────── + +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_broker_simple_roundtrip() { + let url = shared_redis_url().await; + let broker = Broker::redis(url).await.unwrap(); + + let publisher = broker.publisher("bf-simple").await.unwrap(); + let topic = Topic::namespaced("bf-simple", "events"); + + let count = Arc::new(AtomicU32::new(0)); + let count_clone = count.clone(); + let handler = AsyncHandlerPayloadOnly::new(move |_: BlockEvent| { + let c = count_clone.clone(); + async move { + c.fetch_add(1, Ordering::SeqCst); + Ok::<(), std::io::Error>(()) + } + }); + + let consumer_broker = broker.clone(); + let consumer_handle = tokio::spawn(async move { + let _ = consumer_broker + .consumer(&topic) + .group("bf-simple-group") + .consumer_name("consumer-1") + .prefetch(10) + .run(handler) + .await; + }); + + tokio::time::sleep(Duration::from_millis(400)).await; + + publisher + .publish("events", &BlockEvent { block_number: 42 }) + .await + .unwrap(); + + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + while count.load(Ordering::SeqCst) == 0 && tokio::time::Instant::now() < deadline { + tokio::time::sleep(Duration::from_millis(100)).await; + } + + publisher.shutdown().await; + consumer_handle.abort(); + assert_eq!( + count.load(Ordering::SeqCst), + 1, + "handler should be called exactly once" + ); +} + +// ── Test 2: Multi-publish roundtrip ───────────────────────────────────────── + +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_broker_multi_publish_roundtrip() { + let url = shared_redis_url().await; + let broker = Broker::redis(url).await.unwrap(); + + let publisher = broker.publisher("bf-batch").await.unwrap(); + let topic = Topic::namespaced("bf-batch", "events"); + + let count = Arc::new(AtomicU32::new(0)); + let count_clone = count.clone(); + let handler = AsyncHandlerPayloadOnly::new(move |_: BlockEvent| { + let c = count_clone.clone(); + async move { + c.fetch_add(1, Ordering::SeqCst); + Ok::<(), std::io::Error>(()) + } + }); + + let consumer_broker = broker.clone(); + let consumer_handle = tokio::spawn(async move { + let _ = consumer_broker + .consumer(&topic) + .group("bf-batch-group") + .consumer_name("consumer-1") + .prefetch(10) + .run(handler) + .await; + }); + + tokio::time::sleep(Duration::from_millis(400)).await; + + let events = vec![ + BlockEvent { block_number: 1 }, + BlockEvent { block_number: 2 }, + BlockEvent { block_number: 3 }, + ]; + publisher.publish_batch("events", &events).await.unwrap(); + + let deadline = tokio::time::Instant::now() + Duration::from_secs(8); + while count.load(Ordering::SeqCst) < 3 && tokio::time::Instant::now() < deadline { + tokio::time::sleep(Duration::from_millis(100)).await; + } + + publisher.shutdown().await; + consumer_handle.abort(); + assert_eq!( + count.load(Ordering::SeqCst), + 3, + "all 3 batch messages should be received" + ); +} + +// ── Test 3: Retry then succeed ────────────────────────────────────────────── + +/// Handler fails twice, succeeds on third delivery via claim sweeper retry. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_broker_retry_eventually_succeeds() { + let url = shared_redis_url().await; + let broker = Broker::redis(url).await.unwrap(); + + let publisher = broker.publisher("bf-retry").await.unwrap(); + let topic = Topic::namespaced("bf-retry", "events"); + + let call_count = Arc::new(AtomicU32::new(0)); + let call_count_clone = call_count.clone(); + let handler = AsyncHandlerPayloadOnly::new(move |_: BlockEvent| { + let c = call_count_clone.clone(); + async move { + let prev = c.fetch_add(1, Ordering::SeqCst); + if prev < 2 { + Err(std::io::Error::other("simulated failure")) + } else { + Ok(()) + } + } + }); + + let consumer_broker = broker.clone(); + let consumer_handle = tokio::spawn(async move { + let _ = consumer_broker + .consumer(&topic) + .group("bf-retry-group") + .consumer_name("consumer-1") + .prefetch(10) + .max_retries(5) + .redis_claim_min_idle(1) + .redis_claim_interval(1) + .run(handler) + .await; + }); + + tokio::time::sleep(Duration::from_millis(400)).await; + + publisher + .publish("events", &BlockEvent { block_number: 1 }) + .await + .unwrap(); + + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + while call_count.load(Ordering::SeqCst) < 3 && tokio::time::Instant::now() < deadline { + tokio::time::sleep(Duration::from_millis(200)).await; + } + + publisher.shutdown().await; + consumer_handle.abort(); + assert!( + call_count.load(Ordering::SeqCst) >= 3, + "handler should be called at least 3 times (2 failures + 1 success), got {}", + call_count.load(Ordering::SeqCst) + ); +} + +// ── Test 4: DLQ after max_retries ─────────────────────────────────────────── + +/// Handler always fails → message moves to dead stream after max_retries. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_broker_dlq_after_max_retries() { + let url = shared_redis_url().await; + let broker = Broker::redis(url).await.unwrap(); + + let publisher = broker.publisher("bf-dlq").await.unwrap(); + let topic = Topic::namespaced("bf-dlq", "events"); + let dead_stream = topic.dead_key().to_owned(); + + let handler = AsyncHandlerPayloadOnly::new(|_: BlockEvent| async move { + Err::<(), _>(std::io::Error::other("always fails")) + }); + + let consumer_broker = broker.clone(); + let consumer_handle = tokio::spawn(async move { + let _ = consumer_broker + .consumer(&topic) + .group("bf-dlq-group") + .consumer_name("consumer-1") + .prefetch(10) + .max_retries(2) + .redis_claim_min_idle(1) + .redis_claim_interval(1) + .run(handler) + .await; + }); + + tokio::time::sleep(Duration::from_millis(400)).await; + + publisher + .publish("events", &BlockEvent { block_number: 99 }) + .await + .unwrap(); + + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + loop { + tokio::time::sleep(Duration::from_millis(500)).await; + + let len = redis_xlen(url, &dead_stream).await; + if len >= 1 { + publisher.shutdown().await; + consumer_handle.abort(); + assert_eq!(len, 1, "exactly one message should be in the dead stream"); + return; + } + + if tokio::time::Instant::now() >= deadline { + publisher.shutdown().await; + consumer_handle.abort(); + panic!("message was not moved to dead stream within timeout"); + } + } +} + +// ── Test 5: AckDecision::Dead — immediate dead-letter ─────────────────────── + +/// Handler returns `AckDecision::Dead` on first delivery → message goes to +/// dead stream immediately without waiting for claim sweeper or max_retries. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_broker_ack_decision_dead_immediate() { + let url = shared_redis_url().await; + let broker = Broker::redis(url).await.unwrap(); + + let publisher = broker.publisher("bf-dead").await.unwrap(); + let topic = Topic::namespaced("bf-dead", "events"); + let dead_stream = topic.dead_key().to_owned(); + + let call_count = Arc::new(AtomicU32::new(0)); + + #[derive(Clone)] + struct DeadHandler(Arc); + + #[async_trait::async_trait] + impl broker::Handler for DeadHandler { + async fn call(&self, _msg: &Message) -> Result { + self.0.fetch_add(1, Ordering::SeqCst); + Ok(AckDecision::Dead) + } + } + + let handler = DeadHandler(call_count.clone()); + + let consumer_broker = broker.clone(); + let consumer_handle = tokio::spawn(async move { + let _ = consumer_broker + .consumer(&topic) + .group("bf-dead-group") + .consumer_name("consumer-1") + .prefetch(10) + .max_retries(10) + // Long claim_min_idle — message must NOT need the sweeper to route to DLQ + .redis_claim_min_idle(60) + .redis_claim_interval(15) + .run(handler) + .await; + }); + + tokio::time::sleep(Duration::from_millis(400)).await; + + publisher + .publish("events", &BlockEvent { block_number: 42 }) + .await + .unwrap(); + + let deadline = tokio::time::Instant::now() + Duration::from_secs(8); + loop { + tokio::time::sleep(Duration::from_millis(200)).await; + + let len = redis_xlen(url, &dead_stream).await; + if len >= 1 { + // Extra time to confirm no spurious retries + tokio::time::sleep(Duration::from_millis(800)).await; + publisher.shutdown().await; + consumer_handle.abort(); + + assert_eq!( + call_count.load(Ordering::SeqCst), + 1, + "handler should be called exactly once — AckDecision::Dead must not retry" + ); + assert_eq!(len, 1, "exactly one message should be in the dead stream"); + return; + } + + if tokio::time::Instant::now() >= deadline { + publisher.shutdown().await; + consumer_handle.abort(); + panic!( + "message was not moved to dead stream within timeout. \ + handler_calls={}, dead_stream_len={}", + call_count.load(Ordering::SeqCst), + len + ); + } + } +} + +// ── Test 6: Circuit breaker halts then recovers ───────────────────────────── + +/// Transient failures trip the circuit breaker; after cooldown the consumer +/// resumes and processes a signal message. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_broker_circuit_breaker_halts_consumption() { + let url = shared_redis_url().await; + let broker = Broker::redis(url).await.unwrap(); + + let publisher = broker.publisher("bf-cb").await.unwrap(); + let topic = Topic::namespaced("bf-cb", "events"); + + let transient_count = Arc::new(AtomicU32::new(0)); + let signal_received = Arc::new(AtomicBool::new(false)); + + #[derive(Clone)] + struct TrippingHandler { + transient_count: Arc, + signal_received: Arc, + } + + #[async_trait::async_trait] + impl broker::Handler for TrippingHandler { + async fn call(&self, msg: &Message) -> Result { + if let Ok(event) = serde_json::from_slice::(&msg.payload) + && event.block_number == 99 + { + self.signal_received.store(true, Ordering::SeqCst); + return Ok(AckDecision::Ack); + } + let prev = self.transient_count.fetch_add(1, Ordering::SeqCst); + if prev < 3 { + Err(HandlerError::Transient(Box::new(std::io::Error::other( + "simulated infrastructure failure", + )))) + } else { + Ok(AckDecision::Ack) + } + } + } + + let handler = TrippingHandler { + transient_count: transient_count.clone(), + signal_received: signal_received.clone(), + }; + + let consumer_broker = broker.clone(); + let consumer_handle = tokio::spawn(async move { + let _ = consumer_broker + .consumer(&topic) + .group("bf-cb-group") + .consumer_name("consumer-1") + .prefetch(10) + .max_retries(10) + .circuit_breaker(3, Duration::from_millis(2000)) + // Long claim_min_idle keeps sweeper out of test window + .redis_claim_min_idle(60) + .redis_claim_interval(10) + .run(handler) + .await; + }); + + tokio::time::sleep(Duration::from_millis(400)).await; + + // Publish 3 trip messages to open the circuit + for i in 0..3u64 { + publisher + .publish("events", &BlockEvent { block_number: i }) + .await + .unwrap(); + } + + // Wait until at least 3 transient failures recorded + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + while transient_count.load(Ordering::SeqCst) < 3 && tokio::time::Instant::now() < deadline { + tokio::time::sleep(Duration::from_millis(50)).await; + } + assert!( + transient_count.load(Ordering::SeqCst) >= 3, + "expected at least 3 transient failures to trip the circuit" + ); + + // Circuit is Open — signal must NOT be consumed during cooldown + publisher + .publish("events", &BlockEvent { block_number: 99 }) + .await + .unwrap(); + + tokio::time::sleep(Duration::from_millis(1000)).await; + assert!( + !signal_received.load(Ordering::SeqCst), + "signal message must not be consumed while the circuit is Open" + ); + + // After cooldown: PEL drained → Half-Open → signal consumed + let deadline = tokio::time::Instant::now() + Duration::from_secs(15); + while !signal_received.load(Ordering::SeqCst) && tokio::time::Instant::now() < deadline { + tokio::time::sleep(Duration::from_millis(200)).await; + } + + publisher.shutdown().await; + consumer_handle.abort(); + assert!( + signal_received.load(Ordering::SeqCst), + "signal message must be consumed after the CB cooldown expires" + ); +} + +// ── Test 7: Multichain namespace isolation via Broker facade ──────────────── + +/// Two chains (ethereum, polygon) with separate namespaces. Multiple consumer +/// groups on the same stream (fanout) and separate streams (routing). +/// Validates that the Broker facade correctly isolates namespaces. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_broker_multichain_namespace_isolation() { + let url = shared_redis_url().await; + let broker = Broker::redis(url).await.unwrap(); + + // Namespace-scoped publishers + let eth_publisher = broker.publisher("bf-mc-eth").await.unwrap(); + let polygon_publisher = broker.publisher("bf-mc-polygon").await.unwrap(); + + // Topics + let eth_blocks_topic = Topic::namespaced("bf-mc-eth", "blocks"); + let polygon_blocks_topic = Topic::namespaced("bf-mc-polygon", "blocks"); + let eth_control_topic = Topic::namespaced("bf-mc-eth", "control"); + + // Counters + let app_a_eth = Arc::new(AtomicU32::new(0)); + let app_a_eth_numbers = Arc::new(Mutex::new(HashSet::::new())); + let app_b_eth = Arc::new(AtomicU32::new(0)); + let app_b_eth_numbers = Arc::new(Mutex::new(HashSet::::new())); + let app_a_polygon = Arc::new(AtomicU32::new(0)); + let app_a_polygon_numbers = Arc::new(Mutex::new(HashSet::::new())); + let watch_count = Arc::new(AtomicU32::new(0)); + let watch_received = Arc::new(Mutex::new(None::)); + + // App A — ETH blocks + let app_a_eth_handler = { + let count = app_a_eth.clone(); + let numbers = app_a_eth_numbers.clone(); + AsyncHandlerPayloadOnly::new(move |event: ListenerEvent| { + let count = count.clone(); + let numbers = numbers.clone(); + async move { + if let ListenerEvent::Block { block_number, .. } = event { + count.fetch_add(1, Ordering::SeqCst); + numbers.lock().unwrap().insert(block_number); + } + Ok::<(), std::convert::Infallible>(()) + } + }) + }; + + // App B — ETH blocks (different group = fanout) + let app_b_eth_handler = { + let count = app_b_eth.clone(); + let numbers = app_b_eth_numbers.clone(); + AsyncHandlerPayloadOnly::new(move |event: ListenerEvent| { + let count = count.clone(); + let numbers = numbers.clone(); + async move { + if let ListenerEvent::Block { block_number, .. } = event { + count.fetch_add(1, Ordering::SeqCst); + numbers.lock().unwrap().insert(block_number); + } + Ok::<(), std::convert::Infallible>(()) + } + }) + }; + + // App A — Polygon blocks + let app_a_polygon_handler = { + let count = app_a_polygon.clone(); + let numbers = app_a_polygon_numbers.clone(); + AsyncHandlerPayloadOnly::new(move |event: ListenerEvent| { + let count = count.clone(); + let numbers = numbers.clone(); + async move { + if let ListenerEvent::Block { block_number, .. } = event { + count.fetch_add(1, Ordering::SeqCst); + numbers.lock().unwrap().insert(block_number); + } + Ok::<(), std::convert::Infallible>(()) + } + }) + }; + + // Watch handler — ETH control channel + let watch_handler = { + let count = watch_count.clone(); + let received = watch_received.clone(); + AsyncHandlerPayloadOnly::new(move |event: ListenerEvent| { + let count = count.clone(); + let received = received.clone(); + async move { + if let ListenerEvent::WatchRegister { .. } = &event { + count.fetch_add(1, Ordering::SeqCst); + let mut r = received.lock().unwrap(); + if r.is_none() { + *r = Some(event); + } + } + Ok::<(), std::convert::Infallible>(()) + } + }) + }; + + // Spawn consumers + let b = broker.clone(); + let t = eth_blocks_topic.clone(); + let h1 = tokio::spawn(async move { + let _ = b + .consumer(&t) + .group("bf-mc-app-a") + .consumer_name("pod-1") + .prefetch(32) + .redis_block_ms(1000) + .run(app_a_eth_handler) + .await; + }); + + let b = broker.clone(); + let t = eth_blocks_topic.clone(); + let h2 = tokio::spawn(async move { + let _ = b + .consumer(&t) + .group("bf-mc-app-b") + .consumer_name("pod-1") + .prefetch(32) + .redis_block_ms(1000) + .run(app_b_eth_handler) + .await; + }); + + let b = broker.clone(); + let t = polygon_blocks_topic.clone(); + let h3 = tokio::spawn(async move { + let _ = b + .consumer(&t) + .group("bf-mc-app-a-polygon") + .consumer_name("pod-1") + .prefetch(32) + .redis_block_ms(1000) + .run(app_a_polygon_handler) + .await; + }); + + let b = broker.clone(); + let t = eth_control_topic.clone(); + let h4 = tokio::spawn(async move { + let _ = b + .consumer(&t) + .group("bf-mc-listener") + .consumer_name("pod-1") + .prefetch(16) + .redis_block_ms(1000) + .run(watch_handler) + .await; + }); + + tokio::time::sleep(Duration::from_millis(1200)).await; + + // Publish control message + eth_publisher + .publish( + "control", + &ListenerEvent::WatchRegister { + chain_id: 1, + consumer_id: "app-a".to_string(), + contract_addresses: vec!["0xabc".to_string()], + }, + ) + .await + .unwrap(); + + let eth_count: u32 = 20; + let polygon_count: u32 = 10; + + // Publish block events + for i in 0..u64::from(eth_count) { + eth_publisher + .publish( + "blocks", + &ListenerEvent::Block { + chain_id: 1, + block_number: 1_000_000 + i, + }, + ) + .await + .unwrap(); + } + for i in 0..u64::from(polygon_count) { + polygon_publisher + .publish( + "blocks", + &ListenerEvent::Block { + chain_id: 137, + block_number: 2_000_000 + i, + }, + ) + .await + .unwrap(); + } + + // Wait for all consumers to process + let deadline = tokio::time::Instant::now() + Duration::from_secs(20); + while tokio::time::Instant::now() < deadline { + let done = app_a_eth.load(Ordering::SeqCst) == eth_count + && app_b_eth.load(Ordering::SeqCst) == eth_count + && app_a_polygon.load(Ordering::SeqCst) == polygon_count + && watch_count.load(Ordering::SeqCst) == 1; + if done { + break; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + + eth_publisher.shutdown().await; + polygon_publisher.shutdown().await; + h1.abort(); + h2.abort(); + h3.abort(); + h4.abort(); + + // Fanout: both app-a and app-b get all ETH blocks + assert_eq!( + app_a_eth.load(Ordering::SeqCst), + eth_count, + "app-a should receive every ETH block" + ); + assert_eq!( + app_b_eth.load(Ordering::SeqCst), + eth_count, + "app-b should receive every ETH block (fanout)" + ); + + // Routing: polygon stream gets only polygon blocks + assert_eq!( + app_a_polygon.load(Ordering::SeqCst), + polygon_count, + "polygon consumer should receive only polygon blocks" + ); + + // Control channel + assert_eq!( + watch_count.load(Ordering::SeqCst), + 1, + "watch stream should receive exactly one control message" + ); + + // Content validation + let eth_expected: HashSet = (1_000_000..1_000_000 + u64::from(eth_count)).collect(); + let polygon_expected: HashSet = + (2_000_000..2_000_000 + u64::from(polygon_count)).collect(); + + assert_eq!( + *app_a_eth_numbers.lock().unwrap(), + eth_expected, + "app-a ETH should receive exactly the published block numbers" + ); + assert_eq!( + *app_b_eth_numbers.lock().unwrap(), + eth_expected, + "app-b ETH should receive exactly the published block numbers" + ); + assert_eq!( + *app_a_polygon_numbers.lock().unwrap(), + polygon_expected, + "polygon consumer should receive exactly the published block numbers" + ); + + let watch_payload = watch_received.lock().unwrap().clone(); + assert_eq!( + watch_payload, + Some(ListenerEvent::WatchRegister { + chain_id: 1, + consumer_id: "app-a".to_string(), + contract_addresses: vec!["0xabc".to_string()], + }), + "watch queue should receive the exact WatchRegister payload" + ); +} + +// ── Test 8: Classified handler — simple roundtrip ─────────────────────────── + +/// `AsyncHandlerPayloadClassified` processes a valid message and ACKs it. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_broker_classified_handler_roundtrip() { + let url = shared_redis_url().await; + let broker = Broker::redis(url).await.unwrap(); + + let publisher = broker.publisher("bf-cls-rt").await.unwrap(); + let topic = Topic::namespaced("bf-cls-rt", "events"); + + let count = Arc::new(AtomicU32::new(0)); + let count_clone = count.clone(); + let handler = AsyncHandlerPayloadClassified::new(move |event: BlockEvent| { + let c = count_clone.clone(); + async move { + assert_eq!(event.block_number, 42); + c.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + }); + + let consumer_broker = broker.clone(); + let consumer_handle = tokio::spawn(async move { + let _ = consumer_broker + .consumer(&topic) + .group("bf-cls-rt-group") + .consumer_name("consumer-1") + .prefetch(10) + .run(handler) + .await; + }); + + tokio::time::sleep(Duration::from_millis(400)).await; + + publisher + .publish("events", &BlockEvent { block_number: 42 }) + .await + .unwrap(); + + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + while count.load(Ordering::SeqCst) == 0 && tokio::time::Instant::now() < deadline { + tokio::time::sleep(Duration::from_millis(100)).await; + } + + publisher.shutdown().await; + consumer_handle.abort(); + assert_eq!( + count.load(Ordering::SeqCst), + 1, + "classified handler should be called exactly once" + ); +} + +// ── Test 9: Classified handler — transient errors trip circuit breaker ─────── + +/// `AsyncHandlerPayloadClassified` preserves `HandlerError::Transient`, which +/// trips the circuit breaker. After cooldown the consumer recovers and +/// processes a signal message. +/// +/// This is the key difference from `AsyncHandlerPayloadOnly` — that wrapper +/// maps all closure errors to `HandlerError::Execution`, which does NOT trip +/// the circuit breaker. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_broker_classified_transient_trips_circuit_breaker() { + let url = shared_redis_url().await; + let broker = Broker::redis(url).await.unwrap(); + + let publisher = broker.publisher("bf-cls-cb").await.unwrap(); + let topic = Topic::namespaced("bf-cls-cb", "events"); + + let transient_count = Arc::new(AtomicU32::new(0)); + let signal_received = Arc::new(AtomicBool::new(false)); + + let tc = transient_count.clone(); + let sr = signal_received.clone(); + let handler = AsyncHandlerPayloadClassified::new(move |event: BlockEvent| { + let tc = tc.clone(); + let sr = sr.clone(); + async move { + // Signal message — proves the CB recovered + if event.block_number == 99 { + sr.store(true, Ordering::SeqCst); + return Ok(()); + } + // Trip messages — return Transient to open the circuit + tc.fetch_add(1, Ordering::SeqCst); + Err(HandlerError::transient(std::io::Error::other( + "simulated infra failure", + ))) + } + }); + + let consumer_broker = broker.clone(); + let consumer_handle = tokio::spawn(async move { + let _ = consumer_broker + .consumer(&topic) + .group("bf-cls-cb-group") + .consumer_name("consumer-1") + .prefetch(10) + .max_retries(10) + .circuit_breaker(3, Duration::from_millis(2000)) + .redis_claim_min_idle(60) + .redis_claim_interval(10) + .run(handler) + .await; + }); + + tokio::time::sleep(Duration::from_millis(400)).await; + + // Publish 3 trip messages to open the circuit + for i in 0..3u64 { + publisher + .publish("events", &BlockEvent { block_number: i }) + .await + .unwrap(); + } + + // Wait until at least 3 transient failures recorded + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + while transient_count.load(Ordering::SeqCst) < 3 && tokio::time::Instant::now() < deadline { + tokio::time::sleep(Duration::from_millis(50)).await; + } + assert!( + transient_count.load(Ordering::SeqCst) >= 3, + "expected at least 3 transient failures to trip the circuit" + ); + + // Circuit is Open — signal must NOT be consumed during cooldown + publisher + .publish("events", &BlockEvent { block_number: 99 }) + .await + .unwrap(); + + tokio::time::sleep(Duration::from_millis(1000)).await; + assert!( + !signal_received.load(Ordering::SeqCst), + "signal message must not be consumed while the circuit is Open" + ); + + // After cooldown: Half-Open → signal consumed + let deadline = tokio::time::Instant::now() + Duration::from_secs(15); + while !signal_received.load(Ordering::SeqCst) && tokio::time::Instant::now() < deadline { + tokio::time::sleep(Duration::from_millis(200)).await; + } + + publisher.shutdown().await; + consumer_handle.abort(); + assert!( + signal_received.load(Ordering::SeqCst), + "signal message must be consumed after the CB cooldown expires — \ + proves Transient errors from AsyncHandlerPayloadClassified trip the CB" + ); +} + +// ── Test 10: Classified handler — permanent error → DLQ, CB untouched ─────── + +/// `AsyncHandlerPayloadClassified` with `HandlerError::permanent` routes the +/// message to the dead stream after max_retries, without tripping the circuit +/// breaker. A subsequent valid message is processed normally, proving the CB +/// stayed closed. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_broker_classified_permanent_dlq_no_cb_trip() { + let url = shared_redis_url().await; + let broker = Broker::redis(url).await.unwrap(); + + let publisher = broker.publisher("bf-cls-perm").await.unwrap(); + let topic = Topic::namespaced("bf-cls-perm", "events"); + let dead_stream = topic.dead_key().to_owned(); + + let success_count = Arc::new(AtomicU32::new(0)); + + let sc = success_count.clone(); + let handler = AsyncHandlerPayloadClassified::new(move |event: BlockEvent| { + let sc = sc.clone(); + async move { + if event.block_number == 0 { + // Permanent: the message itself is bad + return Err(HandlerError::permanent(std::io::Error::other( + "invalid block", + ))); + } + // Valid message — proves the CB is still closed + sc.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + }); + + let consumer_broker = broker.clone(); + let consumer_handle = tokio::spawn(async move { + let _ = consumer_broker + .consumer(&topic) + .group("bf-cls-perm-group") + .consumer_name("consumer-1") + .prefetch(10) + .max_retries(2) + .circuit_breaker(3, Duration::from_millis(30_000)) + .redis_claim_min_idle(1) + .redis_claim_interval(1) + .run(handler) + .await; + }); + + tokio::time::sleep(Duration::from_millis(400)).await; + + // Publish a permanently-bad message + publisher + .publish("events", &BlockEvent { block_number: 0 }) + .await + .unwrap(); + + // Wait for it to land in the dead stream + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + loop { + tokio::time::sleep(Duration::from_millis(500)).await; + let len = redis_xlen(url, &dead_stream).await; + if len >= 1 { + break; + } + if tokio::time::Instant::now() >= deadline { + publisher.shutdown().await; + consumer_handle.abort(); + panic!("permanent message was not moved to dead stream within timeout"); + } + } + + // Now publish a valid message — it must be processed immediately + // (proves the circuit breaker did NOT trip from permanent errors) + publisher + .publish("events", &BlockEvent { block_number: 42 }) + .await + .unwrap(); + + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + while success_count.load(Ordering::SeqCst) == 0 && tokio::time::Instant::now() < deadline { + tokio::time::sleep(Duration::from_millis(100)).await; + } + + publisher.shutdown().await; + consumer_handle.abort(); + + assert_eq!( + redis_xlen(url, &dead_stream).await, + 1, + "exactly one message should be in the dead stream" + ); + assert_eq!( + success_count.load(Ordering::SeqCst), + 1, + "valid message must be processed — CB should NOT have tripped from permanent errors" + ); +} diff --git a/listener/crates/shared/broker/tests/metrics_e2e.rs b/listener/crates/shared/broker/tests/metrics_e2e.rs new file mode 100644 index 0000000000..bea073b91a --- /dev/null +++ b/listener/crates/shared/broker/tests/metrics_e2e.rs @@ -0,0 +1,655 @@ +#![cfg(feature = "redis")] +//! End-to-end metrics acceptance tests for the `broker` crate. +//! +//! These tests verify that publishing and consuming messages causes the +//! expected `metrics` counters, gauges, and histograms to be recorded. +//! +//! Uses `metrics_util::debugging::DebuggingRecorder` to capture metrics +//! in-process without a Prometheus exporter. Because the global recorder +//! can only be set once per process, all tests in this file share a single +//! recorder installed in a `std::sync::OnceLock`. +//! +//! Run via: +//! +//! ```bash +//! make test-e2e-redis # or: +//! TMPDIR=/tmp/claude cargo test -p broker --features redis --test metrics_e2e -- --ignored --test-threads=1 +//! ``` + +use std::sync::{ + Arc, OnceLock, + atomic::{AtomicU32, Ordering}, +}; +use std::time::Duration; + +use broker::{ + AckDecision, AsyncHandlerPayloadClassified, AsyncHandlerPayloadOnly, Broker, CancellationToken, + Handler, HandlerError, Message, Topic, +}; +use metrics_util::debugging::{DebugValue, DebuggingRecorder, Snapshotter}; +use metrics_util::{CompositeKey, MetricKind}; +use ordered_float::OrderedFloat; +use test_support::shared_redis_url; + +// ── Global test recorder ──────────────────────────────────────────────────── + +static SNAPSHOTTER: OnceLock = OnceLock::new(); + +fn snapshotter() -> &'static Snapshotter { + SNAPSHOTTER.get_or_init(|| { + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + // This will fail if another test binary already set the global recorder, + // but within this test binary it runs exactly once. + let _ = recorder.install(); + snapshotter + }) +} + +// ── Assertion helpers ─────────────────────────────────────────────────────── + +/// Find a metric by name + kind + labels in the snapshot. +fn find_metric( + snap: &[( + CompositeKey, + Option, + Option, + DebugValue, + )], + kind: MetricKind, + name: &str, + labels: &[(&str, &str)], +) -> Option { + snap.iter() + .find(|(ck, _, _, _)| { + ck.kind() == kind + && ck.key().name() == name + && labels + .iter() + .all(|(k, v)| ck.key().labels().any(|l| l.key() == *k && l.value() == *v)) + }) + .map(|(_, _, _, v)| match v { + DebugValue::Counter(c) => DebugValue::Counter(*c), + DebugValue::Gauge(g) => DebugValue::Gauge(*g), + DebugValue::Histogram(h) => DebugValue::Histogram(h.clone()), + }) +} + +fn assert_counter_gte( + snap: &[( + CompositeKey, + Option, + Option, + DebugValue, + )], + name: &str, + min: u64, + labels: &[(&str, &str)], +) { + let value = find_metric(snap, MetricKind::Counter, name, labels); + match value { + Some(DebugValue::Counter(v)) => assert!( + v >= min, + "{name} = {v}, expected >= {min} (labels: {labels:?})" + ), + other => panic!("{name}: expected Counter >= {min}, got {other:?} (labels: {labels:?})"), + } +} + +#[allow(dead_code)] +fn assert_counter_eq( + snap: &[( + CompositeKey, + Option, + Option, + DebugValue, + )], + name: &str, + expected: u64, + labels: &[(&str, &str)], +) { + let value = find_metric(snap, MetricKind::Counter, name, labels); + match value { + Some(DebugValue::Counter(v)) => assert_eq!( + v, expected, + "{name} = {v}, expected {expected} (labels: {labels:?})" + ), + other => panic!("{name}: expected Counter({expected}), got {other:?} (labels: {labels:?})"), + } +} + +fn assert_histogram_has_observations( + snap: &[( + CompositeKey, + Option, + Option, + DebugValue, + )], + name: &str, + min_count: usize, + labels: &[(&str, &str)], +) { + let value = find_metric(snap, MetricKind::Histogram, name, labels); + match value { + Some(DebugValue::Histogram(values)) => assert!( + values.len() >= min_count, + "{name}: expected >= {min_count} observations, got {} (labels: {labels:?})", + values.len() + ), + other => panic!( + "{name}: expected Histogram with >= {min_count} obs, got {other:?} (labels: {labels:?})" + ), + } +} + +fn assert_gauge_eq( + snap: &[( + CompositeKey, + Option, + Option, + DebugValue, + )], + name: &str, + expected: f64, + labels: &[(&str, &str)], +) { + let value = find_metric(snap, MetricKind::Gauge, name, labels); + match value { + Some(DebugValue::Gauge(v)) => assert_eq!( + v, + OrderedFloat(expected), + "{name} = {v}, expected {expected} (labels: {labels:?})" + ), + other => panic!("{name}: expected Gauge({expected}), got {other:?} (labels: {labels:?})"), + } +} + +// ── Shared payload ────────────────────────────────────────────────────────── + +#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)] +struct TestEvent { + value: u64, +} + +/// Wait for an atomic counter to reach `target` within `timeout`. +async fn wait_for(counter: &AtomicU32, target: u32, timeout: Duration) { + let deadline = tokio::time::Instant::now() + timeout; + while counter.load(Ordering::SeqCst) < target && tokio::time::Instant::now() < deadline { + tokio::time::sleep(Duration::from_millis(50)).await; + } +} + +// ═══════════════════════════════════════════════════════════════════════════ +// AC-2.3: Publish N messages -> broker_messages_published_total == N +// ═══════════════════════════════════════════════════════════════════════════ + +#[tokio::test] +#[ignore = "requires Docker"] +async fn ac_2_3_redis_publish_increments_published_total() { + let snap = snapshotter(); + let url = shared_redis_url().await; + let broker = Broker::redis(url).await.unwrap(); + let publisher = broker.publisher("m-pub").await.unwrap(); + + publisher + .publish("test-pub", &TestEvent { value: 1 }) + .await + .unwrap(); + publisher + .publish("test-pub", &TestEvent { value: 2 }) + .await + .unwrap(); + + let snapshot = snap.snapshot().into_vec(); + assert_counter_gte( + &snapshot, + "broker_messages_published_total", + 2, + &[("backend", "redis"), ("topic", "m-pub.test-pub")], + ); +} + +// ═══════════════════════════════════════════════════════════════════════════ +// AC-2.5: Publish -> broker_publish_duration_seconds has observations +// ═══════════════════════════════════════════════════════════════════════════ + +#[tokio::test] +#[ignore = "requires Docker"] +async fn ac_2_5_redis_publish_records_duration_histogram() { + let snap = snapshotter(); + let url = shared_redis_url().await; + let broker = Broker::redis(url).await.unwrap(); + let publisher = broker.publisher("m-dur").await.unwrap(); + + publisher + .publish("test-dur", &TestEvent { value: 1 }) + .await + .unwrap(); + + let snapshot = snap.snapshot().into_vec(); + assert_histogram_has_observations( + &snapshot, + "broker_publish_duration_seconds", + 1, + &[("backend", "redis"), ("topic", "m-dur.test-dur")], + ); +} + +// ═══════════════════════════════════════════════════════════════════════════ +// AC-2.6: Consume ack -> broker_messages_consumed_total{outcome=ack} +// ═══════════════════════════════════════════════════════════════════════════ + +#[tokio::test] +#[ignore = "requires Docker"] +async fn ac_2_6_redis_consume_ack_increments_outcome_counter() { + let snap = snapshotter(); + let url = shared_redis_url().await; + let broker = Broker::redis(url).await.unwrap(); + let topic = Topic::namespaced("m-ack", "events"); + + let publisher = broker.publisher("m-ack").await.unwrap(); + publisher + .publish("events", &TestEvent { value: 42 }) + .await + .unwrap(); + + let processed = Arc::new(AtomicU32::new(0)); + let processed_clone = processed.clone(); + let handler = AsyncHandlerPayloadOnly::new(move |_: TestEvent| { + let c = processed_clone.clone(); + async move { + c.fetch_add(1, Ordering::SeqCst); + Ok::<(), std::io::Error>(()) + } + }); + + let cancel = CancellationToken::new(); + let cancel_clone = cancel.clone(); + let consumer_broker = broker.clone(); + let consumer_handle = tokio::spawn(async move { + let _ = consumer_broker + .consumer(&topic) + .group("m-ack-group") + .prefetch(1) + .with_cancellation(cancel_clone) + .run(handler) + .await; + }); + + wait_for(&processed, 1, Duration::from_secs(8)).await; + cancel.cancel(); + let _ = consumer_handle.await; + + assert!( + processed.load(Ordering::SeqCst) >= 1, + "handler should have processed at least 1 message" + ); + + let snapshot = snap.snapshot().into_vec(); + assert_counter_gte( + &snapshot, + "broker_messages_consumed_total", + 1, + &[ + ("backend", "redis"), + ("topic", "m-ack.events"), + ("outcome", "ack"), + ], + ); +} + +// ═══════════════════════════════════════════════════════════════════════════ +// AC-2.7: Consume -> broker_handler_duration_seconds has observations +// ═══════════════════════════════════════════════════════════════════════════ + +#[tokio::test] +#[ignore = "requires Docker"] +async fn ac_2_7_redis_consume_records_handler_duration() { + let snap = snapshotter(); + let url = shared_redis_url().await; + let broker = Broker::redis(url).await.unwrap(); + let topic = Topic::namespaced("m-hdur", "events"); + + let publisher = broker.publisher("m-hdur").await.unwrap(); + publisher + .publish("events", &TestEvent { value: 99 }) + .await + .unwrap(); + + let processed = Arc::new(AtomicU32::new(0)); + let processed_clone = processed.clone(); + let handler = AsyncHandlerPayloadOnly::new(move |_: TestEvent| { + let c = processed_clone.clone(); + async move { + c.fetch_add(1, Ordering::SeqCst); + Ok::<(), std::io::Error>(()) + } + }); + + let cancel = CancellationToken::new(); + let cancel_clone = cancel.clone(); + let consumer_broker = broker.clone(); + let consumer_handle = tokio::spawn(async move { + let _ = consumer_broker + .consumer(&topic) + .group("m-hdur-group") + .prefetch(1) + .with_cancellation(cancel_clone) + .run(handler) + .await; + }); + + wait_for(&processed, 1, Duration::from_secs(8)).await; + // The histogram is recorded in the worker task after handler.call() returns, + // but processed is incremented inside handler.call(). Give the worker task + // time to record the histogram and send the result through the mpsc channel. + tokio::time::sleep(Duration::from_millis(300)).await; + // Take snapshot BEFORE cancelling — histogram values are cleared on snapshot + let snapshot = snap.snapshot().into_vec(); + cancel.cancel(); + let _ = consumer_handle.await; + + assert_histogram_has_observations( + &snapshot, + "broker_handler_duration_seconds", + 1, + &[("backend", "redis"), ("topic", "m-hdur.events")], + ); +} + +// ═══════════════════════════════════════════════════════════════════════════ +// AC-2.8: Permanent error -> consumed_total{outcome=permanent} +// ═══════════════════════════════════════════════════════════════════════════ + +#[tokio::test] +#[ignore = "requires Docker"] +async fn ac_2_8_redis_permanent_error_records_permanent_outcome() { + let snap = snapshotter(); + let url = shared_redis_url().await; + let broker = Broker::redis(url).await.unwrap(); + let topic = Topic::namespaced("m-perm", "events"); + + let publisher = broker.publisher("m-perm").await.unwrap(); + publisher + .publish("events", &TestEvent { value: 1 }) + .await + .unwrap(); + + let call_count = Arc::new(AtomicU32::new(0)); + let call_count_clone = call_count.clone(); + let handler = AsyncHandlerPayloadClassified::new(move |_: TestEvent| { + let c = call_count_clone.clone(); + async move { + c.fetch_add(1, Ordering::SeqCst); + Err(HandlerError::permanent(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "bad payload", + ))) + } + }); + + let cancel = CancellationToken::new(); + let cancel_clone = cancel.clone(); + let consumer_broker = broker.clone(); + let consumer_handle = tokio::spawn(async move { + let _ = consumer_broker + .consumer(&topic) + .group("m-perm-group") + .prefetch(1) + .max_retries(2) + .with_cancellation(cancel_clone) + .run(handler) + .await; + }); + + wait_for(&call_count, 1, Duration::from_secs(8)).await; + cancel.cancel(); + let _ = consumer_handle.await; + + let snapshot = snap.snapshot().into_vec(); + assert_counter_gte( + &snapshot, + "broker_messages_consumed_total", + 1, + &[ + ("backend", "redis"), + ("topic", "m-perm.events"), + ("outcome", "permanent"), + ], + ); +} + +// ═══════════════════════════════════════════════════════════════════════════ +// AC-2.9: Transient error -> consumed_total{outcome=transient} +// ═══════════════════════════════════════════════════════════════════════════ + +#[tokio::test] +#[ignore = "requires Docker"] +async fn ac_2_9_redis_transient_error_records_transient_outcome() { + let snap = snapshotter(); + let url = shared_redis_url().await; + let broker = Broker::redis(url).await.unwrap(); + let topic = Topic::namespaced("m-trans", "events"); + + let publisher = broker.publisher("m-trans").await.unwrap(); + publisher + .publish("events", &TestEvent { value: 1 }) + .await + .unwrap(); + + let call_count = Arc::new(AtomicU32::new(0)); + let call_count_clone = call_count.clone(); + let handler = AsyncHandlerPayloadClassified::new(move |_: TestEvent| { + let c = call_count_clone.clone(); + async move { + c.fetch_add(1, Ordering::SeqCst); + Err(HandlerError::transient(std::io::Error::new( + std::io::ErrorKind::ConnectionReset, + "db down", + ))) + } + }); + + let cancel = CancellationToken::new(); + let cancel_clone = cancel.clone(); + let consumer_broker = broker.clone(); + let consumer_handle = tokio::spawn(async move { + let _ = consumer_broker + .consumer(&topic) + .group("m-trans-group") + .prefetch(1) + .with_cancellation(cancel_clone) + .run(handler) + .await; + }); + + wait_for(&call_count, 1, Duration::from_secs(8)).await; + // Let it process briefly so the outcome is recorded + tokio::time::sleep(Duration::from_millis(200)).await; + cancel.cancel(); + let _ = consumer_handle.await; + + let snapshot = snap.snapshot().into_vec(); + assert_counter_gte( + &snapshot, + "broker_messages_consumed_total", + 1, + &[ + ("backend", "redis"), + ("topic", "m-trans.events"), + ("outcome", "transient"), + ], + ); +} + +// ═══════════════════════════════════════════════════════════════════════════ +// AC-2.11: AckDecision::Dead -> dead_lettered{reason=handler_requested} +// ═══════════════════════════════════════════════════════════════════════════ + +#[tokio::test] +#[ignore = "requires Docker"] +async fn ac_2_11_redis_dead_decision_increments_dead_lettered() { + let snap = snapshotter(); + let url = shared_redis_url().await; + let broker = Broker::redis(url).await.unwrap(); + let topic = Topic::namespaced("m-dead", "events"); + + let publisher = broker.publisher("m-dead").await.unwrap(); + publisher + .publish("events", &TestEvent { value: 1 }) + .await + .unwrap(); + + /// Handler that always returns `AckDecision::Dead`. + #[derive(Clone)] + struct DeadHandler { + call_count: Arc, + } + #[async_trait::async_trait] + impl Handler for DeadHandler { + async fn call(&self, _msg: &Message) -> Result { + self.call_count.fetch_add(1, Ordering::SeqCst); + Ok(AckDecision::Dead) + } + } + + let call_count = Arc::new(AtomicU32::new(0)); + let handler = DeadHandler { + call_count: call_count.clone(), + }; + + let cancel = CancellationToken::new(); + let cancel_clone = cancel.clone(); + let consumer_broker = broker.clone(); + let consumer_handle = tokio::spawn(async move { + let _ = consumer_broker + .consumer(&topic) + .group("m-dead-group") + .prefetch(1) + .with_cancellation(cancel_clone) + .run(handler) + .await; + }); + + wait_for(&call_count, 1, Duration::from_secs(8)).await; + tokio::time::sleep(Duration::from_millis(200)).await; + cancel.cancel(); + let _ = consumer_handle.await; + + let snapshot = snap.snapshot().into_vec(); + assert_counter_gte( + &snapshot, + "broker_messages_dead_lettered_total", + 1, + &[ + ("backend", "redis"), + ("topic", "m-dead.events"), + ("reason", "handler_requested"), + ], + ); +} + +// ═══════════════════════════════════════════════════════════════════════════ +// AC-2.13: First delivery -> delivery_count histogram observation +// ═══════════════════════════════════════════════════════════════════════════ + +#[tokio::test] +#[ignore = "requires Docker"] +async fn ac_2_13_redis_delivery_count_histogram_recorded() { + let snap = snapshotter(); + let url = shared_redis_url().await; + let broker = Broker::redis(url).await.unwrap(); + let topic = Topic::namespaced("m-delcnt", "events"); + + let publisher = broker.publisher("m-delcnt").await.unwrap(); + publisher + .publish("events", &TestEvent { value: 1 }) + .await + .unwrap(); + + let processed = Arc::new(AtomicU32::new(0)); + let processed_clone = processed.clone(); + let handler = AsyncHandlerPayloadOnly::new(move |_: TestEvent| { + let c = processed_clone.clone(); + async move { + c.fetch_add(1, Ordering::SeqCst); + Ok::<(), std::io::Error>(()) + } + }); + + let cancel = CancellationToken::new(); + let cancel_clone = cancel.clone(); + let consumer_broker = broker.clone(); + let consumer_handle = tokio::spawn(async move { + let _ = consumer_broker + .consumer(&topic) + .group("m-delcnt-group") + .prefetch(1) + .with_cancellation(cancel_clone) + .run(handler) + .await; + }); + + wait_for(&processed, 1, Duration::from_secs(8)).await; + let snapshot = snap.snapshot().into_vec(); + cancel.cancel(); + let _ = consumer_handle.await; + + assert_histogram_has_observations( + &snapshot, + "broker_message_delivery_count", + 1, + &[("backend", "redis"), ("topic", "m-delcnt.events")], + ); +} + +// ═══════════════════════════════════════════════════════════════════════════ +// AC-2.2: record_queue_depths sets correct gauges +// ═══════════════════════════════════════════════════════════════════════════ + +#[tokio::test] +#[ignore = "requires Docker"] +async fn ac_2_2_record_queue_depths_sets_gauges() { + let snap = snapshotter(); + + let depths = broker::QueueDepths { + principal: 42, + retry: Some(5), + dead_letter: 3, + pending: Some(10), + lag: Some(7), + }; + broker::metrics::record_queue_depths(&depths, "redis", "m-depth.test"); + + let snapshot = snap.snapshot().into_vec(); + assert_gauge_eq( + &snapshot, + "broker_queue_depth_principal", + 42.0, + &[("backend", "redis"), ("topic", "m-depth.test")], + ); + assert_gauge_eq( + &snapshot, + "broker_queue_depth_retry", + 5.0, + &[("backend", "redis"), ("topic", "m-depth.test")], + ); + assert_gauge_eq( + &snapshot, + "broker_queue_depth_dead_letter", + 3.0, + &[("backend", "redis"), ("topic", "m-depth.test")], + ); + assert_gauge_eq( + &snapshot, + "broker_queue_depth_pending", + 10.0, + &[("backend", "redis"), ("topic", "m-depth.test")], + ); + assert_gauge_eq( + &snapshot, + "broker_queue_depth_lag", + 7.0, + &[("backend", "redis"), ("topic", "m-depth.test")], + ); +} diff --git a/listener/crates/shared/broker/tests/redis_e2e.rs b/listener/crates/shared/broker/tests/redis_e2e.rs new file mode 100644 index 0000000000..796343ee80 --- /dev/null +++ b/listener/crates/shared/broker/tests/redis_e2e.rs @@ -0,0 +1,2206 @@ +#![cfg(feature = "redis")] +//! End-to-end integration tests for `broker::redis`. +//! +//! Each test follows the three-step pattern from the `broker` README: +//! 1. Define a payload type. +//! 2. Write a handler with `broker::AsyncHandlerPayloadOnly`. +//! 3. Pick the `broker::redis` backend, build config, and call `consumer.run_*()`. +//! +//! All tests share a single Redis container (`e2e-redis`) using +//! `ReuseDirective::CurrentSession` so the container survives the handle drop +//! inside the `OnceCell` init closure. Each test uses unique stream names to +//! avoid cross-contamination. +//! +//! ⚠️ Always run via Make — the Makefile removes the named container before +//! and after the suite so it is not left running between runs: +//! +//! ```bash +//! make test-e2e-redis +//! ``` +//! +//! Running `cargo test -p broker --features redis --test redis_e2e -- --include-ignored` directly +//! will leave `e2e-redis` running after the suite finishes. + +use std::{ + collections::{HashMap, HashSet}, + future::Future, + process::Command, + sync::{ + Arc, Mutex, + atomic::{AtomicBool, AtomicU32, Ordering}, + }, + time::Duration, +}; + +// Traits and shared types come from `broker` — the backend-agnostic layer. +use broker::traits::{Consumer, DynPublisher}; +#[allow(unused_imports)] // Publisher brings .publish() into scope. +use broker::{AckDecision, AsyncHandlerPayloadOnly}; +// Backend-specific construction (connection, config builder, concrete types) comes from `broker::redis`. +use broker::redis::{ + RedisConnectionManager, RedisConsumer, RedisConsumerConfigBuilder, RedisPublisher, + StreamManager, StreamTopology, +}; +use test_support::shared_redis_url; + +// ── Step 1: shared test payload ─────────────────────────────────────────────── + +#[derive(serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq)] +struct BlockEvent { + block_number: u64, +} + +#[derive(serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq)] +#[serde(tag = "kind", rename_all = "snake_case")] +enum ListenerEvent { + Block { + chain_id: u64, + block_number: u64, + }, + WatchRegister { + chain_id: u64, + consumer_id: String, + contract_addresses: Vec, + }, +} + +/// Publish a synthetic block stream at a fixed rate. +/// +/// This simulates the listener's polling loop producing canonical blocks and +/// publishing them to a chain-specific Redis stream through `DynPublisher`. +async fn publish_blocks_at_rps( + publisher: &DynPublisher, + stream: &str, + chain_id: u64, + start_block: u64, + count: u64, + rps: u64, +) -> Result<(), broker::traits::DynPublishError> { + let per_message_delay = Duration::from_millis((1000 / rps.max(1)).max(1)); + for i in 0..count { + publisher + .publish( + stream, + &ListenerEvent::Block { + chain_id, + block_number: start_block + i, + }, + ) + .await?; + tokio::time::sleep(per_message_delay).await; + } + Ok(()) +} + +// ── Generic Consumer trait helper ───────────────────────────────────────────── + +/// Assert that a single message published via `publish_fn` is received exactly once. +/// +/// Bound on `broker::traits::Consumer` — exercises the trait directly, independent of which +/// backend (`RedisConsumer`, `RmqConsumer`, …) is passed in. +async fn assert_simple_roundtrip(consumer: C, config: C::PrefetchConfig, publish_fn: F) +where + C: Consumer + 'static, + F: Future, +{ + let count = Arc::new(AtomicU32::new(0)); + let count_clone = count.clone(); + + // Step 2: handler — from broker + let handler = AsyncHandlerPayloadOnly::new(move |_: serde_json::Value| { + let c = count_clone.clone(); + async move { + c.fetch_add(1, Ordering::SeqCst); + Ok::<(), std::io::Error>(()) + } + }); + + // Step 3: run consumer in background + let handle = tokio::spawn(async move { + let _ = consumer.run(config, handler).await; + }); + + // Give the consumer time to start and create the Redis consumer group. + tokio::time::sleep(Duration::from_millis(400)).await; + + publish_fn.await; + + // Poll until the handler is called or 5 s elapse. + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + while count.load(Ordering::SeqCst) == 0 && tokio::time::Instant::now() < deadline { + tokio::time::sleep(Duration::from_millis(100)).await; + } + + handle.abort(); + assert_eq!( + count.load(Ordering::SeqCst), + 1, + "expected handler to be called exactly once" + ); +} + +fn docker_container_control(action: &str, container_id: &str) { + let status = Command::new("docker") + .arg(action) + .arg(container_id) + .status() + .unwrap_or_else(|e| panic!("failed to run `docker {action}` for {container_id}: {e}")); + assert!( + status.success(), + "`docker {action}` failed for container {container_id}" + ); +} + +async fn read_classification_marker(url: &str, marker_key: &str, msg_id: &str) -> Option { + let client = redis::Client::open(url).unwrap(); + let mut raw_conn = match client.get_multiplexed_async_connection().await { + Ok(conn) => conn, + Err(_) => return None, + }; + + redis::cmd("HGET") + .arg(marker_key) + .arg(msg_id) + .query_async::>(&mut raw_conn) + .await + .ok() + .flatten() +} + +async fn wait_for_classification_marker( + url: &str, + marker_key: &str, + msg_id: &str, + expected: &str, + deadline_timeout: Duration, +) { + let deadline = tokio::time::Instant::now() + deadline_timeout; + + loop { + let marker = read_classification_marker(url, marker_key, msg_id).await; + if marker.as_deref() == Some(expected) { + return; + } + assert!( + tokio::time::Instant::now() < deadline, + "classification marker was not persisted after Redis recovery (msg_id={msg_id}, marker={marker:?})" + ); + tokio::time::sleep(Duration::from_millis(250)).await; + } +} + +// ── Test 1: run_simple roundtrip ────────────────────────────────────────────── + +/// Publish one message → `run_simple` consumer receives and ACKs it. +/// Exercises the generic `broker::traits::Consumer` trait via `assert_simple_roundtrip`. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_run_simple_roundtrip() { + let url = shared_redis_url().await; + + // Step 3 (backend): publisher + consumer from broker_redis + let conn = RedisConnectionManager::new(url).await.unwrap(); + let publisher = RedisPublisher::builder(conn) + .auto_trim(Duration::from_secs(1)) + .build(); + + let config = RedisConsumerConfigBuilder::new() + .stream("simple.events") + .group_name("simple-group") + .consumer_name("consumer-1") + .dead_stream("simple.events:dead") + .prefetch_count(10) + .build_prefetch() + .unwrap(); + + let consumer = RedisConsumer::connect(url).await.unwrap(); + + assert_simple_roundtrip(consumer, config, async move { + // broker::traits::Publisher trait — topic = stream name for Redis + publisher + .publish("simple.events", &BlockEvent { block_number: 42 }) + .await + .unwrap(); + }) + .await; +} + +// ── Test 2: run_with_retry — handler fails then succeeds ────────────────────── + +/// Handler returns `Err` on the first two deliveries and `Ok` on the third. +/// The ClaimSweeper reclaims the message after each failure and re-delivers it. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_run_with_retry_eventually_succeeds() { + let url = shared_redis_url().await; + + // Step 3 (backend): config from broker_redis + let config = RedisConsumerConfigBuilder::new() + .stream("retry.events") + .group_name("retry-group") + .consumer_name("consumer-1") + .dead_stream("retry.events:dead") + .max_retries(5) + // Short idle/interval so the test finishes quickly. + .claim_min_idle(Duration::from_millis(300)) + .claim_interval(Duration::from_millis(400)) + .prefetch_count(10) + .build_prefetch() + .unwrap(); + + // Step 2: handler — from broker; fails on the first two calls, succeeds on the third + let call_count = Arc::new(AtomicU32::new(0)); + let call_count_clone = call_count.clone(); + let handler = AsyncHandlerPayloadOnly::new(move |_: BlockEvent| { + let c = call_count_clone.clone(); + async move { + let prev = c.fetch_add(1, Ordering::SeqCst); + if prev < 2 { + Err(std::io::Error::other("simulated failure")) + } else { + Ok(()) + } + } + }); + + // Step 3 (backend): consumer from broker_redis, run in background + let consumer = RedisConsumer::connect(url).await.unwrap(); + let consumer_handle = tokio::spawn(async move { + let _ = consumer.run(config, handler).await; + }); + + // Give the consumer time to start and create the group. + tokio::time::sleep(Duration::from_millis(400)).await; + + // Publish via mq::Publisher — topic = stream name + let conn = RedisConnectionManager::new(url).await.unwrap(); + let publisher = RedisPublisher::new(conn); + publisher + .publish("retry.events", &BlockEvent { block_number: 1 }) + .await + .unwrap(); + + // Each retry cycle: consumer blocks on XREADGROUP > for up to 5 s, then drains PEL. + // 3 cycles × ~5 s + claim overhead ≈ 17 s — use 30 s for safety. + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + while call_count.load(Ordering::SeqCst) < 3 && tokio::time::Instant::now() < deadline { + tokio::time::sleep(Duration::from_millis(200)).await; + } + + publisher.shutdown().await; + consumer_handle.abort(); + assert!( + call_count.load(Ordering::SeqCst) >= 3, + "handler should be called at least 3 times (2 failures + 1 success), got {}", + call_count.load(Ordering::SeqCst) + ); +} + +// ── Test 3: multi-publish roundtrip ─────────────────────────────────────────── + +/// Publish 3 messages sequentially; consumer receives all 3. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_multi_publish_roundtrip() { + let url = shared_redis_url().await; + + // Step 3 (backend): config from broker_redis + let config = RedisConsumerConfigBuilder::new() + .stream("batch.events") + .group_name("batch-group") + .consumer_name("consumer-1") + .dead_stream("batch.events:dead") + .prefetch_count(10) + .build_prefetch() + .unwrap(); + + // Step 2: handler — from broker + let count = Arc::new(AtomicU32::new(0)); + let count_clone = count.clone(); + let handler = AsyncHandlerPayloadOnly::new(move |_: BlockEvent| { + let c = count_clone.clone(); + async move { + c.fetch_add(1, Ordering::SeqCst); + Ok::<(), std::io::Error>(()) + } + }); + + // Step 3 (backend): consumer from broker_redis, run in background + let consumer = RedisConsumer::connect(url).await.unwrap(); + let consumer_handle = tokio::spawn(async move { + let _ = consumer.run(config, handler).await; + }); + + tokio::time::sleep(Duration::from_millis(400)).await; + + // Publish 3 messages via mq::Publisher — topic = stream name + let conn = RedisConnectionManager::new(url).await.unwrap(); + let publisher = RedisPublisher::new(conn); + let events = vec![ + BlockEvent { block_number: 1 }, + BlockEvent { block_number: 2 }, + BlockEvent { block_number: 3 }, + ]; + for event in &events { + publisher.publish("batch.events", event).await.unwrap(); + } + + // Poll until all 3 are received. + let deadline = tokio::time::Instant::now() + Duration::from_secs(8); + while count.load(Ordering::SeqCst) < 3 && tokio::time::Instant::now() < deadline { + tokio::time::sleep(Duration::from_millis(100)).await; + } + + consumer_handle.abort(); + assert_eq!( + count.load(Ordering::SeqCst), + 3, + "all 3 batch messages should be received" + ); +} + +// ── Test 4: run_with_retry → DLQ after max_retries ─────────────────────────── + +/// Handler always returns `Err`. After `max_retries` failed deliveries the +/// ClaimSweeper routes the message to the dead stream. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_run_with_retry_dlq_after_max_retries() { + let url = shared_redis_url().await; + + let dead_stream = "dlq.events:dead"; + + // Step 3 (backend): config from broker_redis + let config = RedisConsumerConfigBuilder::new() + .stream("dlq.events") + .group_name("dlq-group") + .consumer_name("consumer-1") + .dead_stream(dead_stream) + .max_retries(2) + // Short idle/interval for a fast test. + .claim_min_idle(Duration::from_millis(300)) + .claim_interval(Duration::from_millis(400)) + .prefetch_count(10) + .build_prefetch() + .unwrap(); + + // Step 2: handler — from broker; always fails so the message exhausts max_retries + let handler = AsyncHandlerPayloadOnly::new(|_: BlockEvent| async move { + Err::<(), _>(std::io::Error::other("always fails")) + }); + + // Step 3 (backend): consumer from broker_redis, run in background + let consumer = RedisConsumer::connect(url).await.unwrap(); + let consumer_handle = tokio::spawn(async move { + let _ = consumer.run(config, handler).await; + }); + + // Give the consumer time to start and create the group. + tokio::time::sleep(Duration::from_millis(400)).await; + + // Publish via mq::Publisher — topic = stream name + let conn = RedisConnectionManager::new(url).await.unwrap(); + let publisher = RedisPublisher::new(conn); + publisher + .publish("dlq.events", &BlockEvent { block_number: 99 }) + .await + .unwrap(); + + // Each retry cycle: consumer blocks on XREADGROUP > for up to 5 s before draining PEL. + // With max_retries=2: 2 delivery failures × ~5 s block + claim overhead ≈ 12 s — use 20 s. + let deadline = tokio::time::Instant::now() + Duration::from_secs(20); + loop { + tokio::time::sleep(Duration::from_millis(500)).await; + + // Check dead stream length via a raw Redis command. + let client = redis::Client::open(url).unwrap(); + let mut raw_conn = client.get_multiplexed_async_connection().await.unwrap(); + let len: u64 = redis::cmd("XLEN") + .arg(dead_stream) + .query_async(&mut raw_conn) + .await + .unwrap_or(0); + + if len >= 1 { + consumer_handle.abort(); + assert_eq!(len, 1, "exactly one message should be in the dead stream"); + return; + } + + if tokio::time::Instant::now() >= deadline { + consumer_handle.abort(); + panic!("message was not moved to dead stream within timeout"); + } + } +} + +// ── Test 5: AckDecision::Dead — immediate dead-stream without retry ─────────── + +/// Verifies that a handler returning `AckDecision::Dead` routes the message +/// directly to the dead stream on the **first delivery**, without waiting for +/// `claim_min_idle` to expire or exhausting `max_retries`. +/// +/// Key difference from `test_run_with_retry_dlq_after_max_retries`: +/// - Handler never returns `Err` — it explicitly returns `Ok(AckDecision::Dead)` +/// - Handler is called **exactly once** (no retry cycles) +/// - Message appears in dead stream immediately, not after claim_min_idle +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_ack_decision_dead_immediate() { + let url = shared_redis_url().await; + + let dead_stream = "immediate-dead.events:dead"; + + let config = RedisConsumerConfigBuilder::new() + .stream("immediate-dead.events") + .group_name("dead-group") + .consumer_name("consumer-1") + .dead_stream(dead_stream) + // high max_retries — the handler should never reach them + .max_retries(10) + // Long claim_min_idle — the message must NOT need the sweeper to route to DLQ + .claim_min_idle(Duration::from_secs(60)) + .claim_interval(Duration::from_secs(15)) + .prefetch_count(10) + .build_prefetch() + .unwrap(); + + let call_count = Arc::new(AtomicU32::new(0)); + let call_count_clone = call_count.clone(); + + // Custom handler that always returns AckDecision::Dead + struct DeadHandler(Arc); + + #[async_trait::async_trait] + impl broker::Handler for DeadHandler { + async fn call(&self, _msg: &broker::Message) -> Result { + self.0.fetch_add(1, Ordering::SeqCst); + Ok(AckDecision::Dead) + } + } + + let dead_handler = DeadHandler(call_count.clone()); + let _ = call_count_clone; // suppress unused warning + + let consumer = RedisConsumer::connect(url).await.unwrap(); + let consumer_handle = tokio::spawn(async move { + let _ = consumer.run(config, dead_handler).await; + }); + + tokio::time::sleep(Duration::from_millis(400)).await; + + let conn = RedisConnectionManager::new(url).await.unwrap(); + let publisher = RedisPublisher::new(conn); + publisher + .publish("immediate-dead.events", &BlockEvent { block_number: 42 }) + .await + .unwrap(); + + // Poll for dead stream entry — should appear within a few seconds (no claim delay) + let deadline = tokio::time::Instant::now() + Duration::from_secs(8); + loop { + tokio::time::sleep(Duration::from_millis(200)).await; + + let client = redis::Client::open(url).unwrap(); + let mut raw_conn = client.get_multiplexed_async_connection().await.unwrap(); + let len: u64 = redis::cmd("XLEN") + .arg(dead_stream) + .query_async(&mut raw_conn) + .await + .unwrap_or(0); + + if len >= 1 { + // Give a little extra time to confirm no retry deliveries happen + tokio::time::sleep(Duration::from_millis(800)).await; + consumer_handle.abort(); + + assert_eq!( + call_count.load(Ordering::SeqCst), + 1, + "handler should be called exactly once — AckDecision::Dead must not retry" + ); + assert_eq!(len, 1, "exactly one message should be in the dead stream"); + return; + } + + if tokio::time::Instant::now() >= deadline { + consumer_handle.abort(); + panic!( + "message was not moved to dead stream within timeout. \ + handler_calls={}, dead_stream_len={}", + call_count.load(Ordering::SeqCst), + len + ); + } + } +} + +// ── Test 6: Circuit Breaker — halts consumption during the cooldown window ──── +// +// Handler returns `HandlerError::Transient` for the first 3 deliveries, tripping +// the circuit (threshold=3), then `Ok(Ack)` for all subsequent deliveries. +// +// After the circuit opens we publish a "signal" message (block_number=99) and +// assert it is NOT consumed during the cooldown window, then IS consumed once +// the circuit transitions through Half-Open back to Closed. +// +// Implementation note for Redis: the 3 trip messages remain in the PEL (not +// XACKed). After the cooldown the consumer drains its own PEL first (all return +// Ok at that point), then picks up the signal from the stream. The signal_received +// flag is therefore the cleanest assertion target and works regardless of how many +// PEL re-deliveries occur. +// +// A long claim_min_idle (60 s) prevents the ClaimSweeper from interfering during +// the test window. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_circuit_breaker_halts_consumption() { + let url = shared_redis_url().await; + + let config = RedisConsumerConfigBuilder::new() + .stream("cb-halt.events") + .group_name("cb-halt-group") + .consumer_name("consumer-1") + .dead_stream("cb-halt.events:dead") + .max_retries(10) + // Long claim_min_idle keeps the ClaimSweeper from touching the PEL + // during the test window (< 10 s total). + .claim_min_idle(Duration::from_secs(60)) + .claim_interval(Duration::from_secs(10)) + .circuit_breaker_threshold(3) + .circuit_breaker_cooldown(Duration::from_millis(2000)) + .prefetch_count(10) + .build_prefetch() + .unwrap(); + + let transient_count = Arc::new(AtomicU32::new(0)); + let signal_received = Arc::new(AtomicBool::new(false)); + + // Handler: + // block_number == 99 → signal message: record and ACK immediately. + // otherwise → increment transient_count; return Transient for the + // first 3 calls (trips CB), Ok for subsequent calls + // (Half-Open probe and PEL re-deliveries after cooldown). + struct TrippingHandler { + transient_count: Arc, + signal_received: Arc, + } + + #[async_trait::async_trait] + impl broker::Handler for TrippingHandler { + async fn call(&self, msg: &broker::Message) -> Result { + if let Ok(event) = serde_json::from_slice::(&msg.payload) + && event.block_number == 99 + { + self.signal_received.store(true, Ordering::SeqCst); + return Ok(AckDecision::Ack); + } + let prev = self.transient_count.fetch_add(1, Ordering::SeqCst); + if prev < 3 { + Err(broker::HandlerError::Transient(Box::new( + std::io::Error::other("simulated infrastructure failure"), + ))) + } else { + Ok(AckDecision::Ack) + } + } + } + + let consumer = RedisConsumer::connect(url).await.unwrap(); + // Clone Arcs before the move closure so the outer test retains handles. + let transient_count_for_handler = transient_count.clone(); + let signal_received_for_handler = signal_received.clone(); + let consumer_handle = tokio::spawn(async move { + let _ = consumer + .run( + config, + TrippingHandler { + transient_count: transient_count_for_handler, + signal_received: signal_received_for_handler, + }, + ) + .await; + }); + + // Wait for the consumer to start and create the consumer group. + tokio::time::sleep(Duration::from_millis(400)).await; + + let conn = RedisConnectionManager::new(url).await.unwrap(); + let publisher = RedisPublisher::new(conn); + + // Publish 3 trip messages before the consumer's first XREADGROUP blocking call + // so all three arrive in a single batch and are processed sequentially, + // guaranteeing the CB trips on exactly the 3rd failure. + for i in 0..3u64 { + publisher + .publish("cb-halt.events", &BlockEvent { block_number: i }) + .await + .unwrap(); + } + + // Wait until at least 3 transient failures are recorded (circuit is now Open). + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + while transient_count.load(Ordering::SeqCst) < 3 && tokio::time::Instant::now() < deadline { + tokio::time::sleep(Duration::from_millis(50)).await; + } + assert!( + transient_count.load(Ordering::SeqCst) >= 3, + "expected at least 3 transient failures to trip the circuit" + ); + + // ── Circuit is Open ─────────────────────────────────────────────────────── + // Publish the signal message — it must NOT be consumed during the cooldown. + publisher + .publish("cb-halt.events", &BlockEvent { block_number: 99 }) + .await + .unwrap(); + + // Observe: mid-cooldown (1 s into the 2 s window), signal must still be unprocessed. + tokio::time::sleep(Duration::from_millis(1000)).await; + assert!( + !signal_received.load(Ordering::SeqCst), + "signal message must not be consumed while the circuit is Open" + ); + + // ── After cooldown: PEL drained (all Ok) → Half-Open → signal consumed ─── + let deadline = tokio::time::Instant::now() + Duration::from_secs(15); + while !signal_received.load(Ordering::SeqCst) && tokio::time::Instant::now() < deadline { + tokio::time::sleep(Duration::from_millis(200)).await; + } + + consumer_handle.abort(); + assert!( + signal_received.load(Ordering::SeqCst), + "signal message must be consumed after the CB cooldown expires" + ); +} + +// ── Test 7: Circuit Breaker — prevents DLQ pollution during a sustained outage ─ +// +// A handler that always returns `HandlerError::Transient` simulates a downstream +// outage (e.g., database unavailable). +// +// Transient failures now retry indefinitely and do not consume `max_retries`. +// Without a circuit breaker, the message remains in retry circulation. +// +// With CB (threshold=2, cooldown=5 s) the consumer pauses after 2 failures, +// preventing any further XREADGROUP reads. The ClaimSweeper is neutralized by a +// very long claim_min_idle (60 s) so it cannot accumulate delivery counts within +// the 3 s observation window — keeping the dead stream empty. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_circuit_breaker_prevents_dlq_pollution() { + let url = shared_redis_url().await; + + let dead_stream = "cb-dlq.events:dead"; + + let config = RedisConsumerConfigBuilder::new() + .stream("cb-dlq.events") + .group_name("cb-dlq-group") + .consumer_name("consumer-1") + .dead_stream(dead_stream) + .max_retries(3) + // claim_min_idle=60 s prevents the ClaimSweeper from incrementing + // delivery_count during the 3 s observation window. + .claim_min_idle(Duration::from_secs(60)) + .claim_interval(Duration::from_secs(10)) + .circuit_breaker_threshold(2) + .circuit_breaker_cooldown(Duration::from_secs(5)) + .prefetch_count(10) + .build_prefetch() + .unwrap(); + + // Handler: always returns Transient — simulates a sustained infrastructure outage. + struct AlwaysTransient; + + #[async_trait::async_trait] + impl broker::Handler for AlwaysTransient { + async fn call(&self, _msg: &broker::Message) -> Result { + Err(broker::HandlerError::Transient(Box::new( + std::io::Error::other("downstream is unavailable"), + ))) + } + } + + let consumer = RedisConsumer::connect(url).await.unwrap(); + let consumer_handle = tokio::spawn(async move { + let _ = consumer.run(config, AlwaysTransient).await; + }); + + // Wait for the consumer to start and create the consumer group. + tokio::time::sleep(Duration::from_millis(400)).await; + + let conn = RedisConnectionManager::new(url).await.unwrap(); + let publisher = RedisPublisher::new(conn); + + // Publish 2 messages — enough to trip the circuit (threshold=2). + publisher + .publish("cb-dlq.events", &BlockEvent { block_number: 1 }) + .await + .unwrap(); + publisher + .publish("cb-dlq.events", &BlockEvent { block_number: 2 }) + .await + .unwrap(); + + // Wait for the CB to open and for the observation window to elapse. + // cooldown = 5 s; we check at 3 s — still within the Open window. + tokio::time::sleep(Duration::from_secs(3)).await; + + // Verify the dead stream is empty using a raw Redis command. + let client = redis::Client::open(url).unwrap(); + let mut raw_conn = client.get_multiplexed_async_connection().await.unwrap(); + let dead_len: u64 = redis::cmd("XLEN") + .arg(dead_stream) + .query_async(&mut raw_conn) + .await + .unwrap_or(0); + + consumer_handle.abort(); + + assert_eq!( + dead_len, 0, + "circuit breaker must prevent messages from reaching the dead stream during a transient outage" + ); +} + +// ── Test 8: Transient failures retry indefinitely (bounded retry ignored) ────── +// +// With max_retries=1, transient failures should keep retrying and must never +// be moved to the dead stream by retry-budget exhaustion. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_transient_failures_retry_indefinitely() { + let url = shared_redis_url().await; + + let dead_stream = "transient-infinite.events:dead"; + let config = RedisConsumerConfigBuilder::new() + .stream("transient-infinite.events") + .group_name("transient-infinite-group") + .consumer_name("consumer-1") + .dead_stream(dead_stream) + .max_retries(1) + .claim_min_idle(Duration::from_millis(300)) + .claim_interval(Duration::from_millis(300)) + .prefetch_count(10) + .block_ms(1000) + .build_prefetch() + .unwrap(); + + struct AlwaysTransient(Arc); + + #[async_trait::async_trait] + impl broker::Handler for AlwaysTransient { + async fn call(&self, _msg: &broker::Message) -> Result { + self.0.fetch_add(1, Ordering::SeqCst); + Err(broker::HandlerError::Transient(Box::new( + std::io::Error::other("downstream is unavailable"), + ))) + } + } + + let attempts = Arc::new(AtomicU32::new(0)); + let attempts_for_handler = attempts.clone(); + let consumer = RedisConsumer::connect(url).await.unwrap(); + let consumer_handle = tokio::spawn(async move { + let _ = consumer + .run(config, AlwaysTransient(attempts_for_handler)) + .await; + }); + + tokio::time::sleep(Duration::from_millis(400)).await; + + let conn = RedisConnectionManager::new(url).await.unwrap(); + let publisher = RedisPublisher::new(conn); + publisher + .publish("transient-infinite.events", &BlockEvent { block_number: 1 }) + .await + .unwrap(); + + let attempts_deadline = tokio::time::Instant::now() + Duration::from_secs(12); + while tokio::time::Instant::now() < attempts_deadline { + if attempts.load(Ordering::SeqCst) >= 4 { + break; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + + let client = redis::Client::open(url).unwrap(); + let mut raw_conn = client.get_multiplexed_async_connection().await.unwrap(); + let dead_len: u64 = redis::cmd("XLEN") + .arg(dead_stream) + .query_async(&mut raw_conn) + .await + .unwrap_or(0); + + consumer_handle.abort(); + + assert!( + attempts.load(Ordering::SeqCst) >= 4, + "transient message should keep retrying even with max_retries=1; attempts={}", + attempts.load(Ordering::SeqCst) + ); + assert_eq!( + dead_len, 0, + "transient failures must not be moved to dead stream by max_retries" + ); +} + +// ── Test 9: mark_transient retries across Redis outage and recovers marker ──── +// +// Simulates Redis becoming unavailable exactly between handler completion and +// classification persistence. `mark_transient_safe` must block/retry and write +// the transient marker once Redis is back. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_mark_transient_retries_until_redis_recovers() { + use testcontainers::core::ImageExt; + use testcontainers::runners::AsyncRunner; + use testcontainers_modules::redis::Redis; + + let container = Redis::default().with_tag("6.2.0").start().await.unwrap(); + let port = container.get_host_port_ipv4(6379).await.unwrap(); + let url = format!("redis://127.0.0.1:{port}"); + + let stream = "classification-outage.events"; + let group = "classification-outage-group"; + let config = RedisConsumerConfigBuilder::new() + .stream(stream) + .group_name(group) + .consumer_name("consumer-1") + .dead_stream("classification-outage.events:dead") + .max_retries(2) + .claim_min_idle(Duration::from_secs(60)) + .claim_interval(Duration::from_secs(10)) + .prefetch_count(10) + .block_ms(500) + .build_prefetch() + .unwrap(); + let classification_key = config.retry.classification_marker_key(); + + let entered_handler = Arc::new(tokio::sync::Notify::new()); + let release_handler = Arc::new(tokio::sync::Notify::new()); + + struct BlockThenTransient { + entered: Arc, + release: Arc, + } + + #[async_trait::async_trait] + impl broker::Handler for BlockThenTransient { + async fn call(&self, _msg: &broker::Message) -> Result { + self.entered.notify_one(); + self.release.notified().await; + Err(broker::HandlerError::Transient(Box::new( + std::io::Error::other("simulated transient failure"), + ))) + } + } + + let consumer = RedisConsumer::connect(&url).await.unwrap(); + let entered_for_handler = Arc::clone(&entered_handler); + let release_for_handler = Arc::clone(&release_handler); + let consumer_handle = tokio::spawn(async move { + let _ = consumer + .run( + config, + BlockThenTransient { + entered: entered_for_handler, + release: release_for_handler, + }, + ) + .await; + }); + + tokio::time::sleep(Duration::from_millis(400)).await; + + let conn = RedisConnectionManager::new(&url).await.unwrap(); + let publisher = RedisPublisher::new(conn); + let msg_id = publisher + .publish(stream, &BlockEvent { block_number: 7 }) + .await + .unwrap(); + + tokio::time::timeout(Duration::from_secs(5), entered_handler.notified()) + .await + .expect("handler should receive the first delivery"); + + docker_container_control("pause", container.id()); + release_handler.notify_waiters(); + + tokio::time::sleep(Duration::from_millis(500)).await; + assert!( + !consumer_handle.is_finished(), + "consumer should stay alive and retry marker persistence while Redis is down" + ); + + docker_container_control("unpause", container.id()); + + wait_for_classification_marker( + &url, + &classification_key, + &msg_id, + "transient", + Duration::from_secs(30), + ) + .await; + + let final_marker = read_classification_marker(&url, &classification_key, &msg_id).await; + assert_eq!( + final_marker.as_deref(), + Some("transient"), + "classification marker key {} should contain transient for msg_id={}", + classification_key, + msg_id + ); + + consumer_handle.abort(); +} + +// ── Test 9: Transient -> Permanent transition re-enables bounded DLQ path ────── +// +// First delivery is transient (infinite path), subsequent deliveries are +// permanent; once permanent is observed, the message should be dead-lettered +// according to max_retries. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_transient_then_permanent_eventually_dead_letters() { + let url = shared_redis_url().await; + + let dead_stream = "transient-to-permanent.events:dead"; + let config = RedisConsumerConfigBuilder::new() + .stream("transient-to-permanent.events") + .group_name("transient-to-permanent-group") + .consumer_name("consumer-1") + .dead_stream(dead_stream) + .max_retries(1) + .claim_min_idle(Duration::from_millis(300)) + .claim_interval(Duration::from_millis(300)) + .prefetch_count(10) + .block_ms(1000) + .build_prefetch() + .unwrap(); + + struct TransientThenPermanent(Arc); + + #[async_trait::async_trait] + impl broker::Handler for TransientThenPermanent { + async fn call(&self, _msg: &broker::Message) -> Result { + let call_index = self.0.fetch_add(1, Ordering::SeqCst); + if call_index == 0 { + Err(broker::HandlerError::Transient(Box::new( + std::io::Error::other("downstream is unavailable"), + ))) + } else { + Err(broker::HandlerError::permanent(std::io::Error::other( + "invalid payload", + ))) + } + } + } + + let calls = Arc::new(AtomicU32::new(0)); + let calls_for_handler = calls.clone(); + let consumer = RedisConsumer::connect(url).await.unwrap(); + let consumer_handle = tokio::spawn(async move { + let _ = consumer + .run(config, TransientThenPermanent(calls_for_handler)) + .await; + }); + + tokio::time::sleep(Duration::from_millis(400)).await; + + let conn = RedisConnectionManager::new(url).await.unwrap(); + let publisher = RedisPublisher::new(conn); + publisher + .publish( + "transient-to-permanent.events", + &BlockEvent { block_number: 1 }, + ) + .await + .unwrap(); + + let deadline = tokio::time::Instant::now() + Duration::from_secs(20); + let dead_len = loop { + tokio::time::sleep(Duration::from_millis(250)).await; + let client = redis::Client::open(url).unwrap(); + let mut raw_conn = client.get_multiplexed_async_connection().await.unwrap(); + let len: u64 = redis::cmd("XLEN") + .arg(dead_stream) + .query_async(&mut raw_conn) + .await + .unwrap_or(0); + if len >= 1 || tokio::time::Instant::now() >= deadline { + break len; + } + }; + + consumer_handle.abort(); + + assert!( + calls.load(Ordering::SeqCst) >= 2, + "handler should see transient then permanent calls; calls={}", + calls.load(Ordering::SeqCst) + ); + assert_eq!( + dead_len, 1, + "message should eventually dead-letter after becoming permanent" + ); +} + +// ── Test 10: run — steady-state pending retry ──────────────── + +/// Handler fails on first delivery; message stays in PEL. The periodic +/// XREADGROUP "0" drain picks it up during steady-state and the handler +/// succeeds on retry. Proves failed messages get handler retries before DLQ. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_run_retries_pending_in_steady_state() { + let url = shared_redis_url().await; + + let dead_stream = "prefetch-retry.events:dead"; + + let config = RedisConsumerConfigBuilder::new() + .stream("prefetch-retry.events") + .group_name("prefetch-retry-group") + .consumer_name("consumer-1") + .dead_stream(dead_stream) + .max_retries(3) + .claim_min_idle(Duration::from_millis(300)) + .claim_interval(Duration::from_millis(400)) + .prefetch_count(10) + .block_ms(2000) + .build_prefetch() + .unwrap(); + + let call_count = Arc::new(AtomicU32::new(0)); + + struct PrefetchRetryHandler(Arc); + + #[async_trait::async_trait] + impl broker::Handler for PrefetchRetryHandler { + async fn call(&self, _msg: &broker::Message) -> Result { + let prev = self.0.fetch_add(1, Ordering::SeqCst); + if prev < 1 { + Err(broker::HandlerError::permanent(std::io::Error::other( + "simulated failure", + ))) + } else { + Ok(AckDecision::Ack) + } + } + } + + impl Clone for PrefetchRetryHandler { + fn clone(&self) -> Self { + Self(self.0.clone()) + } + } + + let handler = PrefetchRetryHandler(call_count.clone()); + + let consumer = RedisConsumer::connect(url).await.unwrap(); + let consumer_handle = tokio::spawn(async move { + let _ = consumer.run(config, handler).await; + }); + + tokio::time::sleep(Duration::from_millis(400)).await; + + let conn = RedisConnectionManager::new(url).await.unwrap(); + let publisher = RedisPublisher::new(conn); + publisher + .publish("prefetch-retry.events", &BlockEvent { block_number: 1 }) + .await + .unwrap(); + + // Pending drain runs every 1 s. First delivery fails → PEL. Second delivery + // (from XREADGROUP 0) succeeds. Allow up to 15 s. + let deadline = tokio::time::Instant::now() + Duration::from_secs(15); + while call_count.load(Ordering::SeqCst) < 2 && tokio::time::Instant::now() < deadline { + tokio::time::sleep(Duration::from_millis(200)).await; + } + + consumer_handle.abort(); + + assert!( + call_count.load(Ordering::SeqCst) >= 2, + "handler should be called at least 2 times (1 failure + 1 success from steady-state pending drain), got {}", + call_count.load(Ordering::SeqCst) + ); + + let client = redis::Client::open(url).unwrap(); + let mut raw_conn = client.get_multiplexed_async_connection().await.unwrap(); + let dead_len: u64 = redis::cmd("XLEN") + .arg(dead_stream) + .query_async(&mut raw_conn) + .await + .unwrap_or(0); + + assert_eq!( + dead_len, 0, + "message must not reach dead stream when handler succeeds on retry" + ); +} + +// ── Test 9: DynPublisher multichain fanout + control routing ─────────────────── + +/// Mirrors the AMQP dynpublisher multichain e2e with Redis semantics: +/// - fanout via separate consumer groups on the same stream +/// - routing via separate stream names (`*.blocks` / `*.control`) +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_dynpublisher_multichain_routing_for_multiple_apps() { + let url = shared_redis_url().await; + + let eth_blocks_stream = "dynpub.ethereum.events.blocks"; + let polygon_blocks_stream = "dynpub.polygon.events.blocks"; + let eth_control_stream = "dynpub.ethereum.events.control"; + + let eth_blocks_topology = StreamTopology { + main: eth_blocks_stream.to_string(), + dead: format!("{eth_blocks_stream}:dead"), + }; + let polygon_blocks_topology = StreamTopology { + main: polygon_blocks_stream.to_string(), + dead: format!("{polygon_blocks_stream}:dead"), + }; + let eth_control_topology = StreamTopology { + main: eth_control_stream.to_string(), + dead: format!("{eth_control_stream}:dead"), + }; + + let manager_conn = RedisConnectionManager::new(url).await.unwrap(); + let stream_manager = StreamManager::new(manager_conn); + stream_manager + .ensure_topology(ð_blocks_topology) + .await + .unwrap(); + stream_manager + .ensure_topology(&polygon_blocks_topology) + .await + .unwrap(); + stream_manager + .ensure_topology(ð_control_topology) + .await + .unwrap(); + + let app_a_eth_blocks = Arc::new(AtomicU32::new(0)); + let app_a_eth_block_numbers = Arc::new(Mutex::new(HashSet::::new())); + let app_a_eth_unexpected = Arc::new(AtomicU32::new(0)); + let app_b_eth_blocks = Arc::new(AtomicU32::new(0)); + let app_b_eth_block_numbers = Arc::new(Mutex::new(HashSet::::new())); + let app_b_eth_unexpected = Arc::new(AtomicU32::new(0)); + let app_a_polygon_blocks = Arc::new(AtomicU32::new(0)); + let app_a_polygon_block_numbers = Arc::new(Mutex::new(HashSet::::new())); + let app_a_polygon_unexpected = Arc::new(AtomicU32::new(0)); + let watch_count = Arc::new(AtomicU32::new(0)); + let watch_received = Arc::new(Mutex::new(None::)); + let watch_unexpected = Arc::new(AtomicU32::new(0)); + + let app_a_eth_handler = { + let blocks = app_a_eth_blocks.clone(); + let block_numbers = app_a_eth_block_numbers.clone(); + let unexpected = app_a_eth_unexpected.clone(); + AsyncHandlerPayloadOnly::new(move |event: ListenerEvent| { + let blocks = blocks.clone(); + let block_numbers = block_numbers.clone(); + let unexpected = unexpected.clone(); + async move { + match event { + ListenerEvent::Block { + chain_id: 1, + block_number, + } => { + blocks.fetch_add(1, Ordering::SeqCst); + block_numbers.lock().unwrap().insert(block_number); + } + _ => { + unexpected.fetch_add(1, Ordering::SeqCst); + } + } + Ok::<(), std::convert::Infallible>(()) + } + }) + }; + + let app_b_eth_handler = { + let blocks = app_b_eth_blocks.clone(); + let block_numbers = app_b_eth_block_numbers.clone(); + let unexpected = app_b_eth_unexpected.clone(); + AsyncHandlerPayloadOnly::new(move |event: ListenerEvent| { + let blocks = blocks.clone(); + let block_numbers = block_numbers.clone(); + let unexpected = unexpected.clone(); + async move { + match event { + ListenerEvent::Block { + chain_id: 1, + block_number, + } => { + blocks.fetch_add(1, Ordering::SeqCst); + block_numbers.lock().unwrap().insert(block_number); + } + _ => { + unexpected.fetch_add(1, Ordering::SeqCst); + } + } + Ok::<(), std::convert::Infallible>(()) + } + }) + }; + + let app_a_polygon_handler = { + let blocks = app_a_polygon_blocks.clone(); + let block_numbers = app_a_polygon_block_numbers.clone(); + let unexpected = app_a_polygon_unexpected.clone(); + AsyncHandlerPayloadOnly::new(move |event: ListenerEvent| { + let blocks = blocks.clone(); + let block_numbers = block_numbers.clone(); + let unexpected = unexpected.clone(); + async move { + match event { + ListenerEvent::Block { + chain_id: 137, + block_number, + } => { + blocks.fetch_add(1, Ordering::SeqCst); + block_numbers.lock().unwrap().insert(block_number); + } + _ => { + unexpected.fetch_add(1, Ordering::SeqCst); + } + } + Ok::<(), std::convert::Infallible>(()) + } + }) + }; + + let watch_handler = { + let count = watch_count.clone(); + let received = watch_received.clone(); + let unexpected = watch_unexpected.clone(); + AsyncHandlerPayloadOnly::new(move |event: ListenerEvent| { + let count = count.clone(); + let received = received.clone(); + let unexpected = unexpected.clone(); + async move { + match &event { + ListenerEvent::WatchRegister { chain_id: 1, .. } => { + count.fetch_add(1, Ordering::SeqCst); + let mut r = received.lock().unwrap(); + if r.is_none() { + *r = Some(event.clone()); + } + } + _ => { + unexpected.fetch_add(1, Ordering::SeqCst); + } + } + Ok::<(), std::convert::Infallible>(()) + } + }) + }; + + let app_a_eth_config = RedisConsumerConfigBuilder::new() + .with_topology(ð_blocks_topology) + .group_name("dynpub-app-a-eth-group") + .consumer_name("dynpub-app-a-eth-consumer") + .max_retries(3) + .claim_min_idle(Duration::from_secs(2)) + .claim_interval(Duration::from_secs(1)) + .prefetch_count(32) + .block_ms(1000) + .build_prefetch() + .unwrap(); + + let app_b_eth_config = RedisConsumerConfigBuilder::new() + .with_topology(ð_blocks_topology) + .group_name("dynpub-app-b-eth-group") + .consumer_name("dynpub-app-b-eth-consumer") + .max_retries(3) + .claim_min_idle(Duration::from_secs(2)) + .claim_interval(Duration::from_secs(1)) + .prefetch_count(32) + .block_ms(1000) + .build_prefetch() + .unwrap(); + + let app_a_polygon_config = RedisConsumerConfigBuilder::new() + .with_topology(&polygon_blocks_topology) + .group_name("dynpub-app-a-polygon-group") + .consumer_name("dynpub-app-a-polygon-consumer") + .max_retries(3) + .claim_min_idle(Duration::from_secs(2)) + .claim_interval(Duration::from_secs(1)) + .prefetch_count(32) + .block_ms(1000) + .build_prefetch() + .unwrap(); + + let watch_config = RedisConsumerConfigBuilder::new() + .with_topology(ð_control_topology) + .group_name("dynpub-listener-watch-group") + .consumer_name("dynpub-listener-watch-consumer") + .max_retries(3) + .claim_min_idle(Duration::from_secs(2)) + .claim_interval(Duration::from_secs(1)) + .prefetch_count(16) + .block_ms(1000) + .build_prefetch() + .unwrap(); + + let app_a_eth_consumer = RedisConsumer::connect(url).await.unwrap(); + let app_b_eth_consumer = RedisConsumer::connect(url).await.unwrap(); + let app_a_polygon_consumer = RedisConsumer::connect(url).await.unwrap(); + let watch_consumer = RedisConsumer::connect(url).await.unwrap(); + + let app_a_eth_handle = tokio::spawn(async move { + let _ = app_a_eth_consumer + .run(app_a_eth_config, app_a_eth_handler) + .await; + }); + let app_b_eth_handle = tokio::spawn(async move { + let _ = app_b_eth_consumer + .run(app_b_eth_config, app_b_eth_handler) + .await; + }); + let app_a_polygon_handle = tokio::spawn(async move { + let _ = app_a_polygon_consumer + .run(app_a_polygon_config, app_a_polygon_handler) + .await; + }); + let watch_handle = tokio::spawn(async move { + let _ = watch_consumer.run(watch_config, watch_handler).await; + }); + + tokio::time::sleep(Duration::from_millis(1200)).await; + + let mut publishers: HashMap = HashMap::new(); + publishers.insert( + "ethereum".to_string(), + DynPublisher::new(RedisPublisher::new( + RedisConnectionManager::new(url).await.unwrap(), + )), + ); + publishers.insert( + "polygon".to_string(), + DynPublisher::new(RedisPublisher::new( + RedisConnectionManager::new(url).await.unwrap(), + )), + ); + + publishers["ethereum"] + .publish( + eth_control_stream, + &ListenerEvent::WatchRegister { + chain_id: 1, + consumer_id: "app-a".to_string(), + contract_addresses: vec!["0xabc0000000000000000000000000000000000001".to_string()], + }, + ) + .await + .unwrap(); + + let eth_expected: u32 = 24; + let polygon_expected: u32 = 11; + + let eth_pub = publishers + .get("ethereum") + .expect("ethereum publisher should exist") + .clone(); + let polygon_pub = publishers + .get("polygon") + .expect("polygon publisher should exist") + .clone(); + + let eth_publish_handle = tokio::spawn(async move { + publish_blocks_at_rps( + ð_pub, + eth_blocks_stream, + 1, + 1_000_000, + u64::from(eth_expected), + 20, + ) + .await + .unwrap(); + }); + + let polygon_publish_handle = tokio::spawn(async move { + publish_blocks_at_rps( + &polygon_pub, + polygon_blocks_stream, + 137, + 2_000_000, + u64::from(polygon_expected), + 12, + ) + .await + .unwrap(); + }); + + eth_publish_handle.await.unwrap(); + polygon_publish_handle.await.unwrap(); + + let deadline = tokio::time::Instant::now() + Duration::from_secs(20); + while tokio::time::Instant::now() < deadline { + let done = app_a_eth_blocks.load(Ordering::SeqCst) == eth_expected + && app_b_eth_blocks.load(Ordering::SeqCst) == eth_expected + && app_a_polygon_blocks.load(Ordering::SeqCst) == polygon_expected + && watch_count.load(Ordering::SeqCst) == 1; + if done { + break; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + + for publisher in publishers.values() { + publisher.shutdown().await; + } + + app_a_eth_handle.abort(); + app_b_eth_handle.abort(); + app_a_polygon_handle.abort(); + watch_handle.abort(); + + assert_eq!( + app_a_eth_blocks.load(Ordering::SeqCst), + eth_expected, + "app-a should receive every ETH block" + ); + assert_eq!( + app_b_eth_blocks.load(Ordering::SeqCst), + eth_expected, + "app-b should receive every ETH block" + ); + assert_eq!( + app_a_polygon_blocks.load(Ordering::SeqCst), + polygon_expected, + "polygon stream should receive only polygon blocks" + ); + assert_eq!( + watch_count.load(Ordering::SeqCst), + 1, + "watch stream should receive exactly one control message" + ); + + assert_eq!( + app_a_eth_unexpected.load(Ordering::SeqCst), + 0, + "app-a ETH queue should not receive unexpected payloads" + ); + assert_eq!( + app_b_eth_unexpected.load(Ordering::SeqCst), + 0, + "app-b ETH queue should not receive unexpected payloads" + ); + assert_eq!( + app_a_polygon_unexpected.load(Ordering::SeqCst), + 0, + "polygon queue should not receive unexpected payloads" + ); + assert_eq!( + watch_unexpected.load(Ordering::SeqCst), + 0, + "watch queue should not receive unexpected payloads" + ); + + // Content validation: verify received payloads match what was published. + let eth_expected_blocks: HashSet = + (1_000_000..1_000_000 + u64::from(eth_expected)).collect(); + let polygon_expected_blocks: HashSet = + (2_000_000..2_000_000 + u64::from(polygon_expected)).collect(); + + let app_a_eth_received = app_a_eth_block_numbers.lock().unwrap().clone(); + let app_b_eth_received = app_b_eth_block_numbers.lock().unwrap().clone(); + let app_a_polygon_received = app_a_polygon_block_numbers.lock().unwrap().clone(); + + assert_eq!( + app_a_eth_received, eth_expected_blocks, + "app-a ETH should receive exactly the published block numbers" + ); + assert_eq!( + app_b_eth_received, eth_expected_blocks, + "app-b ETH should receive exactly the published block numbers" + ); + assert_eq!( + app_a_polygon_received, polygon_expected_blocks, + "polygon consumer should receive exactly the published block numbers" + ); + + let watch_payload = watch_received.lock().unwrap().clone(); + let expected_watch = ListenerEvent::WatchRegister { + chain_id: 1, + consumer_id: "app-a".to_string(), + contract_addresses: vec!["0xabc0000000000000000000000000000000000001".to_string()], + }; + assert_eq!( + watch_payload.as_ref(), + Some(&expected_watch), + "watch queue should receive the exact WatchRegister payload" + ); +} + +// ── Test 10: prefetch-safe does not double-dispatch slow handlers ─────────────── + +/// Regression test for pending-drain overlap: +/// when handler latency exceeds the 1s pending tick, each stream ID must still +/// be dispatched exactly once while it is in-flight. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_run_no_duplicate_dispatch_when_handler_slow() { + let url = shared_redis_url().await; + + let stream = "prefetch.no-dup.slow.events"; + let dead_stream = "prefetch.no-dup.slow.events:dead"; + let expected_messages: u32 = 5; + + let config = RedisConsumerConfigBuilder::new() + .stream(stream) + .group_name("prefetch-no-dup-group") + .consumer_name("consumer-1") + .dead_stream(dead_stream) + .max_retries(3) + // Keep ClaimSweeper out of the test window; we only validate local dispatch behavior. + .claim_min_idle(Duration::from_secs(30)) + .claim_interval(Duration::from_secs(10)) + .prefetch_count(8) + .block_ms(1000) + .build_prefetch() + .unwrap(); + + let total_calls = Arc::new(AtomicU32::new(0)); + let duplicate_calls = Arc::new(AtomicU32::new(0)); + let seen_blocks = Arc::new(Mutex::new(HashSet::::new())); + + let handler = { + let total_calls = total_calls.clone(); + let duplicate_calls = duplicate_calls.clone(); + let seen_blocks = seen_blocks.clone(); + AsyncHandlerPayloadOnly::new(move |event: BlockEvent| { + let total_calls = total_calls.clone(); + let duplicate_calls = duplicate_calls.clone(); + let seen_blocks = seen_blocks.clone(); + async move { + // Intentionally slower than the 1s pending tick in run. + tokio::time::sleep(Duration::from_millis(1500)).await; + total_calls.fetch_add(1, Ordering::SeqCst); + let inserted = seen_blocks.lock().unwrap().insert(event.block_number); + if !inserted { + duplicate_calls.fetch_add(1, Ordering::SeqCst); + } + Ok::<(), std::convert::Infallible>(()) + } + }) + }; + + let consumer = RedisConsumer::connect(url).await.unwrap(); + let consumer_handle = tokio::spawn(async move { + let _ = consumer.run(config, handler).await; + }); + + tokio::time::sleep(Duration::from_millis(400)).await; + + let conn = RedisConnectionManager::new(url).await.unwrap(); + let publisher = RedisPublisher::new(conn); + for i in 0..u64::from(expected_messages) { + publisher + .publish(stream, &BlockEvent { block_number: i }) + .await + .unwrap(); + } + + let deadline = tokio::time::Instant::now() + Duration::from_secs(20); + while tokio::time::Instant::now() < deadline { + let unique = seen_blocks.lock().unwrap().len(); + if unique == expected_messages as usize { + break; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + + consumer_handle.abort(); + + let unique = seen_blocks.lock().unwrap().len(); + assert_eq!( + unique, expected_messages as usize, + "every published message should be observed" + ); + assert_eq!( + duplicate_calls.load(Ordering::SeqCst), + 0, + "no message should be dispatched more than once while in-flight" + ); + assert_eq!( + total_calls.load(Ordering::SeqCst), + expected_messages, + "handler should be called exactly once per message" + ); +} + +// ── Test 11: prefetch-safe recovers from worker panic ─────────────────────────── + +/// Regression test for panic-safe in-flight cleanup: +/// one message panics once in a worker, then is retried and successfully +/// processed without blocking subsequent messages. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_run_recovers_from_worker_panic() { + let url = shared_redis_url().await; + + let stream = "prefetch.panic-recovery.events"; + let dead_stream = "prefetch.panic-recovery.events:dead"; + let expected_messages: u32 = 3; + + let config = RedisConsumerConfigBuilder::new() + .stream(stream) + .group_name("prefetch-panic-recovery-group") + .consumer_name("consumer-1") + .dead_stream(dead_stream) + .max_retries(5) + .claim_min_idle(Duration::from_millis(300)) + .claim_interval(Duration::from_millis(400)) + .prefetch_count(8) + .block_ms(1000) + .build_prefetch() + .unwrap(); + + let panic_once = Arc::new(AtomicBool::new(false)); + let panic_count = Arc::new(AtomicU32::new(0)); + let success_calls = Arc::new(AtomicU32::new(0)); + let processed_blocks = Arc::new(Mutex::new(HashSet::::new())); + + let handler = { + let panic_once = panic_once.clone(); + let panic_count = panic_count.clone(); + let success_calls = success_calls.clone(); + let processed_blocks = processed_blocks.clone(); + AsyncHandlerPayloadOnly::new(move |event: BlockEvent| { + let panic_once = panic_once.clone(); + let panic_count = panic_count.clone(); + let success_calls = success_calls.clone(); + let processed_blocks = processed_blocks.clone(); + async move { + if event.block_number == 1 && !panic_once.swap(true, Ordering::SeqCst) { + panic_count.fetch_add(1, Ordering::SeqCst); + panic!("intentional worker panic for recovery regression test"); + } + + success_calls.fetch_add(1, Ordering::SeqCst); + processed_blocks.lock().unwrap().insert(event.block_number); + Ok::<(), std::convert::Infallible>(()) + } + }) + }; + + let consumer = RedisConsumer::connect(url).await.unwrap(); + let consumer_handle = tokio::spawn(async move { + let _ = consumer.run(config, handler).await; + }); + + tokio::time::sleep(Duration::from_millis(400)).await; + + let conn = RedisConnectionManager::new(url).await.unwrap(); + let publisher = RedisPublisher::new(conn); + for block in 1..=u64::from(expected_messages) { + publisher + .publish( + stream, + &BlockEvent { + block_number: block, + }, + ) + .await + .unwrap(); + } + + let deadline = tokio::time::Instant::now() + Duration::from_secs(20); + while tokio::time::Instant::now() < deadline { + let unique = processed_blocks.lock().unwrap().len(); + if unique == expected_messages as usize { + break; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + + consumer_handle.abort(); + + let processed = processed_blocks.lock().unwrap().clone(); + assert_eq!( + panic_count.load(Ordering::SeqCst), + 1, + "handler should panic exactly once for block 1" + ); + assert_eq!( + processed.len(), + expected_messages as usize, + "all published blocks should eventually be processed despite one worker panic" + ); + assert!( + processed.contains(&1), + "the block that panicked initially should be retried and processed" + ); + assert_eq!( + success_calls.load(Ordering::SeqCst), + expected_messages, + "successful handler calls should match published message count" + ); +} + +// Helpers for auto-trim assertions. +async fn redis_xlen(url: &str, stream: &str) -> u64 { + let client = redis::Client::open(url).unwrap(); + let mut raw_conn = client.get_multiplexed_async_connection().await.unwrap(); + redis::cmd("XLEN") + .arg(stream) + .query_async(&mut raw_conn) + .await + .unwrap() +} + +async fn redis_group_count(url: &str, stream: &str) -> usize { + let client = redis::Client::open(url).unwrap(); + let mut raw_conn = client.get_multiplexed_async_connection().await.unwrap(); + let groups: redis::Value = redis::cmd("XINFO") + .arg("GROUPS") + .arg(stream) + .query_async(&mut raw_conn) + .await + .unwrap(); + + match groups { + redis::Value::Array(items) => items.len(), + _ => 0, + } +} + +// ── Test: auto_trim + fallback_maxlen works without groups ───────────────────── + +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_auto_trim_with_fallback_trims_without_groups() { + let url = shared_redis_url().await; + let suffix = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + + let stream = format!("autotrim.fallback.no-groups.{suffix}"); + let fallback_maxlen: u64 = 100; + let publish_count: u64 = 5000; + + let conn = RedisConnectionManager::new(url).await.unwrap(); + let publisher = RedisPublisher::builder(conn) + .auto_trim(Duration::from_millis(200)) + .fallback_maxlen(fallback_maxlen as usize) + .build(); + + for i in 0..publish_count { + publisher + .publish(&stream, &BlockEvent { block_number: i }) + .await + .unwrap(); + } + + assert_eq!( + redis_group_count(url, &stream).await, + 0, + "this test must run with no consumer groups to exercise fallback_maxlen" + ); + + let deadline = tokio::time::Instant::now() + Duration::from_secs(15); + let mut last_len = redis_xlen(url, &stream).await; + + while tokio::time::Instant::now() < deadline { + last_len = redis_xlen(url, &stream).await; + + // MAXLEN ~ is approximate, so use tolerant threshold. + if last_len <= fallback_maxlen * 10 { + assert!( + last_len < publish_count, + "fallback trimming should reduce stream length (len={last_len}, published={publish_count})" + ); + publisher.shutdown().await; + return; + } + + tokio::time::sleep(Duration::from_millis(200)).await; + } + + publisher.shutdown().await; + panic!( + "fallback auto-trim did not converge near target: last_len={last_len}, fallback_maxlen={fallback_maxlen}, published={publish_count}" + ); +} + +// ── Test: auto_trim works without fallback via group progress (MINID path) ──── + +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_auto_trim_without_fallback_trims_with_consumer_group_progress() { + let url = shared_redis_url().await; + let suffix = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + + let stream = format!("autotrim.no-fallback.group.{suffix}"); + let group = format!("autotrim-no-fallback-group-{suffix}"); + let publish_count: u32 = 1500; + + let consumed = Arc::new(AtomicU32::new(0)); + let consumed_clone = consumed.clone(); + + let handler = AsyncHandlerPayloadOnly::new(move |_: BlockEvent| { + let c = consumed_clone.clone(); + async move { + c.fetch_add(1, Ordering::SeqCst); + Ok::<(), std::io::Error>(()) + } + }); + + let config = RedisConsumerConfigBuilder::new() + .stream(&stream) + .group_name(&group) + .consumer_name("consumer-1") + .dead_stream(format!("{stream}:dead")) + .prefetch_count(256) + .block_ms(200) + .build_prefetch() + .unwrap(); + + let consumer = RedisConsumer::connect(url).await.unwrap(); + let consumer_handle = tokio::spawn(async move { + let _ = consumer.run(config, handler).await; + }); + + tokio::time::sleep(Duration::from_millis(500)).await; + + let conn = RedisConnectionManager::new(url).await.unwrap(); + let publisher = RedisPublisher::builder(conn) + .auto_trim(Duration::from_millis(200)) // no fallback_maxlen on purpose + .build(); + + for i in 0..publish_count { + publisher + .publish( + &stream, + &BlockEvent { + block_number: i as u64, + }, + ) + .await + .unwrap(); + } + + let consume_deadline = tokio::time::Instant::now() + Duration::from_secs(25); + while consumed.load(Ordering::SeqCst) < publish_count + && tokio::time::Instant::now() < consume_deadline + { + tokio::time::sleep(Duration::from_millis(100)).await; + } + + assert_eq!( + consumed.load(Ordering::SeqCst), + publish_count, + "consumer should process+ack all messages so trimmer can advance safe MINID" + ); + + assert!( + redis_group_count(url, &stream).await >= 1, + "this test requires a consumer group to exercise MINID trimming path" + ); + + // Give ACKs a brief moment to settle before asserting trim. + tokio::time::sleep(Duration::from_millis(500)).await; + + let trim_deadline = tokio::time::Instant::now() + Duration::from_secs(15); + let mut last_len = redis_xlen(url, &stream).await; + + while tokio::time::Instant::now() < trim_deadline { + last_len = redis_xlen(url, &stream).await; + + // MINID ~ is approximate; tolerate some tail. + if last_len <= 500 { + consumer_handle.abort(); + publisher.shutdown().await; + return; + } + + tokio::time::sleep(Duration::from_millis(200)).await; + } + + consumer_handle.abort(); + publisher.shutdown().await; + panic!( + "no-fallback auto-trim did not trim by group progress: last_len={last_len}, published={publish_count}" + ); +} + +// ── Stream depth introspection tests ──────────────────────────────────────── + +/// Verify `RedisQueueInspector::queue_depths` returns correct counts for +/// principal and dead-letter streams after publishing messages. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_queue_depth_introspection() { + use broker::redis::RedisQueueInspector; + use broker::traits::depth::QueueInspector; + + let url = shared_redis_url().await; + let stream = "depth.events"; + let dead_stream = "depth.events:dead"; + + // Publish 5 messages to the principal stream. + let conn = RedisConnectionManager::new(url).await.unwrap(); + let publisher = RedisPublisher::builder(conn.clone()).build(); + for i in 0..5u64 { + publisher + .publish(stream, &BlockEvent { block_number: i }) + .await + .unwrap(); + } + + // Also push 2 messages directly to the dead-letter stream to simulate DLQ entries. + let publisher_dead = RedisPublisher::builder(conn.clone()).build(); + for i in 100..102u64 { + publisher_dead + .publish(dead_stream, &BlockEvent { block_number: i }) + .await + .unwrap(); + } + + // Query depth without group (stream-level only). + let inspector = RedisQueueInspector::new(conn); + let depths = inspector.queue_depths(stream, None).await.unwrap(); + + assert_eq!( + depths.principal, 5, + "5 messages published to principal stream" + ); + assert_eq!( + depths.retry, None, + "Redis retry is PEL-based, no separate stream" + ); + assert_eq!(depths.dead_letter, 2, "2 messages in dead-letter stream"); + assert_eq!(depths.total(), 7); + assert!(!depths.is_empty()); + assert_eq!(depths.pending, None, "no group queried"); + assert_eq!(depths.lag, None, "no group queried"); + // Without group-level metrics, has_pending_work falls back to principal > 0. + assert!(depths.has_pending_work()); +} + +/// Verify `RedisQueueInspector::queue_depths` returns zeros for a stream +/// that does not exist (XLEN on a non-existent key returns 0). +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_queue_depth_nonexistent_stream_returns_zeros() { + use broker::redis::RedisQueueInspector; + use broker::traits::depth::QueueInspector; + + let url = shared_redis_url().await; + let conn = RedisConnectionManager::new(url).await.unwrap(); + let inspector = RedisQueueInspector::new(conn); + + let depths = inspector + .queue_depths("nonexistent.stream.depth.test", None) + .await + .unwrap(); + + assert_eq!(depths.principal, 0); + assert_eq!(depths.retry, None); + assert_eq!(depths.dead_letter, 0); + assert!(depths.is_empty()); +} + +/// Verify `is_empty` returns `true` when the stream does not exist. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_is_empty_nonexistent_stream() { + use broker::redis::RedisQueueInspector; + use broker::traits::depth::QueueInspector; + + let url = shared_redis_url().await; + let conn = RedisConnectionManager::new(url).await.unwrap(); + let inspector = RedisQueueInspector::new(conn); + + let empty = inspector + .is_empty("nonexistent.is_empty.test", "some-group") + .await + .unwrap(); + + assert!(empty, "non-existent stream should be empty"); +} + +/// Verify `is_empty` returns `false` when a consumer group has pending +/// messages (published, read by group, but not ACKed). +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_is_empty_with_pending_messages() { + use broker::redis::RedisQueueInspector; + use broker::traits::depth::QueueInspector; + use redis::AsyncCommands; + + let url = shared_redis_url().await; + let stream = "is_empty.pending.test"; + let group = "test-group"; + let consumer = "test-consumer"; + + let conn = RedisConnectionManager::new(url).await.unwrap(); + let mut raw_conn = conn.get_connection(); + + // Publish a message. + let _: String = raw_conn + .xadd(stream, "*", &[("data", "hello")]) + .await + .unwrap(); + + // Create group and read (delivers to PEL but don't ACK). + let _: () = redis::cmd("XGROUP") + .arg("CREATE") + .arg(stream) + .arg(group) + .arg("0") + .arg("MKSTREAM") + .query_async(&mut raw_conn) + .await + .unwrap(); + + let _: redis::Value = redis::cmd("XREADGROUP") + .arg("GROUP") + .arg(group) + .arg(consumer) + .arg("COUNT") + .arg(1) + .arg("STREAMS") + .arg(stream) + .arg(">") + .query_async(&mut raw_conn) + .await + .unwrap(); + + // Now pending=1, should NOT be empty. + let inspector = RedisQueueInspector::new(conn); + let empty = inspector.is_empty(stream, group).await.unwrap(); + assert!(!empty, "stream with pending messages should not be empty"); +} + +/// Verify `is_empty` returns `true` when all messages are consumed and ACKed. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_is_empty_all_acked() { + use broker::redis::RedisQueueInspector; + use broker::traits::depth::QueueInspector; + use redis::AsyncCommands; + + let url = shared_redis_url().await; + let stream = "is_empty.acked.test"; + let group = "test-group"; + let consumer = "test-consumer"; + + let conn = RedisConnectionManager::new(url).await.unwrap(); + let mut raw_conn = conn.get_connection(); + + // Publish, create group, read, then ACK. + let id: String = raw_conn + .xadd(stream, "*", &[("data", "hello")]) + .await + .unwrap(); + + let _: () = redis::cmd("XGROUP") + .arg("CREATE") + .arg(stream) + .arg(group) + .arg("0") + .arg("MKSTREAM") + .query_async(&mut raw_conn) + .await + .unwrap(); + + let _: redis::Value = redis::cmd("XREADGROUP") + .arg("GROUP") + .arg(group) + .arg(consumer) + .arg("COUNT") + .arg(1) + .arg("STREAMS") + .arg(stream) + .arg(">") + .query_async(&mut raw_conn) + .await + .unwrap(); + + let _: u64 = raw_conn.xack(stream, group, &[&id]).await.unwrap(); + + // pending=0, lag=0 (all delivered and ACKed). XLEN=1 but is_empty=true. + let inspector = RedisQueueInspector::new(conn); + let empty = inspector.is_empty(stream, group).await.unwrap(); + assert!( + empty, + "stream with all messages ACKed should be empty (even though XLEN > 0)" + ); +} + +/// Verify `exists` returns `false` for a stream that does not exist. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_exists_nonexistent_stream() { + use broker::redis::RedisQueueInspector; + use broker::traits::depth::QueueInspector; + + let url = shared_redis_url().await; + let conn = RedisConnectionManager::new(url).await.unwrap(); + let inspector = RedisQueueInspector::new(conn); + + let found = inspector.exists("nonexistent.exists.test").await.unwrap(); + + assert!(!found, "non-existent stream should return false"); +} + +/// Verify `exists` returns `true` for a stream that has been created. +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_exists_after_xadd() { + use broker::redis::RedisQueueInspector; + use broker::traits::depth::QueueInspector; + use redis::AsyncCommands; + + let url = shared_redis_url().await; + let stream = "exists.after_xadd.test"; + + let conn = RedisConnectionManager::new(url).await.unwrap(); + let mut raw_conn = conn.get_connection(); + + let _: String = raw_conn + .xadd(stream, "*", &[("data", "hello")]) + .await + .unwrap(); + + let inspector = RedisQueueInspector::new(conn); + let found = inspector.exists(stream).await.unwrap(); + + assert!(found, "stream created via XADD should exist"); +} + +/// Verify `exists` returns `false` for a non-stream key (e.g. a plain string). +#[tokio::test] +#[ignore = "requires Docker"] +async fn test_exists_returns_false_for_non_stream_key() { + use broker::redis::RedisQueueInspector; + use broker::traits::depth::QueueInspector; + use redis::AsyncCommands; + + let url = shared_redis_url().await; + let key = "exists.string_key.test"; + + let conn = RedisConnectionManager::new(url).await.unwrap(); + let mut raw_conn = conn.get_connection(); + + let _: () = raw_conn.set(key, "not-a-stream").await.unwrap(); + + let inspector = RedisQueueInspector::new(conn); + let found = inspector.exists(key).await.unwrap(); + + assert!(!found, "TYPE check should return false for non-stream keys"); +} diff --git a/listener/crates/shared/primitives/Cargo.toml b/listener/crates/shared/primitives/Cargo.toml new file mode 100644 index 0000000000..bd63b10ece --- /dev/null +++ b/listener/crates/shared/primitives/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "primitives" +version = "0.1.0" +edition.workspace = true + +[dependencies] +alloy = { version = "1.6.1", features = ["full"] } +serde = { workspace = true, features = ["derive"] } +thiserror.workspace = true + +[dev-dependencies] +serde_json.workspace = true diff --git a/listener/crates/shared/primitives/src/event.rs b/listener/crates/shared/primitives/src/event.rs new file mode 100644 index 0000000000..5055fe2eb1 --- /dev/null +++ b/listener/crates/shared/primitives/src/event.rs @@ -0,0 +1,478 @@ +use alloy::primitives::{Address, B256, Bytes, U256}; +use serde::{Deserialize, Serialize}; +use std::fmt; +use thiserror::Error; + +/// Event payload for the backtrack-reorg consumer. +/// Carries the block number where the reorg was detected. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ReorgBacktrackEvent { + pub block_number: u64, + /// Hash of the new canonical block at `block_number`. + /// The backtrack handler upserts this block itself (idempotent) + pub block_hash: B256, + /// Parent hash of the new block at `block_number`. + /// The backtrack starts by fetching this hash from RPC. + pub parent_hash: B256, +} + +/// Validation errors for [`FilterCommand`]. +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum FilterCommandValidationError { + #[error("consumer_id must not be empty")] + EmptyConsumerId, + #[error("at least one of from or to must be set")] + MissingContractAddresses, +} + +/// Event payload for the control.watch and control.unwatch consumers. +/// Chain ID is omitted because the listener injects chain scope through namespaced routing. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct FilterCommand { + pub consumer_id: String, + pub from: Option
, + pub to: Option
, + pub log_address: Option
, +} + +impl FilterCommand { + /// Validate that the command has a non-empty consumer ID and at least one address. + /// + /// Normalizes `consumer_id` by trimming leading/trailing whitespace so + /// the stored value is always canonical. + #[must_use = "validation result must be checked"] + pub fn validate(&mut self) -> Result<(), FilterCommandValidationError> { + self.consumer_id = self.consumer_id.trim().to_owned(); + if self.consumer_id.is_empty() { + return Err(FilterCommandValidationError::EmptyConsumerId); + } + if self.from.is_none() && self.to.is_none() && self.log_address.is_none() { + return Err(FilterCommandValidationError::MissingContractAddresses); + } + Ok(()) + } +} + +/// A log with its block-global index. +/// +/// Wire format: `{ log_index, address, topics, data }`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct IndexedLog { + /// Block-global log index. + pub log_index: u64, + /// The address which emitted this log. + pub address: Address, + /// The indexed topic list. + pub topics: Vec, + /// The plain data. + pub data: Bytes, +} + +/// A transaction with its associated logs, as published in the block payload. +/// +/// Part of the RFC 006 output contract. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TransactionPayload { + /// Sender address. + pub from: Address, + /// Recipient address, `None` for contract creation. + pub to: Option
, + /// Transaction hash. + pub hash: B256, + /// Position of this transaction within the block. + pub transaction_index: u64, + /// Value transferred in wei. + pub value: U256, + /// Calldata / input data. + pub data: Bytes, + /// Logs emitted during this transaction's execution. + pub logs: Vec, +} + +/// Processing flow that produced a block payload. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub enum BlockFlow { + /// Normal forward chain synchronization. + Live, + /// Block republished during a reorg backtrack. + Reorged, + /// Historical catch-up / replay (reserved for future use). + Catchup, +} + +/// Block payload published to the message broker after a block is validated and persisted. +/// +/// Conforms to the RFC 006 output contract. Embeds block header fields plus full +/// transactions with their logs. Downstream consumers (Host, Gateway, Relayer) +/// deserialize this to learn about new canonical blocks and process their events. +/// +/// Used for both live flow and catchup replay. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct BlockPayload { + /// Processing flow that produced this payload (Live, Reorged, or Catchup). + pub flow: BlockFlow, + /// The chain ID this block belongs to (e.g. 1 for Ethereum mainnet). + pub chain_id: u64, + /// Block number (height). + pub block_number: u64, + /// Block hash. + pub block_hash: B256, + /// Parent block hash. + pub parent_hash: B256, + /// Block timestamp (seconds since Unix epoch). + pub timestamp: u64, + /// Transactions with their logs, ordered by transaction index. + pub transactions: Vec, +} + +// --- Debug-only Display impls (remove when no longer needed) --- + +impl fmt::Display for IndexedLog { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let data_hex = format!("{}", self.data); + write!( + f, + "Log(index={}, addr={}, data={}, topics={})", + self.log_index, + self.address, + data_hex, + self.topics.len(), + )?; + for topic in &self.topics { + writeln!(f, " {topic}")?; + } + Ok(()) + } +} + +impl fmt::Display for TransactionPayload { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let to_str = match &self.to { + Some(addr) => format!("{addr}"), + None => "contract creation".to_string(), + }; + write!( + f, + "Tx(hash={}, from={}, to={}, idx={}, value={}, logs={})", + self.hash, + self.from, + to_str, + self.transaction_index, + self.value, + self.logs.len(), + )?; + for log in &self.logs { + writeln!(f, " {log}")?; + } + Ok(()) + } +} + +impl fmt::Display for BlockPayload { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + writeln!( + f, + "Block(flow={:?}, chain={}, number={}, hash={}, ts={}, txs={})", + self.flow, + self.chain_id, + self.block_number, + self.block_hash, + self.timestamp, + self.transactions.len(), + )?; + for tx in &self.transactions { + writeln!(f, " {tx}")?; + } + Ok(()) + } +} + +// --- End debug-only Display impls --- + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn sample_payload() -> BlockPayload { + BlockPayload { + flow: BlockFlow::Live, + chain_id: 1, + block_number: 12345, + block_hash: "0xabcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890" + .parse() + .unwrap(), + parent_hash: "0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef" + .parse() + .unwrap(), + timestamp: 1700000000, + transactions: vec![TransactionPayload { + from: "0x0000000000000000000000000000000000000001" + .parse() + .unwrap(), + to: Some( + "0x00000000000000000000000000000000deadbeef" + .parse() + .unwrap(), + ), + hash: "0xabcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890" + .parse() + .unwrap(), + transaction_index: 0, + value: U256::from(1_000_000_000_000_000_000u128), + data: Bytes::from_static(&[0xa9, 0x05, 0x9c, 0xbb]), + logs: vec![IndexedLog { + log_index: 0, + address: "0x00000000000000000000000000000000deadbeef" + .parse() + .unwrap(), + topics: vec![ + "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef" + .parse() + .unwrap(), + ], + data: Bytes::from_static(&[0x00, 0x01, 0x02]), + }], + }], + } + } + + #[test] + fn block_payload_serializes_to_expected_json_and_round_trips() { + let payload = sample_payload(); + let json = serde_json::to_value(&payload).unwrap(); + let expected = json!({ + "flow": "LIVE", + "chain_id": 1, + "block_number": 12345, + "block_hash": "0xabcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890", + "parent_hash": "0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", + "timestamp": 1700000000, + "transactions": [{ + "from": "0x0000000000000000000000000000000000000001", + "to": "0x00000000000000000000000000000000deadbeef", + "hash": "0xabcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890", + "transaction_index": 0, + "value": "0xde0b6b3a7640000", + "data": "0xa9059cbb", + "logs": [{ + "log_index": 0, + "address": "0x00000000000000000000000000000000deadbeef", + "topics": [ + "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef" + ], + "data": "0x000102" + }] + }] + }); + + assert_eq!(json, expected); + + // Round-trip + let deserialized: BlockPayload = serde_json::from_value(json).unwrap(); + assert_eq!(payload, deserialized); + } + + #[test] + fn empty_block_serializes_with_empty_transactions() { + let payload = BlockPayload { + flow: BlockFlow::Live, + chain_id: 1, + block_number: 0, + block_hash: B256::ZERO, + parent_hash: B256::ZERO, + timestamp: 0, + transactions: vec![], + }; + + let json = serde_json::to_value(&payload).unwrap(); + assert_eq!(json["transactions"], json!([])); + } + + #[test] + fn filter_command_rejects_empty_consumer_id() { + let mut cmd = FilterCommand { + consumer_id: " ".into(), + from: Some( + "0x00000000000000000000000000000000deadbeef" + .parse() + .unwrap(), + ), + to: None, + log_address: None, + }; + assert_eq!( + cmd.validate().unwrap_err(), + FilterCommandValidationError::EmptyConsumerId, + ); + } + + #[test] + fn filter_command_rejects_literal_empty_consumer_id() { + let mut cmd = FilterCommand { + consumer_id: "".into(), + from: Some( + "0x00000000000000000000000000000000deadbeef" + .parse() + .unwrap(), + ), + to: None, + log_address: None, + }; + assert_eq!( + cmd.validate().unwrap_err(), + FilterCommandValidationError::EmptyConsumerId, + ); + } + + #[test] + fn filter_command_rejects_missing_addresses() { + let mut cmd = FilterCommand { + consumer_id: "gateway".into(), + from: None, + to: None, + log_address: None, + }; + assert_eq!( + cmd.validate().unwrap_err(), + FilterCommandValidationError::MissingContractAddresses, + ); + } + + #[test] + fn filter_command_accepts_valid() { + let mut cmd = FilterCommand { + consumer_id: "gateway".into(), + from: Some( + "0x00000000000000000000000000000000deadbeef" + .parse() + .unwrap(), + ), + to: None, + log_address: None, + }; + cmd.validate().unwrap(); + } + + #[test] + fn filter_command_trims_consumer_id() { + let mut cmd = FilterCommand { + consumer_id: " gateway ".into(), + from: Some( + "0x00000000000000000000000000000000deadbeef" + .parse() + .unwrap(), + ), + to: None, + log_address: None, + }; + cmd.validate().unwrap(); + assert_eq!(cmd.consumer_id, "gateway"); + } + + #[test] + fn filter_command_accepts_to_only() { + let mut cmd = FilterCommand { + consumer_id: "gateway".into(), + from: None, + to: Some( + "0x00000000000000000000000000000000deadbeef" + .parse() + .unwrap(), + ), + log_address: None, + }; + cmd.validate().unwrap(); + } + + #[test] + fn filter_command_round_trips() { + let filter = FilterCommand { + consumer_id: "gateway".into(), + from: Some( + "0x00000000000000000000000000000000deadbeef" + .parse() + .unwrap(), + ), + to: None, + log_address: None, + }; + + let json = serde_json::to_value(&filter).unwrap(); + let expected = json!({ + "consumer_id": "gateway", + "from": "0x00000000000000000000000000000000deadbeef", + "to": null, + "log_address": null, + }); + assert_eq!(json, expected); + + let deserialized: FilterCommand = serde_json::from_value(json).unwrap(); + assert_eq!(filter, deserialized); + } + + #[test] + fn filter_command_accepts_log_address_only() { + let mut cmd = FilterCommand { + consumer_id: "gateway".into(), + from: None, + to: None, + log_address: Some( + "0x00000000000000000000000000000000deadbeef" + .parse() + .unwrap(), + ), + }; + cmd.validate().unwrap(); + } + + #[test] + fn filter_command_round_trips_with_log_address() { + let filter = FilterCommand { + consumer_id: "gateway".into(), + from: None, + to: None, + log_address: Some( + "0x00000000000000000000000000000000deadbeef" + .parse() + .unwrap(), + ), + }; + + let json = serde_json::to_value(&filter).unwrap(); + let expected = json!({ + "consumer_id": "gateway", + "from": null, + "to": null, + "log_address": "0x00000000000000000000000000000000deadbeef", + }); + assert_eq!(json, expected); + + let deserialized: FilterCommand = serde_json::from_value(json).unwrap(); + assert_eq!(filter, deserialized); + } + + #[test] + fn contract_creation_serializes_to_null() { + let tx = TransactionPayload { + from: "0x0000000000000000000000000000000000000001" + .parse() + .unwrap(), + to: None, + hash: "0xabcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890" + .parse() + .unwrap(), + transaction_index: 3, + value: U256::ZERO, + data: Bytes::from_static(&[0x60, 0x60, 0x60, 0x40, 0x52]), + logs: vec![], + }; + + let json = serde_json::to_value(&tx).unwrap(); + assert_eq!(json["to"], json!(null)); + + let deserialized: TransactionPayload = serde_json::from_value(json).unwrap(); + assert_eq!(deserialized.to, None); + } +} diff --git a/listener/crates/shared/primitives/src/lib.rs b/listener/crates/shared/primitives/src/lib.rs new file mode 100644 index 0000000000..cb9bddabd6 --- /dev/null +++ b/listener/crates/shared/primitives/src/lib.rs @@ -0,0 +1,4 @@ +pub mod event; +pub mod namespace; +pub mod routing; +pub mod utils; diff --git a/listener/crates/shared/primitives/src/namespace.rs b/listener/crates/shared/primitives/src/namespace.rs new file mode 100644 index 0000000000..a432255b8d --- /dev/null +++ b/listener/crates/shared/primitives/src/namespace.rs @@ -0,0 +1 @@ +pub const EMPTY_NAMESPACE: &str = ""; diff --git a/listener/crates/shared/primitives/src/routing.rs b/listener/crates/shared/primitives/src/routing.rs new file mode 100644 index 0000000000..20eb0d8849 --- /dev/null +++ b/listener/crates/shared/primitives/src/routing.rs @@ -0,0 +1,11 @@ +// Common routing keys for listener/consumer. +pub const FETCH_NEW_BLOCKS: &str = "fetch-new-blocks"; +pub const BACKTRACK_REORG: &str = "backtrack-reorg"; +pub const WATCH: &str = "control.watch"; +pub const UNWATCH: &str = "control.unwatch"; +pub const CLEAN_BLOCKS: &str = "clean-blocks"; +pub const NEW_EVENT: &str = "new-event"; + +pub fn consumer_new_event_routing(consumer_id: String) -> String { + format!("{}.{}", consumer_id, NEW_EVENT) +} diff --git a/listener/crates/shared/primitives/src/routing.rs.orig b/listener/crates/shared/primitives/src/routing.rs.orig new file mode 100644 index 0000000000..1d2942dcff --- /dev/null +++ b/listener/crates/shared/primitives/src/routing.rs.orig @@ -0,0 +1,14 @@ +// Common routing keys for listener/consumer. +pub const FETCH_NEW_BLOCKS: &str = "fetch-new-blocks"; +pub const BACKTRACK_REORG: &str = "backtrack-reorg"; +pub const WATCH: &str = "control.watch"; +pub const UNWATCH: &str = "control.unwatch"; +pub const CLEAN_BLOCKS: &str = "clean-blocks"; +pub const NEW_EVENT: &str = "new-event"; +<<<<<<< HEAD + +pub fn consumer_new_event_routing(consumer_id: String) -> String { + format!("{}.{}", consumer_id, NEW_EVENT) +} +======= +>>>>>>> c4fb34a (chore(core): publisher, with is exists strategy, and republish staled process) diff --git a/listener/crates/shared/primitives/src/utils.rs b/listener/crates/shared/primitives/src/utils.rs new file mode 100644 index 0000000000..21c9976e93 --- /dev/null +++ b/listener/crates/shared/primitives/src/utils.rs @@ -0,0 +1,47 @@ +use alloy::primitives::Address; + +/// Convert an optional [`Address`] to its EIP-55 checksummed string representation. +pub fn checksum_optional_address(addr: &Option
) -> Option { + addr.map(|a| a.to_checksum(None)) +} + +pub fn chain_id_to_namespace(chain_id: u64) -> String { + format!("chain-id-{}", chain_id) +} + +/// Converts a u64 to i64, clamping to i64::MAX if the value exceeds it. +/// Prevents silent wrapping to negative values which could cause +/// destructive SQL behavior (e.g., deleting all rows instead of keeping N). +pub fn saturating_u64_to_i64(value: u64) -> i64 { + i64::try_from(value).unwrap_or(i64::MAX) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn saturating_u64_to_i64_zero() { + assert_eq!(saturating_u64_to_i64(0), 0); + } + + #[test] + fn saturating_u64_to_i64_normal_value() { + assert_eq!(saturating_u64_to_i64(1000), 1000); + } + + #[test] + fn saturating_u64_to_i64_at_i64_max() { + assert_eq!(saturating_u64_to_i64(i64::MAX as u64), i64::MAX); + } + + #[test] + fn saturating_u64_to_i64_above_i64_max() { + assert_eq!(saturating_u64_to_i64(i64::MAX as u64 + 1), i64::MAX); + } + + #[test] + fn saturating_u64_to_i64_at_u64_max() { + assert_eq!(saturating_u64_to_i64(u64::MAX), i64::MAX); + } +} diff --git a/listener/crates/shared/telemetry/Cargo.toml b/listener/crates/shared/telemetry/Cargo.toml new file mode 100644 index 0000000000..78f125e55d --- /dev/null +++ b/listener/crates/shared/telemetry/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "telemetry" +version = "0.1.0" +edition.workspace = true + +[dependencies] +metrics.workspace = true +metrics-exporter-prometheus = { version = "0.16", default-features = false, features = ["http-listener"] } +thiserror.workspace = true +tracing.workspace = true + +[dev-dependencies] +tokio = { workspace = true, features = ["rt-multi-thread", "macros", "time"] } +reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] } diff --git a/listener/crates/shared/telemetry/src/lib.rs b/listener/crates/shared/telemetry/src/lib.rs new file mode 100644 index 0000000000..e4c29d60d5 --- /dev/null +++ b/listener/crates/shared/telemetry/src/lib.rs @@ -0,0 +1,74 @@ +//! Prometheus metrics exporter bootstrap. +//! +//! Installs the `metrics-exporter-prometheus` recorder and starts an HTTP +//! server that serves the `/metrics` endpoint for Prometheus scraping. +//! +//! Application crates emit metrics via the `metrics` facade crate directly +//! (e.g., `metrics::counter!("broker_messages_published_total")`) — they do +//! not depend on this crate. + +#![deny(clippy::correctness)] +#![warn(clippy::suspicious, clippy::style, clippy::complexity, clippy::perf)] + +use std::net::SocketAddr; + +use metrics_exporter_prometheus::PrometheusBuilder; +use thiserror::Error; + +/// Configuration for the metrics HTTP endpoint. +#[derive(Debug, Clone)] +pub struct MetricsConfig { + /// Address to bind the metrics HTTP server. Default: `0.0.0.0:9090` + pub listen_addr: SocketAddr, +} + +impl Default for MetricsConfig { + fn default() -> Self { + Self { + listen_addr: SocketAddr::from(([0, 0, 0, 0], 9090)), + } + } +} + +/// Errors that can occur during metrics initialization. +#[derive(Error, Debug)] +#[non_exhaustive] +pub enum TelemetryError { + /// Failed to install the Prometheus recorder. + /// Happens if a global recorder is already installed (double-init) + /// or if the HTTP server fails to bind. + #[error("failed to install prometheus exporter: {0}")] + Install(String), +} + +/// Install the Prometheus exporter and start the HTTP server on `/metrics`. +/// +/// Call once at startup, after loading config, before any subsystem that emits +/// metrics. The HTTP server runs until the tokio runtime shuts down. +/// +/// # Errors +/// +/// Returns [`TelemetryError::Install`] if: +/// - A global metrics recorder is already installed (call this once) +/// - The HTTP server fails to bind to the configured address +pub fn init_metrics(config: MetricsConfig) -> Result<(), TelemetryError> { + PrometheusBuilder::new() + .with_http_listener(config.listen_addr) + .install() + .map_err(|e| TelemetryError::Install(e.to_string()))?; + + tracing::info!(addr = %config.listen_addr, "Prometheus metrics endpoint started"); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + // AC-1.1 + #[test] + fn metrics_config_default_binds_to_9090() { + let config = MetricsConfig::default(); + assert_eq!(config.listen_addr, SocketAddr::from(([0, 0, 0, 0], 9090))); + } +} diff --git a/listener/crates/test-support/Cargo.toml b/listener/crates/test-support/Cargo.toml new file mode 100644 index 0000000000..23686ddf8d --- /dev/null +++ b/listener/crates/test-support/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "test-support" +version = "0.1.0" +edition.workspace = true + +[dependencies] +tokio.workspace = true +testcontainers = { version = "0.23", features = ["reusable-containers"] } +testcontainers-modules = { version = "0.11", features = ["redis"] } diff --git a/listener/crates/test-support/src/lib.rs b/listener/crates/test-support/src/lib.rs new file mode 100644 index 0000000000..664dda3de9 --- /dev/null +++ b/listener/crates/test-support/src/lib.rs @@ -0,0 +1,34 @@ +use tokio::sync::OnceCell; + +static REDIS_URL: OnceCell = OnceCell::const_new(); + +/// Lazily start a single Redis container shared across all tests. +/// +/// `ReuseDirective::CurrentSession` prevents the `Drop` impl from stopping +/// the container when the `ContainerAsync` handle goes out of scope at the +/// end of the `OnceCell` init closure. The container therefore stays alive +/// for all tests in the suite. Cleanup is handled by `make test-e2e-*` +/// targets via `docker rm -f e2e-redis`. +pub async fn shared_redis_url() -> &'static str { + REDIS_URL + .get_or_init(|| async { + use testcontainers::core::{ImageExt, ReuseDirective}; + use testcontainers::runners::AsyncRunner; + use testcontainers_modules::redis::Redis; + + let container = Redis::default() + .with_container_name("e2e-redis") + .with_tag("6.2.0") + .with_reuse(ReuseDirective::CurrentSession) + .start() + .await + .expect("failed to start Redis container — is Docker running?"); + let port = container + .get_host_port_ipv4(6379) + .await + .expect("failed to get Redis container port"); + format!("redis://127.0.0.1:{port}") + }) + .await + .as_str() +} diff --git a/listener/ct.yaml b/listener/ct.yaml new file mode 100644 index 0000000000..6f6888668f --- /dev/null +++ b/listener/ct.yaml @@ -0,0 +1,10 @@ +chart-dirs: + - charts +target-branch: main +helm-extra-args: --timeout 600s +validate-maintainers: false +# Fail if chart content changed but Chart.yaml version was not bumped +check-version-increment: true +# Extra validation scripts run per changed chart ($1 = chart path) +additional-commands: + - scripts/ct-lint-extras.sh {{ .Path }} diff --git a/listener/data/canonical-chain-extended.csv b/listener/data/canonical-chain-extended.csv new file mode 100644 index 0000000000..cbc7d79b24 --- /dev/null +++ b/listener/data/canonical-chain-extended.csv @@ -0,0 +1,195 @@ +"2abba6a3-bad4-48bd-a379-10369ae752ae",1,24619316,"5A9A8B1D14B69730720417B0C672C0182A65787F5E239CAA5576B4C669DE25D1","5B465871CB3C50DBF7A6006851F4EA2B854F5B9D48338E2F6A0DCE4B0DFEAD5D","CANONICAL","2026-03-09 10:27:29.000000+00" +"6abc54cb-2716-4b6e-afcb-2af227ba318e",1,24619317,"8F0C1C45351AED996F60BFB3E66B86518EA9098A7E09CF650151B1EB0FFDDBF8","5A9A8B1D14B69730720417B0C672C0182A65787F5E239CAA5576B4C669DE25D1","CANONICAL","2026-03-09 10:27:41.000000+00" +"6660bab2-feb0-4c9e-a596-36b3860b1b19",1,24619318,"2CCD69D886A84B096F59A1CC8CA08442C1A3ED8FB57B07D620C48F79882B2B5F","8F0C1C45351AED996F60BFB3E66B86518EA9098A7E09CF650151B1EB0FFDDBF8","CANONICAL","2026-03-09 10:27:53.000000+00" +"f51a6a4c-ff4b-41ff-9b79-3dee7389c238",1,24619319,"9055E86B7D82EC344E09F58BBFE97A2B2A3E3BC1880A991483AD9AC230097D6A","2CCD69D886A84B096F59A1CC8CA08442C1A3ED8FB57B07D620C48F79882B2B5F","CANONICAL","2026-03-09 10:28:05.000000+00" +"4c2a729d-bc90-4b20-8d8c-58c933bd6b29",1,24619320,"FAFF681785D6F47BCCED3E61A6E245868671A81D9BF5E486DAEE1F86366FDEEA","9055E86B7D82EC344E09F58BBFE97A2B2A3E3BC1880A991483AD9AC230097D6A","CANONICAL","2026-03-09 10:28:17.000000+00" +"5c218cd2-6825-464f-a321-5f3765c077b3",1,24619321,"EFAF54FBE3B42C7EF45D0BF0E466E13D4417F7DA162FD11AA3301C4700BA7A2F","FAFF681785D6F47BCCED3E61A6E245868671A81D9BF5E486DAEE1F86366FDEEA","CANONICAL","2026-03-09 10:28:29.000000+00" +"97106d5b-c095-4b09-91a2-e65d7a03f1e8",1,24619322,"5502CD8D8A22475794A0E714A64B8A4535D630418BD9775AD8BB18F9CA109E1F","EFAF54FBE3B42C7EF45D0BF0E466E13D4417F7DA162FD11AA3301C4700BA7A2F","CANONICAL","2026-03-09 10:28:41.000000+00" +"d29eb8ad-8012-488e-b9ad-58faf51df575",1,24619323,"182C88EB4431F45062D1B5F2FAEDA807B58844B7E2727FB4B60086BF5E5A5BDA","5502CD8D8A22475794A0E714A64B8A4535D630418BD9775AD8BB18F9CA109E1F","CANONICAL","2026-03-09 10:28:53.000000+00" +"399c2bc6-788e-43c4-baa8-6f48acf72868",1,24619324,"A17A63484CE8D88F1CCAD423A9B90750438875A51DC8720E72CFB42BA818D985","182C88EB4431F45062D1B5F2FAEDA807B58844B7E2727FB4B60086BF5E5A5BDA","CANONICAL","2026-03-09 10:29:05.000000+00" +"ac75eb20-f37f-4808-9845-ca35424bd6bd",1,24619325,"CBC0A741C363B24AA24AC9EE6990CAC68B99A8FB1CAF24FBDC8B35A662A723D6","A17A63484CE8D88F1CCAD423A9B90750438875A51DC8720E72CFB42BA818D985","CANONICAL","2026-03-09 10:29:17.000000+00" +"7f5aca40-160a-423c-b691-ab655136695e",1,24619326,"99167E8BE898A784B4A57E7980F8B06FF36444BE9639B835425300192B937BFE","CBC0A741C363B24AA24AC9EE6990CAC68B99A8FB1CAF24FBDC8B35A662A723D6","CANONICAL","2026-03-09 10:29:29.000000+00" +"ce15e9de-719a-489b-85c8-c203e24a0c55",1,24619327,"368CE0108BCE5A27DBB06D378825375426D06065268A5469EFE55011AEFB10BF","99167E8BE898A784B4A57E7980F8B06FF36444BE9639B835425300192B937BFE","CANONICAL","2026-03-09 10:29:41.000000+00" +"c7aa9967-a515-4c51-8b69-a453232e8689",1,24619328,"6B8320AEACB44956DEE451A8927F9DC80026CA034BE570098B44D6FEF39CA8BC","368CE0108BCE5A27DBB06D378825375426D06065268A5469EFE55011AEFB10BF","CANONICAL","2026-03-09 10:29:53.000000+00" +"80af8f04-1ce3-48e5-84a2-f10c18e0d3cf",1,24619329,"B6B2EC4873840E787FB6190A7FFE707F62292AF32425FEF4E0E445E6E3DA0470","6B8320AEACB44956DEE451A8927F9DC80026CA034BE570098B44D6FEF39CA8BC","CANONICAL","2026-03-09 10:30:05.000000+00" +"7e27c254-e8bf-46c4-84c6-2605928686b5",1,24619330,"C35B60A98E0CFCA2E557940B52172A1EED024B62FEB839D3D98AD706A43BAA67","B6B2EC4873840E787FB6190A7FFE707F62292AF32425FEF4E0E445E6E3DA0470","CANONICAL","2026-03-09 10:30:17.000000+00" +"e83a7075-d1e6-4b19-8852-d5240761a34e",1,24619331,"4C19A7E8809C0EFB95D636C75D44F95C04873F3F7D643B59EDC5CD85033479D8","C35B60A98E0CFCA2E557940B52172A1EED024B62FEB839D3D98AD706A43BAA67","CANONICAL","2026-03-09 10:30:29.000000+00" +"604da614-de26-4625-bb0a-a73aa5ef56c8",1,24619332,"0A6BADFE47F1FE1E867F5C59194E282F3B7CAE943A38FFB497531798BB3A221B","4C19A7E8809C0EFB95D636C75D44F95C04873F3F7D643B59EDC5CD85033479D8","CANONICAL","2026-03-09 10:30:41.000000+00" +"b8409228-256a-41c8-90f5-34d099344ff0",1,24619333,"2D6BB334DF38AD4703D15BA0C14A81FEA2BFFA40109499B5675ABC695E14C504","0A6BADFE47F1FE1E867F5C59194E282F3B7CAE943A38FFB497531798BB3A221B","CANONICAL","2026-03-09 10:30:53.000000+00" +"8d149358-c8c1-4755-94b1-ec597d6c4efe",1,24619334,"3A52B52C9ED802F45A3B1A5119EFF6D4896261C8E397CFC0AE0F14CFAB4E1827","2D6BB334DF38AD4703D15BA0C14A81FEA2BFFA40109499B5675ABC695E14C504","CANONICAL","2026-03-09 10:31:05.000000+00" +"cfc6d22f-3db3-457c-811d-c371edb049eb",1,24619335,"496E2411C2DD60E5CB63CDAE1F386B25781F3FBCC61DA8A02D7958E9569679F8","3A52B52C9ED802F45A3B1A5119EFF6D4896261C8E397CFC0AE0F14CFAB4E1827","CANONICAL","2026-03-09 10:31:17.000000+00" +"475be85d-0210-4431-a6b9-cce8b2e378d9",1,24619336,"42D417F32B6DE90152A62FADA60CB4E313BC45696975ADCC86B3C0325A05A5E2","496E2411C2DD60E5CB63CDAE1F386B25781F3FBCC61DA8A02D7958E9569679F8","CANONICAL","2026-03-09 10:31:29.000000+00" +"2ba60e57-2eff-4002-b064-a6f27b7129cf",1,24619337,"74528BF767FB6BCA4652244B0B0CF3B6A90185C2946632686D859EF2AE55B502","42D417F32B6DE90152A62FADA60CB4E313BC45696975ADCC86B3C0325A05A5E2","CANONICAL","2026-03-09 10:31:41.000000+00" +"7009976b-4565-48db-83a4-d9e58256541d",1,24619338,"6FA69B26BF14F1FB436C07875812B3226C6503696B46F889C31AB5810C6EB990","74528BF767FB6BCA4652244B0B0CF3B6A90185C2946632686D859EF2AE55B502","CANONICAL","2026-03-09 10:31:53.000000+00" +"d85562f7-3546-4ffc-8872-dda828a1fb9d",1,24619339,"131554B3CBA7730297F7A14EF17DB6D52E5EF5F23FFC077AFA54F78931808ACE","6FA69B26BF14F1FB436C07875812B3226C6503696B46F889C31AB5810C6EB990","CANONICAL","2026-03-09 10:32:05.000000+00" +"8226eed8-ece7-4a73-8830-daa5a3618e59",1,24619340,"48487E74DD41368A326DD5284325DEE05DA21B5A21C3642C701BDBD4346F13E4","131554B3CBA7730297F7A14EF17DB6D52E5EF5F23FFC077AFA54F78931808ACE","CANONICAL","2026-03-09 10:32:17.000000+00" +"6d7c94b5-790a-41d9-8478-d71da2b0eca0",1,24619341,"34D4287E29653756F9090AED083A3675E4952124E98F20397A2758C2A40C2E40","48487E74DD41368A326DD5284325DEE05DA21B5A21C3642C701BDBD4346F13E4","CANONICAL","2026-03-09 10:32:29.000000+00" +"58d0ee2e-947b-4e0c-834d-f04a90802e46",1,24619342,"BE2FF5375C0204C2295253C244B9C4098872A843E719289410C621CC3B278432","34D4287E29653756F9090AED083A3675E4952124E98F20397A2758C2A40C2E40","CANONICAL","2026-03-09 10:32:41.000000+00" +"0ca351d2-d7c8-40ff-a7d6-bef906293dbc",1,24619343,"632625DE057CCE9ABFCD427229CE53F61814144F298543109C9E632B7A423BBC","BE2FF5375C0204C2295253C244B9C4098872A843E719289410C621CC3B278432","CANONICAL","2026-03-09 10:32:53.000000+00" +"46212442-0eb7-4560-b2a9-48898cce6214",1,24619344,"2A7A5CF57C92ACB089A4FAD6A7D8FAB90019D4EF7D01171486AADC066776CE2D","632625DE057CCE9ABFCD427229CE53F61814144F298543109C9E632B7A423BBC","CANONICAL","2026-03-09 10:33:05.000000+00" +"2f252f7f-467b-4979-9f80-aee55e6b44cb",1,24619345,"6F9F1FA7469CFDBB6D15977E321D664F6B53EB51960EEEB60BAA03F975BB8498","2A7A5CF57C92ACB089A4FAD6A7D8FAB90019D4EF7D01171486AADC066776CE2D","CANONICAL","2026-03-09 10:33:17.000000+00" +"0569022c-5ff1-407b-95ee-53fa95122ef2",1,24619346,"1B6CC0CE578BA5BAD888A8A603BCAE26BC2B6409523007E112AB64CF18A58AA2","6F9F1FA7469CFDBB6D15977E321D664F6B53EB51960EEEB60BAA03F975BB8498","CANONICAL","2026-03-09 10:33:29.000000+00" +"01a7c202-de26-410e-b2e1-b2f830af1cc5",1,24619347,"366181D09A770E02619E18CB59BECC2EAB94DEEFF1BE607D3332AE3DE2FED44F","1B6CC0CE578BA5BAD888A8A603BCAE26BC2B6409523007E112AB64CF18A58AA2","CANONICAL","2026-03-09 10:33:41.000000+00" +"f5dd2e14-1259-424e-93d6-b717ece63deb",1,24619348,"FA1FCA69415E45A4F5EC575585DCBF39BCB5540D43F80AF3484E59FF978B93F6","366181D09A770E02619E18CB59BECC2EAB94DEEFF1BE607D3332AE3DE2FED44F","CANONICAL","2026-03-09 10:33:53.000000+00" +"5baa6ce7-c044-449b-b361-fd1f05333776",1,24619349,"7B6ADAE8CC01EDDF2DA1E47AF285C292F3C8095A53DFA128C6903201B83AD613","FA1FCA69415E45A4F5EC575585DCBF39BCB5540D43F80AF3484E59FF978B93F6","CANONICAL","2026-03-09 10:34:05.000000+00" +"6ac9a78b-e982-47f3-928d-4658d51ee9df",1,24619350,"61830C0BDB6FB60D848159A36C3F7CAFA57935DE5623CD587978DBD811753002","7B6ADAE8CC01EDDF2DA1E47AF285C292F3C8095A53DFA128C6903201B83AD613","CANONICAL","2026-03-09 10:34:17.000000+00" +"1df8fbf8-a7eb-4357-b3d7-a3fc91a80b84",1,24619351,"C3FA7792F81B1100AA5F9944CEA5D6939DBD65E4005093CB84A28C9813FE0843","61830C0BDB6FB60D848159A36C3F7CAFA57935DE5623CD587978DBD811753002","CANONICAL","2026-03-09 10:34:29.000000+00" +"bb9cea12-dfc0-4e5e-89be-111669793289",1,24619352,"1C514CFE867377A615E4C0870023D0D588AB6F29FF70236223B6DD117F12DB78","C3FA7792F81B1100AA5F9944CEA5D6939DBD65E4005093CB84A28C9813FE0843","CANONICAL","2026-03-09 10:34:41.000000+00" +"c9cd505c-f13f-4373-bfdc-9fbf0d6dd7fe",1,24619353,"39E8463E2AEA2F0BB7FB5F1D9ED7C0182DBE3C99EB6FF601A1497046133E8744","1C514CFE867377A615E4C0870023D0D588AB6F29FF70236223B6DD117F12DB78","CANONICAL","2026-03-09 10:34:53.000000+00" +"a99f0f44-780c-4ea1-9187-f502bb36d6c5",1,24619354,"1578EDC4AEFD287A683A8063AC177564B5EDA98CC2320593F22B2071520D0060","39E8463E2AEA2F0BB7FB5F1D9ED7C0182DBE3C99EB6FF601A1497046133E8744","CANONICAL","2026-03-09 10:35:05.000000+00" +"afe10850-f268-4377-a71c-8e676cceba02",1,24619355,"EEB3A4BD3CE9461DD56D9C3BBC101CAC6607C7F32F5157B3E1B9D6ADBE01AFF3","1578EDC4AEFD287A683A8063AC177564B5EDA98CC2320593F22B2071520D0060","CANONICAL","2026-03-09 10:35:17.000000+00" +"666714b9-c47b-4415-be32-a3a709633ff5",1,24619356,"F7065F7AA5E856E3514134A78ACA091394D7974B1DEE04A9F41BB4611EC450B2","EEB3A4BD3CE9461DD56D9C3BBC101CAC6607C7F32F5157B3E1B9D6ADBE01AFF3","CANONICAL","2026-03-09 10:35:29.000000+00" +"3dc17d3e-4c76-4b67-90d9-8cf9ba4615e9",1,24619357,"ADD8342AC34C16A4CB66D12366653FB2B5539942C9333E7C27E25F053EEE59CC","F7065F7AA5E856E3514134A78ACA091394D7974B1DEE04A9F41BB4611EC450B2","CANONICAL","2026-03-09 10:35:41.000000+00" +"30804d0d-2d49-4cf5-bb2f-efaa47816b71",1,24619358,"49520EB769F3F6DBE297478C4A4BF72EABCFA85BE42C17FCF40CE9ED913E68BE","ADD8342AC34C16A4CB66D12366653FB2B5539942C9333E7C27E25F053EEE59CC","CANONICAL","2026-03-09 10:35:53.000000+00" +"ad605709-9183-4f6f-8441-9537582cce61",1,24619359,"BCD5F0D75F32E696E588E69471666FC95E2038365D0DE7834FAF4ABF3002233C","49520EB769F3F6DBE297478C4A4BF72EABCFA85BE42C17FCF40CE9ED913E68BE","CANONICAL","2026-03-09 10:36:05.000000+00" +"17b0e066-7a4d-4e47-8ce4-d91094711537",1,24619360,"20002E63494C236F937C3FAA171F77D54E54D7DE358EA6943AF066FA3C8287AE","BCD5F0D75F32E696E588E69471666FC95E2038365D0DE7834FAF4ABF3002233C","CANONICAL","2026-03-09 10:36:17.000000+00" +"aef4caca-34a1-42be-b560-a954792e01e8",1,24619361,"6E6DD136D59EF052ECDBEEB3D5F302C2D89081F1979629E71577412735C7864E","20002E63494C236F937C3FAA171F77D54E54D7DE358EA6943AF066FA3C8287AE","CANONICAL","2026-03-09 10:36:29.000000+00" +"f97d2cfa-2c39-478d-a827-b2e1b2031892",1,24619362,"9BF60C431E7CE00FAF1E1B049FD1592DFFDC4A58C8547818EABA091731963C05","6E6DD136D59EF052ECDBEEB3D5F302C2D89081F1979629E71577412735C7864E","CANONICAL","2026-03-09 10:36:41.000000+00" +"89503e03-3248-418e-a043-b09943c7d04a",1,24619363,"9BB863AE4E39FA6C85D2BDFFA4E3F63E91A4A16DA4404C35D0B7DEE322C0E2C9","9BF60C431E7CE00FAF1E1B049FD1592DFFDC4A58C8547818EABA091731963C05","CANONICAL","2026-03-09 10:36:53.000000+00" +"2bb2da27-bea8-4e63-b343-d5889c0e9563",1,24619364,"A289B2E77B8C5AAC2BBE9AF2E804F8FBFEB208F144E888475AF7222E38F12864","9BB863AE4E39FA6C85D2BDFFA4E3F63E91A4A16DA4404C35D0B7DEE322C0E2C9","CANONICAL","2026-03-09 10:37:05.000000+00" +"e0a03807-f5e7-4606-b983-3d1244fd64aa",1,24619365,"4AB5DDB57CE4C7D477175670346D6E59C3917722383B413A5048CA8772459233","A289B2E77B8C5AAC2BBE9AF2E804F8FBFEB208F144E888475AF7222E38F12864","CANONICAL","2026-03-09 10:37:17.000000+00" +"8b0bfcd3-f0e3-4385-9e4d-3805126575a5",1,24619366,"B84758EA8D07C9511E9949C5BFA47CE3FCECD8E4F538CC12F5C19FCFF42007B7","4AB5DDB57CE4C7D477175670346D6E59C3917722383B413A5048CA8772459233","CANONICAL","2026-03-09 10:37:29.000000+00" +"e39e8d16-c269-4967-9d0d-f3434ea28628",1,24619367,"6D6EFEC089CA887FEB44C4CBDF2CAF0BA6DDEFAF89401B3F759A76121AE5FAB9","B84758EA8D07C9511E9949C5BFA47CE3FCECD8E4F538CC12F5C19FCFF42007B7","CANONICAL","2026-03-09 10:37:41.000000+00" +"a01f2c22-ea2d-4edc-9b7a-64c774d8c6b9",1,24619368,"31F9E2F0269F3B65888AFD093DDCFCD19DA9DD5FA9574408C2AE23A56363AF3F","6D6EFEC089CA887FEB44C4CBDF2CAF0BA6DDEFAF89401B3F759A76121AE5FAB9","CANONICAL","2026-03-09 10:37:53.000000+00" +"f54eca37-f432-4974-b98d-2982e9c38e2a",1,24619369,"51E1505FBD13B7A98290E032AB43697ED581CA96230A7F7BA066A9E3F33C4A23","31F9E2F0269F3B65888AFD093DDCFCD19DA9DD5FA9574408C2AE23A56363AF3F","CANONICAL","2026-03-09 10:38:05.000000+00" +"66ff462b-8539-4cd7-be04-8b37f6a1950f",1,24619370,"1352882C95295722D32EDD7B9E56B22708DEDB02ECF188BA837B5AF71BDFFB36","51E1505FBD13B7A98290E032AB43697ED581CA96230A7F7BA066A9E3F33C4A23","CANONICAL","2026-03-09 10:38:17.000000+00" +"ebd3dd1c-13b0-434b-bb57-a52b18b12a78",1,24619371,"F34C8F9E9D1BA29074B62909DD154BF692BC58F3391F37BB135E09132C2653FF","1352882C95295722D32EDD7B9E56B22708DEDB02ECF188BA837B5AF71BDFFB36","CANONICAL","2026-03-09 10:38:29.000000+00" +"8b35618b-5954-42d8-866a-3ec9d8f70eb2",1,24619372,"9AED2953AE9835932D12673DAEBD672D3C2AE670F1B085EEEA3D95A2C7F763D9","F34C8F9E9D1BA29074B62909DD154BF692BC58F3391F37BB135E09132C2653FF","CANONICAL","2026-03-09 10:38:41.000000+00" +"b4fa2f68-586b-4939-aa52-722de8c2854d",1,24619373,"2C0648BA1ACE7631FFF66FFCD6BE7070BAA988F0AA93B85E54459E672561F920","9AED2953AE9835932D12673DAEBD672D3C2AE670F1B085EEEA3D95A2C7F763D9","CANONICAL","2026-03-09 10:38:53.000000+00" +"d066ade2-6832-4f60-892e-d7855dfacfa3",1,24619374,"A83D677BDF47207DFBE799344C268CBBD408B87C31F901044E0733182C96E5F7","2C0648BA1ACE7631FFF66FFCD6BE7070BAA988F0AA93B85E54459E672561F920","CANONICAL","2026-03-09 10:39:05.000000+00" +"4bdd0e32-fa16-448e-9fd6-1432571c3d84",1,24619375,"6C9B033CA8B53DA9951A3B769F01C8F26EE2CCCA352076299B789BE8CC4EDE55","A83D677BDF47207DFBE799344C268CBBD408B87C31F901044E0733182C96E5F7","CANONICAL","2026-03-09 10:39:17.000000+00" +"b6aa79c9-7a90-4528-88a1-de9b794f7c86",1,24619376,"47A91D246EED20CFDB845D6D98B9643F4FD3F9E4F6F464A4CAADF517A43A8AC0","6C9B033CA8B53DA9951A3B769F01C8F26EE2CCCA352076299B789BE8CC4EDE55","CANONICAL","2026-03-09 10:39:29.000000+00" +"27dd42c8-2dad-4898-a225-b3a6887a9cf6",1,24619377,"D5ADB72344AD4055F1C075EC9A63AAB1DF0E64E40D1EB674C85827D8C37A4A80","47A91D246EED20CFDB845D6D98B9643F4FD3F9E4F6F464A4CAADF517A43A8AC0","CANONICAL","2026-03-09 10:39:41.000000+00" +"8f724da3-d9a1-49cb-82f3-7e980cf816c0",1,24619378,"222A3F096635C0862D63DF6C0B3B3AC6BC48928277475F328940D496530A76BC","D5ADB72344AD4055F1C075EC9A63AAB1DF0E64E40D1EB674C85827D8C37A4A80","CANONICAL","2026-03-09 10:39:53.000000+00" +"c5d0dd0e-c6b3-4d1d-ba77-47e6c5faec34",1,24619379,"8A7FC23D45E9FB2BEA20A44A779DBE3FF3EA9120DB75B7D761310339181BFFD1","222A3F096635C0862D63DF6C0B3B3AC6BC48928277475F328940D496530A76BC","CANONICAL","2026-03-09 10:40:05.000000+00" +"0b8cbaea-bcd9-4c68-938f-614574d01d5a",1,24619380,"DE3573A3339A8D54D4699F79B9DD538AB628215F918783321498B4F95C6F52F0","8A7FC23D45E9FB2BEA20A44A779DBE3FF3EA9120DB75B7D761310339181BFFD1","CANONICAL","2026-03-09 10:40:17.000000+00" +"121fd81a-171c-4f67-8673-13c332eae68c",1,24619381,"68B962CB2643015029CE02C8F8BBF47CB7E8E4837426F4088949A443ECC2BA45","DE3573A3339A8D54D4699F79B9DD538AB628215F918783321498B4F95C6F52F0","CANONICAL","2026-03-09 10:40:29.000000+00" +"ddaab77c-645a-4776-ab40-0725707eba19",1,24619382,"DDDEB553026DE57F2944F041D1521098F50579A39362B1E89096595E4E574AE6","68B962CB2643015029CE02C8F8BBF47CB7E8E4837426F4088949A443ECC2BA45","CANONICAL","2026-03-09 10:40:41.000000+00" +"70719235-68b6-4eaa-a4b3-acf4ec83d753",1,24619383,"194DD6C05505409570D1CA38C80E5BE1030876773A62B4B2C1D67E8050B7A571","DDDEB553026DE57F2944F041D1521098F50579A39362B1E89096595E4E574AE6","CANONICAL","2026-03-09 10:40:53.000000+00" +"fa5f9154-917a-4d9d-bea6-94aa9f579b7c",1,24619384,"8FF1FE367B00B79031B3F0ED1A7C107ABAA62EE36BB5C4588228CD5766F78AB5","194DD6C05505409570D1CA38C80E5BE1030876773A62B4B2C1D67E8050B7A571","CANONICAL","2026-03-09 10:41:05.000000+00" +"46bae556-206b-4529-94b0-7d372b8939a7",1,24619385,"4F117F0705BC9AD3CFB223E7C87221CFAE043CABBA959F280AA2B67E4E480F92","8FF1FE367B00B79031B3F0ED1A7C107ABAA62EE36BB5C4588228CD5766F78AB5","CANONICAL","2026-03-09 10:41:17.000000+00" +"d8c7ae95-8bf8-4069-9e95-5765d5e41327",1,24619386,"F3E60A43854F05D26C6DBF6AA7879685ED86665F7E6F3D279FF3BB73697B8581","4F117F0705BC9AD3CFB223E7C87221CFAE043CABBA959F280AA2B67E4E480F92","CANONICAL","2026-03-09 10:41:29.000000+00" +"346097fe-2b2f-4699-af26-ab4aeb1a1034",1,24619387,"B8D4E6A27C6D8C90535957AE00A1D16C5748B86D44D01C70064F28F1A7288EE0","F3E60A43854F05D26C6DBF6AA7879685ED86665F7E6F3D279FF3BB73697B8581","CANONICAL","2026-03-09 10:41:41.000000+00" +"82d76198-625d-4b53-bb25-acde0e219aff",1,24619388,"6B92C8F7EC80C68AFFA1F4A01B0BF5F522B8FF2F1AA2477C3AC6FB9AB8ABA382","B8D4E6A27C6D8C90535957AE00A1D16C5748B86D44D01C70064F28F1A7288EE0","CANONICAL","2026-03-09 10:41:53.000000+00" +"c3f4c7d0-2ce2-4b39-b903-0bdc3bf8be3d",1,24619389,"438D640A26F78A792339B529D564FEB64BC61BEC1B1E6836DC9A39F15F0BC415","6B92C8F7EC80C68AFFA1F4A01B0BF5F522B8FF2F1AA2477C3AC6FB9AB8ABA382","CANONICAL","2026-03-09 10:42:05.000000+00" +"710f5f0e-6a78-4b0f-a340-a2b009a26b33",1,24619390,"E3E4D6D2CC20F03EE4C9D8994C38212DD0B7CABEB01E582B66AD7821ED05FD2E","438D640A26F78A792339B529D564FEB64BC61BEC1B1E6836DC9A39F15F0BC415","CANONICAL","2026-03-09 10:42:17.000000+00" +"8ec09ea2-8358-40bf-8c22-d2b837a83d72",1,24619391,"BB28232C0A1B030A87C58FF0E63AF5B51B4F5811CA41BA293702D864F4E489B0","E3E4D6D2CC20F03EE4C9D8994C38212DD0B7CABEB01E582B66AD7821ED05FD2E","CANONICAL","2026-03-09 10:42:29.000000+00" +"de274f43-4661-4c51-bd7b-1da74a03d8a4",1,24619392,"9FA6906D2AC30767B6978C4343CE09A215B2C4F9B52021E6B5264611BE584CF6","BB28232C0A1B030A87C58FF0E63AF5B51B4F5811CA41BA293702D864F4E489B0","CANONICAL","2026-03-09 10:42:41.000000+00" +"1195325b-5dd9-465f-a510-b3b0741ccea4",1,24619393,"301B51825B5DB0E1EF8396BE1F48942608219114D5BD48702D9070B933AE733C","9FA6906D2AC30767B6978C4343CE09A215B2C4F9B52021E6B5264611BE584CF6","CANONICAL","2026-03-09 10:42:53.000000+00" +"56402670-a15a-4c24-b2a8-6b9714160629",1,24619394,"202F00913C99E3ECA8113A8599521451886976EF598A410CE47D4A0D8C4A9215","301B51825B5DB0E1EF8396BE1F48942608219114D5BD48702D9070B933AE733C","CANONICAL","2026-03-09 10:43:05.000000+00" +"28aabcbf-8163-48a4-9ac7-3d08bdca71f5",1,24619395,"0FF3D33C29215A06ABF3E439C3DFA85D43E1B427F8ED47A28E9F66A4565E70A7","202F00913C99E3ECA8113A8599521451886976EF598A410CE47D4A0D8C4A9215","CANONICAL","2026-03-09 10:43:17.000000+00" +"bad4c214-3809-4d5b-b292-6ae52f9e3f06",1,24619396,"E8EBD79AB3828D94A6E34E2B0B48B1C3F3CC339C8FDF2075C03AB21017D5892C","0FF3D33C29215A06ABF3E439C3DFA85D43E1B427F8ED47A28E9F66A4565E70A7","CANONICAL","2026-03-09 10:43:29.000000+00" +"d2f0ac0a-8dae-4f70-b29e-b95675ecae43",1,24619397,"33488FBDAE8BB4C550544921B6635761D7550BF422CF9F23CA9B8F7D48034BED","E8EBD79AB3828D94A6E34E2B0B48B1C3F3CC339C8FDF2075C03AB21017D5892C","CANONICAL","2026-03-09 10:43:41.000000+00" +"b608c355-f92f-48ef-a82e-7d35543c1a6d",1,24619398,"88D96113A45B5AE7E7C7A240895605F61CCFAA0D5F87E2AB122BF387EC0D4C91","33488FBDAE8BB4C550544921B6635761D7550BF422CF9F23CA9B8F7D48034BED","CANONICAL","2026-03-09 10:43:53.000000+00" +"b39d7808-9959-47d1-9e69-fb4a05ee82dc",1,24619399,"989EDA813923FD2E9BF719C53DFA7597D5CB0BAB666FB2DBD9B2149BDEFA7513","88D96113A45B5AE7E7C7A240895605F61CCFAA0D5F87E2AB122BF387EC0D4C91","CANONICAL","2026-03-09 10:44:05.000000+00" +"8367bb2b-dfa7-4d31-9c0c-f3389e2f455d",1,24619400,"C090AF8254DEE02A3F2DDE94A1906BCED411781294A96D4F973AE71CD98F09E5","989EDA813923FD2E9BF719C53DFA7597D5CB0BAB666FB2DBD9B2149BDEFA7513","CANONICAL","2026-03-09 10:44:17.000000+00" +"cfb347ea-f088-4114-904f-0fac28eea5e6",1,24619401,"12CC169C8CCB99FE6339C42D876B5329BCE236334EB271CD11218A164FFF6192","C090AF8254DEE02A3F2DDE94A1906BCED411781294A96D4F973AE71CD98F09E5","CANONICAL","2026-03-09 10:44:29.000000+00" +"a2bfe74e-63ec-4c0e-989b-1bec9c95523b",1,24619402,"B46758FDD5DBCA2C4551B4ACD70E549664C2E7ADEBF9649B19607C7FD55AE2B9","12CC169C8CCB99FE6339C42D876B5329BCE236334EB271CD11218A164FFF6192","CANONICAL","2026-03-09 10:44:41.000000+00" +"ae0a7fd9-cbcc-4697-b893-171df229a159",1,24619403,"CC2E9C9BBA62EAC401C91DF79AAA5A12A22FEC724446539C601312C7984F670F","B46758FDD5DBCA2C4551B4ACD70E549664C2E7ADEBF9649B19607C7FD55AE2B9","CANONICAL","2026-03-09 10:44:53.000000+00" +"00dba402-4f84-4732-adb2-5349af6fd0c6",1,24619404,"58B3A22AADF60FD4B3368A6EF64F051CE88A1A4B541A0DC6C171D0A5DD297641","CC2E9C9BBA62EAC401C91DF79AAA5A12A22FEC724446539C601312C7984F670F","CANONICAL","2026-03-09 10:45:05.000000+00" +"21defd73-52dc-4442-9435-f5cae6aff449",1,24619405,"041B3E0ABA51E8BFFC1E6B85A5B54D98AF47FF9FF50785995B3250B044E73700","58B3A22AADF60FD4B3368A6EF64F051CE88A1A4B541A0DC6C171D0A5DD297641","CANONICAL","2026-03-09 10:45:17.000000+00" +"398a2c20-d1b5-47ba-b8ef-2e6cb5f12282",1,24619406,"97D7FA2B685846ED74CF27FAEF9296D1574BC9A2D7EBFFDB5D83F04A1E0387B1","041B3E0ABA51E8BFFC1E6B85A5B54D98AF47FF9FF50785995B3250B044E73700","CANONICAL","2026-03-09 10:45:29.000000+00" +"b2bd4b21-e886-4f49-872b-cb4aa23b7f65",1,24619407,"C7BEE8A07B34662DAAD9ABC9C6CBFED266ACF7749420824711E5F76DE1F462F9","97D7FA2B685846ED74CF27FAEF9296D1574BC9A2D7EBFFDB5D83F04A1E0387B1","CANONICAL","2026-03-09 10:45:41.000000+00" +"1d9792d1-240b-4cfc-a5c0-b26e6fe517b1",1,24619408,"92328F3DAB31B7603B3422582F85A1A5CB3FFBACE50DF7D299456A9CC3D784D7","C7BEE8A07B34662DAAD9ABC9C6CBFED266ACF7749420824711E5F76DE1F462F9","CANONICAL","2026-03-09 10:45:53.000000+00" +"f5584213-ad6b-4eab-a97c-ac6f433b05bb",1,24619409,"ADDA7FABEC8B4E7AC237F6A113F014298FB412684714B11876CDF8574754D796","92328F3DAB31B7603B3422582F85A1A5CB3FFBACE50DF7D299456A9CC3D784D7","CANONICAL","2026-03-09 10:46:05.000000+00" +"c9f7d8b7-651b-4a11-bbb7-f4930e4429db",1,24619410,"CCB2883942DDF372858C3D315B0619A87E530ABF605959D8E5624DFC43251FFA","ADDA7FABEC8B4E7AC237F6A113F014298FB412684714B11876CDF8574754D796","CANONICAL","2026-03-09 10:46:17.000000+00" +"7f910dcf-fd14-4ddc-898d-759e24c490fc",1,24619411,"FD01FCB76DFC5D0FF09D78ABDA37A146621ECBC84307CB7E3F4006FB77662A00","CCB2883942DDF372858C3D315B0619A87E530ABF605959D8E5624DFC43251FFA","CANONICAL","2026-03-09 10:46:29.000000+00" +"d5bf72b8-9ebe-48b7-8c1e-3d62237db198",1,24619412,"2A3D6557CAAE2D61DBC1368B45F065822800CF19C7E59190B7372CBB83075947","FD01FCB76DFC5D0FF09D78ABDA37A146621ECBC84307CB7E3F4006FB77662A00","CANONICAL","2026-03-09 10:46:41.000000+00" +"d548fd8f-5c1e-4f0e-92c9-b7adb73fe302",1,24619413,"F6025443E741B0A9016679AC9D620416FE6E1319C301E7A094B974FCB37367D5","2A3D6557CAAE2D61DBC1368B45F065822800CF19C7E59190B7372CBB83075947","CANONICAL","2026-03-09 10:46:53.000000+00" +"e7527931-e56d-4784-a05a-409867162890",1,24619414,"9096FF74C1DCFC08D6CF8F82339E6B0898F4AD620F2EFD09A3C58C4D86A1E36E","F6025443E741B0A9016679AC9D620416FE6E1319C301E7A094B974FCB37367D5","CANONICAL","2026-03-09 10:47:05.000000+00" +"a06b0302-64c9-4797-b39e-03271ec5a51d",1,24619415,"A75D3A5CF551C94E775278A440A82E5451F01E063942C2A8759C84C8FC64CB12","9096FF74C1DCFC08D6CF8F82339E6B0898F4AD620F2EFD09A3C58C4D86A1E36E","CANONICAL","2026-03-09 10:47:17.000000+00" +"082d7e2d-7dc9-4ebc-b336-9c135660a20d",1,24619416,"0D2F6EC76AD8050CE6D8B9C3EE3B19BCFC024701A9BCA4EF119EFD4C673433FB","A75D3A5CF551C94E775278A440A82E5451F01E063942C2A8759C84C8FC64CB12","CANONICAL","2026-03-09 10:47:29.000000+00" +"e09fdced-5beb-42ff-975e-b815b2fac069",1,24619417,"5408BAFA2456E6DAB7DF00540CE1C4D2944ECBDFDABF1FB0F684C225A6281D5D","0D2F6EC76AD8050CE6D8B9C3EE3B19BCFC024701A9BCA4EF119EFD4C673433FB","CANONICAL","2026-03-09 10:47:41.000000+00" +"3519dec8-a63b-4e5b-9081-c10dbc978f0c",1,24619418,"F81BABCAB5C7A2E3F2B66B0053227927852F4592CD93695D5768D99A245A5E50","5408BAFA2456E6DAB7DF00540CE1C4D2944ECBDFDABF1FB0F684C225A6281D5D","CANONICAL","2026-03-09 10:47:53.000000+00" +"393d9da5-2b38-4710-81ff-42acd57a41b5",1,24619419,"CDD5141EE9F02F041BDFED03017288308F5387F8B24A95CCBF16D05575D2FBB0","F81BABCAB5C7A2E3F2B66B0053227927852F4592CD93695D5768D99A245A5E50","CANONICAL","2026-03-09 10:48:05.000000+00" +"91689e49-0a94-4e1d-af6c-ff5f3d82c548",1,24619420,"11D7F28ABBF4E3705B471F36ED4F2B3E3B435A9426C89816FB6AE56CA68F542B","CDD5141EE9F02F041BDFED03017288308F5387F8B24A95CCBF16D05575D2FBB0","CANONICAL","2026-03-09 10:48:17.000000+00" +"059b9c12-f7f4-4a4c-86a0-259834ef78de",1,24619421,"1896BB0AFDB893FF9DEEA490B064F6011091A89E3ECBB5C0D717CD654CFCA640","11D7F28ABBF4E3705B471F36ED4F2B3E3B435A9426C89816FB6AE56CA68F542B","CANONICAL","2026-03-09 10:48:29.000000+00" +"897fa6d9-f5c3-4c12-9e20-29980df6d8ee",1,24619422,"9C3C47CCBF054DD17405B9AA468CA420892E6407E7338B1C401A8BBD7B6A6513","1896BB0AFDB893FF9DEEA490B064F6011091A89E3ECBB5C0D717CD654CFCA640","CANONICAL","2026-03-09 10:48:41.000000+00" +"b4751e7c-1ede-4908-80ce-71c7523e6811",1,24619423,"BA937E9A3EB20BBBAA6F37DD63E9D4B45C7D95C8A6EBD7FA47051F2904D8C1EF","9C3C47CCBF054DD17405B9AA468CA420892E6407E7338B1C401A8BBD7B6A6513","CANONICAL","2026-03-09 10:48:53.000000+00" +"c3836fe8-572b-4bfb-8ad0-c18084a2a726",1,24619424,"19E259C73638D2E70FB7ACDB5CF3D5FD2D6F17A4977425243B196159FC3894F2","BA937E9A3EB20BBBAA6F37DD63E9D4B45C7D95C8A6EBD7FA47051F2904D8C1EF","CANONICAL","2026-03-09 10:49:05.000000+00" +"ee1ee4f8-c56c-4d02-b256-775e2bbeec5b",1,24619425,"AAAD541C060C773E4519852D37AA67C0386EC098722E74E384125F426AD05B87","19E259C73638D2E70FB7ACDB5CF3D5FD2D6F17A4977425243B196159FC3894F2","CANONICAL","2026-03-09 10:49:17.000000+00" +"fb778d69-be6e-48d5-a1e8-59916013acd4",1,24619426,"2A79930A9CFDCA7DB515522639308E3335911DFBCB8CFC3E298DB835B974B8EB","AAAD541C060C773E4519852D37AA67C0386EC098722E74E384125F426AD05B87","CANONICAL","2026-03-09 10:49:29.000000+00" +"3b854f9b-3194-47e1-b94b-5e86d005bcf3",1,24619427,"B8BBB2F241F9C22BBFFAF28B243DEC199CC70DB8976B8EC7EFB82488A3DB5A34","2A79930A9CFDCA7DB515522639308E3335911DFBCB8CFC3E298DB835B974B8EB","CANONICAL","2026-03-09 10:49:41.000000+00" +"3db65752-544c-47e8-aea9-63bc507b5b1f",1,24619428,"A302D598F1AC4715434A6E41B4BE5F51484DEF5E4ADB0010B1AADF5103127A0E","B8BBB2F241F9C22BBFFAF28B243DEC199CC70DB8976B8EC7EFB82488A3DB5A34","CANONICAL","2026-03-09 10:49:53.000000+00" +"9570f0f7-faec-4da8-b43b-7a7c0ed3e29f",1,24619429,"802304505B8732210B2ADCAABCFF9269955B2F69AD3ED075C7542C0541B375A0","A302D598F1AC4715434A6E41B4BE5F51484DEF5E4ADB0010B1AADF5103127A0E","CANONICAL","2026-03-09 10:50:05.000000+00" +"c3f3e228-5da2-4124-abbe-91c5d27fdda5",1,24619430,"154BDC7E91AC48E5AFD02F0395BE183C4D37FEB6C0AEBD82F958FEEC7642EFD1","802304505B8732210B2ADCAABCFF9269955B2F69AD3ED075C7542C0541B375A0","CANONICAL","2026-03-09 10:50:17.000000+00" +"3393db27-c8f4-46f9-a7c7-0cef57d7555f",1,24619431,"A30B27973EF51A44D96557B8A179C2376713F048A00EA049C3251004DA9595AE","154BDC7E91AC48E5AFD02F0395BE183C4D37FEB6C0AEBD82F958FEEC7642EFD1","CANONICAL","2026-03-09 10:50:29.000000+00" +"da3e59fa-b7ab-4901-bec0-dd226b7723e3",1,24619432,"1D883ED18AA71F138158EB2330AEC7A4F4E9305142AC80BDB98311E0D3A60914","A30B27973EF51A44D96557B8A179C2376713F048A00EA049C3251004DA9595AE","CANONICAL","2026-03-09 10:50:41.000000+00" +"4600f53e-9141-47ad-9860-9b9aa835a182",1,24619433,"87CB9CB829189FD93AD88B3447CB1457F447D7572BCDB05AE10D19C869EF38A9","1D883ED18AA71F138158EB2330AEC7A4F4E9305142AC80BDB98311E0D3A60914","CANONICAL","2026-03-09 10:50:53.000000+00" +"251e0ff4-7833-4e93-88bb-28f45ac6d603",1,24619434,"6F547010E0E992E9554EDF44DD8D8DC7E278B5AF49809E2F0D0A3BA6C5786F5A","87CB9CB829189FD93AD88B3447CB1457F447D7572BCDB05AE10D19C869EF38A9","CANONICAL","2026-03-09 10:51:05.000000+00" +"c70776d8-a70b-45e7-912b-8bb18c09044e",1,24619435,"0309D0239B45FF6720E41E36E365915188DBB711232E6838BA61E17078762C30","6F547010E0E992E9554EDF44DD8D8DC7E278B5AF49809E2F0D0A3BA6C5786F5A","CANONICAL","2026-03-09 10:51:17.000000+00" +"be0a9711-b375-499b-a9fb-0f310f0a7a9f",1,24619436,"271ABDE2DEDEC49E34BFF873950F62993173387EA4AB91A6B71D0E15F222D2DE","0309D0239B45FF6720E41E36E365915188DBB711232E6838BA61E17078762C30","CANONICAL","2026-03-09 10:51:29.000000+00" +"9aadfc3e-5165-407a-99d3-89f5c7f697bf",1,24619437,"A405B1DE3DCC8BFB144B0132F4BEE5DC09159AB99D9FAA552E1F824C56021EA5","271ABDE2DEDEC49E34BFF873950F62993173387EA4AB91A6B71D0E15F222D2DE","CANONICAL","2026-03-09 10:51:41.000000+00" +"a8661aa5-f15e-4220-8bc5-bcb7ba456d63",1,24619438,"88CCFFB10C0516F08C700C6633B1CEDC8556C5A9C973F27717DFD1058A3C270C","A405B1DE3DCC8BFB144B0132F4BEE5DC09159AB99D9FAA552E1F824C56021EA5","CANONICAL","2026-03-09 10:51:53.000000+00" +"4c1a9a38-9c8a-4c03-ae3b-c33d8d032f01",1,24619439,"7FFBA88A6538257FE7EDE29149F46E18E7502BD3F29042069F60E4D42FC932D2","88CCFFB10C0516F08C700C6633B1CEDC8556C5A9C973F27717DFD1058A3C270C","CANONICAL","2026-03-09 10:52:05.000000+00" +"6e9b557a-ff70-4d68-99c9-71d532c72ad4",1,24619440,"9602449E1AAA6B966D21DC5D25EB42B5C2D7040B845D18E69CC44E17F6873B55","7FFBA88A6538257FE7EDE29149F46E18E7502BD3F29042069F60E4D42FC932D2","CANONICAL","2026-03-09 10:52:17.000000+00" +"39bc5a24-b80e-4029-8bea-2481d304d9a8",1,24619441,"4119556D61F5ED6361F81E8B3A5BCB798DEDD8F43F57C3C8D20D7FA0188E635A","9602449E1AAA6B966D21DC5D25EB42B5C2D7040B845D18E69CC44E17F6873B55","CANONICAL","2026-03-09 10:52:29.000000+00" +"0f2aa194-45a7-49a1-aa4e-575eef128c6d",1,24619442,"8240F5AEF4C9AD26DABC31F831ADE9BCF028B69E91C16AC504DD55F2A3D75AA1","4119556D61F5ED6361F81E8B3A5BCB798DEDD8F43F57C3C8D20D7FA0188E635A","CANONICAL","2026-03-09 10:52:41.000000+00" +"fc87b2ed-52e5-4600-b4c0-16713f5e1bf5",1,24619443,"D7D83CD02890D78AC409AFA207D988E5C7CF5C557104CD55345740E506694D97","8240F5AEF4C9AD26DABC31F831ADE9BCF028B69E91C16AC504DD55F2A3D75AA1","CANONICAL","2026-03-09 10:52:53.000000+00" +"48b60ff4-cfd1-47f3-862b-479cab39019d",1,24619444,"DE6929C19381E7B7C1DCE489CD4C457D2F4B673AFDCE4BB3551B27199F89DEEE","D7D83CD02890D78AC409AFA207D988E5C7CF5C557104CD55345740E506694D97","CANONICAL","2026-03-09 10:53:05.000000+00" +"b4d1fbbd-7d10-456a-958b-f5d61724fe9f",1,24619445,"3C2ABBC4EDF978843CC99665F80BB7B1EBF712B74487A3AE1DB89053339749A4","DE6929C19381E7B7C1DCE489CD4C457D2F4B673AFDCE4BB3551B27199F89DEEE","CANONICAL","2026-03-09 10:53:17.000000+00" +"3476a269-4d8d-4df7-86fb-f8e349a9f6ac",1,24619446,"422D547003BE27BAB142BD61A6ADEDFD1D160EFBC5BA8DE521CD93C99567B19A","3C2ABBC4EDF978843CC99665F80BB7B1EBF712B74487A3AE1DB89053339749A4","CANONICAL","2026-03-09 10:53:29.000000+00" +"6d9c499c-6773-4551-abf2-78bfe84dbf3d",1,24619447,"B8D40F17E01E3FFC116D49EDD7E50070A2095268CF1FE7E0DCD69583744C2463","422D547003BE27BAB142BD61A6ADEDFD1D160EFBC5BA8DE521CD93C99567B19A","CANONICAL","2026-03-09 10:53:41.000000+00" +"ea4f6bb2-54ce-40d2-96be-153b16c940a4",1,24619448,"49FF49527FAB25C58E9B2E7A79B41A5F4C9DDDA938754DA159EF2B265E05DF99","B8D40F17E01E3FFC116D49EDD7E50070A2095268CF1FE7E0DCD69583744C2463","CANONICAL","2026-03-09 10:53:53.000000+00" +"e70ea62f-7956-4cd6-8294-9e3fd72a1662",1,24619449,"4755EAE4909E0A20614782E5033E1405D6A71DDBE6BD9CFD81CA79D19BF1A63D","49FF49527FAB25C58E9B2E7A79B41A5F4C9DDDA938754DA159EF2B265E05DF99","CANONICAL","2026-03-09 10:54:05.000000+00" +"fc2f383f-8acc-4eef-bd3b-0d33113e906f",1,24619450,"E735C29F4CA93BBA549A62AE9C70449928A05470670FA0CDC6FB78C3A6727770","4755EAE4909E0A20614782E5033E1405D6A71DDBE6BD9CFD81CA79D19BF1A63D","CANONICAL","2026-03-09 10:54:17.000000+00" +"f855a62a-266e-4b53-adfb-13ac9f3088bf",1,24619451,"EEA4D2228183E2340339CA33820D412FF4BAA396B9AF0CAFC92FBE1E3E8F210B","E735C29F4CA93BBA549A62AE9C70449928A05470670FA0CDC6FB78C3A6727770","CANONICAL","2026-03-09 10:54:29.000000+00" +"2fcc8425-a747-450e-80c8-e0aa3d708e20",1,24619452,"9203840401118BD083DFBAD81F3A066B9306BCB524271DD010AAD5F38147BF53","EEA4D2228183E2340339CA33820D412FF4BAA396B9AF0CAFC92FBE1E3E8F210B","CANONICAL","2026-03-09 10:54:41.000000+00" +"4554883a-5563-445d-9973-8c67564c1977",1,24619453,"88443281F8043E6B5CE5062042EE3D8C07B7F38D9A56B2AE245959272F2684D9","9203840401118BD083DFBAD81F3A066B9306BCB524271DD010AAD5F38147BF53","CANONICAL","2026-03-09 10:54:53.000000+00" +"fa4f229e-e3de-4576-be87-ac6272022b83",1,24619454,"0C125CDAA9298441BA1E635D5743295EEE52CEBD5AE1B4E07CFC5C5811F235B3","88443281F8043E6B5CE5062042EE3D8C07B7F38D9A56B2AE245959272F2684D9","CANONICAL","2026-03-09 10:55:05.000000+00" +"48370cc0-d457-40e4-ac55-e6a5cfd20d57",1,24619455,"183E06189B1B96C7B934DB895D8431F84D7B80CA49AD863D2F0C7CCAAF1ECDE8","0C125CDAA9298441BA1E635D5743295EEE52CEBD5AE1B4E07CFC5C5811F235B3","CANONICAL","2026-03-09 10:55:17.000000+00" +"2f135c21-24b0-4f86-8025-e85f5270d964",1,24619456,"F1D4ADDBE47A784332097033E9A58CCC7DB6117FF5FA52FB0BA797F5412EC37A","183E06189B1B96C7B934DB895D8431F84D7B80CA49AD863D2F0C7CCAAF1ECDE8","CANONICAL","2026-03-09 10:55:29.000000+00" +"3efee8ca-187e-442a-9344-83fe5746214b",1,24619457,"78D437EA6F63A694E8CF35363A6F1436B4FF6583C88347E6BFD269D3489475B1","F1D4ADDBE47A784332097033E9A58CCC7DB6117FF5FA52FB0BA797F5412EC37A","CANONICAL","2026-03-09 10:55:41.000000+00" +"7d52b80a-eb18-4497-a91d-f4ff4f67ed32",1,24619458,"545DCFF8C2C1E198C0F914872C958A14D505350E4C558C7C1240668212CED7BC","78D437EA6F63A694E8CF35363A6F1436B4FF6583C88347E6BFD269D3489475B1","CANONICAL","2026-03-09 10:55:53.000000+00" +"0b91e3f1-565f-46c9-a3ec-468926e4a978",1,24619459,"D7F90C7674C6EAE999F02901CF9A597DCA0618D9782054793FE6EA91C683FB2C","545DCFF8C2C1E198C0F914872C958A14D505350E4C558C7C1240668212CED7BC","CANONICAL","2026-03-09 10:56:05.000000+00" +"f5258ecd-decb-47d3-896d-ecd2087b5fc8",1,24619460,"55CD0B941E6738D113852711CBD152F2A72C42379BD39097302B1C98C7A7C3CC","D7F90C7674C6EAE999F02901CF9A597DCA0618D9782054793FE6EA91C683FB2C","CANONICAL","2026-03-09 10:56:17.000000+00" +"2a9c297d-f7bf-4cf9-8d59-9c8478cf01da",1,24619461,"626B44426CE526148721EFDFBBC9DE291B5DF09EF9A6FF5ACC60D9B3991CC27F","55CD0B941E6738D113852711CBD152F2A72C42379BD39097302B1C98C7A7C3CC","CANONICAL","2026-03-09 10:56:29.000000+00" +"4fcf11ba-c334-4add-81fa-e48cbd087374",1,24619462,"6580A5216B6C2DC41BF1ED8E5AC8AEF7A4F9DE91C583430AC338020611B8E600","626B44426CE526148721EFDFBBC9DE291B5DF09EF9A6FF5ACC60D9B3991CC27F","CANONICAL","2026-03-09 10:56:41.000000+00" +"16ff7b96-342f-41aa-a933-5db509ada464",1,24619463,"7494D1815AB1044EA606AA2A19BA6529AA8FE83E97513618CD90ABCF03F7B3BB","6580A5216B6C2DC41BF1ED8E5AC8AEF7A4F9DE91C583430AC338020611B8E600","CANONICAL","2026-03-09 10:56:53.000000+00" +"6ace9820-0b7b-4455-8aa7-130ff2e763b3",1,24619464,"DCE890A575CBFE7BFC34418B336698429609D1C87E36DFBA51DB0945206A872E","7494D1815AB1044EA606AA2A19BA6529AA8FE83E97513618CD90ABCF03F7B3BB","CANONICAL","2026-03-09 10:57:05.000000+00" +"e9960e67-8e83-45dc-8f06-283c74c807ad",1,24619465,"339CE019435B6EA1380450DE3BF1FC137007585D27D87585383224B635A0DA37","DCE890A575CBFE7BFC34418B336698429609D1C87E36DFBA51DB0945206A872E","CANONICAL","2026-03-09 10:57:17.000000+00" +"64149cf5-7e35-4f23-ab88-1784714292c1",1,24619466,"DC3E709326258FC25CB7AF8EBD30D56A3D1E7B0F142DBB4B992FC29C2D7A57AD","339CE019435B6EA1380450DE3BF1FC137007585D27D87585383224B635A0DA37","CANONICAL","2026-03-09 10:57:29.000000+00" +"1179bd86-a465-481f-a6e0-9654c3b0d0ac",1,24619467,"008B94A82584E19068912416CA564269F63D24CFBA2AD66FA7CFDADA1E5629CF","DC3E709326258FC25CB7AF8EBD30D56A3D1E7B0F142DBB4B992FC29C2D7A57AD","CANONICAL","2026-03-09 10:57:41.000000+00" +"fe5dd0b0-074d-424a-a4b7-654b4306c4f0",1,24619468,"75909326CF70B4AB5195184E56899DD9070167FF5BB4770846DB088A19D87B3C","008B94A82584E19068912416CA564269F63D24CFBA2AD66FA7CFDADA1E5629CF","CANONICAL","2026-03-09 10:57:53.000000+00" +"38549bb3-c231-427c-907e-018724a3edd4",1,24619469,"365B4715863A6F2075D837176D918E692084C27547996B274EC5BEDC685DA606","75909326CF70B4AB5195184E56899DD9070167FF5BB4770846DB088A19D87B3C","CANONICAL","2026-03-09 10:58:05.000000+00" +"5329d4bc-018f-45f0-b741-863b743a0f13",1,24619470,"F3DA6E0A399C1DABFBD7F38AAEF1840C8BC1BD65513C1713F8A24800489D77B5","365B4715863A6F2075D837176D918E692084C27547996B274EC5BEDC685DA606","CANONICAL","2026-03-09 10:58:17.000000+00" +"7d5b4f05-191c-46ef-b595-fd398002a1b1",1,24619471,"A734696D9117BF771D7F2DCF254029E35E2F346A6443EEB6AA8F7FBAB7B0F348","F3DA6E0A399C1DABFBD7F38AAEF1840C8BC1BD65513C1713F8A24800489D77B5","CANONICAL","2026-03-09 10:58:29.000000+00" +"b2828e27-f266-43f5-bc8d-60ce6087fc90",1,24619472,"AC915E4BD82B71637586A15F8321ECDA8537F99C2CE0DA7BF9C1BA095D38C3F3","A734696D9117BF771D7F2DCF254029E35E2F346A6443EEB6AA8F7FBAB7B0F348","CANONICAL","2026-03-09 10:58:41.000000+00" +"3b8de08a-42ec-4905-86cd-5759334511d2",1,24619473,"D9ABFF9D9C24FB001FB39350B320396BCB1590B3281BD4D801942FB94F350C3A","AC915E4BD82B71637586A15F8321ECDA8537F99C2CE0DA7BF9C1BA095D38C3F3","CANONICAL","2026-03-09 10:58:53.000000+00" +"189c5946-4898-46ef-a6c2-30391c9cb947",1,24619474,"AB565CA93E6B53FACF9143B42F028C2057F99DF2656D25674831C47D087C8A2F","D9ABFF9D9C24FB001FB39350B320396BCB1590B3281BD4D801942FB94F350C3A","CANONICAL","2026-03-09 10:59:05.000000+00" +"986f9b92-9607-4ea3-b8fe-5ad8a3a997c9",1,24619475,"EAB22D61BC867C820B1A7404FD95C9888F73FC1F41E2FAE286584E341510F73F","AB565CA93E6B53FACF9143B42F028C2057F99DF2656D25674831C47D087C8A2F","CANONICAL","2026-03-09 10:59:17.000000+00" +"f4be9ace-3066-4ee4-bc99-a719ef3bbf0a",1,24619476,"A973D54452F8B2DFCCD4E1EB50412E6BC14E8602259779BE41D4CBCB093010D2","EAB22D61BC867C820B1A7404FD95C9888F73FC1F41E2FAE286584E341510F73F","CANONICAL","2026-03-09 10:59:29.000000+00" +"f8184801-4b89-4e40-bbbf-7a9231732fb6",1,24619477,"67E768EFC5F526ED534EF68FDB6AA82D673A5533C857D62CA6E4DA527E6B863B","A973D54452F8B2DFCCD4E1EB50412E6BC14E8602259779BE41D4CBCB093010D2","CANONICAL","2026-03-09 10:59:41.000000+00" +"8181d265-6a54-43f1-b569-a9e4127ba22a",1,24619478,"9F704FA67461FE8C48EE291BD78FE71ECD8851B56D0CDE3D4B61CE0CBF9854DB","67E768EFC5F526ED534EF68FDB6AA82D673A5533C857D62CA6E4DA527E6B863B","CANONICAL","2026-03-09 10:59:53.000000+00" +"7858de4a-d0a4-4c48-9a78-51502bd97f11",1,24619479,"6E821738F0A541CC420838131B9E0399677EBD5891648C6E594C6C2DF1EA9355","9F704FA67461FE8C48EE291BD78FE71ECD8851B56D0CDE3D4B61CE0CBF9854DB","CANONICAL","2026-03-09 11:00:05.000000+00" +"5f593463-5051-4b3b-a749-e0adc39e1b06",1,24619480,"A839254E1496A81B767910C3FE4A2CF9F9BEB2F2669FCA1A2FBF4F8C788739EC","6E821738F0A541CC420838131B9E0399677EBD5891648C6E594C6C2DF1EA9355","CANONICAL","2026-03-09 11:00:17.000000+00" +"0a699864-84d9-4937-a270-0d4cd4656c19",1,24619481,"DA88615CBF8BCA8288F8B7DD69C82EE64EE1E30DD87769055753BDC48D806EE1","A839254E1496A81B767910C3FE4A2CF9F9BEB2F2669FCA1A2FBF4F8C788739EC","CANONICAL","2026-03-09 11:00:29.000000+00" +"c3ab7170-e3f1-4a09-9d53-d03d021aeebf",1,24619482,"6C97FB97B991DBEB64CB484186A5EB30572218C79F6246E1DAD357C88B5B8CCA","DA88615CBF8BCA8288F8B7DD69C82EE64EE1E30DD87769055753BDC48D806EE1","CANONICAL","2026-03-09 11:00:41.000000+00" +"8d3cfe1d-7c6e-4c48-9742-511921d4fa5d",1,24619483,"C9EA483C9C90CB24D519A2A412BE0E94DF93B9C191DBF2E7358FAF785E240617","6C97FB97B991DBEB64CB484186A5EB30572218C79F6246E1DAD357C88B5B8CCA","CANONICAL","2026-03-09 11:00:53.000000+00" +"246f7fef-98ac-4c58-82a8-af92edcd5fef",1,24619484,"17EF5EED341697E3502D1664A65F2C85243CB6A21FAC48B037C1E6A2B2DB0E52","C9EA483C9C90CB24D519A2A412BE0E94DF93B9C191DBF2E7358FAF785E240617","CANONICAL","2026-03-09 11:01:05.000000+00" +"1ac5da54-c695-47dc-a0b3-bbc5ddf0d888",1,24619485,"C16115EA4AEAC176910917C57A99FC55BEA6BE988FE01B3659132FE5CC289A66","17EF5EED341697E3502D1664A65F2C85243CB6A21FAC48B037C1E6A2B2DB0E52","CANONICAL","2026-03-09 11:01:17.000000+00" +"71caaf1d-7df8-45a2-83ef-a3ee36533a0e",1,24619486,"C9344CCC0C72C6234343E722FB9DB441EDA699A4647F56FDC7A84DB88E4C50AD","C16115EA4AEAC176910917C57A99FC55BEA6BE988FE01B3659132FE5CC289A66","CANONICAL","2026-03-09 11:01:29.000000+00" +"fffb567f-6e39-4b3d-bbec-c73166badb4b",1,24619487,"29F1D58686D80DB2FDB0F46C074401BB55C800BE5F59B4E4B1B2CAD9CC70361C","C9344CCC0C72C6234343E722FB9DB441EDA699A4647F56FDC7A84DB88E4C50AD","CANONICAL","2026-03-09 11:01:41.000000+00" +"ba1c7b48-13c3-4f69-8724-c09233b2320e",1,24619488,"9E1890462A562DE668C9E5C27368B3D21CF03DEB4C1F62E2D1F72CE5090A08E0","29F1D58686D80DB2FDB0F46C074401BB55C800BE5F59B4E4B1B2CAD9CC70361C","CANONICAL","2026-03-09 11:01:53.000000+00" +"e251f9a1-bc5d-4e39-b6e2-f00e69262137",1,24619489,"A71080AC72D15520DD80C5AEAD35D6F786DF1EF61E4193D90790191573C90858","9E1890462A562DE668C9E5C27368B3D21CF03DEB4C1F62E2D1F72CE5090A08E0","CANONICAL","2026-03-09 11:02:05.000000+00" +"9aae03e2-ed2a-4e5d-98a9-98f470e32023",1,24619490,"85D1E02268BADD0BD96C6DB03A41F1F2B3BE7A4A71BA5E55C4B38C93374082DD","A71080AC72D15520DD80C5AEAD35D6F786DF1EF61E4193D90790191573C90858","CANONICAL","2026-03-09 11:02:17.000000+00" +"8d66e179-fa1d-41f3-8b7b-1d24238bc8ea",1,24619491,"4318A2FF8C6500657F673C13C3E2309B66E78307D3CC791150CE0F9D70EC0D11","85D1E02268BADD0BD96C6DB03A41F1F2B3BE7A4A71BA5E55C4B38C93374082DD","CANONICAL","2026-03-09 11:02:29.000000+00" +"489bd8b4-d7d7-4b8b-98f4-7157499f247c",1,24619492,"92191D9D6F9505CA862BD4E01EF395AE7324E010F445947245277911587881C2","4318A2FF8C6500657F673C13C3E2309B66E78307D3CC791150CE0F9D70EC0D11","CANONICAL","2026-03-09 11:02:41.000000+00" +"4ddfdf40-c9d0-4585-b168-170fc2a90ee1",1,24619493,"14374924E534B2B9E5B1AF0A01F8498D5496E5FE52A893CF0538388C04BB88FA","92191D9D6F9505CA862BD4E01EF395AE7324E010F445947245277911587881C2","CANONICAL","2026-03-09 11:02:53.000000+00" +"33f6e646-576e-4866-a1ba-705f99f06882",1,24619494,"04BFBDB0623F76AAA1E9E4269E358E4B266D764ADF2AA8270A617C204F960931","14374924E534B2B9E5B1AF0A01F8498D5496E5FE52A893CF0538388C04BB88FA","CANONICAL","2026-03-09 11:03:05.000000+00" +"d84545a8-5dae-4eb0-b964-d7ebe14a3879",1,24619495,"1927AE3E3F241241350827AE033FEC7824C880122F97149B1E92416A67BFC605","04BFBDB0623F76AAA1E9E4269E358E4B266D764ADF2AA8270A617C204F960931","CANONICAL","2026-03-09 11:03:17.000000+00" +"d5545996-c865-411d-8ebe-e590fcc1324d",1,24619496,"A79189227A7468695405B9181E0AB4B37558CF66912984648B30A240C8C8CD2D","1927AE3E3F241241350827AE033FEC7824C880122F97149B1E92416A67BFC605","CANONICAL","2026-03-09 11:03:29.000000+00" +"5aeaed65-f97a-4715-b1c0-d388d9d61838",1,24619497,"9248280982757572826FB2B1018ED79441B9F7515ED9FAF960435FD54FCD5A54","A79189227A7468695405B9181E0AB4B37558CF66912984648B30A240C8C8CD2D","CANONICAL","2026-03-09 11:03:41.000000+00" +"92fbfc86-5a83-443f-a8da-61600e3cdbf3",1,24619498,"04C97BAB9698B0961F0108A558A8075A68EC6BD6278094AFF622F28FD054B1C7","9248280982757572826FB2B1018ED79441B9F7515ED9FAF960435FD54FCD5A54","CANONICAL","2026-03-09 11:03:53.000000+00" +"28dd4d78-3d00-4605-a02f-0dc6875c3dbb",1,24619499,"AE0EF7F1EBB518BE5A80E82F3474AAC00CF42F7D47F745E0114868907C0022E7","04C97BAB9698B0961F0108A558A8075A68EC6BD6278094AFF622F28FD054B1C7","CANONICAL","2026-03-09 11:04:05.000000+00" +"d249817a-9dbc-493d-9094-4453fb307cb4",1,24619500,"AFBA51042EA340873E029806064C47CBE334F08D4E18520B1CD4CC33FF6154FF","AE0EF7F1EBB518BE5A80E82F3474AAC00CF42F7D47F745E0114868907C0022E7","CANONICAL","2026-03-09 11:04:17.000000+00" +"794af3d3-e018-428f-92d8-bbae8f75ea80",1,24619501,"C595BF8FBE38943449957F6FDBEEF2FDC190FDA648DE4FFC8276A97A33398E91","AFBA51042EA340873E029806064C47CBE334F08D4E18520B1CD4CC33FF6154FF","CANONICAL","2026-03-09 11:04:29.000000+00" +"7eb45f91-717e-48c1-9f1d-70370199f253",1,24619502,"5ACFF3205640E7B83F16F8E46FC5021B002D13F79A480D005FE6366E27855556","C595BF8FBE38943449957F6FDBEEF2FDC190FDA648DE4FFC8276A97A33398E91","CANONICAL","2026-03-09 11:04:41.000000+00" +"d768674c-30cb-4880-90a2-a04544978f59",1,24619503,"78A1FE34E7235A6ED5F620C94A2F008396610F2DF12E82D1371D8B72FAB835E7","5ACFF3205640E7B83F16F8E46FC5021B002D13F79A480D005FE6366E27855556","CANONICAL","2026-03-09 11:04:53.000000+00" +"86164d88-4826-429a-9c5b-d168b703e21d",1,24619504,"428ECEC558E23E24CDB84BD0922BADE14DF786B88C9B47B4C9C2804E5ACAD17C","78A1FE34E7235A6ED5F620C94A2F008396610F2DF12E82D1371D8B72FAB835E7","CANONICAL","2026-03-09 11:05:05.000000+00" +"00dacf21-4732-4722-a031-55122755c7d2",1,24619505,"927656C6E99C3BF34ED0864730A99DCA9C6D2F9D0E6F289AA79645C1D19C21AB","428ECEC558E23E24CDB84BD0922BADE14DF786B88C9B47B4C9C2804E5ACAD17C","CANONICAL","2026-03-09 11:05:17.000000+00" +"39028ec6-2220-469f-bdce-8dff5b7914f9",1,24619506,"4C4189AB485DF7C552D6A19573301DA421CF25A42ECF8D2297CEFAA790B0FD59","927656C6E99C3BF34ED0864730A99DCA9C6D2F9D0E6F289AA79645C1D19C21AB","CANONICAL","2026-03-09 11:05:29.000000+00" +"9a065276-64b6-4146-b38c-da91762b6aff",1,24619507,"EE972D1F06E4EA8967BAF7B1E3D4B5CD08BF34D3EC247E7DC6B31D454BD0FCBA","4C4189AB485DF7C552D6A19573301DA421CF25A42ECF8D2297CEFAA790B0FD59","CANONICAL","2026-03-09 11:05:41.000000+00" +"ffee4273-d93a-4078-bb7e-d0463f06384c",1,24619508,"863A85405D7AC5EC3ACB42ADB8124B1EF7DE0690088FF5B5960423573A963F3E","EE972D1F06E4EA8967BAF7B1E3D4B5CD08BF34D3EC247E7DC6B31D454BD0FCBA","CANONICAL","2026-03-09 11:05:53.000000+00" +"901a6112-a736-42e9-97cc-7234a96e6d91",1,24619509,"549A643EC0B10F8797317CDF5850A9036D84283C16C6B9D83C92B2BB1204872A","863A85405D7AC5EC3ACB42ADB8124B1EF7DE0690088FF5B5960423573A963F3E","CANONICAL","2026-03-09 11:06:05.000000+00" +"2c8957d4-5779-4ea3-9f39-652699ef0b93",1,24619510,"9253F540487E5EE1BC1F670BA045956DED86E2EA24214819D95275F3F8BB5090","549A643EC0B10F8797317CDF5850A9036D84283C16C6B9D83C92B2BB1204872A","CANONICAL","2026-03-09 11:06:17.000000+00" diff --git a/listener/data/canonical-chain-no-matching-blocks.csv b/listener/data/canonical-chain-no-matching-blocks.csv new file mode 100644 index 0000000000..58517624fc --- /dev/null +++ b/listener/data/canonical-chain-no-matching-blocks.csv @@ -0,0 +1,30 @@ +"2abba6a3-bad4-48bd-a379-10369ae752ae",1,24619316,"5A9A8B1D14B69730720417B0C672C0182A65787F5E239CAA5576B4C669DE25D1","5B465871CB3C50DBF7A6006851F4EA2B854F5B9D48338E2F6A0DCE4B0DFEAD5D","CANONICAL","2026-03-09 10:27:29.000000+00" +"6abc54cb-2716-4b6e-afcb-2af227ba318e",1,24619317,"8F0C1C45351AED996F60BFB3E66B86518EA9098A7E09CF650151B1EB0FFDDBF8","5A9A8B1D14B69730720417B0C672C0182A65787F5E239CAA5576B4C669DE25D1","CANONICAL","2026-03-09 10:27:41.000000+00" +"6660bab2-feb0-4c9e-a596-36b3860b1b19",1,24619318,"2CCD69D886A84B096F59A1CC8CA08442C1A3ED8FB57B07D620C48F79882B2B5F","8F0C1C45351AED996F60BFB3E66B86518EA9098A7E09CF650151B1EB0FFDDBF8","CANONICAL","2026-03-09 10:27:53.000000+00" +"f51a6a4c-ff4b-41ff-9b79-3dee7389c238",1,24619319,"9055E86B7D82EC344E09F58BBFE97A2B2A3E3BC1880A991483AD9AC230097D6A","2CCD69D886A84B096F59A1CC8CA08442C1A3ED8FB57B07D620C48F79882B2B5F","CANONICAL","2026-03-09 10:28:05.000000+00" +"4c2a729d-bc90-4b20-8d8c-58c933bd6b29",1,24619320,"FAFF681785D6F47BCCED3E61A6E245868671A81D9BF5E486DAEE1F86366FDEEA","9055E86B7D82EC344E09F58BBFE97A2B2A3E3BC1880A991483AD9AC230097D6A","CANONICAL","2026-03-09 10:28:17.000000+00" +"5c218cd2-6825-464f-a321-5f3765c077b3",1,24619321,"EFAF54FBE3B42C7EF45D0BF0E466E13D4417F7DA162FD11AA3301C4700BA7A2F","FAFF681785D6F47BCCED3E61A6E245868671A81D9BF5E486DAEE1F86366FDEEA","CANONICAL","2026-03-09 10:28:29.000000+00" +"97106d5b-c095-4b09-91a2-e65d7a03f1e8",1,24619322,"5502CD8D8A22475794A0E714A64B8A4535D630418BD9775AD8BB18F9CA109E1F","EFAF54FBE3B42C7EF45D0BF0E466E13D4417F7DA162FD11AA3301C4700BA7A2F","CANONICAL","2026-03-09 10:28:41.000000+00" +"d29eb8ad-8012-488e-b9ad-58faf51df575",1,24619323,"182C88EB4431F45062D1B5F2FAEDA807B58844B7E2727FB4B60086BF5E5A5BDA","5502CD8D8A22475794A0E714A64B8A4535D630418BD9775AD8BB18F9CA109E1F","CANONICAL","2026-03-09 10:28:53.000000+00" +"399c2bc6-788e-43c4-baa8-6f48acf72868",1,24619324,"A17A63484CE8D88F1CCAD423A9B90750438875A51DC8720E72CFB42BA818D985","182C88EB4431F45062D1B5F2FAEDA807B58844B7E2727FB4B60086BF5E5A5BDA","CANONICAL","2026-03-09 10:29:05.000000+00" +"ac75eb20-f37f-4808-9845-ca35424bd6bd",1,24619325,"CBC0A741C363B24AA24AC9EE6990CAC68B99A8FB1CAF24FBDC8B35A662A723D6","A17A63484CE8D88F1CCAD423A9B90750438875A51DC8720E72CFB42BA818D985","CANONICAL","2026-03-09 10:29:17.000000+00" +"7f5aca40-160a-423c-b691-ab655136695e",1,24619326,"99167E8BE898A784B4A57E7980F8B06FF36444BE9639B835425300192B937BFE","CBC0A741C363B24AA24AC9EE6990CAC68B99A8FB1CAF24FBDC8B35A662A723D6","CANONICAL","2026-03-09 10:29:29.000000+00" +"ce15e9de-719a-489b-85c8-c203e24a0c55",1,24619327,"368CE0108BCE5A27DBB06D378825375426D06065268A5469EFE55011AEFB10BF","99167E8BE898A784B4A57E7980F8B06FF36444BE9639B835425300192B937BFE","CANONICAL","2026-03-09 10:29:41.000000+00" +"c7aa9967-a515-4c51-8b69-a453232e8689",1,24619328,"6B8320AEACB44956DEE451A8927F9DC80026CA034BE570098B44D6FEF39CA8BC","368CE0108BCE5A27DBB06D378825375426D06065268A5469EFE55011AEFB10BF","CANONICAL","2026-03-09 10:29:53.000000+00" +"80af8f04-1ce3-48e5-84a2-f10c18e0d3cf",1,24619329,"B6B2EC4873840E787FB6190A7FFE707F62292AF32425FEF4E0E445E6E3DA0470","6B8320AEACB44956DEE451A8927F9DC80026CA034BE570098B44D6FEF39CA8BC","CANONICAL","2026-03-09 10:30:05.000000+00" +"7e27c254-e8bf-46c4-84c6-2605928686b5",1,24619330,"C35B60A98E0CFCA2E557940B52172A1EED024B62FEB839D3D98AD706A43BAA67","B6B2EC4873840E787FB6190A7FFE707F62292AF32425FEF4E0E445E6E3DA0470","CANONICAL","2026-03-09 10:30:17.000000+00" +"e83a7075-d1e6-4b19-8852-d5240761a34e",1,24619331,"4C19A7E8809C0EFB95D636C75D44F95C04873F3F7D643B59EDC5CD85033479D8","C35B60A98E0CFCA2E557940B52172A1EED024B62FEB839D3D98AD706A43BAA67","CANONICAL","2026-03-09 10:30:29.000000+00" +"604da614-de26-4625-bb0a-a73aa5ef56c8",1,24619332,"0A6BADFE47F1FE1E867F5C59194E282F3B7CAE943A38FFB497531798BB3A221B","4C19A7E8809C0EFB95D636C75D44F95C04873F3F7D643B59EDC5CD85033479D8","CANONICAL","2026-03-09 10:30:41.000000+00" +"b8409228-256a-41c8-90f5-34d099344ff0",1,24619333,"2D6BB334DF38AD4703D15BA0C14A81FEA2BFFA40109499B5675ABC695E14C504","0A6BADFE47F1FE1E867F5C59194E282F3B7CAE943A38FFB497531798BB3A221B","CANONICAL","2026-03-09 10:30:53.000000+00" +"8d149358-c8c1-4755-94b1-ec597d6c4efe",1,24619334,"3A52B52C9ED802F45A3B1A5119EFF6D4896261C8E397CFC0AE0F14CFAB4E1827","2D6BB334DF38AD4703D15BA0C14A81FEA2BFFA40109499B5675ABC695E14C504","CANONICAL","2026-03-09 10:31:05.000000+00" +"cfc6d22f-3db3-457c-811d-c371edb049eb",1,24619335,"496E2411C2DD60E5CB63CDAE1F386B25781F3FBCC61DA8A02D7958E9569679F8","3A52B52C9ED802F45A3B1A5119EFF6D4896261C8E397CFC0AE0F14CFAB4E1827","CANONICAL","2026-03-09 10:31:17.000000+00" +"475be85d-0210-4431-a6b9-cce8b2e378d9",1,24619336,"42D417F32B6DE90152A62FADA60CB4E313BC45696975ADCC86B3C0325A05A5E2","496E2411C2DD60E5CB63CDAE1F386B25781F3FBCC61DA8A02D7958E9569679F8","CANONICAL","2026-03-09 10:31:29.000000+00" +"2ba60e57-2eff-4002-b064-a6f27b7129cf",1,24619337,"74528BF767FB6BCA4652244B0B0CF3B6A90185C2946632686D859EF2AE55B502","42D417F32B6DE90152A62FADA60CB4E313BC45696975ADCC86B3C0325A05A5E2","CANONICAL","2026-03-09 10:31:41.000000+00" +"7009976b-4565-48db-83a4-d9e58256541d",1,24619338,"6FA69B26BF14F1FB436C07875812B3226C6503696B46F889C31AB5810C6EB990","74528BF767FB6BCA4652244B0B0CF3B6A90185C2946632686D859EF2AE55B502","CANONICAL","2026-03-09 10:31:53.000000+00" +"d85562f7-3546-4ffc-8872-dda828a1fb9d",1,24619339,"131554B3CBA7730297F7A14EF17DB6D52E5EF5F23FFC077AFA54F78931808ACE","6FA69B26BF14F1FB436C07875812B3226C6503696B46F889C31AB5810C6EB990","CANONICAL","2026-03-09 10:32:05.000000+00" +"8226eed8-ece7-4a73-8830-daa5a3618e59",1,24619340,"48487E74DD41368A326DD5284325DEE05DA21B5A21C3642C701BDBD4346F13E4","131554B3CBA7730297F7A14EF17DB6D52E5EF5F23FFC077AFA54F78931808ACE","CANONICAL","2026-03-09 10:32:17.000000+00" +"6d7c94b5-790a-41d9-8478-d71da2b0eca0",1,24619341,"34D4287E29653756F9090AED083A3675E4952124E98F20397A2758C2A40C2E40","48487E74DD41368A326DD5284325DEE05DA21B5A21C3642C701BDBD4346F13E4","CANONICAL","2026-03-09 10:32:29.000000+00" +"58d0ee2e-947b-4e0c-834d-f04a90802e46",1,24619342,"BE2FF5375C0204C2295253C244B9C4098872A843E719289410C621CC3B278432","34D4287E29653756F9090AED083A3675E4952124E98F20397A2758C2A40C2E40","CANONICAL","2026-03-09 10:32:41.000000+00" +"0ca351d2-d7c8-40ff-a7d6-bef906293dbc",1,24619343,"632625DE057CCE9ABFCD427229CE53F61814144F298543109C9E632B7A423BBC","BE2FF5375C0204C2295253C244B9C4098872A843E719289410C621CC3B278432","CANONICAL","2026-03-09 10:32:53.000000+00" +"46212442-0eb7-4560-b2a9-48898cce6214",1,24619344,"2A7A5CF57C92ACB089A4FAD6A7D8FAB90019D4EF7D01171486AADC066776CE2D","632625DE057CCE9ABFCD427229CE53F61814144F298543109C9E632B7A423BBC","CANONICAL","2026-03-09 10:33:05.000000+00" +"2f252f7f-467b-4979-9f80-aee55e6b44cb",1,24619345,"6F9F1FA7469CFDBB6D15977E321D664F6B53EB51960EEEB60BAA03F975BB8498","2A7A5CF57C92ACB089A4FAD6A7D8FAB90019D4EF7D01171486AADC066776CE2D","CANONICAL","2026-03-09 10:33:17.000000+00" \ No newline at end of file diff --git a/listener/data/reorg-testing-data-132.csv b/listener/data/reorg-testing-data-132.csv new file mode 100644 index 0000000000..4d480372d2 --- /dev/null +++ b/listener/data/reorg-testing-data-132.csv @@ -0,0 +1,131 @@ +"9afb4555-4fcf-4b74-aafc-7bce22ec29d6",1,24619310,"5FBB452E14704821AA8A9E07439E5F7B08EC2E39238366E2CA1CC4337F980BAE","A237593CDBA889B58915AA6A61F3EB047DAB2B22BA8DCC0BA9FBF14CAC4AF731","CANONICAL","2026-03-09 10:26:14.391532+00" +"7e893be5-c46c-47ba-9463-c8407a57b920",1,24619311,"0409356A94F49FDBA1F24334E83447A0AD77FEC516ECFDAFED71282A3A1486A7","5FBB452E14704821AA8A9E07439E5F7B08EC2E39238366E2CA1CC4337F980BAE","CANONICAL","2026-03-09 10:26:25.988417+00" +"31874c8c-d96f-4a90-8c21-180e62a889e0",1,24619315,"5B465871CB3C50DBF7A6006851F4EA2B854F5B9D48338E2F6A0DCE4B0DFEAD5D","2DA42BBF62E6AE3602E87AB8EC28F46968B745476A4B8BDBA5AB35980AEB370D","CANONICAL","2026-03-09 10:27:17.405497+00" +"6d626275-a665-438f-a625-408619699199",1,24619312,"CDFE4E3B97099C9D098D3B11D8EDFDC0477E2C53C7A9DF943BE9341BFC869E3F","0409356A94F49FDBA1F24334E83447A0AD77FEC516ECFDAFED71282A3A1486A7","CANONICAL","2026-03-09 10:26:38.950499+00" +"957efb01-426f-455e-afea-c67dc0669df8",1,24619313,"24E3A70842FEA7E5E8A40EDD9BCC9DD2269D25C3939FF075C160B99855017D19","CDFE4E3B97099C9D098D3B11D8EDFDC0477E2C53C7A9DF943BE9341BFC869E3F","CANONICAL","2026-03-09 10:26:49.410316+00" +"8d9ae7fc-e0b3-4ffb-b413-637bf82bc021",1,24619314,"2DA42BBF62E6AE3602E87AB8EC28F46968B745476A4B8BDBA5AB35980AEB370D","24E3A70842FEA7E5E8A40EDD9BCC9DD2269D25C3939FF075C160B99855017D19","CANONICAL","2026-03-09 10:27:06.503866+00" +"9d624758-f71b-4647-b2fd-60f7ae1e93cd",1,24619316,"8BE72344BBDFE6669B995B0A07F45418E8099628D92CD670665C0791A24D44D1","6F779067708D8AAFA4C18E39D82AA03119532D6F4A2E23AB6E47BC5BDECC9DA0","CANONICAL","2026-03-09 10:28:20.000000+00" +"c7e310e9-c26d-49c8-840c-c4832108420a",1,24619317,"6525DAC1A94AE0703C61FADF30AAA7812CFBB73F4D67270BF389A2E264C23EE1","8BE72344BBDFE6669B995B0A07F45418E8099628D92CD670665C0791A24D44D1","CANONICAL","2026-03-09 10:28:32.000000+00" +"afe35ab5-d028-406b-884b-f4b169c8fb4c",1,24619318,"2D95CF00F37EA028A4902BECC1BE0B09E7BD57CC35748341EC6980EB96D1D3D3","6525DAC1A94AE0703C61FADF30AAA7812CFBB73F4D67270BF389A2E264C23EE1","CANONICAL","2026-03-09 10:28:44.000000+00" +"22ce1143-45d7-40da-b907-a8f8a42ca9ae",1,24619319,"F29FA52C499EFB388394D42DF56AA70F7F58A590F79D3B45B55DF7C8A2DB0C97","2D95CF00F37EA028A4902BECC1BE0B09E7BD57CC35748341EC6980EB96D1D3D3","CANONICAL","2026-03-09 10:28:56.000000+00" +"f30b33ee-5fa9-4f4d-9ed6-c780984c8029",1,24619320,"1E5ED0CFD8160D3CFF37D9871170F7AD7FFB65E8587FC73649A4BA4003DBDC0E","F29FA52C499EFB388394D42DF56AA70F7F58A590F79D3B45B55DF7C8A2DB0C97","CANONICAL","2026-03-09 10:29:08.000000+00" +"b974a131-1f40-4f42-a352-191a6856f5a7",1,24619321,"9CE6C32DA04CA83CB3E82DC63833D27080067C5367B4E3B2AE25AB85A824632C","1E5ED0CFD8160D3CFF37D9871170F7AD7FFB65E8587FC73649A4BA4003DBDC0E","CANONICAL","2026-03-09 10:29:20.000000+00" +"96f9b509-8b30-494b-947d-fb4f9bf41cac",1,24619322,"5B984F02979A2457CBC98E8B4F13C252AA53568C59FA2BAE980254ECC00D1AF1","9CE6C32DA04CA83CB3E82DC63833D27080067C5367B4E3B2AE25AB85A824632C","CANONICAL","2026-03-09 10:29:32.000000+00" +"ad5c5e56-d89d-414c-93b4-84f9032ae0c7",1,24619323,"BB3347465E47BFEF22DD7D550D9DD32F249B2D6C20810CF520EF29E2D0D377E2","5B984F02979A2457CBC98E8B4F13C252AA53568C59FA2BAE980254ECC00D1AF1","CANONICAL","2026-03-09 10:29:44.000000+00" +"0e75e9fe-f84f-48d3-9c7c-07d7bd2e90b6",1,24619324,"3DE38180AAF852930D67F189BF3BD13CE610B34F9ADCBF983DF087F17A41A2A5","BB3347465E47BFEF22DD7D550D9DD32F249B2D6C20810CF520EF29E2D0D377E2","CANONICAL","2026-03-09 10:29:56.000000+00" +"1ab25125-31fa-4162-8769-00d389ba7a7f",1,24619325,"66CCBCC0A43A6413CFC119803B94A855CB573D8BB310004DEFEB7D0C2FB5F5AD","3DE38180AAF852930D67F189BF3BD13CE610B34F9ADCBF983DF087F17A41A2A5","CANONICAL","2026-03-09 10:30:08.000000+00" +"293c564f-5eeb-49dd-8bea-f0346349a242",1,24619326,"929F7618F945299B3E10C0F22B7957D0A7F683A8A2B898D099F5E14AB2C4F9DD","66CCBCC0A43A6413CFC119803B94A855CB573D8BB310004DEFEB7D0C2FB5F5AD","CANONICAL","2026-03-09 10:30:20.000000+00" +"733cb3a9-72e6-4130-bb35-aa8ab8afa639",1,24619327,"4B3CB9F8FCFC55AD2D3B7FB7F10F82D6CF6CB8E68FCB85E799E291B1125AAD02","929F7618F945299B3E10C0F22B7957D0A7F683A8A2B898D099F5E14AB2C4F9DD","CANONICAL","2026-03-09 10:30:32.000000+00" +"d0caefaa-8ba6-423c-8d14-199f5378db0b",1,24619328,"3EE2D3FB9208662AF1C988876075C46E43BF3CC85514ED776AB8AE4CE1611612","4B3CB9F8FCFC55AD2D3B7FB7F10F82D6CF6CB8E68FCB85E799E291B1125AAD02","CANONICAL","2026-03-09 10:30:44.000000+00" +"c604b2d7-f7d6-48c2-b4e2-cfa2c870f302",1,24619329,"A8D3DDD25906EFE8233DF194C9429AFAF934843082E01AAC0F2E1651190BBA6B","3EE2D3FB9208662AF1C988876075C46E43BF3CC85514ED776AB8AE4CE1611612","CANONICAL","2026-03-09 10:30:56.000000+00" +"36d972e7-7423-4f21-aaa8-de481e035cb8",1,24619330,"138073F3FB274CBEDB9CD120D4DA698A0DC4930735201F1AD2A8F7F77A1B364B","A8D3DDD25906EFE8233DF194C9429AFAF934843082E01AAC0F2E1651190BBA6B","CANONICAL","2026-03-09 10:31:08.000000+00" +"760c8274-d559-44d8-b063-3f60350a6b8b",1,24619331,"6DF7178EC651F528EBB0F9CC88ABC055032CCCA642539E39B62EC59080D9F933","138073F3FB274CBEDB9CD120D4DA698A0DC4930735201F1AD2A8F7F77A1B364B","CANONICAL","2026-03-09 10:31:20.000000+00" +"6a575a44-6702-47f9-9509-8b681fd12e1d",1,24619332,"C66E82B3325CF536F47F6B711DB5537629C004F9EED2EDC00A58F9D47D5D2D30","6DF7178EC651F528EBB0F9CC88ABC055032CCCA642539E39B62EC59080D9F933","CANONICAL","2026-03-09 10:31:32.000000+00" +"71dc79f0-0a87-4177-93eb-81a9e4a585e1",1,24619333,"3D56DBFC4BCE0CC2D49AEE7B531865DFFDD1916D6FAEB9787B38085F9EDEB39F","C66E82B3325CF536F47F6B711DB5537629C004F9EED2EDC00A58F9D47D5D2D30","CANONICAL","2026-03-09 10:31:44.000000+00" +"223b0b93-2278-414e-ac6e-878052e94c85",1,24619334,"E457EE89AB3D2E31B5AEA59FC3466188C4568B05293BB92296225A539858C2FA","3D56DBFC4BCE0CC2D49AEE7B531865DFFDD1916D6FAEB9787B38085F9EDEB39F","CANONICAL","2026-03-09 10:31:56.000000+00" +"56205a9f-bd02-4430-afe2-64651dd1fb79",1,24619335,"159BA172F42D660875E8DE84C18C027249611C70380B4D1EB466AACE403B783A","E457EE89AB3D2E31B5AEA59FC3466188C4568B05293BB92296225A539858C2FA","CANONICAL","2026-03-09 10:32:08.000000+00" +"24168ea8-ef04-48ec-83ca-60ee4408e206",1,24619336,"49BFA5F6EF0E4BB1D3A0D875461B3123F8CA3592B16742141602C1D63FBA7D77","159BA172F42D660875E8DE84C18C027249611C70380B4D1EB466AACE403B783A","CANONICAL","2026-03-09 10:32:20.000000+00" +"e87d8912-7213-404b-a68f-375821c1d1f2",1,24619337,"EDEA8C507424653C545EAFF7CDB1749668E91A47F1ACBF8C946FCEC01BA48293","49BFA5F6EF0E4BB1D3A0D875461B3123F8CA3592B16742141602C1D63FBA7D77","CANONICAL","2026-03-09 10:32:32.000000+00" +"3feaa057-bc9d-4548-ac8a-cc9f72d8bbf6",1,24619338,"49874AC1FC41B7DAE7FBDEA6ED03F6AE40012B72B628CB66A7377EF27A4E251D","EDEA8C507424653C545EAFF7CDB1749668E91A47F1ACBF8C946FCEC01BA48293","CANONICAL","2026-03-09 10:32:44.000000+00" +"b81f7925-4c86-422a-9e7b-109075b984de",1,24619339,"78F4600F025EF87B843CC349E72AC24D9BB9A2E6BFA0047F17D173D7F6CAE150","49874AC1FC41B7DAE7FBDEA6ED03F6AE40012B72B628CB66A7377EF27A4E251D","CANONICAL","2026-03-09 10:32:56.000000+00" +"02b7eeb2-cb0f-408f-934d-fc74b51b4c42",1,24619340,"0A6A6706819DF5D3479787F476AAD222712EC411FE9E5AEA8F2D3C1D437335EF","78F4600F025EF87B843CC349E72AC24D9BB9A2E6BFA0047F17D173D7F6CAE150","CANONICAL","2026-03-09 10:33:08.000000+00" +"29b6f606-4ec0-4e7f-8fe8-69df9a6dc0a0",1,24619341,"0EA220D4A243613DE07827AB7765EC7C3CFF4EB9D11642E7749276A31280F182","0A6A6706819DF5D3479787F476AAD222712EC411FE9E5AEA8F2D3C1D437335EF","CANONICAL","2026-03-09 10:33:20.000000+00" +"6630ed31-e0eb-42d3-a742-fef0386aa258",1,24619342,"A259329A782D380CC3A04AAD87093D6826869FCEBEB08605F9A1D32FC784F2F5","0EA220D4A243613DE07827AB7765EC7C3CFF4EB9D11642E7749276A31280F182","CANONICAL","2026-03-09 10:33:32.000000+00" +"57966c2a-f2a2-42ab-bf84-0d3bd422bc9f",1,24619343,"177066839776137D85F6C7669509E8DB4239F78E6F32582095A819530D4A01E3","A259329A782D380CC3A04AAD87093D6826869FCEBEB08605F9A1D32FC784F2F5","CANONICAL","2026-03-09 10:33:44.000000+00" +"c6cbefb6-bb5a-4142-9b48-65cd71608692",1,24619344,"26599DD98A4C2E0B569B8983F4C0BB34CC23DB3FAAB64386CD8CFC29429B87C8","177066839776137D85F6C7669509E8DB4239F78E6F32582095A819530D4A01E3","CANONICAL","2026-03-09 10:33:56.000000+00" +"c08ec4d9-91f5-431f-bd42-c124eaef1a70",1,24619345,"578DF90C9B526F7A79B7DC2C8386E5EA44ED230690B9C271E4B8B997D3C62717","26599DD98A4C2E0B569B8983F4C0BB34CC23DB3FAAB64386CD8CFC29429B87C8","CANONICAL","2026-03-09 10:34:08.000000+00" +"c831af74-e55d-4369-8edf-cedd1389cd1d",1,24619346,"14077FDC1A791726FB4994F6E3CC23B501C005751AB8AD254EC2B7CD0965A2F5","578DF90C9B526F7A79B7DC2C8386E5EA44ED230690B9C271E4B8B997D3C62717","CANONICAL","2026-03-09 10:34:20.000000+00" +"5189bcda-a9a4-41f4-9b40-5078cc1c7670",1,24619347,"C52126B339CCB59196EFE2984B3ADE193FC9F3CD6566B5598635AC0351CE9B19","14077FDC1A791726FB4994F6E3CC23B501C005751AB8AD254EC2B7CD0965A2F5","CANONICAL","2026-03-09 10:34:32.000000+00" +"606708aa-3f1a-49d4-a0bb-1f619880cd51",1,24619348,"A18E72979006EEF6E33BFD70A2F3F05EDEEF2EFA596DB4C3E44FBFB6D0A5402A","C52126B339CCB59196EFE2984B3ADE193FC9F3CD6566B5598635AC0351CE9B19","CANONICAL","2026-03-09 10:34:44.000000+00" +"d57a82c7-5deb-4d13-9446-15f035b14656",1,24619349,"238C5C5EFA79936B6AD693E8C0B147EB728E0B8587CEDF086B8AFBC5F1238A35","A18E72979006EEF6E33BFD70A2F3F05EDEEF2EFA596DB4C3E44FBFB6D0A5402A","CANONICAL","2026-03-09 10:34:56.000000+00" +"8715010c-04da-4bfe-bb08-e0cd34f2aba5",1,24619350,"625921F0D0FF2F314F5787083DCFB510C2E66557371E6ACF232456C7857F9C0C","238C5C5EFA79936B6AD693E8C0B147EB728E0B8587CEDF086B8AFBC5F1238A35","CANONICAL","2026-03-09 10:35:08.000000+00" +"3a8006fb-3eac-47a8-8c27-1d270dd4e80d",1,24619351,"7D746867478DCF482ED3B9849851889032B92ED4FAD7802A77066DB69795049C","625921F0D0FF2F314F5787083DCFB510C2E66557371E6ACF232456C7857F9C0C","CANONICAL","2026-03-09 10:35:20.000000+00" +"2dbdd28a-02ab-4bba-9b4a-02010d2fe201",1,24619352,"95AD323C221CA9C38202DDD221174307D567B3802FCF5E81F767D14DD2A5B5D6","7D746867478DCF482ED3B9849851889032B92ED4FAD7802A77066DB69795049C","CANONICAL","2026-03-09 10:35:32.000000+00" +"205cf145-8e0a-4432-866c-6b97c032f920",1,24619353,"9118451EEFD64E1261A61CE1321F08FB7A820FD19CA1B1D20A6111CFB5CDAE62","95AD323C221CA9C38202DDD221174307D567B3802FCF5E81F767D14DD2A5B5D6","CANONICAL","2026-03-09 10:35:44.000000+00" +"f5a21077-63d7-4de8-9e7f-80b0c56e2a73",1,24619354,"E449E1A259EE70E369092FC9C4487864A8B73D6F6F7F511A31DE2238B90897F5","9118451EEFD64E1261A61CE1321F08FB7A820FD19CA1B1D20A6111CFB5CDAE62","CANONICAL","2026-03-09 10:35:56.000000+00" +"9291a982-db2d-4b9d-a666-1720bf157aac",1,24619355,"348D150577570F45FA2DC6F8FFAF788CBCAA853D56F50F1CFCC36C2B7BA2C17D","E449E1A259EE70E369092FC9C4487864A8B73D6F6F7F511A31DE2238B90897F5","CANONICAL","2026-03-09 10:36:08.000000+00" +"66df9d28-21ef-4608-9f44-327258dcd786",1,24619356,"36E6461FABEA07506AD2DAE091ADD209131877DAA1D2F11DB2EBE74B5C302D73","348D150577570F45FA2DC6F8FFAF788CBCAA853D56F50F1CFCC36C2B7BA2C17D","CANONICAL","2026-03-09 10:36:20.000000+00" +"348d8d44-fea7-4b20-a377-5be71cd5a5e8",1,24619357,"4002679A865D64E18C4E30B63F9C2AF2F7A5C652FFF418D88DDFC0549C015D7D","36E6461FABEA07506AD2DAE091ADD209131877DAA1D2F11DB2EBE74B5C302D73","CANONICAL","2026-03-09 10:36:32.000000+00" +"96c4934d-ea95-4e69-84b2-743efbcdd8a1",1,24619358,"0DF97B29B70CC0F0D04ED9476B1A01BE4064A4CB53DA3B9157C6F3685EA595EC","4002679A865D64E18C4E30B63F9C2AF2F7A5C652FFF418D88DDFC0549C015D7D","CANONICAL","2026-03-09 10:36:44.000000+00" +"e9e6a606-29e9-4120-8b2d-504e8d752b6f",1,24619359,"DC012EA8A88033F7771EB7BF9538300CDC2CD00310812984198EE17DCB98170A","0DF97B29B70CC0F0D04ED9476B1A01BE4064A4CB53DA3B9157C6F3685EA595EC","CANONICAL","2026-03-09 10:36:56.000000+00" +"702727bc-bdc3-4359-a86b-128b05224866",1,24619360,"F72E5BB92C9BF4FB70F07F1E0202B15671E7844968A964AC1B87F40B4D120891","DC012EA8A88033F7771EB7BF9538300CDC2CD00310812984198EE17DCB98170A","CANONICAL","2026-03-09 10:37:08.000000+00" +"7c0d207c-fcb2-4923-a7ed-1e36d05587eb",1,24619361,"8DBEDA261B06C5186449D45F6B54BEF75E060A65C95FA60284EB09ADB1CFE4F9","F72E5BB92C9BF4FB70F07F1E0202B15671E7844968A964AC1B87F40B4D120891","CANONICAL","2026-03-09 10:37:20.000000+00" +"174068ce-2426-4c97-a3ef-597897578a05",1,24619362,"2E8D43D3AFD3AA6CC95B61C9B5998668C1561DA7128136B0FD2F1DF89DA26723","8DBEDA261B06C5186449D45F6B54BEF75E060A65C95FA60284EB09ADB1CFE4F9","CANONICAL","2026-03-09 10:37:32.000000+00" +"fbafbe54-ae24-48b2-8a7c-2c5fad28d5e3",1,24619363,"22696D6723031D4B83743231AE27CE59A69A403E0F2267296FEEB3944027D12C","2E8D43D3AFD3AA6CC95B61C9B5998668C1561DA7128136B0FD2F1DF89DA26723","CANONICAL","2026-03-09 10:37:44.000000+00" +"60a1218f-79fb-46ce-ad09-47c3e92bda45",1,24619364,"B1F6FF6F365EA14935749612BD52DF6734F0F15AEBEE06D19A327F510699E693","22696D6723031D4B83743231AE27CE59A69A403E0F2267296FEEB3944027D12C","CANONICAL","2026-03-09 10:37:56.000000+00" +"c61ea4be-935d-473d-8b88-67c7baa6a870",1,24619365,"62B0F2007235239D14B1EBEEF5E905E113AAF8D767F50E18E65D7742F0F911B3","B1F6FF6F365EA14935749612BD52DF6734F0F15AEBEE06D19A327F510699E693","CANONICAL","2026-03-09 10:38:08.000000+00" +"5c322f5d-0358-4678-877c-2fc7ff119ed6",1,24619366,"6BCE71998F976B8A11E45EBA72B1DD6D4A0C3CD06FE48671773D5F7998CE39B4","62B0F2007235239D14B1EBEEF5E905E113AAF8D767F50E18E65D7742F0F911B3","CANONICAL","2026-03-09 10:38:20.000000+00" +"8d5ff8a8-a0b5-41c3-a50d-cb59e6fce1d6",1,24619367,"435296960BFA257AE3A50B3EF79399ADA5A2100D9CFFF001E6773E4E778E2136","6BCE71998F976B8A11E45EBA72B1DD6D4A0C3CD06FE48671773D5F7998CE39B4","CANONICAL","2026-03-09 10:38:32.000000+00" +"d6a69bf6-2497-45a4-ad6a-ac5a6006cdd1",1,24619368,"A3B8FD145E0BEC1143182EB1838FEDCED09BB912B987556A4A51BAD910FE3774","435296960BFA257AE3A50B3EF79399ADA5A2100D9CFFF001E6773E4E778E2136","CANONICAL","2026-03-09 10:38:44.000000+00" +"52fc972e-d76c-41e4-af5d-7c9768ba4166",1,24619369,"D00B789270420441F36D46DD0CEA19EEBD7CBB99F7D5E37A306CB5A736409A9A","A3B8FD145E0BEC1143182EB1838FEDCED09BB912B987556A4A51BAD910FE3774","CANONICAL","2026-03-09 10:38:56.000000+00" +"7db03865-35c3-45bd-a10a-3ee5232e7da2",1,24619370,"D3A5EBD9C0462ACE1C86BBE0AF059ADF7B233975751E27A23F52D69E84DAA7CD","D00B789270420441F36D46DD0CEA19EEBD7CBB99F7D5E37A306CB5A736409A9A","CANONICAL","2026-03-09 10:39:08.000000+00" +"9466a972-4065-4dc6-822a-04aac77d7289",1,24619371,"705C4E183C836453191AFDCCDEAEB26B0E7539860E0693F5EFC0CBAB3F062B2C","D3A5EBD9C0462ACE1C86BBE0AF059ADF7B233975751E27A23F52D69E84DAA7CD","CANONICAL","2026-03-09 10:39:20.000000+00" +"8504ee96-16c9-4b47-8701-4ab5301fd963",1,24619372,"DEC8964351B4787E143BF52413CAB18B34A6B49A564BD4ADC45626573C500AB2","705C4E183C836453191AFDCCDEAEB26B0E7539860E0693F5EFC0CBAB3F062B2C","CANONICAL","2026-03-09 10:39:32.000000+00" +"da118db1-f39f-4f33-b46a-0f65745474ee",1,24619373,"51564EA8A0FA67B981DB9875A48C0C83BF5EFB4A4AEEB222F1F375ACE6A32572","DEC8964351B4787E143BF52413CAB18B34A6B49A564BD4ADC45626573C500AB2","CANONICAL","2026-03-09 10:39:44.000000+00" +"5bbc53f0-0291-4f70-826c-d268241804c1",1,24619374,"8B4EF13FAE627461B469BF5E70EBD2899FF1D0C7F3F3558272C3BE46B9554E37","51564EA8A0FA67B981DB9875A48C0C83BF5EFB4A4AEEB222F1F375ACE6A32572","CANONICAL","2026-03-09 10:39:56.000000+00" +"8e407c6f-b33d-4d5a-88e0-8bc89c5df6f4",1,24619375,"1839347ACDD1B14311D705BF436926A54F5EA1ACB55298545B3266302E5C80DA","8B4EF13FAE627461B469BF5E70EBD2899FF1D0C7F3F3558272C3BE46B9554E37","CANONICAL","2026-03-09 10:40:08.000000+00" +"1c0bff87-0ec2-468c-b50a-f5f9e29eec24",1,24619376,"955BB81214BB7E3DF428FCBA1670DB6D09EFDC6505B9AB40D594246630BC13B9","1839347ACDD1B14311D705BF436926A54F5EA1ACB55298545B3266302E5C80DA","CANONICAL","2026-03-09 10:40:20.000000+00" +"e8fc357a-d5bd-48f9-bcb7-bcfd076d34a8",1,24619377,"C047493295C71AAAC5F5BBFFB3AEDB733DEA0B84F6565C0CB897199676D06D83","955BB81214BB7E3DF428FCBA1670DB6D09EFDC6505B9AB40D594246630BC13B9","CANONICAL","2026-03-09 10:40:32.000000+00" +"6e15d2de-6000-47ec-8ba0-de8436cd027e",1,24619378,"51EAE16046E8ACDF38C5AFE9A18B023FFEB8D8284281BBB65C7E9939D8A14E66","C047493295C71AAAC5F5BBFFB3AEDB733DEA0B84F6565C0CB897199676D06D83","CANONICAL","2026-03-09 10:40:44.000000+00" +"a770974a-606d-442d-a4ed-a221def70a1a",1,24619379,"6A8B48E6C646C2D0460379E962379416A0DEDBC7462F53AC6CE5C78BDB43782F","51EAE16046E8ACDF38C5AFE9A18B023FFEB8D8284281BBB65C7E9939D8A14E66","CANONICAL","2026-03-09 10:40:56.000000+00" +"2fd073ba-f371-423e-9dc2-314c4cb52f85",1,24619380,"FE70EABCF903D7B8BC759D8204CB1D1F3308D802E2646104D3EA21237B31C861","6A8B48E6C646C2D0460379E962379416A0DEDBC7462F53AC6CE5C78BDB43782F","CANONICAL","2026-03-09 10:41:08.000000+00" +"a722d706-4c2d-4632-ab05-de6531b42a31",1,24619381,"B58EC557C1136878D6A3CF012904C89C849ED6016FB91934BEA7B151F47D12D3","FE70EABCF903D7B8BC759D8204CB1D1F3308D802E2646104D3EA21237B31C861","CANONICAL","2026-03-09 10:41:20.000000+00" +"16ae02ac-12d3-415f-9819-6d6136e24e28",1,24619382,"6879CFAC93AE5456D392C98DA2DE02D64787F5F6544F58A7EED029BFE6FA9971","B58EC557C1136878D6A3CF012904C89C849ED6016FB91934BEA7B151F47D12D3","CANONICAL","2026-03-09 10:41:32.000000+00" +"d1c694c3-05f9-485f-adf9-632565b1a387",1,24619383,"A04EFFB7165A0CA0A2DBB7B58852EECF1775C99AC2453C68073B72D7F2BA3B39","6879CFAC93AE5456D392C98DA2DE02D64787F5F6544F58A7EED029BFE6FA9971","CANONICAL","2026-03-09 10:41:44.000000+00" +"0d3a628c-04a6-4f97-a7a3-e77dc213db12",1,24619384,"CC2366E009DEA0933C38D30EA63BF5DA8E0A8CF9C0A9BE101BAC93E104EEE219","A04EFFB7165A0CA0A2DBB7B58852EECF1775C99AC2453C68073B72D7F2BA3B39","CANONICAL","2026-03-09 10:41:56.000000+00" +"2571b8d7-9773-4c28-8381-abc4e66ed47a",1,24619385,"6833E4B7BD52E28F5DFC784E4EE9EE96E367DA92A901123BE24966AEDC5B01F9","CC2366E009DEA0933C38D30EA63BF5DA8E0A8CF9C0A9BE101BAC93E104EEE219","CANONICAL","2026-03-09 10:42:08.000000+00" +"ee8819ee-1e7f-48f7-b8ab-60339d0d6f58",1,24619386,"942B0FA0C7CF27DE10D07DEBC726AD7D40E194A17965678D08E1F064E26E17B8","6833E4B7BD52E28F5DFC784E4EE9EE96E367DA92A901123BE24966AEDC5B01F9","CANONICAL","2026-03-09 10:42:20.000000+00" +"eefe9305-406c-4866-8715-31cba02ab9b3",1,24619387,"E55544EB7B4EE3F551233030FAC72D5D4564823E136666D9D019C0F295756088","942B0FA0C7CF27DE10D07DEBC726AD7D40E194A17965678D08E1F064E26E17B8","CANONICAL","2026-03-09 10:42:32.000000+00" +"8c7c7bed-f9a8-4797-88ed-79080ec492e3",1,24619388,"1823ED9D066EA3B2AC87645E25C76FB49AA867AD0F416312457827B6AEB3D462","E55544EB7B4EE3F551233030FAC72D5D4564823E136666D9D019C0F295756088","CANONICAL","2026-03-09 10:42:44.000000+00" +"35598025-ab86-4946-98e7-2443410813e2",1,24619389,"E00C26ED6068FE56CB43FBF8B9A67834FF4C4FC6907D782FE8BCBAC46BEABE3A","1823ED9D066EA3B2AC87645E25C76FB49AA867AD0F416312457827B6AEB3D462","CANONICAL","2026-03-09 10:42:56.000000+00" +"0c003c3f-9164-494f-a1cb-fe1fb40b72c7",1,24619390,"3E1387C3D118AFEEE3D17C98CEEABE0FD86273114808C45C7CB2F090DD3A6049","E00C26ED6068FE56CB43FBF8B9A67834FF4C4FC6907D782FE8BCBAC46BEABE3A","CANONICAL","2026-03-09 10:43:08.000000+00" +"f03719b4-56dd-44e5-adc7-6fcb4c04451a",1,24619391,"612497E3E1E1EBBA0826CA5FD682E22F44F16C8CDF176C43876C662F79DA900A","3E1387C3D118AFEEE3D17C98CEEABE0FD86273114808C45C7CB2F090DD3A6049","CANONICAL","2026-03-09 10:43:20.000000+00" +"3b3ee62a-7bed-421a-8ae0-00e07655a908",1,24619392,"4646AD9392BC3AC08A36FABCB2ECE27B787B77427280B3B9553A36FAB422DCF5","612497E3E1E1EBBA0826CA5FD682E22F44F16C8CDF176C43876C662F79DA900A","CANONICAL","2026-03-09 10:43:32.000000+00" +"57e4f50d-70f0-4635-b073-778e27ea9820",1,24619393,"6B2ABB7AFE4FA2EFA9CF993D21491B5B3E092684C7075F78335BE3B96B890891","4646AD9392BC3AC08A36FABCB2ECE27B787B77427280B3B9553A36FAB422DCF5","CANONICAL","2026-03-09 10:43:44.000000+00" +"beaf8393-f8ab-4546-83b5-3c67dfd1430f",1,24619394,"8F1D3CB47E30E2296D3D8B65D4F458804509ADCD82B7A3B3901DA000BB9ED8FA","6B2ABB7AFE4FA2EFA9CF993D21491B5B3E092684C7075F78335BE3B96B890891","CANONICAL","2026-03-09 10:43:56.000000+00" +"cef0f644-3b8e-499e-8381-2f8342851d78",1,24619395,"653685691B4C940C215879912A041742AE617AAE630F62C7E196E36F69178C1B","8F1D3CB47E30E2296D3D8B65D4F458804509ADCD82B7A3B3901DA000BB9ED8FA","CANONICAL","2026-03-09 10:44:08.000000+00" +"6f3aa571-71c6-4c3d-894d-8c037db62eb7",1,24619396,"5BB41B53F001703C57C79F98529B2A148072BA864CB19CF969140166EDE13DBE","653685691B4C940C215879912A041742AE617AAE630F62C7E196E36F69178C1B","CANONICAL","2026-03-09 10:44:20.000000+00" +"a7b0def4-7b0a-4a2e-aa45-d377fa5bb126",1,24619397,"74F884FD8ABD0870AF49BBB16D3942A8F51B65A9025F9AB062F595E852C53E61","5BB41B53F001703C57C79F98529B2A148072BA864CB19CF969140166EDE13DBE","CANONICAL","2026-03-09 10:44:32.000000+00" +"e847b666-88d2-45b3-9398-dff30c1094eb",1,24619398,"1FDE958CE7218F36649BD309209946FF3BA78B617C41E838726FC4A749AFDAFB","74F884FD8ABD0870AF49BBB16D3942A8F51B65A9025F9AB062F595E852C53E61","CANONICAL","2026-03-09 10:44:44.000000+00" +"1e574b7a-c5d0-4ff9-946d-c5f1fb4eae38",1,24619399,"ABAFDD7E0565CAD918E87D889471C022E66033B8717E26B120ABEBF08CC32043","1FDE958CE7218F36649BD309209946FF3BA78B617C41E838726FC4A749AFDAFB","CANONICAL","2026-03-09 10:44:56.000000+00" +"042b0f4b-0c33-4190-98b5-806546753e8a",1,24619400,"BD7816E61849DF2808012C208AC7B055A1EFD73866CD0044CA77CD1284C14E1A","ABAFDD7E0565CAD918E87D889471C022E66033B8717E26B120ABEBF08CC32043","CANONICAL","2026-03-09 10:45:08.000000+00" +"02b264f3-9d07-433a-a4c7-f07629aaa896",1,24619401,"8F657289DD0D61DBE7B133B6B91FFF71266F2FFE568F37F4CD1BEC0821AFF383","BD7816E61849DF2808012C208AC7B055A1EFD73866CD0044CA77CD1284C14E1A","CANONICAL","2026-03-09 10:45:20.000000+00" +"6a6727b9-3bc7-45a5-a708-4fd7c0710547",1,24619402,"FE197FBD0848EF4E6AAA48DDD120AA8B85E18205268A2615FC67D7C6427140ED","8F657289DD0D61DBE7B133B6B91FFF71266F2FFE568F37F4CD1BEC0821AFF383","CANONICAL","2026-03-09 10:45:32.000000+00" +"49ae35ac-6302-44ba-8ea7-b57976ec3dc6",1,24619403,"7F38950C04EAF5CDA7FEE046B8A05D50C7AB060C07E533F754A1133BC7415A46","FE197FBD0848EF4E6AAA48DDD120AA8B85E18205268A2615FC67D7C6427140ED","CANONICAL","2026-03-09 10:45:44.000000+00" +"f63c2599-16a1-47f5-8cd0-82b4b0686af8",1,24619404,"ADCDB37BB87D7B45E567E9091BCEE05E6D38D6E665E60D10FA69D07CF6AC8B60","7F38950C04EAF5CDA7FEE046B8A05D50C7AB060C07E533F754A1133BC7415A46","CANONICAL","2026-03-09 10:45:56.000000+00" +"b1c1699d-9946-4009-b784-d71657fa9ee8",1,24619405,"F242CF9BEEA8C1C6E53D3F03811CA708127C8200BCFAAAA51CA2599D37799590","ADCDB37BB87D7B45E567E9091BCEE05E6D38D6E665E60D10FA69D07CF6AC8B60","CANONICAL","2026-03-09 10:46:08.000000+00" +"907f7b76-727d-47df-83a9-a399b458017c",1,24619406,"888678081F6B38337CC3717BCF1C661E301A1B80E7EBDCAEFCB95C20A260A19C","F242CF9BEEA8C1C6E53D3F03811CA708127C8200BCFAAAA51CA2599D37799590","CANONICAL","2026-03-09 10:46:20.000000+00" +"e332adbc-0668-4c52-8bf9-f102b810ff48",1,24619407,"6AE5944E0380802D0107E4CD8B2551BF69972C86A3859147051DC1D394188F45","888678081F6B38337CC3717BCF1C661E301A1B80E7EBDCAEFCB95C20A260A19C","CANONICAL","2026-03-09 10:46:32.000000+00" +"cd71e518-cf62-4975-9c0b-8b0e1c25161d",1,24619408,"4530F4C82E16FCC852C8CD968D44193B4344C3C5CEA3761DD49CAAB5FFF990D8","6AE5944E0380802D0107E4CD8B2551BF69972C86A3859147051DC1D394188F45","CANONICAL","2026-03-09 10:46:44.000000+00" +"8129b7b3-3b3e-4a72-ba97-a9e5d59ccdf3",1,24619409,"108E9BFA51E7EE8D321CD0CB5D6FC6C04571569F42B1F62B696A3B0B8FA327B3","4530F4C82E16FCC852C8CD968D44193B4344C3C5CEA3761DD49CAAB5FFF990D8","CANONICAL","2026-03-09 10:46:56.000000+00" +"3c74d4bd-8f4d-4e74-88e1-c5a4e732c913",1,24619410,"E2CEB35DB79F403AE61753D4E0D114EFDDD92528E36DADF17F16EDFA216E3130","108E9BFA51E7EE8D321CD0CB5D6FC6C04571569F42B1F62B696A3B0B8FA327B3","CANONICAL","2026-03-09 10:47:08.000000+00" +"7d821dda-9347-4fd8-a619-88a626275eee",1,24619411,"2E279E287A94837DFF9303E52AB4A0FB3AE4DD436D9313B0BDCE0AC0FA7FB648","E2CEB35DB79F403AE61753D4E0D114EFDDD92528E36DADF17F16EDFA216E3130","CANONICAL","2026-03-09 10:47:20.000000+00" +"e2bd7991-dc2f-436f-a107-e5fbb3755ba0",1,24619412,"1D955C33D14DE42B27B61D5D8D3FB0C178E96997F1ED6EAB1A96F46C2E2627E8","2E279E287A94837DFF9303E52AB4A0FB3AE4DD436D9313B0BDCE0AC0FA7FB648","CANONICAL","2026-03-09 10:47:32.000000+00" +"29608feb-8e78-4c47-b233-97f45de6c5fc",1,24619413,"DD503EC9752747FAA8ABA65CB89E857DFDCA31868C486AB3ED29DCCE162FF1C3","1D955C33D14DE42B27B61D5D8D3FB0C178E96997F1ED6EAB1A96F46C2E2627E8","CANONICAL","2026-03-09 10:47:44.000000+00" +"85093d3c-6b14-4684-974e-eb117957a1cf",1,24619414,"27CA4082B48E132335D5268EA9379661346FD5124FD8585B2A6A9898109DCEC6","DD503EC9752747FAA8ABA65CB89E857DFDCA31868C486AB3ED29DCCE162FF1C3","CANONICAL","2026-03-09 10:47:56.000000+00" +"6483b7dc-6234-4465-b560-e41857721d71",1,24619415,"B10F262C6664E651A64FCDB79489EBFC0A5A8B1B0B34B97284548F2397869E76","27CA4082B48E132335D5268EA9379661346FD5124FD8585B2A6A9898109DCEC6","CANONICAL","2026-03-09 10:48:08.000000+00" +"e3a6a38c-076e-451e-88e9-4d46cb67f41f",1,24619416,"8BB092084A51CE568532EB7886DB5EFEEFF7003189D12C7B464F7A568650D988","B10F262C6664E651A64FCDB79489EBFC0A5A8B1B0B34B97284548F2397869E76","CANONICAL","2026-03-09 10:48:20.000000+00" +"6984c974-a78e-4bb9-be87-da74dbc7c95d",1,24619417,"1AEBB0FD86CC30B9AC11027D207467A57E8DCF8C45FECF1A58AE3B8FBC011CB3","8BB092084A51CE568532EB7886DB5EFEEFF7003189D12C7B464F7A568650D988","CANONICAL","2026-03-09 10:48:32.000000+00" +"170dffc7-64ae-4afe-91ec-96a929635cde",1,24619418,"81139D95D8A07685387D974A48A9283837365E6B4836595E0253BCC75DEC3870","1AEBB0FD86CC30B9AC11027D207467A57E8DCF8C45FECF1A58AE3B8FBC011CB3","CANONICAL","2026-03-09 10:48:44.000000+00" +"4a9b53ac-ba8c-43f3-96b0-ae4b64480196",1,24619419,"3089A432B425B1C34A15BC9DD317FC21A225B78206B269F762BAB7B270024D8A","81139D95D8A07685387D974A48A9283837365E6B4836595E0253BCC75DEC3870","CANONICAL","2026-03-09 10:48:56.000000+00" +"e5bb3055-6492-4238-bbce-481635c7d44d",1,24619420,"040FEA2B0FD36FB3DEA413EC141EB0734FC1C95023AD2BBA9FD5625554276E7A","3089A432B425B1C34A15BC9DD317FC21A225B78206B269F762BAB7B270024D8A","CANONICAL","2026-03-09 10:49:08.000000+00" +"486f70fb-8fab-4b55-9f4b-ffab82677957",1,24619421,"FC5C425F721CDE4F917FEB447FC410DE291720F2F2AC69511551AD98DBDC2F88","040FEA2B0FD36FB3DEA413EC141EB0734FC1C95023AD2BBA9FD5625554276E7A","CANONICAL","2026-03-09 10:49:20.000000+00" +"84938765-e981-4665-92a0-2eaeffe5499d",1,24619422,"6988BBB125ED4A558DBE817A03BCDF75DD1FE7439BD5E8F623369F0A846C7A23","FC5C425F721CDE4F917FEB447FC410DE291720F2F2AC69511551AD98DBDC2F88","CANONICAL","2026-03-09 10:49:32.000000+00" +"4ed5f250-2673-4759-b005-40679d06eede",1,24619423,"65D51489AEBD82F9AB9C12DF39D8FE81F13BE02E0C56332B889FA6F2965C04B9","6988BBB125ED4A558DBE817A03BCDF75DD1FE7439BD5E8F623369F0A846C7A23","CANONICAL","2026-03-09 10:49:44.000000+00" +"471b1820-bce9-48a7-874c-9cc4d844df8e",1,24619424,"141CC3A72FFA0BDE12702AFB2F3E6F7B3CF269A6732FD493A44968F7D31FF8BD","65D51489AEBD82F9AB9C12DF39D8FE81F13BE02E0C56332B889FA6F2965C04B9","CANONICAL","2026-03-09 10:49:56.000000+00" +"a7751af7-9fa8-4339-b151-6e18bc08775f",1,24619425,"6927F7636E72B94C6CA7C1231D74341EFFA87B3B48197C815C83D178F2FC8079","141CC3A72FFA0BDE12702AFB2F3E6F7B3CF269A6732FD493A44968F7D31FF8BD","CANONICAL","2026-03-09 10:50:08.000000+00" +"4b8025a9-61ef-47cd-b379-c909a23719a0",1,24619426,"E84D02CFC5AF99DF26632ED591B7F4154F00AA13108DB13F4CB1A207868C6E08","6927F7636E72B94C6CA7C1231D74341EFFA87B3B48197C815C83D178F2FC8079","CANONICAL","2026-03-09 10:50:20.000000+00" +"3647cdb2-aa35-4d9a-aab1-fc4b66f820c7",1,24619427,"087DCBC2A347308BBEC76A290D2B56121A60291A3A37DE1432F7A8C1F98F31DE","E84D02CFC5AF99DF26632ED591B7F4154F00AA13108DB13F4CB1A207868C6E08","CANONICAL","2026-03-09 10:50:32.000000+00" +"417070bd-f0d9-4612-b25c-162025a8a592",1,24619428,"7FDD16872CAC3FD1B650586C6A8323882A4962EC4C0EDB01927B07120BF27861","087DCBC2A347308BBEC76A290D2B56121A60291A3A37DE1432F7A8C1F98F31DE","CANONICAL","2026-03-09 10:50:44.000000+00" +"a2bb6f84-67c4-405f-951f-4e2e3d7adec3",1,24619429,"01685FF14D51D1871CA529628B05062B5D3607A68AD80A5EDB0745E5575D4F2A","7FDD16872CAC3FD1B650586C6A8323882A4962EC4C0EDB01927B07120BF27861","CANONICAL","2026-03-09 10:50:56.000000+00" +"890804e5-dc5a-4ddc-a52a-f574a2dc07db",1,24619430,"4244C1698C5B92876CA557DFB995E9641FB1787431082CBC292E01DF6BBB41FD","01685FF14D51D1871CA529628B05062B5D3607A68AD80A5EDB0745E5575D4F2A","CANONICAL","2026-03-09 10:51:08.000000+00" +"b5539464-3d41-41dc-872f-724933380ef8",1,24619431,"EC02FDF7C1C8AC3F47DA9AB78320D22D01928C605A52D539AE4E32992E6F2BAE","4244C1698C5B92876CA557DFB995E9641FB1787431082CBC292E01DF6BBB41FD","CANONICAL","2026-03-09 10:51:20.000000+00" +"12f64326-10c0-454a-bfc3-698c7b7c9aa5",1,24619432,"2A849F8E241EF71C7C4F9199B5A64325AD3CE5A7781A1302A02B63CFBBC53585","EC02FDF7C1C8AC3F47DA9AB78320D22D01928C605A52D539AE4E32992E6F2BAE","CANONICAL","2026-03-09 10:51:32.000000+00" +"3cc81181-a522-46fc-9623-d85fa3999ea9",1,24619433,"F210947CB567E8ABC8996CBD8CD0819CBFBC79A38CF3A483CAB6B0D3C4AF097D","2A849F8E241EF71C7C4F9199B5A64325AD3CE5A7781A1302A02B63CFBBC53585","CANONICAL","2026-03-09 10:51:44.000000+00" +"8751f04d-05c0-4033-81ac-7192f8c147a3",1,24619434,"43239631A22093BF67F17888CBA13427A1211A3F76380EA5C7E89DA3F1FFEED2","F210947CB567E8ABC8996CBD8CD0819CBFBC79A38CF3A483CAB6B0D3C4AF097D","CANONICAL","2026-03-09 10:51:56.000000+00" +"7be5f2cc-1c3e-421b-b802-83e65a7db919",1,24619435,"A8AF94CD02EA70D81FCAE49A64D188AC5DE6BDE3CE65820A414A1269C2737511","43239631A22093BF67F17888CBA13427A1211A3F76380EA5C7E89DA3F1FFEED2","CANONICAL","2026-03-09 10:52:08.000000+00" +"a4471d9f-8cc8-4172-b598-f34e13780520",1,24619436,"5285E6874A150BE2EE87F9523B3A802EBF15DCD4125307E2F77A208FF5AD4623","A8AF94CD02EA70D81FCAE49A64D188AC5DE6BDE3CE65820A414A1269C2737511","CANONICAL","2026-03-09 10:52:20.000000+00" +"882b7a96-6276-4037-8cd3-7d18ec58f494",1,24619437,"40189A421387F01657360B4D2729523FD80714B36F78AB96B1DDE53EE18FDAB5","5285E6874A150BE2EE87F9523B3A802EBF15DCD4125307E2F77A208FF5AD4623","CANONICAL","2026-03-09 10:52:32.000000+00" +"0ce21b35-8561-4b00-a644-1b6b585a37ba",1,24619438,"D787C439C6E5358DC5A015DED4E6509F22BC5F80CF6099621144EE58083C0EED","40189A421387F01657360B4D2729523FD80714B36F78AB96B1DDE53EE18FDAB5","CANONICAL","2026-03-09 10:52:44.000000+00" +"52de7876-dc1d-4114-a991-b55f8e979bf2",1,24619439,"4B987F50FD41FCA21A9BBA86B45BD221DE893F8B4F48CC6103C0006D6A376BCF","D787C439C6E5358DC5A015DED4E6509F22BC5F80CF6099621144EE58083C0EED","CANONICAL","2026-03-09 10:52:56.000000+00" +"6e90fbbd-f8ad-4e21-bc57-6a263d11e23e",1,24619440,"B066F240C325E265A0C0CD0083AF5FB8172F8D87786701514A6AB612A7D0BFBD","4B987F50FD41FCA21A9BBA86B45BD221DE893F8B4F48CC6103C0006D6A376BCF","CANONICAL","2026-03-09 10:53:08.000000+00" \ No newline at end of file diff --git a/listener/data/reorg-testing-data-50.csv b/listener/data/reorg-testing-data-50.csv new file mode 100644 index 0000000000..792a1ca46c --- /dev/null +++ b/listener/data/reorg-testing-data-50.csv @@ -0,0 +1,49 @@ +"9afb4555-4fcf-4b74-aafc-7bce22ec29d6",1,24619310,"5FBB452E14704821AA8A9E07439E5F7B08EC2E39238366E2CA1CC4337F980BAE","A237593CDBA889B58915AA6A61F3EB047DAB2B22BA8DCC0BA9FBF14CAC4AF731","CANONICAL","2026-03-09 10:26:14.391532+00" +"7e893be5-c46c-47ba-9463-c8407a57b920",1,24619311,"0409356A94F49FDBA1F24334E83447A0AD77FEC516ECFDAFED71282A3A1486A7","5FBB452E14704821AA8A9E07439E5F7B08EC2E39238366E2CA1CC4337F980BAE","CANONICAL","2026-03-09 10:26:25.988417+00" +"31874c8c-d96f-4a90-8c21-180e62a889e0",1,24619315,"5B465871CB3C50DBF7A6006851F4EA2B854F5B9D48338E2F6A0DCE4B0DFEAD5D","2DA42BBF62E6AE3602E87AB8EC28F46968B745476A4B8BDBA5AB35980AEB370D","CANONICAL","2026-03-09 10:27:17.405497+00" +"6d626275-a665-438f-a625-408619699199",1,24619312,"CDFE4E3B97099C9D098D3B11D8EDFDC0477E2C53C7A9DF943BE9341BFC869E3F","0409356A94F49FDBA1F24334E83447A0AD77FEC516ECFDAFED71282A3A1486A7","CANONICAL","2026-03-09 10:26:38.950499+00" +"957efb01-426f-455e-afea-c67dc0669df8",1,24619313,"24E3A70842FEA7E5E8A40EDD9BCC9DD2269D25C3939FF075C160B99855017D19","CDFE4E3B97099C9D098D3B11D8EDFDC0477E2C53C7A9DF943BE9341BFC869E3F","CANONICAL","2026-03-09 10:26:49.410316+00" +"8d9ae7fc-e0b3-4ffb-b413-637bf82bc021",1,24619314,"2DA42BBF62E6AE3602E87AB8EC28F46968B745476A4B8BDBA5AB35980AEB370D","24E3A70842FEA7E5E8A40EDD9BCC9DD2269D25C3939FF075C160B99855017D19","CANONICAL","2026-03-09 10:27:06.503866+00" +"9d624758-f71b-4647-b2fd-60f7ae1e93cd",1,24619316,"8BE72344BBDFE6669B995B0A07F45418E8099628D92CD670665C0791A24D44D1","6F779067708D8AAFA4C18E39D82AA03119532D6F4A2E23AB6E47BC5BDECC9DA0","CANONICAL","2026-03-09 10:28:20.000000+00" +"c7e310e9-c26d-49c8-840c-c4832108420a",1,24619317,"6525DAC1A94AE0703C61FADF30AAA7812CFBB73F4D67270BF389A2E264C23EE1","8BE72344BBDFE6669B995B0A07F45418E8099628D92CD670665C0791A24D44D1","CANONICAL","2026-03-09 10:28:32.000000+00" +"afe35ab5-d028-406b-884b-f4b169c8fb4c",1,24619318,"2D95CF00F37EA028A4902BECC1BE0B09E7BD57CC35748341EC6980EB96D1D3D3","6525DAC1A94AE0703C61FADF30AAA7812CFBB73F4D67270BF389A2E264C23EE1","CANONICAL","2026-03-09 10:28:44.000000+00" +"22ce1143-45d7-40da-b907-a8f8a42ca9ae",1,24619319,"F29FA52C499EFB388394D42DF56AA70F7F58A590F79D3B45B55DF7C8A2DB0C97","2D95CF00F37EA028A4902BECC1BE0B09E7BD57CC35748341EC6980EB96D1D3D3","CANONICAL","2026-03-09 10:28:56.000000+00" +"f30b33ee-5fa9-4f4d-9ed6-c780984c8029",1,24619320,"1E5ED0CFD8160D3CFF37D9871170F7AD7FFB65E8587FC73649A4BA4003DBDC0E","F29FA52C499EFB388394D42DF56AA70F7F58A590F79D3B45B55DF7C8A2DB0C97","CANONICAL","2026-03-09 10:29:08.000000+00" +"b974a131-1f40-4f42-a352-191a6856f5a7",1,24619321,"9CE6C32DA04CA83CB3E82DC63833D27080067C5367B4E3B2AE25AB85A824632C","1E5ED0CFD8160D3CFF37D9871170F7AD7FFB65E8587FC73649A4BA4003DBDC0E","CANONICAL","2026-03-09 10:29:20.000000+00" +"96f9b509-8b30-494b-947d-fb4f9bf41cac",1,24619322,"5B984F02979A2457CBC98E8B4F13C252AA53568C59FA2BAE980254ECC00D1AF1","9CE6C32DA04CA83CB3E82DC63833D27080067C5367B4E3B2AE25AB85A824632C","CANONICAL","2026-03-09 10:29:32.000000+00" +"ad5c5e56-d89d-414c-93b4-84f9032ae0c7",1,24619323,"BB3347465E47BFEF22DD7D550D9DD32F249B2D6C20810CF520EF29E2D0D377E2","5B984F02979A2457CBC98E8B4F13C252AA53568C59FA2BAE980254ECC00D1AF1","CANONICAL","2026-03-09 10:29:44.000000+00" +"0e75e9fe-f84f-48d3-9c7c-07d7bd2e90b6",1,24619324,"3DE38180AAF852930D67F189BF3BD13CE610B34F9ADCBF983DF087F17A41A2A5","BB3347465E47BFEF22DD7D550D9DD32F249B2D6C20810CF520EF29E2D0D377E2","CANONICAL","2026-03-09 10:29:56.000000+00" +"1ab25125-31fa-4162-8769-00d389ba7a7f",1,24619325,"66CCBCC0A43A6413CFC119803B94A855CB573D8BB310004DEFEB7D0C2FB5F5AD","3DE38180AAF852930D67F189BF3BD13CE610B34F9ADCBF983DF087F17A41A2A5","CANONICAL","2026-03-09 10:30:08.000000+00" +"293c564f-5eeb-49dd-8bea-f0346349a242",1,24619326,"929F7618F945299B3E10C0F22B7957D0A7F683A8A2B898D099F5E14AB2C4F9DD","66CCBCC0A43A6413CFC119803B94A855CB573D8BB310004DEFEB7D0C2FB5F5AD","CANONICAL","2026-03-09 10:30:20.000000+00" +"733cb3a9-72e6-4130-bb35-aa8ab8afa639",1,24619327,"4B3CB9F8FCFC55AD2D3B7FB7F10F82D6CF6CB8E68FCB85E799E291B1125AAD02","929F7618F945299B3E10C0F22B7957D0A7F683A8A2B898D099F5E14AB2C4F9DD","CANONICAL","2026-03-09 10:30:32.000000+00" +"d0caefaa-8ba6-423c-8d14-199f5378db0b",1,24619328,"3EE2D3FB9208662AF1C988876075C46E43BF3CC85514ED776AB8AE4CE1611612","4B3CB9F8FCFC55AD2D3B7FB7F10F82D6CF6CB8E68FCB85E799E291B1125AAD02","CANONICAL","2026-03-09 10:30:44.000000+00" +"c604b2d7-f7d6-48c2-b4e2-cfa2c870f302",1,24619329,"A8D3DDD25906EFE8233DF194C9429AFAF934843082E01AAC0F2E1651190BBA6B","3EE2D3FB9208662AF1C988876075C46E43BF3CC85514ED776AB8AE4CE1611612","CANONICAL","2026-03-09 10:30:56.000000+00" +"36d972e7-7423-4f21-aaa8-de481e035cb8",1,24619330,"138073F3FB274CBEDB9CD120D4DA698A0DC4930735201F1AD2A8F7F77A1B364B","A8D3DDD25906EFE8233DF194C9429AFAF934843082E01AAC0F2E1651190BBA6B","CANONICAL","2026-03-09 10:31:08.000000+00" +"760c8274-d559-44d8-b063-3f60350a6b8b",1,24619331,"6DF7178EC651F528EBB0F9CC88ABC055032CCCA642539E39B62EC59080D9F933","138073F3FB274CBEDB9CD120D4DA698A0DC4930735201F1AD2A8F7F77A1B364B","CANONICAL","2026-03-09 10:31:20.000000+00" +"6a575a44-6702-47f9-9509-8b681fd12e1d",1,24619332,"C66E82B3325CF536F47F6B711DB5537629C004F9EED2EDC00A58F9D47D5D2D30","6DF7178EC651F528EBB0F9CC88ABC055032CCCA642539E39B62EC59080D9F933","CANONICAL","2026-03-09 10:31:32.000000+00" +"71dc79f0-0a87-4177-93eb-81a9e4a585e1",1,24619333,"3D56DBFC4BCE0CC2D49AEE7B531865DFFDD1916D6FAEB9787B38085F9EDEB39F","C66E82B3325CF536F47F6B711DB5537629C004F9EED2EDC00A58F9D47D5D2D30","CANONICAL","2026-03-09 10:31:44.000000+00" +"223b0b93-2278-414e-ac6e-878052e94c85",1,24619334,"E457EE89AB3D2E31B5AEA59FC3466188C4568B05293BB92296225A539858C2FA","3D56DBFC4BCE0CC2D49AEE7B531865DFFDD1916D6FAEB9787B38085F9EDEB39F","CANONICAL","2026-03-09 10:31:56.000000+00" +"56205a9f-bd02-4430-afe2-64651dd1fb79",1,24619335,"159BA172F42D660875E8DE84C18C027249611C70380B4D1EB466AACE403B783A","E457EE89AB3D2E31B5AEA59FC3466188C4568B05293BB92296225A539858C2FA","CANONICAL","2026-03-09 10:32:08.000000+00" +"24168ea8-ef04-48ec-83ca-60ee4408e206",1,24619336,"49BFA5F6EF0E4BB1D3A0D875461B3123F8CA3592B16742141602C1D63FBA7D77","159BA172F42D660875E8DE84C18C027249611C70380B4D1EB466AACE403B783A","CANONICAL","2026-03-09 10:32:20.000000+00" +"e87d8912-7213-404b-a68f-375821c1d1f2",1,24619337,"EDEA8C507424653C545EAFF7CDB1749668E91A47F1ACBF8C946FCEC01BA48293","49BFA5F6EF0E4BB1D3A0D875461B3123F8CA3592B16742141602C1D63FBA7D77","CANONICAL","2026-03-09 10:32:32.000000+00" +"3feaa057-bc9d-4548-ac8a-cc9f72d8bbf6",1,24619338,"49874AC1FC41B7DAE7FBDEA6ED03F6AE40012B72B628CB66A7377EF27A4E251D","EDEA8C507424653C545EAFF7CDB1749668E91A47F1ACBF8C946FCEC01BA48293","CANONICAL","2026-03-09 10:32:44.000000+00" +"b81f7925-4c86-422a-9e7b-109075b984de",1,24619339,"78F4600F025EF87B843CC349E72AC24D9BB9A2E6BFA0047F17D173D7F6CAE150","49874AC1FC41B7DAE7FBDEA6ED03F6AE40012B72B628CB66A7377EF27A4E251D","CANONICAL","2026-03-09 10:32:56.000000+00" +"02b7eeb2-cb0f-408f-934d-fc74b51b4c42",1,24619340,"0A6A6706819DF5D3479787F476AAD222712EC411FE9E5AEA8F2D3C1D437335EF","78F4600F025EF87B843CC349E72AC24D9BB9A2E6BFA0047F17D173D7F6CAE150","CANONICAL","2026-03-09 10:33:08.000000+00" +"29b6f606-4ec0-4e7f-8fe8-69df9a6dc0a0",1,24619341,"0EA220D4A243613DE07827AB7765EC7C3CFF4EB9D11642E7749276A31280F182","0A6A6706819DF5D3479787F476AAD222712EC411FE9E5AEA8F2D3C1D437335EF","CANONICAL","2026-03-09 10:33:20.000000+00" +"6630ed31-e0eb-42d3-a742-fef0386aa258",1,24619342,"A259329A782D380CC3A04AAD87093D6826869FCEBEB08605F9A1D32FC784F2F5","0EA220D4A243613DE07827AB7765EC7C3CFF4EB9D11642E7749276A31280F182","CANONICAL","2026-03-09 10:33:32.000000+00" +"57966c2a-f2a2-42ab-bf84-0d3bd422bc9f",1,24619343,"177066839776137D85F6C7669509E8DB4239F78E6F32582095A819530D4A01E3","A259329A782D380CC3A04AAD87093D6826869FCEBEB08605F9A1D32FC784F2F5","CANONICAL","2026-03-09 10:33:44.000000+00" +"c6cbefb6-bb5a-4142-9b48-65cd71608692",1,24619344,"26599DD98A4C2E0B569B8983F4C0BB34CC23DB3FAAB64386CD8CFC29429B87C8","177066839776137D85F6C7669509E8DB4239F78E6F32582095A819530D4A01E3","CANONICAL","2026-03-09 10:33:56.000000+00" +"c08ec4d9-91f5-431f-bd42-c124eaef1a70",1,24619345,"578DF90C9B526F7A79B7DC2C8386E5EA44ED230690B9C271E4B8B997D3C62717","26599DD98A4C2E0B569B8983F4C0BB34CC23DB3FAAB64386CD8CFC29429B87C8","CANONICAL","2026-03-09 10:34:08.000000+00" +"c831af74-e55d-4369-8edf-cedd1389cd1d",1,24619346,"14077FDC1A791726FB4994F6E3CC23B501C005751AB8AD254EC2B7CD0965A2F5","578DF90C9B526F7A79B7DC2C8386E5EA44ED230690B9C271E4B8B997D3C62717","CANONICAL","2026-03-09 10:34:20.000000+00" +"5189bcda-a9a4-41f4-9b40-5078cc1c7670",1,24619347,"C52126B339CCB59196EFE2984B3ADE193FC9F3CD6566B5598635AC0351CE9B19","14077FDC1A791726FB4994F6E3CC23B501C005751AB8AD254EC2B7CD0965A2F5","CANONICAL","2026-03-09 10:34:32.000000+00" +"606708aa-3f1a-49d4-a0bb-1f619880cd51",1,24619348,"A18E72979006EEF6E33BFD70A2F3F05EDEEF2EFA596DB4C3E44FBFB6D0A5402A","C52126B339CCB59196EFE2984B3ADE193FC9F3CD6566B5598635AC0351CE9B19","CANONICAL","2026-03-09 10:34:44.000000+00" +"d57a82c7-5deb-4d13-9446-15f035b14656",1,24619349,"238C5C5EFA79936B6AD693E8C0B147EB728E0B8587CEDF086B8AFBC5F1238A35","A18E72979006EEF6E33BFD70A2F3F05EDEEF2EFA596DB4C3E44FBFB6D0A5402A","CANONICAL","2026-03-09 10:34:56.000000+00" +"8715010c-04da-4bfe-bb08-e0cd34f2aba5",1,24619350,"625921F0D0FF2F314F5787083DCFB510C2E66557371E6ACF232456C7857F9C0C","238C5C5EFA79936B6AD693E8C0B147EB728E0B8587CEDF086B8AFBC5F1238A35","CANONICAL","2026-03-09 10:35:08.000000+00" +"3a8006fb-3eac-47a8-8c27-1d270dd4e80d",1,24619351,"7D746867478DCF482ED3B9849851889032B92ED4FAD7802A77066DB69795049C","625921F0D0FF2F314F5787083DCFB510C2E66557371E6ACF232456C7857F9C0C","CANONICAL","2026-03-09 10:35:20.000000+00" +"2dbdd28a-02ab-4bba-9b4a-02010d2fe201",1,24619352,"95AD323C221CA9C38202DDD221174307D567B3802FCF5E81F767D14DD2A5B5D6","7D746867478DCF482ED3B9849851889032B92ED4FAD7802A77066DB69795049C","CANONICAL","2026-03-09 10:35:32.000000+00" +"205cf145-8e0a-4432-866c-6b97c032f920",1,24619353,"9118451EEFD64E1261A61CE1321F08FB7A820FD19CA1B1D20A6111CFB5CDAE62","95AD323C221CA9C38202DDD221174307D567B3802FCF5E81F767D14DD2A5B5D6","CANONICAL","2026-03-09 10:35:44.000000+00" +"f5a21077-63d7-4de8-9e7f-80b0c56e2a73",1,24619354,"E449E1A259EE70E369092FC9C4487864A8B73D6F6F7F511A31DE2238B90897F5","9118451EEFD64E1261A61CE1321F08FB7A820FD19CA1B1D20A6111CFB5CDAE62","CANONICAL","2026-03-09 10:35:56.000000+00" +"9291a982-db2d-4b9d-a666-1720bf157aac",1,24619355,"348D150577570F45FA2DC6F8FFAF788CBCAA853D56F50F1CFCC36C2B7BA2C17D","E449E1A259EE70E369092FC9C4487864A8B73D6F6F7F511A31DE2238B90897F5","CANONICAL","2026-03-09 10:36:08.000000+00" +"66df9d28-21ef-4608-9f44-327258dcd786",1,24619356,"36E6461FABEA07506AD2DAE091ADD209131877DAA1D2F11DB2EBE74B5C302D73","348D150577570F45FA2DC6F8FFAF788CBCAA853D56F50F1CFCC36C2B7BA2C17D","CANONICAL","2026-03-09 10:36:20.000000+00" +"348d8d44-fea7-4b20-a377-5be71cd5a5e8",1,24619357,"4002679A865D64E18C4E30B63F9C2AF2F7A5C652FFF418D88DDFC0549C015D7D","36E6461FABEA07506AD2DAE091ADD209131877DAA1D2F11DB2EBE74B5C302D73","CANONICAL","2026-03-09 10:36:32.000000+00" +"96c4934d-ea95-4e69-84b2-743efbcdd8a1",1,24619358,"0DF97B29B70CC0F0D04ED9476B1A01BE4064A4CB53DA3B9157C6F3685EA595EC","4002679A865D64E18C4E30B63F9C2AF2F7A5C652FFF418D88DDFC0549C015D7D","CANONICAL","2026-03-09 10:36:44.000000+00" \ No newline at end of file diff --git a/listener/data/reorg-testing-data.csv b/listener/data/reorg-testing-data.csv new file mode 100644 index 0000000000..504d4dc2e5 --- /dev/null +++ b/listener/data/reorg-testing-data.csv @@ -0,0 +1,6 @@ +"9afb4555-4fcf-4b74-aafc-7bce22ec29d6",1,24619310,"5FBB452E14704821AA8A9E07439E5F7B08EC2E39238366E2CA1CC4337F980BAE","A237593CDBA889B58915AA6A61F3EB047DAB2B22BA8DCC0BA9FBF14CAC4AF731","CANONICAL","2026-03-09 10:26:14.391532+00" +"7e893be5-c46c-47ba-9463-c8407a57b920",1,24619311,"0409356A94F49FDBA1F24334E83447A0AD77FEC516ECFDAFED71282A3A1486A7","5FBB452E14704821AA8A9E07439E5F7B08EC2E39238366E2CA1CC4337F980BAE","CANONICAL","2026-03-09 10:26:25.988417+00" +"31874c8c-d96f-4a90-8c21-180e62a889e0",1,24619315,"5B465871CB3C50DBF7A6006851F4EA2B854F5B9D48338E2F6A0DCE4B0DFEAD5D","2DA42BBF62E6AE3602E87AB8EC28F46968B745476A4B8BDBA5AB35980AEB370D","CANONICAL","2026-03-09 10:27:17.405497+00" +"6d626275-a665-438f-a625-408619699199",1,24619312,"CDFE4E3B97099C9D098D3B11D8EDFDC0477E2C53C7A9DF943BE9341BFC869E3F","0409356A94F49FDBA1F24334E83447A0AD77FEC516ECFDAFED71282A3A1486A7","CANONICAL","2026-03-09 10:26:38.950499+00" +"957efb01-426f-455e-afea-c67dc0669df8",1,24619313,"24E3A70842FEA7E5E8A40EDD9BCC9DD2269D25C3939FF075C160B99855017D19","CDFE4E3B97099C9D098D3B11D8EDFDC0477E2C53C7A9DF943BE9341BFC869E3F","CANONICAL","2026-03-09 10:26:49.410316+00" +"8d9ae7fc-e0b3-4ffb-b413-637bf82bc021",1,24619314,"2DA42BBF62E6AE3602E87AB8EC28F46968B745476A4B8BDBA5AB35980AEB370D","24E3A70842FEA7E5E8A40EDD9BCC9DD2269D25C3939FF075C160B99855017D19","CANONICAL","2026-03-09 10:27:06.503866+00" diff --git a/listener/dev/create-database.sql b/listener/dev/create-database.sql new file mode 100644 index 0000000000..226d1bc980 --- /dev/null +++ b/listener/dev/create-database.sql @@ -0,0 +1 @@ +CREATE DATABASE listener; \ No newline at end of file diff --git a/listener/docker-compose.yaml b/listener/docker-compose.yaml new file mode 100644 index 0000000000..d5596140ba --- /dev/null +++ b/listener/docker-compose.yaml @@ -0,0 +1,173 @@ +# Unified Docker Compose for the listener project. +# +# Usage: +# docker compose up # Base (PostgreSQL + Redis + RabbitMQ) +# docker compose --profile erpc up # Base + eRPC proxy +# docker compose --profile monitoring up # Base + Prometheus + Grafana +# +# Listener instances (one per chain, erpc auto-starts): +# docker compose --profile listener-1 up # Chain 1 (see config/listener-1.yaml) +# docker compose --profile listener-2 up # Chain 2 (see config/listener-2.yaml) +# docker compose --profile listener-1 --profile listener-2 up # Both chains +# +# Combine with monitoring: +# docker compose --profile listener-1 --profile monitoring up +# + +x-listener-base: &listener-base + build: + context: . + dockerfile: Dockerfile + restart: unless-stopped + networks: + - listener + +services: + # ── Base infrastructure (always starts) ────────────────────────────── + + postgres: + image: postgres:17-alpine + container_name: listener-postgres + environment: + POSTGRES_DB: postgres + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 10s + timeout: 5s + retries: 5 + ports: + - "5432:5432" + networks: + - listener + volumes: + - postgres_data:/var/lib/postgresql/data + - ./dev/create-database.sql:/docker-entrypoint-initdb.d/create-database.sql + + # ── Message brokers (always start) ─────────────────────────────────── + + redis: + image: redis:7-alpine + container_name: listener-redis + ports: + - "6379:6379" + networks: + - listener + restart: unless-stopped + command: redis-server --appendonly yes --appendfsync always + volumes: + - redis_data:/data + + rabbitmq: + image: rabbitmq:3-management-alpine + container_name: listener-rabbitmq + ports: + - "5672:5672" + - "15672:15672" + environment: + RABBITMQ_DEFAULT_USER: user + RABBITMQ_DEFAULT_PASS: pass + networks: + - listener + restart: unless-stopped + volumes: + - rabbitmq_data:/var/lib/rabbitmq + + # ── RPC proxy (optional) ───────────────────────────────────────────── + + erpc: + image: ghcr.io/erpc/erpc:0.0.63 + container_name: listener-erpc + profiles: [erpc, listener-1, listener-2] + volumes: + - ./config/erpc-public.yaml:/erpc.yaml + ports: + - "4000:4000" # HTTP API + - "4001:4001" # Metrics + networks: + - listener + + # ── Listener instances (one per chain, pick via --profile) ─────────── + + listener-1: + <<: *listener-base + container_name: listener-1 + profiles: [listener-1] + depends_on: + postgres: { condition: service_healthy } + rabbitmq: { condition: service_started } + erpc: { condition: service_started } + volumes: + - ./config/listener-1.yaml:/config.yaml:ro + + listener-2: + <<: *listener-base + container_name: listener-2 + profiles: [listener-2] + depends_on: + postgres: { condition: service_healthy } + redis: { condition: service_started } + erpc: { condition: service_started } + volumes: + - ./config/listener-2.yaml:/config.yaml:ro + + # ── Monitoring (optional) ──────────────────────────────────────────── + # Note: Prometheus scrapes erpc:4001 — combine with --profile erpc + # for full metrics. Without eRPC, Prometheus still starts but that + # target shows as down. + + prometheus: + image: prom/prometheus:v3.9.1 + container_name: listener-prometheus + profiles: [monitoring] + ports: + - "9090:9090" + networks: + - listener + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--web.console.libraries=/etc/prometheus/console_libraries' + - '--web.console.templates=/etc/prometheus/consoles' + - '--web.enable-lifecycle' + volumes: + - ./monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./monitoring/prometheus/alert.rules:/etc/prometheus/alert.rules:ro + - prometheus_data:/prometheus + restart: unless-stopped + + grafana: + image: grafana/grafana:12.3.2 + container_name: listener-grafana + profiles: [monitoring] + ports: + - "3000:3000" + networks: + - listener + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD: admin + GF_USERS_ALLOW_SIGN_UP: "false" + volumes: + - ./monitoring/grafana/grafana.ini:/etc/grafana/grafana.ini:ro + - ./monitoring/grafana/datasources/prometheus.yml:/etc/grafana/provisioning/datasources/prometheus.yml:ro + - ./monitoring/grafana/dashboards/default.yml:/etc/grafana/provisioning/dashboards/default.yml:ro + - ./monitoring/grafana/dashboards/erpc.json:/etc/grafana/dashboards/erpc.json:ro + - ./monitoring/grafana/dashboards/listener.json:/etc/grafana/dashboards/listener.json:ro + - ./monitoring/grafana/dashboards/listener-fleet.json:/etc/grafana/dashboards/listener-fleet.json:ro + - grafana_data:/var/lib/grafana + depends_on: + - prometheus + restart: unless-stopped + +networks: + listener: + driver: bridge + +volumes: + postgres_data: + redis_data: + rabbitmq_data: + prometheus_data: + grafana_data: diff --git a/listener/docs/abstract.md b/listener/docs/abstract.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/listener/docs/assets/assets.md b/listener/docs/assets/assets.md new file mode 100644 index 0000000000..03c8c1fc43 --- /dev/null +++ b/listener/docs/assets/assets.md @@ -0,0 +1 @@ +### Folder containing assets, schemes etc. diff --git a/listener/docs/assets/listener_v2_algorithm.md b/listener/docs/assets/listener_v2_algorithm.md new file mode 100644 index 0000000000..9e5d2d9cf6 --- /dev/null +++ b/listener/docs/assets/listener_v2_algorithm.md @@ -0,0 +1,152 @@ +# Listener Core - Algorithm V2 (Cursor Algorithm) + +## Overview + +The Cursor Algorithm solves the block production speed problem by parallelizing block fetching while maintaining sequential validation through a cursor. + +## Architecture Diagram + +```mermaid +flowchart TB + subgraph INIT["Initialization"] + DB[(Database)] -->|"Last canonical block hash"| TIP[Chain Tip Hash] + RPC[RPC Node] -->|"eth_blockNumber"| HEIGHT[Current Height] + TIP --> RANGE[Calculate Block Range] + HEIGHT --> RANGE + end + + subgraph TASK1["Task 1: Parallel Poller"] + RANGE -->|"spawn N tasks"| P1[Poller 1] + RANGE --> P2[Poller 2] + RANGE --> P3[Poller 3] + RANGE --> PN[Poller N] + + P1 -->|"fetch block by number + receipts + compute"| RPC1[RPC Call] + P2 -->|"fetch block by number + receipts + compute"| RPC2[RPC Call] + P3 -->|"fetch block by number + receipts + compute"| RPC3[RPC Call] + PN -->|"fetch block by number + receipts + compute"| RPCN[RPC Call] + + RPC1 -->|"set_once(0)"| BUFFER + RPC2 -->|"set_once(1)"| BUFFER + RPC3 -->|"set_once(2)"| BUFFER + RPCN -->|"set_once(N)"| BUFFER + end + + subgraph BUFFER_ZONE["AsyncSlotBuffer (In-Memory)"] + BUFFER[/"Slot 0 | Slot 1 | Slot 2 | ... | Slot N"/] + end + + subgraph TASK2["Task 2: Cursor (Sequential Validator)"] + BUFFER -->|"get(i) - waits if empty"| CURSOR[Cursor] + CURSOR -->|"compare hashes"| CHECK{parent_hash == prev_hash?} + + CHECK -->|"YES"| BROADCAST[Broadcast to Message Broker] + BROADCAST --> NEXT[Move to next slot] + NEXT -->|"i++"| CURSOR + + CHECK -->|"NO - REORG!"| REORG[Reorg Handler] + end + + subgraph REORG_HANDLING["Reorg Handling"] + REORG -->|"1. Stop cursor"| STOP[Halt Progression] + STOP -->|"2. Backtrack by hash"| BACKTRACK[Fetch blocks + receipt + compute by hash] + BACKTRACK -->|"3. Mark old as UNCLE"| UNCLE[Update Status] + UNCLE -->|"4. Restart from block N"| RESTART[New Iteration] + RESTART --> RANGE + end + + subgraph OUTPUT["Output"] + BROADCAST -->|"blocks"| QUEUE1[Block Queue] + BROADCAST -->|"transactions + receipts"| QUEUE2[Transaction Queue] + end + + NEXT -->|"buffer exhausted"| NEWITER[Start Next Iteration] + NEWITER -->|"keep last block as tip"| RANGE +``` + +## Sequence Diagram + +```mermaid +sequenceDiagram + participant DB as Database + participant Main as Main Loop + participant Pollers as Parallel Pollers + participant Buffer as AsyncSlotBuffer + participant Cursor as Cursor + participant Broker as Message Broker + + Main->>DB: Get last canonical block hash + DB-->>Main: chain_tip_hash + Main->>Main: Calculate range [start, start+batch_size] + + par Parallel Block Fetching + Main->>Pollers: Spawn N parallel tasks + Pollers->>Buffer: set_once(0, block_0) [arrives T+500ms] + Pollers->>Buffer: set_once(1, block_1) [arrives T+100ms] + Pollers->>Buffer: set_once(2, block_2) [arrives T+800ms] + Note over Pollers,Buffer: Blocks arrive in random order + end + + loop Sequential Validation + Cursor->>Buffer: get(i) - blocks until filled + Buffer-->>Cursor: block_i + + alt parent_hash matches + Cursor->>Cursor: Validate hash chain + Cursor->>Broker: Broadcast block + txs + Cursor->>Cursor: i++, prev_hash = block.hash + else REORG DETECTED + Cursor->>Main: Signal reorg at block N + Main->>Main: Backtrack by hash + Main->>DB: Mark old blocks as UNCLE + Main->>Main: Restart iteration from N + end + end + + Cursor->>Main: Buffer exhausted + Main->>Main: Next iteration (keep last block) +``` + +## Component State Diagram + +```mermaid +stateDiagram-v2 + [*] --> Initializing: Start + + Initializing --> Polling: Calculate range + + Polling --> Filling: Spawn parallel fetchers + Filling --> Filling: Blocks arriving (random order) + + Filling --> Validating: Cursor starts reading + + Validating --> Validating: Hash matches, broadcast + Validating --> ReorgDetected: Hash mismatch! + + ReorgDetected --> Backtracking: Stop cursor + Backtracking --> Backtracking: Fetch by hash + Backtracking --> Polling: Canonical chain rebuilt + + Validating --> Polling: Buffer exhausted, next batch +``` + +## Key Components + +| Component | Status | Description | +| ------------------- | -------------- | ------------------------------------------------------ | +| `AsyncSlotBuffer` | ✅ Implemented | Thread-safe buffer for parallel write, sequential read | +| `EvmBlockFetcher` | ✅ Implemented | 5 strategies for fetching blocks + receipts | +| `EvmBlockComputer` | ✅ Implemented | Block hash verification (receiptRoot, txRoot) | +| `SemEvmRpcProvider` | ✅ Implemented | Semaphore-controlled RPC provider | +| Cursor Loop | ⚠️ Simulated | `slot_buffer_sim_flow.rs` demo | +| Reorg Handler | ❌ Not Yet | Backtracking logic | +| Message Broker | ❌ Not Yet | Redis Streams / RabbitMQ integration | +| Database Layer | ❌ Not Yet | Block status persistence (CANONICAL/UNCLE) | + +## Algorithm Properties + +- **Parallel Fetching**: Overcomes HTTP latency for fast chains (Arbitrum, Monad) +- **Sequential Validation**: Cursor ensures hash chain integrity +- **Reorg Safe**: Detects reorgs via parent_hash comparison +- **Cancellation Support**: CancellationToken propagates through all tasks +- **Rate Limit Friendly**: Semaphore controls concurrent RPC calls diff --git a/listener/docs/dashboard.md b/listener/docs/dashboard.md new file mode 100644 index 0000000000..d5ed1aa687 --- /dev/null +++ b/listener/docs/dashboard.md @@ -0,0 +1,255 @@ +# Listener Grafana Dashboards + +Two dashboards ship with the project: + +| File | Dashboard | Audience | +|---|---|---| +| `monitoring/grafana/dashboards/listener.json` | **Evm Listener - Per Chain** | On-call engineer: deep dive into a single chain | +| `monitoring/grafana/dashboards/listener-fleet.json` | **Evm Listener - Fleet and Infrastructure** | SRE lead: cross-chain health + shared RPC/broker | + +Both are valid Grafana 12.x exports. They consume metrics described in [`docs/metrics.md`](./metrics.md). + +--- + +## Why two dashboards? + +The listener emits **three classes of metrics** that label their data differently: + +| Class | Example metrics | Intrinsic labels | +|---|---|---| +| **Chain-intrinsic** | `listener_cursor_iterations_total`, `listener_reorgs_total`, `listener_*_block_number`, `listener_*_fetch_duration_seconds`, `listener_transient/permanent/publish_errors_total` | `chain_id` | +| **RPC provider** | `listener_rpc_*` | `method`, `endpoint` (no `chain_id`) | +| **Broker** | `broker_*` | `topic`, `backend`, `outcome`, ... (no `chain_id`) | + +If we put all three classes on the same dashboard filtered by `$chain_id`, the RPC and broker panels would show **"No data"** because those metrics don't carry `chain_id`. The broker is a shared Redis/RabbitMQ instance, and RPC metrics aggregate across listener instances. + +The clean solution: **split along the real data boundary.** + +- `listener.json` = only chain-intrinsic metrics, filtered by `$chain_id`. Every panel always has data. +- `listener-fleet.json` = cross-chain summary (by `chain_id`) + shared RPC + shared broker. No chain filter on RPC/broker panels. + +### Note: `chain_id` alone, no `network` label + +`chain_id` is the EIP-155 network identifier — uniquely identifying each EVM network (`1` = Ethereum Mainnet, `11155111` = Sepolia, etc.). Sorting and grouping by `chain_id` is sufficient; the dashboards deliberately do NOT rely on a separate `network` label, so they work without any special Prometheus scrape configuration. + +--- + +## Import + +### Option A — Auto-provisioning (recommended) + +The provisioning config at `monitoring/grafana/dashboards/default.yml` watches the `dashboards/` folder and imports every JSON file automatically. Both dashboards are already mounted into the Grafana container by `docker-compose.yaml` (service `grafana`, `profiles: [monitoring]`). + +```bash +docker compose --profile monitoring up +# Grafana → http://localhost:3000 (admin/admin) +``` + +### Option B — Manual upload + +Grafana UI → **Dashboards** → **New** → **Import** → upload the JSON file → pick the Prometheus datasource. + +--- + +## Dashboard 1 — Evm Listener - Per Chain + +**`listener.json`**, uid: `listener-per-chain` + +### Template variables + +| Variable | Type | Purpose | +|---|---|---| +| `datasource` | datasource | Prometheus datasource | +| `chain_id` | query, single-select | Pick which chain to view. Populated from `label_values(listener_cursor_iterations_total, chain_id)` | + +### Rows + +1. **Overview** (5 stats, always expanded): Chain Height, DB Tip, Sync Lag, Cursor Rate (per min), Reorgs 24h — all filtered by `$chain_id`. +2. **Cursor & Sync**: block heights (chain vs DB), sync-lag timeseries, cursor iteration rate, reorgs bar chart. +3. **Block Fetch Performance**: block-fetch heatmap + p50/p95/p99, range-fetch heatmap + p50/p95/p99. +4. **Listener Errors**: transient by kind, permanent by kind, publish errors rate. +5. **Block Compute Verification**: two sub-sections per failure type (transaction root, receipt root, block hash): + - **24h counters** (stat panels, top row): total number of compute failures over the last 24 hours — quick "has anything gone wrong today?" glance. + - **Rate timeseries** (bottom row): failure rate split by `stalling` label — `stalling=false` are skipped permissively (data quality issues), `stalling=true` are hard halts (invariant concerns). + +### Healthy vs degraded readings + +| Panel | Healthy | Degraded | Action | +|---|---|---|---| +| Chain Height | monotonically increasing | flat | check RPC panels in fleet dashboard | +| DB Tip | monotonically increasing | flat, far below chain height | check fetch performance row | +| Sync Lag | `< 5` blocks (green) | `≥ 20` (red) | check block/range fetch duration | +| Cursor Rate | `> 0` iterations/min | `0` (red) | cursor stalled — check errors row + fleet RPC panels | +| Reorgs 24h | chain-dependent | unexpected spike | correlate with upstream chain events | +| Transient errors | low and bursty | sustained rate | usually RPC or broker infra — jump to fleet dashboard | +| Permanent errors | **always zero** | any non-zero | invariant violation = logic bug, investigate immediately | +| Publish errors | zero | sustained > 0 | broker outage or missing consumer queue — jump to fleet broker row | +| Compute failures 24h (counter) | 0 on strict chains; small/chain-dependent on L2s with skipping | sustained increase | cross-check with rate panel to see if it's a burst or a trend | +| Compute failures rate (stalling=false) | low, chain-dependent | sustained rate | investigate RPC data quality, likely unsupported L2 tx types | +| Compute failures rate (stalling=true) | **always zero** | any non-zero | block verification failing hard — check RPC node or block computer encoding | + +--- + +## Dashboard 2 — Evm Listener - Fleet and Infrastructure + +**`listener-fleet.json`**, uid: `listener-fleet` + +### Template variables + +| Variable | Type | Applies to | +|---|---|---| +| `datasource` | datasource | all panels | +| `endpoint` | query, multi, all | RPC row only — filter by RPC provider host | +| `topic` | query, multi, all | Broker rows — filter by routing key | +| `backend` | query, multi, all | Broker rows — `redis` / `amqp` | + +No `chain_id` variable — this dashboard is deliberately cross-chain. + +### Rows + +#### Fleet Overview + +- **Fleet Health (table)**: one row per `chain_id` with columns `chain height`, `db tip`, `sync lag`, `cursor/min`, `reorgs 24h`, `transient 1h`, `permanent 1h`. Sorted by sync lag descending; conditional coloring flags degraded chains. **This is the go-to "how is the fleet doing?" view.** +- **Sync Lag per Chain** (timeseries): one series per chain, `listener_chain_height_block_number - listener_db_tip_block_number`. +- **Cursor Iteration Rate per Chain**: detect stalls across fleet. +- **Reorgs per Chain (1h buckets)**: spike detector. +- **Listener Errors per Chain**: transient / permanent / publish stacked, one series per `(chain_id, kind)`. + +Click **"Per-chain deep dive"** (top-right link) to jump to `listener.json` for a single chain. + +#### RPC Provider (shared across chains) + +- **RPC Request Rate** by `(method, endpoint)`: traffic profile. +- **RPC Success Ratio by endpoint**: `success / (success+error)` per provider. Compare Alchemy vs Infura vs self-hosted at a glance. +- **RPC p95 Latency** by `(method, endpoint)`: per-provider slowness. +- **RPC Errors** by `(endpoint, error_kind)`: error taxonomy. +- **RPC Semaphore Available by endpoint**: saturation indicator (0 = saturated). +- **Top Failing (method, endpoint, error_kind) — 24h**: instant diagnosis table. + +#### Broker — Publishing (collapsed) + +Publish rate by topic, errors by kind, duration heatmap, p95 by topic. + +#### Broker — Consuming (collapsed) + +Consumed rate by outcome (`ack`/`nack`/`dead`/`delay`/`transient`/`permanent`), handler duration heatmap + p95, dead-letter by reason, delivery count distribution. + +#### Broker — Queue Depth (collapsed) + +Principal / retry / dead-letter / pending / lag, one series per topic. **DLQ depth climbing = messages systematically failing.** + +#### Broker — Circuit Breaker & Connection (collapsed) + +Breaker state (0=closed, 1=open, 2=half-open), trips, consecutive failures, consumer connected stat, reconnection rate, claim sweeper stats. + +--- + +## Multi-chain deployment + +Deploy one listener per chain, each with its own config pointing to its RPC provider. Each listener emits metrics with its own `chain_id` label. Prometheus automatically aggregates them; both dashboards will populate: + +- `listener.json` → `chain_id` dropdown auto-populates from `label_values(listener_cursor_iterations_total, chain_id)`. +- `listener-fleet.json` → fleet table auto-grows one row per chain; RPC/broker rows aggregate across all listener instances. + +No scrape-config gymnastics required. Just add one scrape job per chain to `monitoring/prometheus/prometheus.yml`: + +```yaml +scrape_configs: + - job_name: "listener-ethereum" + static_configs: + - targets: ["listener-eth:9090"] + + - job_name: "listener-sepolia" + static_configs: + - targets: ["listener-sep:9090"] +``` + +The dashboards don't need any external `chain_id` label — they read the intrinsic `chain_id` label emitted by the listener itself. + +> Optional: if you want to display a human-readable network name alongside `chain_id`, add a `network: "ethereum-mainnet"` label to the scrape job. Nothing in the dashboards uses it today, but it's a harmless free-form annotation you can reference in alerts or custom panels. + +--- + +## Suggested Prometheus alert rules + +Add to `monitoring/prometheus/alert.rules`: + +```yaml +groups: + - name: listener + rules: + - alert: ListenerCursorStall + expr: rate(listener_cursor_iterations_total[5m]) == 0 + for: 5m + labels: { severity: critical } + annotations: + summary: "Listener cursor stalled on chain {{ $labels.chain_id }}" + + - alert: ListenerSyncLagHigh + expr: (listener_chain_height_block_number - listener_db_tip_block_number) > 50 + for: 10m + labels: { severity: warning } + annotations: + summary: "Sync lag > 50 blocks on chain {{ $labels.chain_id }}" + + - alert: ListenerReorgStorm + expr: increase(listener_reorgs_total[1h]) > 10 + labels: { severity: warning } + annotations: + summary: "More than 10 reorgs in the last hour on chain {{ $labels.chain_id }}" + + - alert: ListenerPermanentError + expr: increase(listener_permanent_errors_total[5m]) > 0 + labels: { severity: critical } + annotations: + summary: "Permanent (invariant) error on chain {{ $labels.chain_id }}: {{ $labels.error_kind }}" + + - alert: ListenerRpcErrorRateHigh + expr: | + sum by (endpoint) (rate(listener_rpc_errors_total[5m])) + / + sum by (endpoint) (rate(listener_rpc_requests_total[5m])) + > 0.05 + for: 5m + labels: { severity: warning } + annotations: + summary: "RPC error rate > 5% on {{ $labels.endpoint }}" + + - alert: ListenerRpcSemaphoreExhausted + expr: listener_rpc_semaphore_available == 0 + for: 2m + labels: { severity: warning } + annotations: + summary: "RPC semaphore saturated on {{ $labels.endpoint }}" + + - alert: BrokerCircuitBreakerOpen + expr: broker_circuit_breaker_state == 1 + for: 2m + labels: { severity: critical } + annotations: + summary: "Broker circuit breaker OPEN on {{ $labels.topic }}" + + - alert: BrokerDlqGrowing + expr: deriv(broker_queue_depth_dead_letter[15m]) > 0 + for: 15m + labels: { severity: warning } + annotations: + summary: "Dead-letter queue growing on {{ $labels.topic }}" +``` + +--- + +## Customization + +- **Rename per environment**: change `title` and `uid` fields at the bottom of each JSON (e.g. `"Evm Listener [staging] - Per Chain"`). +- **Add a new chain**: no dashboard changes needed — the `chain_id` variable in `listener.json` auto-populates, and the fleet table auto-grows one row. +- **Change refresh / default time range**: edit the `refresh` and `time` fields at the bottom of each JSON. +- **Add panels**: open the dashboard in Grafana UI → edit → save → export JSON → replace the file. Panel `id` values must be unique within the dashboard. + +--- + +## Related docs + +- [`docs/metrics.md`](./metrics.md) — raw metric definitions, labels, Grafana query examples +- `monitoring/prometheus/prometheus.yml` — scrape config with commented multi-chain example +- `monitoring/grafana/dashboards/default.yml` — Grafana provisioning config diff --git a/listener/docs/guidelines.md b/listener/docs/guidelines.md new file mode 100644 index 0000000000..82abcdfc17 --- /dev/null +++ b/listener/docs/guidelines.md @@ -0,0 +1,15 @@ +# Guidelines + +The rationnale is: GIVE NO ROOM TO MISS SOMETHING (a block, a transaction, a receipt, a log), crash or skip a processing is not permitted. + +- MUST NEVER panic ! +- No uncontrolled unwraps. +- Should retry indefinitely most of the time if there is a encountered error and raise an alert if needed to look after it. +- Be consistent in error management, anyhow or box dyn +- Think alerting. +- Think profiling if needed. +- Think strategy pattern. +- Think separations of concerns regarding logic. +- Think reusable code. + +(The rationnale is: IMPOSSIBILITY TO MISS SOMETHING, crash or skip a processing) diff --git a/listener/docs/infra.md b/listener/docs/infra.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/listener/docs/library_notifier.md b/listener/docs/library_notifier.md new file mode 100644 index 0000000000..f3de0f05d9 --- /dev/null +++ b/listener/docs/library_notifier.md @@ -0,0 +1,70 @@ +# Notifier library + +## Goal + +Integrate event consuming for specific logs and filters for zama components, and mimic the behaviour of a poller or a websocket stream, but from the listener standalone component. + +## Logic: + +This component consume blocks, transasctions, receipts from the different queues declared on rabbitmq, checks into its table to see if there is relevant filters or abi to watch, or even "from" or "to" sources if we need to watch for some transactions, register logs in a table, and forward to the internal logic of the components that need logs. + +Basically, the library is receipt parser. +You refer a watcher, and match the current watcher from the receipt if we need to get process an event. + +For inspiration regarding this library, There is an existing implementation for logic, ask for access if needed. + +## Features: + +### General: + +- Rust library. +- Resilient to HPA: (Ok by design: consuming from rmq) +- Clear api for subscribe and consume events from the rabbitmq. +- Find the ability to call the consume functions and pass hooks or handlers to trigger zama internal logic (should be easily integrable to existing components). +- Prepare strategy pattern for Solana. + +### Queue consumer: + +- Use Shared rmq library between listener, and notifier library, with retry queues, etc. + +### Data storage: + +- RDS database (Could leverage on rds for different components) +- postgres tables for logs and different watchers. + +### Blockchain related: + +#### Minimal features: + +- Persist block height and resilient to failure mode. +- Multichain by design (consuming multiple queues (blocks, transactions with receipts -> e.g logs) for each networks). +- Should consume all events even if they are not used (or rmq memory will grow). +- Declare notifiers for dynamical events ABI + - Store log watchers types into a postgres. + - Store watched logs into a postgres. + - Declare new watchers dynamically. + +#### Should have features: + +- Declare multiple watcher with a number of block confirmations if a block confirmation number is required (RPC url could be required for this). (e.g finality, safe or n confirmations blocks) based on events. +- Ability to be aware of new available chain from rabbitmq. +- different types of watchers (logs, tx) +- OPTIONAL: Cancel reorged events. +- OPTIONAL: Replay past blocks (should not need this with rmq, since its queuing messages). +- Check altogether if problems with duplicate logs, and how to manage them in the zama internals (could be handled optionally) To get a unicity regarding logs and handle deduplication, we can if needed apply a semantic hash regarding log. +- Metrics +- Alerting. + +### Example: + +- Should implement a minimal working example on how to use the library. + +### Postgres table: + +- table 1: watcher + - uuid, chainId, number of conf block ?, ABI, watcher type (tx, contract) +- table 2: logs + - uuid, watcher_uuid, block_number, released (TRUE, FALSE), log (deserialised or not), UNCLE? (Not mandatory if leverage on block confirmations) + +## Schemes: + diff --git a/listener/docs/listener_core.md b/listener/docs/listener_core.md new file mode 100644 index 0000000000..18e9222ade --- /dev/null +++ b/listener/docs/listener_core.md @@ -0,0 +1,92 @@ +# Listener Core + +## Problematics + +- TOP PRIORITY: Miss zero events. +- Simple Reorg: Parent hash different previous hash +- Back and forth reorgs: reorged branch becomes canonical again +- Multi branching reorgs: Multiple reorgs can be detected at the same time, and only one will become canonical. +- Must be trustless as much as possible regarding the RPC node we are hitting on, or the load balancer we are hitting on that serves multiple nodes +- Caching problem, corrupted data etc... + +## Common knowledge: + +- Counter-intuitively, it is always the fresher information that will bring us the truth, especially regarding the past. +- Transaction receipts contains all the logs +- ReceiptRoot calculation, and block hash calculation ensure there is no missing logs for a given block. +- Zero websocket, not resilient. + +## Goal + +The goal of this core algorithm is to fetch (http polling), blocks, transactions, receipt, by polling, handle reorogs properly by checking hash and parent hash, fetch new information if needed to be consistent and aware of canonical chain and broadcast blocks, and transaction to the message broker for the library to be aware of new events. + +## Logic / Algorithms + +### Algorithm v1: Sequential poller and reorg checker + +This is a descriptive of a basic algorithm, which could be sufficient with chains that produces blocks in more time than an http call duration. +This algorithm is sequential, and is just referred here for knowledge. + +If you need access to an existing implementation of this algorithm, ask and I will share you the implementation. + +This algorithm leverages mostly on database, to perform checks, states updates, and branching. + +1. polling loop for getting the next block +2. we register block, transactions of this block, and associated receipt. + 1. The receipt contains all the logs. + 2. We broadcast the block, and the transactions with receipts to given queues with chainId, to get almost real time performance, for being consumed and filtered by the library notifier over abi filter and contract address +3. we compare current block parent hash, and previous block parent hash to detect if a reorg occurred. + 1. if it matches, we go back to the beginning of the algorithm. + 2. if it didn't match: Reorg is detected. + 1. we fetch one by one all the previous blocks by hash, we broadcast events in the same fashion we did previously. (BACKTRACKING) + 2. pass the other ones to UNCLES status. + 3. Optionally, we broadcast cancelation events for old blocks for the library, but its not needed + 4. Then we go to 1. with the next iteration to fetch new blocks. + +### Algorithm v2: Cursor Algorithm + +The major flaw with the v1 iterative poller algorithm, is the block production time for faster chains, such as Arbitrum, Monad, or even Solana later could be faster than a single http response call, database operations, and network time calls if levraging on rabbitmq to trigger block fetch and polling operations, cumulated operations could be more than 100/200ms only in average time. It does not keep up with chain with a smaller block time duration. +Also, if later a full chain indexer is needed, this is impossible to leverage on the first algorithm. + +Here is the proposed algorithm to address this flaw. + +#### task one: parallel poller + +Resolving the http latency, and ensure no event is missed. + +1. We calculate a block range: + 1. `min(blockHeight - currentRegisteredBlock, maxParallelBlockFetch + currentRegisteredBlock)` + 2. or range given from an order to fetch the next block. +2. We spawn parallel task to fetch blocks (http polling), and register them in an in memory datastructure (slots for new blocks). And we fetch receipt for those blocks (strategy pattern could be required for diffrernt chains implems (`eth_getBlockReceipts` or `eth_getTransactionReceipt`for each transaction)) +3. Optional: we recompute block hash: The rationale behind this, calculate receiptRoot and then block hash from receipt root and all other headers: this ensure that there is no inconsistency in receipts, hence logs contained in the receipts. + +#### task two: cursor, reorg check and event broadcaster + +1. The cursor, check the data structure up to its length, comparing current parent hash and previous hash, and wait if there is no block yet available at a certain slot to continue its progression. + 1. we broadcast blocks and transaction to the message broker. +2. if no reorg, continue and check the next block with strategy described in 1. +3. if a reorg is detected + 1. the cursor stops its progression. + 2. we backtrack from block `n` to handle the reorg and get matching blocks by hash sequentially until the canonical chain is build again, we pass previous data to uncle (refer to algo 1) in a new data structure. + 3. and we start a new task one to get fresher information from block `n`, and a new cursor strategy on top of this one. + +4. When the cursor arrives to the end of the data structure, it launches the next iteration for the task one parallel poller, and keep at least the latest block from the previous iteration to compare hash and parent hash. + +## Features + +- Data structure for getting in memory blocks +- Event driven system to react to multiple events for the algorithm described above. +- strategy pattern (handling chains that doesn't support `eth_getBlockReceipts` method, and solana later) +- algorithm v2 implementation with eth_getBlockReceipt first. +- tables to store minimal metadata (blocks, transactions and receipts) with status (CANONICAL, UNCLE). +- Sql cleaner feature. +- OPTIONAL: Finalized status. +- Block computer. +- OPTIONAL: data storage layer no-sql or S3. +- push rabbitmq messages. + +## Additional: + + + +## Schemes diff --git a/listener/docs/listener_overview.md b/listener/docs/listener_overview.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/listener/docs/load_balancer.md b/listener/docs/load_balancer.md new file mode 100644 index 0000000000..10b86a5684 --- /dev/null +++ b/listener/docs/load_balancer.md @@ -0,0 +1,4 @@ +# Load balancer + +## Goal: +Solve SPOF regarding nodes, and mitigate response inconsistency. \ No newline at end of file diff --git a/listener/docs/metrics.md b/listener/docs/metrics.md new file mode 100644 index 0000000000..a2bdf4c87d --- /dev/null +++ b/listener/docs/metrics.md @@ -0,0 +1,222 @@ +# Listener Metrics Catalog + +Prometheus metrics exposed on the `/metrics` endpoint (default `:9090`). +Enabled via `telemetry.enabled: true` in config. + +--- + +## Listener Core Metrics + +### Cursor Liveness + +| Metric | Type | Labels | Description | +|--------|------|--------|-------------| +| `listener_cursor_iterations_total` | Counter | `chain_id` | Total main cursor loop iterations. **Stall detection:** `rate(...[5m]) == 0` means the cursor is stuck. | + +### Chain Sync + +| Metric | Type | Labels | Description | +|--------|------|--------|-------------| +| `listener_db_tip_block_number` | Gauge | `chain_id` | Latest canonical block number persisted in the database. | +| `listener_chain_height_block_number` | Gauge | `chain_id` | Latest block number reported by the RPC node. | + +**Sync lag** = `listener_chain_height_block_number - listener_db_tip_block_number` + +### Reorgs + +| Metric | Type | Labels | Description | +|--------|------|--------|-------------| +| `listener_reorgs_total` | Counter | `chain_id` | Total chain reorganizations detected by the cursor. | + +### Fetch Timing + +| Metric | Type | Labels | Description | +|--------|------|--------|-------------| +| `listener_block_fetch_duration_seconds` | Histogram | `chain_id` | Wall-clock time to fetch a single block (RPC call + receipts). | +| `listener_range_fetch_duration_seconds` | Histogram | `chain_id` | Wall-clock time to fetch and process an entire block range (producer + consumer). | + +### Publishing + +| Metric | Type | Labels | Description | +|--------|------|--------|-------------| +| `listener_publish_errors_total` | Counter | `chain_id` | Failures when publishing block events to the broker. Incremented per failed publish attempt (after broker-level retries are exhausted). | + +### Error Classification + +| Metric | Type | Labels | Description | +|--------|------|--------|-------------| +| `listener_transient_errors_total` | Counter | `chain_id`, `error_kind` | Transient (infrastructure) errors from the handler error classifier. These trigger circuit breaker and broker retries. | +| `listener_permanent_errors_total` | Counter | `chain_id`, `error_kind` | Permanent (logic) errors from the handler error classifier. These are dead-lettered without retry. | + +**`error_kind` label values:** + +| Value | Source Error | Transient/Permanent | +|-------|-------------|---------------------| +| `block_fetch` | `CouldNotFetchBlock` | Transient | +| `block_compute` | `CouldNotComputeBlock` | Transient | +| `database` | `DatabaseError` | Transient | +| `chain_height` | `ChainHeightError` | Transient | +| `slot_buffer` | `SlotBufferError` | Transient | +| `broker_publish` | `BrokerPublishError` | Transient | +| `payload_build` | `PayloadBuildError` | Transient | +| `message_processing` | `MessageProcessingError` | Transient | +| `invariant_violation` | `InvariantViolation` | Permanent | + +### Block Compute Verification + +| Metric | Type | Labels | Description | +|--------|------|--------|-------------| +| `listener_compute_block_failure_total` | Counter | `chain_id`, `stalling` | Block hash verification failures. `stalling=true` means the listener halted on this error; `stalling=false` means it was skipped (allow_skipping mode). | +| `listener_compute_transaction_failure_total` | Counter | `chain_id`, `stalling` | Transaction root verification failures. Same `stalling` semantics. | +| `listener_compute_receipt_failure_total` | Counter | `chain_id`, `stalling` | Receipt root verification failures. Same `stalling` semantics. | + +**`stalling` label values:** + +| Value | Meaning | +|-------|---------| +| `true` | `compute_block_allow_skipping: false` — verification failure stalls the listener (error propagated) | +| `false` | `compute_block_allow_skipping: true` — verification failure logged and skipped (permissive mode) | + +--- + +## RPC Provider Metrics (`SemEvmRpcProvider`) + +| Metric | Type | Labels | Description | +|--------|------|--------|-------------| +| `listener_rpc_request_duration_seconds` | Histogram | `method`, `endpoint` | Wall-clock time per JSON-RPC call (includes semaphore wait time). | +| `listener_rpc_requests_total` | Counter | `method`, `endpoint`, `status` | Total RPC requests. `status` is `success` or `error`. | +| `listener_rpc_errors_total` | Counter | `method`, `endpoint`, `error_kind` | RPC errors broken down by failure type. | +| `listener_rpc_semaphore_available` | Gauge | `endpoint` | Available permits in the RPC concurrency semaphore. Low values indicate RPC saturation. | + +### `endpoint` label + +The `endpoint` label is the **host** portion of the configured `rpc_url`, extracted +via `url::Url::host_str()` at provider construction time (falling back to +`"unknown"` if the URL cannot be parsed). + +**Examples:** + +| Configured `rpc_url` | Emitted `endpoint` label | +|----------------------|--------------------------| +| `https://ethereum-rpc.publicnode.com` | `ethereum-rpc.publicnode.com` | +| `https://eth-mainnet.g.alchemy.com/v2/MY_SECRET_KEY` | `eth-mainnet.g.alchemy.com` | +| `https://mainnet.infura.io/v3/MY_PROJECT_ID` | `mainnet.infura.io` | + +API keys embedded in the URL **path** or **query** are deliberately NOT emitted — +only the host is exposed. This keeps Prometheus scrapes free of secrets. + +**Use cases:** +- Compare latency and error rates across multiple RPC providers (Alchemy vs Infura vs self-hosted). +- Detect provider-specific outages (one endpoint's error rate spikes while others are healthy). +- Attribute semaphore saturation to a specific endpoint when rotating providers. + +### `method` label values + +`eth_blockNumber`, `eth_chainId`, `eth_getBlockByNumber`, `eth_getBlockByHash`, +`eth_getTransactionReceipt`, `eth_getBlockReceipts`, `eth_getTransactionReceipt_batch` + +**`error_kind` label values:** + +| Value | Description | +|-------|-------------| +| `deserialization` | Response could not be deserialized (likely node bug or schema mismatch) | +| `unsupported_method` | RPC method not supported by the node | +| `rate_limited` | HTTP 429 or rate-limit error from the node | +| `transport` | Network/connection failure | +| `not_found` | Block or receipt returned null | +| `batch_error` | Batch request failed (HTTP error or parse failure) | +| `batch_unsupported` | Node does not support batch JSON-RPC | + +--- + +## Broker Metrics (reference) + +The `broker` crate emits its own metrics (prefixed `broker_*`). +See `crates/shared/broker/src/metrics.rs` for the full catalog, including: +- `broker_messages_published_total` +- `broker_publish_errors_total` +- `broker_publish_duration_seconds` +- `broker_messages_consumed_total` +- `broker_handler_duration_seconds` +- `broker_circuit_breaker_state` +- `broker_queue_depth_*` + +--- + +## Grafana Query Examples + +### Sync lag (blocks behind chain tip) +```promql +listener_chain_height_block_number - listener_db_tip_block_number +``` + +### Cursor stall alert (no iterations in 5 minutes) +```promql +rate(listener_cursor_iterations_total[5m]) == 0 +``` + +### Reorg rate (per hour) +```promql +increase(listener_reorgs_total[1h]) +``` + +### P99 block fetch latency +```promql +histogram_quantile(0.99, rate(listener_block_fetch_duration_seconds_bucket[5m])) +``` + +### RPC error rate by method and endpoint +```promql +sum by (method, endpoint) (rate(listener_rpc_errors_total[5m])) +``` + +### RPC latency by method and endpoint (P95) +```promql +histogram_quantile(0.95, sum by (method, endpoint, le) (rate(listener_rpc_request_duration_seconds_bucket[5m]))) +``` + +### Per-endpoint error ratio (compare RPC providers) +```promql +sum by (endpoint) (rate(listener_rpc_errors_total[5m])) + / +sum by (endpoint) (rate(listener_rpc_requests_total[5m])) +``` + +### Transient error rate by kind +```promql +sum by (error_kind) (rate(listener_transient_errors_total[5m])) +``` + +### RPC semaphore saturation +```promql +listener_rpc_semaphore_available == 0 +``` + +### Compute verification failure rate (by stalling) +```promql +sum by (stalling) (rate(listener_compute_transaction_failure_total{chain_id="$chain_id"}[5m])) +``` + +### Any stalling compute failure (alert candidate) +```promql +sum(increase(listener_compute_block_failure_total{stalling="true"}[5m]) + + increase(listener_compute_transaction_failure_total{stalling="true"}[5m]) + + increase(listener_compute_receipt_failure_total{stalling="true"}[5m])) by (chain_id) > 0 +``` + +--- + +## Initialization + +All gauges are initialized to `0` at startup so that Grafana discovers the time series +on the first scrape, even before the first cursor iteration completes. + +Block-compute failure counters (`listener_compute_{block,transaction,receipt}_failure_total`) +are **also** pre-seeded at `0` at startup for every `{chain_id, stalling}` combination +(`stalling=true` and `stalling=false`). + +Why: Prometheus `increase()` / `rate()` needs at least two samples in the lookback window +to compute a delta. Without pre-seeding, a counter going from "absent" directly to `1` +on the first failure would make `increase(...[24h])` report `0` — there is no baseline +to compare against. Seeding the series at `0` makes the first real failure show up +immediately as `1` in stat panels and alerts. diff --git a/listener/docs/nodes.md b/listener/docs/nodes.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/listener/docs/spec-block-computer-tx-types.md b/listener/docs/spec-block-computer-tx-types.md new file mode 100644 index 0000000000..c5bf4c2f5e --- /dev/null +++ b/listener/docs/spec-block-computer-tx-types.md @@ -0,0 +1,502 @@ +# Block Computer — Transaction Type Encoding Specification + +**Status**: Specification +**Author**: nboisde +**Date**: 2026-04-14 +**File**: `crates/listener_core/src/blockchain/evm/evm_block_computer.rs` + +--- + +## 1. Problem Statement + +The block computer verifies block integrity by recomputing the transaction trie root +from parsed RPC data and comparing it to the header's `transactionsRoot`. This requires +RLP-encoding every transaction in the block. + +The current implementation only handles: +- Standard Ethereum types 0–4 (via `AnyTxEnvelope::Ethereum` + alloy's `encoded_2718()`) +- Optimism/Base deposit type 126 (0x7E) +- Arbitrum internal type 106 (0x6A) + +Any other type falls into the `_` branch and returns `UnsupportedTransactionType`, which +currently blocks cursor processing. As of block 85,523,136 on Polygon mainnet, type 127 +(0x7F — PIP-74 state sync) causes this failure. + +## 2. Complete Transaction Type Inventory + +### 2.1 Standard Ethereum (types 0–4) + +Handled by alloy natively via `AnyTxEnvelope::Ethereum` → `encoded_2718()`. + +| Type | EIP | Name | Encoding | +|------|-----|------|----------| +| 0 | — | Legacy | RLP([nonce, gasPrice, gasLimit, to, value, data, v, r, s]) | +| 1 | 2930 | Access list | 0x01 + RLP([chainId, nonce, gasPrice, gasLimit, to, value, data, accessList, signatureYParity, signatureR, signatureS]) | +| 2 | 1559 | Dynamic fee | 0x02 + RLP([chainId, nonce, maxPriorityFeePerGas, maxFeePerGas, gasLimit, to, value, data, accessList, signatureYParity, signatureR, signatureS]) | +| 3 | 4844 | Blob | 0x03 + RLP([chainId, nonce, maxPriorityFeePerGas, maxFeePerGas, gasLimit, to, value, data, accessList, maxFeePerBlobGas, blobVersionedHashes, signatureYParity, signatureR, signatureS]) | +| 4 | 7702 | Set code | 0x04 + RLP([chainId, nonce, maxPriorityFeePerGas, maxFeePerGas, gasLimit, to, value, data, accessList, authorizationList, signatureYParity, signatureR, signatureS]) | + +**No action needed** — alloy handles these. + +### 2.2 Optimism / Base (type 126) + +**Status**: Implemented. + +| Type | Name | Encoding | +|------|------|----------| +| 126 (0x7E) | Deposit transaction | 0x7E + RLP([sourceHash, from, to, mint, value, gas, isSystemTx, data]) | + +Source: OP Stack specification. `from` comes from the recovered signer, not from `OtherFields`. + +### 2.3 Arbitrum (types 100–106) + +Source: [OffchainLabs/go-ethereum — core/types/arb_types.go](https://github.com/OffchainLabs/go-ethereum/blob/master/core/types/arb_types.go) + +Arbitrum's modified Geth marshals all custom fields to JSON, so all types below **can** be +encoded from `OtherFields`. + +#### Type 100 (0x64) — ArbitrumDepositTx + +L1→L2 ETH deposit via the bridge. Appears in every block that processes a bridge deposit. + +``` +0x64 + RLP([chainId, l1RequestId, from, to, value]) +``` + +| Field | JSON key | Rust type | Notes | +|-------|----------|-----------|-------| +| chainId | `"chainId"` | u64 | hex string | +| l1RequestId | `"requestId"` | B256 | | +| from | `"from"` | Address | use recovered signer | +| to | `"to"` | Address | always present (not optional) | +| value | `"value"` | U256 | | + +```rust +fn encode_arbitrum_deposit_transaction( + unknown: &UnknownTxEnvelope, + from: Address, + index: usize, +) -> Result, BlockVerificationError> { + let fields = &unknown.inner.fields; + let chain_id: u64 = extract_u64(fields, "chainId", index)?; + let l1_request_id: B256 = extract_field(fields, "requestId", index)?; + let to: Address = extract_field(fields, "to", index)?; + let value: U256 = extract_field(fields, "value", index)?; + + let mut payload = Vec::new(); + chain_id.encode(&mut payload); + l1_request_id.encode(&mut payload); + from.encode(&mut payload); + to.encode(&mut payload); + value.encode(&mut payload); + + encode_typed_tx(0x64, payload) +} +``` + +#### Type 101 (0x65) — ArbitrumUnsignedTx + +L1 user calling an L2 contract via the bridge (unsigned, no signature). + +``` +0x65 + RLP([chainId, from, nonce, gasFeeCap, gas, to, value, data]) +``` + +| Field | JSON key | Rust type | Notes | +|-------|----------|-----------|-------| +| chainId | `"chainId"` | u64 | | +| from | `"from"` | Address | use recovered signer | +| nonce | `"nonce"` | u64 | | +| gasFeeCap | `"maxFeePerGas"` | U256 | Arbitrum maps gasFeeCap to maxFeePerGas in JSON | +| gas | `"gas"` | u64 | | +| to | `"to"` | Option\ | can be None for contract creation | +| value | `"value"` | U256 | | +| data | `"input"` | Bytes | "input" not "data" | + +```rust +fn encode_arbitrum_unsigned_transaction( + unknown: &UnknownTxEnvelope, + from: Address, + index: usize, +) -> Result, BlockVerificationError> { + let fields = &unknown.inner.fields; + let chain_id: u64 = extract_u64(fields, "chainId", index)?; + let nonce: u64 = extract_u64(fields, "nonce", index)?; + let gas_fee_cap: U256 = extract_field(fields, "maxFeePerGas", index)?; + let gas: u64 = extract_u64(fields, "gas", index)?; + let to: Option
= extract_optional_address(fields, "to"); + let value: U256 = extract_field(fields, "value", index)?; + let data: Bytes = extract_field(fields, "input", index)?; + + let mut payload = Vec::new(); + chain_id.encode(&mut payload); + from.encode(&mut payload); + nonce.encode(&mut payload); + gas_fee_cap.encode(&mut payload); + gas.encode(&mut payload); + encode_optional_address(to, &mut payload); + value.encode(&mut payload); + data.encode(&mut payload); + + encode_typed_tx(0x65, payload) +} +``` + +#### Type 102 (0x66) — ArbitrumContractTx + +L1 contract calling an L2 contract. Similar to ArbitrumUnsignedTx but with `requestId` +instead of `nonce`. + +``` +0x66 + RLP([chainId, requestId, from, gasFeeCap, gas, to, value, data]) +``` + +| Field | JSON key | Rust type | Notes | +|-------|----------|-----------|-------| +| chainId | `"chainId"` | u64 | | +| requestId | `"requestId"` | B256 | | +| from | `"from"` | Address | use recovered signer | +| gasFeeCap | `"maxFeePerGas"` | U256 | | +| gas | `"gas"` | u64 | | +| to | `"to"` | Option\ | | +| value | `"value"` | U256 | | +| data | `"input"` | Bytes | | + +```rust +fn encode_arbitrum_contract_transaction( + unknown: &UnknownTxEnvelope, + from: Address, + index: usize, +) -> Result, BlockVerificationError> { + let fields = &unknown.inner.fields; + let chain_id: u64 = extract_u64(fields, "chainId", index)?; + let request_id: B256 = extract_field(fields, "requestId", index)?; + let gas_fee_cap: U256 = extract_field(fields, "maxFeePerGas", index)?; + let gas: u64 = extract_u64(fields, "gas", index)?; + let to: Option
= extract_optional_address(fields, "to"); + let value: U256 = extract_field(fields, "value", index)?; + let data: Bytes = extract_field(fields, "input", index)?; + + let mut payload = Vec::new(); + chain_id.encode(&mut payload); + request_id.encode(&mut payload); + from.encode(&mut payload); + gas_fee_cap.encode(&mut payload); + gas.encode(&mut payload); + encode_optional_address(to, &mut payload); + value.encode(&mut payload); + data.encode(&mut payload); + + encode_typed_tx(0x66, payload) +} +``` + +#### Type 104 (0x68) — ArbitrumRetryTx + +Retry execution of a failed L1→L2 retryable ticket. + +``` +0x68 + RLP([chainId, nonce, from, gasFeeCap, gas, to, value, data, ticketId, refundTo, maxRefund, submissionFeeRefund]) +``` + +| Field | JSON key | Rust type | Notes | +|-------|----------|-----------|-------| +| chainId | `"chainId"` | u64 | | +| nonce | `"nonce"` | u64 | | +| from | `"from"` | Address | use recovered signer | +| gasFeeCap | `"maxFeePerGas"` | U256 | | +| gas | `"gas"` | u64 | | +| to | `"to"` | Option\ | | +| value | `"value"` | U256 | | +| data | `"input"` | Bytes | | +| ticketId | `"ticketId"` | B256 | Arbitrum-specific field | +| refundTo | `"refundTo"` | Address | Arbitrum-specific field | +| maxRefund | `"maxRefund"` | U256 | Arbitrum-specific field | +| submissionFeeRefund | `"submissionFeeRefund"` | U256 | Arbitrum-specific field | + +```rust +fn encode_arbitrum_retry_transaction( + unknown: &UnknownTxEnvelope, + from: Address, + index: usize, +) -> Result, BlockVerificationError> { + let fields = &unknown.inner.fields; + let chain_id: u64 = extract_u64(fields, "chainId", index)?; + let nonce: u64 = extract_u64(fields, "nonce", index)?; + let gas_fee_cap: U256 = extract_field(fields, "maxFeePerGas", index)?; + let gas: u64 = extract_u64(fields, "gas", index)?; + let to: Option
= extract_optional_address(fields, "to"); + let value: U256 = extract_field(fields, "value", index)?; + let data: Bytes = extract_field(fields, "input", index)?; + let ticket_id: B256 = extract_field(fields, "ticketId", index)?; + let refund_to: Address = extract_field(fields, "refundTo", index)?; + let max_refund: U256 = extract_field(fields, "maxRefund", index)?; + let submission_fee_refund: U256 = extract_field(fields, "submissionFeeRefund", index)?; + + let mut payload = Vec::new(); + chain_id.encode(&mut payload); + nonce.encode(&mut payload); + from.encode(&mut payload); + gas_fee_cap.encode(&mut payload); + gas.encode(&mut payload); + encode_optional_address(to, &mut payload); + value.encode(&mut payload); + data.encode(&mut payload); + ticket_id.encode(&mut payload); + refund_to.encode(&mut payload); + max_refund.encode(&mut payload); + submission_fee_refund.encode(&mut payload); + + encode_typed_tx(0x68, payload) +} +``` + +#### Type 105 (0x69) — ArbitrumSubmitRetryableTx + +Creates a retryable ticket with L1→L2 fee escrow. + +``` +0x69 + RLP([chainId, requestId, from, l1BaseFee, depositValue, gasFeeCap, gas, retryTo, retryValue, beneficiary, maxSubmissionFee, feeRefundAddr, retryData]) +``` + +| Field | JSON key | Rust type | Notes | +|-------|----------|-----------|-------| +| chainId | `"chainId"` | u64 | | +| requestId | `"requestId"` | B256 | | +| from | `"from"` | Address | use recovered signer | +| l1BaseFee | `"l1BaseFee"` | U256 | Arbitrum-specific field | +| depositValue | `"depositValue"` | U256 | Arbitrum-specific field | +| gasFeeCap | `"maxFeePerGas"` | U256 | | +| gas | `"gas"` | u64 | | +| retryTo | `"retryTo"` | Option\ | destination on L2 | +| retryValue | `"retryValue"` | U256 | Arbitrum-specific field | +| beneficiary | `"beneficiary"` | Address | Arbitrum-specific field | +| maxSubmissionFee | `"maxSubmissionFee"` | U256 | Arbitrum-specific field | +| feeRefundAddr | `"feeRefundAddr"` | Address | Arbitrum-specific field | +| retryData | `"retryData"` | Bytes | Arbitrum-specific field | + +```rust +fn encode_arbitrum_submit_retryable_transaction( + unknown: &UnknownTxEnvelope, + from: Address, + index: usize, +) -> Result, BlockVerificationError> { + let fields = &unknown.inner.fields; + let chain_id: u64 = extract_u64(fields, "chainId", index)?; + let request_id: B256 = extract_field(fields, "requestId", index)?; + let l1_base_fee: U256 = extract_field(fields, "l1BaseFee", index)?; + let deposit_value: U256 = extract_field(fields, "depositValue", index)?; + let gas_fee_cap: U256 = extract_field(fields, "maxFeePerGas", index)?; + let gas: u64 = extract_u64(fields, "gas", index)?; + let retry_to: Option
= extract_optional_address(fields, "retryTo"); + let retry_value: U256 = extract_field(fields, "retryValue", index)?; + let beneficiary: Address = extract_field(fields, "beneficiary", index)?; + let max_submission_fee: U256 = extract_field(fields, "maxSubmissionFee", index)?; + let fee_refund_addr: Address = extract_field(fields, "feeRefundAddr", index)?; + let retry_data: Bytes = extract_field(fields, "retryData", index)?; + + let mut payload = Vec::new(); + chain_id.encode(&mut payload); + request_id.encode(&mut payload); + from.encode(&mut payload); + l1_base_fee.encode(&mut payload); + deposit_value.encode(&mut payload); + gas_fee_cap.encode(&mut payload); + gas.encode(&mut payload); + encode_optional_address(retry_to, &mut payload); + retry_value.encode(&mut payload); + beneficiary.encode(&mut payload); + max_submission_fee.encode(&mut payload); + fee_refund_addr.encode(&mut payload); + retry_data.encode(&mut payload); + + encode_typed_tx(0x69, payload) +} +``` + +#### Type 106 (0x6A) — ArbitrumInternalTx + +**Status**: Implemented. + +``` +0x6A + RLP([chainId, data]) +``` + +### 2.4 Polygon Bor (type 127) + +Source: [PIP-74 — Canonical Inclusion of StateSync Transactions](https://forum.polygon.technology/t/pip-74-canonical-inclusion-of-statesync-transactions-in-block-bodies/21331) + +Introduced by the **Madhugiri hardfork** (Dec 2024, activation block 80,084,800). + +| Type | Name | Encoding | +|------|------|----------| +| 127 (0x7F) | State sync (PIP-74) | 0x7F + RLP([\[encStateSyncData, ...]]) | + +Each `encStateSyncData`: + +| Field | Type | Description | +|-------|------|-------------| +| ID | uint64 | State sync event ID | +| Contract | address | Receiver contract on L2 | +| Data | bytes | ABI-encoded payload | +| TxHash | bytes32 | L1 transaction hash | + +**Cannot be encoded from JSON RPC** — the inner payload is not in the transaction fields. +The RPC returns a normalized view: `from=0x0, to=0x0, gas=0, input=0x, v/r/s=0`. +The actual state sync data is only available via `eth_getRawTransactionByHash`. + +State sync txs appear on **sprint blocks** (every 16 blocks on Polygon, e.g. block numbers +divisible by 16). + +Example raw bytes for block 85,523,136 tx at index 375: +``` +0x7f <- type byte +f9025f <- RLP list header (607 bytes) + f9013d <- first encStateSyncData + 83 30364e <- ID: 3159630 + 94 a6fa...c0aa <- Contract: 0xa6fa4fb5f76172d178d61b04b0ecd319c5d1c0aa + b90100 87a7811f... <- Data: 256 bytes + a0 378a5f6c...1e1e <- TxHash: 0x378a5f6c... + f9011c <- second encStateSyncData + 83 30364f <- ID: 3159631 + 94 8397...8a28a <- Contract: 0x8397259c983751daf40400790063935a11afa28a + b8e0 0000... <- Data: 224 bytes + a0 458eea10...a51f <- TxHash: 0x458eea10... +``` + +#### Options for supporting type 127 + +**Option A — Skip tx root verification (current approach)** + +When encountering type 0x7F, skip the entire transaction root verification for this block. +Block hash + receipt root verification still run. Block hash verification alone proves header +authenticity (including the `transactionsRoot` stored in the header). + +Tradeoff: a malicious RPC could serve tampered transactions for state-sync-containing blocks +without detection. Acceptable for trusted RPC endpoints. + +**Option B — Fetch raw bytes via `eth_getRawTransactionByHash`** + +Add a `get_raw_transaction_by_hash` method to `SemEvmRpcProvider`. In the fetcher, before +calling verification, scan for unknown types and fetch raw bytes. Pass them to the block +computer via a `HashMap>`. + +Changes required: +- `sem_evm_rpc_provider.rs`: add `get_raw_transaction_by_hash` +- `evm_block_fetcher.rs`: add `fetch_raw_unknown_txs`, make `build_fetched_block` async +- `evm_block_computer.rs`: accept `raw_txs` parameter in `verify_block`/`verify_transaction_root`/`safe_encode_transaction` + +Tradeoff: extra RPC call per state-sync block, architectural changes to fetcher. + +**Option C — Reconstruct from receipt logs** + +The receipt logs contain state sync event IDs (in topics of 0x0000...1001 contract events) +and contract addresses, but NOT the full payload data or L1 tx hash. **Not viable.** + +### 2.5 Other chains + +| Chain | Custom types? | Notes | +|-------|--------------|-------| +| BNB/BSC | No | Standard 0–2 only | +| Avalanche C-Chain | No | Standard 0–2 (custom block header handled in `verify_block_hash`) | +| Monad | No | Standard 0–4 | +| Base | Type 126 only | Same as Optimism (OP Stack) | +| zkSync | Type 113 (0x71) | EIP-712 zkSync tx. Not relevant unless we add zkSync support. | +| Polygon zkEVM | No | Standard types only | + +## 3. Shared Helper + +Extract a shared `encode_typed_tx` to reduce duplication across all encoders: + +```rust +/// Build a typed transaction envelope: type_byte + RLP list wrapping the payload. +fn encode_typed_tx(type_byte: u8, payload: Vec) -> Result, BlockVerificationError> { + let mut result = vec![type_byte]; + let header = alloy::rlp::Header { + list: true, + payload_length: payload.len(), + }; + header.encode(&mut result); + result.extend_from_slice(&payload); + Ok(result) +} +``` + +Refactor the existing `encode_deposit_transaction` (0x7E) and +`encode_arbitrum_internal_transaction` (0x6A) to use this helper too. + +## 4. Skip Mechanism Design + +The `_` fallback and type 127 should both use a non-blocking skip. The current code at +`verify_block` (lines 296–305) already catches `verify_transaction_root` errors and continues +— but the error still propagates from `safe_encode_transaction` through `verify_transaction_root`. + +Proposed flow: + +``` +safe_encode_transaction + └── type 127 or _ → Err(UnsupportedTransactionType { tx_type, index }) + +verify_transaction_root + └── on UnsupportedTransactionType: + error!(tx_type, index, block_number, "Skipping tx root verification: ...") + return Ok(()) // skip, don't propagate + +verify_block (unchanged) + └── verify_transaction_root → Ok (skipped) or Ok (verified) + └── verify_receipt_root → must pass + └── verify_block_hash → must pass +``` + +The ERROR log ensures visibility. A `listener_block_verification_skipped_total` metric +counter (labels: `chain_id`, `reason`) should be incremented for alerting. + +## 5. Receipt Encoding for Custom Types + +Receipt encoding uses `AnyReceiptEnvelope::encoded_2718()` for all non-0x7E types, which +produces `type_byte || rlp(status, cumulativeGasUsed, bloom, logs)`. This works for: + +- Arbitrum types 100–106: standard receipt format, just different type byte +- Polygon type 127: standard receipt format (status=1, gasUsed=0, standard logs) + +**No receipt encoding changes needed** for any of the types above. The Optimism-specific +deposit receipt handling (type 0x7E with `depositNonce`/`depositReceiptVersion`) is already +implemented. + +## 6. Testing + +### Polygon state sync block + +Block **85,523,136** (0x518FAC0) on Polygon mainnet. 376 transactions, index 375 is type 0x7F. +Sprint blocks occur every 16 blocks. + +``` +TX hash: 0x4f8e7a02f12c3573bf9a7c83ac19f77f0537d614a1f05f68b39278cc02d652e5 +RPC: https://polygon-bor-rpc.publicnode.com +``` + +Test should verify: +1. `verify_block` does NOT error (tx root skipped, receipt root + block hash pass) +2. `verify_receipt_root` passes independently +3. `verify_block_hash` passes independently + +### Arbitrum + +Enable `Arbitrum One` in `report_block_verification` test (already commented out at line 118). +Use `https://arb1.arbitrum.io/rpc`. Most blocks contain type 106 (internal) and +occasionally types 100, 104, 105. + +### Field mapping validation + +For each new Arbitrum encoder, validate by picking a real transaction of that type from +Arbiscan and comparing: +1. Fetch via `eth_getBlockByNumber(..., true)` — check JSON field names match spec +2. Fetch via `eth_getRawTransactionByHash` — compare our encoding against raw bytes + +## 7. References + +- [PIP-74 forum post](https://forum.polygon.technology/t/pip-74-canonical-inclusion-of-statesync-transactions-in-block-bodies/21331) +- [Bor v2.5.0 release (PIP-74 implementation)](https://github.com/0xPolygon/bor/releases/tag/v2.5.0) +- [Arbitrum Nitro arb_types.go](https://github.com/OffchainLabs/go-ethereum/blob/master/core/types/arb_types.go) +- [Arbitrum inside Nitro docs](https://docs.arbitrum.io/how-arbitrum-works/inside-arbitrum-nitro) +- [EIP-2718: Typed Transaction Envelope](https://eips.ethereum.org/EIPS/eip-2718) diff --git a/listener/listener-core.yaml b/listener/listener-core.yaml new file mode 100644 index 0000000000..f389320f1b --- /dev/null +++ b/listener/listener-core.yaml @@ -0,0 +1,142 @@ +# Unified Docker Compose for the listener project. + +x-listener-base: &listener-base + build: + context: . + dockerfile: Dockerfile + restart: unless-stopped + +services: + # ── Base infrastructure (always starts) ────────────────────────────── + + # ── Message brokers (always start) ─────────────────────────────────── + + redis: + image: redis:7-alpine + container_name: listener-redis + ports: + - "6379:6379" + # networks: + # - listener + restart: unless-stopped + command: redis-server --appendonly yes + volumes: + - redis_data:/data + networks: + - listener + + rabbitmq: + image: rabbitmq:3-management-alpine + container_name: listener-rabbitmq + ports: + - "5672:5672" + - "15672:15672" + environment: + RABBITMQ_DEFAULT_USER: user + RABBITMQ_DEFAULT_PASS: pass + # networks: + # - listener + restart: unless-stopped + networks: + - listener + + # ── RPC proxy (optional) ───────────────────────────────────────────── + + erpc: + image: ghcr.io/erpc/erpc:0.0.63 + container_name: listener-erpc + profiles: [erpc, listener-1, listener-2] + volumes: + - ./config/erpc-public.yaml:/erpc.yaml + ports: + - "4000:4000" # HTTP API + - "4001:4001" # Metrics + networks: + - listener + + # ── Listener instances (one per chain, pick via --profile) ─────────── + + listener-1: + <<: *listener-base + container_name: listener-1 + profiles: [listener-1] + depends_on: + rabbitmq: { condition: service_started } + erpc: { condition: service_started } + networks: + - listener + - fhevm_default + volumes: + - ./config/listener-1.yaml:/config.yaml:ro + + listener-2: + <<: *listener-base + container_name: listener-2 + profiles: [listener-2] + depends_on: + redis: { condition: service_started } + erpc: { condition: service_started } + networks: + - listener + - fhevm_default + volumes: + - ./config/listener-2.yaml:/config.yaml:ro + + # ── Monitoring (optional) ──────────────────────────────────────────── + # Note: Prometheus scrapes erpc:4001 — combine with --profile erpc + # for full metrics. Without eRPC, Prometheus still starts but that + # target shows as down. + + prometheus: + image: prom/prometheus:v3.9.1 + container_name: listener-prometheus + profiles: [monitoring] + ports: + - "9090:9090" + networks: + - listener + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--web.console.libraries=/etc/prometheus/console_libraries' + - '--web.console.templates=/etc/prometheus/consoles' + - '--web.enable-lifecycle' + volumes: + - ./monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./monitoring/prometheus/alert.rules:/etc/prometheus/alert.rules:ro + - prometheus_data:/prometheus + restart: unless-stopped + + grafana: + image: grafana/grafana:12.3.2 + container_name: listener-grafana + profiles: [monitoring] + ports: + - "3000:3000" + networks: + - listener + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD: admin + GF_USERS_ALLOW_SIGN_UP: "false" + volumes: + - ./monitoring/grafana/grafana.ini:/etc/grafana/grafana.ini:ro + - ./monitoring/grafana/datasources/prometheus.yml:/etc/grafana/provisioning/datasources/prometheus.yml:ro + - ./monitoring/grafana/dashboards/default.yml:/etc/grafana/provisioning/dashboards/default.yml:ro + - ./monitoring/grafana/dashboards/erpc.json:/etc/grafana/dashboards/erpc.json:ro + - grafana_data:/var/lib/grafana + depends_on: + - prometheus + restart: unless-stopped + + +networks: + listener: + driver: bridge + fhevm_default: + external: true + +volumes: + redis_data: + prometheus_data: + grafana_data: diff --git a/listener/monitoring/grafana/dashboards/default.yml b/listener/monitoring/grafana/dashboards/default.yml new file mode 100644 index 0000000000..f684fc01d5 --- /dev/null +++ b/listener/monitoring/grafana/dashboards/default.yml @@ -0,0 +1,13 @@ +apiVersion: 1 + +providers: + - name: 'default' + orgId: 1 + folder: '' + type: file + disableDeletion: false + updateIntervalSeconds: 10 + allowUiUpdates: true + options: + path: /etc/grafana/dashboards + foldersFromFilesStructure: true diff --git a/listener/monitoring/grafana/dashboards/erpc.json b/listener/monitoring/grafana/dashboards/erpc.json new file mode 100644 index 0000000000..b1bea51e99 --- /dev/null +++ b/listener/monitoring/grafana/dashboards/erpc.json @@ -0,0 +1,10663 @@ +{ + "__inputs": [ + { + "name": "DS_GROUNDCOVER-PROMETHEUS", + "label": "groundcover-prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + }, + { + "name": "DS_FLY-PROMETHEUS", + "label": "fly-prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "panel", + "id": "bargauge", + "name": "Bar gauge", + "version": "" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "12.0.1" + }, + { + "type": "panel", + "id": "heatmap", + "name": "Heatmap", + "version": "" + }, + { + "type": "panel", + "id": "piechart", + "name": "Pie chart", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 35, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "fieldMinMax": false, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 19, + "w": 11, + "x": 0, + "y": 2017 + }, + "id": 90, + "options": { + "displayMode": "lcd", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 49, + "minVizHeight": 0, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "text": {}, + "valueMode": "text" + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "topk(20,\n sum by (region, network, vendor, finality, category) (\n increase(erpc_network_successful_request_total{\n network=~\"${network:regex}\",\n _network!~\"${ignore.network:regex}\",\n project=~\"${project:regex}\",\n client_name=~\"${client_name:regex}\",\n cluster_key=~\"${cluster_key:regex}\",\n vendor=~\"${vendor:regex}\",\n vendor!~\"${ignore.vendor:regex}\",\n category=~\"${category:regex}\",\n finality=~\"${finality:regex}\",\n user=~\"${user:regex}\"\n }[$__range])\n )\n)", + "format": "time_series", + "instant": true, + "interval": "10m", + "legendFormat": "{{region}}, {{network}}, {{vendor}}, {{finality}}, {{category}}", + "range": false, + "refId": "A" + } + ], + "title": "Share Breakdown (Top 20)", + "transformations": [ + { + "id": "sortBy", + "options": { + "desc": true, + "fields": [ + "Value" + ], + "sort": [ + { + "field": "Value" + } + ] + } + }, + { + "id": "limit", + "options": { + "limit": 10, + "offset": 0 + } + } + ], + "transparent": true, + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "fieldMinMax": false, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 19, + "w": 13, + "x": 11, + "y": 2017 + }, + "id": 100, + "options": { + "displayLabels": [ + "name", + "value" + ], + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "values": [ + "percent" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "topk(20,\n sum by (vendor) (\n increase(erpc_network_successful_request_total{\n network=~\"${network:regex}\",\n network!~\"${ignore_network:regex}\",\n project=~\"${project:regex}\",\n client_name=~\"${client_name:regex}\",\n cluster_key=~\"${cluster_key:regex}\",\n vendor=~\"${vendor:regex}\",\n vendor!~\"${ignore_vendor:regex}\",\n category=~\"${category:regex}\",\n finality=~\"${finality:regex}\",\n user=~\"${user:regex}\"\n }[$__range])\n )\n)", + "format": "time_series", + "instant": true, + "interval": "1m", + "legendFormat": "{{region}} {{vendor}} {{category}}", + "range": false, + "refId": "A" + } + ], + "title": "Vendors Share (Top 20)", + "transformations": [ + { + "id": "sortBy", + "options": { + "desc": true, + "fields": [ + "Value" + ], + "sort": [ + { + "field": "Value" + } + ] + } + }, + { + "id": "limit", + "options": { + "limit": 10, + "offset": 0 + } + } + ], + "transparent": true, + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "fieldMinMax": false, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 17, + "w": 8, + "x": 0, + "y": 2287 + }, + "id": 118, + "options": { + "displayLabels": [ + "name", + "value" + ], + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "values": [ + "percent" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "topk(20,\n sum by (region, network) (\n increase(erpc_network_successful_request_total{\n network=~\"${network:regex}\",\n _network!~\"${ignore_network:regex}\",\n project=~\"${project:regex}\",\n client_name=~\"${client_name:regex}\",\n cluster_key=~\"${cluster_key:regex}\",\n vendor=~\"${vendor:regex}\",\n vendor!~\"${ignore_vendor:regex}\",\n category=~\"${category:regex}\",\n finality=~\"${finality:regex}\",\n user=~\"${user:regex}\"\n }[$__range])\n )\n)", + "format": "time_series", + "instant": true, + "interval": "1m", + "legendFormat": "{{region}} {{network}}", + "range": false, + "refId": "A" + } + ], + "title": "Networks Share (Top 20)", + "transformations": [ + { + "id": "sortBy", + "options": { + "desc": true, + "fields": [ + "Value" + ], + "sort": [ + { + "field": "Value" + } + ] + } + }, + { + "id": "limit", + "options": { + "limit": 10, + "offset": 0 + } + } + ], + "transparent": true, + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "fieldMinMax": false, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 17, + "w": 8, + "x": 8, + "y": 2287 + }, + "id": 119, + "options": { + "displayLabels": [ + "name", + "value" + ], + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "values": [ + "percent" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "topk(20,\n sum by (category) (\n increase(erpc_network_successful_request_total{\n network=~\"${network:regex}\",\n _network!~\"${ignore_network:regex}\",\n project=~\"${project:regex}\",\n client_name=~\"${client_name:regex}\",\n cluster_key=~\"${cluster_key:regex}\",\n vendor=~\"${vendor:regex}\",\n vendor!~\"${ignore_vendor:regex}\",\n category=~\"${category:regex}\",\n finality=~\"${finality:regex}\",\n user=~\"${user:regex}\"\n }[$__range])\n )\n)", + "format": "time_series", + "instant": true, + "interval": "1m", + "legendFormat": "{{region}} {{category}}", + "range": false, + "refId": "A" + } + ], + "title": "Method Share (Top 20)", + "transformations": [ + { + "id": "sortBy", + "options": { + "desc": true, + "fields": [ + "Value" + ], + "sort": [ + { + "field": "Value" + } + ] + } + }, + { + "id": "limit", + "options": { + "limit": 10, + "offset": 0 + } + } + ], + "transparent": true, + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "fieldMinMax": false, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 17, + "w": 8, + "x": 16, + "y": 2287 + }, + "id": 120, + "options": { + "displayLabels": [ + "name", + "value" + ], + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "values": [ + "percent" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "topk(20,\n sum by (finality) (\n increase(erpc_network_successful_request_total{\n network=~\"${network:regex}\",\n _network!~\"${ignore_network:regex}\",\n project=~\"${project:regex}\",\n client_name=~\"${client_name:regex}\",\n cluster_key=~\"${cluster_key:regex}\",\n vendor=~\"${vendor:regex}\",\n vendor!~\"${ignore_vendor:regex}\",\n category=~\"${category:regex}\",\n finality=~\"${finality:regex}\",\n user=~\"${user:regex}\"\n }[$__range])\n )\n)", + "format": "time_series", + "instant": true, + "interval": "1m", + "legendFormat": "{{region}} {{finality}}", + "range": false, + "refId": "A" + } + ], + "title": "Finality Share (Top 20)", + "transformations": [ + { + "id": "sortBy", + "options": { + "desc": true, + "fields": [ + "Value" + ], + "sort": [ + { + "field": "Value" + } + ] + } + }, + { + "id": "limit", + "options": { + "limit": 10, + "offset": 0 + } + } + ], + "transparent": true, + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total successful responses sent back to the client", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.8, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 0, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 24, + "x": 0, + "y": 2304 + }, + "id": 85, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Total", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(increase(erpc_network_successful_request_total{\n \n network=~\"${network:regex}\",\n project=~\"${project:regex}\",\n client_name=~\"${client_name:regex}\",\n cluster_key=~\"${cluster_key:regex}\",\n vendor=~\"${vendor:regex}\",\n upstream=~\"${upstream:regex}\",\n vendor!~\"${ignore_vendor:regex}\",\n category=~\"${category:regex}\",\n finality=~\"${finality:regex}\",user=~\"${user:regex}\"\n }[1m])) by (region, agent_name) > 0", + "interval": "", + "legendFormat": "{{region}} {{agent_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Total Responses (by Region and User-Agent)", + "type": "timeseries" + } + ], + "title": "Overall", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 116, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total requests received by eRPC for a specific network and method", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.8, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 0, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 8, + "x": 0, + "y": 2 + }, + "id": 2, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Total", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(increase(erpc_network_request_received_total{network=~\"${network:regex}\",network!~\"${ignore_network:regex}\",upstream=~\"${upstream:regexp}\",category=~\"${category:regex}\",project=~\"${project:regex}\",finality=~\"${finality:regex}\",user=~\"${user:regex}\"}[1m])) by (network, category, finality)", + "interval": "", + "legendFormat": "{{network}}, {{category}}, {{finality}}", + "range": true, + "refId": "A" + } + ], + "title": "Incoming Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total requests received by eRPC for a specific network and method", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 0, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 8, + "x": 8, + "y": 2 + }, + "id": 28, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [ + "min", + "p70", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(erpc_network_request_received_total{network=~\"${network:regex}\",network!~\"${ignore_network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",project=~\"${project:regex}\",finality=~\"${finality:regex}\",user=~\"${user:regex}\"}[1m])) by (network, user, category, finality) > 0", + "interval": "", + "legendFormat": "{{network}}, {{user}}, {{category}}, {{finality}}", + "range": true, + "refId": "A" + } + ], + "title": "Incoming RPS", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total successful responses sent back to the client", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.8, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 0, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 8, + "x": 16, + "y": 2 + }, + "id": 122, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Total", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(increase(erpc_network_successful_request_total{\n \n network=~\"${network:regex}\",\n project=~\"${project:regex}\",\n client_name=~\"${client_name:regex}\",\n cluster_key=~\"${cluster_key:regex}\",\n vendor=~\"${vendor:regex}\",\n upstream=~\"${upstream:regex}\",\n vendor!~\"${ignore_vendor:regex}\",\n category=~\"${category:regex}\",\n finality=~\"${finality:regex}\",user=~\"${user:regex}\"\n }[1m])) by (region, network, vendor, category, finality, emptyish) > 0", + "interval": "", + "legendFormat": "{{region}} {{network}}, {{vendor}}, {{category}}, {{finality}}, empty={{emptyish}}", + "range": true, + "refId": "A" + } + ], + "title": "Successful Responses", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "When a request fails even after trying all upstreams and retries, we send a error response to the user. This chart shows critical errors that need attention and wake-up alarms.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 8, + "x": 0, + "y": 1575 + }, + "id": 14, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(erpc_network_failed_request_total{network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",severity=\"critical\",project=~\"${project:regex}\",finality=~\"${finality:regex}\"}[1m])) by (network,category, finality,error) > 0", + "instant": false, + "interval": "", + "legendFormat": "{{network}}, {{category}}, {{finality}}, {{error}}", + "range": true, + "refId": "A" + } + ], + "title": "Network-level Critical Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Issues where clients got an error but it's most probably due to bad requests or a transient problem with upstreams", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 8, + "x": 8, + "y": 1575 + }, + "id": 66, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(erpc_network_failed_request_total{network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",severity=\"warning\",project=~\"${project:regex}\",finality=~\"${finality:regex}\"}[1m])) by (network,category,error) > 0", + "instant": false, + "interval": "", + "legendFormat": "{{network}}, {{category}}, {{error}}", + "range": true, + "refId": "A" + } + ], + "title": "Network-level Warnings", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Client-side errors where it's most probably a normal operation (eth_call reverts) or bad request by the user", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 8, + "x": 16, + "y": 1575 + }, + "id": 67, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(erpc_network_failed_request_total{network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",severity=\"info\",project=~\"${project:regex}\",finality=~\"${finality:regex}\"}[1m])) by (network,category,error) > 0", + "instant": false, + "interval": "", + "legendFormat": "{{network}}, {{category}}, {{error}}", + "range": true, + "refId": "A" + } + ], + "title": "Network-level Notices", + "type": "timeseries" + } + ], + "title": "Networks - Requests", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 2 + }, + "id": 19, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 8, + "x": 0, + "y": 2951 + }, + "id": 8, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(erpc_network_request_duration_seconds_bucket{network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",vendor=~\"${vendor:regex}\",vendor!=\"\",project=~\"${project:regex}\",finality=~\"${finality:regex}\",user=~\"${user:regex}\"}[1m])) by (le, network, category, finality))", + "hide": false, + "legendFormat": "{{network}}, {{category}}, {{finality}}", + "range": true, + "refId": "B" + } + ], + "title": "Network-method P99 Resp. time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 8, + "x": 8, + "y": 2951 + }, + "id": 44, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum(rate(erpc_network_request_duration_seconds_bucket{network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",vendor=~\"${vendor:regex}\",vendor!=\"\",project=~\"${project:regex}\",finality=~\"${finality:regex}\",user=~\"${user:regex}\"}[1m])) by (le, network, category, finality))", + "legendFormat": "{{network}}, {{category}}, {{finality}}", + "range": true, + "refId": "A" + } + ], + "title": "Network-method P90 Resp. time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 8, + "x": 16, + "y": 2951 + }, + "id": 45, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum(rate(erpc_network_request_duration_seconds_bucket{network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",vendor=~\"${vendor:regex}\",vendor!=\"\",project=~\"${project:regex}\",finality=~\"${finality:regex}\",user=~\"${user:regex}\"}[1m])) by (le, network, category, finality))", + "hide": false, + "legendFormat": "{{network}}, {{category}}, {{finality}}", + "range": true, + "refId": "C" + } + ], + "title": "Network-method P50 Resp. time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 2997 + }, + "id": 103, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(erpc_network_request_duration_seconds_bucket{network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",vendor=~\"${vendor:regex}\",vendor!=\"\",project=~\"${project:regex}\",finality=~\"${finality:regex}\",user=~\"${user:regex}\"}[1m])) by (le, network, category, finality, vendor))", + "hide": false, + "legendFormat": "{{network}}, {{category}}, {{finality}}, {{vendor}}", + "range": true, + "refId": "B" + } + ], + "title": "Network vendors P99 Resp. time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 2997 + }, + "id": 104, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum(rate(erpc_network_request_duration_seconds_bucket{network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",vendor=~\"${vendor:regex}\",vendor!=\"\",project=~\"${project:regex}\",finality=~\"${finality:regex}\",user=~\"${user:regex}\"}[1m])) by (le, network, category, finality, vendor))", + "legendFormat": "{{network}}, {{category}}, {{vendor}}, {{finality}}", + "range": true, + "refId": "A" + } + ], + "title": "Network vendors P90 Resp. time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 2997 + }, + "id": 105, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [ + "min", + "max", + "delta" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Delta", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum(rate(erpc_network_request_duration_seconds_bucket{network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",vendor=~\"${vendor:regex}\",category=~\"${category:regex}\",vendor!=\"\",project=~\"${project:regex}\",finality=~\"${finality:regex}\",user=~\"${user:regex}\"}[1m])) by (le, network, category, finality, vendor))", + "hide": false, + "legendFormat": "{{network}}, {{category}}, {{vendor}}, {{finality}}", + "range": true, + "refId": "C" + } + ], + "title": "Network vendor P50 Resp. time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 3004 + }, + "id": 106, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(erpc_network_request_duration_seconds_bucket{network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",vendor=\"\",project=~\"${project:regex}\",finality=~\"${finality:regex}\",user=~\"${user:regex}\"}[1m])) by (le, network, category, finality, vendor))", + "hide": false, + "legendFormat": "{{network}}, {{category}}, {{finality}}, {{vendor}}", + "range": true, + "refId": "B" + } + ], + "title": "Network-errors P99 Resp. time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 3004 + }, + "id": 107, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum(rate(erpc_network_request_duration_seconds_bucket{network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",vendor=\"\",project=~\"${project:regex}\",finality=~\"${finality:regex}\",user=~\"${user:regex}\"}[1m])) by (le, network, category, finality, vendor))", + "legendFormat": "{{network}}, {{category}}, {{vendor}}, {{finality}}", + "range": true, + "refId": "A" + } + ], + "title": "Network-errors P90 Resp. time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 3004 + }, + "id": 108, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum(rate(erpc_network_request_duration_seconds_bucket{network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",vendor=\"\",project=~\"${project:regex}\",finality=~\"${finality:regex}\"}[1m])) by (le, network, category, finality, vendor))", + "hide": false, + "legendFormat": "{{network}}, {{category}}, {{vendor}}, {{finality}}", + "range": true, + "refId": "C" + } + ], + "title": "Network errors P50 Resp. time", + "type": "timeseries" + } + ], + "title": "Networks - Latency", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 117, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "When identical requests arrive at the same time they'll be merged, this number shows number of these requests", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.5, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 0, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 8, + "x": 0, + "y": 4 + }, + "id": 9, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Total", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(increase(erpc_network_multiplexed_request_total{network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",project=~\"${project:regex}\",finality=~\"${finality:regex}\"}[1m])) by (project, network, category, finality)", + "legendFormat": "{{network}}, {{category}}, {{finality}}", + "range": true, + "refId": "A" + } + ], + "title": "Multiplexed requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total hedge requests fired due to initial upstream being slow", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 8, + "x": 8, + "y": 4 + }, + "id": 68, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(erpc_network_hedged_request_total{network=~\"${network:regex}\",category=~\"${category:regex}\",project=~\"${project:regex}\",finality=~\"${finality:regex}\"}[1m])) by (project, network, category, finality)", + "interval": "", + "legendFormat": "{{project}}, {{network}}, {{category}}, {{finality}}", + "range": true, + "refId": "A" + } + ], + "title": "Hedge RPS", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total hedged requests that ended up being actually faster and useful. The higher the better. If this value is too low try increasing the failsafe.hedge.quantile to higher value e.g. 0.99", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-RdYlGr", + "seriesBy": "last" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 17, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "dash": [ + 0, + 10 + ], + "fill": "dot" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 8, + "x": 16, + "y": 4 + }, + "id": 33, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "1 - ((sum(increase(erpc_network_hedge_discards_total{attempt!=\"1\",network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",project=~\"${project:regex}\",finality=~\"${finality:regex}\"}[30s])) by (project, network)) / (sum(increase(erpc_network_hedged_request_total{network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",project=~\"${project:regex}\",finality=~\"${finality:regex}\"}[1m])) by (project, network))) > 0", + "legendFormat": "{{network}}", + "range": true, + "refId": "A" + } + ], + "title": "Hedge Effectiveness", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Shows the finality state of the block related to the response, this shows if your workload is mainly around head of chain unfinalized data, or historical finalized data.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 86, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(increase(erpc_network_successful_request_total{network=~\"${network:regex}\",vendor=~\"${vendor:regexp}\",upstream=~\"${upstream:regexp}\",category=~\"${category:regex}\",project=~\"${project:regex}\",finality=~\"${finality:regex}\"}[1m])) by (finality)", + "interval": "", + "legendFormat": "{{finality}}", + "range": true, + "refId": "A" + } + ], + "title": "Response Finality Share Overall", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Shows the finality state of the block related to the response, this shows if your workload is mainly around head of chain unfinalized data, or historical finalized data.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 89, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(increase(erpc_network_successful_request_total{network=~\"${network:regex}\",vendor=~\"${vendor:regexp}\",upstream=~\"${upstream:regexp}\",category=~\"${category:regex}\",project=~\"${project:regex}\",finality=~\"${finality:regex}\"}[1m])) by (vendor, finality)", + "interval": "", + "legendFormat": "{{vendor}}, {{finality}}", + "range": true, + "refId": "A" + } + ], + "title": "Response Finality Share By Vendor", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Highest value for latest block polled from all upstreams across all replicas", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisGridShow": false, + "axisLabel": "", + "axisPlacement": "hidden", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "dash": [ + 0, + 10 + ], + "fill": "dot" + }, + "lineWidth": 1, + "pointSize": 7, + "scaleDistribution": { + "linearThreshold": 1, + "log": 10, + "type": "symlog" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 63, + "interval": "1m", + "maxDataPoints": 20, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(erpc_upstream_finalized_block_number{upstream=\"*\",network!=\"\",network=~\"${network:regex}\",project=~\"${project:regex}\"}) by (project, network)", + "format": "time_series", + "instant": false, + "legendFormat": "{{project}}, {{network}}", + "range": true, + "refId": "A" + } + ], + "title": "Highest Finalized Block", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Highest value for latest block polled from all upstreams across all replicas", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisGridShow": false, + "axisLabel": "", + "axisPlacement": "hidden", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "dash": [ + 0, + 10 + ], + "fill": "dot" + }, + "lineWidth": 1, + "pointSize": 7, + "scaleDistribution": { + "linearThreshold": 1, + "log": 10, + "type": "symlog" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 28 + }, + "id": 25, + "interval": "1m", + "maxDataPoints": 20, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(erpc_upstream_latest_block_number{upstream=\"*\",network!=\"\",network=~\"${network:regex}\",project=~\"${project:regex}\"}) by (project, network)", + "format": "time_series", + "instant": false, + "legendFormat": "{{project}}, {{network}}", + "range": true, + "refId": "A" + } + ], + "title": "Highest Latest Block", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Distance in seconds between the latest block timestamp and current time, indicating chain synchronization and block production latency", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 38 + }, + "id": 99, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [ + "lastNotNull", + "mean" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_FLY-PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "erpc_network_latest_block_timestamp_distance_seconds{network!=\"\",network=~\"${network:regex}\",project=~\"${project:regex}\",origin=\"evm_state_poller\"}", + "format": "time_series", + "instant": false, + "legendFormat": "{{project}}, {{network}}", + "range": true, + "refId": "A" + } + ], + "title": "Block Timestamp Distance - From Upstream Polling", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Comparison of block timestamp distance between EVM State Poller and Network Responses to identify cache staleness and synchronization issues", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "vis": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 38 + }, + "id": 130, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [ + "lastNotNull", + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_FLY-PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "erpc_network_latest_block_timestamp_distance_seconds{network!=\"\",network=~\"${network:regex}\",project=~\"${project:regex}\",origin=\"network_response\"}", + "format": "time_series", + "instant": false, + "legendFormat": "{{project}}, {{network}}", + "range": true, + "refId": "A" + } + ], + "title": "Block Timestamp Distance - From Latest Block Responses", + "type": "timeseries" + } + ], + "title": "Networks - Advanced", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 69, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "fieldMinMax": false, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 2020 + }, + "id": 81, + "options": { + "displayLabels": [ + "name", + "value" + ], + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "values": [ + "percent" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.3.7", + "repeat": "finality", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (vendor, finality) (\n erpc_network_successful_request_total{\n network=~\"${network:regex}\",\n project=~\"${project:regex}\",\n client_name=~\"${client_name:regex}\",\n cluster_key=~\"${cluster_key:regex}\",\n vendor=~\"${vendor:regex}\", vendor!=\"n/a\",\n category=~\"${category:regex}\",\n finality=~\"${finality:regex}\"\n }\n)", + "format": "time_series", + "instant": true, + "interval": "1m", + "legendFormat": "{{vendor}}", + "range": false, + "refId": "A" + } + ], + "title": "Vendors Share: finality=$finality", + "transformations": [ + { + "id": "sortBy", + "options": { + "desc": true, + "fields": [ + "Value" + ], + "sort": [ + { + "field": "Value" + } + ] + } + }, + { + "id": "limit", + "options": { + "limit": 10, + "offset": 0 + } + } + ], + "transparent": true, + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "These are real requests sent to upstream endpoints, including retries, grouped by Vendor Name", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 2088 + }, + "id": 70, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(increase(erpc_upstream_request_total{network=~\"${network:regex}\",vendor=~\"${vendor:regex}\",category=~\"${category:regex}\",composite=\"none\",finality=~\"${finality:regex}\"}[1m])) by (network, vendor, category)", + "interval": "", + "legendFormat": "{{network}}, {{vendor}}, {{category}}", + "range": true, + "refId": "A" + } + ], + "title": "Vendor Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "These are real requests sent to upstream endpoints, including retries", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 2088 + }, + "id": 29, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(erpc_upstream_request_total{network=~\"${network:regex}\",vendor=~\"${vendor:regex}\",category=~\"${category:regex}\",composite=\"none\",project=~\"${project:regex}\",finality=~\"${finality:regex}\"}[1m])) by (network, vendor, finality, category)", + "interval": "", + "legendFormat": "{{network}}, {{vendor}}, {{category}}, {{finality}}", + "range": true, + "refId": "A" + } + ], + "title": "Vendor RPS", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Failures received when attempting upstream endpoints, these errors mean upstream has failures", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 12, + "x": 0, + "y": 2100 + }, + "id": 24, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(increase(erpc_upstream_request_errors_total{severity=\"critical\",network=~\"${network:regex}\",vendor=~\"${vendor:regex}\",category=~\"${category:regex}\",composite=\"none\",vendor=~\"${vendor:regex}\",project=~\"${project:regex}\"}[1m])) by (region, project, upstream, category, error) > 0", + "interval": "", + "legendFormat": "{{region}} {{project}}, {{upstream}}, {{category}}, {{error}}", + "range": true, + "refId": "A" + } + ], + "title": "Upstream-level critical errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Failures received when attempting upstream endpoints, these errors mean upstream has failures", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 12, + "x": 12, + "y": 2100 + }, + "id": 82, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(increase(erpc_upstream_request_errors_total{severity=\"warning\",network=~\"${network:regex}\",vendor=~\"${vendor:regex}\",category=~\"${category:regex}\",composite=\"none\",project=~\"${project:regex}\",finality=~\"${finality:regex}\"}[1m])) by (project, network, vendor, category, finality, error) > 0", + "interval": "", + "legendFormat": "{{project}}, {{network}}, {{category}}, {{vendor}}, {{finality}}, {{error}}", + "range": true, + "refId": "A" + } + ], + "title": "Vendor warnings", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 8, + "x": 0, + "y": 2114 + }, + "id": 78, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(erpc_upstream_request_duration_seconds_bucket{network=~\"${network:regex}\",vendor=~\"${vendor:regex}\",category=~\"${category:regex}\",composite=\"none\",project=~\"${project:regex}\",vendor=~\"${vendor:regex}\"}[1m])) by (le, project, vendor, category))", + "legendFormat": "{{project}}, {{vendor}}, {{category}}", + "range": true, + "refId": "C" + } + ], + "title": "Vendor P99 Resp. time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 8, + "x": 8, + "y": 2114 + }, + "id": 79, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [ + "logmin", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum(rate(erpc_upstream_request_duration_seconds_bucket{network=~\"${network:regex}\",vendor=~\"${vendor:regex}\",category=~\"${category:regex}\",composite=\"none\",project=~\"${project:regex}\"}[1m])) by (le, project, vendor, category))", + "legendFormat": "{{project}}, {{vendor}}, {{category}}", + "range": true, + "refId": "B" + } + ], + "title": "Vendor P90 Resp. time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 8, + "x": 16, + "y": 2114 + }, + "id": 80, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [ + "logmin", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum(rate(erpc_upstream_request_duration_seconds_bucket{network=~\"${network:regex}\",vendor=~\"${vendor:regex}\",category=~\"${category:regex}\",composite=\"none\",project=~\"${project:regex}\"}[30s])) by (le, project, vendor, category))", + "legendFormat": "{{project}}, {{vendor}}, {{category}}", + "range": true, + "refId": "A" + } + ], + "title": "Vendor P50 Resp. time", + "type": "timeseries" + } + ], + "title": "Vendors", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 20, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 8, + "x": 0, + "y": 2021 + }, + "id": 91, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(erpc_upstream_request_duration_seconds_bucket{network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",composite=\"none\",vendor=~\"${vendor:regex}\",project=~\"${project:regex}\",finality=~\"${finality:regex}\"}[1m])) by (le, project, upstream, category, finality))", + "legendFormat": "{{project}}, {{upstream}}, {{category}}, {{finality}}", + "range": true, + "refId": "B" + } + ], + "title": "Upstream-level P99 Resp. time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 8, + "x": 8, + "y": 2021 + }, + "id": 46, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum(rate(erpc_upstream_request_duration_seconds_bucket{network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",composite=\"none\",vendor=~\"${vendor:regex}\",project=~\"${project:regex}\",finality=~\"${finality:regex}\"}[1m])) by (le, project, upstream, category, finality))", + "legendFormat": "{{project}}, {{upstream}}, {{category}}, {{finality}}", + "range": true, + "refId": "B" + } + ], + "title": "Upstream-level P90 Resp. time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 8, + "x": 16, + "y": 2021 + }, + "id": 47, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum(rate(erpc_upstream_request_duration_seconds_bucket{network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",composite=\"none\",vendor=~\"${vendor:regex}\",project=~\"${project:regex}\",finality=~\"${finality:regex}\"}[30s])) by (le, project, upstream, category, finality))", + "legendFormat": "{{project}}, {{upstream}}, {{category}}, {{finality}}", + "range": true, + "refId": "A" + } + ], + "title": "Upstream-level P50 Resp. time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "These are real requests sent to upstream endpoints, including retries", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 2033 + }, + "id": 4, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(increase(erpc_upstream_request_total{network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",vendor=~\"${vendor:regex}\",category=~\"${category:regex}\",composite=\"none\",project=~\"${project:regex}\"}[1m])) by (project, network, upstream, category, attempt)", + "interval": "", + "legendFormat": "{{project}}, {{network}}, {{upstream}}, {{category}}, attempt={{attempt}}", + "range": true, + "refId": "A" + } + ], + "title": "Upstream-level Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "These are real requests sent to upstream endpoints, including retries", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 2033 + }, + "id": 71, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(erpc_upstream_request_total{network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",vendor=~\"${vendor:regex}\",category=~\"${category:regex}\",composite=\"none\",project=~\"${project:regex}\"}[1m])) by (project, network, upstream, category, finality)", + "interval": "", + "legendFormat": "{{project}}, {{network}}, {{upstream}}, {{category}}, {{finality}}", + "range": true, + "refId": "A" + } + ], + "title": "Upstream-level RPS", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Failures received when attempting upstream endpoints, these errors mean upstream has failures", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 2045 + }, + "id": 83, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(increase(erpc_upstream_request_errors_total{severity=\"critical\",network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",composite=\"none\",vendor=~\"${vendor:regex}\",project=~\"${project:regex}\"}[1m])) by (project, network, upstream, category, error) > 0", + "interval": "", + "legendFormat": "{{project}}, {{network}}, {{category}}, {{upstream}}, {{error}}", + "range": true, + "refId": "A" + } + ], + "title": "Upstream-level critical errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Failures received when attempting upstream endpoints, these errors mean upstream has failures", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 2045 + }, + "id": 73, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(increase(erpc_upstream_request_errors_total{severity=\"warning\",network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",composite=\"none\",vendor=~\"${vendor:regex}\",project=~\"${project:regex}\"}[1m])) by (project, network, upstream, category, error) > 0", + "interval": "", + "legendFormat": "{{project}}, {{network}}, {{category}}, {{upstream}}, {{error}}", + "range": true, + "refId": "A" + } + ], + "title": "Upstream-level warnings", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 2057 + }, + "id": 102, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(increase(erpc_upstream_wrong_empty_response_total{network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",vendor=~\"${vendor:regex}\",project=~\"${project:regex}\"}[1m])) by (project, network, upstream, category) > 0", + "interval": "", + "legendFormat": "{{project}}, {{network}}, {{category}}, {{upstream}}, {{error}}", + "range": true, + "refId": "A" + } + ], + "title": "Upstream wrong empty responses", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Failures received when attempting upstream endpoints, these errors mean upstream has failures", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 2057 + }, + "id": 74, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(increase(erpc_upstream_request_errors_total{severity=\"info\",network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",composite=\"none\",vendor=~\"${vendor:regex}\",project=~\"${project:regex}\"}[1m])) by (project, network, upstream, category, error) > 0", + "interval": "", + "legendFormat": "{{project}}, {{network}}, {{category}}, {{upstream}}, {{error}}", + "range": true, + "refId": "A" + } + ], + "title": "Upstream-level notices", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total number of times a request was skipped due to upstream latest block being less than requested upper bound block", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 12, + "x": 0, + "y": 2066 + }, + "id": 87, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(increase(erpc_upstream_stale_upper_bound_total{network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",vendor=~\"${vendor:regex}\",project=~\"${project:regex}\"}[1m])) by (project, network, vendor, category)", + "interval": "", + "legendFormat": "{{project}}, {{network}}, {{category}}, {{vendor}}", + "range": true, + "refId": "A" + } + ], + "title": "Upstream stale upper-bound", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total number of times a request was skipped due to requested lower bound block being less than upstream's available block range.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 12, + "x": 12, + "y": 2066 + }, + "id": 88, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(increase(erpc_upstream_stale_lower_bound_total{network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",vendor=~\"${vendor:regex}\",project=~\"${project:regex}\"}[1m])) by (project, network, vendor, category)", + "interval": "", + "legendFormat": "{{project}}, {{network}}, {{category}}, {{vendor}}", + "range": true, + "refId": "A" + } + ], + "title": "Upstream stale lower-bound", + "type": "timeseries" + } + ], + "title": "Upstreams", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 53, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "fieldMinMax": false, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 19, + "w": 12, + "x": 0, + "y": 1175 + }, + "id": 72, + "options": { + "displayLabels": [ + "name", + "percent" + ], + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "values": [ + "percent" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "topk(8,\n sum by (vendor) (\n avg_over_time(erpc_upstream_score_overall{\n network=\"${network}\",\n vendor=~\"${vendor}\",\n category=\"${category}\"\n }[$__range])\n )\n)", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{vendor}}", + "range": true, + "refId": "A" + } + ], + "title": "Top Score Vendors", + "transformations": [ + { + "id": "sortBy", + "options": { + "desc": true, + "fields": [ + "Value" + ], + "sort": [ + { + "field": "Value" + } + ] + } + }, + { + "id": "limit", + "options": { + "limit": 10, + "offset": 0 + } + } + ], + "transparent": true, + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 19, + "w": 12, + "x": 12, + "y": 1175 + }, + "id": 50, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [ + "max", + "logmin", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg(erpc_upstream_score_overall{category=~\"${category:regex}\",upstream=~\"${upstream:regex}\",vendor=~\"${vendor:regex}\",vendor!~\"${ignore_vendor:regex}\",network=~\"${network:regex}\",project=~\"${project:regex}\"}) by (project, vendor)", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{project}}, {{vendor}}", + "range": true, + "refId": "A" + } + ], + "title": "Avg. Score by Vendor", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 1400 + }, + "id": 26, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg(erpc_upstream_score_overall{category=~\"${category:regex}\",network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",vendor=~\"${vendor:regex}\",project=~\"${project:regex}\"}) by (project, upstream, category)", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{project}}, {{upstream}}, {{category}}", + "range": true, + "refId": "A" + } + ], + "title": "Avg. Score by Upstream", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd", + "seriesBy": "last" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1024, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 6, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "dark-green" + }, + { + "color": "green", + "value": 1 + }, + { + "color": "yellow", + "value": 2 + }, + { + "color": "orange", + "value": 3 + }, + { + "color": "dark-orange", + "value": 4 + }, + { + "color": "light-red", + "value": 5 + }, + { + "color": "semi-dark-red", + "value": 10 + }, + { + "color": "dark-red", + "value": 20 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 20, + "w": 12, + "x": 0, + "y": 1410 + }, + "id": 49, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg(erpc_upstream_block_head_lag{upstream!=\"*\",network!=\"\",network=~\"${network:regex}\",project=~\"${project:regex}\",vendor=~\"${vendor:regex}\"}) by (project, network, upstream) <= 5000", + "format": "time_series", + "instant": false, + "interval": "1m", + "legendFormat": "{{project}}, {{network}}, {{upstream}}", + "range": true, + "refId": "A" + } + ], + "title": "Latest Block Lag by Upstream", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "A special lagging tracker showing only lags of above 5000 blocks which indicates node too far off.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "semi-dark-red", + "mode": "fixed", + "seriesBy": "last" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisGridShow": false, + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "dash": [ + 0, + 10 + ], + "fill": "dot" + }, + "lineWidth": 1, + "pointSize": 6, + "scaleDistribution": { + "log": 10, + "type": "log" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "dark-green" + }, + { + "color": "green", + "value": 1 + }, + { + "color": "#EAB839", + "value": 5 + }, + { + "color": "dark-orange", + "value": 10 + }, + { + "color": "semi-dark-red", + "value": 20 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 20, + "w": 12, + "x": 12, + "y": 1410 + }, + "id": 59, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg(erpc_upstream_block_head_lag{upstream!=\"*\",network!=\"\",network=~\"${network:regex}\",project=~\"${project:regex}\",vendor=~\"${vendor:regex}\"}) by (project, upstream, network,category) > 5000", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{project}}, {{network}}, {{upstream}}", + "range": true, + "refId": "A" + } + ], + "title": "Out-of-sync latest block lag by Upstream", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "decimals": 0, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "locale" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "upstream" + }, + "properties": [ + { + "id": "custom.width", + "value": 363 + } + ] + } + ] + }, + "gridPos": { + "h": 15, + "w": 12, + "x": 0, + "y": 1430 + }, + "id": 51, + "interval": "1m", + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Value" + } + ] + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg(erpc_upstream_latest_block_number{upstream!=\"*\",network!=\"\",network=~\"${network:regex}\",project=~\"${project:regex}\",vendor=~\"${vendor:regex}\"}) by (project, upstream, network)", + "format": "table", + "instant": true, + "interval": "1m", + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Latest Block by Upstream", + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "How many times we had to Poll for latest block number. This number can go up by integrity module if debounce config is too low.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 15, + "w": 12, + "x": 12, + "y": 1430 + }, + "id": 52, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(increase(erpc_upstream_latest_block_polled_total{network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",vendor=~\"${vendor:regex}\",project=~\"${project:regex}\"}[1m])) by (project, network, upstream)", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{project}}, {{network}}, {{upstream}}", + "range": true, + "refId": "A" + } + ], + "title": "Latest Block Polls", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Difference between lowest and highest finalized block as reported by upstreams. Min/Max/Mean are values across multiple replicas (if you have erpc in a multi-container setup)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd", + "seriesBy": "last" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisGridShow": false, + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "dash": [ + 0, + 10 + ], + "fill": "dot" + }, + "lineWidth": 1, + "pointSize": 6, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "max": 1024, + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "dark-green" + }, + { + "color": "green", + "value": 1 + }, + { + "color": "#EAB839", + "value": 5 + }, + { + "color": "dark-orange", + "value": 10 + }, + { + "color": "semi-dark-red", + "value": 20 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 15, + "w": 12, + "x": 0, + "y": 1445 + }, + "id": 54, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg(erpc_upstream_finalization_lag{upstream!=\"*\",network!=\"\",network=~\"${network:regex}\",project=~\"${project:regex}\",vendor=~\"${vendor:regex}\"}) by (project, upstream, network) <= 5000", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{project}}, {{network}}, {{upstream}}", + "range": true, + "refId": "A" + } + ], + "title": "Finalized Block Lag by Upstream", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "A special lagging tracker showing only lags of above 5000 blocks which indicates node too far off.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "semi-dark-red", + "mode": "fixed", + "seriesBy": "last" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisGridShow": false, + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "dash": [ + 0, + 10 + ], + "fill": "dot" + }, + "lineWidth": 1, + "pointSize": 6, + "scaleDistribution": { + "log": 10, + "type": "log" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "dark-green" + }, + { + "color": "green", + "value": 1 + }, + { + "color": "#EAB839", + "value": 5 + }, + { + "color": "dark-orange", + "value": 10 + }, + { + "color": "semi-dark-red", + "value": 20 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 15, + "w": 12, + "x": 12, + "y": 1445 + }, + "id": 57, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg(erpc_upstream_finalization_lag{upstream!=\"*\",network!=\"\",network=~\"${network:regex}\",project=~\"${project:regex}\",vendor=~\"${vendor:regex}\"}) by (project, upstream, network) > 5000", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{project}}, {{network}}, {{upstream}}", + "range": true, + "refId": "A" + } + ], + "title": "Out-of-sync Finalized Blocks by Upstream", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "locale" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "upstream" + }, + "properties": [ + { + "id": "custom.width", + "value": 287 + } + ] + } + ] + }, + "gridPos": { + "h": 13, + "w": 12, + "x": 0, + "y": 1460 + }, + "id": 55, + "interval": "1m", + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg(erpc_upstream_finalized_block_number{upstream!=\"*\",network!=\"\",network=~\"${network:regex}\",vendor=~\"${vendor:regex}\",project=~\"${project:regex}\"}) by (project, upstream, network)", + "format": "table", + "instant": true, + "interval": "1m", + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Finalized Block by Upstream", + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "How many times we had to Poll for finalized block number. ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 12, + "x": 12, + "y": 1460 + }, + "id": 56, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(increase(erpc_upstream_finalized_block_polled_total{network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",vendor=~\"${vendor:regex}\",project=~\"${project:regex}\"}[1m])) by (project, network, upstream)", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{project}}, {{network}}, {{upstream}}", + "range": true, + "refId": "A" + } + ], + "title": "Finalized Block Polls", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "How many times upstream returned a stale latest block (compared to other nodes) for eth_getBlockByNumber(latest) and eth_blockNumber", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 12, + "x": 0, + "y": 1473 + }, + "id": 60, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(increase(erpc_upstream_stale_latest_block_total{network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",vendor=~\"${vendor:regex}\",project=~\"${project:regex}\"}[1m])) by (project, network, upstream, category) ", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{project}}, {{network}}, {{upstream}}, {{category}}", + "range": true, + "refId": "A" + } + ], + "title": "Stale latest block by Upstream", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "How many times upstream returned stale finalized block (compared to other nodes) for eth_getBlockByNumber(finalized) and eth_blockNumber", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 12, + "x": 12, + "y": 1473 + }, + "id": 61, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(increase(erpc_upstream_stale_finalized_block_total{network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",project=~\"${project:regex}\"}[1m])) by (project, network, upstream, category) ", + "format": "time_series", + "interval": "1m", + "legendFormat": "{{project}}, {{network}}, {{upstream}}, {{category}}", + "range": true, + "refId": "A" + } + ], + "title": "Stale finalized block by Upstream", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "These are issues that are most often edge-cases that are not covered within eRPC logic, and must be reported to the team.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 1486 + }, + "id": 65, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(erpc_unexpected_panic_total{project=~\"${project:regex}\"}[1m])) by (scope, extra, error)", + "instant": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Unexpected panics", + "type": "timeseries" + } + ], + "title": "Health", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 64, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 1176 + }, + "id": 75, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(erpc_shadow_response_identical_total{project=~\"${project:regexp}\",network=~\"${network:regexp}\"}[1m])) by (project, network, upstream, category, finality)", + "instant": false, + "interval": "", + "legendFormat": "{{network}}, {{category}}, {{upstream}}, {{finality}}", + "range": true, + "refId": "A" + } + ], + "title": "Shadow Identical", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 1176 + }, + "id": 76, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(increase(erpc_shadow_response_mismatch_total{project=~\"${project:regexp}\",network=~\"${network:regexp}\"}[1m])) by (project, network, upstream, category, emptyish, larger)", + "hide": false, + "instant": false, + "legendFormat": "{{network}}, {{category}}, {{upstream}}, emt={{emptyish}}, lgr={{larger}}", + "range": true, + "refId": "B" + } + ], + "title": "Shadow Mismatches", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 1271 + }, + "id": 77, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(erpc_shadow_response_error_total{project=~\"${project:regexp}\",network=~\"${network:regexp}\"}[1m])) by (project, network, upstream, category, finality, error)", + "instant": false, + "interval": "", + "legendFormat": "{{network}}, {{category}}, {{upstream}}, {{error}}, {{finality}}", + "range": true, + "refId": "A" + } + ], + "title": "Shadow Errors", + "type": "timeseries" + } + ], + "title": "Shadow Traffic", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 21, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 9538 + }, + "id": 17, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Total", + "sortDesc": true + }, + "tooltip": { + "hideZeros": true, + "maxHeight": -2, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(increase(erpc_cache_set_success_total{network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",project=~\"${project:regex}\"}[1m])) by (project, network, category, connector, policy, ttl)", + "legendFormat": "{{project}}, {{network}}, {{category}}, {{connector}}, {{policy}}, TTL={{ttl}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache SET success total", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 9538 + }, + "id": 16, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum(rate(erpc_cache_set_success_duration_seconds_bucket{network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",project=~\"${project:regex}\"}[1m])) by (le, project, network, category, connector, policy, ttl))", + "hide": false, + "legendFormat": "p50 - {{project}}, {{network}}, {{category}}, {{connector}}, {{policy}}, TTL={{ttl}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(erpc_cache_set_success_duration_seconds_bucket{network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",project=~\"${project:regex}\"}[1m])) by (le, project, network, category, connector, policy, ttl))", + "hide": true, + "legendFormat": "p95 - {{project}}, {{network}}, {{category}}, {{connector}}, {{policy}}, TTL={{ttl}}", + "range": true, + "refId": "B" + } + ], + "title": "Cache SET success duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 9548 + }, + "id": 36, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(increase(erpc_cache_set_error_total{network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",project=~\"${project:regex}\"}[1m])) by (project, network, category, connector, policy, error, ttl)", + "legendFormat": "{{project}}, {{network}}, {{category}}, {{connector}}, {{error}}, {{policy}}, TTL={{ttl}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache SET error total", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 9548 + }, + "id": 37, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum(rate(erpc_cache_set_error_duration_seconds_bucket{network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",project=~\"${project:regex}\"}[1m])) by (le, project, network, category, connector, policy, ttl))", + "legendFormat": "p50 - {{project}}, {{network}}, {{category}}, {{connector}}, {{policy}}, TTL={{ttl}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(erpc_cache_set_error_duration_seconds_bucket{network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",project=~\"${project:regex}\"}[1m])) by (le, project, network, category, connector, policy, ttl))", + "hide": false, + "legendFormat": "p95 - {{project}}, {{network}}, {{category}}, {{connector}}, {{policy}}, TTL={{ttl}}", + "range": true, + "refId": "B" + } + ], + "title": "Cache SET error duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 9558 + }, + "id": 38, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Total", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(increase(erpc_cache_get_success_hit_total{network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",project=~\"${project:regex}\"}[30s])) by (project, network, category, connector)", + "legendFormat": "{{project}}, {{network}}, {{category}}, {{connector}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache GET success-hit total", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 9558 + }, + "id": 39, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [ + "p90", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "90th %", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum(rate(erpc_cache_get_success_hit_duration_seconds_bucket{network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",project=~\"${project:regex}\"}[30s])) by (le, project, network, category, connector))", + "legendFormat": "p50 - {{project}}, {{network}}, {{category}}, {{connector}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(erpc_cache_get_success_hit_duration_seconds_bucket{network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",project=~\"${project:regex}\"}[30s])) by (le, project, network, category, connector))", + "hide": false, + "legendFormat": "p95 - {{project}}, {{network}}, {{category}}, {{connector}}", + "range": true, + "refId": "B" + } + ], + "title": "Cache GET success-hit duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 9568 + }, + "id": 42, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Total", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(increase(erpc_cache_get_success_miss_total{network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",project=~\"${project:regex}\"}[1m])) by (project, network, category, connector)", + "legendFormat": "{{project}}, {{network}}, {{category}}, {{connector}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache GET success-miss total", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 9568 + }, + "id": 43, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum(rate(erpc_cache_get_success_miss_duration_seconds_bucket{network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",project=~\"${project:regex}\"}[30s])) by (le, project, network, category, connector))", + "legendFormat": "p50 - {{project}}, {{network}}, {{category}}, {{connector}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(erpc_cache_get_success_miss_duration_seconds_bucket{network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",project=~\"${project:regex}\"}[30s])) by (le, project, network, category, connector))", + "hide": false, + "legendFormat": "p95 - {{project}}, {{network}}, {{category}}, {{connector}}", + "range": true, + "refId": "B" + } + ], + "title": "Cache GET success-miss duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 9578 + }, + "id": 40, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Total", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(increase(erpc_cache_get_error_total{network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",error!~\".*ErrRecordNotFound.*\",project=~\"${project:regex}\"}[1m])) by (region, project, network, category, connector, policy, ttl, error)", + "legendFormat": "{{region}} {{project}}, {{network}}, {{category}}, {{connector}}, {{policy}}, {{error}}, TTL={{ttl}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache GET error total", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 9578 + }, + "id": 41, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum(rate(erpc_cache_get_error_duration_seconds_bucket{network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",error!~\"ErrRecordNotFound.*\",project=~\"${project:regex}\"}[30s])) by (le, project, network, category, connector, policy, ttl, error))", + "legendFormat": "p50 - {{project}}, {{network}}, {{category}}, {{connector}}, {{policy}}, {{error}}, TTL={{ttl}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(erpc_cache_get_error_duration_seconds_bucket{network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",error!~\"ErrRecordNotFound.*\",project=~\"${project:regex}\"}[30s])) by (le, project, network, category, connector, policy, ttl, error))", + "hide": false, + "legendFormat": "p95 - {{project}}, {{network}}, {{category}}, {{connector}}, {{policy}}, {{error}}, TTL={{ttl}}", + "range": true, + "refId": "B" + } + ], + "title": "Cache GET error duration", + "type": "timeseries" + } + ], + "title": "Cache", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 84, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-RdYlGr" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 1285 + }, + "id": 121, + "interval": "1m", + "options": { + "displayMode": "basic", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "bottomk(30, sum by (network, category, finality) (\n erpc_consensus_misbehavior_detected_total{\n network=~\"${network:regex}\",\n project=~\"${project:regex}\",\n category=~\"${category:regex}\"\n }\n))", + "format": "heatmap", + "instant": true, + "interval": "", + "legendFormat": "{{network}}, {{category}}, {{finality}}", + "range": false, + "refId": "A" + } + ], + "title": "Misbehavior totals", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1296 + }, + "id": 92, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(increase(erpc_consensus_total{network=~\"${network:regex}\",project=~\"${project:regex}\",category=~\"${category:regex}\"}[1m])) by (network, category)", + "interval": "", + "legendFormat": "{{network}}, {{category}}", + "range": true, + "refId": "A" + } + ], + "title": "Total consensus operations", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1296 + }, + "id": 95, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(1, sum(rate(erpc_consensus_agreement_count_bucket{network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",project=~\"${project:regex}\",finality=~\"${finality:regex}\"}[1m])) by (le, network, category, finality))", + "interval": "", + "legendFormat": "{{network}}, {{category}}, {{finality}}", + "range": true, + "refId": "A" + } + ], + "title": "Avg. consensus agreements per operation", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1304 + }, + "id": 94, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Total", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(increase(erpc_consensus_responses_collected_count{network=~\"${network:regex}\",project=~\"${project:regex}\",category=~\"${category:regex}\",finality=~\"${finality:regex}\"}[1m])) by (network, category, finality, vendors) > 0", + "interval": "", + "legendFormat": "{{network}}, {{category}}, {{finality}}, {{vendors}}", + "range": true, + "refId": "A" + } + ], + "title": "Total consensus responses", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1304 + }, + "id": 93, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(1, sum(rate(erpc_consensus_responses_collected_bucket{network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",project=~\"${project:regex}\",finality=~\"${finality:regex}\"}[1m])) by (le, network, category, short_circuited, finality))", + "interval": "", + "legendFormat": "{{network}}, {{category}}, {{finality}}, short-circuited={{short_circuited}}", + "range": true, + "refId": "A" + } + ], + "title": "Avg. consensus responses per operation", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1312 + }, + "id": 96, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Total", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(increase(erpc_consensus_misbehavior_detected_total{network=~\"${network:regex}\",project=~\"${project:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",finality=~\"${finality:regex}\"}[1m])) by (network, upstream, category, finality, response_type, larger_than_consensus) > 0", + "interval": "", + "legendFormat": "{{network}}, {{upstream}}, {{category}}, {{finality}}, type={{response_type}} larger={{larger_than_consensus}}", + "range": true, + "refId": "A" + } + ], + "title": "Total consensus misbehaviors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1312 + }, + "id": 131, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(increase(erpc_consensus_short_circuit_total{network=~\"${network:regex}\",project=~\"${project:regex}\",category=~\"${category:regex}\"}[1m])) by (network, category, reason, finality) > 0", + "interval": "", + "legendFormat": "{{network}}, {{category}}, {{reason}}, {{finality}} ", + "range": true, + "refId": "A" + } + ], + "title": "Total consensus short-circuits", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1320 + }, + "id": 98, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(increase(erpc_consensus_errors_total{network=~\"${network:regex}\",project=~\"${project:regex}\",category=~\"${category:regex}\"}[1m])) by (network, category, category, error, finality) > 0", + "interval": "", + "legendFormat": "{{network}}, {{category}}, {{finality}}, {{error}}", + "range": true, + "refId": "A" + } + ], + "title": "Total consensus errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1320 + }, + "id": 97, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(increase(erpc_consensus_cancellations_total{network=~\"${network:regex}\",project=~\"${project:regex}\",category=~\"${category:regex}\"}[1m])) by (network, category, phase, finality) > 0", + "interval": "", + "legendFormat": "{{network}}, {{category}}, {{phase}}, {{finality}} ", + "range": true, + "refId": "A" + } + ], + "title": "Total consensus cancellations", + "type": "timeseries" + } + ], + "title": "Consensus", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 111, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 17, + "w": 16, + "x": 0, + "y": 1179 + }, + "id": 109, + "interval": "1m", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size", + "value": "" + }, + "yBuckets": { + "scale": { + "log": 2, + "type": "log" + }, + "value": "" + } + }, + "cellGap": 0, + "cellValues": { + "unit": "short" + }, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdYlGn", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1000 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto", + "value": "{{bucket}}" + }, + "tooltip": { + "mode": "multi", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "topk(20, sum by (bucket, network) (\n increase(erpc_network_evm_block_range_requested_total{\n network=~\"${network:regex}\",network!~\"${ignore_network:regex}\",category=~\"${category:regex}\",project=~\"${project:regex}\",vendor=~\"${vendor:regex}\",\n category=~\"(eth_getBlockByNumber|eth_getBlockByHash|eth_blockNumber|eth_getLogs|eth_getTransactionByHash|eth_getTransactionReceipt|eth_getBlockReceipts)\"\n }[$__range])\n))", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{bucket}} {{network}}", + "range": true, + "refId": "A" + } + ], + "title": "Access Blocks Heatmap Top 20 (Cacheable)", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 17, + "w": 8, + "x": 16, + "y": 1179 + }, + "id": 110, + "interval": "1m", + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "topk(10,\n (\n sum by (network, bucket) (\n increase(erpc_network_evm_block_range_requested_total{\n network=~\"${network:regex}\",network!~\"${ignore_network:regex}\",\n category=~\"${category:regex}\",\n project=~\"${project:regex}\",\n vendor=~\"${vendor:regex}\",\n # bucket!~\"(TIP|LATEST|FUTURE|FINALIZED)\",\n category=~\"(eth_getBlockByNumber|eth_getBlockByHash|eth_getLogs|eth_blockNumber|eth_getTransactionByHash|eth_getTransactionReceipt|eth_getBlockReceipts)\"\n }[$__range])\n )\n )\n)", + "instant": false, + "interval": "", + "legendFormat": "{{network}} / {{bucket}}", + "range": true, + "refId": "A" + } + ], + "title": "Top Block Buckets by Network (Cacheable)", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 17, + "w": 16, + "x": 0, + "y": 1196 + }, + "id": 112, + "interval": "1m", + "options": { + "calculate": false, + "cellGap": 6, + "cellValues": { + "unit": "short" + }, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdYlGn", + "steps": 45 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "multi", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (category, bucket) (\n increase(erpc_network_evm_block_range_requested_total{\n network=~\"${network:regex}\",category=~\"${category:regex}\",project=~\"${project:regex}\",\n category!~\"(eth_getBlockByNumber|eth_getBlockByHash|eth_getLogs|eth_blockNumber|eth_getTransactionByHash|eth_getTransactionReceipt|eth_getBlockReceipts)\"\n }[5m])\n)", + "instant": false, + "interval": "", + "legendFormat": "{{category}} / {{bucket}}", + "range": true, + "refId": "A" + } + ], + "title": "Access Blocks Heatmap by Method (Non-cacheable)", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 17, + "w": 8, + "x": 16, + "y": 1196 + }, + "id": 113, + "interval": "1m", + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "topk(20,\n sum by (network, bucket) (\n increase(erpc_network_evm_block_range_requested_total{\n network=~\"${network:regex}\",category=~\"${category:regex}\",project=~\"${project:regex}\",\n category!~\"(eth_getBlockByNumber|eth_getBlockByHash|eth_getLogs|eth_blockNumber|eth_getTransactionByHash|eth_getTransactionReceipt|eth_getBlockReceipts)\"\n }[$__range])\n )\n)", + "instant": false, + "interval": "", + "legendFormat": "{{network}} / {{bucket}}", + "range": true, + "refId": "A" + } + ], + "title": "Top Block Buckets (Non-cacheable)", + "type": "bargauge" + } + ], + "title": "Blocks Heatmap", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 123, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "description": "Dynamically tuned value for maxCount of different rate limit budgets. NOTE This value is SUM of all replicas (in a multiple container setup)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 114 + }, + "id": 30, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": true, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg by (budget, method) (erpc_rate_limiter_budget_max_count)\n", + "legendFormat": "{{budget}}, config_method={{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Rate-limit budget MaxCount", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 12, + "x": 0, + "y": 123 + }, + "id": 12, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(erpc_rate_limits_total{origin=\"project\",network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\"}[1m])) by (project, network, budget)", + "legendFormat": "{{project}}, {{network}}, {{budget}}", + "range": true, + "refId": "A" + } + ], + "title": "Project-level RPS rate limits", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 12, + "x": 12, + "y": 123 + }, + "id": 124, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(erpc_rate_limits_total{origin=\"network\",network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",budget!=\"\"}[1m])) by (project, network, user)", + "legendFormat": "{{project}}, {{network}}, {{user}}", + "range": true, + "refId": "A" + } + ], + "title": "Network-level RPS rate limits", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 12, + "x": 0, + "y": 136 + }, + "id": 125, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(erpc_rate_limits_total{origin=\"upstream\",budget=\"\",network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\"}[1m])) by (project, network, vendor)", + "legendFormat": "{{project}}, {{network}}, {{vendor}}", + "range": true, + "refId": "A" + } + ], + "title": "Upstream remote RPS rate limits", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 12, + "x": 12, + "y": 136 + }, + "id": 128, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(erpc_rate_limits_total{origin=\"upstream\",budget!=\"\",network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\"}[1m])) by (project, network, vendor)", + "legendFormat": "{{project}}, {{network}}, {{vendor}}", + "range": true, + "refId": "A" + } + ], + "title": "Upstream self-imposed RPS rate limits", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 12, + "x": 0, + "y": 149 + }, + "id": 126, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(erpc_rate_limits_total{origin=\"auth\",network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\"}[1m])) by (project, network, user, auth)", + "legendFormat": "{{project}}, {{network}}, {{user}}, {{auth}}", + "range": true, + "refId": "A" + } + ], + "title": "Auth-level RPS rate limits", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 12, + "x": 12, + "y": 149 + }, + "id": 129, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(erpc_rate_limits_total{network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\"}[1m])) by (project, network, category, origin)", + "legendFormat": "origin={{origin}}, project={{project}}, {{network}}, {{category}}", + "range": true, + "refId": "A" + } + ], + "title": "Method RPS rate limits", + "type": "timeseries" + } + ], + "title": "Rate Limiters", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 127, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 12, + "x": 0, + "y": 1181 + }, + "id": 114, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(erpc_network_evm_get_logs_range_requested_bucket{network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",vendor=~\"${vendor:regex}\",vendor!=\"\",project=~\"${project:regex}\",finality=~\"${finality:regex}\"}[1m])) by (le, project, network, user, finality))", + "legendFormat": "{{project}}, {{network}}, {{user}}, {{finality}}", + "range": true, + "refId": "A" + } + ], + "title": "eth_getLogs P99 ranges", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 12, + "x": 12, + "y": 1181 + }, + "id": 115, + "interval": "1m", + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum(rate(erpc_network_evm_get_logs_range_requested_bucket{network=~\"${network:regex}\",upstream=~\"${upstream:regex}\",category=~\"${category:regex}\",vendor=~\"${vendor:regex}\",vendor!=\"\",project=~\"${project:regex}\",finality=~\"${finality:regex}\"}[1m])) by (le, project, network, user, finality))", + "legendFormat": "{{project}}, {{network}}, {{user}}, {{finality}}", + "range": true, + "refId": "A" + } + ], + "title": "eth_getLogs P50 ranges", + "type": "timeseries" + } + ], + "title": "evm eth_getLogs", + "type": "row" + } + ], + "refresh": "10s", + "schemaVersion": 41, + "tags": [], + "templating": { + "list": [ + { + "current": {}, + "includeAll": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "definition": "label_values(client_name)", + "includeAll": true, + "multi": true, + "name": "client_name", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(client_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(cluster_key)", + "includeAll": true, + "multi": true, + "name": "cluster_key", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(cluster_key)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "definition": "label_values(project)", + "includeAll": true, + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(vendor)", + "includeAll": true, + "multi": true, + "name": "vendor", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(vendor)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "allValue": "NOTHING_TO_IGNORE", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "definition": "label_values(vendor)", + "includeAll": true, + "multi": true, + "name": "ignore_vendor", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(vendor)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(network)", + "includeAll": true, + "multi": true, + "name": "network", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(network)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "allValue": "NOTHING_TO_IGNORE", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "definition": "label_values(erpc_upstream_latest_block_number,network)", + "includeAll": true, + "multi": true, + "name": "ignore_network", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(erpc_upstream_latest_block_number,network)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(upstream)", + "includeAll": true, + "multi": true, + "name": "upstream", + "options": [], + "query": { + "query": "label_values(upstream)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "definition": "label_values(user)", + "includeAll": true, + "multi": true, + "name": "user", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(user)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(category)", + "includeAll": true, + "multi": true, + "name": "category", + "options": [], + "query": { + "query": "label_values(category)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_GROUNDCOVER-PROMETHEUS}" + }, + "definition": "label_values(finality)", + "includeAll": true, + "multi": true, + "name": "finality", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(finality)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "baseFilters": [], + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "filters": [], + "name": "Filters", + "type": "adhoc" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "eRPC Metrics", + "uid": "erpc_metrics", + "version": 160, + "weekStart": "" +} \ No newline at end of file diff --git a/listener/monitoring/grafana/dashboards/listener-fleet.json b/listener/monitoring/grafana/dashboards/listener-fleet.json new file mode 100644 index 0000000000..93784240de --- /dev/null +++ b/listener/monitoring/grafana/dashboards/listener-fleet.json @@ -0,0 +1,1272 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "Prometheus datasource that scrapes every listener deployment and the shared broker", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "12.0.0" }, + { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" }, + { "type": "panel", "id": "stat", "name": "Stat", "version": "" }, + { "type": "panel", "id": "timeseries", "name": "Time series", "version": "" }, + { "type": "panel", "id": "table", "name": "Table", "version": "" }, + { "type": "panel", "id": "row", "name": "Row", "version": "" } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { "type": "grafana", "uid": "-- Grafana --" }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Cross-chain fleet view + shared infrastructure (RPC provider, broker). Groups by chain_id (EIP-155 is unique per network, so no separate network label is required). No chain_id filter on RPC/broker panels since those metrics are shared across all listener instances.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": true, + "tags": [], + "targetBlank": false, + "title": "Per-chain deep dive", + "tooltip": "Jump to the chain-intrinsic dashboard", + "type": "link", + "url": "/d/listener-per-chain" + } + ], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "id": 1, + "title": "Fleet Overview", + "type": "row", + "panels": [] + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "description": "One row per chain. Sort by sync lag descending to surface degraded chains first.", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "custom": { "align": "auto", "cellOptions": { "type": "auto" } }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "sync lag" }, + "properties": [ + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 5 }, + { "color": "red", "value": 20 } + ] + } + }, + { "id": "custom.cellOptions", "value": { "type": "color-background", "mode": "basic" } } + ] + }, + { + "matcher": { "id": "byName", "options": "cursor/min" }, + "properties": [ + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "green", "value": 0.1 } + ] + } + }, + { "id": "custom.cellOptions", "value": { "type": "color-background", "mode": "basic" } } + ] + }, + { + "matcher": { "id": "byName", "options": "permanent 1h" }, + "properties": [ + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "red", "value": 1 } + ] + } + }, + { "id": "custom.cellOptions", "value": { "type": "color-background", "mode": "basic" } } + ] + } + ] + }, + "gridPos": { "h": 10, "w": 24, "x": 0, "y": 1 }, + "id": 2, + "options": { + "cellHeight": "sm", + "footer": { "countRows": false, "fields": "", "reducer": ["sum"], "show": false }, + "showHeader": true, + "sortBy": [{ "desc": true, "displayName": "sync lag" }] + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (chain_id) (listener_chain_height_block_number)", + "format": "table", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "chain height" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (chain_id) (listener_db_tip_block_number)", + "format": "table", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "db tip" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (chain_id) (listener_chain_height_block_number - listener_db_tip_block_number)", + "format": "table", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "sync lag" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "exemplar": false, + "expr": "60 * sum by (chain_id) (rate(listener_cursor_iterations_total[5m]))", + "format": "table", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "cursor/min" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (chain_id) (increase(listener_reorgs_total[24h]))", + "format": "table", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "reorgs 24h" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (chain_id) (increase(listener_transient_errors_total[1h]))", + "format": "table", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "transient 1h" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (chain_id) (increase(listener_permanent_errors_total[1h]))", + "format": "table", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "permanent 1h" + } + ], + "title": "Fleet Health (per chain_id)", + "transformations": [ + { + "id": "joinByField", + "options": { "byField": "chain_id", "mode": "outer" } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time 1": true, "Time 2": true, "Time 3": true, "Time 4": true, + "Time 5": true, "Time 6": true, "Time 7": true + }, + "renameByName": { + "Value #chain height": "chain height", + "Value #db tip": "db tip", + "Value #sync lag": "sync lag", + "Value #cursor/min": "cursor/min", + "Value #reorgs 24h": "reorgs 24h", + "Value #transient 1h": "transient 1h", + "Value #permanent 1h": "permanent 1h" + } + } + } + ], + "type": "table" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never" }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 11 }, + "id": 3, + "options": { + "legend": { "displayMode": "table", "placement": "right", "calcs": ["last", "max"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "sum by (chain_id) (listener_chain_height_block_number - listener_db_tip_block_number)", + "legendFormat": "chain {{chain_id}}", + "refId": "A" + } + ], + "title": "Sync Lag per Chain", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never" }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 11 }, + "id": 4, + "options": { + "legend": { "displayMode": "table", "placement": "right", "calcs": ["last", "mean"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "sum by (chain_id) (rate(listener_cursor_iterations_total[5m]))", + "legendFormat": "chain {{chain_id}}", + "refId": "A" + } + ], + "title": "Cursor Iteration Rate per Chain", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "drawStyle": "bars", "lineWidth": 1, "fillOpacity": 80, "showPoints": "never" }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 19 }, + "id": 5, + "options": { + "legend": { "displayMode": "table", "placement": "right", "calcs": ["sum"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "sum by (chain_id) (increase(listener_reorgs_total[1h]))", + "legendFormat": "chain {{chain_id}}", + "refId": "A" + } + ], + "title": "Reorgs per Chain (1h buckets)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 20, "showPoints": "never", "stacking": { "mode": "normal" } }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 19 }, + "id": 6, + "options": { + "legend": { "displayMode": "table", "placement": "right", "calcs": ["sum"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "sum by (chain_id) (rate(listener_transient_errors_total[$__rate_interval]))", + "legendFormat": "transient · chain {{chain_id}}", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "sum by (chain_id) (rate(listener_permanent_errors_total[$__rate_interval]))", + "legendFormat": "permanent · chain {{chain_id}}", + "refId": "B" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "sum by (chain_id) (rate(listener_publish_errors_total[$__rate_interval]))", + "legendFormat": "publish · chain {{chain_id}}", + "refId": "C" + } + ], + "title": "Listener Errors per Chain (transient / permanent / publish)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 27 }, + "id": 20, + "title": "RPC Provider (shared across chains)", + "type": "row", + "panels": [] + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never", "stacking": { "mode": "none" } }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 28 }, + "id": 21, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last", "mean"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "sum by (method, endpoint) (rate(listener_rpc_requests_total{endpoint=~\"$endpoint\"}[$__rate_interval]))", + "legendFormat": "{{method}} @ {{endpoint}}", + "refId": "A" + } + ], + "title": "RPC Request Rate (by method + endpoint)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never" }, + "unit": "percentunit", + "min": 0, + "max": 1 + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 28 }, + "id": 22, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last", "min"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "sum by (endpoint) (rate(listener_rpc_requests_total{endpoint=~\"$endpoint\", status=\"success\"}[$__rate_interval])) / clamp_min(sum by (endpoint) (rate(listener_rpc_requests_total{endpoint=~\"$endpoint\"}[$__rate_interval])), 0.001)", + "legendFormat": "{{endpoint}}", + "refId": "A" + } + ], + "title": "RPC Success Ratio (by endpoint)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never" }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 36 }, + "id": 23, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last", "max"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "listener_rpc_request_duration_seconds{endpoint=~\"$endpoint\", quantile=\"0.95\"}", + "legendFormat": "p95 · {{method}} @ {{endpoint}}", + "refId": "A" + } + ], + "title": "RPC p95 Latency (by method + endpoint)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 20, "showPoints": "never", "stacking": { "mode": "normal" } }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 36 }, + "id": 24, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["sum"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "sum by (endpoint, error_kind) (rate(listener_rpc_errors_total{endpoint=~\"$endpoint\"}[$__rate_interval]))", + "legendFormat": "{{error_kind}} @ {{endpoint}}", + "refId": "A" + } + ], + "title": "RPC Errors (by kind + endpoint)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never" }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 44 }, + "id": 25, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last", "min"] }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "listener_rpc_semaphore_available{endpoint=~\"$endpoint\"}", + "legendFormat": "{{endpoint}}", + "refId": "A" + } + ], + "title": "RPC Semaphore Available (by endpoint)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { "defaults": { "custom": { "align": "auto" } }, "overrides": [] }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 44 }, + "id": 26, + "options": { + "cellHeight": "sm", + "footer": { "countRows": false, "fields": "", "reducer": ["sum"], "show": false }, + "showHeader": true + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "exemplar": false, + "expr": "topk(10, sum by (method, endpoint, error_kind) (increase(listener_rpc_errors_total{endpoint=~\"$endpoint\"}[24h])))", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Top Failing (method, endpoint, error_kind) — 24h", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { "Time": true, "__name__": true }, + "renameByName": { "Value": "errors (24h)" } + } + }, + { + "id": "sortBy", + "options": { "fields": {}, "sort": [{ "desc": true, "field": "errors (24h)" }] } + } + ], + "type": "table" + }, + { + "collapsed": true, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 52 }, + "id": 30, + "title": "Broker — Publishing", + "type": "row", + "panels": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never" }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 53 }, + "id": 31, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last", "mean"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "sum by (topic) (rate(broker_messages_published_total{backend=~\"$backend\", topic=~\"$topic\"}[$__rate_interval]))", + "legendFormat": "{{topic}}", + "refId": "A" + } + ], + "title": "Messages Published (rate by topic)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 20, "showPoints": "never", "stacking": { "mode": "normal" } }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 53 }, + "id": 32, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["sum"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "sum by (topic, error_kind) (rate(broker_publish_errors_total{backend=~\"$backend\", topic=~\"$topic\"}[$__rate_interval]))", + "legendFormat": "{{error_kind}} · {{topic}}", + "refId": "A" + } + ], + "title": "Publish Errors (by kind + topic)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "description": "Average publish duration (_sum/_count) and publish throughput per topic.", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never" }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 61 }, + "id": 33, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last", "mean", "max"] }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "rate(broker_publish_duration_seconds_sum{backend=~\"$backend\", topic=~\"$topic\"}[$__rate_interval]) / clamp_min(rate(broker_publish_duration_seconds_count{backend=~\"$backend\", topic=~\"$topic\"}[$__rate_interval]), 0.0001)", + "legendFormat": "avg · {{topic}}", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "rate(broker_publish_duration_seconds_count{backend=~\"$backend\", topic=~\"$topic\"}[$__rate_interval])", + "legendFormat": "msgs/s · {{topic}}", + "refId": "B" + } + ], + "title": "Publish — Avg Duration & Throughput", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never" }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 61 }, + "id": 34, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last", "max"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "broker_publish_duration_seconds{backend=~\"$backend\", topic=~\"$topic\", quantile=\"0.95\"}", + "legendFormat": "p95 · {{topic}}", + "refId": "A" + } + ], + "title": "Publish p95 Latency (by topic)", + "type": "timeseries" + } + ] + }, + { + "collapsed": true, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 53 }, + "id": 40, + "title": "Broker — Consuming", + "type": "row", + "panels": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 20, "showPoints": "never", "stacking": { "mode": "normal" } }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 54 }, + "id": 41, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["sum"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "sum by (topic, outcome) (rate(broker_messages_consumed_total{backend=~\"$backend\", topic=~\"$topic\"}[$__rate_interval]))", + "legendFormat": "{{outcome}} · {{topic}}", + "refId": "A" + } + ], + "title": "Consumed Rate (by outcome)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "description": "Average handler duration (_sum/_count) and message throughput per topic.", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never" }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 54 }, + "id": 42, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last", "mean", "max"] }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "rate(broker_handler_duration_seconds_sum{backend=~\"$backend\", topic=~\"$topic\"}[$__rate_interval]) / clamp_min(rate(broker_handler_duration_seconds_count{backend=~\"$backend\", topic=~\"$topic\"}[$__rate_interval]), 0.0001)", + "legendFormat": "avg · {{topic}}", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "rate(broker_handler_duration_seconds_count{backend=~\"$backend\", topic=~\"$topic\"}[$__rate_interval])", + "legendFormat": "msgs/s · {{topic}}", + "refId": "B" + } + ], + "title": "Handler — Avg Duration & Throughput", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never" }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 62 }, + "id": 43, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last", "max"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "broker_handler_duration_seconds{backend=~\"$backend\", topic=~\"$topic\", quantile=\"0.95\"}", + "legendFormat": "p95 · {{topic}}", + "refId": "A" + } + ], + "title": "Handler p95 Latency (by topic)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 20, "showPoints": "never" }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 62 }, + "id": 44, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["sum"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "sum by (topic, reason) (rate(broker_messages_dead_lettered_total{backend=~\"$backend\", topic=~\"$topic\"}[$__rate_interval]))", + "legendFormat": "{{reason}} · {{topic}}", + "refId": "A" + } + ], + "title": "Dead-Letter Rate (by reason)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "description": "Average delivery count (_sum/_count) and p95 — Redis only. Sustained avg > 1 means messages routinely redelivered.", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never" }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 70 }, + "id": 45, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last", "mean", "max"] }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "rate(broker_message_delivery_count_sum{backend=~\"$backend\", topic=~\"$topic\"}[$__rate_interval]) / clamp_min(rate(broker_message_delivery_count_count{backend=~\"$backend\", topic=~\"$topic\"}[$__rate_interval]), 0.0001)", + "legendFormat": "avg · {{topic}}", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "broker_message_delivery_count{backend=~\"$backend\", topic=~\"$topic\", quantile=\"0.95\"}", + "legendFormat": "p95 · {{topic}}", + "refId": "B" + } + ], + "title": "Delivery Count (avg + p95)", + "type": "timeseries" + } + ] + }, + { + "collapsed": true, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 54 }, + "id": 50, + "title": "Broker — Queue Depth", + "type": "row", + "panels": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never" }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 55 }, + "id": 51, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last", "max"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "broker_queue_depth_principal{backend=~\"$backend\", topic=~\"$topic\"}", + "legendFormat": "{{topic}}", + "refId": "A" + } + ], + "title": "Principal Queue Depth", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never" }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 55 }, + "id": 52, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last", "max"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "broker_queue_depth_retry{backend=~\"$backend\", topic=~\"$topic\"}", + "legendFormat": "{{topic}}", + "refId": "A" + } + ], + "title": "Retry Queue Depth (AMQP)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "fixed", "fixedColor": "red" }, + "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 20, "showPoints": "never" }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 63 }, + "id": 53, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last", "max"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "broker_queue_depth_dead_letter{backend=~\"$backend\", topic=~\"$topic\"}", + "legendFormat": "{{topic}}", + "refId": "A" + } + ], + "title": "Dead-Letter Queue Depth", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never" }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 63 }, + "id": 54, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last", "max"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "broker_queue_depth_pending{backend=~\"$backend\", topic=~\"$topic\"}", + "legendFormat": "pending · {{topic}}", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "broker_queue_depth_lag{backend=~\"$backend\", topic=~\"$topic\"}", + "legendFormat": "lag · {{topic}}", + "refId": "B" + } + ], + "title": "Pending Entries / Consumer Group Lag (Redis)", + "type": "timeseries" + } + ] + }, + { + "collapsed": true, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 55 }, + "id": 60, + "title": "Broker — Circuit Breaker & Connection", + "type": "row", + "panels": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 30, "showPoints": "never" }, + "mappings": [ + { "type": "value", "options": { "0": { "text": "closed", "color": "green" } } }, + { "type": "value", "options": { "1": { "text": "open", "color": "red" } } }, + { "type": "value", "options": { "2": { "text": "half-open", "color": "yellow" } } } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 0.5 }, + { "color": "red", "value": 1 } + ] + }, + "max": 2, + "min": 0, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 56 }, + "id": 61, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last"] }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "broker_circuit_breaker_state{backend=~\"$backend\", topic=~\"$topic\"}", + "legendFormat": "{{topic}}", + "refId": "A" + } + ], + "title": "Circuit Breaker State (0=closed, 1=open, 2=half-open)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "fixed", "fixedColor": "red" }, + "custom": { "drawStyle": "bars", "lineWidth": 1, "fillOpacity": 80, "showPoints": "never" }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 56 }, + "id": 62, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["sum"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "increase(broker_circuit_breaker_trips_total{backend=~\"$backend\", topic=~\"$topic\"}[$__rate_interval])", + "legendFormat": "{{topic}}", + "refId": "A" + } + ], + "title": "Circuit Breaker Trips", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never" }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 64 }, + "id": 63, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last", "max"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "broker_circuit_breaker_consecutive_failures{backend=~\"$backend\", topic=~\"$topic\"}", + "legendFormat": "{{topic}}", + "refId": "A" + } + ], + "title": "Consecutive Failures", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [ + { "type": "value", "options": { "0": { "text": "disconnected", "color": "red" } } }, + { "type": "value", "options": { "1": { "text": "connected", "color": "green" } } } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "green", "value": 1 } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 6, "x": 12, "y": 64 }, + "id": 64, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "value_and_name" + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "broker_consumer_connected{backend=~\"$backend\"}", + "legendFormat": "{{backend}}", + "refId": "A" + } + ], + "title": "Consumer Connected", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never" }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 6, "x": 18, "y": 64 }, + "id": 65, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "rate(broker_consumer_reconnections_total{backend=~\"$backend\"}[$__rate_interval])", + "legendFormat": "{{backend}}", + "refId": "A" + } + ], + "title": "Reconnection Rate", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never" }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 72 }, + "id": 66, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["sum"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "rate(broker_claim_sweeper_messages_claimed_total{backend=~\"$backend\"}[$__rate_interval])", + "legendFormat": "claimed · {{backend}}", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "rate(broker_claim_sweeper_messages_dead_lettered_total{backend=~\"$backend\"}[$__rate_interval])", + "legendFormat": "dead-lettered · {{backend}}", + "refId": "B" + } + ], + "title": "Claim Sweeper (claimed vs dead-lettered)", + "type": "timeseries" + } + ] + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": ["listener", "blockchain", "evm", "fleet", "infrastructure"], + "templating": { + "list": [ + { + "current": { "selected": false, "text": "Prometheus", "value": "Prometheus" }, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { "selected": false, "text": "All", "value": "$__all" }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "definition": "label_values(listener_rpc_requests_total, endpoint)", + "hide": 0, + "includeAll": true, + "label": "Endpoint", + "multi": true, + "name": "endpoint", + "options": [], + "query": { "query": "label_values(listener_rpc_requests_total, endpoint)", "refId": "StandardVariableQuery" }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { "selected": false, "text": "All", "value": "$__all" }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "definition": "label_values(broker_messages_published_total, topic)", + "hide": 0, + "includeAll": true, + "label": "Topic", + "multi": true, + "name": "topic", + "options": [], + "query": { "query": "label_values(broker_messages_published_total, topic)", "refId": "StandardVariableQuery" }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { "selected": false, "text": "All", "value": "$__all" }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "definition": "label_values(broker_messages_published_total, backend)", + "hide": 0, + "includeAll": true, + "label": "Backend", + "multi": true, + "name": "backend", + "options": [], + "query": { "query": "label_values(broker_messages_published_total, backend)", "refId": "StandardVariableQuery" }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { "from": "now-6h", "to": "now" }, + "timepicker": {}, + "timezone": "", + "title": "Evm Listener - Fleet and Infrastructure", + "uid": "listener-fleet", + "version": 3, + "weekStart": "" +} diff --git a/listener/monitoring/grafana/dashboards/listener.json b/listener/monitoring/grafana/dashboards/listener.json new file mode 100644 index 0000000000..7600394ed9 --- /dev/null +++ b/listener/monitoring/grafana/dashboards/listener.json @@ -0,0 +1,876 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "Prometheus datasource that scrapes the listener `/metrics` endpoint", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "12.0.0" }, + { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" }, + { "type": "panel", "id": "stat", "name": "Stat", "version": "" }, + { "type": "panel", "id": "timeseries", "name": "Time series", "version": "" }, + { "type": "panel", "id": "row", "name": "Row", "version": "" } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { "type": "grafana", "uid": "-- Grafana --" }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Per-chain view of the EVM listener. Shows only metrics intrinsically labeled by chain_id. For RPC provider, broker, and cross-chain fleet metrics, see the 'Evm Listener - Fleet and Infrastructure' dashboard.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": true, + "tags": [], + "targetBlank": false, + "title": "Fleet & Infrastructure", + "tooltip": "Jump to the cross-chain + shared-infra dashboard", + "type": "link", + "url": "/d/listener-fleet" + } + ], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "id": 1, + "title": "Overview", + "type": "row", + "panels": [] + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "blue", "value": null }] }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 5, "x": 0, "y": 1 }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "listener_chain_height_block_number{chain_id=\"$chain_id\"}", + "legendFormat": "chain height", + "refId": "A" + } + ], + "title": "Chain Height", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 5, "x": 5, "y": 1 }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "listener_db_tip_block_number{chain_id=\"$chain_id\"}", + "legendFormat": "db tip", + "refId": "A" + } + ], + "title": "DB Tip", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 5 }, + { "color": "red", "value": 20 } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 10, "y": 1 }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "listener_chain_height_block_number{chain_id=\"$chain_id\"} - listener_db_tip_block_number{chain_id=\"$chain_id\"}", + "legendFormat": "lag", + "refId": "A" + } + ], + "title": "Sync Lag (blocks)", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "green", "value": 0.1 } + ] + }, + "unit": "cpm" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 5, "x": 14, "y": 1 }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "60 * rate(listener_cursor_iterations_total{chain_id=\"$chain_id\"}[5m])", + "legendFormat": "iterations/min", + "refId": "A" + } + ], + "title": "Cursor Rate (per min)", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "fixed", "fixedColor": "orange" }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "orange", "value": null }] }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 5, "x": 19, "y": 1 }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "increase(listener_reorgs_total{chain_id=\"$chain_id\"}[24h])", + "legendFormat": "reorgs 24h", + "refId": "A" + } + ], + "title": "Reorgs (24h)", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, + "id": 10, + "title": "Cursor & Sync", + "type": "row", + "panels": [] + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never" }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, + "id": 11, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last", "max"] }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "listener_chain_height_block_number{chain_id=\"$chain_id\"}", + "legendFormat": "chain height", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "listener_db_tip_block_number{chain_id=\"$chain_id\"}", + "legendFormat": "db tip", + "refId": "B" + } + ], + "title": "Block Heights (chain vs DB)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 20, "showPoints": "never" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 5 }, + { "color": "red", "value": 20 } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, + "id": 12, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "single", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "listener_chain_height_block_number{chain_id=\"$chain_id\"} - listener_db_tip_block_number{chain_id=\"$chain_id\"}", + "legendFormat": "lag", + "refId": "A" + } + ], + "title": "Sync Lag over Time", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never" }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 14 }, + "id": 13, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "single", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "rate(listener_cursor_iterations_total{chain_id=\"$chain_id\"}[$__rate_interval])", + "legendFormat": "iterations/s", + "refId": "A" + } + ], + "title": "Cursor Iteration Rate", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "fixed", "fixedColor": "orange" }, + "custom": { "drawStyle": "bars", "lineWidth": 1, "fillOpacity": 80, "showPoints": "never" }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 14 }, + "id": 14, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "single", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "increase(listener_reorgs_total{chain_id=\"$chain_id\"}[$__rate_interval])", + "legendFormat": "reorgs", + "refId": "A" + } + ], + "title": "Reorgs over Time", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 22 }, + "id": 20, + "title": "Block Fetch Performance", + "type": "row", + "panels": [] + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "description": "Average block fetch duration computed from summary _sum / _count. Summary metric — rolling quantiles are in the next panel.", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never" }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 23 }, + "id": 21, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last", "mean", "max"] }, + "tooltip": { "mode": "single", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "rate(listener_block_fetch_duration_seconds_sum{chain_id=\"$chain_id\"}[$__rate_interval]) / clamp_min(rate(listener_block_fetch_duration_seconds_count{chain_id=\"$chain_id\"}[$__rate_interval]), 0.0001)", + "legendFormat": "avg", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "rate(listener_block_fetch_duration_seconds_count{chain_id=\"$chain_id\"}[$__rate_interval])", + "legendFormat": "blocks/sec", + "refId": "B" + } + ], + "title": "Block Fetch — Avg Duration & Throughput", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never" }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 23 }, + "id": 22, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last", "max"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "listener_block_fetch_duration_seconds{chain_id=\"$chain_id\", quantile=\"0.5\"}", + "legendFormat": "p50", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "listener_block_fetch_duration_seconds{chain_id=\"$chain_id\", quantile=\"0.95\"}", + "legendFormat": "p95", + "refId": "B" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "listener_block_fetch_duration_seconds{chain_id=\"$chain_id\", quantile=\"0.99\"}", + "legendFormat": "p99", + "refId": "C" + } + ], + "title": "Block Fetch — Quantiles (rolling)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "description": "Average range fetch duration (time from cursor start to all blocks inserted) computed from summary _sum / _count. Plus throughput in ranges/sec.", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never" }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 31 }, + "id": 23, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last", "mean", "max"] }, + "tooltip": { "mode": "single", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "rate(listener_range_fetch_duration_seconds_sum{chain_id=\"$chain_id\"}[$__rate_interval]) / clamp_min(rate(listener_range_fetch_duration_seconds_count{chain_id=\"$chain_id\"}[$__rate_interval]), 0.0001)", + "legendFormat": "avg", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "rate(listener_range_fetch_duration_seconds_count{chain_id=\"$chain_id\"}[$__rate_interval])", + "legendFormat": "ranges/sec", + "refId": "B" + } + ], + "title": "Range Fetch — Avg Duration & Throughput", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never" }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 31 }, + "id": 24, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last", "max"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "listener_range_fetch_duration_seconds{chain_id=\"$chain_id\", quantile=\"0.5\"}", + "legendFormat": "p50", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "listener_range_fetch_duration_seconds{chain_id=\"$chain_id\", quantile=\"0.95\"}", + "legendFormat": "p95", + "refId": "B" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "listener_range_fetch_duration_seconds{chain_id=\"$chain_id\", quantile=\"0.99\"}", + "legendFormat": "p99", + "refId": "C" + } + ], + "title": "Range Fetch — Quantiles (rolling)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 39 }, + "id": 30, + "title": "Listener Errors", + "type": "row", + "panels": [] + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 20, "showPoints": "never", "stacking": { "mode": "normal" } }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 40 }, + "id": 31, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["sum"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "sum by (error_kind) (rate(listener_transient_errors_total{chain_id=\"$chain_id\"}[$__rate_interval]))", + "legendFormat": "{{error_kind}}", + "refId": "A" + } + ], + "title": "Transient Errors (by kind)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 20, "showPoints": "never", "stacking": { "mode": "normal" } }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 40 }, + "id": 32, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["sum"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "sum by (error_kind) (rate(listener_permanent_errors_total{chain_id=\"$chain_id\"}[$__rate_interval]))", + "legendFormat": "{{error_kind}}", + "refId": "A" + } + ], + "title": "Permanent Errors (by kind)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "fixed", "fixedColor": "red" }, + "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 20, "showPoints": "never" }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 40 }, + "id": 33, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "single", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "rate(listener_publish_errors_total{chain_id=\"$chain_id\"}[$__rate_interval])", + "legendFormat": "publish errors", + "refId": "A" + } + ], + "title": "Publish Errors Rate", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 48 }, + "id": 40, + "title": "Block Compute Verification", + "type": "row", + "panels": [] + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "fixed", "fixedColor": "orange" }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "orange", "value": null }] }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 8, "x": 0, "y": 49 }, + "id": 44, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "sum(increase(listener_compute_transaction_failure_total{chain_id=\"$chain_id\"}[24h]))", + "legendFormat": "tx root failures 24h", + "refId": "A" + } + ], + "title": "Transaction Root Failures (24h)", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "fixed", "fixedColor": "orange" }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "orange", "value": null }] }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 8, "x": 8, "y": 49 }, + "id": 45, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "sum(increase(listener_compute_receipt_failure_total{chain_id=\"$chain_id\"}[24h]))", + "legendFormat": "receipt root failures 24h", + "refId": "A" + } + ], + "title": "Receipt Root Failures (24h)", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "fixed", "fixedColor": "red" }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }] }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 8, "x": 16, "y": 49 }, + "id": 46, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "sum(increase(listener_compute_block_failure_total{chain_id=\"$chain_id\"}[24h]))", + "legendFormat": "block hash failures 24h", + "refId": "A" + } + ], + "title": "Block Hash Failures (24h)", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 20, "showPoints": "never", "stacking": { "mode": "normal" } }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 53 }, + "id": 41, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["sum"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "sum by (stalling) (rate(listener_compute_transaction_failure_total{chain_id=\"$chain_id\"}[$__rate_interval]))", + "legendFormat": "stalling={{stalling}}", + "refId": "A" + } + ], + "title": "Transaction Root Failures Rate", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 20, "showPoints": "never", "stacking": { "mode": "normal" } }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 53 }, + "id": 42, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["sum"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "sum by (stalling) (rate(listener_compute_receipt_failure_total{chain_id=\"$chain_id\"}[$__rate_interval]))", + "legendFormat": "stalling={{stalling}}", + "refId": "A" + } + ], + "title": "Receipt Root Failures Rate", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 20, "showPoints": "never", "stacking": { "mode": "normal" } }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 53 }, + "id": 43, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["sum"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "editorMode": "code", + "expr": "sum by (stalling) (rate(listener_compute_block_failure_total{chain_id=\"$chain_id\"}[$__rate_interval]))", + "legendFormat": "stalling={{stalling}}", + "refId": "A" + } + ], + "title": "Block Hash Failures Rate", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": ["listener", "blockchain", "evm", "per-chain"], + "templating": { + "list": [ + { + "current": { "selected": false, "text": "Prometheus", "value": "Prometheus" }, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { "selected": false, "text": "", "value": "" }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "definition": "label_values(listener_cursor_iterations_total, chain_id)", + "hide": 0, + "includeAll": false, + "label": "Chain ID", + "multi": false, + "name": "chain_id", + "options": [], + "query": { "query": "label_values(listener_cursor_iterations_total, chain_id)", "refId": "StandardVariableQuery" }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { "from": "now-1h", "to": "now" }, + "timepicker": {}, + "timezone": "", + "title": "Evm Listener - Per Chain", + "uid": "listener-per-chain", + "version": 3, + "weekStart": "" +} diff --git a/listener/monitoring/grafana/datasources/prometheus.yml b/listener/monitoring/grafana/datasources/prometheus.yml new file mode 100644 index 0000000000..1a57b69c8a --- /dev/null +++ b/listener/monitoring/grafana/datasources/prometheus.yml @@ -0,0 +1,9 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: true diff --git a/listener/monitoring/grafana/grafana.ini b/listener/monitoring/grafana/grafana.ini new file mode 100644 index 0000000000..71876dbda0 --- /dev/null +++ b/listener/monitoring/grafana/grafana.ini @@ -0,0 +1,41 @@ +[paths] +provisioning = /etc/grafana/provisioning + +[server] +http_addr = 0.0.0.0 +http_port = 3000 + +[security] +# If you're exposing Grafana to the internet, consider setting this to true +allow_embedding = false + +[users] +# Disable user signup / registration +allow_sign_up = false + +[auth.anonymous] +# Enable anonymous access +enabled = false + +[dashboards] +# This setting is crucial for discovering and loading dashboards +default_home_dashboard_path = /etc/grafana/provisioning/dashboards/erpc.json + +[datasources] +# Automatically update/delete datasources at Grafana startup +datasource_sync_ttl = 600 + +[unified_alerting] +# Enable the new alerting system if you plan to use Grafana for alerting +enabled = true + +[alerting] +# Disable the old alerting system +enabled = false + +[feature_toggles] +# Enable new features as needed +enable = publicDashboards + +[rendering] +server_url = http://localhost:3000 \ No newline at end of file diff --git a/listener/monitoring/prometheus/alert.rules b/listener/monitoring/prometheus/alert.rules new file mode 100644 index 0000000000..3e958e3da1 --- /dev/null +++ b/listener/monitoring/prometheus/alert.rules @@ -0,0 +1,56 @@ +groups: +- name: eRPC_Alerts + rules: + - alert: HighErrorRate + expr: sum(rate(erpc_upstream_request_errors_total[5m])) by (upstream) / sum(rate(erpc_upstream_request_total[5m])) by (upstream) > 0.05 + for: 5m + labels: + severity: warning + annotations: + summary: "High error rate for upstream {{ $labels.upstream }}" + description: "Error rate for upstream {{ $labels.upstream }} is {{ $value | humanizePercentage }} over the last 5 minutes" + + - alert: SlowRequests + expr: histogram_quantile(0.95, sum(rate(erpc_upstream_request_duration_seconds_budget[5m])) by (le, upstream)) > 1 + for: 5m + labels: + severity: warning + annotations: + summary: "Slow requests for upstream {{ $labels.upstream }}" + description: "95th percentile of request duration for upstream {{ $labels.upstream }} is {{ $value | humanizeDuration }} over the last 5 minutes" + + - alert: HighRateLimiting + expr: sum(rate(erpc_upstream_request_self_rate_limited_total[5m])) by (upstream) / sum(rate(erpc_upstream_request_total[5m])) by (upstream) > 0.1 + for: 5m + labels: + severity: warning + annotations: + summary: "High rate limiting for upstream {{ $labels.upstream }}" + description: "Rate limiting for upstream {{ $labels.upstream }} is {{ $value | humanizePercentage }} over the last 5 minutes" + + - alert: NetworkRateLimiting + expr: sum(rate(erpc_network_request_self_rate_limited_total[5m])) by (network) > 10 + for: 5m + labels: + severity: warning + annotations: + summary: "High network rate limiting for {{ $labels.network }}" + description: "Network rate limiting for {{ $labels.network }} is {{ $value | humanize }} requests/second over the last 5 minutes" + + - alert: HighRequestRate + expr: sum(rate(erpc_upstream_request_total[5m])) by (upstream) > 1000 + for: 5m + labels: + severity: warning + annotations: + summary: "High request rate for upstream {{ $labels.upstream }}" + description: "Request rate for upstream {{ $labels.upstream }} is {{ $value | humanize }} requests/second over the last 5 minutes" + + - alert: LowRequestRate + expr: sum(rate(erpc_upstream_request_total[5m])) by (upstream) < 1 + for: 15m + labels: + severity: warning + annotations: + summary: "Low request rate for upstream {{ $labels.upstream }}" + description: "Request rate for upstream {{ $labels.upstream }} is {{ $value | humanize }} requests/second over the last 15 minutes" \ No newline at end of file diff --git a/listener/monitoring/prometheus/prometheus.yml b/listener/monitoring/prometheus/prometheus.yml new file mode 100644 index 0000000000..d5b35c3cc8 --- /dev/null +++ b/listener/monitoring/prometheus/prometheus.yml @@ -0,0 +1,57 @@ +global: + scrape_interval: 10s + evaluation_interval: 10s + +rule_files: + - alert.rules + +scrape_configs: + - job_name: "prometheus" + static_configs: + - targets: + - "localhost:9090" + + - job_name: "erpc" + static_configs: + - targets: + - "erpc:4001" + + # ───────────────────────────────────────────────────────────────────── + # Listener deployments — one scrape job per chain. + # + # The `labels:` block attaches `chain_id` and `network` as external labels + # to EVERY metric scraped from that target, including broker metrics that + # do not carry `chain_id` intrinsically. This makes the same Grafana + # dashboard (monitoring/grafana/dashboards/listener.json) work for every + # chain without code changes, and lets the fleet dashboard + # (listener-fleet.json) slice metrics by chain. + # + # Uncomment and duplicate the block per chain you deploy. + # ───────────────────────────────────────────────────────────────────── + + - job_name: "listener-ethereum" + static_configs: + - targets: ["host.docker.internal:9091"] + labels: + chain_id: "1" + network: "ethereum-mainnet" + + - job_name: "polygon-mainnet" + static_configs: + - targets: ["host.docker.internal:9092"] + labels: + chain_id: "127" + network: "polygon-mainnet" + + - job_name: "avalanche-mainnet" + static_configs: + - targets: ["host.docker.internal:9093"] + labels: + chain_id: "43114" + network: "avalanche-mainnet" + +alerting: + alertmanagers: + - static_configs: + - targets: + # - alertmanager:9093 diff --git a/listener/recipe.json b/listener/recipe.json new file mode 100644 index 0000000000..7fcc47d336 --- /dev/null +++ b/listener/recipe.json @@ -0,0 +1 @@ +{"skeleton":{"manifests":[{"relative_path":"Cargo.toml","contents":"[workspace]\nmembers = [\"crates/consumer\", \"crates/listener_core\", \"crates/shared/broker\", \"crates/test-support\"]\nresolver = \"2\"\n\n[workspace.dependencies]\nalloy-json-rpc = \"1.6.1\"\nalloy-transport-ipc = \"1.6.1\"\nasync-trait = \"0.1.89\"\nfutures = \"0.3.31\"\nserde = \"1.0.228\"\nserde_json = \"1.0.149\"\nthiserror = \"2.0.18\"\ntokio-util = \"0.7\"\ntracing = \"0.1.44\"\n\n[workspace.dependencies.alloy]\nversion = \"1.6.1\"\nfeatures = [\"full\", \"trie\", \"rpc-types\", \"network\", \"consensus\", \"rlp\"]\n\n[workspace.dependencies.alloy-primitives]\nversion = \"1\"\nfeatures = [\"serde\"]\n\n[workspace.dependencies.tokio]\nversion = \"1.49.0\"\nfeatures = [\"rt-multi-thread\", \"sync\", \"time\"]\n\n[workspace.package]\nedition = \"2024\"\n\n[profile.release]\nopt-level = 3\nlto = true\ncodegen-units = 1\n\n[profile.release.package]\n","targets":[]},{"relative_path":"crates/consumer/Cargo.toml","contents":"[package]\nname = \"consumer\"\nversion = \"0.0.1\"\n\n[package.edition]\nworkspace = true\n\n[dependencies.alloy]\nworkspace = true\n\n[dependencies.async-trait]\nworkspace = true\n\n[dependencies.broker]\npath = \"../shared/broker\"\n\n[dependencies.primitives]\npath = \"../shared/primitives\"\n\n[dependencies.serde]\nworkspace = true\n\n[dependencies.serde_json]\nworkspace = true\n\n[dependencies.thiserror]\nworkspace = true\n\n[dependencies.tracing]\nworkspace = true\n\n[dev-dependencies.redis]\nversion = \"0.29\"\nfeatures = [\"tokio-comp\", \"aio\"]\n\n[dev-dependencies.test-support]\npath = \"../test-support\"\n\n[dev-dependencies.tokio]\nworkspace = true\nfeatures = [\"rt-multi-thread\", \"sync\", \"time\", \"macros\"]\n\n[[test]]\npath = \"tests/watch_e2e.rs\"\nname = \"watch_e2e\"\nrequired-features = []\n\n[lib]\npath = \"src/lib.rs\"\nname = \"consumer\"\nrequired-features = []\ncrate-type = [\"lib\"]\n","targets":[{"path":"src/lib.rs","kind":{"Lib":{"is_proc_macro":false}},"name":"consumer"},{"path":"tests/watch_e2e.rs","kind":"Test","name":"watch_e2e"}]},{"relative_path":"crates/listener_core/Cargo.toml","contents":"[package]\nname = \"listener_core\"\nversion = \"0.0.1\"\n\n[package.edition]\nworkspace = true\n\n[dependencies]\nanyhow = \"1.0\"\nderivative = \"2.2\"\nrand = \"0.9.2\"\ntracing-subscriber = \"0.3.22\"\nurl = \"2.5.8\"\n\n[dependencies.alloy]\nworkspace = true\n\n[dependencies.alloy-json-rpc]\nworkspace = true\n\n[dependencies.alloy-transport-ipc]\nworkspace = true\n\n[dependencies.async-trait]\nworkspace = true\n\n[dependencies.broker]\npath = \"../shared/broker\"\n\n[dependencies.chrono]\nversion = \"0.4\"\nfeatures = [\"serde\"]\n\n[dependencies.config]\nversion = \"0.14\"\nfeatures = [\"yaml\"]\n\n[dependencies.futures]\nworkspace = true\n\n[dependencies.primitives]\npath = \"../shared/primitives\"\n\n[dependencies.reqwest]\nversion = \"0.13.2\"\nfeatures = [\"json\"]\n\n[dependencies.serde]\nworkspace = true\nfeatures = [\"derive\"]\n\n[dependencies.serde_json]\nworkspace = true\n\n[dependencies.sqlx]\nversion = \"0.8\"\nfeatures = [\"runtime-tokio\", \"postgres\", \"migrate\", \"uuid\", \"chrono\"]\n\n[dependencies.thiserror]\nworkspace = true\n\n[dependencies.tokio]\nworkspace = true\nfeatures = [\"rt-multi-thread\", \"signal\"]\n\n[dependencies.tokio-util]\nworkspace = true\n\n[dependencies.tracing]\nworkspace = true\n\n[dependencies.uuid]\nversion = \"1\"\nfeatures = [\"v4\", \"serde\"]\n\n[[bin]]\npath = \"src/main.rs\"\nname = \"listener_core\"\nrequired-features = []\n\n[lib]\npath = \"src/lib.rs\"\nname = \"listener_core\"\nrequired-features = []\ncrate-type = [\"lib\"]\n","targets":[{"path":"src/lib.rs","kind":{"Lib":{"is_proc_macro":false}},"name":"listener_core"},{"path":"src/main.rs","kind":"Bin","name":"listener_core"}]},{"relative_path":"crates/shared/broker/Cargo.toml","contents":"[package]\nname = \"broker\"\nversion = \"0.0.1\"\n\n[package.edition]\nworkspace = true\n\n[dependencies]\narc-swap = \"1\"\n\n[dependencies.async-trait]\nworkspace = true\n\n[dependencies.futures]\nworkspace = true\n\n[dependencies.lapin]\nversion = \"4.0.0\"\noptional = true\n\n[dependencies.redis]\nversion = \"0.29\"\nfeatures = [\"tokio-comp\", \"aio\", \"streams\", \"connection-manager\"]\noptional = true\n\n[dependencies.serde]\nworkspace = true\nfeatures = [\"derive\"]\n\n[dependencies.serde_json]\nworkspace = true\nfeatures = [\"raw_value\"]\n\n[dependencies.thiserror]\nworkspace = true\n\n[dependencies.tokio]\nworkspace = true\nfeatures = [\"rt-multi-thread\", \"sync\", \"time\", \"macros\"]\n\n[dependencies.tokio-util]\nworkspace = true\n\n[dependencies.tracing]\nworkspace = true\n\n[dependencies.uuid]\nversion = \"1\"\nfeatures = [\"v7\"]\n\n[dev-dependencies.test-support]\npath = \"../../test-support\"\n\n[dev-dependencies.testcontainers]\nversion = \"0.23\"\nfeatures = [\"reusable-containers\"]\n\n[dev-dependencies.testcontainers-modules]\nversion = \"0.11\"\nfeatures = [\"redis\", \"rabbitmq\"]\n\n[features]\namqp = [\"dep:lapin\"]\ndefault = [\"redis\", \"amqp\"]\nredis = [\"dep:redis\"]\n\n[[test]]\npath = \"tests/amqp_e2e.rs\"\nname = \"amqp_e2e\"\nrequired-features = []\n\n[[test]]\npath = \"tests/broker_e2e.rs\"\nname = \"broker_e2e\"\nrequired-features = []\n\n[[test]]\npath = \"tests/redis_e2e.rs\"\nname = \"redis_e2e\"\nrequired-features = []\n\n[lib]\npath = \"src/lib.rs\"\nname = \"broker\"\nrequired-features = []\ncrate-type = [\"lib\"]\n","targets":[{"path":"src/lib.rs","kind":{"Lib":{"is_proc_macro":false}},"name":"broker"},{"path":"tests/amqp_e2e.rs","kind":"Test","name":"amqp_e2e"},{"path":"tests/broker_e2e.rs","kind":"Test","name":"broker_e2e"},{"path":"tests/redis_e2e.rs","kind":"Test","name":"redis_e2e"}]},{"relative_path":"crates/shared/primitives/Cargo.toml","contents":"[package]\nname = \"primitives\"\nversion = \"0.0.1\"\n\n[package.edition]\nworkspace = true\n\n[dependencies.alloy]\nversion = \"1.6.1\"\nfeatures = [\"full\"]\n\n[dependencies.serde]\nworkspace = true\nfeatures = [\"derive\"]\n\n[dependencies.thiserror]\nworkspace = true\n\n[dev-dependencies.serde_json]\nworkspace = true\n\n[lib]\npath = \"src/lib.rs\"\nname = \"primitives\"\nrequired-features = []\ncrate-type = [\"lib\"]\n","targets":[{"path":"src/lib.rs","kind":{"Lib":{"is_proc_macro":false}},"name":"primitives"}]},{"relative_path":"crates/test-support/Cargo.toml","contents":"[package]\nname = \"test-support\"\nversion = \"0.0.1\"\n\n[package.edition]\nworkspace = true\n\n[dependencies.testcontainers]\nversion = \"0.23\"\nfeatures = [\"reusable-containers\"]\n\n[dependencies.testcontainers-modules]\nversion = \"0.11\"\nfeatures = [\"redis\"]\n\n[dependencies.tokio]\nworkspace = true\n\n[lib]\npath = \"src/lib.rs\"\nname = \"test_support\"\nrequired-features = []\ncrate-type = [\"lib\"]\n","targets":[{"path":"src/lib.rs","kind":{"Lib":{"is_proc_macro":false}},"name":"test_support"}]}],"config_file":null,"lock_file":"version = 4\n\n[[package]]\nname = \"aes\"\nversion = \"0.8.4\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0\"\ndependencies = [\"cfg-if\", \"cipher\", \"cpufeatures 0.2.17\"]\n\n[[package]]\nname = \"ahash\"\nversion = \"0.8.12\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75\"\ndependencies = [\"cfg-if\", \"once_cell\", \"version_check\", \"zerocopy\"]\n\n[[package]]\nname = \"aho-corasick\"\nversion = \"1.1.4\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301\"\ndependencies = [\"memchr\"]\n\n[[package]]\nname = \"allocator-api2\"\nversion = \"0.2.21\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923\"\n\n[[package]]\nname = \"alloy\"\nversion = \"1.8.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"50ab0cd8afe573d1f7dc2353698a51b1f93aec362c8211e28cfd3948c6adba39\"\ndependencies = [\"alloy-consensus\", \"alloy-contract\", \"alloy-core\", \"alloy-eips\", \"alloy-genesis\", \"alloy-network\", \"alloy-provider\", \"alloy-pubsub\", \"alloy-rpc-client\", \"alloy-rpc-types\", \"alloy-serde\", \"alloy-signer\", \"alloy-signer-local\", \"alloy-transport\", \"alloy-transport-http\", \"alloy-transport-ipc\", \"alloy-transport-ws\", \"alloy-trie\"]\n\n[[package]]\nname = \"alloy-chains\"\nversion = \"0.2.33\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"f4e9e31d834fe25fe991b8884e4b9f0e59db4a97d86e05d1464d6899c013cd62\"\ndependencies = [\"alloy-primitives\", \"num_enum\", \"strum\"]\n\n[[package]]\nname = \"alloy-consensus\"\nversion = \"1.8.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"7f16daaf7e1f95f62c6c3bf8a3fc3d78b08ae9777810c0bb5e94966c7cd57ef0\"\ndependencies = [\"alloy-eips\", \"alloy-primitives\", \"alloy-rlp\", \"alloy-serde\", \"alloy-trie\", \"alloy-tx-macros\", \"auto_impl\", \"borsh\", \"c-kzg\", \"derive_more\", \"either\", \"k256\", \"once_cell\", \"rand 0.8.5\", \"secp256k1\", \"serde\", \"serde_json\", \"serde_with\", \"thiserror 2.0.18\"]\n\n[[package]]\nname = \"alloy-consensus-any\"\nversion = \"1.8.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"118998d9015332ab1b4720ae1f1e3009491966a0349938a1f43ff45a8a4c6299\"\ndependencies = [\"alloy-consensus\", \"alloy-eips\", \"alloy-primitives\", \"alloy-rlp\", \"alloy-serde\", \"serde\"]\n\n[[package]]\nname = \"alloy-contract\"\nversion = \"1.8.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"7ac9e0c34dc6bce643b182049cdfcca1b8ce7d9c260cbdd561f511873b7e26cd\"\ndependencies = [\"alloy-consensus\", \"alloy-dyn-abi\", \"alloy-json-abi\", \"alloy-network\", \"alloy-network-primitives\", \"alloy-primitives\", \"alloy-provider\", \"alloy-pubsub\", \"alloy-rpc-types-eth\", \"alloy-sol-types\", \"alloy-transport\", \"futures\", \"futures-util\", \"serde_json\", \"thiserror 2.0.18\", \"tracing\"]\n\n[[package]]\nname = \"alloy-core\"\nversion = \"1.5.7\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"23e8604b0c092fabc80d075ede181c9b9e596249c70b99253082d7e689836529\"\ndependencies = [\"alloy-dyn-abi\", \"alloy-json-abi\", \"alloy-primitives\", \"alloy-rlp\", \"alloy-sol-types\"]\n\n[[package]]\nname = \"alloy-dyn-abi\"\nversion = \"1.5.7\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"cc2db5c583aaef0255aa63a4fe827f826090142528bba48d1bf4119b62780cad\"\ndependencies = [\"alloy-json-abi\", \"alloy-primitives\", \"alloy-sol-type-parser\", \"alloy-sol-types\", \"itoa\", \"serde\", \"serde_json\", \"winnow 0.7.15\"]\n\n[[package]]\nname = \"alloy-eip2124\"\nversion = \"0.2.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"741bdd7499908b3aa0b159bba11e71c8cddd009a2c2eb7a06e825f1ec87900a5\"\ndependencies = [\"alloy-primitives\", \"alloy-rlp\", \"crc\", \"serde\", \"thiserror 2.0.18\"]\n\n[[package]]\nname = \"alloy-eip2930\"\nversion = \"0.2.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"9441120fa82df73e8959ae0e4ab8ade03de2aaae61be313fbf5746277847ce25\"\ndependencies = [\"alloy-primitives\", \"alloy-rlp\", \"borsh\", \"serde\"]\n\n[[package]]\nname = \"alloy-eip7702\"\nversion = \"0.6.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"2919c5a56a1007492da313e7a3b6d45ef5edc5d33416fdec63c0d7a2702a0d20\"\ndependencies = [\"alloy-primitives\", \"alloy-rlp\", \"borsh\", \"k256\", \"serde\", \"thiserror 2.0.18\"]\n\n[[package]]\nname = \"alloy-eip7928\"\nversion = \"0.3.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"f8222b1d88f9a6d03be84b0f5e76bb60cd83991b43ad8ab6477f0e4a7809b98d\"\ndependencies = [\"alloy-primitives\", \"alloy-rlp\", \"borsh\", \"serde\"]\n\n[[package]]\nname = \"alloy-eips\"\nversion = \"1.8.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"e6ef28c9fdad22d4eec52d894f5f2673a0895f1e5ef196734568e68c0f6caca8\"\ndependencies = [\"alloy-eip2124\", \"alloy-eip2930\", \"alloy-eip7702\", \"alloy-eip7928\", \"alloy-primitives\", \"alloy-rlp\", \"alloy-serde\", \"auto_impl\", \"borsh\", \"c-kzg\", \"derive_more\", \"either\", \"serde\", \"serde_with\", \"sha2\"]\n\n[[package]]\nname = \"alloy-genesis\"\nversion = \"1.8.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"bbf9480307b09d22876efb67d30cadd9013134c21f3a17ec9f93fd7536d38024\"\ndependencies = [\"alloy-eips\", \"alloy-primitives\", \"alloy-serde\", \"alloy-trie\", \"borsh\", \"serde\", \"serde_with\"]\n\n[[package]]\nname = \"alloy-json-abi\"\nversion = \"1.5.7\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"e9dbe713da0c737d9e5e387b0ba790eb98b14dd207fe53eef50e19a5a8ec3dac\"\ndependencies = [\"alloy-primitives\", \"alloy-sol-type-parser\", \"serde\", \"serde_json\"]\n\n[[package]]\nname = \"alloy-json-rpc\"\nversion = \"1.8.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"422d110f1c40f1f8d0e5562b0b649c35f345fccb7093d9f02729943dcd1eef71\"\ndependencies = [\"alloy-primitives\", \"alloy-sol-types\", \"http\", \"serde\", \"serde_json\", \"thiserror 2.0.18\", \"tracing\"]\n\n[[package]]\nname = \"alloy-network\"\nversion = \"1.8.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"7197a66d94c4de1591cdc16a9bcea5f8cccd0da81b865b49aef97b1b4016e0fa\"\ndependencies = [\"alloy-consensus\", \"alloy-consensus-any\", \"alloy-eips\", \"alloy-json-rpc\", \"alloy-network-primitives\", \"alloy-primitives\", \"alloy-rpc-types-any\", \"alloy-rpc-types-eth\", \"alloy-serde\", \"alloy-signer\", \"alloy-sol-types\", \"async-trait\", \"auto_impl\", \"derive_more\", \"futures-utils-wasm\", \"serde\", \"serde_json\", \"thiserror 2.0.18\"]\n\n[[package]]\nname = \"alloy-network-primitives\"\nversion = \"1.8.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"eb82711d59a43fdfd79727c99f270b974c784ec4eb5728a0d0d22f26716c87ef\"\ndependencies = [\"alloy-consensus\", \"alloy-eips\", \"alloy-primitives\", \"alloy-serde\", \"serde\"]\n\n[[package]]\nname = \"alloy-primitives\"\nversion = \"1.5.7\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"de3b431b4e72cd8bd0ec7a50b4be18e73dab74de0dba180eef171055e5d5926e\"\ndependencies = [\"alloy-rlp\", \"bytes\", \"cfg-if\", \"const-hex\", \"derive_more\", \"foldhash 0.2.0\", \"hashbrown 0.16.1\", \"indexmap 2.13.1\", \"itoa\", \"k256\", \"keccak-asm\", \"paste\", \"proptest\", \"rand 0.9.2\", \"rapidhash\", \"ruint\", \"rustc-hash\", \"serde\", \"sha3\"]\n\n[[package]]\nname = \"alloy-provider\"\nversion = \"1.8.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"bf6b18b929ef1d078b834c3631e9c925177f3b23ddc6fa08a722d13047205876\"\ndependencies = [\"alloy-chains\", \"alloy-consensus\", \"alloy-eips\", \"alloy-json-rpc\", \"alloy-network\", \"alloy-network-primitives\", \"alloy-primitives\", \"alloy-pubsub\", \"alloy-rpc-client\", \"alloy-rpc-types-anvil\", \"alloy-rpc-types-debug\", \"alloy-rpc-types-eth\", \"alloy-rpc-types-trace\", \"alloy-rpc-types-txpool\", \"alloy-signer\", \"alloy-sol-types\", \"alloy-transport\", \"alloy-transport-http\", \"alloy-transport-ipc\", \"alloy-transport-ws\", \"async-stream\", \"async-trait\", \"auto_impl\", \"dashmap\", \"either\", \"futures\", \"futures-utils-wasm\", \"lru\", \"parking_lot\", \"pin-project\", \"reqwest\", \"serde\", \"serde_json\", \"thiserror 2.0.18\", \"tokio\", \"tracing\", \"url\", \"wasmtimer\"]\n\n[[package]]\nname = \"alloy-pubsub\"\nversion = \"1.8.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"5ad54073131e7292d4e03e1aa2287730f737280eb160d8b579fb31939f558c11\"\ndependencies = [\"alloy-json-rpc\", \"alloy-primitives\", \"alloy-transport\", \"auto_impl\", \"bimap\", \"futures\", \"parking_lot\", \"serde\", \"serde_json\", \"tokio\", \"tokio-stream\", \"tower\", \"tracing\", \"wasmtimer\"]\n\n[[package]]\nname = \"alloy-rlp\"\nversion = \"0.3.15\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"dc90b1e703d3c03f4ff7f48e82dd0bc1c8211ab7d079cd836a06fcfeb06651cb\"\ndependencies = [\"alloy-rlp-derive\", \"arrayvec\", \"bytes\"]\n\n[[package]]\nname = \"alloy-rlp-derive\"\nversion = \"0.3.15\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"f36834a5c0a2fa56e171bf256c34d70fca07d0c0031583edea1c4946b7889c9e\"\ndependencies = [\"proc-macro2\", \"quote\", \"syn 2.0.117\"]\n\n[[package]]\nname = \"alloy-rpc-client\"\nversion = \"1.8.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"94fcc9604042ca80bd37aa5e232ea1cd851f337e31e2babbbb345bc0b1c30de3\"\ndependencies = [\"alloy-json-rpc\", \"alloy-primitives\", \"alloy-pubsub\", \"alloy-transport\", \"alloy-transport-http\", \"alloy-transport-ipc\", \"alloy-transport-ws\", \"futures\", \"pin-project\", \"reqwest\", \"serde\", \"serde_json\", \"tokio\", \"tokio-stream\", \"tower\", \"tracing\", \"url\", \"wasmtimer\"]\n\n[[package]]\nname = \"alloy-rpc-types\"\nversion = \"1.8.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"4faad925d3a669ffc15f43b3deec7fbdf2adeb28a4d6f9cf4bc661698c0f8f4b\"\ndependencies = [\"alloy-primitives\", \"alloy-rpc-types-anvil\", \"alloy-rpc-types-debug\", \"alloy-rpc-types-engine\", \"alloy-rpc-types-eth\", \"alloy-rpc-types-trace\", \"alloy-rpc-types-txpool\", \"alloy-serde\", \"serde\"]\n\n[[package]]\nname = \"alloy-rpc-types-anvil\"\nversion = \"1.8.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"47df51bedb3e6062cb9981187a51e86d0d64a4de66eb0855e9efe6574b044ddf\"\ndependencies = [\"alloy-primitives\", \"alloy-rpc-types-eth\", \"alloy-serde\", \"serde\"]\n\n[[package]]\nname = \"alloy-rpc-types-any\"\nversion = \"1.8.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"3823026d1ed239a40f12364fac50726c8daf1b6ab8077a97212c5123910429ed\"\ndependencies = [\"alloy-consensus-any\", \"alloy-rpc-types-eth\", \"alloy-serde\"]\n\n[[package]]\nname = \"alloy-rpc-types-debug\"\nversion = \"1.8.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"2145138f3214928f08cd13da3cb51ef7482b5920d8ac5a02ecd4e38d1a8f6d1e\"\ndependencies = [\"alloy-primitives\", \"derive_more\", \"serde\", \"serde_with\"]\n\n[[package]]\nname = \"alloy-rpc-types-engine\"\nversion = \"1.8.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"bb9b97b6e7965679ad22df297dda809b11cebc13405c1b537e5cffecc95834fa\"\ndependencies = [\"alloy-consensus\", \"alloy-eips\", \"alloy-primitives\", \"alloy-rlp\", \"alloy-serde\", \"derive_more\", \"rand 0.8.5\", \"serde\", \"strum\"]\n\n[[package]]\nname = \"alloy-rpc-types-eth\"\nversion = \"1.8.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"59c095f92c4e1ff4981d89e9aa02d5f98c762a1980ab66bec49c44be11349da2\"\ndependencies = [\"alloy-consensus\", \"alloy-consensus-any\", \"alloy-eips\", \"alloy-network-primitives\", \"alloy-primitives\", \"alloy-rlp\", \"alloy-serde\", \"alloy-sol-types\", \"itertools 0.14.0\", \"serde\", \"serde_json\", \"serde_with\", \"thiserror 2.0.18\"]\n\n[[package]]\nname = \"alloy-rpc-types-trace\"\nversion = \"1.8.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"2e5a4d010f86cd4e01e5205ec273911e538e1738e76d8bafe9ecd245910ea5a3\"\ndependencies = [\"alloy-primitives\", \"alloy-rpc-types-eth\", \"alloy-serde\", \"serde\", \"serde_json\", \"thiserror 2.0.18\"]\n\n[[package]]\nname = \"alloy-rpc-types-txpool\"\nversion = \"1.8.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"942d26a2ca8891b26de4a8529d21091e21c1093e27eb99698f1a86405c76b1ff\"\ndependencies = [\"alloy-primitives\", \"alloy-rpc-types-eth\", \"alloy-serde\", \"serde\"]\n\n[[package]]\nname = \"alloy-serde\"\nversion = \"1.8.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"11ece63b89294b8614ab3f483560c08d016930f842bf36da56bf0b764a15c11e\"\ndependencies = [\"alloy-primitives\", \"serde\", \"serde_json\"]\n\n[[package]]\nname = \"alloy-signer\"\nversion = \"1.8.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"43f447aefab0f1c0649f71edc33f590992d4e122bc35fb9cdbbf67d4421ace85\"\ndependencies = [\"alloy-primitives\", \"async-trait\", \"auto_impl\", \"either\", \"elliptic-curve\", \"k256\", \"thiserror 2.0.18\"]\n\n[[package]]\nname = \"alloy-signer-local\"\nversion = \"1.8.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"f721f4bf2e4812e5505aaf5de16ef3065a8e26b9139ac885862d00b5a55a659a\"\ndependencies = [\"alloy-consensus\", \"alloy-network\", \"alloy-primitives\", \"alloy-signer\", \"async-trait\", \"k256\", \"rand 0.8.5\", \"thiserror 2.0.18\"]\n\n[[package]]\nname = \"alloy-sol-macro\"\nversion = \"1.5.7\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"ab81bab693da9bb79f7a95b64b394718259fdd7e41dceeced4cad57cb71c4f6a\"\ndependencies = [\"alloy-sol-macro-expander\", \"alloy-sol-macro-input\", \"proc-macro-error2\", \"proc-macro2\", \"quote\", \"syn 2.0.117\"]\n\n[[package]]\nname = \"alloy-sol-macro-expander\"\nversion = \"1.5.7\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"489f1620bb7e2483fb5819ed01ab6edc1d2f93939dce35a5695085a1afd1d699\"\ndependencies = [\"alloy-json-abi\", \"alloy-sol-macro-input\", \"const-hex\", \"heck\", \"indexmap 2.13.1\", \"proc-macro-error2\", \"proc-macro2\", \"quote\", \"sha3\", \"syn 2.0.117\", \"syn-solidity\"]\n\n[[package]]\nname = \"alloy-sol-macro-input\"\nversion = \"1.5.7\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"56cef806ad22d4392c5fc83cf8f2089f988eb99c7067b4e0c6f1971fc1cca318\"\ndependencies = [\"alloy-json-abi\", \"const-hex\", \"dunce\", \"heck\", \"macro-string\", \"proc-macro2\", \"quote\", \"serde_json\", \"syn 2.0.117\", \"syn-solidity\"]\n\n[[package]]\nname = \"alloy-sol-type-parser\"\nversion = \"1.5.7\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"a6df77fea9d6a2a75c0ef8d2acbdfd92286cc599983d3175ccdc170d3433d249\"\ndependencies = [\"serde\", \"winnow 0.7.15\"]\n\n[[package]]\nname = \"alloy-sol-types\"\nversion = \"1.5.7\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"64612d29379782a5dde6f4b6570d9c756d734d760c0c94c254d361e678a6591f\"\ndependencies = [\"alloy-json-abi\", \"alloy-primitives\", \"alloy-sol-macro\", \"serde\"]\n\n[[package]]\nname = \"alloy-transport\"\nversion = \"1.8.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"8098f965442a9feb620965ba4b4be5e2b320f4ec5a3fff6bfa9e1ff7ef42bed1\"\ndependencies = [\"alloy-json-rpc\", \"auto_impl\", \"base64 0.22.1\", \"derive_more\", \"futures\", \"futures-utils-wasm\", \"parking_lot\", \"serde\", \"serde_json\", \"thiserror 2.0.18\", \"tokio\", \"tower\", \"tracing\", \"url\", \"wasmtimer\"]\n\n[[package]]\nname = \"alloy-transport-http\"\nversion = \"1.8.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"e8597d36d546e1dab822345ad563243ec3920e199322cb554ce56c8ef1a1e2e7\"\ndependencies = [\"alloy-json-rpc\", \"alloy-transport\", \"itertools 0.14.0\", \"reqwest\", \"serde_json\", \"tower\", \"tracing\", \"url\"]\n\n[[package]]\nname = \"alloy-transport-ipc\"\nversion = \"1.8.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"a1bd98c3870b8a44b79091dde5216a81d58ffbc1fd8ed61b776f9fee0f3bdf20\"\ndependencies = [\"alloy-json-rpc\", \"alloy-pubsub\", \"alloy-transport\", \"bytes\", \"futures\", \"interprocess\", \"pin-project\", \"serde\", \"serde_json\", \"tokio\", \"tokio-util\", \"tracing\"]\n\n[[package]]\nname = \"alloy-transport-ws\"\nversion = \"1.8.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"ec3ab7a72b180992881acc112628b7668337a19ce15293ee974600ea7b693691\"\ndependencies = [\"alloy-pubsub\", \"alloy-transport\", \"futures\", \"http\", \"rustls\", \"serde_json\", \"tokio\", \"tokio-tungstenite\", \"tracing\", \"url\", \"ws_stream_wasm\"]\n\n[[package]]\nname = \"alloy-trie\"\nversion = \"0.9.5\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"3f14b5d9b2c2173980202c6ff470d96e7c5e202c65a9f67884ad565226df7fbb\"\ndependencies = [\"alloy-primitives\", \"alloy-rlp\", \"derive_more\", \"nybbles\", \"serde\", \"smallvec\", \"thiserror 2.0.18\", \"tracing\"]\n\n[[package]]\nname = \"alloy-tx-macros\"\nversion = \"1.8.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"d69722eddcdf1ce096c3ab66cf8116999363f734eb36fe94a148f4f71c85da84\"\ndependencies = [\"darling\", \"proc-macro2\", \"quote\", \"syn 2.0.117\"]\n\n[[package]]\nname = \"amq-protocol\"\nversion = \"10.0.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"8032525e9bb1bb8aa556476de729106e972b9fb811e5db21ce462a4f0f057d03\"\ndependencies = [\"amq-protocol-tcp\", \"amq-protocol-types\", \"amq-protocol-uri\", \"cookie-factory\", \"nom 8.0.0\", \"serde\"]\n\n[[package]]\nname = \"amq-protocol-tcp\"\nversion = \"10.0.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"22f50ebc589843a42a1428b3e1b149164645bfe8c22a7ed0f128ad0af4aaad84\"\ndependencies = [\"amq-protocol-uri\", \"async-rs\", \"cfg-if\", \"tcp-stream\", \"tracing\"]\n\n[[package]]\nname = \"amq-protocol-types\"\nversion = \"10.0.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"12ffea0c942eb17ea55262e4cc57b223d8d6f896269b1313153f9215784dc2b8\"\ndependencies = [\"cookie-factory\", \"nom 8.0.0\", \"serde\", \"serde_json\"]\n\n[[package]]\nname = \"amq-protocol-uri\"\nversion = \"10.0.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"baa9f65c896cb658503e5547e262132ac356c26bc477afedfd8d3f324f4c5006\"\ndependencies = [\"amq-protocol-types\", \"percent-encoding\", \"url\"]\n\n[[package]]\nname = \"android_system_properties\"\nversion = \"0.1.5\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311\"\ndependencies = [\"libc\"]\n\n[[package]]\nname = \"anyhow\"\nversion = \"1.0.102\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c\"\n\n[[package]]\nname = \"arc-swap\"\nversion = \"1.9.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"a07d1f37ff60921c83bdfc7407723bdefe89b44b98a9b772f225c8f9d67141a6\"\ndependencies = [\"rustversion\"]\n\n[[package]]\nname = \"ark-ff\"\nversion = \"0.3.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"6b3235cc41ee7a12aaaf2c575a2ad7b46713a8a50bda2fc3b003a04845c05dd6\"\ndependencies = [\"ark-ff-asm 0.3.0\", \"ark-ff-macros 0.3.0\", \"ark-serialize 0.3.0\", \"ark-std 0.3.0\", \"derivative\", \"num-bigint\", \"num-traits\", \"paste\", \"rustc_version 0.3.3\", \"zeroize\"]\n\n[[package]]\nname = \"ark-ff\"\nversion = \"0.4.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"ec847af850f44ad29048935519032c33da8aa03340876d351dfab5660d2966ba\"\ndependencies = [\"ark-ff-asm 0.4.2\", \"ark-ff-macros 0.4.2\", \"ark-serialize 0.4.2\", \"ark-std 0.4.0\", \"derivative\", \"digest 0.10.7\", \"itertools 0.10.5\", \"num-bigint\", \"num-traits\", \"paste\", \"rustc_version 0.4.1\", \"zeroize\"]\n\n[[package]]\nname = \"ark-ff\"\nversion = \"0.5.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"a177aba0ed1e0fbb62aa9f6d0502e9b46dad8c2eab04c14258a1212d2557ea70\"\ndependencies = [\"ark-ff-asm 0.5.0\", \"ark-ff-macros 0.5.0\", \"ark-serialize 0.5.0\", \"ark-std 0.5.0\", \"arrayvec\", \"digest 0.10.7\", \"educe\", \"itertools 0.13.0\", \"num-bigint\", \"num-traits\", \"paste\", \"zeroize\"]\n\n[[package]]\nname = \"ark-ff-asm\"\nversion = \"0.3.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"db02d390bf6643fb404d3d22d31aee1c4bc4459600aef9113833d17e786c6e44\"\ndependencies = [\"quote\", \"syn 1.0.109\"]\n\n[[package]]\nname = \"ark-ff-asm\"\nversion = \"0.4.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"3ed4aa4fe255d0bc6d79373f7e31d2ea147bcf486cba1be5ba7ea85abdb92348\"\ndependencies = [\"quote\", \"syn 1.0.109\"]\n\n[[package]]\nname = \"ark-ff-asm\"\nversion = \"0.5.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"62945a2f7e6de02a31fe400aa489f0e0f5b2502e69f95f853adb82a96c7a6b60\"\ndependencies = [\"quote\", \"syn 2.0.117\"]\n\n[[package]]\nname = \"ark-ff-macros\"\nversion = \"0.3.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"db2fd794a08ccb318058009eefdf15bcaaaaf6f8161eb3345f907222bac38b20\"\ndependencies = [\"num-bigint\", \"num-traits\", \"quote\", \"syn 1.0.109\"]\n\n[[package]]\nname = \"ark-ff-macros\"\nversion = \"0.4.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"7abe79b0e4288889c4574159ab790824d0033b9fdcb2a112a3182fac2e514565\"\ndependencies = [\"num-bigint\", \"num-traits\", \"proc-macro2\", \"quote\", \"syn 1.0.109\"]\n\n[[package]]\nname = \"ark-ff-macros\"\nversion = \"0.5.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"09be120733ee33f7693ceaa202ca41accd5653b779563608f1234f78ae07c4b3\"\ndependencies = [\"num-bigint\", \"num-traits\", \"proc-macro2\", \"quote\", \"syn 2.0.117\"]\n\n[[package]]\nname = \"ark-serialize\"\nversion = \"0.3.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"1d6c2b318ee6e10f8c2853e73a83adc0ccb88995aa978d8a3408d492ab2ee671\"\ndependencies = [\"ark-std 0.3.0\", \"digest 0.9.0\"]\n\n[[package]]\nname = \"ark-serialize\"\nversion = \"0.4.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"adb7b85a02b83d2f22f89bd5cac66c9c89474240cb6207cb1efc16d098e822a5\"\ndependencies = [\"ark-std 0.4.0\", \"digest 0.10.7\", \"num-bigint\"]\n\n[[package]]\nname = \"ark-serialize\"\nversion = \"0.5.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"3f4d068aaf107ebcd7dfb52bc748f8030e0fc930ac8e360146ca54c1203088f7\"\ndependencies = [\"ark-std 0.5.0\", \"arrayvec\", \"digest 0.10.7\", \"num-bigint\"]\n\n[[package]]\nname = \"ark-std\"\nversion = \"0.3.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"1df2c09229cbc5a028b1d70e00fdb2acee28b1055dfb5ca73eea49c5a25c4e7c\"\ndependencies = [\"num-traits\", \"rand 0.8.5\"]\n\n[[package]]\nname = \"ark-std\"\nversion = \"0.4.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"94893f1e0c6eeab764ade8dc4c0db24caf4fe7cbbaafc0eba0a9030f447b5185\"\ndependencies = [\"num-traits\", \"rand 0.8.5\"]\n\n[[package]]\nname = \"ark-std\"\nversion = \"0.5.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"246a225cc6131e9ee4f24619af0f19d67761fff15d7ccc22e42b80846e69449a\"\ndependencies = [\"num-traits\", \"rand 0.8.5\"]\n\n[[package]]\nname = \"arraydeque\"\nversion = \"0.5.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236\"\n\n[[package]]\nname = \"arrayvec\"\nversion = \"0.7.6\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50\"\n\n[[package]]\nname = \"asn1-rs\"\nversion = \"0.7.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"56624a96882bb8c26d61312ae18cb45868e5a9992ea73c58e45c3101e56a1e60\"\ndependencies = [\"asn1-rs-derive\", \"asn1-rs-impl\", \"displaydoc\", \"nom 7.1.3\", \"num-traits\", \"rusticata-macros\", \"thiserror 2.0.18\", \"time\"]\n\n[[package]]\nname = \"asn1-rs-derive\"\nversion = \"0.6.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c\"\ndependencies = [\"proc-macro2\", \"quote\", \"syn 2.0.117\", \"synstructure\"]\n\n[[package]]\nname = \"asn1-rs-impl\"\nversion = \"0.2.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7\"\ndependencies = [\"proc-macro2\", \"quote\", \"syn 2.0.117\"]\n\n[[package]]\nname = \"async-channel\"\nversion = \"2.5.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2\"\ndependencies = [\"concurrent-queue\", \"event-listener-strategy\", \"futures-core\", \"pin-project-lite\"]\n\n[[package]]\nname = \"async-compat\"\nversion = \"0.2.5\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"a1ba85bc55464dcbf728b56d97e119d673f4cf9062be330a9a26f3acf504a590\"\ndependencies = [\"futures-core\", \"futures-io\", \"once_cell\", \"pin-project-lite\", \"tokio\"]\n\n[[package]]\nname = \"async-executor\"\nversion = \"1.14.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"c96bf972d85afc50bf5ab8fe2d54d1586b4e0b46c97c50a0c9e71e2f7bcd812a\"\ndependencies = [\"async-task\", \"concurrent-queue\", \"fastrand\", \"futures-lite\", \"pin-project-lite\", \"slab\"]\n\n[[package]]\nname = \"async-global-executor\"\nversion = \"3.1.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"13f937e26114b93193065fd44f507aa2e9169ad0cdabbb996920b1fe1ddea7ba\"\ndependencies = [\"async-channel\", \"async-executor\", \"async-lock\", \"blocking\", \"futures-lite\", \"tokio\"]\n\n[[package]]\nname = \"async-lock\"\nversion = \"3.4.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311\"\ndependencies = [\"event-listener\", \"event-listener-strategy\", \"pin-project-lite\"]\n\n[[package]]\nname = \"async-rs\"\nversion = \"0.8.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"1e7d98bcae2752f5f3edb17288ff34b799760be54f63261073eed9f6982367b5\"\ndependencies = [\"async-compat\", \"async-global-executor\", \"async-trait\", \"cfg-if\", \"futures-core\", \"futures-io\", \"hickory-resolver\", \"tokio\", \"tokio-stream\"]\n\n[[package]]\nname = \"async-stream\"\nversion = \"0.3.6\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476\"\ndependencies = [\"async-stream-impl\", \"futures-core\", \"pin-project-lite\"]\n\n[[package]]\nname = \"async-stream-impl\"\nversion = \"0.3.6\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d\"\ndependencies = [\"proc-macro2\", \"quote\", \"syn 2.0.117\"]\n\n[[package]]\nname = \"async-task\"\nversion = \"4.7.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de\"\n\n[[package]]\nname = \"async-trait\"\nversion = \"0.1.89\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb\"\ndependencies = [\"proc-macro2\", \"quote\", \"syn 2.0.117\"]\n\n[[package]]\nname = \"async_io_stream\"\nversion = \"0.3.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"b6d7b9decdf35d8908a7e3ef02f64c5e9b1695e230154c0e8de3969142d9b94c\"\ndependencies = [\"futures\", \"pharos\", \"rustc_version 0.4.1\"]\n\n[[package]]\nname = \"atoi\"\nversion = \"2.0.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528\"\ndependencies = [\"num-traits\"]\n\n[[package]]\nname = \"atomic-waker\"\nversion = \"1.1.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0\"\n\n[[package]]\nname = \"auto_impl\"\nversion = \"1.3.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"ffdcb70bdbc4d478427380519163274ac86e52916e10f0a8889adf0f96d3fee7\"\ndependencies = [\"proc-macro2\", \"quote\", \"syn 2.0.117\"]\n\n[[package]]\nname = \"autocfg\"\nversion = \"1.5.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8\"\n\n[[package]]\nname = \"aws-lc-rs\"\nversion = \"1.16.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc\"\ndependencies = [\"aws-lc-sys\", \"zeroize\"]\n\n[[package]]\nname = \"aws-lc-sys\"\nversion = \"0.39.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"83a25cf98105baa966497416dbd42565ce3a8cf8dbfd59803ec9ad46f3126399\"\ndependencies = [\"cc\", \"cmake\", \"dunce\", \"fs_extra\"]\n\n[[package]]\nname = \"backon\"\nversion = \"1.6.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef\"\ndependencies = [\"fastrand\"]\n\n[[package]]\nname = \"base16ct\"\nversion = \"0.2.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf\"\n\n[[package]]\nname = \"base64\"\nversion = \"0.21.7\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567\"\n\n[[package]]\nname = \"base64\"\nversion = \"0.22.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6\"\n\n[[package]]\nname = \"base64ct\"\nversion = \"1.8.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06\"\n\n[[package]]\nname = \"bimap\"\nversion = \"0.6.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"230c5f1ca6a325a32553f8640d31ac9b49f2411e901e427570154868b46da4f7\"\n\n[[package]]\nname = \"bit-set\"\nversion = \"0.8.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3\"\ndependencies = [\"bit-vec\"]\n\n[[package]]\nname = \"bit-vec\"\nversion = \"0.8.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7\"\n\n[[package]]\nname = \"bitcoin-io\"\nversion = \"0.1.4\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"2dee39a0ee5b4095224a0cfc6bf4cc1baf0f9624b96b367e53b66d974e51d953\"\n\n[[package]]\nname = \"bitcoin_hashes\"\nversion = \"0.14.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"26ec84b80c482df901772e931a9a681e26a1b9ee2302edeff23cb30328745c8b\"\ndependencies = [\"bitcoin-io\", \"hex-conservative\"]\n\n[[package]]\nname = \"bitflags\"\nversion = \"1.3.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a\"\n\n[[package]]\nname = \"bitflags\"\nversion = \"2.11.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af\"\ndependencies = [\"serde_core\"]\n\n[[package]]\nname = \"bitvec\"\nversion = \"1.0.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c\"\ndependencies = [\"funty\", \"radium\", \"tap\", \"wyz\"]\n\n[[package]]\nname = \"block-buffer\"\nversion = \"0.10.4\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71\"\ndependencies = [\"generic-array\"]\n\n[[package]]\nname = \"block-padding\"\nversion = \"0.3.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93\"\ndependencies = [\"generic-array\"]\n\n[[package]]\nname = \"blocking\"\nversion = \"1.6.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"e83f8d02be6967315521be875afa792a316e28d57b5a2d401897e2a7921b7f21\"\ndependencies = [\"async-channel\", \"async-task\", \"futures-io\", \"futures-lite\", \"piper\"]\n\n[[package]]\nname = \"blst\"\nversion = \"0.3.16\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"dcdb4c7013139a150f9fc55d123186dbfaba0d912817466282c73ac49e71fb45\"\ndependencies = [\"cc\", \"glob\", \"threadpool\", \"zeroize\"]\n\n[[package]]\nname = \"bollard\"\nversion = \"0.18.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"97ccca1260af6a459d75994ad5acc1651bcabcbdbc41467cc9786519ab854c30\"\ndependencies = [\"base64 0.22.1\", \"bollard-stubs\", \"bytes\", \"futures-core\", \"futures-util\", \"hex\", \"home\", \"http\", \"http-body-util\", \"hyper\", \"hyper-named-pipe\", \"hyper-rustls\", \"hyper-util\", \"hyperlocal\", \"log\", \"pin-project-lite\", \"rustls\", \"rustls-native-certs\", \"rustls-pemfile\", \"rustls-pki-types\", \"serde\", \"serde_derive\", \"serde_json\", \"serde_repr\", \"serde_urlencoded\", \"thiserror 2.0.18\", \"tokio\", \"tokio-util\", \"tower-service\", \"url\", \"winapi\"]\n\n[[package]]\nname = \"bollard-stubs\"\nversion = \"1.47.1-rc.27.3.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"3f179cfbddb6e77a5472703d4b30436bff32929c0aa8a9008ecf23d1d3cdd0da\"\ndependencies = [\"serde\", \"serde_repr\", \"serde_with\"]\n\n[[package]]\nname = \"borsh\"\nversion = \"1.6.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"cfd1e3f8955a5d7de9fab72fc8373fade9fb8a703968cb200ae3dc6cf08e185a\"\ndependencies = [\"borsh-derive\", \"bytes\", \"cfg_aliases\"]\n\n[[package]]\nname = \"borsh-derive\"\nversion = \"1.6.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"bfcfdc083699101d5a7965e49925975f2f55060f94f9a05e7187be95d530ca59\"\ndependencies = [\"once_cell\", \"proc-macro-crate\", \"proc-macro2\", \"quote\", \"syn 2.0.117\"]\n\n[[package]]\nname = \"broker\"\nversion = \"0.0.1\"\ndependencies = [\"arc-swap\", \"async-trait\", \"futures\", \"lapin\", \"redis\", \"serde\", \"serde_json\", \"test-support\", \"testcontainers\", \"testcontainers-modules\", \"thiserror 2.0.18\", \"tokio\", \"tokio-util\", \"tracing\", \"uuid\"]\n\n[[package]]\nname = \"bumpalo\"\nversion = \"3.20.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb\"\n\n[[package]]\nname = \"byte-slice-cast\"\nversion = \"1.2.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"7575182f7272186991736b70173b0ea045398f984bf5ebbb3804736ce1330c9d\"\n\n[[package]]\nname = \"byteorder\"\nversion = \"1.5.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b\"\n\n[[package]]\nname = \"bytes\"\nversion = \"1.11.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33\"\ndependencies = [\"serde\"]\n\n[[package]]\nname = \"c-kzg\"\nversion = \"2.1.7\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"6648ed1e4ea8e8a1a4a2c78e1cda29a3fd500bc622899c340d8525ea9a76b24a\"\ndependencies = [\"blst\", \"cc\", \"glob\", \"hex\", \"libc\", \"once_cell\", \"serde\"]\n\n[[package]]\nname = \"cbc\"\nversion = \"0.1.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6\"\ndependencies = [\"cipher\"]\n\n[[package]]\nname = \"cc\"\nversion = \"1.2.59\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"b7a4d3ec6524d28a329fc53654bbadc9bdd7b0431f5d65f1a56ffb28a1ee5283\"\ndependencies = [\"find-msvc-tools\", \"jobserver\", \"libc\", \"shlex\"]\n\n[[package]]\nname = \"cesu8\"\nversion = \"1.1.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c\"\n\n[[package]]\nname = \"cfg-if\"\nversion = \"1.0.4\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801\"\n\n[[package]]\nname = \"cfg_aliases\"\nversion = \"0.2.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724\"\n\n[[package]]\nname = \"chacha20\"\nversion = \"0.10.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601\"\ndependencies = [\"cfg-if\", \"cpufeatures 0.3.0\", \"rand_core 0.10.0\"]\n\n[[package]]\nname = \"chrono\"\nversion = \"0.4.44\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0\"\ndependencies = [\"iana-time-zone\", \"js-sys\", \"num-traits\", \"serde\", \"wasm-bindgen\", \"windows-link\"]\n\n[[package]]\nname = \"cipher\"\nversion = \"0.4.4\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad\"\ndependencies = [\"crypto-common\", \"inout\"]\n\n[[package]]\nname = \"cmake\"\nversion = \"0.1.58\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678\"\ndependencies = [\"cc\"]\n\n[[package]]\nname = \"cms\"\nversion = \"0.2.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"7b77c319abfd5219629c45c34c89ba945ed3c5e49fcde9d16b6c3885f118a730\"\ndependencies = [\"const-oid\", \"der\", \"spki\", \"x509-cert\"]\n\n[[package]]\nname = \"combine\"\nversion = \"4.6.7\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd\"\ndependencies = [\"bytes\", \"futures-core\", \"memchr\", \"pin-project-lite\", \"tokio\", \"tokio-util\"]\n\n[[package]]\nname = \"concurrent-queue\"\nversion = \"2.5.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973\"\ndependencies = [\"crossbeam-utils\"]\n\n[[package]]\nname = \"config\"\nversion = \"0.14.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"68578f196d2a33ff61b27fae256c3164f65e36382648e30666dde05b8cc9dfdf\"\ndependencies = [\"async-trait\", \"convert_case 0.6.0\", \"json5\", \"nom 7.1.3\", \"pathdiff\", \"ron\", \"rust-ini\", \"serde\", \"serde_json\", \"toml\", \"yaml-rust2\"]\n\n[[package]]\nname = \"const-hex\"\nversion = \"1.18.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"531185e432bb31db1ecda541e9e7ab21468d4d844ad7505e0546a49b4945d49b\"\ndependencies = [\"cfg-if\", \"cpufeatures 0.2.17\", \"proptest\", \"serde_core\"]\n\n[[package]]\nname = \"const-oid\"\nversion = \"0.9.6\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8\"\n\n[[package]]\nname = \"const-random\"\nversion = \"0.1.18\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359\"\ndependencies = [\"const-random-macro\"]\n\n[[package]]\nname = \"const-random-macro\"\nversion = \"0.1.16\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e\"\ndependencies = [\"getrandom 0.2.17\", \"once_cell\", \"tiny-keccak\"]\n\n[[package]]\nname = \"const_format\"\nversion = \"0.2.35\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"7faa7469a93a566e9ccc1c73fe783b4a65c274c5ace346038dca9c39fe0030ad\"\ndependencies = [\"const_format_proc_macros\"]\n\n[[package]]\nname = \"const_format_proc_macros\"\nversion = \"0.2.34\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"1d57c2eccfb16dbac1f4e61e206105db5820c9d26c3c472bc17c774259ef7744\"\ndependencies = [\"proc-macro2\", \"quote\", \"unicode-xid\"]\n\n[[package]]\nname = \"consumer\"\nversion = \"0.0.1\"\ndependencies = [\"alloy\", \"async-trait\", \"broker\", \"primitives\", \"redis\", \"serde\", \"serde_json\", \"test-support\", \"thiserror 2.0.18\", \"tokio\", \"tracing\"]\n\n[[package]]\nname = \"convert_case\"\nversion = \"0.6.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca\"\ndependencies = [\"unicode-segmentation\"]\n\n[[package]]\nname = \"convert_case\"\nversion = \"0.10.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9\"\ndependencies = [\"unicode-segmentation\"]\n\n[[package]]\nname = \"cookie-factory\"\nversion = \"0.3.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"9885fa71e26b8ab7855e2ec7cae6e9b380edff76cd052e07c683a0319d51b3a2\"\n\n[[package]]\nname = \"core-foundation\"\nversion = \"0.9.4\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f\"\ndependencies = [\"core-foundation-sys\", \"libc\"]\n\n[[package]]\nname = \"core-foundation\"\nversion = \"0.10.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6\"\ndependencies = [\"core-foundation-sys\", \"libc\"]\n\n[[package]]\nname = \"core-foundation-sys\"\nversion = \"0.8.7\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b\"\n\n[[package]]\nname = \"cpufeatures\"\nversion = \"0.2.17\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280\"\ndependencies = [\"libc\"]\n\n[[package]]\nname = \"cpufeatures\"\nversion = \"0.3.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201\"\ndependencies = [\"libc\"]\n\n[[package]]\nname = \"crc\"\nversion = \"3.4.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d\"\ndependencies = [\"crc-catalog\"]\n\n[[package]]\nname = \"crc-catalog\"\nversion = \"2.4.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5\"\n\n[[package]]\nname = \"critical-section\"\nversion = \"1.2.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b\"\n\n[[package]]\nname = \"crossbeam-channel\"\nversion = \"0.5.15\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2\"\ndependencies = [\"crossbeam-utils\"]\n\n[[package]]\nname = \"crossbeam-epoch\"\nversion = \"0.9.18\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e\"\ndependencies = [\"crossbeam-utils\"]\n\n[[package]]\nname = \"crossbeam-queue\"\nversion = \"0.3.12\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115\"\ndependencies = [\"crossbeam-utils\"]\n\n[[package]]\nname = \"crossbeam-utils\"\nversion = \"0.8.21\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28\"\n\n[[package]]\nname = \"crunchy\"\nversion = \"0.2.4\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5\"\n\n[[package]]\nname = \"crypto-bigint\"\nversion = \"0.5.5\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76\"\ndependencies = [\"generic-array\", \"rand_core 0.6.4\", \"subtle\", \"zeroize\"]\n\n[[package]]\nname = \"crypto-common\"\nversion = \"0.1.7\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a\"\ndependencies = [\"generic-array\", \"typenum\"]\n\n[[package]]\nname = \"darling\"\nversion = \"0.23.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d\"\ndependencies = [\"darling_core\", \"darling_macro\"]\n\n[[package]]\nname = \"darling_core\"\nversion = \"0.23.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0\"\ndependencies = [\"ident_case\", \"proc-macro2\", \"quote\", \"serde\", \"strsim\", \"syn 2.0.117\"]\n\n[[package]]\nname = \"darling_macro\"\nversion = \"0.23.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d\"\ndependencies = [\"darling_core\", \"quote\", \"syn 2.0.117\"]\n\n[[package]]\nname = \"dashmap\"\nversion = \"6.1.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf\"\ndependencies = [\"cfg-if\", \"crossbeam-utils\", \"hashbrown 0.14.5\", \"lock_api\", \"once_cell\", \"parking_lot_core\"]\n\n[[package]]\nname = \"data-encoding\"\nversion = \"2.10.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea\"\n\n[[package]]\nname = \"der\"\nversion = \"0.7.10\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb\"\ndependencies = [\"const-oid\", \"der_derive\", \"flagset\", \"pem-rfc7468\", \"zeroize\"]\n\n[[package]]\nname = \"der-parser\"\nversion = \"10.0.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6\"\ndependencies = [\"asn1-rs\", \"displaydoc\", \"nom 7.1.3\", \"num-bigint\", \"num-traits\", \"rusticata-macros\"]\n\n[[package]]\nname = \"der_derive\"\nversion = \"0.7.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"8034092389675178f570469e6c3b0465d3d30b4505c294a6550db47f3c17ad18\"\ndependencies = [\"proc-macro2\", \"quote\", \"syn 2.0.117\"]\n\n[[package]]\nname = \"deranged\"\nversion = \"0.5.8\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c\"\ndependencies = [\"powerfmt\", \"serde_core\"]\n\n[[package]]\nname = \"derivative\"\nversion = \"2.2.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b\"\ndependencies = [\"proc-macro2\", \"quote\", \"syn 1.0.109\"]\n\n[[package]]\nname = \"derive_more\"\nversion = \"2.1.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134\"\ndependencies = [\"derive_more-impl\"]\n\n[[package]]\nname = \"derive_more-impl\"\nversion = \"2.1.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb\"\ndependencies = [\"convert_case 0.10.0\", \"proc-macro2\", \"quote\", \"rustc_version 0.4.1\", \"syn 2.0.117\", \"unicode-xid\"]\n\n[[package]]\nname = \"des\"\nversion = \"0.8.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"ffdd80ce8ce993de27e9f063a444a4d53ce8e8db4c1f00cc03af5ad5a9867a1e\"\ndependencies = [\"cipher\"]\n\n[[package]]\nname = \"digest\"\nversion = \"0.9.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066\"\ndependencies = [\"generic-array\"]\n\n[[package]]\nname = \"digest\"\nversion = \"0.10.7\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292\"\ndependencies = [\"block-buffer\", \"const-oid\", \"crypto-common\", \"subtle\"]\n\n[[package]]\nname = \"displaydoc\"\nversion = \"0.2.5\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0\"\ndependencies = [\"proc-macro2\", \"quote\", \"syn 2.0.117\"]\n\n[[package]]\nname = \"dlv-list\"\nversion = \"0.5.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"442039f5147480ba31067cb00ada1adae6892028e40e45fc5de7b7df6dcc1b5f\"\ndependencies = [\"const-random\"]\n\n[[package]]\nname = \"docker_credential\"\nversion = \"1.3.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"1d89dfcba45b4afad7450a99b39e751590463e45c04728cf555d36bb66940de8\"\ndependencies = [\"base64 0.21.7\", \"serde\", \"serde_json\"]\n\n[[package]]\nname = \"doctest-file\"\nversion = \"1.1.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"c2db04e74f0a9a93103b50e90b96024c9b2bdca8bce6a632ec71b88736d3d359\"\n\n[[package]]\nname = \"dotenvy\"\nversion = \"0.15.7\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b\"\n\n[[package]]\nname = \"dunce\"\nversion = \"1.0.5\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813\"\n\n[[package]]\nname = \"dyn-clone\"\nversion = \"1.0.20\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555\"\n\n[[package]]\nname = \"ecdsa\"\nversion = \"0.16.9\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca\"\ndependencies = [\"der\", \"digest 0.10.7\", \"elliptic-curve\", \"rfc6979\", \"serdect\", \"signature\", \"spki\"]\n\n[[package]]\nname = \"educe\"\nversion = \"0.6.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"1d7bc049e1bd8cdeb31b68bbd586a9464ecf9f3944af3958a7a9d0f8b9799417\"\ndependencies = [\"enum-ordinalize\", \"proc-macro2\", \"quote\", \"syn 2.0.117\"]\n\n[[package]]\nname = \"either\"\nversion = \"1.15.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719\"\ndependencies = [\"serde\"]\n\n[[package]]\nname = \"elliptic-curve\"\nversion = \"0.13.8\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47\"\ndependencies = [\"base16ct\", \"crypto-bigint\", \"digest 0.10.7\", \"ff\", \"generic-array\", \"group\", \"pkcs8\", \"rand_core 0.6.4\", \"sec1\", \"serdect\", \"subtle\", \"zeroize\"]\n\n[[package]]\nname = \"encoding_rs\"\nversion = \"0.8.35\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3\"\ndependencies = [\"cfg-if\"]\n\n[[package]]\nname = \"enum-as-inner\"\nversion = \"0.6.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc\"\ndependencies = [\"heck\", \"proc-macro2\", \"quote\", \"syn 2.0.117\"]\n\n[[package]]\nname = \"enum-ordinalize\"\nversion = \"4.3.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"4a1091a7bb1f8f2c4b28f1fe2cef4980ca2d410a3d727d67ecc3178c9b0800f0\"\ndependencies = [\"enum-ordinalize-derive\"]\n\n[[package]]\nname = \"enum-ordinalize-derive\"\nversion = \"4.3.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"8ca9601fb2d62598ee17836250842873a413586e5d7ed88b356e38ddbb0ec631\"\ndependencies = [\"proc-macro2\", \"quote\", \"syn 2.0.117\"]\n\n[[package]]\nname = \"equivalent\"\nversion = \"1.0.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f\"\n\n[[package]]\nname = \"errno\"\nversion = \"0.3.14\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb\"\ndependencies = [\"libc\", \"windows-sys 0.61.2\"]\n\n[[package]]\nname = \"etcetera\"\nversion = \"0.8.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943\"\ndependencies = [\"cfg-if\", \"home\", \"windows-sys 0.48.0\"]\n\n[[package]]\nname = \"event-listener\"\nversion = \"5.4.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab\"\ndependencies = [\"concurrent-queue\", \"parking\", \"pin-project-lite\"]\n\n[[package]]\nname = \"event-listener-strategy\"\nversion = \"0.5.4\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93\"\ndependencies = [\"event-listener\", \"pin-project-lite\"]\n\n[[package]]\nname = \"fastrand\"\nversion = \"2.3.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be\"\n\n[[package]]\nname = \"fastrlp\"\nversion = \"0.3.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"139834ddba373bbdd213dffe02c8d110508dcf1726c2be27e8d1f7d7e1856418\"\ndependencies = [\"arrayvec\", \"auto_impl\", \"bytes\"]\n\n[[package]]\nname = \"fastrlp\"\nversion = \"0.4.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"ce8dba4714ef14b8274c371879b175aa55b16b30f269663f19d576f380018dc4\"\ndependencies = [\"arrayvec\", \"auto_impl\", \"bytes\"]\n\n[[package]]\nname = \"ff\"\nversion = \"0.13.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393\"\ndependencies = [\"rand_core 0.6.4\", \"subtle\"]\n\n[[package]]\nname = \"filetime\"\nversion = \"0.2.27\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"f98844151eee8917efc50bd9e8318cb963ae8b297431495d3f758616ea5c57db\"\ndependencies = [\"cfg-if\", \"libc\", \"libredox\"]\n\n[[package]]\nname = \"find-msvc-tools\"\nversion = \"0.1.9\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582\"\n\n[[package]]\nname = \"fixed-hash\"\nversion = \"0.8.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"835c052cb0c08c1acf6ffd71c022172e18723949c8282f2b9f27efbc51e64534\"\ndependencies = [\"byteorder\", \"rand 0.8.5\", \"rustc-hex\", \"static_assertions\"]\n\n[[package]]\nname = \"flagset\"\nversion = \"0.4.7\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"b7ac824320a75a52197e8f2d787f6a38b6718bb6897a35142d749af3c0e8f4fe\"\n\n[[package]]\nname = \"flume\"\nversion = \"0.11.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095\"\ndependencies = [\"futures-core\", \"futures-sink\", \"spin\"]\n\n[[package]]\nname = \"flume\"\nversion = \"0.12.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"5e139bc46ca777eb5efaf62df0ab8cc5fd400866427e56c68b22e414e53bd3be\"\ndependencies = [\"futures-core\", \"futures-sink\", \"spin\"]\n\n[[package]]\nname = \"fnv\"\nversion = \"1.0.7\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1\"\n\n[[package]]\nname = \"foldhash\"\nversion = \"0.1.5\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2\"\n\n[[package]]\nname = \"foldhash\"\nversion = \"0.2.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb\"\n\n[[package]]\nname = \"form_urlencoded\"\nversion = \"1.2.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf\"\ndependencies = [\"percent-encoding\"]\n\n[[package]]\nname = \"fs_extra\"\nversion = \"1.3.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c\"\n\n[[package]]\nname = \"funty\"\nversion = \"2.0.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c\"\n\n[[package]]\nname = \"futures\"\nversion = \"0.3.32\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d\"\ndependencies = [\"futures-channel\", \"futures-core\", \"futures-executor\", \"futures-io\", \"futures-sink\", \"futures-task\", \"futures-util\"]\n\n[[package]]\nname = \"futures-channel\"\nversion = \"0.3.32\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d\"\ndependencies = [\"futures-core\", \"futures-sink\"]\n\n[[package]]\nname = \"futures-core\"\nversion = \"0.3.32\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d\"\n\n[[package]]\nname = \"futures-executor\"\nversion = \"0.3.32\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d\"\ndependencies = [\"futures-core\", \"futures-task\", \"futures-util\"]\n\n[[package]]\nname = \"futures-intrusive\"\nversion = \"0.5.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f\"\ndependencies = [\"futures-core\", \"lock_api\", \"parking_lot\"]\n\n[[package]]\nname = \"futures-io\"\nversion = \"0.3.32\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718\"\n\n[[package]]\nname = \"futures-lite\"\nversion = \"2.6.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad\"\ndependencies = [\"fastrand\", \"futures-core\", \"futures-io\", \"parking\", \"pin-project-lite\"]\n\n[[package]]\nname = \"futures-macro\"\nversion = \"0.3.32\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b\"\ndependencies = [\"proc-macro2\", \"quote\", \"syn 2.0.117\"]\n\n[[package]]\nname = \"futures-rustls\"\nversion = \"0.26.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"a8f2f12607f92c69b12ed746fabf9ca4f5c482cba46679c1a75b874ed7c26adb\"\ndependencies = [\"futures-io\", \"rustls\", \"rustls-pki-types\"]\n\n[[package]]\nname = \"futures-sink\"\nversion = \"0.3.32\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893\"\n\n[[package]]\nname = \"futures-task\"\nversion = \"0.3.32\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393\"\n\n[[package]]\nname = \"futures-util\"\nversion = \"0.3.32\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6\"\ndependencies = [\"futures-channel\", \"futures-core\", \"futures-io\", \"futures-macro\", \"futures-sink\", \"futures-task\", \"memchr\", \"pin-project-lite\", \"slab\"]\n\n[[package]]\nname = \"futures-utils-wasm\"\nversion = \"0.1.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"42012b0f064e01aa58b545fe3727f90f7dd4020f4a3ea735b50344965f5a57e9\"\n\n[[package]]\nname = \"generic-array\"\nversion = \"0.14.7\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a\"\ndependencies = [\"typenum\", \"version_check\", \"zeroize\"]\n\n[[package]]\nname = \"getrandom\"\nversion = \"0.2.17\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0\"\ndependencies = [\"cfg-if\", \"js-sys\", \"libc\", \"wasi\", \"wasm-bindgen\"]\n\n[[package]]\nname = \"getrandom\"\nversion = \"0.3.4\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd\"\ndependencies = [\"cfg-if\", \"js-sys\", \"libc\", \"r-efi 5.3.0\", \"wasip2\", \"wasm-bindgen\"]\n\n[[package]]\nname = \"getrandom\"\nversion = \"0.4.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555\"\ndependencies = [\"cfg-if\", \"libc\", \"r-efi 6.0.0\", \"rand_core 0.10.0\", \"wasip2\", \"wasip3\"]\n\n[[package]]\nname = \"glob\"\nversion = \"0.3.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280\"\n\n[[package]]\nname = \"group\"\nversion = \"0.13.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63\"\ndependencies = [\"ff\", \"rand_core 0.6.4\", \"subtle\"]\n\n[[package]]\nname = \"h2\"\nversion = \"0.4.13\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54\"\ndependencies = [\"atomic-waker\", \"bytes\", \"fnv\", \"futures-core\", \"futures-sink\", \"http\", \"indexmap 2.13.1\", \"slab\", \"tokio\", \"tokio-util\", \"tracing\"]\n\n[[package]]\nname = \"hashbrown\"\nversion = \"0.12.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888\"\n\n[[package]]\nname = \"hashbrown\"\nversion = \"0.14.5\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1\"\ndependencies = [\"ahash\", \"allocator-api2\"]\n\n[[package]]\nname = \"hashbrown\"\nversion = \"0.15.5\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1\"\ndependencies = [\"allocator-api2\", \"equivalent\", \"foldhash 0.1.5\"]\n\n[[package]]\nname = \"hashbrown\"\nversion = \"0.16.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100\"\ndependencies = [\"allocator-api2\", \"equivalent\", \"foldhash 0.2.0\", \"serde\", \"serde_core\"]\n\n[[package]]\nname = \"hashlink\"\nversion = \"0.8.4\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7\"\ndependencies = [\"hashbrown 0.14.5\"]\n\n[[package]]\nname = \"hashlink\"\nversion = \"0.10.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1\"\ndependencies = [\"hashbrown 0.15.5\"]\n\n[[package]]\nname = \"heck\"\nversion = \"0.5.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea\"\n\n[[package]]\nname = \"hermit-abi\"\nversion = \"0.5.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c\"\n\n[[package]]\nname = \"hex\"\nversion = \"0.4.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70\"\n\n[[package]]\nname = \"hex-conservative\"\nversion = \"0.2.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"fda06d18ac606267c40c04e41b9947729bf8b9efe74bd4e82b61a5f26a510b9f\"\ndependencies = [\"arrayvec\"]\n\n[[package]]\nname = \"hickory-proto\"\nversion = \"0.25.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"f8a6fe56c0038198998a6f217ca4e7ef3a5e51f46163bd6dd60b5c71ca6c6502\"\ndependencies = [\"async-trait\", \"cfg-if\", \"data-encoding\", \"enum-as-inner\", \"futures-channel\", \"futures-io\", \"futures-util\", \"idna\", \"ipnet\", \"once_cell\", \"rand 0.9.2\", \"ring\", \"thiserror 2.0.18\", \"tinyvec\", \"tokio\", \"tracing\", \"url\"]\n\n[[package]]\nname = \"hickory-resolver\"\nversion = \"0.25.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"dc62a9a99b0bfb44d2ab95a7208ac952d31060efc16241c87eaf36406fecf87a\"\ndependencies = [\"cfg-if\", \"futures-util\", \"hickory-proto\", \"ipconfig\", \"moka\", \"once_cell\", \"parking_lot\", \"rand 0.9.2\", \"resolv-conf\", \"smallvec\", \"thiserror 2.0.18\", \"tokio\", \"tracing\"]\n\n[[package]]\nname = \"hkdf\"\nversion = \"0.12.4\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7\"\ndependencies = [\"hmac\"]\n\n[[package]]\nname = \"hmac\"\nversion = \"0.12.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e\"\ndependencies = [\"digest 0.10.7\"]\n\n[[package]]\nname = \"home\"\nversion = \"0.5.12\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d\"\ndependencies = [\"windows-sys 0.61.2\"]\n\n[[package]]\nname = \"http\"\nversion = \"1.4.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a\"\ndependencies = [\"bytes\", \"itoa\"]\n\n[[package]]\nname = \"http-body\"\nversion = \"1.0.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184\"\ndependencies = [\"bytes\", \"http\"]\n\n[[package]]\nname = \"http-body-util\"\nversion = \"0.1.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a\"\ndependencies = [\"bytes\", \"futures-core\", \"http\", \"http-body\", \"pin-project-lite\"]\n\n[[package]]\nname = \"httparse\"\nversion = \"1.10.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87\"\n\n[[package]]\nname = \"httpdate\"\nversion = \"1.0.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9\"\n\n[[package]]\nname = \"hyper\"\nversion = \"1.9.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca\"\ndependencies = [\"atomic-waker\", \"bytes\", \"futures-channel\", \"futures-core\", \"h2\", \"http\", \"http-body\", \"httparse\", \"httpdate\", \"itoa\", \"pin-project-lite\", \"smallvec\", \"tokio\", \"want\"]\n\n[[package]]\nname = \"hyper-named-pipe\"\nversion = \"0.1.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"73b7d8abf35697b81a825e386fc151e0d503e8cb5fcb93cc8669c376dfd6f278\"\ndependencies = [\"hex\", \"hyper\", \"hyper-util\", \"pin-project-lite\", \"tokio\", \"tower-service\", \"winapi\"]\n\n[[package]]\nname = \"hyper-rustls\"\nversion = \"0.27.7\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58\"\ndependencies = [\"http\", \"hyper\", \"hyper-util\", \"rustls\", \"rustls-pki-types\", \"tokio\", \"tokio-rustls\", \"tower-service\"]\n\n[[package]]\nname = \"hyper-util\"\nversion = \"0.1.20\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0\"\ndependencies = [\"base64 0.22.1\", \"bytes\", \"futures-channel\", \"futures-util\", \"http\", \"http-body\", \"hyper\", \"ipnet\", \"libc\", \"percent-encoding\", \"pin-project-lite\", \"socket2 0.6.3\", \"system-configuration\", \"tokio\", \"tower-service\", \"tracing\", \"windows-registry\"]\n\n[[package]]\nname = \"hyperlocal\"\nversion = \"0.9.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7\"\ndependencies = [\"hex\", \"http-body-util\", \"hyper\", \"hyper-util\", \"pin-project-lite\", \"tokio\", \"tower-service\"]\n\n[[package]]\nname = \"iana-time-zone\"\nversion = \"0.1.65\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470\"\ndependencies = [\"android_system_properties\", \"core-foundation-sys\", \"iana-time-zone-haiku\", \"js-sys\", \"log\", \"wasm-bindgen\", \"windows-core\"]\n\n[[package]]\nname = \"iana-time-zone-haiku\"\nversion = \"0.1.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f\"\ndependencies = [\"cc\"]\n\n[[package]]\nname = \"icu_collections\"\nversion = \"2.2.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c\"\ndependencies = [\"displaydoc\", \"potential_utf\", \"utf8_iter\", \"yoke\", \"zerofrom\", \"zerovec\"]\n\n[[package]]\nname = \"icu_locale_core\"\nversion = \"2.2.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29\"\ndependencies = [\"displaydoc\", \"litemap\", \"tinystr\", \"writeable\", \"zerovec\"]\n\n[[package]]\nname = \"icu_normalizer\"\nversion = \"2.2.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4\"\ndependencies = [\"icu_collections\", \"icu_normalizer_data\", \"icu_properties\", \"icu_provider\", \"smallvec\", \"zerovec\"]\n\n[[package]]\nname = \"icu_normalizer_data\"\nversion = \"2.2.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38\"\n\n[[package]]\nname = \"icu_properties\"\nversion = \"2.2.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de\"\ndependencies = [\"icu_collections\", \"icu_locale_core\", \"icu_properties_data\", \"icu_provider\", \"zerotrie\", \"zerovec\"]\n\n[[package]]\nname = \"icu_properties_data\"\nversion = \"2.2.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14\"\n\n[[package]]\nname = \"icu_provider\"\nversion = \"2.2.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421\"\ndependencies = [\"displaydoc\", \"icu_locale_core\", \"writeable\", \"yoke\", \"zerofrom\", \"zerotrie\", \"zerovec\"]\n\n[[package]]\nname = \"id-arena\"\nversion = \"2.3.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954\"\n\n[[package]]\nname = \"ident_case\"\nversion = \"1.0.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39\"\n\n[[package]]\nname = \"idna\"\nversion = \"1.1.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de\"\ndependencies = [\"idna_adapter\", \"smallvec\", \"utf8_iter\"]\n\n[[package]]\nname = \"idna_adapter\"\nversion = \"1.2.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344\"\ndependencies = [\"icu_normalizer\", \"icu_properties\"]\n\n[[package]]\nname = \"impl-codec\"\nversion = \"0.6.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"ba6a270039626615617f3f36d15fc827041df3b78c439da2cadfa47455a77f2f\"\ndependencies = [\"parity-scale-codec\"]\n\n[[package]]\nname = \"impl-trait-for-tuples\"\nversion = \"0.2.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"a0eb5a3343abf848c0984fe4604b2b105da9539376e24fc0a3b0007411ae4fd9\"\ndependencies = [\"proc-macro2\", \"quote\", \"syn 2.0.117\"]\n\n[[package]]\nname = \"indexmap\"\nversion = \"1.9.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99\"\ndependencies = [\"autocfg\", \"hashbrown 0.12.3\", \"serde\"]\n\n[[package]]\nname = \"indexmap\"\nversion = \"2.13.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"45a8a2b9cb3e0b0c1803dbb0758ffac5de2f425b23c28f518faabd9d805342ff\"\ndependencies = [\"equivalent\", \"hashbrown 0.16.1\", \"serde\", \"serde_core\"]\n\n[[package]]\nname = \"inout\"\nversion = \"0.1.4\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01\"\ndependencies = [\"block-padding\", \"generic-array\"]\n\n[[package]]\nname = \"interprocess\"\nversion = \"2.4.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"6be5e5c847dbdb44564bd85294740d031f4f8aeb3464e5375ef7141f7538db69\"\ndependencies = [\"doctest-file\", \"futures-core\", \"libc\", \"recvmsg\", \"tokio\", \"widestring\", \"windows-sys 0.52.0\"]\n\n[[package]]\nname = \"ipconfig\"\nversion = \"0.3.4\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"4d40460c0ce33d6ce4b0630ad68ff63d6661961c48b6dba35e5a4d81cfb48222\"\ndependencies = [\"socket2 0.6.3\", \"widestring\", \"windows-registry\", \"windows-result\", \"windows-sys 0.61.2\"]\n\n[[package]]\nname = \"ipnet\"\nversion = \"2.12.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2\"\n\n[[package]]\nname = \"iri-string\"\nversion = \"0.7.12\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"25e659a4bb38e810ebc252e53b5814ff908a8c58c2a9ce2fae1bbec24cbf4e20\"\ndependencies = [\"memchr\", \"serde\"]\n\n[[package]]\nname = \"itertools\"\nversion = \"0.10.5\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473\"\ndependencies = [\"either\"]\n\n[[package]]\nname = \"itertools\"\nversion = \"0.13.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186\"\ndependencies = [\"either\"]\n\n[[package]]\nname = \"itertools\"\nversion = \"0.14.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285\"\ndependencies = [\"either\"]\n\n[[package]]\nname = \"itoa\"\nversion = \"1.0.18\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682\"\n\n[[package]]\nname = \"jni\"\nversion = \"0.21.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97\"\ndependencies = [\"cesu8\", \"cfg-if\", \"combine\", \"jni-sys 0.3.1\", \"log\", \"thiserror 1.0.69\", \"walkdir\", \"windows-sys 0.45.0\"]\n\n[[package]]\nname = \"jni-sys\"\nversion = \"0.3.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258\"\ndependencies = [\"jni-sys 0.4.1\"]\n\n[[package]]\nname = \"jni-sys\"\nversion = \"0.4.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2\"\ndependencies = [\"jni-sys-macros\"]\n\n[[package]]\nname = \"jni-sys-macros\"\nversion = \"0.4.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264\"\ndependencies = [\"quote\", \"syn 2.0.117\"]\n\n[[package]]\nname = \"jobserver\"\nversion = \"0.1.34\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33\"\ndependencies = [\"getrandom 0.3.4\", \"libc\"]\n\n[[package]]\nname = \"js-sys\"\nversion = \"0.3.94\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"2e04e2ef80ce82e13552136fabeef8a5ed1f985a96805761cbb9a2c34e7664d9\"\ndependencies = [\"cfg-if\", \"futures-util\", \"once_cell\", \"wasm-bindgen\"]\n\n[[package]]\nname = \"json5\"\nversion = \"0.4.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"96b0db21af676c1ce64250b5f40f3ce2cf27e4e47cb91ed91eb6fe9350b430c1\"\ndependencies = [\"pest\", \"pest_derive\", \"serde\"]\n\n[[package]]\nname = \"k256\"\nversion = \"0.13.4\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"f6e3919bbaa2945715f0bb6d3934a173d1e9a59ac23767fbaaef277265a7411b\"\ndependencies = [\"cfg-if\", \"ecdsa\", \"elliptic-curve\", \"once_cell\", \"serdect\", \"sha2\"]\n\n[[package]]\nname = \"keccak\"\nversion = \"0.1.6\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"cb26cec98cce3a3d96cbb7bced3c4b16e3d13f27ec56dbd62cbc8f39cfb9d653\"\ndependencies = [\"cpufeatures 0.2.17\"]\n\n[[package]]\nname = \"keccak-asm\"\nversion = \"0.1.6\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"fa468878266ad91431012b3e5ef1bf9b170eab22883503a318d46857afa4579a\"\ndependencies = [\"digest 0.10.7\", \"sha3-asm\"]\n\n[[package]]\nname = \"lapin\"\nversion = \"4.4.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"39338badb3f992d800f6964501b056b575bdf142eb288202f973d218fe253b90\"\ndependencies = [\"amq-protocol\", \"async-rs\", \"async-trait\", \"backon\", \"cfg-if\", \"flume 0.12.0\", \"futures-core\", \"futures-io\", \"tracing\"]\n\n[[package]]\nname = \"lazy_static\"\nversion = \"1.5.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe\"\ndependencies = [\"spin\"]\n\n[[package]]\nname = \"leb128fmt\"\nversion = \"0.1.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2\"\n\n[[package]]\nname = \"libc\"\nversion = \"0.2.184\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af\"\n\n[[package]]\nname = \"libm\"\nversion = \"0.2.16\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981\"\n\n[[package]]\nname = \"libredox\"\nversion = \"0.1.15\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"7ddbf48fd451246b1f8c2610bd3b4ac0cc6e149d89832867093ab69a17194f08\"\ndependencies = [\"bitflags 2.11.0\", \"libc\", \"plain\", \"redox_syscall 0.7.3\"]\n\n[[package]]\nname = \"libsqlite3-sys\"\nversion = \"0.30.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149\"\ndependencies = [\"pkg-config\", \"vcpkg\"]\n\n[[package]]\nname = \"linux-raw-sys\"\nversion = \"0.12.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53\"\n\n[[package]]\nname = \"listener_core\"\nversion = \"0.0.1\"\ndependencies = [\"alloy\", \"alloy-json-rpc\", \"alloy-transport-ipc\", \"anyhow\", \"async-trait\", \"broker\", \"chrono\", \"config\", \"derivative\", \"futures\", \"primitives\", \"rand 0.9.2\", \"reqwest\", \"serde\", \"serde_json\", \"sqlx\", \"thiserror 2.0.18\", \"tokio\", \"tokio-util\", \"tracing\", \"tracing-subscriber\", \"url\", \"uuid\"]\n\n[[package]]\nname = \"litemap\"\nversion = \"0.8.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0\"\n\n[[package]]\nname = \"lock_api\"\nversion = \"0.4.14\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965\"\ndependencies = [\"scopeguard\"]\n\n[[package]]\nname = \"log\"\nversion = \"0.4.29\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897\"\n\n[[package]]\nname = \"lru\"\nversion = \"0.16.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"a1dc47f592c06f33f8e3aea9591776ec7c9f9e4124778ff8a3c3b87159f7e593\"\ndependencies = [\"hashbrown 0.16.1\"]\n\n[[package]]\nname = \"lru-slab\"\nversion = \"0.1.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154\"\n\n[[package]]\nname = \"macro-string\"\nversion = \"0.1.4\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"1b27834086c65ec3f9387b096d66e99f221cf081c2b738042aa252bcd41204e3\"\ndependencies = [\"proc-macro2\", \"quote\", \"syn 2.0.117\"]\n\n[[package]]\nname = \"md-5\"\nversion = \"0.10.6\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf\"\ndependencies = [\"cfg-if\", \"digest 0.10.7\"]\n\n[[package]]\nname = \"memchr\"\nversion = \"2.8.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79\"\n\n[[package]]\nname = \"mime\"\nversion = \"0.3.17\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a\"\n\n[[package]]\nname = \"minimal-lexical\"\nversion = \"0.2.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a\"\n\n[[package]]\nname = \"mio\"\nversion = \"1.2.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1\"\ndependencies = [\"libc\", \"wasi\", \"windows-sys 0.61.2\"]\n\n[[package]]\nname = \"moka\"\nversion = \"0.12.15\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046\"\ndependencies = [\"crossbeam-channel\", \"crossbeam-epoch\", \"crossbeam-utils\", \"equivalent\", \"parking_lot\", \"portable-atomic\", \"smallvec\", \"tagptr\", \"uuid\"]\n\n[[package]]\nname = \"nom\"\nversion = \"7.1.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a\"\ndependencies = [\"memchr\", \"minimal-lexical\"]\n\n[[package]]\nname = \"nom\"\nversion = \"8.0.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405\"\ndependencies = [\"memchr\"]\n\n[[package]]\nname = \"nu-ansi-term\"\nversion = \"0.50.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5\"\ndependencies = [\"windows-sys 0.61.2\"]\n\n[[package]]\nname = \"num-bigint\"\nversion = \"0.4.6\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9\"\ndependencies = [\"num-integer\", \"num-traits\"]\n\n[[package]]\nname = \"num-bigint-dig\"\nversion = \"0.8.6\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7\"\ndependencies = [\"lazy_static\", \"libm\", \"num-integer\", \"num-iter\", \"num-traits\", \"rand 0.8.5\", \"smallvec\", \"zeroize\"]\n\n[[package]]\nname = \"num-conv\"\nversion = \"0.2.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967\"\n\n[[package]]\nname = \"num-integer\"\nversion = \"0.1.46\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f\"\ndependencies = [\"num-traits\"]\n\n[[package]]\nname = \"num-iter\"\nversion = \"0.1.45\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf\"\ndependencies = [\"autocfg\", \"num-integer\", \"num-traits\"]\n\n[[package]]\nname = \"num-traits\"\nversion = \"0.2.19\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841\"\ndependencies = [\"autocfg\", \"libm\"]\n\n[[package]]\nname = \"num_cpus\"\nversion = \"1.17.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b\"\ndependencies = [\"hermit-abi\", \"libc\"]\n\n[[package]]\nname = \"num_enum\"\nversion = \"0.7.6\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26\"\ndependencies = [\"num_enum_derive\", \"rustversion\"]\n\n[[package]]\nname = \"num_enum_derive\"\nversion = \"0.7.6\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8\"\ndependencies = [\"proc-macro2\", \"quote\", \"syn 2.0.117\"]\n\n[[package]]\nname = \"nybbles\"\nversion = \"0.4.8\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"0d49ff0c0d00d4a502b39df9af3a525e1efeb14b9dabb5bb83335284c1309210\"\ndependencies = [\"alloy-rlp\", \"cfg-if\", \"proptest\", \"ruint\", \"serde\", \"smallvec\"]\n\n[[package]]\nname = \"oid-registry\"\nversion = \"0.8.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7\"\ndependencies = [\"asn1-rs\"]\n\n[[package]]\nname = \"once_cell\"\nversion = \"1.21.4\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50\"\ndependencies = [\"critical-section\", \"portable-atomic\"]\n\n[[package]]\nname = \"openssl-probe\"\nversion = \"0.2.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe\"\n\n[[package]]\nname = \"ordered-multimap\"\nversion = \"0.7.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"49203cdcae0030493bad186b28da2fa25645fa276a51b6fec8010d281e02ef79\"\ndependencies = [\"dlv-list\", \"hashbrown 0.14.5\"]\n\n[[package]]\nname = \"p12-keystore\"\nversion = \"0.2.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"ffb9bf5222606eb712d3bb30e01bc9420545b00859970897e70c682353a034f2\"\ndependencies = [\"base64 0.22.1\", \"cbc\", \"cms\", \"der\", \"des\", \"hex\", \"hmac\", \"pkcs12\", \"pkcs5\", \"rand 0.10.0\", \"rc2\", \"sha1\", \"sha2\", \"thiserror 2.0.18\", \"x509-parser\"]\n\n[[package]]\nname = \"parity-scale-codec\"\nversion = \"3.7.5\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"799781ae679d79a948e13d4824a40970bfa500058d245760dd857301059810fa\"\ndependencies = [\"arrayvec\", \"bitvec\", \"byte-slice-cast\", \"const_format\", \"impl-trait-for-tuples\", \"parity-scale-codec-derive\", \"rustversion\", \"serde\"]\n\n[[package]]\nname = \"parity-scale-codec-derive\"\nversion = \"3.7.5\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"34b4653168b563151153c9e4c08ebed57fb8262bebfa79711552fa983c623e7a\"\ndependencies = [\"proc-macro-crate\", \"proc-macro2\", \"quote\", \"syn 2.0.117\"]\n\n[[package]]\nname = \"parking\"\nversion = \"2.2.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba\"\n\n[[package]]\nname = \"parking_lot\"\nversion = \"0.12.5\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a\"\ndependencies = [\"lock_api\", \"parking_lot_core\"]\n\n[[package]]\nname = \"parking_lot_core\"\nversion = \"0.9.12\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1\"\ndependencies = [\"cfg-if\", \"libc\", \"redox_syscall 0.5.18\", \"smallvec\", \"windows-link\"]\n\n[[package]]\nname = \"parse-display\"\nversion = \"0.9.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"914a1c2265c98e2446911282c6ac86d8524f495792c38c5bd884f80499c7538a\"\ndependencies = [\"parse-display-derive\", \"regex\", \"regex-syntax\"]\n\n[[package]]\nname = \"parse-display-derive\"\nversion = \"0.9.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"2ae7800a4c974efd12df917266338e79a7a74415173caf7e70aa0a0707345281\"\ndependencies = [\"proc-macro2\", \"quote\", \"regex\", \"regex-syntax\", \"structmeta\", \"syn 2.0.117\"]\n\n[[package]]\nname = \"paste\"\nversion = \"1.0.15\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a\"\n\n[[package]]\nname = \"pathdiff\"\nversion = \"0.2.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3\"\n\n[[package]]\nname = \"pbkdf2\"\nversion = \"0.12.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2\"\ndependencies = [\"digest 0.10.7\", \"hmac\"]\n\n[[package]]\nname = \"pem-rfc7468\"\nversion = \"0.7.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412\"\ndependencies = [\"base64ct\"]\n\n[[package]]\nname = \"percent-encoding\"\nversion = \"2.3.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220\"\n\n[[package]]\nname = \"pest\"\nversion = \"2.8.6\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"e0848c601009d37dfa3430c4666e147e49cdcf1b92ecd3e63657d8a5f19da662\"\ndependencies = [\"memchr\", \"ucd-trie\"]\n\n[[package]]\nname = \"pest_derive\"\nversion = \"2.8.6\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"11f486f1ea21e6c10ed15d5a7c77165d0ee443402f0780849d1768e7d9d6fe77\"\ndependencies = [\"pest\", \"pest_generator\"]\n\n[[package]]\nname = \"pest_generator\"\nversion = \"2.8.6\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"8040c4647b13b210a963c1ed407c1ff4fdfa01c31d6d2a098218702e6664f94f\"\ndependencies = [\"pest\", \"pest_meta\", \"proc-macro2\", \"quote\", \"syn 2.0.117\"]\n\n[[package]]\nname = \"pest_meta\"\nversion = \"2.8.6\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"89815c69d36021a140146f26659a81d6c2afa33d216d736dd4be5381a7362220\"\ndependencies = [\"pest\", \"sha2\"]\n\n[[package]]\nname = \"pharos\"\nversion = \"0.5.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"e9567389417feee6ce15dd6527a8a1ecac205ef62c2932bcf3d9f6fc5b78b414\"\ndependencies = [\"futures\", \"rustc_version 0.4.1\"]\n\n[[package]]\nname = \"pin-project\"\nversion = \"1.1.11\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"f1749c7ed4bcaf4c3d0a3efc28538844fb29bcdd7d2b67b2be7e20ba861ff517\"\ndependencies = [\"pin-project-internal\"]\n\n[[package]]\nname = \"pin-project-internal\"\nversion = \"1.1.11\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6\"\ndependencies = [\"proc-macro2\", \"quote\", \"syn 2.0.117\"]\n\n[[package]]\nname = \"pin-project-lite\"\nversion = \"0.2.17\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd\"\n\n[[package]]\nname = \"pin-utils\"\nversion = \"0.1.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184\"\n\n[[package]]\nname = \"piper\"\nversion = \"0.2.5\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"c835479a4443ded371d6c535cbfd8d31ad92c5d23ae9770a61bc155e4992a3c1\"\ndependencies = [\"atomic-waker\", \"fastrand\", \"futures-io\"]\n\n[[package]]\nname = \"pkcs1\"\nversion = \"0.7.5\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f\"\ndependencies = [\"der\", \"pkcs8\", \"spki\"]\n\n[[package]]\nname = \"pkcs12\"\nversion = \"0.1.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"695b3df3d3cc1015f12d70235e35b6b79befc5fa7a9b95b951eab1dd07c9efc2\"\ndependencies = [\"cms\", \"const-oid\", \"der\", \"digest 0.10.7\", \"spki\", \"x509-cert\", \"zeroize\"]\n\n[[package]]\nname = \"pkcs5\"\nversion = \"0.7.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6\"\ndependencies = [\"aes\", \"cbc\", \"der\", \"pbkdf2\", \"scrypt\", \"sha2\", \"spki\"]\n\n[[package]]\nname = \"pkcs8\"\nversion = \"0.10.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7\"\ndependencies = [\"der\", \"spki\"]\n\n[[package]]\nname = \"pkg-config\"\nversion = \"0.3.32\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c\"\n\n[[package]]\nname = \"plain\"\nversion = \"0.2.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6\"\n\n[[package]]\nname = \"portable-atomic\"\nversion = \"1.13.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49\"\n\n[[package]]\nname = \"potential_utf\"\nversion = \"0.1.5\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564\"\ndependencies = [\"zerovec\"]\n\n[[package]]\nname = \"powerfmt\"\nversion = \"0.2.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391\"\n\n[[package]]\nname = \"ppv-lite86\"\nversion = \"0.2.21\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9\"\ndependencies = [\"zerocopy\"]\n\n[[package]]\nname = \"prettyplease\"\nversion = \"0.2.37\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b\"\ndependencies = [\"proc-macro2\", \"syn 2.0.117\"]\n\n[[package]]\nname = \"primitive-types\"\nversion = \"0.12.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"0b34d9fd68ae0b74a41b21c03c2f62847aa0ffea044eee893b4c140b37e244e2\"\ndependencies = [\"fixed-hash\", \"impl-codec\", \"uint\"]\n\n[[package]]\nname = \"primitives\"\nversion = \"0.0.1\"\ndependencies = [\"alloy\", \"serde\", \"serde_json\", \"thiserror 2.0.18\"]\n\n[[package]]\nname = \"proc-macro-crate\"\nversion = \"3.5.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f\"\ndependencies = [\"toml_edit 0.25.10+spec-1.1.0\"]\n\n[[package]]\nname = \"proc-macro-error-attr2\"\nversion = \"2.0.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5\"\ndependencies = [\"proc-macro2\", \"quote\"]\n\n[[package]]\nname = \"proc-macro-error2\"\nversion = \"2.0.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802\"\ndependencies = [\"proc-macro-error-attr2\", \"proc-macro2\", \"quote\", \"syn 2.0.117\"]\n\n[[package]]\nname = \"proc-macro2\"\nversion = \"1.0.106\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934\"\ndependencies = [\"unicode-ident\"]\n\n[[package]]\nname = \"proptest\"\nversion = \"1.11.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744\"\ndependencies = [\"bit-set\", \"bit-vec\", \"bitflags 2.11.0\", \"num-traits\", \"rand 0.9.2\", \"rand_chacha 0.9.0\", \"rand_xorshift\", \"regex-syntax\", \"rusty-fork\", \"tempfile\", \"unarray\"]\n\n[[package]]\nname = \"quick-error\"\nversion = \"1.2.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0\"\n\n[[package]]\nname = \"quinn\"\nversion = \"0.11.9\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20\"\ndependencies = [\"bytes\", \"cfg_aliases\", \"pin-project-lite\", \"quinn-proto\", \"quinn-udp\", \"rustc-hash\", \"rustls\", \"socket2 0.6.3\", \"thiserror 2.0.18\", \"tokio\", \"tracing\", \"web-time\"]\n\n[[package]]\nname = \"quinn-proto\"\nversion = \"0.11.14\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098\"\ndependencies = [\"aws-lc-rs\", \"bytes\", \"getrandom 0.3.4\", \"lru-slab\", \"rand 0.9.2\", \"ring\", \"rustc-hash\", \"rustls\", \"rustls-pki-types\", \"slab\", \"thiserror 2.0.18\", \"tinyvec\", \"tracing\", \"web-time\"]\n\n[[package]]\nname = \"quinn-udp\"\nversion = \"0.5.14\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd\"\ndependencies = [\"cfg_aliases\", \"libc\", \"once_cell\", \"socket2 0.6.3\", \"tracing\", \"windows-sys 0.60.2\"]\n\n[[package]]\nname = \"quote\"\nversion = \"1.0.45\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924\"\ndependencies = [\"proc-macro2\"]\n\n[[package]]\nname = \"r-efi\"\nversion = \"5.3.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f\"\n\n[[package]]\nname = \"r-efi\"\nversion = \"6.0.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf\"\n\n[[package]]\nname = \"radium\"\nversion = \"0.7.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09\"\n\n[[package]]\nname = \"rand\"\nversion = \"0.8.5\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404\"\ndependencies = [\"libc\", \"rand_chacha 0.3.1\", \"rand_core 0.6.4\", \"serde\"]\n\n[[package]]\nname = \"rand\"\nversion = \"0.9.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1\"\ndependencies = [\"rand_chacha 0.9.0\", \"rand_core 0.9.5\", \"serde\"]\n\n[[package]]\nname = \"rand\"\nversion = \"0.10.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"bc266eb313df6c5c09c1c7b1fbe2510961e5bcd3add930c1e31f7ed9da0feff8\"\ndependencies = [\"chacha20\", \"getrandom 0.4.2\", \"rand_core 0.10.0\"]\n\n[[package]]\nname = \"rand_chacha\"\nversion = \"0.3.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88\"\ndependencies = [\"ppv-lite86\", \"rand_core 0.6.4\"]\n\n[[package]]\nname = \"rand_chacha\"\nversion = \"0.9.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb\"\ndependencies = [\"ppv-lite86\", \"rand_core 0.9.5\"]\n\n[[package]]\nname = \"rand_core\"\nversion = \"0.6.4\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c\"\ndependencies = [\"getrandom 0.2.17\"]\n\n[[package]]\nname = \"rand_core\"\nversion = \"0.9.5\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c\"\ndependencies = [\"getrandom 0.3.4\", \"serde\"]\n\n[[package]]\nname = \"rand_core\"\nversion = \"0.10.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"0c8d0fd677905edcbeedbf2edb6494d676f0e98d54d5cf9bda0b061cb8fb8aba\"\n\n[[package]]\nname = \"rand_xorshift\"\nversion = \"0.4.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a\"\ndependencies = [\"rand_core 0.9.5\"]\n\n[[package]]\nname = \"rapidhash\"\nversion = \"4.4.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"b5e48930979c155e2f33aa36ab3119b5ee81332beb6482199a8ecd6029b80b59\"\ndependencies = [\"rustversion\"]\n\n[[package]]\nname = \"rc2\"\nversion = \"0.8.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"62c64daa8e9438b84aaae55010a93f396f8e60e3911590fcba770d04643fc1dd\"\ndependencies = [\"cipher\"]\n\n[[package]]\nname = \"recvmsg\"\nversion = \"1.0.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"d3edd4d5d42c92f0a659926464d4cce56b562761267ecf0f469d85b7de384175\"\n\n[[package]]\nname = \"redis\"\nversion = \"0.29.5\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"1bc42f3a12fd4408ce64d8efef67048a924e543bd35c6591c0447fda9054695f\"\ndependencies = [\"arc-swap\", \"backon\", \"bytes\", \"combine\", \"futures-channel\", \"futures-util\", \"itoa\", \"num-bigint\", \"percent-encoding\", \"pin-project-lite\", \"ryu\", \"sha1_smol\", \"socket2 0.5.10\", \"tokio\", \"tokio-util\", \"url\"]\n\n[[package]]\nname = \"redox_syscall\"\nversion = \"0.3.5\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29\"\ndependencies = [\"bitflags 1.3.2\"]\n\n[[package]]\nname = \"redox_syscall\"\nversion = \"0.5.18\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d\"\ndependencies = [\"bitflags 2.11.0\"]\n\n[[package]]\nname = \"redox_syscall\"\nversion = \"0.7.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"6ce70a74e890531977d37e532c34d45e9055d2409ed08ddba14529471ed0be16\"\ndependencies = [\"bitflags 2.11.0\"]\n\n[[package]]\nname = \"ref-cast\"\nversion = \"1.0.25\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d\"\ndependencies = [\"ref-cast-impl\"]\n\n[[package]]\nname = \"ref-cast-impl\"\nversion = \"1.0.25\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da\"\ndependencies = [\"proc-macro2\", \"quote\", \"syn 2.0.117\"]\n\n[[package]]\nname = \"regex\"\nversion = \"1.12.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276\"\ndependencies = [\"aho-corasick\", \"memchr\", \"regex-automata\", \"regex-syntax\"]\n\n[[package]]\nname = \"regex-automata\"\nversion = \"0.4.14\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f\"\ndependencies = [\"aho-corasick\", \"memchr\", \"regex-syntax\"]\n\n[[package]]\nname = \"regex-syntax\"\nversion = \"0.8.10\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a\"\n\n[[package]]\nname = \"reqwest\"\nversion = \"0.13.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"ab3f43e3283ab1488b624b44b0e988d0acea0b3214e694730a055cb6b2efa801\"\ndependencies = [\"base64 0.22.1\", \"bytes\", \"encoding_rs\", \"futures-core\", \"h2\", \"http\", \"http-body\", \"http-body-util\", \"hyper\", \"hyper-rustls\", \"hyper-util\", \"js-sys\", \"log\", \"mime\", \"percent-encoding\", \"pin-project-lite\", \"quinn\", \"rustls\", \"rustls-pki-types\", \"rustls-platform-verifier\", \"serde\", \"serde_json\", \"sync_wrapper\", \"tokio\", \"tokio-rustls\", \"tower\", \"tower-http\", \"tower-service\", \"url\", \"wasm-bindgen\", \"wasm-bindgen-futures\", \"web-sys\"]\n\n[[package]]\nname = \"resolv-conf\"\nversion = \"0.7.6\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7\"\n\n[[package]]\nname = \"rfc6979\"\nversion = \"0.4.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2\"\ndependencies = [\"hmac\", \"subtle\"]\n\n[[package]]\nname = \"ring\"\nversion = \"0.17.14\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7\"\ndependencies = [\"cc\", \"cfg-if\", \"getrandom 0.2.17\", \"libc\", \"untrusted\", \"windows-sys 0.52.0\"]\n\n[[package]]\nname = \"rlp\"\nversion = \"0.5.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"bb919243f34364b6bd2fc10ef797edbfa75f33c252e7998527479c6d6b47e1ec\"\ndependencies = [\"bytes\", \"rustc-hex\"]\n\n[[package]]\nname = \"ron\"\nversion = \"0.8.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"b91f7eff05f748767f183df4320a63d6936e9c6107d97c9e6bdd9784f4289c94\"\ndependencies = [\"base64 0.21.7\", \"bitflags 2.11.0\", \"serde\", \"serde_derive\"]\n\n[[package]]\nname = \"rsa\"\nversion = \"0.9.10\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d\"\ndependencies = [\"const-oid\", \"digest 0.10.7\", \"num-bigint-dig\", \"num-integer\", \"num-traits\", \"pkcs1\", \"pkcs8\", \"rand_core 0.6.4\", \"signature\", \"spki\", \"subtle\", \"zeroize\"]\n\n[[package]]\nname = \"ruint\"\nversion = \"1.17.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"c141e807189ad38a07276942c6623032d3753c8859c146104ac2e4d68865945a\"\ndependencies = [\"alloy-rlp\", \"ark-ff 0.3.0\", \"ark-ff 0.4.2\", \"ark-ff 0.5.0\", \"bytes\", \"fastrlp 0.3.1\", \"fastrlp 0.4.0\", \"num-bigint\", \"num-integer\", \"num-traits\", \"parity-scale-codec\", \"primitive-types\", \"proptest\", \"rand 0.8.5\", \"rand 0.9.2\", \"rlp\", \"ruint-macro\", \"serde_core\", \"valuable\", \"zeroize\"]\n\n[[package]]\nname = \"ruint-macro\"\nversion = \"1.2.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"48fd7bd8a6377e15ad9d42a8ec25371b94ddc67abe7c8b9127bec79bebaaae18\"\n\n[[package]]\nname = \"rust-ini\"\nversion = \"0.20.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"3e0698206bcb8882bf2a9ecb4c1e7785db57ff052297085a6efd4fe42302068a\"\ndependencies = [\"cfg-if\", \"ordered-multimap\"]\n\n[[package]]\nname = \"rustc-hash\"\nversion = \"2.1.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe\"\n\n[[package]]\nname = \"rustc-hex\"\nversion = \"2.1.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"3e75f6a532d0fd9f7f13144f392b6ad56a32696bfcd9c78f797f16bbb6f072d6\"\n\n[[package]]\nname = \"rustc_version\"\nversion = \"0.3.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"f0dfe2087c51c460008730de8b57e6a320782fbfb312e1f4d520e6c6fae155ee\"\ndependencies = [\"semver 0.11.0\"]\n\n[[package]]\nname = \"rustc_version\"\nversion = \"0.4.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92\"\ndependencies = [\"semver 1.0.27\"]\n\n[[package]]\nname = \"rusticata-macros\"\nversion = \"4.1.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632\"\ndependencies = [\"nom 7.1.3\"]\n\n[[package]]\nname = \"rustix\"\nversion = \"1.1.4\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190\"\ndependencies = [\"bitflags 2.11.0\", \"errno\", \"libc\", \"linux-raw-sys\", \"windows-sys 0.61.2\"]\n\n[[package]]\nname = \"rustls\"\nversion = \"0.23.37\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4\"\ndependencies = [\"aws-lc-rs\", \"once_cell\", \"ring\", \"rustls-pki-types\", \"rustls-webpki\", \"subtle\", \"zeroize\"]\n\n[[package]]\nname = \"rustls-connector\"\nversion = \"0.22.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"f510f2d983baf4a45354ae8ca5abf5a6cdb3c47244ea22f705499d6d9c09a912\"\ndependencies = [\"futures-io\", \"futures-rustls\", \"log\", \"rustls\", \"rustls-native-certs\", \"rustls-pki-types\", \"rustls-webpki\"]\n\n[[package]]\nname = \"rustls-native-certs\"\nversion = \"0.8.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63\"\ndependencies = [\"openssl-probe\", \"rustls-pki-types\", \"schannel\", \"security-framework\"]\n\n[[package]]\nname = \"rustls-pemfile\"\nversion = \"2.2.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50\"\ndependencies = [\"rustls-pki-types\"]\n\n[[package]]\nname = \"rustls-pki-types\"\nversion = \"1.14.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd\"\ndependencies = [\"web-time\", \"zeroize\"]\n\n[[package]]\nname = \"rustls-platform-verifier\"\nversion = \"0.6.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784\"\ndependencies = [\"core-foundation 0.10.1\", \"core-foundation-sys\", \"jni\", \"log\", \"once_cell\", \"rustls\", \"rustls-native-certs\", \"rustls-platform-verifier-android\", \"rustls-webpki\", \"security-framework\", \"security-framework-sys\", \"webpki-root-certs\", \"windows-sys 0.61.2\"]\n\n[[package]]\nname = \"rustls-platform-verifier-android\"\nversion = \"0.1.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f\"\n\n[[package]]\nname = \"rustls-webpki\"\nversion = \"0.103.10\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef\"\ndependencies = [\"aws-lc-rs\", \"ring\", \"rustls-pki-types\", \"untrusted\"]\n\n[[package]]\nname = \"rustversion\"\nversion = \"1.0.22\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d\"\n\n[[package]]\nname = \"rusty-fork\"\nversion = \"0.3.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2\"\ndependencies = [\"fnv\", \"quick-error\", \"tempfile\", \"wait-timeout\"]\n\n[[package]]\nname = \"ryu\"\nversion = \"1.0.23\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f\"\n\n[[package]]\nname = \"salsa20\"\nversion = \"0.10.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213\"\ndependencies = [\"cipher\"]\n\n[[package]]\nname = \"same-file\"\nversion = \"1.0.6\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502\"\ndependencies = [\"winapi-util\"]\n\n[[package]]\nname = \"schannel\"\nversion = \"0.1.29\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939\"\ndependencies = [\"windows-sys 0.61.2\"]\n\n[[package]]\nname = \"schemars\"\nversion = \"0.9.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"4cd191f9397d57d581cddd31014772520aa448f65ef991055d7f61582c65165f\"\ndependencies = [\"dyn-clone\", \"ref-cast\", \"serde\", \"serde_json\"]\n\n[[package]]\nname = \"schemars\"\nversion = \"1.2.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc\"\ndependencies = [\"dyn-clone\", \"ref-cast\", \"serde\", \"serde_json\"]\n\n[[package]]\nname = \"scopeguard\"\nversion = \"1.2.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49\"\n\n[[package]]\nname = \"scrypt\"\nversion = \"0.11.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f\"\ndependencies = [\"pbkdf2\", \"salsa20\", \"sha2\"]\n\n[[package]]\nname = \"sec1\"\nversion = \"0.7.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc\"\ndependencies = [\"base16ct\", \"der\", \"generic-array\", \"pkcs8\", \"serdect\", \"subtle\", \"zeroize\"]\n\n[[package]]\nname = \"secp256k1\"\nversion = \"0.30.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"b50c5943d326858130af85e049f2661ba3c78b26589b8ab98e65e80ae44a1252\"\ndependencies = [\"bitcoin_hashes\", \"rand 0.8.5\", \"secp256k1-sys\", \"serde\"]\n\n[[package]]\nname = \"secp256k1-sys\"\nversion = \"0.10.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"d4387882333d3aa8cb20530a17c69a3752e97837832f34f6dccc760e715001d9\"\ndependencies = [\"cc\"]\n\n[[package]]\nname = \"security-framework\"\nversion = \"3.7.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d\"\ndependencies = [\"bitflags 2.11.0\", \"core-foundation 0.10.1\", \"core-foundation-sys\", \"libc\", \"security-framework-sys\"]\n\n[[package]]\nname = \"security-framework-sys\"\nversion = \"2.17.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3\"\ndependencies = [\"core-foundation-sys\", \"libc\"]\n\n[[package]]\nname = \"semver\"\nversion = \"0.11.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"f301af10236f6df4160f7c3f04eec6dbc70ace82d23326abad5edee88801c6b6\"\ndependencies = [\"semver-parser\"]\n\n[[package]]\nname = \"semver\"\nversion = \"1.0.27\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2\"\n\n[[package]]\nname = \"semver-parser\"\nversion = \"0.10.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"9900206b54a3527fdc7b8a938bffd94a568bac4f4aa8113b209df75a09c0dec2\"\ndependencies = [\"pest\"]\n\n[[package]]\nname = \"send_wrapper\"\nversion = \"0.6.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73\"\n\n[[package]]\nname = \"serde\"\nversion = \"1.0.228\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e\"\ndependencies = [\"serde_core\", \"serde_derive\"]\n\n[[package]]\nname = \"serde_core\"\nversion = \"1.0.228\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad\"\ndependencies = [\"serde_derive\"]\n\n[[package]]\nname = \"serde_derive\"\nversion = \"1.0.228\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79\"\ndependencies = [\"proc-macro2\", \"quote\", \"syn 2.0.117\"]\n\n[[package]]\nname = \"serde_json\"\nversion = \"1.0.149\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86\"\ndependencies = [\"itoa\", \"memchr\", \"serde\", \"serde_core\", \"zmij\"]\n\n[[package]]\nname = \"serde_repr\"\nversion = \"0.1.20\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c\"\ndependencies = [\"proc-macro2\", \"quote\", \"syn 2.0.117\"]\n\n[[package]]\nname = \"serde_spanned\"\nversion = \"0.6.9\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3\"\ndependencies = [\"serde\"]\n\n[[package]]\nname = \"serde_urlencoded\"\nversion = \"0.7.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd\"\ndependencies = [\"form_urlencoded\", \"itoa\", \"ryu\", \"serde\"]\n\n[[package]]\nname = \"serde_with\"\nversion = \"3.18.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"dd5414fad8e6907dbdd5bc441a50ae8d6e26151a03b1de04d89a5576de61d01f\"\ndependencies = [\"base64 0.22.1\", \"chrono\", \"hex\", \"indexmap 1.9.3\", \"indexmap 2.13.1\", \"schemars 0.9.0\", \"schemars 1.2.1\", \"serde_core\", \"serde_json\", \"serde_with_macros\", \"time\"]\n\n[[package]]\nname = \"serde_with_macros\"\nversion = \"3.18.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"d3db8978e608f1fe7357e211969fd9abdcae80bac1ba7a3369bb7eb6b404eb65\"\ndependencies = [\"darling\", \"proc-macro2\", \"quote\", \"syn 2.0.117\"]\n\n[[package]]\nname = \"serdect\"\nversion = \"0.2.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"a84f14a19e9a014bb9f4512488d9829a68e04ecabffb0f9904cd1ace94598177\"\ndependencies = [\"base16ct\", \"serde\"]\n\n[[package]]\nname = \"sha1\"\nversion = \"0.10.6\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba\"\ndependencies = [\"cfg-if\", \"cpufeatures 0.2.17\", \"digest 0.10.7\"]\n\n[[package]]\nname = \"sha1_smol\"\nversion = \"1.0.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d\"\n\n[[package]]\nname = \"sha2\"\nversion = \"0.10.9\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283\"\ndependencies = [\"cfg-if\", \"cpufeatures 0.2.17\", \"digest 0.10.7\"]\n\n[[package]]\nname = \"sha3\"\nversion = \"0.10.8\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"75872d278a8f37ef87fa0ddbda7802605cb18344497949862c0d4dcb291eba60\"\ndependencies = [\"digest 0.10.7\", \"keccak\"]\n\n[[package]]\nname = \"sha3-asm\"\nversion = \"0.1.6\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"59cbb88c189d6352cc8ae96a39d19c7ecad8f7330b29461187f2587fdc2988d5\"\ndependencies = [\"cc\", \"cfg-if\"]\n\n[[package]]\nname = \"sharded-slab\"\nversion = \"0.1.7\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6\"\ndependencies = [\"lazy_static\"]\n\n[[package]]\nname = \"shlex\"\nversion = \"1.3.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64\"\n\n[[package]]\nname = \"signal-hook-registry\"\nversion = \"1.4.8\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b\"\ndependencies = [\"errno\", \"libc\"]\n\n[[package]]\nname = \"signature\"\nversion = \"2.2.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de\"\ndependencies = [\"digest 0.10.7\", \"rand_core 0.6.4\"]\n\n[[package]]\nname = \"slab\"\nversion = \"0.4.12\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5\"\n\n[[package]]\nname = \"smallvec\"\nversion = \"1.15.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03\"\ndependencies = [\"serde\"]\n\n[[package]]\nname = \"socket2\"\nversion = \"0.5.10\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678\"\ndependencies = [\"libc\", \"windows-sys 0.52.0\"]\n\n[[package]]\nname = \"socket2\"\nversion = \"0.6.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e\"\ndependencies = [\"libc\", \"windows-sys 0.61.2\"]\n\n[[package]]\nname = \"spin\"\nversion = \"0.9.8\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67\"\ndependencies = [\"lock_api\"]\n\n[[package]]\nname = \"spki\"\nversion = \"0.7.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d\"\ndependencies = [\"base64ct\", \"der\"]\n\n[[package]]\nname = \"sqlx\"\nversion = \"0.8.6\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"1fefb893899429669dcdd979aff487bd78f4064e5e7907e4269081e0ef7d97dc\"\ndependencies = [\"sqlx-core\", \"sqlx-macros\", \"sqlx-mysql\", \"sqlx-postgres\", \"sqlx-sqlite\"]\n\n[[package]]\nname = \"sqlx-core\"\nversion = \"0.8.6\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6\"\ndependencies = [\"base64 0.22.1\", \"bytes\", \"chrono\", \"crc\", \"crossbeam-queue\", \"either\", \"event-listener\", \"futures-core\", \"futures-intrusive\", \"futures-io\", \"futures-util\", \"hashbrown 0.15.5\", \"hashlink 0.10.0\", \"indexmap 2.13.1\", \"log\", \"memchr\", \"once_cell\", \"percent-encoding\", \"serde\", \"serde_json\", \"sha2\", \"smallvec\", \"thiserror 2.0.18\", \"tokio\", \"tokio-stream\", \"tracing\", \"url\", \"uuid\"]\n\n[[package]]\nname = \"sqlx-macros\"\nversion = \"0.8.6\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"a2d452988ccaacfbf5e0bdbc348fb91d7c8af5bee192173ac3636b5fb6e6715d\"\ndependencies = [\"proc-macro2\", \"quote\", \"sqlx-core\", \"sqlx-macros-core\", \"syn 2.0.117\"]\n\n[[package]]\nname = \"sqlx-macros-core\"\nversion = \"0.8.6\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"19a9c1841124ac5a61741f96e1d9e2ec77424bf323962dd894bdb93f37d5219b\"\ndependencies = [\"dotenvy\", \"either\", \"heck\", \"hex\", \"once_cell\", \"proc-macro2\", \"quote\", \"serde\", \"serde_json\", \"sha2\", \"sqlx-core\", \"sqlx-mysql\", \"sqlx-postgres\", \"sqlx-sqlite\", \"syn 2.0.117\", \"tokio\", \"url\"]\n\n[[package]]\nname = \"sqlx-mysql\"\nversion = \"0.8.6\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526\"\ndependencies = [\"atoi\", \"base64 0.22.1\", \"bitflags 2.11.0\", \"byteorder\", \"bytes\", \"chrono\", \"crc\", \"digest 0.10.7\", \"dotenvy\", \"either\", \"futures-channel\", \"futures-core\", \"futures-io\", \"futures-util\", \"generic-array\", \"hex\", \"hkdf\", \"hmac\", \"itoa\", \"log\", \"md-5\", \"memchr\", \"once_cell\", \"percent-encoding\", \"rand 0.8.5\", \"rsa\", \"serde\", \"sha1\", \"sha2\", \"smallvec\", \"sqlx-core\", \"stringprep\", \"thiserror 2.0.18\", \"tracing\", \"uuid\", \"whoami\"]\n\n[[package]]\nname = \"sqlx-postgres\"\nversion = \"0.8.6\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46\"\ndependencies = [\"atoi\", \"base64 0.22.1\", \"bitflags 2.11.0\", \"byteorder\", \"chrono\", \"crc\", \"dotenvy\", \"etcetera\", \"futures-channel\", \"futures-core\", \"futures-util\", \"hex\", \"hkdf\", \"hmac\", \"home\", \"itoa\", \"log\", \"md-5\", \"memchr\", \"once_cell\", \"rand 0.8.5\", \"serde\", \"serde_json\", \"sha2\", \"smallvec\", \"sqlx-core\", \"stringprep\", \"thiserror 2.0.18\", \"tracing\", \"uuid\", \"whoami\"]\n\n[[package]]\nname = \"sqlx-sqlite\"\nversion = \"0.8.6\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"c2d12fe70b2c1b4401038055f90f151b78208de1f9f89a7dbfd41587a10c3eea\"\ndependencies = [\"atoi\", \"chrono\", \"flume 0.11.1\", \"futures-channel\", \"futures-core\", \"futures-executor\", \"futures-intrusive\", \"futures-util\", \"libsqlite3-sys\", \"log\", \"percent-encoding\", \"serde\", \"serde_urlencoded\", \"sqlx-core\", \"thiserror 2.0.18\", \"tracing\", \"url\", \"uuid\"]\n\n[[package]]\nname = \"stable_deref_trait\"\nversion = \"1.2.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596\"\n\n[[package]]\nname = \"static_assertions\"\nversion = \"1.1.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f\"\n\n[[package]]\nname = \"stringprep\"\nversion = \"0.1.5\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1\"\ndependencies = [\"unicode-bidi\", \"unicode-normalization\", \"unicode-properties\"]\n\n[[package]]\nname = \"strsim\"\nversion = \"0.11.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f\"\n\n[[package]]\nname = \"structmeta\"\nversion = \"0.3.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"2e1575d8d40908d70f6fd05537266b90ae71b15dbbe7a8b7dffa2b759306d329\"\ndependencies = [\"proc-macro2\", \"quote\", \"structmeta-derive\", \"syn 2.0.117\"]\n\n[[package]]\nname = \"structmeta-derive\"\nversion = \"0.3.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc\"\ndependencies = [\"proc-macro2\", \"quote\", \"syn 2.0.117\"]\n\n[[package]]\nname = \"strum\"\nversion = \"0.27.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf\"\ndependencies = [\"strum_macros\"]\n\n[[package]]\nname = \"strum_macros\"\nversion = \"0.27.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7\"\ndependencies = [\"heck\", \"proc-macro2\", \"quote\", \"syn 2.0.117\"]\n\n[[package]]\nname = \"subtle\"\nversion = \"2.6.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292\"\n\n[[package]]\nname = \"syn\"\nversion = \"1.0.109\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237\"\ndependencies = [\"proc-macro2\", \"quote\", \"unicode-ident\"]\n\n[[package]]\nname = \"syn\"\nversion = \"2.0.117\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99\"\ndependencies = [\"proc-macro2\", \"quote\", \"unicode-ident\"]\n\n[[package]]\nname = \"syn-solidity\"\nversion = \"1.5.7\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"53f425ae0b12e2f5ae65542e00898d500d4d318b4baf09f40fd0d410454e9947\"\ndependencies = [\"paste\", \"proc-macro2\", \"quote\", \"syn 2.0.117\"]\n\n[[package]]\nname = \"sync_wrapper\"\nversion = \"1.0.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263\"\ndependencies = [\"futures-core\"]\n\n[[package]]\nname = \"synstructure\"\nversion = \"0.13.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2\"\ndependencies = [\"proc-macro2\", \"quote\", \"syn 2.0.117\"]\n\n[[package]]\nname = \"system-configuration\"\nversion = \"0.7.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b\"\ndependencies = [\"bitflags 2.11.0\", \"core-foundation 0.9.4\", \"system-configuration-sys\"]\n\n[[package]]\nname = \"system-configuration-sys\"\nversion = \"0.6.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4\"\ndependencies = [\"core-foundation-sys\", \"libc\"]\n\n[[package]]\nname = \"tagptr\"\nversion = \"0.2.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417\"\n\n[[package]]\nname = \"tap\"\nversion = \"1.0.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369\"\n\n[[package]]\nname = \"tcp-stream\"\nversion = \"0.34.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"ff68da304b44cbdfcb3c084ef27d8ddb8bdfba1dab36b1469513c6fe1e1c2b58\"\ndependencies = [\"async-rs\", \"cfg-if\", \"futures-io\", \"p12-keystore\", \"rustls-connector\"]\n\n[[package]]\nname = \"tempfile\"\nversion = \"3.27.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd\"\ndependencies = [\"fastrand\", \"getrandom 0.4.2\", \"once_cell\", \"rustix\", \"windows-sys 0.61.2\"]\n\n[[package]]\nname = \"test-support\"\nversion = \"0.0.1\"\ndependencies = [\"testcontainers\", \"testcontainers-modules\", \"tokio\"]\n\n[[package]]\nname = \"testcontainers\"\nversion = \"0.23.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"59a4f01f39bb10fc2a5ab23eb0d888b1e2bb168c157f61a1b98e6c501c639c74\"\ndependencies = [\"async-trait\", \"bollard\", \"bollard-stubs\", \"bytes\", \"docker_credential\", \"either\", \"etcetera\", \"futures\", \"log\", \"memchr\", \"parse-display\", \"pin-project-lite\", \"serde\", \"serde_json\", \"serde_with\", \"thiserror 2.0.18\", \"tokio\", \"tokio-stream\", \"tokio-tar\", \"tokio-util\", \"ulid\", \"url\"]\n\n[[package]]\nname = \"testcontainers-modules\"\nversion = \"0.11.6\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"4d43ed4e8f58424c3a2c6c56dbea6643c3c23e8666a34df13c54f0a184e6c707\"\ndependencies = [\"testcontainers\"]\n\n[[package]]\nname = \"thiserror\"\nversion = \"1.0.69\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52\"\ndependencies = [\"thiserror-impl 1.0.69\"]\n\n[[package]]\nname = \"thiserror\"\nversion = \"2.0.18\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4\"\ndependencies = [\"thiserror-impl 2.0.18\"]\n\n[[package]]\nname = \"thiserror-impl\"\nversion = \"1.0.69\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1\"\ndependencies = [\"proc-macro2\", \"quote\", \"syn 2.0.117\"]\n\n[[package]]\nname = \"thiserror-impl\"\nversion = \"2.0.18\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5\"\ndependencies = [\"proc-macro2\", \"quote\", \"syn 2.0.117\"]\n\n[[package]]\nname = \"thread_local\"\nversion = \"1.1.9\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185\"\ndependencies = [\"cfg-if\"]\n\n[[package]]\nname = \"threadpool\"\nversion = \"1.8.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa\"\ndependencies = [\"num_cpus\"]\n\n[[package]]\nname = \"time\"\nversion = \"0.3.47\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c\"\ndependencies = [\"deranged\", \"itoa\", \"num-conv\", \"powerfmt\", \"serde_core\", \"time-core\", \"time-macros\"]\n\n[[package]]\nname = \"time-core\"\nversion = \"0.1.8\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca\"\n\n[[package]]\nname = \"time-macros\"\nversion = \"0.2.27\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215\"\ndependencies = [\"num-conv\", \"time-core\"]\n\n[[package]]\nname = \"tiny-keccak\"\nversion = \"2.0.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237\"\ndependencies = [\"crunchy\"]\n\n[[package]]\nname = \"tinystr\"\nversion = \"0.8.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d\"\ndependencies = [\"displaydoc\", \"zerovec\"]\n\n[[package]]\nname = \"tinyvec\"\nversion = \"1.11.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3\"\ndependencies = [\"tinyvec_macros\"]\n\n[[package]]\nname = \"tinyvec_macros\"\nversion = \"0.1.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20\"\n\n[[package]]\nname = \"tokio\"\nversion = \"1.51.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"2bd1c4c0fc4a7ab90fc15ef6daaa3ec3b893f004f915f2392557ed23237820cd\"\ndependencies = [\"bytes\", \"libc\", \"mio\", \"pin-project-lite\", \"signal-hook-registry\", \"socket2 0.6.3\", \"tokio-macros\", \"windows-sys 0.61.2\"]\n\n[[package]]\nname = \"tokio-macros\"\nversion = \"2.7.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496\"\ndependencies = [\"proc-macro2\", \"quote\", \"syn 2.0.117\"]\n\n[[package]]\nname = \"tokio-rustls\"\nversion = \"0.26.4\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61\"\ndependencies = [\"rustls\", \"tokio\"]\n\n[[package]]\nname = \"tokio-stream\"\nversion = \"0.1.18\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70\"\ndependencies = [\"futures-core\", \"pin-project-lite\", \"tokio\", \"tokio-util\"]\n\n[[package]]\nname = \"tokio-tar\"\nversion = \"0.3.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"9d5714c010ca3e5c27114c1cdeb9d14641ace49874aa5626d7149e47aedace75\"\ndependencies = [\"filetime\", \"futures-core\", \"libc\", \"redox_syscall 0.3.5\", \"tokio\", \"tokio-stream\", \"xattr\"]\n\n[[package]]\nname = \"tokio-tungstenite\"\nversion = \"0.28.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"d25a406cddcc431a75d3d9afc6a7c0f7428d4891dd973e4d54c56b46127bf857\"\ndependencies = [\"futures-util\", \"log\", \"rustls\", \"rustls-pki-types\", \"tokio\", \"tokio-rustls\", \"tungstenite\", \"webpki-roots 0.26.11\"]\n\n[[package]]\nname = \"tokio-util\"\nversion = \"0.7.18\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098\"\ndependencies = [\"bytes\", \"futures-core\", \"futures-sink\", \"pin-project-lite\", \"tokio\"]\n\n[[package]]\nname = \"toml\"\nversion = \"0.8.23\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362\"\ndependencies = [\"serde\", \"serde_spanned\", \"toml_datetime 0.6.11\", \"toml_edit 0.22.27\"]\n\n[[package]]\nname = \"toml_datetime\"\nversion = \"0.6.11\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c\"\ndependencies = [\"serde\"]\n\n[[package]]\nname = \"toml_datetime\"\nversion = \"1.1.1+spec-1.1.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7\"\ndependencies = [\"serde_core\"]\n\n[[package]]\nname = \"toml_edit\"\nversion = \"0.22.27\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a\"\ndependencies = [\"indexmap 2.13.1\", \"serde\", \"serde_spanned\", \"toml_datetime 0.6.11\", \"toml_write\", \"winnow 0.7.15\"]\n\n[[package]]\nname = \"toml_edit\"\nversion = \"0.25.10+spec-1.1.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"a82418ca169e235e6c399a84e395ab6debeb3bc90edc959bf0f48647c6a32d1b\"\ndependencies = [\"indexmap 2.13.1\", \"toml_datetime 1.1.1+spec-1.1.0\", \"toml_parser\", \"winnow 1.0.1\"]\n\n[[package]]\nname = \"toml_parser\"\nversion = \"1.1.2+spec-1.1.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526\"\ndependencies = [\"winnow 1.0.1\"]\n\n[[package]]\nname = \"toml_write\"\nversion = \"0.1.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801\"\n\n[[package]]\nname = \"tower\"\nversion = \"0.5.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4\"\ndependencies = [\"futures-core\", \"futures-util\", \"pin-project-lite\", \"sync_wrapper\", \"tokio\", \"tower-layer\", \"tower-service\"]\n\n[[package]]\nname = \"tower-http\"\nversion = \"0.6.8\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8\"\ndependencies = [\"bitflags 2.11.0\", \"bytes\", \"futures-util\", \"http\", \"http-body\", \"iri-string\", \"pin-project-lite\", \"tower\", \"tower-layer\", \"tower-service\"]\n\n[[package]]\nname = \"tower-layer\"\nversion = \"0.3.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e\"\n\n[[package]]\nname = \"tower-service\"\nversion = \"0.3.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3\"\n\n[[package]]\nname = \"tracing\"\nversion = \"0.1.44\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100\"\ndependencies = [\"log\", \"pin-project-lite\", \"tracing-attributes\", \"tracing-core\"]\n\n[[package]]\nname = \"tracing-attributes\"\nversion = \"0.1.31\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da\"\ndependencies = [\"proc-macro2\", \"quote\", \"syn 2.0.117\"]\n\n[[package]]\nname = \"tracing-core\"\nversion = \"0.1.36\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a\"\ndependencies = [\"once_cell\", \"valuable\"]\n\n[[package]]\nname = \"tracing-log\"\nversion = \"0.2.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3\"\ndependencies = [\"log\", \"once_cell\", \"tracing-core\"]\n\n[[package]]\nname = \"tracing-subscriber\"\nversion = \"0.3.23\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319\"\ndependencies = [\"nu-ansi-term\", \"sharded-slab\", \"smallvec\", \"thread_local\", \"tracing-core\", \"tracing-log\"]\n\n[[package]]\nname = \"try-lock\"\nversion = \"0.2.5\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b\"\n\n[[package]]\nname = \"tungstenite\"\nversion = \"0.28.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"8628dcc84e5a09eb3d8423d6cb682965dea9133204e8fb3efee74c2a0c259442\"\ndependencies = [\"bytes\", \"data-encoding\", \"http\", \"httparse\", \"log\", \"rand 0.9.2\", \"rustls\", \"rustls-pki-types\", \"sha1\", \"thiserror 2.0.18\", \"utf-8\"]\n\n[[package]]\nname = \"typenum\"\nversion = \"1.19.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb\"\n\n[[package]]\nname = \"ucd-trie\"\nversion = \"0.1.7\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971\"\n\n[[package]]\nname = \"uint\"\nversion = \"0.9.5\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"76f64bba2c53b04fcab63c01a7d7427eadc821e3bc48c34dc9ba29c501164b52\"\ndependencies = [\"byteorder\", \"crunchy\", \"hex\", \"static_assertions\"]\n\n[[package]]\nname = \"ulid\"\nversion = \"1.2.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"470dbf6591da1b39d43c14523b2b469c86879a53e8b758c8e090a470fe7b1fbe\"\ndependencies = [\"rand 0.9.2\", \"web-time\"]\n\n[[package]]\nname = \"unarray\"\nversion = \"0.1.4\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94\"\n\n[[package]]\nname = \"unicode-bidi\"\nversion = \"0.3.18\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5\"\n\n[[package]]\nname = \"unicode-ident\"\nversion = \"1.0.24\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75\"\n\n[[package]]\nname = \"unicode-normalization\"\nversion = \"0.1.25\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8\"\ndependencies = [\"tinyvec\"]\n\n[[package]]\nname = \"unicode-properties\"\nversion = \"0.1.4\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d\"\n\n[[package]]\nname = \"unicode-segmentation\"\nversion = \"1.13.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c\"\n\n[[package]]\nname = \"unicode-xid\"\nversion = \"0.2.6\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853\"\n\n[[package]]\nname = \"untrusted\"\nversion = \"0.9.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1\"\n\n[[package]]\nname = \"url\"\nversion = \"2.5.8\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed\"\ndependencies = [\"form_urlencoded\", \"idna\", \"percent-encoding\", \"serde\", \"serde_derive\"]\n\n[[package]]\nname = \"utf-8\"\nversion = \"0.7.6\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9\"\n\n[[package]]\nname = \"utf8_iter\"\nversion = \"1.0.4\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be\"\n\n[[package]]\nname = \"uuid\"\nversion = \"1.23.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"5ac8b6f42ead25368cf5b098aeb3dc8a1a2c05a3eee8a9a1a68c640edbfc79d9\"\ndependencies = [\"getrandom 0.4.2\", \"js-sys\", \"serde_core\", \"wasm-bindgen\"]\n\n[[package]]\nname = \"valuable\"\nversion = \"0.1.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65\"\n\n[[package]]\nname = \"vcpkg\"\nversion = \"0.2.15\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426\"\n\n[[package]]\nname = \"version_check\"\nversion = \"0.9.5\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a\"\n\n[[package]]\nname = \"wait-timeout\"\nversion = \"0.2.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11\"\ndependencies = [\"libc\"]\n\n[[package]]\nname = \"walkdir\"\nversion = \"2.5.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b\"\ndependencies = [\"same-file\", \"winapi-util\"]\n\n[[package]]\nname = \"want\"\nversion = \"0.3.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e\"\ndependencies = [\"try-lock\"]\n\n[[package]]\nname = \"wasi\"\nversion = \"0.11.1+wasi-snapshot-preview1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b\"\n\n[[package]]\nname = \"wasip2\"\nversion = \"1.0.2+wasi-0.2.9\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5\"\ndependencies = [\"wit-bindgen\"]\n\n[[package]]\nname = \"wasip3\"\nversion = \"0.4.0+wasi-0.3.0-rc-2026-01-06\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5\"\ndependencies = [\"wit-bindgen\"]\n\n[[package]]\nname = \"wasite\"\nversion = \"0.1.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b\"\n\n[[package]]\nname = \"wasm-bindgen\"\nversion = \"0.2.117\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"0551fc1bb415591e3372d0bc4780db7e587d84e2a7e79da121051c5c4b89d0b0\"\ndependencies = [\"cfg-if\", \"once_cell\", \"rustversion\", \"wasm-bindgen-macro\", \"wasm-bindgen-shared\"]\n\n[[package]]\nname = \"wasm-bindgen-futures\"\nversion = \"0.4.67\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"03623de6905b7206edd0a75f69f747f134b7f0a2323392d664448bf2d3c5d87e\"\ndependencies = [\"js-sys\", \"wasm-bindgen\"]\n\n[[package]]\nname = \"wasm-bindgen-macro\"\nversion = \"0.2.117\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"7fbdf9a35adf44786aecd5ff89b4563a90325f9da0923236f6104e603c7e86be\"\ndependencies = [\"quote\", \"wasm-bindgen-macro-support\"]\n\n[[package]]\nname = \"wasm-bindgen-macro-support\"\nversion = \"0.2.117\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"dca9693ef2bab6d4e6707234500350d8dad079eb508dca05530c85dc3a529ff2\"\ndependencies = [\"bumpalo\", \"proc-macro2\", \"quote\", \"syn 2.0.117\", \"wasm-bindgen-shared\"]\n\n[[package]]\nname = \"wasm-bindgen-shared\"\nversion = \"0.2.117\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"39129a682a6d2d841b6c429d0c51e5cb0ed1a03829d8b3d1e69a011e62cb3d3b\"\ndependencies = [\"unicode-ident\"]\n\n[[package]]\nname = \"wasm-encoder\"\nversion = \"0.244.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319\"\ndependencies = [\"leb128fmt\", \"wasmparser\"]\n\n[[package]]\nname = \"wasm-metadata\"\nversion = \"0.244.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909\"\ndependencies = [\"anyhow\", \"indexmap 2.13.1\", \"wasm-encoder\", \"wasmparser\"]\n\n[[package]]\nname = \"wasmparser\"\nversion = \"0.244.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe\"\ndependencies = [\"bitflags 2.11.0\", \"hashbrown 0.15.5\", \"indexmap 2.13.1\", \"semver 1.0.27\"]\n\n[[package]]\nname = \"wasmtimer\"\nversion = \"0.4.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"1c598d6b99ea013e35844697fc4670d08339d5cda15588f193c6beedd12f644b\"\ndependencies = [\"futures\", \"js-sys\", \"parking_lot\", \"pin-utils\", \"slab\", \"wasm-bindgen\"]\n\n[[package]]\nname = \"web-sys\"\nversion = \"0.3.94\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"cd70027e39b12f0849461e08ffc50b9cd7688d942c1c8e3c7b22273236b4dd0a\"\ndependencies = [\"js-sys\", \"wasm-bindgen\"]\n\n[[package]]\nname = \"web-time\"\nversion = \"1.1.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb\"\ndependencies = [\"js-sys\", \"wasm-bindgen\"]\n\n[[package]]\nname = \"webpki-root-certs\"\nversion = \"1.0.6\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"804f18a4ac2676ffb4e8b5b5fa9ae38af06df08162314f96a68d2a363e21a8ca\"\ndependencies = [\"rustls-pki-types\"]\n\n[[package]]\nname = \"webpki-roots\"\nversion = \"0.26.11\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9\"\ndependencies = [\"webpki-roots 1.0.6\"]\n\n[[package]]\nname = \"webpki-roots\"\nversion = \"1.0.6\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"22cfaf3c063993ff62e73cb4311efde4db1efb31ab78a3e5c457939ad5cc0bed\"\ndependencies = [\"rustls-pki-types\"]\n\n[[package]]\nname = \"whoami\"\nversion = \"1.6.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d\"\ndependencies = [\"libredox\", \"wasite\"]\n\n[[package]]\nname = \"widestring\"\nversion = \"1.2.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471\"\n\n[[package]]\nname = \"winapi\"\nversion = \"0.3.9\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419\"\ndependencies = [\"winapi-i686-pc-windows-gnu\", \"winapi-x86_64-pc-windows-gnu\"]\n\n[[package]]\nname = \"winapi-i686-pc-windows-gnu\"\nversion = \"0.4.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6\"\n\n[[package]]\nname = \"winapi-util\"\nversion = \"0.1.11\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22\"\ndependencies = [\"windows-sys 0.61.2\"]\n\n[[package]]\nname = \"winapi-x86_64-pc-windows-gnu\"\nversion = \"0.4.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f\"\n\n[[package]]\nname = \"windows-core\"\nversion = \"0.62.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb\"\ndependencies = [\"windows-implement\", \"windows-interface\", \"windows-link\", \"windows-result\", \"windows-strings\"]\n\n[[package]]\nname = \"windows-implement\"\nversion = \"0.60.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf\"\ndependencies = [\"proc-macro2\", \"quote\", \"syn 2.0.117\"]\n\n[[package]]\nname = \"windows-interface\"\nversion = \"0.59.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358\"\ndependencies = [\"proc-macro2\", \"quote\", \"syn 2.0.117\"]\n\n[[package]]\nname = \"windows-link\"\nversion = \"0.2.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5\"\n\n[[package]]\nname = \"windows-registry\"\nversion = \"0.6.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720\"\ndependencies = [\"windows-link\", \"windows-result\", \"windows-strings\"]\n\n[[package]]\nname = \"windows-result\"\nversion = \"0.4.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5\"\ndependencies = [\"windows-link\"]\n\n[[package]]\nname = \"windows-strings\"\nversion = \"0.5.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091\"\ndependencies = [\"windows-link\"]\n\n[[package]]\nname = \"windows-sys\"\nversion = \"0.45.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0\"\ndependencies = [\"windows-targets 0.42.2\"]\n\n[[package]]\nname = \"windows-sys\"\nversion = \"0.48.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9\"\ndependencies = [\"windows-targets 0.48.5\"]\n\n[[package]]\nname = \"windows-sys\"\nversion = \"0.52.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d\"\ndependencies = [\"windows-targets 0.52.6\"]\n\n[[package]]\nname = \"windows-sys\"\nversion = \"0.60.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb\"\ndependencies = [\"windows-targets 0.53.5\"]\n\n[[package]]\nname = \"windows-sys\"\nversion = \"0.61.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc\"\ndependencies = [\"windows-link\"]\n\n[[package]]\nname = \"windows-targets\"\nversion = \"0.42.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071\"\ndependencies = [\"windows_aarch64_gnullvm 0.42.2\", \"windows_aarch64_msvc 0.42.2\", \"windows_i686_gnu 0.42.2\", \"windows_i686_msvc 0.42.2\", \"windows_x86_64_gnu 0.42.2\", \"windows_x86_64_gnullvm 0.42.2\", \"windows_x86_64_msvc 0.42.2\"]\n\n[[package]]\nname = \"windows-targets\"\nversion = \"0.48.5\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c\"\ndependencies = [\"windows_aarch64_gnullvm 0.48.5\", \"windows_aarch64_msvc 0.48.5\", \"windows_i686_gnu 0.48.5\", \"windows_i686_msvc 0.48.5\", \"windows_x86_64_gnu 0.48.5\", \"windows_x86_64_gnullvm 0.48.5\", \"windows_x86_64_msvc 0.48.5\"]\n\n[[package]]\nname = \"windows-targets\"\nversion = \"0.52.6\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973\"\ndependencies = [\"windows_aarch64_gnullvm 0.52.6\", \"windows_aarch64_msvc 0.52.6\", \"windows_i686_gnu 0.52.6\", \"windows_i686_gnullvm 0.52.6\", \"windows_i686_msvc 0.52.6\", \"windows_x86_64_gnu 0.52.6\", \"windows_x86_64_gnullvm 0.52.6\", \"windows_x86_64_msvc 0.52.6\"]\n\n[[package]]\nname = \"windows-targets\"\nversion = \"0.53.5\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3\"\ndependencies = [\"windows-link\", \"windows_aarch64_gnullvm 0.53.1\", \"windows_aarch64_msvc 0.53.1\", \"windows_i686_gnu 0.53.1\", \"windows_i686_gnullvm 0.53.1\", \"windows_i686_msvc 0.53.1\", \"windows_x86_64_gnu 0.53.1\", \"windows_x86_64_gnullvm 0.53.1\", \"windows_x86_64_msvc 0.53.1\"]\n\n[[package]]\nname = \"windows_aarch64_gnullvm\"\nversion = \"0.42.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8\"\n\n[[package]]\nname = \"windows_aarch64_gnullvm\"\nversion = \"0.48.5\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8\"\n\n[[package]]\nname = \"windows_aarch64_gnullvm\"\nversion = \"0.52.6\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3\"\n\n[[package]]\nname = \"windows_aarch64_gnullvm\"\nversion = \"0.53.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53\"\n\n[[package]]\nname = \"windows_aarch64_msvc\"\nversion = \"0.42.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43\"\n\n[[package]]\nname = \"windows_aarch64_msvc\"\nversion = \"0.48.5\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc\"\n\n[[package]]\nname = \"windows_aarch64_msvc\"\nversion = \"0.52.6\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469\"\n\n[[package]]\nname = \"windows_aarch64_msvc\"\nversion = \"0.53.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006\"\n\n[[package]]\nname = \"windows_i686_gnu\"\nversion = \"0.42.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f\"\n\n[[package]]\nname = \"windows_i686_gnu\"\nversion = \"0.48.5\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e\"\n\n[[package]]\nname = \"windows_i686_gnu\"\nversion = \"0.52.6\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b\"\n\n[[package]]\nname = \"windows_i686_gnu\"\nversion = \"0.53.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3\"\n\n[[package]]\nname = \"windows_i686_gnullvm\"\nversion = \"0.52.6\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66\"\n\n[[package]]\nname = \"windows_i686_gnullvm\"\nversion = \"0.53.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c\"\n\n[[package]]\nname = \"windows_i686_msvc\"\nversion = \"0.42.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060\"\n\n[[package]]\nname = \"windows_i686_msvc\"\nversion = \"0.48.5\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406\"\n\n[[package]]\nname = \"windows_i686_msvc\"\nversion = \"0.52.6\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66\"\n\n[[package]]\nname = \"windows_i686_msvc\"\nversion = \"0.53.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2\"\n\n[[package]]\nname = \"windows_x86_64_gnu\"\nversion = \"0.42.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36\"\n\n[[package]]\nname = \"windows_x86_64_gnu\"\nversion = \"0.48.5\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e\"\n\n[[package]]\nname = \"windows_x86_64_gnu\"\nversion = \"0.52.6\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78\"\n\n[[package]]\nname = \"windows_x86_64_gnu\"\nversion = \"0.53.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499\"\n\n[[package]]\nname = \"windows_x86_64_gnullvm\"\nversion = \"0.42.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3\"\n\n[[package]]\nname = \"windows_x86_64_gnullvm\"\nversion = \"0.48.5\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc\"\n\n[[package]]\nname = \"windows_x86_64_gnullvm\"\nversion = \"0.52.6\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d\"\n\n[[package]]\nname = \"windows_x86_64_gnullvm\"\nversion = \"0.53.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1\"\n\n[[package]]\nname = \"windows_x86_64_msvc\"\nversion = \"0.42.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0\"\n\n[[package]]\nname = \"windows_x86_64_msvc\"\nversion = \"0.48.5\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538\"\n\n[[package]]\nname = \"windows_x86_64_msvc\"\nversion = \"0.52.6\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec\"\n\n[[package]]\nname = \"windows_x86_64_msvc\"\nversion = \"0.53.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650\"\n\n[[package]]\nname = \"winnow\"\nversion = \"0.7.15\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945\"\ndependencies = [\"memchr\"]\n\n[[package]]\nname = \"winnow\"\nversion = \"1.0.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"09dac053f1cd375980747450bfc7250c264eaae0583872e845c0c7cd578872b5\"\ndependencies = [\"memchr\"]\n\n[[package]]\nname = \"wit-bindgen\"\nversion = \"0.51.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5\"\ndependencies = [\"wit-bindgen-rust-macro\"]\n\n[[package]]\nname = \"wit-bindgen-core\"\nversion = \"0.51.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc\"\ndependencies = [\"anyhow\", \"heck\", \"wit-parser\"]\n\n[[package]]\nname = \"wit-bindgen-rust\"\nversion = \"0.51.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21\"\ndependencies = [\"anyhow\", \"heck\", \"indexmap 2.13.1\", \"prettyplease\", \"syn 2.0.117\", \"wasm-metadata\", \"wit-bindgen-core\", \"wit-component\"]\n\n[[package]]\nname = \"wit-bindgen-rust-macro\"\nversion = \"0.51.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a\"\ndependencies = [\"anyhow\", \"prettyplease\", \"proc-macro2\", \"quote\", \"syn 2.0.117\", \"wit-bindgen-core\", \"wit-bindgen-rust\"]\n\n[[package]]\nname = \"wit-component\"\nversion = \"0.244.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2\"\ndependencies = [\"anyhow\", \"bitflags 2.11.0\", \"indexmap 2.13.1\", \"log\", \"serde\", \"serde_derive\", \"serde_json\", \"wasm-encoder\", \"wasm-metadata\", \"wasmparser\", \"wit-parser\"]\n\n[[package]]\nname = \"wit-parser\"\nversion = \"0.244.0\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736\"\ndependencies = [\"anyhow\", \"id-arena\", \"indexmap 2.13.1\", \"log\", \"semver 1.0.27\", \"serde\", \"serde_derive\", \"serde_json\", \"unicode-xid\", \"wasmparser\"]\n\n[[package]]\nname = \"writeable\"\nversion = \"0.6.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4\"\n\n[[package]]\nname = \"ws_stream_wasm\"\nversion = \"0.7.5\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"6c173014acad22e83f16403ee360115b38846fe754e735c5d9d3803fe70c6abc\"\ndependencies = [\"async_io_stream\", \"futures\", \"js-sys\", \"log\", \"pharos\", \"rustc_version 0.4.1\", \"send_wrapper\", \"thiserror 2.0.18\", \"wasm-bindgen\", \"wasm-bindgen-futures\", \"web-sys\"]\n\n[[package]]\nname = \"wyz\"\nversion = \"0.5.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed\"\ndependencies = [\"tap\"]\n\n[[package]]\nname = \"x509-cert\"\nversion = \"0.2.5\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"1301e935010a701ae5f8655edc0ad17c44bad3ac5ce8c39185f75453b720ae94\"\ndependencies = [\"const-oid\", \"der\", \"spki\"]\n\n[[package]]\nname = \"x509-parser\"\nversion = \"0.18.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202\"\ndependencies = [\"asn1-rs\", \"data-encoding\", \"der-parser\", \"lazy_static\", \"nom 7.1.3\", \"oid-registry\", \"rusticata-macros\", \"thiserror 2.0.18\", \"time\"]\n\n[[package]]\nname = \"xattr\"\nversion = \"1.6.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156\"\ndependencies = [\"libc\", \"rustix\"]\n\n[[package]]\nname = \"yaml-rust2\"\nversion = \"0.8.1\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"8902160c4e6f2fb145dbe9d6760a75e3c9522d8bf796ed7047c85919ac7115f8\"\ndependencies = [\"arraydeque\", \"encoding_rs\", \"hashlink 0.8.4\"]\n\n[[package]]\nname = \"yoke\"\nversion = \"0.8.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca\"\ndependencies = [\"stable_deref_trait\", \"yoke-derive\", \"zerofrom\"]\n\n[[package]]\nname = \"yoke-derive\"\nversion = \"0.8.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e\"\ndependencies = [\"proc-macro2\", \"quote\", \"syn 2.0.117\", \"synstructure\"]\n\n[[package]]\nname = \"zerocopy\"\nversion = \"0.8.48\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9\"\ndependencies = [\"zerocopy-derive\"]\n\n[[package]]\nname = \"zerocopy-derive\"\nversion = \"0.8.48\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4\"\ndependencies = [\"proc-macro2\", \"quote\", \"syn 2.0.117\"]\n\n[[package]]\nname = \"zerofrom\"\nversion = \"0.1.7\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df\"\ndependencies = [\"zerofrom-derive\"]\n\n[[package]]\nname = \"zerofrom-derive\"\nversion = \"0.1.7\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1\"\ndependencies = [\"proc-macro2\", \"quote\", \"syn 2.0.117\", \"synstructure\"]\n\n[[package]]\nname = \"zeroize\"\nversion = \"1.8.2\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0\"\ndependencies = [\"zeroize_derive\"]\n\n[[package]]\nname = \"zeroize_derive\"\nversion = \"1.4.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e\"\ndependencies = [\"proc-macro2\", \"quote\", \"syn 2.0.117\"]\n\n[[package]]\nname = \"zerotrie\"\nversion = \"0.2.4\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf\"\ndependencies = [\"displaydoc\", \"yoke\", \"zerofrom\"]\n\n[[package]]\nname = \"zerovec\"\nversion = \"0.11.6\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239\"\ndependencies = [\"yoke\", \"zerofrom\", \"zerovec-derive\"]\n\n[[package]]\nname = \"zerovec-derive\"\nversion = \"0.11.3\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555\"\ndependencies = [\"proc-macro2\", \"quote\", \"syn 2.0.117\"]\n\n[[package]]\nname = \"zmij\"\nversion = \"1.0.21\"\nsource = \"registry+https://github.com/rust-lang/crates.io-index\"\nchecksum = \"b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa\"\n","rust_toolchain_file":["Toml","[toolchain]\nchannel = \"1.91.1\"\n"]}} \ No newline at end of file diff --git a/listener/rust-toolchain.toml b/listener/rust-toolchain.toml new file mode 100644 index 0000000000..d6d3381977 --- /dev/null +++ b/listener/rust-toolchain.toml @@ -0,0 +1,2 @@ +[toolchain] +channel = "1.91.1" diff --git a/listener/scripts/ct-lint-extras.sh b/listener/scripts/ct-lint-extras.sh new file mode 100755 index 0000000000..b8434fd0ce --- /dev/null +++ b/listener/scripts/ct-lint-extras.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Extra lint checks invoked by chart-testing (ct lint). +# ct passes the chart path as $1. +set -euo pipefail + +CHART_DIR="${1:?usage: ct-lint-extras.sh }" +TARGET_BRANCH="${CT_TARGET_BRANCH:-main}" + +# ── 1. Broken symlink check ───────────────────────────────────────────────── +echo "── symlink check: $CHART_DIR ──" +broken=0 +while IFS= read -r link; do + [ -z "$link" ] && continue + if [ ! -e "$link" ]; then + echo "ERROR: broken symlink: $link -> $(readlink "$link")" + broken=1 + fi +done < <(find "$CHART_DIR" -type l 2>/dev/null) + +if [ $broken -ne 0 ]; then + exit 1 +fi +echo "OK: all symlinks resolve" + +# ── 2. Symlink target version-bump check ───────────────────────────────────── +# ct only detects changes inside chart-dirs. If a symlink target outside the +# chart changed, ct won't flag the chart as modified and check-version-increment +# never fires. This section covers that gap. +echo "" +echo "── symlink target change check: $CHART_DIR ──" + +merge_base=$(git merge-base HEAD "origin/${TARGET_BRANCH}" 2>/dev/null || echo "") +if [ -z "$merge_base" ]; then + echo "SKIP: could not find merge base with origin/${TARGET_BRANCH}" + exit 0 +fi + +targets_changed=0 +while IFS= read -r link; do + [ -z "$link" ] && continue + target=$(readlink "$link") + # Resolve to repo-relative path + abs_target=$(cd "$(dirname "$link")" && realpath -q "$target" 2>/dev/null || echo "") + [ -z "$abs_target" ] && continue + repo_root=$(git rev-parse --show-toplevel) + rel_target="${abs_target#"${repo_root}"/}" + # Check if this target changed vs the target branch + if git diff --quiet "$merge_base" -- "$rel_target" 2>/dev/null; then + : + else + echo "CHANGED: symlink target $rel_target (via $link)" + targets_changed=1 + fi +done < <(find "$CHART_DIR" -type l 2>/dev/null) + +if [ $targets_changed -eq 1 ]; then + old_ver=$(git show "${merge_base}:${CHART_DIR}/Chart.yaml" 2>/dev/null | grep '^version:' | awk '{print $2}') + new_ver=$(grep '^version:' "${CHART_DIR}/Chart.yaml" | awk '{print $2}') + if [ "$old_ver" = "$new_ver" ]; then + echo "FAIL: symlink targets outside chart changed but Chart.yaml version is still $old_ver" + exit 1 + fi + echo "OK: version bumped $old_ver -> $new_ver" +else + echo "OK: no symlink targets changed" +fi diff --git a/typos.toml b/typos.toml index 77ad6bdac9..d1369a2879 100644 --- a/typos.toml +++ b/typos.toml @@ -119,6 +119,7 @@ einput = "einput" # encrypted input reencrypt = "reencrypt" # re-encryption operation reencryption = "reencryption" decryptor = "decryptor" # entity that decrypts +bimap = "bimap" # Rust crate name ciphertext = "ciphertext" # encrypted data ciphertexts = "ciphertexts" plaintext = "plaintext" # unencrypted data @@ -157,3 +158,6 @@ caf = "caf" # appears in AWS AMI IDs # Test variable naming patterns BA = "BA" # used in test variable names (e.g., wrongOrderBAInsteadOfAB) + +# Doc +PN = "PN" # abbreviation P1, ..., PN \ No newline at end of file