diff --git a/Cargo.lock b/Cargo.lock index bb594bd74..f9fa20d88 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8369,11 +8369,13 @@ dependencies = [ "hex", "katana-chain-spec", "katana-genesis", + "katana-metrics", "katana-primitives", "katana-provider", "katana-provider-api", "katana-rpc-types", "katana-tee", + "metrics", "piltover", "starknet 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)", "thiserror 1.0.69", diff --git a/crates/settlement/Cargo.toml b/crates/settlement/Cargo.toml index 980c2070b..62ad24260 100644 --- a/crates/settlement/Cargo.toml +++ b/crates/settlement/Cargo.toml @@ -8,6 +8,7 @@ version.workspace = true [dependencies] katana-chain-spec.workspace = true +katana-metrics.workspace = true katana-primitives.workspace = true katana-provider.workspace = true katana-provider-api.workspace = true @@ -23,6 +24,7 @@ x509-verifier-rust-crypto.workspace = true alloy-primitives.workspace = true async-trait.workspace = true +metrics.workspace = true cainome.workspace = true hex.workspace = true starknet.workspace = true diff --git a/crates/settlement/src/backend/mod.rs b/crates/settlement/src/backend/mod.rs index 5e6bf7d17..a60b6b616 100644 --- a/crates/settlement/src/backend/mod.rs +++ b/crates/settlement/src/backend/mod.rs @@ -29,6 +29,10 @@ pub trait ProvingBackend: Send + Sync { /// Human-readable backend name, for logs. fn name(&self) -> &'static str; + /// Short, label-friendly identifier of the proof system (e.g. `sp1`, + /// `mock`), used as a metric label. + fn proof_type(&self) -> &'static str; + /// Builds the `update_state` payload settling the state transition /// `(prev_block, block]`. /// diff --git a/crates/settlement/src/backend/tee/mod.rs b/crates/settlement/src/backend/tee/mod.rs index 9f7091bad..88abb71c6 100644 --- a/crates/settlement/src/backend/tee/mod.rs +++ b/crates/settlement/src/backend/tee/mod.rs @@ -70,6 +70,13 @@ where } } + fn proof_type(&self) -> &'static str { + match self.prover { + TeeProver::Mock => "mock", + TeeProver::Sp1 { .. } => "sp1", + } + } + async fn prove( &self, prev_block: Option, diff --git a/crates/settlement/src/lib.rs b/crates/settlement/src/lib.rs index 42667283b..acffb54ab 100644 --- a/crates/settlement/src/lib.rs +++ b/crates/settlement/src/lib.rs @@ -20,6 +20,7 @@ pub mod backend; mod config; pub mod error; +mod metrics; mod piltover; mod service; diff --git a/crates/settlement/src/metrics.rs b/crates/settlement/src/metrics.rs new file mode 100644 index 000000000..8880f52a0 --- /dev/null +++ b/crates/settlement/src/metrics.rs @@ -0,0 +1,52 @@ +//! Metrics for the settlement service. +//! +//! Instrumented at the service layer so they are agnostic to the proving +//! backend: `proof_generation_seconds` times the whole [`ProvingBackend::prove`] +//! call (attestation build + proving), regardless of whether it is the TEE/SP1 +//! backend or a future validity-proof one. +//! +//! [`ProvingBackend::prove`]: crate::backend::ProvingBackend::prove + +use katana_metrics::Metrics; +use metrics::{Counter, Gauge, Histogram}; + +/// Proof-related settlement metrics. +/// +/// Constructed with a `proof_type` label (e.g. `sp1`, `mock`) identifying the +/// proving backend, so proof timings can be broken down and compared across +/// backends. +#[derive(Metrics)] +#[metrics(scope = "settlement")] +pub(crate) struct SettlementProofMetrics { + /// Time spent generating a proof for a batch, in seconds — the backend + /// `prove` call (attestation build + proving). This is the dominant cost of + /// settlement and the primary thing to watch. + pub(crate) proof_generation_seconds: Histogram, + + /// End-to-end time to settle a batch (prove + submit), in seconds. + pub(crate) settle_batch_seconds: Histogram, +} + +/// Metrics for the settlement service's settle loop. +#[derive(Metrics)] +#[metrics(scope = "settlement")] +pub(crate) struct SettlementMetrics { + /// Time spent submitting the `update_state` transaction to the Piltover core + /// contract and waiting for it to land, in seconds. + pub(crate) state_update_seconds: Histogram, + + /// Number of blocks in each settled batch. + pub(crate) blocks_per_batch: Histogram, + + /// Total number of batches successfully settled. + pub(crate) batches_settled_total: Counter, + + /// Total number of blocks successfully settled. + pub(crate) blocks_settled_total: Counter, + + /// Total number of failed settlement attempts. + pub(crate) settlement_failures_total: Counter, + + /// The last block number successfully settled to the core contract. + pub(crate) settled_block: Gauge, +} diff --git a/crates/settlement/src/service.rs b/crates/settlement/src/service.rs index ede164027..a61f8f22b 100644 --- a/crates/settlement/src/service.rs +++ b/crates/settlement/src/service.rs @@ -20,6 +20,7 @@ use tracing::{error, info, warn}; use crate::backend::ProvingBackend; use crate::error::SettlementError; +use crate::metrics::{SettlementMetrics, SettlementProofMetrics}; use crate::piltover::PiltoverClient; use crate::SettlementConfig; @@ -90,6 +91,11 @@ where provider: self.provider.clone(), batch_size: self.config.batch_size.max(1) as u64, idle_flush_interval: self.config.idle_flush_interval, + metrics: SettlementMetrics::default(), + proof_metrics: SettlementProofMetrics::new_with_labels(&[( + "proof_type", + self.backend.proof_type(), + )]), }; let notify_rx = self.block_notify.subscribe(); @@ -147,6 +153,8 @@ struct Worker

{ idle_flush_interval: tokio::time::Duration, /// Last settled block, from Piltover's `get_state()`. `None` = nothing settled yet. cursor: Option, + metrics: SettlementMetrics, + proof_metrics: SettlementProofMetrics, } /// Persists the settled-block cursor to the durable [`tables::SettlementCheckpoints`] index, read @@ -231,8 +239,18 @@ where match next_action(self.cursor, head, self.batch_size, idle_elapsed) { Action::Settle { first, last } => { + let batch_start = Instant::now(); match self.settle_batch(first, last).await { Ok(tx_hash) => { + let blocks = last - first + 1; + self.proof_metrics + .settle_batch_seconds + .record(batch_start.elapsed().as_secs_f64()); + self.metrics.blocks_per_batch.record(blocks as f64); + self.metrics.batches_settled_total.increment(1); + self.metrics.blocks_settled_total.increment(blocks); + self.metrics.settled_block.set(last as f64); + info!( first, last, @@ -248,6 +266,7 @@ where } Err(error) => { + self.metrics.settlement_failures_total.increment(1); consecutive_failures += 1; error!( first, @@ -354,8 +373,16 @@ where last: BlockNumber, ) -> Result { let prev_block = if first == 0 { None } else { Some(first - 1) }; + + let proof_start = Instant::now(); let update = self.backend.prove(prev_block, last).await?; - self.piltover.update_state(update).await.map_err(Into::into) + self.proof_metrics.proof_generation_seconds.record(proof_start.elapsed().as_secs_f64()); + + let update_start = Instant::now(); + let tx_hash = self.piltover.update_state(update).await?; + self.metrics.state_update_seconds.record(update_start.elapsed().as_secs_f64()); + + Ok(tx_hash) } } diff --git a/monitoring/grafana/dashboards/overview.json b/monitoring/grafana/dashboards/overview.json index bf72d26ed..f9dcf74b1 100644 --- a/monitoring/grafana/dashboards/overview.json +++ b/monitoring/grafana/dashboards/overview.json @@ -3034,6 +3034,664 @@ ], "title": "Stage Prune Duration", "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 102 + }, + "id": 141, + "title": "Settlement", + "type": "row" + }, + { + "datasource": "Prometheus", + "description": "Time spent generating a proof for a batch (attestation build + proving), by proof type. The dominant cost of settlement.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Duration", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 103 + }, + "id": 142, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "katana_settlement_proof_generation_seconds{instance=~\"$instance\", quantile=~\"0.5|0.9|0.99\"}", + "instant": false, + "legendFormat": "p{{quantile}} · {{proof_type}}", + "range": true, + "refId": "A" + } + ], + "title": "Proof Generation Duration", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "description": "End-to-end time to settle a batch (prove + submit), by proof type. The gap over proof time is submission overhead.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Duration", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 103 + }, + "id": 143, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "katana_settlement_settle_batch_seconds{instance=~\"$instance\", quantile=~\"0.5|0.9|0.99\"}", + "instant": false, + "legendFormat": "p{{quantile}} · {{proof_type}}", + "range": true, + "refId": "A" + } + ], + "title": "Settle Batch Duration", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "description": "Time to submit the update_state transaction to the Piltover core contract and wait for it to land.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Duration", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 111 + }, + "id": 144, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "katana_settlement_state_update_seconds{instance=~\"$instance\", quantile=~\"0.5|0.9|0.99\"}", + "instant": false, + "legendFormat": "p{{quantile}}", + "range": true, + "refId": "A" + } + ], + "title": "State Update Latency", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "description": "Number of blocks in each settled batch. Small values mean idle-flushing; full values mean the backlog is driving batching.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Blocks", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 111 + }, + "id": 145, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "katana_settlement_blocks_per_batch{instance=~\"$instance\", quantile=~\"0.5|0.9|0.99\"}", + "instant": false, + "legendFormat": "p{{quantile}}", + "range": true, + "refId": "A" + } + ], + "title": "Blocks per Batch", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "description": "Blocks and batches successfully settled per second.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Rate", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 119 + }, + "id": 146, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "rate(katana_settlement_blocks_settled_total{instance=~\"$instance\"}[$__rate_interval])", + "instant": false, + "legendFormat": "Blocks/s", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "rate(katana_settlement_batches_settled_total{instance=~\"$instance\"}[$__rate_interval])", + "instant": false, + "legendFormat": "Batches/s", + "range": true, + "refId": "B" + } + ], + "title": "Settlement Throughput", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "description": "Rate of failed settlement attempts (each triggers a backoff + retry).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Rate", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 119 + }, + "id": 147, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "rate(katana_settlement_settlement_failures_total{instance=~\"$instance\"}[$__rate_interval])", + "instant": false, + "legendFormat": "Failures/s", + "range": true, + "refId": "A" + } + ], + "title": "Settlement Failures", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "description": "Last block number successfully settled to the core contract.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 127 + }, + "id": 148, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "editorMode": "code", + "expr": "katana_settlement_settled_block{instance=~\"$instance\"}", + "instant": false, + "legendFormat": "Settled Block", + "range": true, + "refId": "A" + } + ], + "title": "Settled Block", + "type": "stat" } ], "refresh": "30s",