Merge pull request #1037 from Dstack-TEE/feat/gateway-metrics

feat(gateway): add a /metrics endpoint on the admin listener

Author: kvinwang

docs/dstack-gateway.md

@@ -88,3 +88,49 @@ insecure_no_auth = false
 The admin server is fail-closed: if it is enabled with no `admin_token` and no `htpasswd_file`, and `insecure_no_auth` is `false`, it refuses to start rather than exposing an unauthenticated admin API.
 
 Clients authenticate by sending `Authorization: Bearer <token>` or the `X-Admin-Token: <token>` header.
+
+## Metrics
+
+The admin server exposes Prometheus metrics at `GET /metrics`. It is part of the
+admin API, so it is only reachable when `core.admin.enabled` is true and it
+requires the same credentials — unless `insecure_no_auth` is set, which exposes
+it along with the rest of the admin API. The series name domains, node ids and
+instance counts, which is topology that should not be readable without
+authentication.
+
+```yaml
+scrape_configs:
+  - job_name: dstack-gateway
+    static_configs:
+      - targets: ["<core.admin.address>"]
+    authorization:
+      credentials: "<the admin token>"
+```
+
+### Cluster-scoped vs node-local series
+
+`dstack_gateway_cluster_*` describes replicated state: every node in the cluster
+reports the same value, so summing across targets multiplies it by the number of
+nodes. Everything else describes what one process did and sums normally.
+
+```promql
+# Instances in the routing table — replicated, so take one node's view
+max(dstack_gateway_cluster_instances)
+
+# Connections across the fleet — node-local, so add them up
+sum(dstack_gateway_connections)
+
+# Nodes disagreeing about who is up: this is the replication-lag signal
+max(dstack_gateway_cluster_nodes_active) - min(dstack_gateway_cluster_nodes_active)
+```
+
+### Series worth alerting on
+
+| Metric | Why |
+|---|---|
+| `dstack_gateway_wg_reconfigure_failures_total` | The gateway could not push a WireGuard config: it failed to render, failed to write, or `wg syncconf` rejected the whole file over one bad peer stanza. Routing updates have stopped reaching the data plane while the gateway still looks healthy. |
+| `dstack_gateway_kv_decode_failures_total` | A replicated record that fails to decode is skipped, which makes the CVM behind it silently unroutable. Labelled by key prefix. Alert on `> 0`; the magnitude counts how often a bad record was *read*, not how many are bad, so do not read it as a severity. |
+| `dstack_gateway_kv_peer_buffered_logs` | Entries still buffered for a peer. Sustained growth means that peer stopped acknowledging and the two nodes are drifting apart. |
+| `dstack_gateway_cluster_cert_not_after_seconds` | Certificate expiry per domain; alert on `- time()` falling under the renewal window. Capped at 256 series — compare `dstack_gateway_cluster_cert_domains` to see whether the cap was hit. |
+| `dstack_gateway_kv_persist_failures_total` | Periodic snapshots are failing, so a restart replays a growing WAL. |
+| `dstack_gateway_kv_persist_failures_total` | Periodic snapshots are failing, so a restart replays a growing WAL. |

dstack/gateway/src/kv/mod.rs

@@ -265,6 +265,8 @@ pub mod keys {
     pub const CERT_PREFIX: &str = "cert/";
     pub const DNS_CRED_PREFIX: &str = "dns_cred/";
     pub const DNS_CRED_DEFAULT: &str = "dns_cred_default";
+    /// Shared by the `GLOBAL_*` keys below; not itself a key.
+    pub const GLOBAL_PREFIX: &str = "global/";
     pub const GLOBAL_CERTBOT_CONFIG: &str = "global/certbot_config";
     pub const GLOBAL_ACME_CREDENTIALS: &str = "global/acme_credentials";
     pub const GLOBAL_ACME_ATTESTATION: &str = "global/acme_attestation";
@@ -440,6 +442,7 @@ impl GetPutCodec for NodeState {
             .and_then(|entry| match decode(entry.value.as_ref()?) {
                 Ok(value) => Some(value),
                 Err(e) => {
+                    crate::metrics::record_decode_failure(key);
                     warn!("failed to decode value for key {key}: {e:?}");
                     None
                 }
@@ -460,6 +463,7 @@ impl GetPutCodec for NodeState {
             let value = match decode(entry.value.as_ref()?) {
                 Ok(value) => value,
                 Err(e) => {
+                    crate::metrics::record_decode_failure(key);
                     warn!("failed to decode value for key {key}: {e:?}");
                     return None;
                 }
@@ -476,6 +480,7 @@ impl GetPutCodec for NodeState {
             let value = match decode(entry.value.as_ref()?) {
                 Ok(value) => value,
                 Err(e) => {
+                    crate::metrics::record_decode_failure(key);
                     warn!("failed to decode value for key {key}: {e:?}");
                     return None;
                 }
@@ -626,6 +631,32 @@ impl KvStore {
             .collect()
     }
 
+    /// Whether a node counts as active. A node with no recorded status is up.
+    ///
+    /// The routing path and the metrics sampler both filter on this, and they
+    /// have to agree: a gauge that counts a node the router has dropped is
+    /// describing a routing table that does not exist.
+    pub(crate) fn node_is_active(status: Option<&NodeStatus>) -> bool {
+        !matches!(status, Some(NodeStatus::Down))
+    }
+
+    /// Count all and active nodes, without materialising `GatewayNodeInfo`.
+    ///
+    /// A scrape wants two numbers. Reaching them through `get_all_nodes()` and
+    /// `get_active_nodes()` instead means loading the node table twice, cloning
+    /// five strings per node, and taking the ephemeral lock once per node for a
+    /// `last_seen` that the count never reads -- all of it under the proxy lock
+    /// that the data path takes on every connection.
+    pub fn count_nodes(&self) -> (u64, u64) {
+        let statuses = self.load_all_node_statuses();
+        let nodes = self.load_all_nodes();
+        let active = nodes
+            .keys()
+            .filter(|id| Self::node_is_active(statuses.get(id)))
+            .count() as u64;
+        (nodes.len() as u64, active)
+    }
+
     // ==================== Connection Count Sync ====================
 
     /// Sync connection count for an instance (from this node)
@@ -879,6 +910,7 @@ impl KvStore {
                 match decode(value) {
                     Ok(config) => Some(config),
                     Err(e) => {
+                        crate::metrics::record_decode_failure(key);
                         warn!("failed to decode cert config for key {key}: {e:?}");
                         None
                     }
@@ -951,6 +983,7 @@ impl KvStore {
                 match decode(value) {
                     Ok(data) => Some((domain.to_string(), data)),
                     Err(e) => {
+                        crate::metrics::record_decode_failure(key);
                         warn!("failed to decode cert data for key {key}: {e:?}");
                         None
                     }
@@ -1146,6 +1179,7 @@ impl KvStore {
                 match decode(value) {
                     Ok(att) => Some(att),
                     Err(e) => {
+                        crate::metrics::record_decode_failure(key);
                         warn!("failed to decode attestation for key {key}: {e:?}");
                         None
                     }

dstack/gateway/src/main.rs

@@ -29,6 +29,7 @@ mod debug_service;
 mod distributed_certbot;
 mod kv;
 mod main_service;
+mod metrics;
 mod models;
 mod pp;
 mod proxy;

dstack/gateway/src/main_service.rs

@@ -845,7 +845,10 @@ fn start_wavekv_watch_task(proxy: Proxy) -> Result<()> {
                 match kv_store_for_persist.persist_if_dirty() {
                     Ok(true) => info!("WaveKV: periodic persist completed"),
                     Ok(false) => {} // No changes to persist
-                    Err(err) => error!("WaveKV: periodic persist failed: {err:?}"),
+                    Err(err) => {
+                        crate::metrics::record_kv_persist_failure();
+                        error!("WaveKV: periodic persist failed: {err:?}");
+                    }
                 }
             }
         });
@@ -1183,6 +1186,20 @@ impl ProxyState {
     }
 
     pub(crate) fn reconfigure(&mut self) -> Result<()> {
+        // Every way out of here that is not a clean apply leaves the data plane
+        // on the routing table it already had, so they all feed one counter --
+        // the early returns included. A config that cannot be rendered or
+        // written never reaches `wg` at all, and both call sites of this
+        // function only log the `Err`, so a full disk would otherwise look
+        // exactly like having nothing to apply.
+        let result = self.reconfigure_inner();
+        if result.is_err() {
+            crate::metrics::record_wg_reconfigure(false);
+        }
+        result
+    }
+
+    fn reconfigure_inner(&mut self) -> Result<()> {
         let wg_config = self.generate_wg_config()?;
         // the rendered config carries the interface's WireGuard private key.
         safe_write_with_mode(&self.config.wg.config_path, wg_config, 0o600)
@@ -1192,8 +1209,18 @@ impl ProxyState {
         let config_path = &self.config.wg.config_path;
 
         match cmd!(wg syncconf $ifname $config_path) {
-            Ok(_) => info!("wg config updated"),
-            Err(err) => error!("failed to set wg config: {err:?}"),
+            Ok(_) => {
+                crate::metrics::record_wg_reconfigure(true);
+                info!("wg config updated");
+            }
+            Err(err) => {
+                // `wg syncconf` rejects the whole file when one peer stanza is
+                // bad, and this stays `Ok` for the caller as it always has, so
+                // the counter is the only signal that routing updates stopped
+                // reaching the data plane.
+                crate::metrics::record_wg_reconfigure(false);
+                error!("failed to set wg config: {err:?}");
+            }
         }
         Ok(())
     }
@@ -1469,16 +1496,9 @@ impl ProxyState {
         self.kv_store
             .load_all_nodes()
             .into_iter()
-            .filter(|(id, _)| {
-                if !exclude_down {
-                    return true;
-                }
-                // Exclude nodes with status "down"
-                match node_statuses.get(id) {
-                    Some(NodeStatus::Down) => false,
-                    _ => true, // Include Up or nodes without explicit status
-                }
-            })
+            // Shared with the metrics sampler so the gauge and the routing
+            // table cannot disagree about what "active" means.
+            .filter(|(id, _)| !exclude_down || KvStore::node_is_active(node_statuses.get(id)))
             .map(|(id, node)| GatewayNodeInfo {
                 id,
                 uuid: node.uuid,