Sgl-router Prometheus metrics endpoint and usage track metrics (#6537)

Author: upfixer

sgl-router/Cargo.toml

@@ -30,6 +30,9 @@ tracing-appender = "0.2.3"
 kube = { version = "0.88.1", features = ["runtime", "derive"] }
 k8s-openapi = { version = "0.21.0", features = ["v1_29"] }
 futures = "0.3"
+# Added for metrics
+metrics = "0.24.2"
+metrics-exporter-prometheus = "0.17.0"
 [profile.release]
 lto = "thin"
 codegen-units = 1

sgl-router/README.md

@@ -81,6 +81,18 @@ router = Router(
 
 Use the `--verbose` flag with the CLI for more detailed logs.
 
+### Metrics
+
+SGL Router exposes a Prometheus HTTP scrape endpoint for monitoring, which by default listens at 127.0.0.1:29000.
+
+To change the endpoint to listen on all network interfaces and set the port to 9000, configure the following options when launching the router:
+```
+python -m sglang_router.launch_router \
+  --worker-urls http://localhost:8080 http://localhost:8081 \
+  --prometheus-host 0.0.0.0 \
+  --prometheus-port 9000
+```
+
 ### Kubernetes Service Discovery
 
 SGL Router supports automatic service discovery for worker nodes in Kubernetes environments. When enabled, the router will automatically:

sgl-router/py_src/sglang_router/launch_router.py

@@ -48,6 +48,9 @@ class RouterArgs:
     selector: Dict[str, str] = dataclasses.field(default_factory=dict)
     service_discovery_port: int = 80
     service_discovery_namespace: Optional[str] = None
+    # Prometheus configuration
+    prometheus_port: Optional[int] = None
+    prometheus_host: Optional[str] = None
 
     @staticmethod
     def add_cli_args(
@@ -176,6 +179,19 @@ def add_cli_args(
             type=str,
             help="Kubernetes namespace to watch for pods. If not provided, watches all namespaces (requires cluster-wide permissions)",
         )
+        # Prometheus configuration
+        parser.add_argument(
+            f"--{prefix}prometheus-port",
+            type=int,
+            default=29000,
+            help="Port to expose Prometheus metrics. If not specified, Prometheus metrics are disabled",
+        )
+        parser.add_argument(
+            f"--{prefix}prometheus-host",
+            type=str,
+            default="127.0.0.1",
+            help="Host address to bind the Prometheus metrics server",
+        )
 
     @classmethod
     def from_cli_args(
@@ -215,6 +231,8 @@ def from_cli_args(
             service_discovery_namespace=getattr(
                 args, f"{prefix}service_discovery_namespace", None
             ),
+            prometheus_port=getattr(args, f"{prefix}prometheus_port", None),
+            prometheus_host=getattr(args, f"{prefix}prometheus_host", None),
         )
 
     @staticmethod
@@ -278,6 +296,8 @@ def launch_router(args: argparse.Namespace) -> Optional[Router]:
             selector=router_args.selector,
             service_discovery_port=router_args.service_discovery_port,
             service_discovery_namespace=router_args.service_discovery_namespace,
+            prometheus_port=router_args.prometheus_port,
+            prometheus_host=router_args.prometheus_host,
         )
 
         router.start()

sgl-router/py_src/sglang_router/router.py

@@ -40,6 +40,8 @@ class Router:
             worker URLs using this port. Default: 80
         service_discovery_namespace: Kubernetes namespace to watch for pods. If not provided,
             watches pods across all namespaces (requires cluster-wide permissions). Default: None
+        prometheus_port: Port to expose Prometheus metrics. Default: None
+        prometheus_host: Host address to bind the Prometheus metrics server. Default: None
     """
 
     def __init__(
@@ -62,6 +64,8 @@ def __init__(
         selector: Dict[str, str] = None,
         service_discovery_port: int = 80,
         service_discovery_namespace: Optional[str] = None,
+        prometheus_port: Optional[int] = None,
+        prometheus_host: Optional[str] = None,
     ):
         if selector is None:
             selector = {}
@@ -85,6 +89,8 @@ def __init__(
             selector=selector,
             service_discovery_port=service_discovery_port,
             service_discovery_namespace=service_discovery_namespace,
+            prometheus_port=prometheus_port,
+            prometheus_host=prometheus_host,
         )
 
     def start(self) -> None:

sgl-router/py_test/test_launch_server.py

@@ -28,6 +28,8 @@ def popen_launch_router(
     selector: list = None,
     service_discovery_port: int = 80,
     service_discovery_namespace: str = None,
+    prometheus_port: int = None,
+    prometheus_host: str = None,
 ):
     """
     Launch the router server process.
@@ -45,6 +47,8 @@ def popen_launch_router(
         selector: List of label selectors in format ["key1=value1", "key2=value2"]
         service_discovery_port: Port to use for service discovery
         service_discovery_namespace: Kubernetes namespace to watch for pods. If None, watches all namespaces.
+        prometheus_port: Port to expose Prometheus metrics. If None, Prometheus metrics are disabled.
+        prometheus_host: Host address to bind the Prometheus metrics server.
     """
     _, host, port = base_url.split(":")
     host = host[2:]
@@ -87,6 +91,12 @@ def popen_launch_router(
             ["--router-service-discovery-namespace", service_discovery_namespace]
         )
 
+    if prometheus_port is not None:
+        command.extend(["--router-prometheus-port", str(prometheus_port)])
+
+    if prometheus_host is not None:
+        command.extend(["--router-prometheus-host", prometheus_host])
+
     if log_dir is not None:
         command.extend(["--log-dir", log_dir])
 

sgl-router/src/lib.rs

@@ -1,10 +1,12 @@
 use pyo3::prelude::*;
 pub mod logging;
 use std::collections::HashMap;
+pub mod prometheus;
 pub mod router;
 pub mod server;
 pub mod service_discovery;
 pub mod tree;
+use crate::prometheus::PrometheusConfig;
 
 #[pyclass(eq)]
 #[derive(Clone, PartialEq, Debug)]
@@ -35,6 +37,8 @@ struct Router {
     selector: HashMap<String, String>,
     service_discovery_port: u16,
     service_discovery_namespace: Option<String>,
+    prometheus_port: Option<u16>,
+    prometheus_host: Option<String>,
 }
 
 #[pymethods]
@@ -58,7 +62,9 @@ impl Router {
         service_discovery = false,
         selector = HashMap::new(),
         service_discovery_port = 80,
-        service_discovery_namespace = None
+        service_discovery_namespace = None,
+        prometheus_port = None,
+        prometheus_host = None
     ))]
     fn new(
         worker_urls: Vec<String>,
@@ -79,6 +85,8 @@ impl Router {
         selector: HashMap<String, String>,
         service_discovery_port: u16,
         service_discovery_namespace: Option<String>,
+        prometheus_port: Option<u16>,
+        prometheus_host: Option<String>,
     ) -> PyResult<Self> {
         Ok(Router {
             host,
@@ -99,6 +107,8 @@ impl Router {
             selector,
             service_discovery_port,
             service_discovery_namespace,
+            prometheus_port,
+            prometheus_host,
         })
     }
 
@@ -136,6 +146,15 @@ impl Router {
             None
         };
 
+        // Create Prometheus config if enabled
+        let prometheus_config = Some(PrometheusConfig {
+            port: self.prometheus_port.unwrap_or(29000),
+            host: self
+                .prometheus_host
+                .clone()
+                .unwrap_or_else(|| "127.0.0.1".to_string()),
+        });
+
         actix_web::rt::System::new().block_on(async move {
             server::startup(server::ServerConfig {
                 host: self.host.clone(),
@@ -146,6 +165,7 @@ impl Router {
                 max_payload_size: self.max_payload_size,
                 log_dir: self.log_dir.clone(),
                 service_discovery_config,
+                prometheus_config,
             })
             .await
             .map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string()))?;

sgl-router/src/prometheus.rs

@@ -0,0 +1,40 @@
+use metrics_exporter_prometheus::{Matcher, PrometheusBuilder};
+use std::net::{IpAddr, Ipv4Addr, SocketAddr};
+use std::time::Duration;
+
+#[derive(Debug, Clone)]
+pub struct PrometheusConfig {
+    pub port: u16,
+    pub host: String,
+}
+
+impl Default for PrometheusConfig {
+    fn default() -> Self {
+        Self {
+            port: 29000,
+            host: "0.0.0.0".to_string(),
+        }
+    }
+}
+
+pub fn start_prometheus(config: PrometheusConfig) {
+    let duration_matcher = Matcher::Suffix(String::from("duration"));
+    let duration_bucket = [
+        0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 15.0, 30.0, 45.0,
+        60.0, 90.0, 120.0, 180.0, 240.0,
+    ];
+
+    let ip_addr: IpAddr = config
+        .host
+        .parse()
+        .unwrap_or(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)));
+    let socket_addr = SocketAddr::new(ip_addr, config.port);
+
+    PrometheusBuilder::new()
+        .with_http_listener(socket_addr)
+        .upkeep_timeout(Duration::from_secs(5 * 60))
+        .set_buckets_for_metric(duration_matcher, &duration_bucket)
+        .expect("failed to set duration bucket")
+        .install()
+        .expect("failed to install Prometheus metrics exporter");
+}