feat: Introduce service_is_healthy metric

Author: zuston

src/heartbeat.rs

@@ -3,6 +3,7 @@ use crate::config::Config;
 use crate::grpc::protobuf::uniffle::coordinator_server_client::CoordinatorServerClient;
 use crate::grpc::protobuf::uniffle::{ShuffleServerHeartBeatRequest, ShuffleServerId};
 use crate::health_service::HealthService;
+use crate::metric::SERVICE_IS_HEALTHY;
 use crate::runtime::manager::RuntimeManager;
 use log::info;
 use std::time::Duration;
@@ -58,6 +59,8 @@ impl HeartbeatTask {
                 all_tags.extend_from_slice(&*tags);
 
                 let healthy = health_service.is_healthy().await.unwrap_or(false);
+                SERVICE_IS_HEALTHY.set(if healthy { 0 } else { 1 });
+
                 let memory_snapshot = app_manager
                     .store_memory_snapshot()
                     .await

src/metric.rs

@@ -398,6 +398,9 @@ pub static GAUGE_LOCAL_DISK_IS_HEALTHY: Lazy<IntGaugeVec> = Lazy::new(|| {
     .unwrap()
 });
 
+pub static SERVICE_IS_HEALTHY: Lazy<IntGauge> =
+    Lazy::new(|| IntGauge::new("service_is_healthy", "service_is_healthy").expect(""));
+
 pub static GAUGE_RUNTIME_ALIVE_THREAD_NUM: Lazy<IntGaugeVec> = Lazy::new(|| {
     register_int_gauge_vec!(
         "runtime_thread_alive_gauge",
@@ -520,6 +523,9 @@ pub static EVENT_BUS_HANDLE_DURATION: Lazy<HistogramVec> = Lazy::new(|| {
 });
 
 fn register_custom_metrics() {
+    REGISTRY
+        .register(Box::new(SERVICE_IS_HEALTHY.clone()))
+        .expect("");
     REGISTRY
         .register(Box::new(GAUGE_MEMORY_SPILL_IN_QUEUE_BYTES.clone()))
         .expect("");