[GCM] Add schedule_exit and bf_exit sdiag counters to slurm monitor (#142)

* [GCM] Add schedule_exit and bf_exit sdiag counters to slurm monitor

Summary:
Extend the slurm monitor sdiag telemetry with the schedule_exit and
bf_exit sub-section counters surfaced by sdiag --json. These counters
expose why the main scheduler and the backfill scheduler stopped each
cycle (e.g., end of job queue, max time, max job start, max RPC count),
which gives much better visibility into scheduler tuning and saturation
than the existing aggregate cycle stats alone.

Test Plan:
Updated and ran the existing sdiag JSON parsing tests against the
checked-in sample-sdiag-output.json, including the missing-fields case.
Ran 'pytest gcm/tests/test_slurm.py gcm/tests/test_slurm_rest_client.py'
- all 25 tests passed.

* [GCM] Add schedule_exit and bf_exit sdiag counters to slurm monitor

Summary:
Extend the slurm monitor sdiag telemetry with the schedule_exit and
bf_exit sub-section counters surfaced by sdiag --json. These counters
expose why the main scheduler and the backfill scheduler stopped each
cycle (e.g., end of job queue, max time, max job start, max RPC count),
which gives much better visibility into scheduler tuning and saturation
than the existing aggregate cycle stats alone.

Test Plan:
Updated and ran the existing sdiag JSON parsing tests against the
checked-in sample-sdiag-output.json, including the missing-fields case.
Ran 'pytest gcm/tests/test_slurm.py gcm/tests/test_slurm_rest_client.py'
- all 25 tests passed.

---------

Co-authored-by: yongl user <yongl@yongl-login-0.yongl-login.tenant-slurm.svc.cluster.local>

Author: yonglimeta

gcm/monitoring/slurm/client.py

@@ -199,6 +199,8 @@ def sdiag_structured(self) -> Sdiag:
                 subprocess.check_output(["sdiag", "--all", "--json"], text=True)
             )
             stats = sdiag_output["statistics"]
+            schedule_exit = stats.get("schedule_exit") or {}
+            bf_exit = stats.get("bf_exit") or {}
 
             result = Sdiag(
                 server_thread_count=stats.get("server_thread_count"),
@@ -224,6 +226,20 @@ def sdiag_structured(self) -> Sdiag:
                 bf_cycle_sum=stats.get("bf_cycle_sum"),
                 bf_cycle_max=stats.get("bf_cycle_max"),
                 bf_queue_len=stats.get("bf_queue_len"),
+                schedule_exit_end_job_queue=schedule_exit.get("end_job_queue"),
+                schedule_exit_default_queue_depth=schedule_exit.get(
+                    "default_queue_depth"
+                ),
+                schedule_exit_max_job_start=schedule_exit.get("max_job_start"),
+                schedule_exit_max_rpc_cnt=schedule_exit.get("max_rpc_cnt"),
+                schedule_exit_max_sched_time=schedule_exit.get("max_sched_time"),
+                schedule_exit_licenses=schedule_exit.get("licenses"),
+                bf_exit_end_job_queue=bf_exit.get("end_job_queue"),
+                bf_exit_max_job_start=bf_exit.get("bf_max_job_start"),
+                bf_exit_max_job_test=bf_exit.get("bf_max_job_test"),
+                bf_exit_max_time=bf_exit.get("bf_max_time"),
+                bf_exit_node_space_size=bf_exit.get("bf_node_space_size"),
+                bf_exit_state_changed=bf_exit.get("state_changed"),
             )
 
             # Reset sdiag counters after collection

gcm/monitoring/slurm/rest_client.py

@@ -113,6 +113,8 @@ def sinfo(self) -> Iterable[str]:
     def sdiag_structured(self) -> Sdiag:
         data = self._get(f"/slurm/{self.api_version}/diag")
         stats = data.get("statistics", {})
+        schedule_exit = stats.get("schedule_exit") or {}
+        bf_exit = stats.get("bf_exit") or {}
         result = Sdiag(
             server_thread_count=stats.get("server_thread_count"),
             agent_queue_size=stats.get("agent_queue_size"),
@@ -137,6 +139,18 @@ def sdiag_structured(self) -> Sdiag:
             bf_cycle_sum=stats.get("bf_cycle_sum"),
             bf_cycle_max=stats.get("bf_cycle_max"),
             bf_queue_len=stats.get("bf_queue_len"),
+            schedule_exit_end_job_queue=schedule_exit.get("end_job_queue"),
+            schedule_exit_default_queue_depth=schedule_exit.get("default_queue_depth"),
+            schedule_exit_max_job_start=schedule_exit.get("max_job_start"),
+            schedule_exit_max_rpc_cnt=schedule_exit.get("max_rpc_cnt"),
+            schedule_exit_max_sched_time=schedule_exit.get("max_sched_time"),
+            schedule_exit_licenses=schedule_exit.get("licenses"),
+            bf_exit_end_job_queue=bf_exit.get("end_job_queue"),
+            bf_exit_max_job_start=bf_exit.get("bf_max_job_start"),
+            bf_exit_max_job_test=bf_exit.get("bf_max_job_test"),
+            bf_exit_max_time=bf_exit.get("bf_max_time"),
+            bf_exit_node_space_size=bf_exit.get("bf_node_space_size"),
+            bf_exit_state_changed=bf_exit.get("state_changed"),
         )
         return result
 

gcm/schemas/slurm/sdiag.py

@@ -35,3 +35,19 @@ class Sdiag:
     bf_cycle_sum: Optional[int] = None
     bf_cycle_max: Optional[int] = None
     bf_queue_len: Optional[int] = None
+
+    # Schedule exit statistics
+    schedule_exit_end_job_queue: Optional[int] = None
+    schedule_exit_default_queue_depth: Optional[int] = None
+    schedule_exit_max_job_start: Optional[int] = None
+    schedule_exit_max_rpc_cnt: Optional[int] = None
+    schedule_exit_max_sched_time: Optional[int] = None
+    schedule_exit_licenses: Optional[int] = None
+
+    # Backfill exit statistics
+    bf_exit_end_job_queue: Optional[int] = None
+    bf_exit_max_job_start: Optional[int] = None
+    bf_exit_max_job_test: Optional[int] = None
+    bf_exit_max_time: Optional[int] = None
+    bf_exit_node_space_size: Optional[int] = None
+    bf_exit_state_changed: Optional[int] = None

gcm/tests/test_slurm.py

@@ -513,6 +513,18 @@ def test_parse_sdiag_json(
             bf_cycle_sum=371434634,
             bf_cycle_max=47125449,
             bf_queue_len=411,
+            schedule_exit_end_job_queue=54,
+            schedule_exit_default_queue_depth=0,
+            schedule_exit_max_job_start=0,
+            schedule_exit_max_rpc_cnt=0,
+            schedule_exit_max_sched_time=281,
+            schedule_exit_licenses=0,
+            bf_exit_end_job_queue=10,
+            bf_exit_max_job_start=0,
+            bf_exit_max_job_test=0,
+            bf_exit_max_time=0,
+            bf_exit_node_space_size=0,
+            bf_exit_state_changed=0,
         )
 
         assert result == expected
@@ -572,6 +584,18 @@ def test_parse_sdiag_json_with_missing_fields(
             bf_cycle_sum=None,
             bf_cycle_max=None,
             bf_queue_len=None,
+            schedule_exit_end_job_queue=None,
+            schedule_exit_default_queue_depth=None,
+            schedule_exit_max_job_start=None,
+            schedule_exit_max_rpc_cnt=None,
+            schedule_exit_max_sched_time=None,
+            schedule_exit_licenses=None,
+            bf_exit_end_job_queue=None,
+            bf_exit_max_job_start=None,
+            bf_exit_max_job_test=None,
+            bf_exit_max_time=None,
+            bf_exit_node_space_size=None,
+            bf_exit_state_changed=None,
         )
 
         assert result == expected