refactor(BA-5714): drop stray deployment changes and restore dropped comments

- Revert deployment/executor.py, deployment/route/coordinator.py,
  deployment/route/executor.py, deployment/route/handlers/observer/
  health_check.py to the parent branch state — those edits pre-dated
  main and are unrelated to BA-5650; they slipped in when this slice
  pulled the whole sokovan/ subtree.
- sokovan/data/allocation.py: restore the explanatory comment above
  ``main_access_key`` and fix ``from_agent_selections`` to pass
  ``main_access_key=`` (matches the renamed field).
- sokovan/data/workload.py: restore the explanatory comment above
  ``SessionWorkload.main_access_key``.

Author: jopemachine

src/ai/backend/manager/sokovan/data/allocation.py

@@ -80,6 +80,7 @@ class SessionAllocation:
     kernel_allocations: list[KernelAllocation]
     # List of agent allocations for this session
     agent_allocations: list[AgentAllocation]
+    # Owner's resolved main_access_key; required for keypair-scoped concurrency tracking and resource policy lookups.
     main_access_key: AccessKey
     # Phases that passed during scheduling
     passed_phases: list[SchedulingPredicate] = field(default_factory=list)
@@ -140,7 +141,7 @@ def from_agent_selections(
             scaling_group=scaling_group,
             kernel_allocations=kernel_allocations,
             agent_allocations=agent_allocations,
-            access_key=session_workload.main_access_key,
+            main_access_key=session_workload.main_access_key,
         )
 
     def unique_agent_ids(self) -> list[AgentId]:

src/ai/backend/manager/sokovan/data/workload.py

@@ -77,6 +77,7 @@ class SessionWorkload:
 
     # Session identifier
     session_id: SessionId
+    # Owner's resolved main_access_key; required for keypair-scoped concurrency tracking and resource policy lookups.
     main_access_key: AccessKey
     # Resource requirements
     requested_slots: ResourceSlot

src/ai/backend/manager/sokovan/deployment/executor.py

@@ -12,7 +12,7 @@
     ClientPool,
 )
 from ai.backend.common.clients.prometheus.client import PrometheusClient
-from ai.backend.common.clients.prometheus.preset import MetricPreset
+from ai.backend.common.clients.prometheus.preset import LabelMatcher, MetricPreset
 from ai.backend.common.clients.valkey_client.valkey_stat.client import ValkeyStatClient
 from ai.backend.common.config import ModelHealthCheck
 from ai.backend.common.data.permission.types import RBACElementType
@@ -762,7 +762,7 @@ async def _fetch_prometheus_metric(
         preset_data: PrometheusQueryPresetData = presets[preset_id]
 
         # Auto-inject deployment-specific label for scoping
-        labels: dict[str, str] = {"model_service_name": deployment.metadata.name}
+        labels = {"model_service_name": LabelMatcher.exact(deployment.metadata.name)}
 
         # time_window: preset default → fallback to "5m"
         time_window = preset_data.time_window or "5m"

src/ai/backend/manager/sokovan/deployment/route/coordinator.py

@@ -370,6 +370,15 @@ async def _handle_status_transitions(
                 batch_updaters, BulkCreator(specs=all_history_specs)
             )
 
+        # Record running_at in Valkey for routes that just transitioned to RUNNING
+        if (
+            transitions.success is not None
+            and transitions.success.status == RouteStatus.RUNNING
+            and result.successes
+        ):
+            for route in result.successes:
+                await self._valkey_schedule.mark_route_running_at(str(route.route_id))
+
     async def process_if_needed(self, lifecycle_type: RouteLifecycleType) -> None:
         """
         Process route lifecycle operation if needed (based on internal state).

src/ai/backend/manager/sokovan/deployment/route/executor.py

@@ -193,6 +193,11 @@ async def check_running_routes(self, routes: Sequence[RouteData]) -> RouteExecut
                 with RouteRecorderContext.shared_step("fetch_kernel_connection_info"):
                     await self._populate_replica_info(routes_missing_replica)
 
+        # Phase 4: Ensure RouteHealthRecords exist in Valkey for routes with replica info
+        routes_with_replica = [r for r in successes if r.replica_host and r.replica_port]
+        if routes_with_replica:
+            await self._ensure_health_records(routes_with_replica)
+
         return RouteExecutionResult(
             successes=successes,
             errors=errors,
@@ -220,6 +225,29 @@ async def _populate_replica_info(self, routes: Sequence[RouteData]) -> None:
         if populated_routes:
             await self._initialize_health_records(populated_routes, updates)
 
+    async def _ensure_health_records(self, routes: Sequence[RouteData]) -> None:
+        """Ensure RouteHealthRecords exist in Valkey for routes that already have replica info.
+
+        Routes may already have replica_host/port in DB (set by a previous cycle or legacy code)
+        but lack a RouteHealthRecord in Valkey. This method checks and initializes missing records.
+        """
+        route_id_strs = [str(r.route_id) for r in routes]
+        existing = await self._valkey_schedule.get_route_health_records_batch(route_id_strs)
+        missing = [r for r in routes if existing.get(str(r.route_id)) is None]
+        if not missing:
+            return
+        log.warning(
+            "RouteHealthRecord missing in Valkey for {} routes, re-initializing: {}",
+            len(missing),
+            [str(r.route_id)[:8] for r in missing],
+        )
+        replica_info = {
+            r.route_id: (r.replica_host, r.replica_port)
+            for r in missing
+            if r.replica_host and r.replica_port
+        }
+        await self._initialize_health_records(missing, replica_info)
+
     async def _initialize_health_records(
         self,
         routes: Sequence[RouteData],
@@ -230,6 +258,14 @@ async def _initialize_health_records(
         health_configs = await self._deployment_repo.fetch_health_check_configs_by_revision_ids(
             revision_ids
         )
+        redis_time = await self._valkey_schedule.get_redis_time()
+
+        # Read existing running_at values that were set when routes transitioned to RUNNING
+        # These may be in partial hashes (only running_at field), so read raw field directly
+        running_at_map = await self._valkey_schedule.get_route_running_at_batch([
+            str(r.route_id) for r in routes
+        ])
+
         records: list[RouteHealthRecord] = []
         for route in routes:
             host, port = replica_info[route.route_id]
@@ -238,16 +274,21 @@ async def _initialize_health_records(
             health_path = health_config.path if health_config else "/"
             initial_delay = health_config.initial_delay if health_config else 60.0
             created_at = int(route.created_at.timestamp())
-            initial_delay_until = created_at + int(initial_delay)
+
+            # Use running_at from Valkey (set at RUNNING transition), fallback to redis_time
+            route_id_str = str(route.route_id)
+            running_at = running_at_map.get(route_id_str) or redis_time
+            initial_delay_until = running_at + int(initial_delay)
 
             records.append(
                 RouteHealthRecord(
-                    route_id=str(route.route_id),
+                    route_id=route_id_str,
                     created_at=created_at,
                     initial_delay_until=initial_delay_until,
                     health_path=health_path,
                     inference_port=port,
                     replica_host=host,
+                    running_at=running_at,
                 )
             )
 

src/ai/backend/manager/sokovan/deployment/route/handlers/observer/health_check.py

@@ -73,6 +73,11 @@ async def observe(self, routes: Sequence[RouteData]) -> RouteObservationResult:
                 targets.append((route_id_str, record))
 
         if not targets:
+            if checkable:
+                log.warning(
+                    "Health observer: {} checkable routes but 0 have records in Valkey",
+                    len(checkable),
+                )
             return RouteObservationResult(observed_count=0)
 
         # Perform HTTP health checks in parallel