feat(agent): runtime operations manager — idempotency-keyed switches, health, hot reload

New `packages/agent/src/runtime/operations/` module:
- `manager.ts` — RuntimeOperationManager, the single-flight gate for
  provider switches, restarts, and reloads. Replaces the ad-hoc
  `providerSwitchInProgress` boolean.
- `repository.ts` — operation-state store, surfaces pending/active/done.
- `classifier.ts` + `classifier.test.ts` — decides whether an inbound
  request is a duplicate of an in-flight op via the idempotency key.
- `health.ts` + `health.test.ts` + `health-checks.ts` — runtime health
  predicates used by the reload/cold strategies.
- `cold-strategy.ts`, `reload-hot.ts` — strategy implementations for
  full restart vs hot reload.
- `index.ts` + `types.ts` — module barrel and shared types.

Wires the manager through:
- `api/provider-switch-routes.ts` — reads Idempotency-Key header,
  routes through the manager rather than the legacy boolean gate.
- `api/server.ts`, `runtime/restart.ts` — refactored against the new
  manager; the old single-flight scaffolding is gone.
- `app-core/scripts/dev-platform.mjs`, `app-core/src/api/client-base.ts`,
  `client-types-core.ts`, `cli/run-main.ts`, `runtime/error-handlers.ts`,
  `shared/scripts/generate-keywords.mjs` — companion changes the
  manager required (dev-platform integration, error surfacing,
  client-side typing for the new op-status responses).

WIP — preserving in a commit so it doesn't sit as uncommitted state on
the feature branch. Squash / split as needed in the PR.

Author: Dexploarer

packages/agent/src/api/provider-switch-routes.ts

@@ -2,6 +2,10 @@ import type http from "node:http";
 import { logger } from "@elizaos/core";
 import type { ElizaConfig } from "../config/config.js";
 import { normalizeOnboardingProviderId } from "../contracts/onboarding.js";
+import type {
+  ProviderSwitchIntent,
+  RuntimeOperationManager,
+} from "../runtime/operations/index.js";
 import type { ReadJsonBodyOptions } from "./http-helpers.js";
 import {
   applyOnboardingConnectionConfig,
@@ -27,15 +31,38 @@ export interface ProviderSwitchRouteContext {
   ) => Promise<T | null>;
   saveElizaConfig: (config: ElizaConfig) => void;
   scheduleRuntimeRestart: (reason: string) => void;
+  /**
+   * Legacy single-flight gate — kept on the context type for now because
+   * other call sites still set it. This route no longer reads or writes
+   * the flag; the runtime operation repo's active-op slot is the gate.
+   */
   providerSwitchInProgress: boolean;
   setProviderSwitchInProgress: (value: boolean) => void;
-  restartRuntime?: (reason: string) => Promise<boolean>;
+  runtimeOperationManager: RuntimeOperationManager;
 }
 
 // ---------------------------------------------------------------------------
 // Route handler
 // ---------------------------------------------------------------------------
 
+function readIdempotencyKey(
+  headers: http.IncomingHttpHeaders,
+): string | undefined {
+  for (const [key, value] of Object.entries(headers)) {
+    if (key.toLowerCase() !== "idempotency-key") continue;
+    if (typeof value === "string" && value.trim().length > 0) {
+      return value.trim();
+    }
+    if (Array.isArray(value)) {
+      const first = value.find(
+        (v) => typeof v === "string" && v.trim().length > 0,
+      );
+      if (first) return first.trim();
+    }
+  }
+  return undefined;
+}
+
 export async function handleProviderSwitchRoutes(
   ctx: ProviderSwitchRouteContext,
 ): Promise<boolean> {
@@ -59,21 +86,14 @@ export async function handleProviderSwitchRoutes(
       return true;
     }
 
-    if (ctx.providerSwitchInProgress) {
-      error(res, "Provider switch already in progress", 409);
+    const trimmedApiKey =
+      typeof body.apiKey === "string" ? body.apiKey.trim() : undefined;
+    if (trimmedApiKey && trimmedApiKey.length > 512) {
+      error(res, "API key is too long", 400);
       return true;
     }
-    ctx.setProviderSwitchInProgress(true);
 
     try {
-      const trimmedApiKey =
-        typeof body.apiKey === "string" ? body.apiKey.trim() : undefined;
-      if (trimmedApiKey && trimmedApiKey.length > 512) {
-        ctx.setProviderSwitchInProgress(false);
-        error(res, "API key is too long", 400);
-        return true;
-      }
-
       const config = state.config;
       let connection:
         | ReturnType<typeof createProviderSwitchConnection>
@@ -97,7 +117,7 @@ export async function handleProviderSwitchRoutes(
           process.env.OPENAI_BASE_URL = `${cloudBaseUrl}/api/v1`;
           process.env.OPENAI_API_KEY = cloudApiKey;
         }
-      } else if (normalizedProvider) {
+      } else {
         connection = createProviderSwitchConnection({
           provider: normalizedProvider,
           apiKey: trimmedApiKey,
@@ -106,36 +126,75 @@ export async function handleProviderSwitchRoutes(
               ? body.primaryModel.trim()
               : undefined,
         });
-      } else {
-        connection = null;
       }
 
       if (!connection) {
-        ctx.setProviderSwitchInProgress(false);
         error(res, "Invalid provider", 400);
         return true;
       }
 
       await applyOnboardingConnectionConfig(config, connection);
       ctx.saveElizaConfig(config);
 
-      const restartReason = `provider switch to ${normalizedProvider}`;
-      const restarted = ctx.restartRuntime
-        ? await ctx.restartRuntime(restartReason)
-        : false;
-      if (!restarted) {
-        ctx.scheduleRuntimeRestart(restartReason);
+      const intent: ProviderSwitchIntent = {
+        kind: "provider-switch",
+        provider: normalizedProvider,
+        apiKey: trimmedApiKey,
+        primaryModel:
+          typeof body.primaryModel === "string"
+            ? body.primaryModel.trim()
+            : undefined,
+      };
+      const idempotencyKey = readIdempotencyKey(req.headers);
+
+      const outcome = await ctx.runtimeOperationManager.start({
+        intent,
+        idempotencyKey,
+      });
+
+      if (outcome.kind === "accepted") {
+        logger.info(
+          `[api] Provider switch accepted: provider=${normalizedProvider} op=${outcome.operation.id}`,
+        );
+        json(
+          res,
+          {
+            success: true,
+            provider: normalizedProvider,
+            restarting: true,
+            operationId: outcome.operation.id,
+          },
+          202,
+        );
+        return true;
       }
 
-      ctx.setProviderSwitchInProgress(false);
+      if (outcome.kind === "deduped") {
+        const op = outcome.operation;
+        logger.info(
+          `[api] Provider switch deduped: provider=${normalizedProvider} op=${op.id} status=${op.status}`,
+        );
+        json(res, {
+          success: true,
+          provider: normalizedProvider,
+          restarting: op.status === "running" || op.status === "pending",
+          operationId: op.id,
+          deduped: true,
+        });
+        return true;
+      }
 
-      json(res, {
-        success: true,
-        provider: normalizedProvider,
-        restarting: restarted,
-      });
+      // outcome.kind === "rejected-busy"
+      json(
+        res,
+        {
+          error: "Provider switch already in progress",
+          activeOperationId: outcome.activeOperationId,
+        },
+        409,
+      );
+      return true;
     } catch (err) {
-      ctx.setProviderSwitchInProgress(false);
       logger.error(
         `[api] Provider switch failed: ${err instanceof Error ? err.stack : err}`,
       );

packages/agent/src/api/server.ts

@@ -205,6 +205,20 @@ import type { PTYService } from "./parse-action-block.js";
 import { handlePermissionRoutes } from "./permissions-routes.js";
 import { handlePermissionsExtraRoutes } from "./permissions-routes-extra.js";
 import { handlePluginRoutes } from "./plugin-routes.js";
+import {
+  type ClassifyContext,
+  createColdStrategy,
+  createHotStrategy,
+  defaultClassifier,
+  DefaultRuntimeOperationManager,
+  getDefaultHealthChecker,
+  getDefaultRepository,
+  type RuntimeOperationManager,
+} from "../runtime/operations/index.js";
+import {
+  resolvePreferredProviderId,
+  resolvePrimaryModel,
+} from "../runtime/eliza.js";
 import { handleProviderSwitchRoutes } from "./provider-switch-routes.js";
 import { handleRegistryRoutes } from "./registry-routes.js";
 import { RegistryService } from "./registry-service.js";
@@ -1044,6 +1058,46 @@ const clearPairing = _clearPairing;
 /** Guard against concurrent provider switch requests (P0 §3). */
 let providerSwitchInProgress = false;
 
+/**
+ * Lazy per-process runtime operation manager. Constructed on first
+ * request because it needs the per-server `state` reference + the
+ * `onRestart` closure. Cached so subsequent requests see the same
+ * active-op slot and execution chain.
+ */
+let cachedRuntimeOperationManager: RuntimeOperationManager | null = null;
+
+function getOrCreateRuntimeOperationManager(
+  state: ServerState,
+  restartRuntime: (reason: string) => Promise<boolean>,
+): RuntimeOperationManager {
+  if (cachedRuntimeOperationManager) {
+    return cachedRuntimeOperationManager;
+  }
+  const repository = getDefaultRepository();
+  const healthChecker = getDefaultHealthChecker();
+  const coldStrategy = createColdStrategy({
+    restartRuntime: async (reason) => {
+      const ok = await restartRuntime(reason);
+      if (!ok) return null;
+      return state.runtime;
+    },
+  });
+  const hotStrategy = createHotStrategy({});
+  const classifyContext = (): ClassifyContext => ({
+    currentProvider: resolvePreferredProviderId(state.config),
+    currentPrimaryModel: resolvePrimaryModel(state.config),
+  });
+  cachedRuntimeOperationManager = new DefaultRuntimeOperationManager({
+    repository,
+    runtime: () => state.runtime,
+    classifyContext,
+    classifier: defaultClassifier,
+    healthChecker,
+    strategies: { cold: coldStrategy, hot: hotStrategy },
+  });
+  return cachedRuntimeOperationManager;
+}
+
 // PluginConfigMutationRejection, resolvePluginConfigMutationRejections,
 // WalletExportRejection, resolveWalletExportRejection
 // extracted to server-helpers-plugin.ts and server-helpers-wallet.ts respectively.
@@ -1323,6 +1377,10 @@ async function handleRequest(
   };
 
   // ── POST /api/provider/switch (extracted to provider-switch-routes.ts) ──
+  const runtimeOperationManager = getOrCreateRuntimeOperationManager(
+    state,
+    restartRuntime,
+  );
   if (
     await handleProviderSwitchRoutes({
       req,
@@ -1339,7 +1397,7 @@ async function handleRequest(
       setProviderSwitchInProgress: (v: boolean) => {
         providerSwitchInProgress = v;
       },
-      restartRuntime,
+      runtimeOperationManager,
     })
   ) {
     return;