fix(e2e): converge on OpenShell live state

Author: ericksoa

scripts/runtime-state-mutation-control.py

@@ -2588,7 +2588,11 @@ def _signal_exact_process(process: ProcessIdentity, requested_signal: int) -> No
         if current is None:
             return
         if current.identity_key() != process.identity_key():
-            _fail("writer-pid-reused")
+            # The pidfd remains bound to the original process, so a numeric PID
+            # replacement cannot receive this signal. Treat the old writer as
+            # gone; the caller's next complete writer scan will independently
+            # discover and handle the replacement identity.
+            return
         try:
             signal.pidfd_send_signal(pidfd, requested_signal)
         except ProcessLookupError:

scripts/state-dir-guard.py

@@ -47,7 +47,7 @@
     {"/sandbox/.openclaw", "/sandbox/.hermes", "/sandbox/.deepagents"}
 )
 OPENCLAW_CONFIG_DIR = "/sandbox/.openclaw"
-OPENCLAW_NATIVE_MUTABLE_ROOT = "devices"
+OPENCLAW_NATIVE_MUTABLE_ROOTS = ("devices", "identity")
 OPENCLAW_MUTATION_MUTEX_PATH = "/run/nemoclaw/openclaw-config-mutation.lock"
 MAX_TRANSITION_LOCK_BYTES = 16 * 1024
 # Keep this exact source/target contract aligned with
@@ -699,9 +699,9 @@ def is_private_writable_root(self, relative_path: str) -> bool:
         )
 
     def is_openclaw_native_mutable_path(self, relative_path: str) -> bool:
-        return self.config_path == OPENCLAW_CONFIG_DIR and (
-            relative_path == OPENCLAW_NATIVE_MUTABLE_ROOT
-            or relative_path.startswith(f"{OPENCLAW_NATIVE_MUTABLE_ROOT}/")
+        return self.config_path == OPENCLAW_CONFIG_DIR and any(
+            relative_path == root or relative_path.startswith(f"{root}/")
+            for root in OPENCLAW_NATIVE_MUTABLE_ROOTS
         )
 
     def is_under_writable_root(self, relative_path: str) -> bool:

src/lib/onboard/experimental/hermes-portable-onboarding-policy-source.test.ts

@@ -110,6 +110,7 @@ function policyRequirementsInput(
     sandboxName: boundary.sandboxName,
     gatewayName: boundary.gatewayName,
     gatewayPort: boundary.gatewayPort,
+    lifecycleLiveIdentityFingerprint: boundary.lifecycleLiveIdentityFingerprint,
     policySourcePath: boundary.policySourcePath,
     operation: "continue composed Hermes Portable onboarding",
   };
@@ -210,6 +211,7 @@ network_policies:
             sandboxName: "alpha",
             gatewayName: "nemoclaw",
             gatewayPort: GATEWAY_PORT,
+            lifecycleLiveIdentityFingerprint: HERMES_PORTABLE_TEST_LIVE_IDENTITY,
             policySourcePath: effectivePolicySourcePath,
             operation: "verify composed Hermes Portable policy",
           });

src/lib/onboard/managed-workload/onboard-orchestration.test.ts

@@ -51,6 +51,7 @@ function createFreshOnboardingRuntime(
   options: {
     readonly stockManagedRuntime?: boolean;
     readonly tempManagedRuntime?: boolean;
+    readonly tempManagedRuntimeCatalog?: string | null;
     readonly unavailableCatalog?: boolean;
   } = {},
 ) {
@@ -73,7 +74,7 @@ function createFreshOnboardingRuntime(
       managedWorkloadRebuild: null,
       tempManagedRuntime: options.tempManagedRuntime ?? false,
       stockManagedRuntime: options.stockManagedRuntime ?? false,
-      tempManagedRuntimeCatalog: null,
+      tempManagedRuntimeCatalog: options.tempManagedRuntimeCatalog ?? null,
       agentName: "openclaw",
       legacyDockerfilePath: "agents/openclaw/Dockerfile",
       customDockerfilePath: null,
@@ -211,6 +212,26 @@ describe("managed workload onboard orchestration", () => {
     await expect(runtime.ensurePreparedWorkload()).rejects.toThrow("registry offline");
   });
 
+  it("treats an explicit temporary catalog as strict managed-image selection", async () => {
+    const { prepared, runtime } = createFreshOnboardingRuntime(
+      {},
+      { tempManagedRuntimeCatalog: "/tmp/pi-candidate-catalog.json" },
+    );
+
+    await expect(runtime.ensurePreparedWorkload()).resolves.toBe(prepared);
+    expect(prepareSandboxWorkloadSource).toHaveBeenCalledExactlyOnceWith(
+      expect.objectContaining({
+        catalogPath: "/tmp/pi-candidate-catalog.json",
+        runtime: expect.objectContaining({
+          driverName: "docker",
+          managedImages: expect.objectContaining({
+            exactDigestReferences: true,
+          }),
+        }),
+      }),
+    );
+  });
+
   it("selects only the shipped Hermes Dockerfile fallback without profile or prebuild work", async () => {
     const expectedDockerfilePath = "/workspace/agents/hermes/Dockerfile";
     const ensurePreparedProfile = vi.fn(() => null);

src/lib/onboard/managed-workload/onboard-orchestration.ts

@@ -227,7 +227,10 @@ export function createManagedWorkloadOnboardRuntime(
   const discoveredRuntimeCapabilities = resolveSandboxWorkloadRuntimeCapabilities(
     input.computePlan,
   );
-  const strictManagedRuntime = input.tempManagedRuntime || input.managedWorkloadRebuild !== null;
+  const strictManagedRuntime =
+    input.tempManagedRuntime ||
+    input.tempManagedRuntimeCatalog !== null ||
+    input.managedWorkloadRebuild !== null;
   const runtimeCapabilities =
     strictManagedRuntime || input.stockManagedRuntime
       ? discoveredRuntimeCapabilities

src/lib/onboard/sandbox-create/live-policy-requirements.test.ts

@@ -0,0 +1,100 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+const state = vi.hoisted(() => ({
+  assertGateway: vi.fn(),
+  inspectPolicy: vi.fn(),
+  inspectReadiness: vi.fn(),
+}));
+
+vi.mock("../../adapters/openshell/policy-state", async (importOriginal) => ({
+  ...(await importOriginal<typeof import("../../adapters/openshell/policy-state")>()),
+  assertOpenShellGatewayPortBinding: state.assertGateway,
+  inspectOpenShellSandboxPolicyReadiness: state.inspectReadiness,
+  inspectSandboxPolicy: state.inspectPolicy,
+}));
+
+import { verifyLiveCreatedSandboxPolicyRequirements } from "./live-policy-requirements";
+
+const IDENTITY = "a".repeat(64);
+const REQUIRED_POLICY = `
+version: 1
+network_policies:
+  required:
+    name: required
+    endpoints:
+      - host: example.com
+        port: 443
+`;
+
+function inspection(activeVersion: number) {
+  return {
+    policySource: "sandbox" as const,
+    effectivePolicy: {
+      version: 1,
+      network_policies: {
+        required: {
+          name: "required",
+          endpoints: [{ host: "example.com", port: 443 }],
+        },
+      },
+    },
+    policyIdentity: { hash: `hash-${String(activeVersion)}`, activeVersion },
+  };
+}
+
+function verify(sleep = vi.fn()) {
+  verifyLiveCreatedSandboxPolicyRequirements(
+    {
+      sandboxName: "alpha",
+      gatewayName: "nemoclaw",
+      gatewayPort: 8080,
+      lifecycleLiveIdentityFingerprint: IDENTITY,
+      policySourcePath: "/tmp/required-policy.yaml",
+      operation: "continue onboarding",
+    },
+    { readFile: () => REQUIRED_POLICY, sleep },
+  );
+  return sleep;
+}
+
+describe("live created sandbox policy requirements", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    state.inspectPolicy.mockReturnValue(inspection(7));
+  });
+
+  it("waits for OpenShell's exact live policy version without recording ownership", () => {
+    state.inspectReadiness
+      .mockReturnValueOnce({ state: "transient", reason: "policy-version-pending" })
+      .mockReturnValueOnce({ state: "ready" });
+
+    const sleep = verify();
+
+    expect(sleep).toHaveBeenCalledExactlyOnceWith(1_000);
+    expect(state.inspectReadiness).toHaveBeenCalledTimes(2);
+    expect(state.inspectReadiness).toHaveBeenLastCalledWith({
+      sandboxName: "alpha",
+      gatewayName: "nemoclaw",
+      sandboxIdentityFingerprint: IDENTITY,
+      policyVersion: 7,
+    });
+    expect(state.inspectPolicy).toHaveBeenCalledTimes(3);
+  });
+
+  it("fails after the bounded OpenShell convergence window", () => {
+    state.inspectReadiness.mockReturnValue({
+      state: "transient",
+      reason: "sandbox-not-ready",
+    });
+    const sleep = vi.fn();
+
+    expect(() => verify(sleep)).toThrow(
+      "Refusing to continue onboarding: the exact sandbox is not Ready.",
+    );
+    expect(state.inspectReadiness).toHaveBeenCalledTimes(5);
+    expect(sleep).toHaveBeenCalledTimes(4);
+  });
+});

src/lib/onboard/sandbox-create/live-policy-requirements.ts

@@ -5,20 +5,32 @@ import fs from "node:fs";
 
 import {
   assertOpenShellGatewayPortBinding,
+  inspectOpenShellSandboxPolicyReadiness,
   inspectSandboxPolicy,
   PolicyObservationError,
 } from "../../adapters/openshell/policy-state";
 import { assertPolicyRequirementContainment, parseOpenShellPolicy } from "../../policy/merge";
 
+const POLICY_READINESS_MAX_OBSERVATIONS = 5;
+const POLICY_READINESS_POLL_INTERVAL_MS = 1_000;
+
+function sleepForPolicyConvergence(milliseconds: number): void {
+  Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, milliseconds);
+}
+
 export interface LiveCreatedSandboxPolicyRequirementsInput {
   readonly sandboxName: string;
   readonly gatewayName: string;
   readonly gatewayPort: number;
+  readonly lifecycleLiveIdentityFingerprint: string;
   readonly policySourcePath: string;
 }
 
 export interface LiveCreatedSandboxPolicyRequirementsDeps {
-  readonly readFile?: typeof fs.readFileSync;
+  readonly readFile?: (path: string, encoding: "utf8") => string;
+  readonly inspectPolicy?: typeof inspectSandboxPolicy;
+  readonly inspectPolicyReadiness?: typeof inspectOpenShellSandboxPolicyReadiness;
+  readonly sleep?: (milliseconds: number) => void;
 }
 
 export interface LiveCreatedSandboxPolicyRequirementsCheck extends LiveCreatedSandboxPolicyRequirementsInput {
@@ -34,10 +46,6 @@ export function verifyLiveCreatedSandboxPolicyRequirements(
     gatewayName: input.gatewayName,
     gatewayPort: input.gatewayPort,
   });
-  const inspection = inspectSandboxPolicy({
-    sandboxName: input.sandboxName,
-    gatewayName: input.gatewayName,
-  });
   let requiredPolicy: ReturnType<typeof parseOpenShellPolicy>["policy"];
   try {
     requiredPolicy = parseOpenShellPolicy(
@@ -48,10 +56,61 @@ export function verifyLiveCreatedSandboxPolicyRequirements(
       `Refusing to ${input.operation}: the required sandbox policy could not be read.`,
     );
   }
-  try {
-    assertPolicyRequirementContainment(inspection, requiredPolicy);
-  } catch (error) {
-    const detail = error instanceof Error ? error.message : String(error);
-    throw new PolicyObservationError(`Refusing to ${input.operation}: ${detail}.`);
+  const inspectPolicy = deps.inspectPolicy ?? inspectSandboxPolicy;
+  const inspectPolicyReadiness =
+    deps.inspectPolicyReadiness ?? inspectOpenShellSandboxPolicyReadiness;
+  let lastFailure = "the exact sandbox policy did not converge";
+  let ready = false;
+  for (let attempt = 0; attempt < POLICY_READINESS_MAX_OBSERVATIONS; attempt += 1) {
+    ready = (() => {
+      const before = inspectPolicy({
+        sandboxName: input.sandboxName,
+        gatewayName: input.gatewayName,
+      });
+      try {
+        assertPolicyRequirementContainment(before, requiredPolicy);
+      } catch (error) {
+        lastFailure = error instanceof Error ? error.message : String(error);
+        return false;
+      }
+      const readiness = inspectPolicyReadiness({
+        sandboxName: input.sandboxName,
+        gatewayName: input.gatewayName,
+        sandboxIdentityFingerprint: input.lifecycleLiveIdentityFingerprint,
+        policyVersion: before.policyIdentity.activeVersion,
+      });
+      if (readiness.state !== "ready") {
+        lastFailure =
+          readiness.reason === "sandbox-not-ready"
+            ? "the exact sandbox is not Ready"
+            : "the observed policy version is not active";
+        return false;
+      }
+      const after = inspectPolicy({
+        sandboxName: input.sandboxName,
+        gatewayName: input.gatewayName,
+      });
+      if (
+        after.policyIdentity.hash !== before.policyIdentity.hash ||
+        after.policyIdentity.activeVersion !== before.policyIdentity.activeVersion
+      ) {
+        lastFailure = "the live OpenShell policy changed during verification";
+        return false;
+      }
+      try {
+        assertPolicyRequirementContainment(after, requiredPolicy);
+        return true;
+      } catch (error) {
+        lastFailure = error instanceof Error ? error.message : String(error);
+        return false;
+      }
+    })();
+    if (ready) break;
+    if (attempt + 1 < POLICY_READINESS_MAX_OBSERVATIONS) {
+      (deps.sleep ?? sleepForPolicyConvergence)(POLICY_READINESS_POLL_INTERVAL_MS);
+    }
+  }
+  if (!ready) {
+    throw new PolicyObservationError(`Refusing to ${input.operation}: ${lastFailure}.`);
   }
 }

src/lib/onboard/sandbox-create/orchestration.ts

@@ -2217,6 +2217,7 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche
         sandboxName,
         gatewayName: GATEWAY_NAME,
         gatewayPort: GATEWAY_PORT,
+        lifecycleLiveIdentityFingerprint: checkpoint.sandboxIdentityFingerprint,
         policySourcePath,
         operation: `resume sandbox creation for '${sandboxName}'`,
       });
@@ -2247,6 +2248,7 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche
         sandboxName,
         gatewayName: GATEWAY_NAME,
         gatewayPort: GATEWAY_PORT,
+        lifecycleLiveIdentityFingerprint: boundary.lifecycleLiveIdentityFingerprint,
         policySourcePath: boundary.policySourcePath,
         operation,
       });

src/lib/onboard/sandbox-gpu-create-run-attempt.ts

@@ -732,10 +732,12 @@ export function createSandboxGpuCreateAttemptRunner(
                 : resolveOpenShellSandboxId(input.sandboxName, deps.runCaptureOpenshell);
             } catch (error) {
               if (createAttemptNonce) persistIdentitySettlementRecovery();
+              const diagnostic =
+                error instanceof Error ? ` ${error.message}` : " Identity settlement failed.";
               throw new Error(
                 createFailure?.kind === "sandbox_create_incomplete"
-                  ? "Managed bootstrap incomplete create did not return one exact durable sandbox identity after Ready."
-                  : "Managed bootstrap create did not return one exact durable sandbox identity after Ready.",
+                  ? `Managed bootstrap incomplete create did not return one exact durable sandbox identity after Ready.${diagnostic}`
+                  : `Managed bootstrap create did not return one exact durable sandbox identity after Ready.${diagnostic}`,
                 { cause: error },
               );
             }