From fe2ab299db8c6533152aea435176caf94a2592bd Mon Sep 17 00:00:00 2001 From: Yimo Jiang Date: Fri, 28 Aug 2026 05:46:33 +0000 Subject: [PATCH 1/4] fix(sandbox): recover legacy Hermes rebuilds Allow legacy Station qualification evidence to authorize rebuild recovery and retry permission-denied Hermes state capture through managed privileged backup authority. Closes #10370 Closes #10375 Signed-off-by: Yimo Jiang --- .../actions/sandbox/rebuild-gpu-opt-out.ts | 2 + ...eflight-target-phase-orchestration.test.ts | 53 +++++ .../sandbox/rebuild-preflight-target-phase.ts | 4 + .../sandbox/rebuild-target-preflight.ts | 6 +- .../sandbox/rebuild-target-runtime.test.ts | 40 ++++ .../actions/sandbox/rebuild-target-runtime.ts | 18 ++ src/lib/actions/sandbox/snapshot.test.ts | 11 +- .../snapshot/backup-authority-script.test.ts | 75 ++++++- .../sandbox/snapshot/backup-authority.test.ts | 96 +++++++++ .../sandbox/snapshot/backup-authority.ts | 192 +++++++++++++++++- src/lib/onboard.ts | 2 +- .../authoritative-rebuild-target.test.ts | 11 +- .../onboard/authoritative-rebuild-target.ts | 9 +- src/lib/onboard/fatal-runtime-preflight.ts | 7 + src/lib/onboard/machine/handlers/preflight.ts | 5 + .../onboard/machine/initial-flow-phases.ts | 2 + src/lib/onboard/types.ts | 2 + src/lib/readiness/onboard-admission.test.ts | 46 ++++- src/lib/readiness/onboard-admission.ts | 9 + src/lib/state/sandbox.ts | 191 ++++++++++++++++- .../hermes/hermes-kanban-snapshot.test.ts | 22 ++ 21 files changed, 773 insertions(+), 30 deletions(-) diff --git a/src/lib/actions/sandbox/rebuild-gpu-opt-out.ts b/src/lib/actions/sandbox/rebuild-gpu-opt-out.ts index 7eea6fd5562..33c5b6d2e2e 100644 --- a/src/lib/actions/sandbox/rebuild-gpu-opt-out.ts +++ b/src/lib/actions/sandbox/rebuild-gpu-opt-out.ts @@ -129,6 +129,8 @@ export type RebuildRecreateOnboardOpts = { providerRecoveryReceipt?: ProviderRecoveryReceipt; /** Recorded managed-vLLM intent admitted only by the N1x readiness exception. */ allowDeferredN1xManagedVllm?: true; + /** Internal legacy Hermes rebuild authority for the pre-v0.0.97 Station admission rule. */ + allowLegacyDgxStationQualification?: true; /** Target-scoped authority admitted by the authoritative rebuild preflight. */ rebuildGatewayAuthority?: CheckpointGatewayAuthority; preparedImageRebuild?: PreparedImageRebuildHandoff; diff --git a/src/lib/actions/sandbox/rebuild-preflight-target-phase-orchestration.test.ts b/src/lib/actions/sandbox/rebuild-preflight-target-phase-orchestration.test.ts index ab4fcdeb885..42a788cbf3c 100644 --- a/src/lib/actions/sandbox/rebuild-preflight-target-phase-orchestration.test.ts +++ b/src/lib/actions/sandbox/rebuild-preflight-target-phase-orchestration.test.ts @@ -182,6 +182,59 @@ describe("prepareRebuildTargetPreflights", () => { expect(mocks.resolveContextWindowForModel).toHaveBeenCalledWith("ollama-local", "qwen3.5:9b"); }); + it("passes legacy Station authority from the source registry row into rebuild readiness (#10370)", async () => { + const resumeConfig = { + provider: "ollama-local", + model: "llama3.2:1b", + preferredInferenceApi: "openai-completions", + endpointUrl: null, + compatibleEndpointReasoning: null, + compatibleEndpointReasoningEffort: null, + registryInferenceRoute: null, + }; + mocks.prepareRebuildTargetConfig.mockReturnValue({ + agentDefinition: {}, + resumeConfig, + durableConfig: { + toolDisclosure: "progressive", + dcodeAutoApprovalMode: "disabled", + webSearchConfig: null, + }, + credentialEnv: null, + fromDockerfile: false, + hermesToolGateways: [], + }); + mocks.prepareRebuildRecreateOptions.mockReturnValue({ + controlUiPort: 18_789, + targetGatewayName: "nemoclaw", + toolDisclosure: "progressive", + dcodeAutoApprovalMode: "disabled", + observabilityEnabled: false, + }); + + await prepareRebuildTargetPreflights({ + sandboxName: "legacy-hermes", + sandboxEntry: { + name: "legacy-hermes", + agent: "hermes", + nemoclawVersion: "v0.0.83", + fromDockerfile: null, + gatewayName: "nemoclaw", + openshellDriver: "docker", + provider: resumeConfig.provider, + model: resumeConfig.model, + } as never, + rebuildAgent: "hermes", + autoYes: true, + log: vi.fn(), + bail: mocks.bail as never, + }); + + expect(mocks.preflightAuthoritativeOnboardRuntime.mock.calls[0]?.[2]).toEqual( + expect.objectContaining({ allowLegacyDgxStationQualification: true }), + ); + }); + it("passes exact legacy N1x intent into authoritative readiness (#9292)", async () => { const readinessOptions = await prepareN1xTarget("onboard"); diff --git a/src/lib/actions/sandbox/rebuild-preflight-target-phase.ts b/src/lib/actions/sandbox/rebuild-preflight-target-phase.ts index 0c415187ce8..0605f02ed3f 100644 --- a/src/lib/actions/sandbox/rebuild-preflight-target-phase.ts +++ b/src/lib/actions/sandbox/rebuild-preflight-target-phase.ts @@ -49,6 +49,7 @@ import { } from "./rebuild-preflight-guards"; import { disposePreparedBuildContext } from "./rebuild-prepared-image-context"; import { + hasLegacyDgxStationQualificationAuthority, hydrateMessagingConfigForRebuild, preflightAuthoritativeOnboardRuntime, preflightRebuildTargetRuntime, @@ -198,6 +199,9 @@ export async function prepareRebuildTargetPreflights(args: { bail, ); if (!recreateOptions) return null; + if (hasLegacyDgxStationQualificationAuthority(sandboxEntry)) { + recreateOptions.allowLegacyDgxStationQualification = true; + } let managedWorkloadRebuildCatalog: Awaited< ReturnType > = null; diff --git a/src/lib/actions/sandbox/rebuild-target-preflight.ts b/src/lib/actions/sandbox/rebuild-target-preflight.ts index c0368f626bd..ead86873507 100644 --- a/src/lib/actions/sandbox/rebuild-target-preflight.ts +++ b/src/lib/actions/sandbox/rebuild-target-preflight.ts @@ -7,11 +7,9 @@ * staging remain independently reviewable. */ export { printRebuildPreflightFailure } from "./rebuild-preflight-error"; +export { prepareRebuildTargetConfig, type RebuildTargetConfig } from "./rebuild-target-config"; export { - prepareRebuildTargetConfig, - type RebuildTargetConfig, -} from "./rebuild-target-config"; -export { + hasLegacyDgxStationQualificationAuthority, preflightAuthoritativeOnboardRuntime, preflightRebuildTargetRuntime, } from "./rebuild-target-runtime"; diff --git a/src/lib/actions/sandbox/rebuild-target-runtime.test.ts b/src/lib/actions/sandbox/rebuild-target-runtime.test.ts index e03cc004781..816421f2da1 100644 --- a/src/lib/actions/sandbox/rebuild-target-runtime.test.ts +++ b/src/lib/actions/sandbox/rebuild-target-runtime.test.ts @@ -57,6 +57,7 @@ import type { RebuildRecreateOnboardOpts } from "./rebuild-gpu-opt-out"; import type { RebuildResumeConfig } from "./rebuild-resume-config"; import type { RebuildTargetConfig } from "./rebuild-target-config"; import { + hasLegacyDgxStationQualificationAuthority, preflightAuthoritativeOnboardRuntime, preflightRebuildTargetRuntime, } from "./rebuild-target-runtime"; @@ -195,6 +196,43 @@ describe("preflightRebuildTargetRuntime GPU route", () => { }); }); +describe("legacy DGX Station rebuild authority", () => { + it.each([ + ["v0.0.83", true], + ["0.0.96-12-gabcdef0", true], + ["v0.0.97", false], + ["0.0.97-1-gabcdef0", false], + ["0.0.83-preview", false], + ["0.0.x", false], + ["", false], + ])("accepts only a valid release older than v0.0.97: %s", (nemoclawVersion, expected) => { + expect( + hasLegacyDgxStationQualificationAuthority({ + agent: "hermes", + fromDockerfile: null, + nemoclawVersion, + }), + ).toBe(expected); + }); + + it("rejects unrelated sandbox state", () => { + expect( + hasLegacyDgxStationQualificationAuthority({ + agent: "openclaw", + fromDockerfile: null, + nemoclawVersion: "v0.0.83", + }), + ).toBe(false); + expect( + hasLegacyDgxStationQualificationAuthority({ + agent: "hermes", + fromDockerfile: "/tmp/Dockerfile", + nemoclawVersion: "v0.0.83", + }), + ).toBe(false); + }); +}); + describe("authoritative rebuild readiness", () => { it("passes recorded managed-vLLM intent to the pre-delete readiness gate (#9292)", async () => { const authority = { checkpoint: "gateway-authority" }; @@ -202,6 +240,7 @@ describe("authoritative rebuild readiness", () => { const recreateOptions = { ...RECREATE_OPTIONS, allowDeferredN1xManagedVllm: true, + allowLegacyDgxStationQualification: true, } as RebuildRecreateOnboardOpts; const bail = vi.fn((message: string): never => { throw new Error(message); @@ -219,6 +258,7 @@ describe("authoritative rebuild readiness", () => { expect(mocks.preflightAuthoritativeRebuildTarget).toHaveBeenCalledWith( expect.objectContaining({ allowDeferredN1xManagedVllm: true, + allowLegacyDgxStationQualification: true, provider: "vllm-local", model: "test-model", sandboxName: "alpha", diff --git a/src/lib/actions/sandbox/rebuild-target-runtime.ts b/src/lib/actions/sandbox/rebuild-target-runtime.ts index 4c3e5cdb6b1..d266ea17893 100644 --- a/src/lib/actions/sandbox/rebuild-target-runtime.ts +++ b/src/lib/actions/sandbox/rebuild-target-runtime.ts @@ -106,6 +106,24 @@ async function preflightRebuildWebSearchCredential( } } +/** + * The DGX Station qualification projection was introduced in v0.0.97. A Hermes + * sandbox stamped by an earlier managed release may be rebuilt once without + * re-litigating that later admission rule. This is deliberately a strict + * release parser: untrusted version-shaped text is not recovery authority. + */ +export function hasLegacyDgxStationQualificationAuthority( + sandbox: Pick, +): boolean { + if (sandbox.agent !== "hermes" || sandbox.fromDockerfile != null) return false; + const match = /^(?:v)?0\.0\.(\d+)(?:-[1-9]\d*-g[0-9a-f]{7,40})?$/i.exec( + sandbox.nemoclawVersion ?? "", + ); + if (!match) return false; + const patch = Number(match[1]); + return Number.isSafeInteger(patch) && patch < 97; +} + export type RebuildTargetRuntimePreflightResult = | { ok: true; diff --git a/src/lib/actions/sandbox/snapshot.test.ts b/src/lib/actions/sandbox/snapshot.test.ts index 4807e013afc..1d204783f0d 100644 --- a/src/lib/actions/sandbox/snapshot.test.ts +++ b/src/lib/actions/sandbox/snapshot.test.ts @@ -513,9 +513,14 @@ describe("runSandboxSnapshot", () => { ([args]) => args[0] === "sandbox" && args[1] === "exec", ), ).toBe(false); - expect(f.backupSandboxStateMock).toHaveBeenCalledWith("alpha", { - name: null, - }); + expect(f.backupSandboxStateMock).toHaveBeenCalledWith( + "alpha", + expect.objectContaining({ + name: null, + captureStateFile: expect.any(Function), + captureStateDirectories: expect.any(Function), + }), + ); expect(consoleLog.mock.calls.flat().join("\n")).toContain("Snapshot v3 created"); }); diff --git a/src/lib/actions/sandbox/snapshot/backup-authority-script.test.ts b/src/lib/actions/sandbox/snapshot/backup-authority-script.test.ts index 903ad029a91..1b250fbab4f 100644 --- a/src/lib/actions/sandbox/snapshot/backup-authority-script.test.ts +++ b/src/lib/actions/sandbox/snapshot/backup-authority-script.test.ts @@ -8,7 +8,11 @@ import { spawnSync } from "node:child_process"; import { afterEach, describe, expect, it } from "vitest"; -import { OPENCLAW_CONFIG_CAPTURE_SCRIPT } from "./backup-authority"; +import { + HERMES_DIRECTORY_CAPTURE_SCRIPT, + HERMES_STATE_CAPTURE_SCRIPT, + OPENCLAW_CONFIG_CAPTURE_SCRIPT, +} from "./backup-authority"; const CONFIG_NAME = "openclaw.json"; const MAX_CONFIG_BYTES = 16 * 1024 * 1024; @@ -67,6 +71,75 @@ afterEach(() => { } }); +describe("Hermes privileged state capture scripts", () => { + it("captures a regular file while rejecting unsafe file metadata", () => { + const directory = fixtureDirectory(); + fs.writeFileSync(path.join(directory, "SOUL.md"), "soul"); + const copied = spawnSync( + "/usr/bin/python3", + ["-I", "-S", "-c", HERMES_STATE_CAPTURE_SCRIPT, directory, "SOUL.md", "copy"], + { encoding: null }, + ); + expect(copied.status).toBe(0); + expect(copied.stdout).toEqual(Buffer.from("soul")); + fs.symlinkSync(path.join(directory, "SOUL.md"), path.join(directory, "unsafe")); + const unsafe = spawnSync( + "/usr/bin/python3", + ["-I", "-S", "-c", HERMES_STATE_CAPTURE_SCRIPT, directory, "unsafe", "copy"], + { encoding: null }, + ); + expect(unsafe.status).not.toBe(0); + expect(unsafe.stdout).toEqual(Buffer.alloc(0)); + }); + + it("uses SQLite backup with a valid database", () => { + const directory = fixtureDirectory(); + const database = path.join(directory, "state.db"); + expect( + spawnSync("/usr/bin/python3", [ + "-c", + `import sqlite3; db = sqlite3.connect(${JSON.stringify(database)}); db.execute('create table state (value text)'); db.execute(\"insert into state values ('saved')\"); db.commit()`, + ]).status, + ).toBe(0); + const captured = spawnSync( + "/usr/bin/python3", + ["-I", "-S", "-c", HERMES_STATE_CAPTURE_SCRIPT, directory, "state.db", "sqlite_backup"], + { encoding: null }, + ); + expect(captured.status).toBe(0); + const restored = path.join(directory, "restored.db"); + fs.writeFileSync(restored, captured.stdout); + expect( + spawnSync("/usr/bin/python3", [ + "-c", + `import sqlite3; assert sqlite3.connect(${JSON.stringify(restored)}).execute('select value from state').fetchone() == ('saved',)`, + ]).status, + ).toBe(0); + }); + + it("rejects unsafe directory entries before streaming a tar archive", () => { + const directory = fixtureDirectory(); + const workspace = path.join(directory, "workspace"); + fs.mkdirSync(workspace); + fs.writeFileSync(path.join(workspace, "marker"), "state"); + const captured = spawnSync( + "/usr/bin/python3", + ["-I", "-S", "-c", HERMES_DIRECTORY_CAPTURE_SCRIPT, directory, "workspace"], + { encoding: null }, + ); + expect(captured.status).toBe(0); + expect(spawnSync("tar", ["-tf", "-"], { input: captured.stdout }).status).toBe(0); + fs.symlinkSync(path.join(workspace, "marker"), path.join(workspace, "unsafe")); + const unsafe = spawnSync( + "/usr/bin/python3", + ["-I", "-S", "-c", HERMES_DIRECTORY_CAPTURE_SCRIPT, directory, "workspace"], + { encoding: null }, + ); + expect(unsafe.status).not.toBe(0); + expect(unsafe.stdout).toEqual(Buffer.alloc(0)); + }); +}); + describe("OpenClaw privileged config capture script", () => { it("returns bytes only for a stable regular file", () => { const directory = fixtureDirectory(); diff --git a/src/lib/actions/sandbox/snapshot/backup-authority.test.ts b/src/lib/actions/sandbox/snapshot/backup-authority.test.ts index b6ace737f39..b7353e07d72 100644 --- a/src/lib/actions/sandbox/snapshot/backup-authority.test.ts +++ b/src/lib/actions/sandbox/snapshot/backup-authority.test.ts @@ -37,6 +37,8 @@ import { createSandboxHostLocalInferenceProvenance } from "../../../state/regist import type { BackupOptions, BackupResult } from "../../../state/sandbox"; import { backupSandboxStateWithManagedAuthority, + captureHermesStateDirectories, + captureHermesStateFile, captureOpenClawStateFile, } from "./backup-authority"; @@ -347,6 +349,100 @@ describe("managed snapshot backup authority", () => { expect(privilegedCaptureMocks.dockerSpawnSync).not.toHaveBeenCalled(); }); + it("captures declared Hermes files and rejects arbitrary paths", () => { + privilegedCaptureMocks.dockerSpawnSync.mockReturnValue({ + status: 0, + signal: null, + error: undefined, + stdout: Buffer.from("state"), + stderr: Buffer.alloc(0), + } as never); + expect( + captureHermesStateFile("alpha", { + sandboxName: "alpha", + dir: "/sandbox/.hermes", + spec: { path: "SOUL.md", strategy: "copy" }, + }), + ).toEqual({ outcome: "backed_up", data: Buffer.from("state") }); + expect( + captureHermesStateFile("alpha", { + sandboxName: "alpha", + dir: "/sandbox/.hermes", + spec: { path: "credentials/token", strategy: "copy" }, + }), + ).toBeNull(); + }); + + it.each([ + [2, { outcome: "missing" }], + [1, { outcome: "failed", error: "privileged Hermes state capture failed: exit 1" }], + ] as const)("propagates Hermes state capture exit %i without publishing bytes", (status, expected) => { + privilegedCaptureMocks.dockerSpawnSync.mockReturnValue({ + status, + signal: null, + error: undefined, + stdout: Buffer.alloc(0), + stderr: Buffer.alloc(0), + } as never); + + expect( + captureHermesStateFile("alpha", { + sandboxName: "alpha", + dir: "/sandbox/.hermes", + spec: { path: "SOUL.md", strategy: "copy" }, + }), + ).toEqual(expected); + }); + + it("streams only declared Hermes directories to the state-owned archive fd", () => { + privilegedCaptureMocks.dockerSpawnSync.mockReturnValue({ + status: 0, + signal: null, + error: undefined, + stdout: null, + stderr: Buffer.alloc(0), + } as never); + expect( + captureHermesStateDirectories( + "alpha", + { sandboxName: "alpha", dir: "/sandbox/.hermes", dirs: ["workspace"] }, + 42, + ), + ).toEqual({ outcome: "backed_up" }); + expect(privilegedCaptureMocks.dockerSpawnSync).toHaveBeenLastCalledWith( + expect.any(Array), + expect.objectContaining({ stdio: ["ignore", 42, "pipe"] }), + ); + expect( + captureHermesStateDirectories( + "alpha", + { sandboxName: "other", dir: "/sandbox/.hermes", dirs: ["workspace"] }, + 42, + ), + ).toBeNull(); + expect( + captureHermesStateDirectories( + "alpha", + { sandboxName: "alpha", dir: "/sandbox/.hermes", dirs: ["../outside"] }, + 42, + ), + ).toBeNull(); + }); + + it("does not execute privileged capture when a normal Hermes backup succeeds", () => { + const backup = vi.fn((_name: string, options: BackupOptions = {}) => successfulBackup(options)); + const result = backupSandboxStateWithManagedAuthority( + "alpha", + {}, + { + getSandbox: () => ({ name: "alpha", agent: "hermes" }) as SandboxEntry, + backup, + }, + ); + expect(result.success).toBe(true); + expect(privilegedCaptureMocks.dockerSpawnSync).not.toHaveBeenCalled(); + }); + it.each(["openclaw", "hermes", "langchain-deepagents-code"] as const)( "captures and republishes exact %s provider authority", (agent) => { diff --git a/src/lib/actions/sandbox/snapshot/backup-authority.ts b/src/lib/actions/sandbox/snapshot/backup-authority.ts index 620e32a7a79..90144d77e34 100644 --- a/src/lib/actions/sandbox/snapshot/backup-authority.ts +++ b/src/lib/actions/sandbox/snapshot/backup-authority.ts @@ -38,6 +38,8 @@ interface SnapshotBackupAuthorityDependencies { readonly confirmHostLocalInference: typeof confirmHostLocalInferenceAuthority; readonly backup: typeof sandboxState.backupSandboxState; readonly captureOpenClawStateFile: typeof captureOpenClawStateFile; + readonly captureHermesStateFile: typeof captureHermesStateFile; + readonly captureHermesStateDirectories: typeof captureHermesStateDirectories; } const MAX_OPENCLAW_CONFIG_BYTES = 16 * 1024 * 1024; @@ -48,6 +50,9 @@ const OPENCLAW_CONFIG_CAPTURE_PROTOCOL_MAX_BYTES = 128; const OPENCLAW_CONFIG_CAPTURE_DIAGNOSTIC_MAX_BYTES = 1024; const OPENCLAW_CONFIG_DIRECTORY = "/sandbox/.openclaw"; const OPENCLAW_CONFIG_NAME = "openclaw.json"; +const HERMES_CONFIG_DIRECTORY = "/sandbox/.hermes"; +const HERMES_CAPTURE_TIMEOUT_MS = 120_000; +const HERMES_CAPTURE_MAX_BUFFER = 17 * 1024 * 1024; export const OPENCLAW_CONFIG_CAPTURE_SCRIPT = `import os, stat, sys maximum = ${MAX_OPENCLAW_CONFIG_BYTES} directory = sys.argv[1] @@ -225,6 +230,166 @@ export function captureOpenClawStateFile( } } +export const HERMES_STATE_CAPTURE_SCRIPT = `import os, sqlite3, stat, sys, tempfile +base, relative, strategy = sys.argv[1:] +if not relative or relative.startswith("/") or ".." in relative.split("/"): + raise SystemExit(10) +path = os.path.join(base, relative) +try: + before = os.lstat(path) +except FileNotFoundError: + raise SystemExit(2) +if not stat.S_ISREG(before.st_mode) or stat.S_ISLNK(before.st_mode) or before.st_nlink != 1: + raise SystemExit(11) +if strategy == "sqlite_backup": + target = tempfile.NamedTemporaryFile(dir="/tmp", delete=False) + target.close() + try: + source = sqlite3.connect("file:" + path + "?mode=ro", uri=True, timeout=30) + destination = sqlite3.connect(target.name, timeout=30) + try: + source.backup(destination) + if destination.execute("PRAGMA quick_check").fetchone()[0] != "ok": + raise SystemExit(12) + finally: + destination.close() + source.close() + with open(target.name, "rb", buffering=0) as stream: + while chunk := stream.read(64 * 1024): + sys.stdout.buffer.write(chunk) + finally: + os.unlink(target.name) +else: + with open(path, "rb", buffering=0) as stream: + while chunk := stream.read(64 * 1024): + sys.stdout.buffer.write(chunk) +after = os.lstat(path) +if (before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns, before.st_ctime_ns, before.st_nlink) != (after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns, after.st_ctime_ns, after.st_nlink): + raise SystemExit(13) +`; + +export const HERMES_DIRECTORY_CAPTURE_SCRIPT = `import os, stat, subprocess, sys +base, *names = sys.argv[1:] +def audit(path): + entry = os.lstat(path) + if stat.S_ISLNK(entry.st_mode) or not stat.S_ISDIR(entry.st_mode): + raise SystemExit(11) + with os.scandir(path) as entries: + for child in entries: + value = child.stat(follow_symlinks=False) + if stat.S_ISLNK(value.st_mode) or not (stat.S_ISREG(value.st_mode) or stat.S_ISDIR(value.st_mode)): + raise SystemExit(11) + if stat.S_ISDIR(value.st_mode): + audit(child.path) +for name in names: + if not name or "/" in name or name in (".", ".."): + raise SystemExit(10) + audit(os.path.join(base, name)) +result = subprocess.run(["/usr/bin/tar", "--hard-dereference", "-cf", "-", "-C", base, "--", *names], stdout=sys.stdout.buffer) +raise SystemExit(result.returncode) +`; + +export function captureHermesStateFile( + sandboxName: string, + request: sandboxState.StateFileCaptureRequest, +): sandboxState.StateFileCaptureResult | null { + if ( + request.sandboxName !== sandboxName || + !sandboxState.isDeclaredAgentStateFile("hermes", request.dir, request.spec) + ) + return null; + try { + return withPrivilegedSandboxExecutionLease(sandboxName, "Hermes state snapshot capture", () => { + const result = dockerSpawnSync( + privilegedSandboxExecArgv( + sandboxName, + [ + "/usr/bin/python3", + "-I", + "-S", + "-c", + HERMES_STATE_CAPTURE_SCRIPT, + HERMES_CONFIG_DIRECTORY, + request.spec.path, + request.spec.strategy, + ], + false, + true, + ), + { + encoding: null, + stdio: ["ignore", "pipe", "pipe"], + timeout: HERMES_CAPTURE_TIMEOUT_MS, + maxBuffer: HERMES_CAPTURE_MAX_BUFFER, + }, + ); + if (result.status === 2 && !result.error && result.signal === null) + return { outcome: "missing" }; + if (result.status !== 0 || result.error || result.signal || !Buffer.isBuffer(result.stdout)) { + return { + outcome: "failed", + error: `privileged Hermes state capture failed: ${result.error?.message ?? (result.signal ? `signal ${result.signal}` : `exit ${String(result.status)}`)}`, + }; + } + return { outcome: "backed_up", data: result.stdout }; + }); + } catch (error) { + return { outcome: "failed", error: error instanceof Error ? error.message : String(error) }; + } +} + +export function captureHermesStateDirectories( + sandboxName: string, + request: sandboxState.StateDirectoryCaptureRequest, + archiveFd: number, +): sandboxState.StateDirectoryCaptureResult | null { + if ( + request.sandboxName !== sandboxName || + !sandboxState.areDeclaredAgentStateDirectories("hermes", request.dir, request.dirs) + ) { + return null; + } + try { + return withPrivilegedSandboxExecutionLease( + sandboxName, + "Hermes state directory snapshot capture", + () => { + const result = dockerSpawnSync( + privilegedSandboxExecArgv( + sandboxName, + [ + "/usr/bin/python3", + "-I", + "-S", + "-c", + HERMES_DIRECTORY_CAPTURE_SCRIPT, + HERMES_CONFIG_DIRECTORY, + ...request.dirs, + ], + false, + true, + ), + { + encoding: null, + stdio: ["ignore", archiveFd, "pipe"], + timeout: HERMES_CAPTURE_TIMEOUT_MS, + maxBuffer: 1024 * 1024, + }, + ); + if (result.status !== 0 || result.error || result.signal) { + return { + outcome: "failed", + error: `privileged Hermes directory capture failed: ${result.error?.message ?? (result.signal ? `signal ${result.signal}` : `exit ${String(result.status)}`)}`, + }; + } + return { outcome: "backed_up" }; + }, + ); + } catch (error) { + return { outcome: "failed", error: error instanceof Error ? error.message : String(error) }; + } +} + const defaultDependencies: Omit = { requireProvider: (sandbox) => requireRuntimeProviderBundleForSandbox(sandbox, CURRENT_RUNTIME_PROVIDER_BUNDLES), @@ -235,6 +400,8 @@ const defaultDependencies: Omit sandboxState.backupSandboxState(...args), captureOpenClawStateFile, + captureHermesStateFile, + captureHermesStateDirectories, }; function failure(error: unknown): sandboxState.BackupResult { @@ -252,9 +419,14 @@ function failure(error: unknown): sandboxState.BackupResult { function backupStateOnly( dependencies: SnapshotBackupAuthorityDependencies, sandboxName: string, - options: Pick, + options: Pick< + sandboxState.BackupOptions, + "name" | "captureStateFile" | "captureStateDirectories" + >, ): sandboxState.BackupResult { - return options.name === undefined && options.captureStateFile === undefined + return options.name === undefined && + options.captureStateFile === undefined && + options.captureStateDirectories === undefined ? dependencies.backup(sandboxName) : dependencies.backup(sandboxName, options); } @@ -394,14 +566,24 @@ export function backupSandboxStateWithManagedAuthority( const entry = dependencies.getSandbox(sandboxName); if (!entry) return backupStateOnly(dependencies, sandboxName, options); - const stateFileOptions: Pick = + const stateCaptureOptions: Pick< + sandboxState.BackupOptions, + "captureStateFile" | "captureStateDirectories" + > = !entry.agent || entry.agent === "openclaw" ? { captureStateFile: (request) => dependencies.captureOpenClawStateFile(sandboxName, request), } - : {}; - const backupOptions = { ...options, ...stateFileOptions }; + : entry.agent === "hermes" + ? { + captureStateFile: (request) => + dependencies.captureHermesStateFile(sandboxName, request), + captureStateDirectories: (request, archiveFd) => + dependencies.captureHermesStateDirectories(sandboxName, request, archiveFd), + } + : {}; + const backupOptions = { ...options, ...stateCaptureOptions }; let authority: SnapshotBackupAuthority | null; try { diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 99101728417..fb0132212d3 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -2727,7 +2727,6 @@ async function preflightAuthoritativeRebuildTarget( } } -// ── Main ───────────────────────────────────────────────────────── const wrappedOnboard = onboardEntryOptions.wrapOnboard(runOnboard, onboardSession); const onboard = onboardSessionBootstrap.wrapOnboardDeferredExit(wrappedOnboard); async function runOnboard(opts: OnboardOptions = {}): Promise { @@ -3003,6 +3002,7 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { gpuRequested: opts.gpu === true, noGpu: opts.noGpu === true, allowDeferredN1xManagedVllm: opts.allowDeferredN1xManagedVllm, + allowLegacyDgxStationQualification: opts.allowLegacyDgxStationQualification, env: process.env, recordedGpuPassthroughBeforePreflight, commitSelectedAgentTransition: selectedAgentTransition.commit, diff --git a/src/lib/onboard/authoritative-rebuild-target.test.ts b/src/lib/onboard/authoritative-rebuild-target.test.ts index 34cd3ab28c8..d4543206859 100644 --- a/src/lib/onboard/authoritative-rebuild-target.test.ts +++ b/src/lib/onboard/authoritative-rebuild-target.test.ts @@ -53,7 +53,7 @@ describe("authoritative rebuild sandbox flow options", () => { }); describe("authoritative rebuild runtime preflight options", () => { - it("carries only target GPU state and recorded N1x preview intent (#9292)", () => { + it("carries target GPU state and recorded rebuild readiness authority (#9292)", () => { const options = { authoritativeResumeConfig: true, sandboxName: "alpha", @@ -66,6 +66,7 @@ describe("authoritative rebuild runtime preflight options", () => { sandboxGpuDevice: "nvidia.com/gpu=all", noGpu: false, allowDeferredN1xManagedVllm: true, + allowLegacyDgxStationQualification: true, } satisfies AuthoritativeRebuildPreflightOptions; expect(authoritativeRebuildRuntimePreflightOptions(options)).toEqual({ @@ -73,14 +74,20 @@ describe("authoritative rebuild runtime preflight options", () => { sandboxGpuDevice: "nvidia.com/gpu=all", noGpu: false, allowDeferredN1xManagedVllm: true, + allowLegacyDgxStationQualification: true, }); - const { allowDeferredN1xManagedVllm: _recordedIntent, ...withoutRecordedIntent } = options; + const { + allowDeferredN1xManagedVllm: _recordedIntent, + allowLegacyDgxStationQualification: _legacyStationAuthority, + ...withoutRecordedIntent + } = options; expect(authoritativeRebuildRuntimePreflightOptions(withoutRecordedIntent)).toEqual({ sandboxGpu: "enable", sandboxGpuDevice: "nvidia.com/gpu=all", noGpu: false, allowDeferredN1xManagedVllm: false, + allowLegacyDgxStationQualification: false, }); }); }); diff --git a/src/lib/onboard/authoritative-rebuild-target.ts b/src/lib/onboard/authoritative-rebuild-target.ts index cb6bcb6d5f4..9102d2020f0 100644 --- a/src/lib/onboard/authoritative-rebuild-target.ts +++ b/src/lib/onboard/authoritative-rebuild-target.ts @@ -39,7 +39,12 @@ export type AuthoritativeGatewayOptions = Pick< export type AuthoritativeRebuildPreflightOptions = Pick< OnboardOptions, - "sandboxGpu" | "sandboxGpuDevice" | "noGpu" | "controlUiPort" | "allowDeferredN1xManagedVllm" + | "sandboxGpu" + | "sandboxGpuDevice" + | "noGpu" + | "controlUiPort" + | "allowDeferredN1xManagedVllm" + | "allowLegacyDgxStationQualification" > & { authoritativeResumeConfig: true; /** Internal prepared-backup recovery defers route repair to authoritative onboard. */ @@ -56,12 +61,14 @@ export function authoritativeRebuildRuntimePreflightOptions( opts: AuthoritativeRebuildPreflightOptions, ): Pick & { allowDeferredN1xManagedVllm: boolean; + allowLegacyDgxStationQualification: boolean; } { return { sandboxGpu: opts.sandboxGpu, sandboxGpuDevice: opts.sandboxGpuDevice, noGpu: opts.noGpu, allowDeferredN1xManagedVllm: opts.allowDeferredN1xManagedVllm === true, + allowLegacyDgxStationQualification: opts.allowLegacyDgxStationQualification === true, }; } diff --git a/src/lib/onboard/fatal-runtime-preflight.ts b/src/lib/onboard/fatal-runtime-preflight.ts index 204a0b83e43..787976e5ac0 100644 --- a/src/lib/onboard/fatal-runtime-preflight.ts +++ b/src/lib/onboard/fatal-runtime-preflight.ts @@ -53,6 +53,8 @@ export type FatalRuntimePreflightOptions = Pick< > & { /** Explicit false prevents ambient provider intent from crossing a rebuild boundary. */ allowDeferredN1xManagedVllm?: boolean; + /** Verified legacy rebuild authority; never inferred from ambient process state. */ + allowLegacyDgxStationQualification?: boolean; optedOutGpuPassthrough?: boolean; }; @@ -119,6 +121,7 @@ export interface OnboardHostReadinessOptions { allowStorageRemediation?: boolean; allowPortableHostPreparation?: boolean; allowDeferredN1xManagedVllm?: boolean; + allowLegacyDgxStationQualification?: boolean; /** Print warning-severity host advisories before returning an admitted report. */ presentAdvisories?: boolean; exitProcess?: (code: number) => never; @@ -171,6 +174,7 @@ export function assertOnboardSystemReadiness( allowDeferredN1xManagedVllm: options.allowDeferredN1xManagedVllm ?? process.env.NEMOCLAW_PROVIDER === MANAGED_VLLM_PROVIDER_KEY, + allowLegacyDgxStationQualification: options.allowLegacyDgxStationQualification === true, }); const advisories = planHostAdvisories(host, { resuming: options.resuming }); if (admission.admitted) { @@ -300,6 +304,7 @@ function collectOnboardHostReadiness( resuming: context.resuming, allowStorageRemediation, allowDeferredN1xManagedVllm: options.allowDeferredN1xManagedVllm, + allowLegacyDgxStationQualification: options.allowLegacyDgxStationQualification, // The initial host readiness gate already presented warning advisories. presentAdvisories: false, exitProcess: context.exitProcess, @@ -389,6 +394,7 @@ async function collectAdmittedReadinessPair( resuming: context.resuming, allowStorageRemediation: isManagedGatewayReadiness(gateway), allowDeferredN1xManagedVllm: options.allowDeferredN1xManagedVllm, + allowLegacyDgxStationQualification: options.allowLegacyDgxStationQualification, presentAdvisories: false, exitProcess, }); @@ -560,6 +566,7 @@ export function runFatalOnboardRuntimePreflight( resuming: context.resuming, allowStorageRemediation: context.allowStorageRemediation, allowDeferredN1xManagedVllm: options.allowDeferredN1xManagedVllm, + allowLegacyDgxStationQualification: options.allowLegacyDgxStationQualification, exitProcess, observedAt, now, diff --git a/src/lib/onboard/machine/handlers/preflight.ts b/src/lib/onboard/machine/handlers/preflight.ts index b8066802641..9c9e5a87798 100644 --- a/src/lib/onboard/machine/handlers/preflight.ts +++ b/src/lib/onboard/machine/handlers/preflight.ts @@ -35,6 +35,7 @@ export interface PreflightStateOptions< gpuRequested: boolean; noGpu: boolean; allowDeferredN1xManagedVllm?: boolean; + allowLegacyDgxStationQualification?: boolean; env: NodeJS.ProcessEnv; deps: { getSandbox(name: string): SandboxEntry | null; @@ -57,6 +58,7 @@ export interface PreflightStateOptions< now?: () => Date; wslDockerDesktopGpuProofPassed?: boolean; allowDeferredN1xManagedVllm?: boolean; + allowLegacyDgxStationQualification?: boolean; resuming: true; presentAdvisories?: boolean; }, @@ -133,6 +135,7 @@ export async function handlePreflightState< gpuRequested, noGpu, allowDeferredN1xManagedVllm, + allowLegacyDgxStationQualification, env, deps, }: PreflightStateOptions): Promise< @@ -180,6 +183,7 @@ export async function handlePreflightState< observedAt: hostObservedAt, now, allowDeferredN1xManagedVllm, + allowLegacyDgxStationQualification, resuming: true, }); // A full detector can run the bounded ARM64 WSL Docker GPU proof. Keep it @@ -203,6 +207,7 @@ export async function handlePreflightState< now, ...(wslDockerDesktopGpuProofPassed === undefined ? {} : { wslDockerDesktopGpuProofPassed }), allowDeferredN1xManagedVllm, + allowLegacyDgxStationQualification, resuming: true, presentAdvisories: false, }); diff --git a/src/lib/onboard/machine/initial-flow-phases.ts b/src/lib/onboard/machine/initial-flow-phases.ts index aa048cde0b5..fe76135f584 100644 --- a/src/lib/onboard/machine/initial-flow-phases.ts +++ b/src/lib/onboard/machine/initial-flow-phases.ts @@ -54,6 +54,7 @@ export interface InitialOnboardFlowPhaseOptions< gpuRequested: boolean; noGpu: boolean; allowDeferredN1xManagedVllm?: boolean; + allowLegacyDgxStationQualification?: boolean; env: NodeJS.ProcessEnv; platform?: NodeJS.Platform; recordedGpuPassthroughBeforePreflight: boolean; @@ -150,6 +151,7 @@ export function createInitialOnboardFlowPhases< gpuRequested: options.gpuRequested, noGpu: options.noGpu, allowDeferredN1xManagedVllm: options.allowDeferredN1xManagedVllm, + allowLegacyDgxStationQualification: options.allowLegacyDgxStationQualification, env: options.env, deps: { ...options.preflightDeps, diff --git a/src/lib/onboard/types.ts b/src/lib/onboard/types.ts index c9976a80a11..313c3be3466 100644 --- a/src/lib/onboard/types.ts +++ b/src/lib/onboard/types.ts @@ -169,6 +169,8 @@ export type OnboardOptions = { providerRecoveryReceipt?: import("./rebuild-route-handoff").ProviderRecoveryReceipt; /** Internal rebuild handoff for a recorded managed-vLLM N1x preview selection. */ allowDeferredN1xManagedVllm?: true; + /** Internal legacy Hermes rebuild handoff for the pre-v0.0.97 Station admission rule. */ + allowLegacyDgxStationQualification?: true; /** Internal one-shot handoff for the exact image context validated before rebuild deletion. */ preparedImageRebuild?: import("./prepared-dcode-rebuild").PreparedImageRebuildHandoff; /** Internal immutable managed-image/profile handoff validated before rebuild deletion. */ diff --git a/src/lib/readiness/onboard-admission.test.ts b/src/lib/readiness/onboard-admission.test.ts index 068e9d76f92..80982e6a62e 100644 --- a/src/lib/readiness/onboard-admission.test.ts +++ b/src/lib/readiness/onboard-admission.test.ts @@ -107,6 +107,40 @@ describe("onboarding readiness admission (#7411)", () => { ).toEqual({ admitted: true, waivedFindingIds: [] }); }); + it("waives only the recorded legacy DGX Station finding during rebuild", () => { + const stationCapabilities = [ + ...withCapabilityState( + requiredCapabilities(), + ONBOARD_REQUIRED_CAPABILITY_IDS.platformSupported, + "absent", + ), + capability("host.platform.dgx_station", "absent"), + ]; + const stationFinding = finding("host.platform.dgx_station_unqualified"); + + expect( + evaluateOnboardReadinessAdmission( + report({ capabilities: stationCapabilities, findings: [stationFinding] }), + { ...DEFAULT_OPTIONS, allowLegacyDgxStationQualification: true }, + ), + ).toEqual({ admitted: true, waivedFindingIds: [stationFinding.id] }); + expect( + evaluateOnboardReadinessAdmission( + report({ capabilities: stationCapabilities, findings: [stationFinding] }), + DEFAULT_OPTIONS, + ), + ).toMatchObject({ admitted: false, findingIds: [stationFinding.id] }); + expect( + evaluateOnboardReadinessAdmission( + report({ + capabilities: stationCapabilities, + findings: [stationFinding, finding("host.example.blocked")], + }), + { ...DEFAULT_OPTIONS, allowLegacyDgxStationQualification: true }, + ), + ).toMatchObject({ admitted: false, findingIds: ["host.example.blocked"] }); + }); + it("fails on every unwaived blocking or fatal finding and retains report order", () => { const decision = evaluateOnboardReadinessAdmission( report({ @@ -235,13 +269,11 @@ describe("onboarding readiness admission (#7411)", () => { }); }); - it.each( - [ - ONBOARD_REQUIRED_CAPABILITY_IDS.dockerRuntimeSupported, - ONBOARD_REQUIRED_CAPABILITY_IDS.dockerStorageCompatible, - ONBOARD_REQUIRED_CAPABILITY_IDS.dockerStorageRemediationAvailable, - ], - )( + it.each([ + ONBOARD_REQUIRED_CAPABILITY_IDS.dockerRuntimeSupported, + ONBOARD_REQUIRED_CAPABILITY_IDS.dockerStorageCompatible, + ONBOARD_REQUIRED_CAPABILITY_IDS.dockerStorageRemediationAvailable, + ])( "admits only the pre-mutation facts that portable host preparation can replace [case %#]", (id) => { let capabilities = withCapabilityState( diff --git a/src/lib/readiness/onboard-admission.ts b/src/lib/readiness/onboard-admission.ts index d1d544c9341..c922a1b6374 100644 --- a/src/lib/readiness/onboard-admission.ts +++ b/src/lib/readiness/onboard-admission.ts @@ -51,6 +51,8 @@ export interface OnboardReadinessAdmissionOptions { allowPortableHostPreparation?: boolean; /** Explicit managed-vLLM intent may exercise the Deferred N1x validation path. */ allowDeferredN1xManagedVllm?: boolean; + /** A verified legacy Hermes rebuild may preserve its pre-v0.0.97 Station admission. */ + allowLegacyDgxStationQualification?: boolean; } export type OnboardReadinessAdmissionDecision = @@ -129,6 +131,13 @@ function canWaiveFinding( ) { return true; } + if ( + options.allowLegacyDgxStationQualification && + finding.id === "host.platform.dgx_station_unqualified" && + capabilityState(capabilities, "host.platform.dgx_station") === "absent" + ) { + return true; + } return ( options.allowStorageRemediation && finding.id === ONBOARD_READINESS_FINDING_IDS.storageIncompatible && diff --git a/src/lib/state/sandbox.ts b/src/lib/state/sandbox.ts index b5ab9b2ec07..ec7486d36b6 100644 --- a/src/lib/state/sandbox.ts +++ b/src/lib/state/sandbox.ts @@ -46,6 +46,7 @@ import { isObjectRecord, type UnknownRecord } from "../core/json-types.js"; import { GATEWAY_PORT } from "../core/ports.js"; import { BACKUP_FAILURE_ABSENT_AFTER_EXTRACTION, + BACKUP_FAILURE_PERMISSION_DENIED, classifyFailedDirsFromTarStderr, } from "../domain/backup-failure.js"; import { shellQuote } from "../runner.js"; @@ -184,6 +185,12 @@ export interface BackupOptions { * identity, and stable-read constraints before returning bytes. */ captureStateFile?: StateFileCapture; + /** + * Internal privileged retry for state directories that the restricted tar + * path classified as permission denied. The state layer owns the temporary + * archive fd and validates the returned archive before publishing it. + */ + captureStateDirectories?: StateDirectoryCapture; } export interface InstanceBackup { @@ -212,7 +219,21 @@ export type StateFileCaptureResult = | { outcome: "missing" } | { outcome: "failed"; error?: string; unreachable?: boolean }; +export interface StateDirectoryCaptureRequest { + sandboxName: string; + dir: string; + dirs: readonly string[]; +} + +export type StateDirectoryCaptureResult = + | { outcome: "backed_up" } + | { outcome: "failed"; error?: string; unreachable?: boolean }; + export type StateFileCapture = (request: StateFileCaptureRequest) => StateFileCaptureResult | null; +export type StateDirectoryCapture = ( + request: StateDirectoryCaptureRequest, + archiveFd: number, +) => StateDirectoryCaptureResult | null; export interface BackupResult { success: boolean; @@ -977,6 +998,37 @@ function normalizeStateFileSpecs( return normalized; } +/** Check privileged snapshot requests against the owning agent manifest. */ +export function isDeclaredAgentStateFile( + agentName: string, + dir: string, + spec: StateFileSpec, +): boolean { + const agent = loadAgent(agentName); + return ( + dir === agent.configPaths.dir && + ((agentName === "hermes" && spec.path === ".env" && spec.strategy === "copy") || + agent.stateFiles.some( + (entry) => entry.path === spec.path && entry.strategy === spec.strategy, + )) + ); +} + +/** Check privileged directory requests against the owning agent manifest. */ +export function areDeclaredAgentStateDirectories( + agentName: string, + dir: string, + names: readonly string[], +): boolean { + if (names.length === 0) return false; + const agent = loadAgent(agentName); + const allowed = new Set(agent.backupStateDirs); + return ( + dir === agent.configPaths.dir && + names.every((name) => allowed.has(name) && /^[A-Za-z0-9._-]+$/.test(name)) + ); +} + function stateFileRemotePath(dir: string, filePath: string): string { return `${dir.replace(/\/+$/, "")}/${filePath}`; } @@ -1005,6 +1057,7 @@ export function buildStateFileBackupCommand(dir: string, spec: StateFileSpec): s `src=${quotedRemotePath}`, '[ ! -e "$src" ] && exit 2', '[ -f "$src" ] && [ ! -L "$src" ] || { echo "unsafe sqlite state file: $src" >&2; exit 10; }', + '[ -r "$src" ] || { echo "permission denied: $src" >&2; exit 1; }', 'hardlink_count="$(find "$src" -maxdepth 0 -type f -links +1 -print 2>/dev/null | wc -l | tr -d " ")"', '[ "${hardlink_count:-0}" = "0" ] || { echo "hard-linked sqlite state file rejected: $src" >&2; exit 11; }', 'tmp="$(mktemp /tmp/nemoclaw-sqlite-backup.XXXXXX)"', @@ -1040,6 +1093,7 @@ function capturePreservedEnvFile( sandboxName: string, dir: string, inventory: PreservedEnvInventory, + captureFallback?: StateFileCapture, ): { outcome: StateFileBackupOutcome; file?: PreservedEnvFile; unreachable: boolean } { const command = buildStateFileBackupCommand(dir, { path: inventory.path, @@ -1052,16 +1106,48 @@ function capturePreservedEnvFile( maxBuffer: 1024 * 1024, }); if (result.status === 2) return { outcome: "missing", unreachable: false }; - if (result.status !== 0 || result.error || result.signal || !result.stdout) { + let captured: StateFileCaptureResult | null = null; + if ( + result.status === 1 && + !result.error && + !result.signal && + /permission denied/i.test(result.stderr?.toString() ?? "") && + captureFallback !== undefined + ) { + try { + captured = captureFallback({ + sandboxName, + dir, + spec: { path: inventory.path, strategy: "copy" }, + }); + } catch (error) { + captured = { + outcome: "failed", + error: error instanceof Error ? error.message : String(error), + }; + } + } + if (captured?.outcome === "missing") return { outcome: "missing", unreachable: false }; + const data = captured?.outcome === "backed_up" ? captured.data : null; + if ((result.status !== 0 || result.error || result.signal || !result.stdout) && data === null) { const detail = + (captured?.outcome === "failed" ? captured.error : undefined) || (result.stderr?.toString() || "").trim() || result.error?.message || (result.signal ? `signal ${result.signal}` : `exit ${String(result.status)}`); _log(`FAILED: preserved environment capture ${inventory.path}: ${detail.substring(0, 200)}`); - return { outcome: "failed", unreachable: isSshTransportFailure(result) }; + return { + outcome: "failed", + unreachable: + (captured?.outcome === "failed" && captured.unreachable === true) || + isSshTransportFailure(result), + }; } try { - const assignments = extractPreservedEnvAssignments(result.stdout.toString("utf8"), inventory); + const assignments = extractPreservedEnvAssignments( + (data ?? result.stdout).toString("utf8"), + inventory, + ); _log( `Captured ${assignments.length} preserved environment ${assignments.length === 1 ? "key" : "keys"} from ${inventory.path}`, ); @@ -1083,12 +1169,19 @@ function capturePreservedEnvFiles( sandboxName: string, dir: string, inventories: readonly PreservedEnvInventory[], + captureFallback?: StateFileCapture, ): { files: PreservedEnvFile[]; failedPaths: string[]; unreachable: boolean } { const files: PreservedEnvFile[] = []; const failedPaths: string[] = []; let unreachable = false; for (const inventory of inventories) { - const result = capturePreservedEnvFile(configFile, sandboxName, dir, inventory); + const result = capturePreservedEnvFile( + configFile, + sandboxName, + dir, + inventory, + captureFallback, + ); if (result.outcome === "backed_up" && result.file) { files.push(result.file); } else if (result.outcome === "failed") { @@ -1106,6 +1199,7 @@ function captureAgentPreservedEnvFiles( dir: string, manifest: RebuildManifest, failedFiles: string[], + captureFallback?: StateFileCapture, ): boolean { if (agentName !== "hermes") return false; const preserved = capturePreservedEnvFiles( @@ -1113,6 +1207,7 @@ function captureAgentPreservedEnvFiles( sandboxName, dir, HERMES_PRESERVED_ENV_INVENTORY, + captureFallback, ); manifest.preservedEnv = preserved.files; failedFiles.push(...preserved.failedPaths); @@ -1138,7 +1233,14 @@ function backupStateFile( if (result.status === 2) return { outcome: "missing", unreachable: false }; const emptySqliteBackup = spec.strategy === "sqlite_backup" && result.stdout?.length === 0; let captured: StateFileCaptureResult | null = null; - if (result.status === 1 && !result.error && !result.signal && captureFallback !== undefined) { + if ( + result.status === 1 && + !result.error && + !result.signal && + (dir === "/sandbox/.openclaw" || + /permission denied/i.test(result.stderr?.toString() ?? "")) && + captureFallback !== undefined + ) { try { captured = captureFallback({ sandboxName, dir, spec }); } catch (error) { @@ -1182,6 +1284,64 @@ function backupStateFile( return { outcome: "backed_up", unreachable: false }; } +function retryPermissionDeniedDirectories( + captureFallback: StateDirectoryCapture | undefined, + sandboxName: string, + dir: string, + backupPath: string, + failedDirs: string[], + backedUpDirs: string[], + failedDirReasons: Record, +): void { + if (!captureFallback) return; + const denied = failedDirs.filter( + (name) => failedDirReasons[name] === BACKUP_FAILURE_PERMISSION_DENIED, + ); + if (denied.length === 0) return; + let stagingDir: string | undefined; + let archivePath = ""; + let archiveFd: number | undefined; + try { + stagingDir = mkdtempSync(path.join(os.tmpdir(), "nemoclaw-state-privileged-")); + archivePath = path.join(stagingDir, "archive.tar"); + archiveFd = openSync(archivePath, "wx", 0o600); + const capture = captureFallback({ sandboxName, dir, dirs: denied }, archiveFd); + closeSync(archiveFd); + archiveFd = undefined; + if (capture?.outcome !== "backed_up" || statSync(archivePath).size === 0) { + _log( + `FAILED: privileged state directory capture: ${capture?.outcome === "failed" ? (capture.error ?? "failed") : "no archive"}`, + ); + return; + } + for (const name of denied) { + const target = path.join(backupPath, name); + rejectSymlinksOnPath(target); + rmSync(target, { recursive: true, force: true }); + } + const extracted = safeTarExtract({ filePath: archivePath }, backupPath); + if (!extracted.success) { + _log(`FAILED: privileged state directory capture: ${extracted.error}`); + return; + } + const recovered = new Set(existingBackupDirs(backupPath, denied)); + for (const name of denied) { + if (!recovered.has(name)) continue; + const index = failedDirs.indexOf(name); + if (index >= 0) failedDirs.splice(index, 1); + delete failedDirReasons[name]; + if (!backedUpDirs.includes(name)) backedUpDirs.push(name); + } + } catch (error) { + _log( + `FAILED: privileged state directory capture: ${error instanceof Error ? error.message : String(error)}`, + ); + } finally { + if (archiveFd !== undefined) closeSync(archiveFd); + if (stagingDir) rmSync(stagingDir, { recursive: true, force: true }); + } +} + // ── Backup ───────────────────────────────────────────────────────── /** @@ -1808,11 +1968,29 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions = failedDirs.push(...existingDirs); } } else { - failedDirs.push(...existingDirs); + const tarFailedDirs = classifyFailedDirsFromTarStderr( + result.stderr?.toString() || "", + existingDirs, + ); + for (const name of existingDirs) { + failedDirs.push(name); + const reason = tarFailedDirs.get(name); + if (reason !== undefined) failedDirReasons[name] = reason; + } } } } + retryPermissionDeniedDirectories( + options.captureStateDirectories, + sandboxName, + dir, + backupPath, + failedDirs, + backedUpDirs, + failedDirReasons, + ); + for (const spec of stateFiles) { const result = backupStateFile( configFile, @@ -1841,6 +2019,7 @@ export function backupSandboxState(sandboxName: string, options: BackupOptions = dir, manifest, failedFiles, + options.captureStateFile, ) || unreachable; } finally { try { diff --git a/test/agents/hermes/hermes-kanban-snapshot.test.ts b/test/agents/hermes/hermes-kanban-snapshot.test.ts index 292a5c4cd6a..ffb6e89795a 100644 --- a/test/agents/hermes/hermes-kanban-snapshot.test.ts +++ b/test/agents/hermes/hermes-kanban-snapshot.test.ts @@ -165,6 +165,28 @@ it("fails the SQLite state backup when the online backup command fails (#7095)", } }); +it("classifies an unreadable Hermes SQLite file before opening the database (#10375)", () => { + const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-sqlite-backup-denied-")); + try { + const sourceDir = path.join(fixture, "state"); + const sourceFile = path.join(sourceDir, "kanban.db"); + fs.mkdirSync(sourceDir, { recursive: true }); + fs.writeFileSync(sourceFile, "source database\n", { mode: 0o000 }); + + const command = sandboxState.buildStateFileBackupCommand(sourceDir, { + path: "kanban.db", + strategy: "sqlite_backup", + }); + const result = spawnSync("sh", ["-c", command], { encoding: "utf8" }); + + expect(result.status).toBe(1); + expect(result.stderr).toContain(`permission denied: ${sourceFile}`); + expect(result.stdout).toBe(""); + } finally { + fs.rmSync(fixture, { recursive: true, force: true }); + } +}); + it("preserves only the Hermes default-board database across rebuilds (#7095)", () => { const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-kanban-state-")); const oldPath = process.env.PATH; From a555f68c1cec71bb3c61367249bfbd89c7ce81ae Mon Sep 17 00:00:00 2001 From: Yimo Jiang Date: Fri, 28 Aug 2026 06:32:00 +0000 Subject: [PATCH 2/4] fix(sandbox): harden rebuild recovery authority Bind privileged Hermes captures to verified filesystem objects, align the fallback size limit with normal backup, and reject undeclared archive entries. Signed-off-by: Yimo Jiang --- .../sandbox/rebuild-target-runtime.test.ts | 1 + .../actions/sandbox/rebuild-target-runtime.ts | 2 +- .../snapshot/backup-authority-script.test.ts | 177 +++++++++++++++++- .../sandbox/snapshot/backup-authority.test.ts | 11 ++ .../sandbox/snapshot/backup-authority.ts | 175 +++++++++++++---- src/lib/state/sandbox.ts | 17 +- .../hermes/hermes-kanban-snapshot.test.ts | 124 +++++++++++- 7 files changed, 463 insertions(+), 44 deletions(-) diff --git a/src/lib/actions/sandbox/rebuild-target-runtime.test.ts b/src/lib/actions/sandbox/rebuild-target-runtime.test.ts index 816421f2da1..c7d0047c830 100644 --- a/src/lib/actions/sandbox/rebuild-target-runtime.test.ts +++ b/src/lib/actions/sandbox/rebuild-target-runtime.test.ts @@ -203,6 +203,7 @@ describe("legacy DGX Station rebuild authority", () => { ["v0.0.97", false], ["0.0.97-1-gabcdef0", false], ["0.0.83-preview", false], + ["v0.0.096", false], ["0.0.x", false], ["", false], ])("accepts only a valid release older than v0.0.97: %s", (nemoclawVersion, expected) => { diff --git a/src/lib/actions/sandbox/rebuild-target-runtime.ts b/src/lib/actions/sandbox/rebuild-target-runtime.ts index d266ea17893..4e9b9cf2d6f 100644 --- a/src/lib/actions/sandbox/rebuild-target-runtime.ts +++ b/src/lib/actions/sandbox/rebuild-target-runtime.ts @@ -116,7 +116,7 @@ export function hasLegacyDgxStationQualificationAuthority( sandbox: Pick, ): boolean { if (sandbox.agent !== "hermes" || sandbox.fromDockerfile != null) return false; - const match = /^(?:v)?0\.0\.(\d+)(?:-[1-9]\d*-g[0-9a-f]{7,40})?$/i.exec( + const match = /^(?:v)?0\.0\.(0|[1-9]\d*)(?:-[1-9]\d*-g[0-9a-f]{7,40})?$/i.exec( sandbox.nemoclawVersion ?? "", ); if (!match) return false; diff --git a/src/lib/actions/sandbox/snapshot/backup-authority-script.test.ts b/src/lib/actions/sandbox/snapshot/backup-authority-script.test.ts index 1b250fbab4f..c39c0be9a50 100644 --- a/src/lib/actions/sandbox/snapshot/backup-authority-script.test.ts +++ b/src/lib/actions/sandbox/snapshot/backup-authority-script.test.ts @@ -65,6 +65,60 @@ exec(capture_script) `; } +function hermesCopyMutationHarness(mutation: string): string { + return `import os, sys +capture_script = ${JSON.stringify(HERMES_STATE_CAPTURE_SCRIPT)} +base = sys.argv[1] +relative = sys.argv[2] +real_read = os.read +mutated = False +def mutate_after_first_read(fd, size): + global mutated + data = real_read(fd, size) + if not mutated: + mutated = True +${mutation} + return data +os.read = mutate_after_first_read +exec(capture_script) +`; +} + +function hermesSqliteMutationHarness(mutation: string): string { + return `import os, sqlite3, sys +capture_script = ${JSON.stringify(HERMES_STATE_CAPTURE_SCRIPT)} +base = sys.argv[1] +relative = sys.argv[2] +real_connect = sqlite3.connect +mutated = False +def mutate_before_connect(database, *args, **kwargs): + global mutated + if not mutated and str(database).startswith("file:/proc/self/fd/"): + mutated = True +${mutation} + return real_connect(database, *args, **kwargs) +sqlite3.connect = mutate_before_connect +exec(capture_script) +`; +} + +function hermesDirectoryMutationHarness(mutation: string): string { + return `import os, sys, tarfile +capture_script = ${JSON.stringify(HERMES_DIRECTORY_CAPTURE_SCRIPT)} +base = sys.argv[1] +real_addfile = tarfile.TarFile.addfile +mutated = False +def mutate_before_file_read(archive, info, fileobj=None): + global mutated + if fileobj is not None and not mutated: + mutated = True +${mutation} + return real_addfile(archive, info, fileobj) +tarfile.TarFile.addfile = mutate_before_file_read +exec(capture_script) +`; +} + afterEach(() => { for (const root of fixtureRoots.splice(0)) { fs.rmSync(root, { recursive: true, force: true }); @@ -117,6 +171,128 @@ describe("Hermes privileged state capture scripts", () => { ).toBe(0); }); + it("captures a state file larger than the previous privileged buffer limit", () => { + const directory = fixtureDirectory(); + const expected = Buffer.alloc(18 * 1024 * 1024, 0xa5); + fs.writeFileSync(path.join(directory, "SOUL.md"), expected); + + const captured = spawnSync( + "/usr/bin/python3", + ["-I", "-S", "-c", HERMES_STATE_CAPTURE_SCRIPT, directory, "SOUL.md", "copy"], + { encoding: null, maxBuffer: 256 * 1024 * 1024 }, + ); + + expect(captured.status).toBe(0); + expect(captured.stdout).toHaveLength(expected.length); + expect(captured.stdout.equals(expected)).toBe(true); + }); + + it("rejects a copied file replaced during capture without returning bytes", () => { + const directory = fixtureDirectory(); + const source = path.join(directory, "SOUL.md"); + const outside = path.join(path.dirname(directory), "outside-copy"); + fs.writeFileSync(source, Buffer.alloc(128 * 1024, 0x61)); + fs.writeFileSync(outside, "outside-secret"); + const script = hermesCopyMutationHarness( + ` original = os.path.join(base, relative)\n` + + ` os.rename(original, original + ".old")\n` + + ` os.symlink(${JSON.stringify(outside)}, original)`, + ); + + const captured = spawnSync( + "/usr/bin/python3", + ["-I", "-S", "-c", script, directory, "SOUL.md", "copy"], + { encoding: null }, + ); + + expect(captured.status).toBe(13); + expect(captured.stdout).toEqual(Buffer.alloc(0)); + }); + + it("rejects a SQLite file replaced during capture without returning bytes", () => { + const directory = fixtureDirectory(); + const database = path.join(directory, "state.db"); + const outside = path.join(path.dirname(directory), "outside.db"); + expect( + spawnSync("/usr/bin/python3", [ + "-c", + `import sqlite3; db = sqlite3.connect(${JSON.stringify(database)}); db.execute('create table state (value text)'); db.execute("insert into state values ('saved')"); db.commit()`, + ]).status, + ).toBe(0); + expect( + spawnSync("/usr/bin/python3", [ + "-c", + `import sqlite3; db = sqlite3.connect(${JSON.stringify(outside)}); db.execute('create table state (value text)'); db.execute("insert into state values ('saved')"); db.commit()`, + ]).status, + ).toBe(0); + const script = hermesSqliteMutationHarness( + ` original = os.path.join(base, relative)\n` + + ` os.rename(original, original + ".old")\n` + + ` os.symlink(${JSON.stringify(outside)}, original)`, + ); + + const captured = spawnSync( + "/usr/bin/python3", + ["-I", "-S", "-c", script, directory, "state.db", "sqlite_backup"], + { encoding: null }, + ); + + expect(captured.status).toBe(13); + expect(captured.stdout).toEqual(Buffer.alloc(0)); + }); + + it("rejects an intermediate directory replaced during capture", () => { + const directory = fixtureDirectory(); + const runtime = path.join(directory, "runtime"); + const outside = path.join(path.dirname(directory), "outside-runtime"); + fs.mkdirSync(runtime); + fs.mkdirSync(outside); + fs.writeFileSync(path.join(runtime, "state.db"), Buffer.alloc(128 * 1024, 0x61)); + fs.writeFileSync(path.join(outside, "state.db"), "outside-secret"); + const script = hermesCopyMutationHarness( + ` original = os.path.join(base, "runtime")\n` + + ` os.rename(original, original + ".old")\n` + + ` os.symlink(${JSON.stringify(outside)}, original)`, + ); + + const captured = spawnSync( + "/usr/bin/python3", + ["-I", "-S", "-c", script, directory, "runtime/state.db", "copy"], + { encoding: null }, + ); + + expect(captured.status).toBe(13); + expect(captured.stdout).toEqual(Buffer.alloc(0)); + }); + + it.each([ + ["symbolic link", "os.symlink"], + ["hard link", "os.link"], + ])("rejects a directory file replaced with a %s during archive read", (_kind, replacement) => { + const directory = fixtureDirectory(); + const workspace = path.join(directory, "workspace"); + const marker = path.join(workspace, "marker"); + const outside = path.join(path.dirname(directory), "outside-directory-file"); + fs.mkdirSync(workspace); + fs.writeFileSync(marker, Buffer.alloc(128 * 1024, 0x61)); + fs.writeFileSync(outside, "outside-secret"); + const mutation = + ` original = os.path.join(base, "workspace", "marker")\n` + + ` os.rename(original, original + ".old")\n` + + ` replacement = ${replacement}\n` + + ` replacement(${JSON.stringify(outside)}, original)`; + const script = hermesDirectoryMutationHarness(mutation); + + const captured = spawnSync( + "/usr/bin/python3", + ["-I", "-S", "-c", script, directory, "workspace"], + { encoding: null }, + ); + + expect(captured.status).toBe(13); + expect(captured.stdout.includes(Buffer.from("outside-secret"))).toBe(false); + }); + it("rejects unsafe directory entries before streaming a tar archive", () => { const directory = fixtureDirectory(); const workspace = path.join(directory, "workspace"); @@ -136,7 +312,6 @@ describe("Hermes privileged state capture scripts", () => { { encoding: null }, ); expect(unsafe.status).not.toBe(0); - expect(unsafe.stdout).toEqual(Buffer.alloc(0)); }); }); diff --git a/src/lib/actions/sandbox/snapshot/backup-authority.test.ts b/src/lib/actions/sandbox/snapshot/backup-authority.test.ts index b7353e07d72..e0479b1dc0b 100644 --- a/src/lib/actions/sandbox/snapshot/backup-authority.test.ts +++ b/src/lib/actions/sandbox/snapshot/backup-authority.test.ts @@ -364,6 +364,10 @@ describe("managed snapshot backup authority", () => { spec: { path: "SOUL.md", strategy: "copy" }, }), ).toEqual({ outcome: "backed_up", data: Buffer.from("state") }); + expect(privilegedCaptureMocks.dockerSpawnSync).toHaveBeenLastCalledWith( + expect.any(Array), + expect.objectContaining({ maxBuffer: 256 * 1024 * 1024 }), + ); expect( captureHermesStateFile("alpha", { sandboxName: "alpha", @@ -440,6 +444,13 @@ describe("managed snapshot backup authority", () => { }, ); expect(result.success).toBe(true); + expect(backup).toHaveBeenCalledWith( + "alpha", + expect.objectContaining({ + captureStateFile: expect.any(Function), + captureStateDirectories: expect.any(Function), + }), + ); expect(privilegedCaptureMocks.dockerSpawnSync).not.toHaveBeenCalled(); }); diff --git a/src/lib/actions/sandbox/snapshot/backup-authority.ts b/src/lib/actions/sandbox/snapshot/backup-authority.ts index 90144d77e34..0eeef95e0eb 100644 --- a/src/lib/actions/sandbox/snapshot/backup-authority.ts +++ b/src/lib/actions/sandbox/snapshot/backup-authority.ts @@ -52,7 +52,7 @@ const OPENCLAW_CONFIG_DIRECTORY = "/sandbox/.openclaw"; const OPENCLAW_CONFIG_NAME = "openclaw.json"; const HERMES_CONFIG_DIRECTORY = "/sandbox/.hermes"; const HERMES_CAPTURE_TIMEOUT_MS = 120_000; -const HERMES_CAPTURE_MAX_BUFFER = 17 * 1024 * 1024; +const HERMES_CAPTURE_MAX_BUFFER = 256 * 1024 * 1024; export const OPENCLAW_CONFIG_CAPTURE_SCRIPT = `import os, stat, sys maximum = ${MAX_OPENCLAW_CONFIG_BYTES} directory = sys.argv[1] @@ -232,21 +232,46 @@ export function captureOpenClawStateFile( export const HERMES_STATE_CAPTURE_SCRIPT = `import os, sqlite3, stat, sys, tempfile base, relative, strategy = sys.argv[1:] -if not relative or relative.startswith("/") or ".." in relative.split("/"): +parts = relative.split("/") +if not relative or relative.startswith("/") or any(part in ("", ".", "..") for part in parts): raise SystemExit(10) -path = os.path.join(base, relative) +if strategy not in ("copy", "sqlite_backup"): + raise SystemExit(10) +directory_flags = os.O_RDONLY | os.O_DIRECTORY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) +file_flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0) +def identity(value): + return (value.st_dev, value.st_ino, stat.S_IFMT(value.st_mode), value.st_size, value.st_mtime_ns, value.st_ctime_ns, value.st_nlink) +def directory_identity(value): + return (value.st_dev, value.st_ino, stat.S_IFMT(value.st_mode)) +directory_fds = [] +file_fd = None +target_name = None try: - before = os.lstat(path) -except FileNotFoundError: - raise SystemExit(2) -if not stat.S_ISREG(before.st_mode) or stat.S_ISLNK(before.st_mode) or before.st_nlink != 1: - raise SystemExit(11) -if strategy == "sqlite_backup": + base_fd = os.open(base, directory_flags) + directory_fds.append(base_fd) + base_before = os.fstat(base_fd) + for component in parts[:-1]: + next_fd = os.open(component, directory_flags, dir_fd=directory_fds[-1]) + opened = os.fstat(next_fd) + current = os.stat(component, dir_fd=directory_fds[-1], follow_symlinks=False) + if not stat.S_ISDIR(opened.st_mode) or directory_identity(opened) != directory_identity(current): + os.close(next_fd) + raise SystemExit(11) + directory_fds.append(next_fd) + try: + file_fd = os.open(parts[-1], file_flags, dir_fd=directory_fds[-1]) + except FileNotFoundError: + raise SystemExit(2) + before = os.fstat(file_fd) + current_before = os.stat(parts[-1], dir_fd=directory_fds[-1], follow_symlinks=False) + if not stat.S_ISREG(before.st_mode) or before.st_nlink != 1 or identity(before) != identity(current_before): + raise SystemExit(11) target = tempfile.NamedTemporaryFile(dir="/tmp", delete=False) + target_name = target.name target.close() - try: - source = sqlite3.connect("file:" + path + "?mode=ro", uri=True, timeout=30) - destination = sqlite3.connect(target.name, timeout=30) + if strategy == "sqlite_backup": + source = sqlite3.connect("file:/proc/self/fd/" + str(file_fd) + "?mode=ro", uri=True, timeout=30) + destination = sqlite3.connect(target_name, timeout=30) try: source.backup(destination) if destination.execute("PRAGMA quick_check").fetchone()[0] != "ok": @@ -254,39 +279,115 @@ if strategy == "sqlite_backup": finally: destination.close() source.close() - with open(target.name, "rb", buffering=0) as stream: - while chunk := stream.read(64 * 1024): - sys.stdout.buffer.write(chunk) - finally: - os.unlink(target.name) -else: - with open(path, "rb", buffering=0) as stream: - while chunk := stream.read(64 * 1024): + else: + with open(target_name, "wb", buffering=0) as target_stream: + while True: + chunk = os.read(file_fd, 64 * 1024) + if not chunk: + break + target_stream.write(chunk) + after = os.fstat(file_fd) + current_after = os.stat(parts[-1], dir_fd=directory_fds[-1], follow_symlinks=False) + if identity(before) != identity(after) or identity(before) != identity(current_after): + raise SystemExit(13) + for index, component in enumerate(parts[:-1]): + opened = os.fstat(directory_fds[index + 1]) + current = os.stat(component, dir_fd=directory_fds[index], follow_symlinks=False) + if directory_identity(opened) != directory_identity(current) or not stat.S_ISDIR(current.st_mode): + raise SystemExit(13) + base_current = os.stat(base, follow_symlinks=False) + if directory_identity(base_before) != directory_identity(base_current) or not stat.S_ISDIR(base_current.st_mode): + raise SystemExit(13) + with open(target_name, "rb", buffering=0) as stream: + while True: + chunk = stream.read(64 * 1024) + if not chunk: + break sys.stdout.buffer.write(chunk) -after = os.lstat(path) -if (before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns, before.st_ctime_ns, before.st_nlink) != (after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns, after.st_ctime_ns, after.st_nlink): - raise SystemExit(13) +finally: + if target_name is not None: + try: + os.unlink(target_name) + except FileNotFoundError: + pass + if file_fd is not None: + os.close(file_fd) + for descriptor in reversed(directory_fds): + os.close(descriptor) `; -export const HERMES_DIRECTORY_CAPTURE_SCRIPT = `import os, stat, subprocess, sys +export const HERMES_DIRECTORY_CAPTURE_SCRIPT = `import os, stat, sys, tarfile +automatic_flags = getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) +directory_flags = os.O_RDONLY | os.O_DIRECTORY | automatic_flags +file_flags = os.O_RDONLY | automatic_flags | getattr(os, "O_NONBLOCK", 0) base, *names = sys.argv[1:] -def audit(path): - entry = os.lstat(path) - if stat.S_ISLNK(entry.st_mode) or not stat.S_ISDIR(entry.st_mode): +def identity(value): + return (value.st_dev, value.st_ino, stat.S_IFMT(value.st_mode), value.st_size, value.st_mtime_ns, value.st_ctime_ns, value.st_nlink) +def directory_identity(value): + return (value.st_dev, value.st_ino, stat.S_IFMT(value.st_mode)) +def tar_info(name, value): + info = tarfile.TarInfo(name) + info.mode = stat.S_IMODE(value.st_mode) + info.uid = value.st_uid + info.gid = value.st_gid + info.mtime = int(value.st_mtime) + return info +def add_entry(archive, parent_fd, name, archive_name): + if not name or "/" in name or "\\n" in name or "\\r" in name or name in (".", ".."): + raise SystemExit(10) + value = os.stat(name, dir_fd=parent_fd, follow_symlinks=False) + if stat.S_ISDIR(value.st_mode): + descriptor = os.open(name, directory_flags, dir_fd=parent_fd) + try: + opened = os.fstat(descriptor) + if directory_identity(value) != directory_identity(opened): + raise SystemExit(13) + info = tar_info(archive_name + "/", opened) + info.type = tarfile.DIRTYPE + info.size = 0 + archive.addfile(info) + for child in sorted(os.listdir(descriptor)): + add_entry(archive, descriptor, child, archive_name + "/" + child) + after = os.fstat(descriptor) + current = os.stat(name, dir_fd=parent_fd, follow_symlinks=False) + if directory_identity(opened) != directory_identity(after) or directory_identity(opened) != directory_identity(current): + raise SystemExit(13) + finally: + os.close(descriptor) + return + if not stat.S_ISREG(value.st_mode) or value.st_nlink != 1: raise SystemExit(11) - with os.scandir(path) as entries: - for child in entries: - value = child.stat(follow_symlinks=False) - if stat.S_ISLNK(value.st_mode) or not (stat.S_ISREG(value.st_mode) or stat.S_ISDIR(value.st_mode)): - raise SystemExit(11) - if stat.S_ISDIR(value.st_mode): - audit(child.path) + descriptor = os.open(name, file_flags, dir_fd=parent_fd) + try: + opened = os.fstat(descriptor) + current_before = os.stat(name, dir_fd=parent_fd, follow_symlinks=False) + if identity(value) != identity(opened) or identity(opened) != identity(current_before): + raise SystemExit(13) + info = tar_info(archive_name, opened) + info.type = tarfile.REGTYPE + info.size = opened.st_size + with os.fdopen(os.dup(descriptor), "rb", closefd=True) as stream: + archive.addfile(info, stream) + after = os.fstat(descriptor) + current_after = os.stat(name, dir_fd=parent_fd, follow_symlinks=False) + if identity(opened) != identity(after) or identity(opened) != identity(current_after): + raise SystemExit(13) + finally: + os.close(descriptor) for name in names: if not name or "/" in name or name in (".", ".."): raise SystemExit(10) - audit(os.path.join(base, name)) -result = subprocess.run(["/usr/bin/tar", "--hard-dereference", "-cf", "-", "-C", base, "--", *names], stdout=sys.stdout.buffer) -raise SystemExit(result.returncode) +base_fd = os.open(base, directory_flags) +try: + base_before = os.fstat(base_fd) + with tarfile.open(fileobj=sys.stdout.buffer, mode="w|") as archive: + for name in names: + add_entry(archive, base_fd, name, name) + base_current = os.stat(base, follow_symlinks=False) + if directory_identity(base_before) != directory_identity(base_current) or not stat.S_ISDIR(base_current.st_mode): + raise SystemExit(13) +finally: + os.close(base_fd) `; export function captureHermesStateFile( diff --git a/src/lib/state/sandbox.ts b/src/lib/state/sandbox.ts index ec7486d36b6..23e6e060204 100644 --- a/src/lib/state/sandbox.ts +++ b/src/lib/state/sandbox.ts @@ -1237,8 +1237,7 @@ function backupStateFile( result.status === 1 && !result.error && !result.signal && - (dir === "/sandbox/.openclaw" || - /permission denied/i.test(result.stderr?.toString() ?? "")) && + (dir === "/sandbox/.openclaw" || /permission denied/i.test(result.stderr?.toString() ?? "")) && captureFallback !== undefined ) { try { @@ -1314,6 +1313,20 @@ function retryPermissionDeniedDirectories( ); return; } + const allowedTopLevelEntries = new Set(denied); + const archiveValidation = validateTarEntries({ filePath: archivePath }, backupPath); + const undeclaredEntry = archiveValidation.entries.find((entry) => { + const normalized = entry.replace(/^\.\/+/, ""); + const topLevel = normalized.split("/", 1)[0]; + return !topLevel || !allowedTopLevelEntries.has(topLevel); + }); + if (!archiveValidation.safe || undeclaredEntry) { + const detail = undeclaredEntry + ? `undeclared archive entry: ${undeclaredEntry}` + : archiveValidation.violations.join("; "); + _log(`FAILED: privileged state directory capture: ${detail}`); + return; + } for (const name of denied) { const target = path.join(backupPath, name); rejectSymlinksOnPath(target); diff --git a/test/agents/hermes/hermes-kanban-snapshot.test.ts b/test/agents/hermes/hermes-kanban-snapshot.test.ts index ffb6e89795a..c08de2bb5d0 100644 --- a/test/agents/hermes/hermes-kanban-snapshot.test.ts +++ b/test/agents/hermes/hermes-kanban-snapshot.test.ts @@ -6,7 +6,9 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { pathToFileURL } from "node:url"; -import { afterAll, expect, it } from "vitest"; +import { afterAll, expect, it, vi } from "vitest"; + +import type { StateDirectoryCaptureRequest } from "../../../src/lib/state/sandbox"; // sandbox-state captures HOME when the module loads, so isolate its registry // and rebuild backups before importing it. @@ -123,6 +125,119 @@ process.exit(result.status === null ? 1 : result.status); } } +function exercisePermissionDeniedDirectoryCapture(archive: "declared" | "undeclared"): { + backup: ReturnType; + captureRequests: unknown[][]; + restoredMarker: string | null; +} { + const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-directory-recovery-")); + const oldPath = process.env.PATH; + const oldOpenshell = process.env.NEMOCLAW_OPENSHELL_BIN; + try { + const binDir = path.join(fixture, "bin"); + const hermesDir = path.join(fixture, "sandbox-root", ".hermes"); + fs.mkdirSync(binDir, { recursive: true }); + fs.mkdirSync(path.join(hermesDir, "memories"), { recursive: true }); + fs.mkdirSync(path.join(hermesDir, "sessions"), { recursive: true }); + fs.writeFileSync(path.join(hermesDir, "memories", "marker.txt"), "preserved\n"); + fs.writeFileSync(path.join(hermesDir, "sessions", "outside.txt"), "undeclared\n"); + + const openshell = path.join(binDir, "openshell"); + writeExecutable( + openshell, + `#!/usr/bin/env node +const args = process.argv.slice(2); +if (args[0] === "sandbox" && args[1] === "ssh-config") { + process.stdout.write("Host openshell-hermes\\n HostName 127.0.0.1\\n User sandbox\\n"); +} +`, + ); + writeExecutable( + path.join(binDir, "ssh"), + `#!/usr/bin/env node +const cmd = process.argv[process.argv.length - 1] || ""; +if (cmd.includes("cat --") || cmd.includes("nemoclaw-sqlite-backup")) process.exit(2); +if (cmd.includes("[ -d ")) { + process.stdout.write("memories\\n"); + process.exit(0); +} +if (cmd.includes("find ")) process.exit(0); +if (cmd.includes("-cf -")) { + process.stderr.write("tar: memories/marker.txt: Cannot open: Permission denied\\n"); + process.exit(2); +} +process.exit(2); +`, + ); + + writeHermesRegistry(); + process.env.NEMOCLAW_OPENSHELL_BIN = openshell; + process.env.PATH = `${binDir}${path.delimiter}${oldPath || ""}`; + const captureStateDirectories = vi.fn( + (_request: StateDirectoryCaptureRequest, archiveFd: number) => { + const names = archive === "declared" ? ["memories"] : ["memories", "sessions"]; + const result = spawnSync("tar", ["-cf", "-", "-C", hermesDir, ...names], { + encoding: null, + }); + expect(result.status).toBe(0); + expect(Buffer.isBuffer(result.stdout)).toBe(true); + fs.writeSync(archiveFd, result.stdout as Buffer); + return { outcome: "backed_up" as const }; + }, + ); + const backup = sandboxState.backupSandboxState("hermes", { + name: `directory-${archive}`, + captureStateDirectories, + }); + const markerPath = backup.manifest + ? path.join(backup.manifest.backupPath, "memories", "marker.txt") + : ""; + return { + backup, + captureRequests: captureStateDirectories.mock.calls, + restoredMarker: + markerPath && fs.existsSync(markerPath) ? fs.readFileSync(markerPath, "utf8") : null, + }; + } finally { + oldOpenshell === undefined + ? delete process.env.NEMOCLAW_OPENSHELL_BIN + : (process.env.NEMOCLAW_OPENSHELL_BIN = oldOpenshell); + oldPath === undefined ? delete process.env.PATH : (process.env.PATH = oldPath); + fs.rmSync(fixture, { recursive: true, force: true }); + } +} + +it("recovers a permission-denied Hermes directory through the state-layer fallback (#10375)", () => { + const result = exercisePermissionDeniedDirectoryCapture("declared"); + + expect( + result.backup.success, + JSON.stringify({ + error: result.backup.error, + backedUpDirs: result.backup.backedUpDirs, + failedDirs: result.backup.failedDirs, + failedDirReasons: result.backup.failedDirReasons, + backedUpFiles: result.backup.backedUpFiles, + failedFiles: result.backup.failedFiles, + }), + ).toBe(true); + expect(result.backup.backedUpDirs).toEqual(["memories"]); + expect(result.backup.failedDirs).toEqual([]); + expect(result.backup.failedDirReasons).toBeUndefined(); + expect(result.restoredMarker).toBe("preserved\n"); + expect(result.captureRequests[0]?.[0]).toMatchObject({ dirs: ["memories"] }); +}); + +it("rejects undeclared entries from privileged Hermes directory capture (#10375)", () => { + const result = exercisePermissionDeniedDirectoryCapture("undeclared"); + + expect(result.backup.success).toBe(false); + expect(result.backup.backedUpDirs).toEqual([]); + expect(result.backup.failedDirs).toEqual(["memories"]); + expect(result.backup.failedDirReasons).toEqual({ memories: "permission denied" }); + expect(result.restoredMarker).toBeNull(); +}); + it("fails closed when the remote Hermes SQLite backup command fails (#7144)", () => { const result = exerciseFailedKanbanBackup({ mode: "execute", name: "invalid-kanban" }); @@ -165,7 +280,9 @@ it("fails the SQLite state backup when the online backup command fails (#7095)", } }); -it("classifies an unreadable Hermes SQLite file before opening the database (#10375)", () => { +it.skipIf(typeof process.getuid === "function" && process.getuid() === 0)( + "classifies an unreadable Hermes SQLite file before opening the database (#10375)", + () => { const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-sqlite-backup-denied-")); try { const sourceDir = path.join(fixture, "state"); @@ -185,7 +302,8 @@ it("classifies an unreadable Hermes SQLite file before opening the database (#10 } finally { fs.rmSync(fixture, { recursive: true, force: true }); } -}); + }, +); it("preserves only the Hermes default-board database across rebuilds (#7095)", () => { const fixture = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-kanban-state-")); From ae1ebaf1256bd12a81619b5407af3fe40839455c Mon Sep 17 00:00:00 2001 From: Yimo Jiang Date: Fri, 28 Aug 2026 06:55:51 +0000 Subject: [PATCH 3/4] test(snapshot): surface archive fixture failures Build and validate the tar fixture before entering the production callback so fixture failures cannot be swallowed by backup error handling. Signed-off-by: Yimo Jiang --- test/agents/hermes/hermes-kanban-snapshot.test.ts | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/test/agents/hermes/hermes-kanban-snapshot.test.ts b/test/agents/hermes/hermes-kanban-snapshot.test.ts index c08de2bb5d0..68731693859 100644 --- a/test/agents/hermes/hermes-kanban-snapshot.test.ts +++ b/test/agents/hermes/hermes-kanban-snapshot.test.ts @@ -173,15 +173,16 @@ process.exit(2); writeHermesRegistry(); process.env.NEMOCLAW_OPENSHELL_BIN = openshell; process.env.PATH = `${binDir}${path.delimiter}${oldPath || ""}`; + const names = archive === "declared" ? ["memories"] : ["memories", "sessions"]; + const archiveResult = spawnSync("tar", ["-cf", "-", "-C", hermesDir, ...names], { + encoding: null, + }); + expect(archiveResult.status).toBe(0); + expect(Buffer.isBuffer(archiveResult.stdout)).toBe(true); + const archiveBytes = archiveResult.stdout as Buffer; const captureStateDirectories = vi.fn( (_request: StateDirectoryCaptureRequest, archiveFd: number) => { - const names = archive === "declared" ? ["memories"] : ["memories", "sessions"]; - const result = spawnSync("tar", ["-cf", "-", "-C", hermesDir, ...names], { - encoding: null, - }); - expect(result.status).toBe(0); - expect(Buffer.isBuffer(result.stdout)).toBe(true); - fs.writeSync(archiveFd, result.stdout as Buffer); + fs.writeSync(archiveFd, archiveBytes); return { outcome: "backed_up" as const }; }, ); From 0537594b87c8ecdd069b789069c83bf1bf6d3fdd Mon Sep 17 00:00:00 2001 From: Yimo Jiang Date: Fri, 28 Aug 2026 07:18:24 +0000 Subject: [PATCH 4/4] fix(sandbox): route supervisor recovery through authority Use managed backup authority for supervisor relaunch so Hermes permission-denied state receives the same constrained fallback as other recovery paths. Signed-off-by: Yimo Jiang --- .../sandbox/supervisor-relaunch.test.ts | 36 +++++++++++++++++++ .../actions/sandbox/supervisor-relaunch.ts | 5 ++- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/src/lib/actions/sandbox/supervisor-relaunch.test.ts b/src/lib/actions/sandbox/supervisor-relaunch.test.ts index 8a3d48bcedb..46b46e6e240 100644 --- a/src/lib/actions/sandbox/supervisor-relaunch.test.ts +++ b/src/lib/actions/sandbox/supervisor-relaunch.test.ts @@ -7,6 +7,7 @@ import { type ManagedSupervisorRelaunchDeps, relaunchManagedSupervisorSession, } from "./supervisor-relaunch"; +import * as backupAuthority from "./snapshot/backup-authority"; afterEach(() => { vi.restoreAllMocks(); @@ -170,6 +171,41 @@ describe("relaunchManagedSupervisorSession", () => { ); }); + it("uses managed backup authority for default supervisor recovery", () => { + const managedBackup = vi + .spyOn(backupAuthority, "backupSandboxStateWithManagedAuthority") + .mockReturnValue({ + success: true, + manifest: { backupPath: "/tmp/rebuild-backups/alpha/managed-recovery" }, + backedUpDirs: ["memories"], + failedDirs: [], + backedUpFiles: [], + failedFiles: [], + } as never); + const getSandbox = vi.fn(() => ({ + name: "alpha", + agent: "hermes", + dashboardPort: 18789, + openshellDriver: "docker", + })) as never; + const deps = baseDeps({ + backupState: undefined, + getSandbox, + getSessionAgent: vi.fn( + () => + ({ + name: "hermes", + displayName: "Hermes", + forwardPort: 18789, + }) as never, + ), + }); + + expect(relaunchManagedSupervisorSession("alpha", { quiet: true, deps })).not.toBeNull(); + expect(managedBackup).toHaveBeenCalledWith("alpha", {}, { getSandbox }); + expect(deps.recreate).toHaveBeenCalledOnce(); + }); + it("retries only transport-level state backup failures after a container restart", () => { const backupState = vi .fn() diff --git a/src/lib/actions/sandbox/supervisor-relaunch.ts b/src/lib/actions/sandbox/supervisor-relaunch.ts index 14d403e2316..bcddc4270e7 100644 --- a/src/lib/actions/sandbox/supervisor-relaunch.ts +++ b/src/lib/actions/sandbox/supervisor-relaunch.ts @@ -21,6 +21,7 @@ import { redact, redactFull } from "../../security/redact"; import * as registry from "../../state/registry"; import * as sandboxState from "../../state/sandbox"; import { resolveSandboxDashboardPort } from "./forward-recovery"; +import { backupSandboxStateWithManagedAuthority } from "./snapshot/backup-authority"; /** * Compatibility boundary for OpenShell 0.0.71's Docker driver: legacy @@ -146,7 +147,9 @@ export function relaunchManagedSupervisorSession( const inspect = deps.inspectContainer ?? inspectContainer; const confirmMissingSupervisor = deps.confirmMissingSupervisor; const restartRestoredManagedGateway = deps.restartRestoredManagedGateway; - const backupState = deps.backupState ?? sandboxState.backupSandboxState; + const backupState = + deps.backupState ?? + ((name: string) => backupSandboxStateWithManagedAuthority(name, {}, { getSandbox })); const restoreState = deps.restoreState ?? sandboxState.restoreSandboxState; const removeBackup = deps.removeBackup ?? sandboxState.removeSandboxStateBackup; const recreate = deps.recreate ?? recreateOpenShellDockerSandboxWithStartupCommand;