Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
116 changes: 113 additions & 3 deletions test/automation/pull-requests/pr-risk-plan.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,45 @@ const HERMES_WRAPPER_FOCUSED_JOBS = [
"security-posture",
];
const HERMES_WRAPPER_REQUIRED_JOBS = [...HERMES_MANAGED_POLICY_REQUIRED_JOBS, "mcp-bridge"];
const SHARED_MESSAGING_RUNTIME_E2E_JOBS = [
"channels-add-remove",
"channels-stop-start",
"hermes-discord",
"messaging-providers",
"openclaw-discord-pairing",
"openclaw-slack-pairing",
];
const HERMES_MESSAGING_RUNTIME_E2E_JOBS = [
"channels-stop-start",
"hermes-discord",
"messaging-providers",
];
const OPENCLAW_MESSAGING_RUNTIME_E2E_JOBS = [
"channels-stop-start",
"messaging-providers",
"openclaw-discord-pairing",
"openclaw-slack-pairing",
];
const HERMES_START_FOCUSED_JOBS = [
"bedrock-runtime-compatible-anthropic",
"channels-stop-start",
"dashboard-remote-bind",
"hermes-discord",
"hermes-e2e",
"hermes-inference-switch",
"hermes-shields-config",
"messaging-providers",
"security-posture",
];
const HERMES_START_REQUIRED_JOBS = [
...HERMES_SANDBOX_BOUNDARY_JOBS,
"bedrock-runtime-compatible-anthropic",
"channels-stop-start",
"dashboard-remote-bind",
"hermes-discord",
"hermes-shields-config",
"messaging-providers",
];
const HERMES_MANAGED_POLICY_FILES = [
"agents/hermes/config/managed-policy.ts",
"agents/hermes/hermes-wrapper.py",
Expand Down Expand Up @@ -110,7 +149,7 @@ describe("deterministic PR risk plan", () => {
const second = plan("src/lib/onboard.ts", "src/lib/state/registry.ts");

expect(first).toEqual(second);
expect(first.version).toBe(19);
expect(first.version).toBe(20);
expect(first.headSha).toBe(HEAD_SHA);
expect(first.planHash).toMatch(/^[a-f0-9]{64}$/u);
expect(first.changedFiles).toEqual(["src/lib/onboard.ts", "src/lib/state/registry.ts"]);
Expand Down Expand Up @@ -320,12 +359,16 @@ describe("deterministic PR risk plan", () => {
const isWrapper = changedFile === "agents/hermes/hermes-wrapper.py";
const expectedFocusedJobs = isWrapper
? HERMES_WRAPPER_FOCUSED_JOBS
: HERMES_MANAGED_POLICY_JOBS;
: changedFile === "agents/hermes/start.sh"
? HERMES_START_FOCUSED_JOBS
: HERMES_MANAGED_POLICY_JOBS;
const expectedRequiredJobs = isWrapper
? HERMES_WRAPPER_REQUIRED_JOBS
: changedFile === "src/lib/hermes-managed-route.ts"
? HERMES_MANAGED_POLICY_JOBS
: HERMES_MANAGED_POLICY_REQUIRED_JOBS;
: changedFile === "agents/hermes/start.sh"
? HERMES_START_REQUIRED_JOBS
: HERMES_MANAGED_POLICY_REQUIRED_JOBS;

const focusedFamily = result.families.find((family) => family.id === "focused-e2e");
expect(focusedFamily).toEqual(
Expand Down Expand Up @@ -362,6 +405,69 @@ describe("deterministic PR risk plan", () => {
);
expect(riskPlanRequiredJobIds(result)).toEqual(HERMES_WRAPPER_REQUIRED_JOBS);
});

it.each([
"src/lib/messaging/applier/openshell-provider.ts",
"src/lib/onboard/credential-provider-registration.ts",
"src/lib/onboard/messaging-policy-presets.ts",
])("selects the full messaging runtime proof set for %s (#10153)", (changedFile) => {
const result = plan(changedFile);
const focusedFamily = result.families.find((family) => family.id === "focused-e2e");

expect(focusedFamily?.matchedFiles).toContain(changedFile);
expect(focusedFamily?.requiredJobs).toEqual(
expect.arrayContaining(SHARED_MESSAGING_RUNTIME_E2E_JOBS),
);
expect(riskPlanRequiredJobIds(result)).toEqual(
expect.arrayContaining(SHARED_MESSAGING_RUNTIME_E2E_JOBS),
);
});

it("selects OpenClaw pairing and restart proofs for OpenClaw startup changes (#10153)", () => {
const changedFile = "scripts/nemoclaw-start.sh";
const result = plan(changedFile);
const focusedFamily = result.families.find((family) => family.id === "focused-e2e");

expect(focusedFamily?.matchedFiles).toEqual([changedFile]);
expect(focusedFamily?.requiredJobs).toEqual(
expect.arrayContaining([...OPENCLAW_MESSAGING_RUNTIME_E2E_JOBS, "shields-config"]),
);
expect(riskPlanRequiredJobIds(result)).toEqual(
expect.arrayContaining([...OPENCLAW_MESSAGING_RUNTIME_E2E_JOBS, "shields-config"]),
);
});

it.each([
"scripts/runtime-state-mutation-control.py",
"src/lib/onboard/runtime-provider/docker-state-mutation.ts",
"src/lib/shields/relock-reconfirm.ts",
])("selects both Shields runtime proofs for %s (#10155)", (changedFile) => {
const result = plan(changedFile);
const focusedFamily = result.families.find((family) => family.id === "focused-e2e");

expect(focusedFamily?.matchedFiles).toContain(changedFile);
expect(focusedFamily?.requiredJobs).toEqual(
expect.arrayContaining(["hermes-shields-config", "shields-config"]),
);
expect(riskPlanRequiredJobIds(result)).toEqual(
expect.arrayContaining(["hermes-shields-config", "shields-config"]),
);
});

it("selects Hermes Shields and messaging proofs for the shared Hermes runtime guard (#10155)", () => {
const changedFile = "agents/hermes/runtime-config-guard.py";
const result = plan(changedFile);
const focusedFamily = result.families.find((family) => family.id === "focused-e2e");

expect(focusedFamily?.matchedFiles).toEqual([changedFile]);
expect(focusedFamily?.requiredJobs).toEqual(
expect.arrayContaining([...HERMES_MESSAGING_RUNTIME_E2E_JOBS, "hermes-shields-config"]),
);
expect(riskPlanRequiredJobIds(result)).toEqual(
expect.arrayContaining([...HERMES_MESSAGING_RUNTIME_E2E_JOBS, "hermes-shields-config"]),
);
});

it("leaves E2E support-only changes in the fast e2e-support project (#7921)", () => {
const changedFiles = ["test/e2e/support/workflow-plan.test.ts"];
const focusedE2eJobs = focusedE2eJobsForChangedFiles(changedFiles);
Expand Down Expand Up @@ -989,11 +1095,15 @@ describe("deterministic PR risk plan", () => {
"channels-add-remove",
"channels-stop-start",
"full-e2e",
"hermes-discord",
"hermes-e2e",
"inference-routing",
"messaging-providers",
"network-policy",
"onboard-repair",
"onboard-resume",
"openclaw-discord-pairing",
"openclaw-slack-pairing",
"rebuild-openclaw",
"state-backup-restore",
]);
Expand Down
23 changes: 21 additions & 2 deletions test/e2e/support/workflow-plan.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -789,19 +789,38 @@ describe("E2E workflow plan", () => {
expect(targetIds).toEqual(expect.arrayContaining(["onboard-repair", "onboard-resume"]));
});

it("uses one risk rule for catalogue targets and workflow jobs", () => {
it("selects the full messaging proof set for messaging runtime changes", () => {
const plan = buildE2eWorkflowPlan(
{},
{ changedFiles: ["src/lib/messaging/applier/agent-config.ts"] },
);

expect(plan.catalogueMatrices.standard.map((row) => row.id)).toContain("channels-add-remove");
expect(plan.catalogueMatrices["nvidia-inference"].map((row) => row.id)).toEqual(
expect.arrayContaining(["channels-stop-start-openclaw", "channels-stop-start-hermes"]),
expect.arrayContaining([
"channels-stop-start-openclaw",
"channels-stop-start-hermes",
"hermes-discord",
"openclaw-discord-pairing",
"openclaw-slack-pairing",
]),
);
expect(plan.selectedJobs).toContain("messaging-providers");
expect(plan.selectedJobs).not.toContain("channels-stop-start");
});

it("selects both Shields proof lanes for shared Shields runtime changes", () => {
const plan = buildE2eWorkflowPlan(
{},
{ changedFiles: ["src/lib/shields/relock-reconfirm.ts"] },
);

expect(plan.catalogueMatrices.standard.map((row) => row.id)).toContain("hermes-shields-config");
expect(plan.catalogueMatrices["nvidia-inference"].map((row) => row.id)).toContain(
"shields-config",
);
});

it.each(["jobs", "targets"] as const)(
"maps the retired Hermes dashboard %s selector to the canonical lane",
(kind) => {
Expand Down
111 changes: 110 additions & 1 deletion tools/advisors/risk-plan.mts
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ const protectedManagedImageContract = (
const { PROTECTED_MANAGED_IMAGE_ACTIVATION_PATH, PROTECTED_MANAGED_IMAGE_MULTIARCH_JOB_ID } =
protectedManagedImageContract;

export const RISK_PLAN_VERSION = 19 as const;
export const RISK_PLAN_VERSION = 20 as const;

export const PR_E2E_TYPED_TARGET_IDS = [
"ubuntu-repo-cloud-langchain-deepagents-code",
Expand Down Expand Up @@ -105,6 +105,56 @@ const HERMES_MANAGED_POLICY_FILES = new Set([
"agents/hermes/start.sh",
"src/lib/hermes-managed-route.ts",
]);
const SHARED_MESSAGING_RUNTIME_E2E_JOB_IDS = [
"channels-add-remove",
"channels-stop-start",
"hermes-discord",
"messaging-providers",
"openclaw-discord-pairing",
"openclaw-slack-pairing",
] as const;
const HERMES_MESSAGING_RUNTIME_E2E_JOB_IDS = [
"channels-stop-start",
"hermes-discord",
"messaging-providers",
] as const;
const OPENCLAW_MESSAGING_RUNTIME_E2E_JOB_IDS = [
"channels-stop-start",
"messaging-providers",
"openclaw-discord-pairing",
"openclaw-slack-pairing",
] as const;
const MESSAGING_RUNTIME_FILES = new Set([
"src/lib/actions/sandbox/rebuild-backup-phase.ts",
"src/lib/actions/sandbox/rebuild-target-runtime.ts",
"src/lib/onboard/credential-provider-registration.ts",
"src/lib/onboard/extra-placeholder-keys.ts",
"src/lib/onboard/gateway-provider-metadata.ts",
"src/lib/onboard/messaging-policy-presets.ts",
"src/lib/onboard/messaging-prep.ts",
"src/lib/onboard/policy-preset-persistence.ts",
"src/lib/onboard/policy-preset-reconciliation.ts",
"src/lib/onboard/policy-selection.ts",
"src/lib/onboard/providers.ts",
"src/lib/onboard/sandbox-create-plan-materialization.ts",
"src/lib/onboard/sandbox-create/provider-publication.ts",
"src/lib/onboard/sandbox-messaging-preflight.ts",
]);
const MESSAGING_RUNTIME_PREFIXES = [
"src/lib/actions/sandbox/policy-channel",
"src/lib/messaging/",
] as const;
const SHARED_SHIELDS_E2E_JOB_IDS = ["hermes-shields-config", "shields-config"] as const;
const SHARED_SHIELDS_RUNTIME_FILES = new Set([
"scripts/runtime-state-mutation-control.py",
"scripts/runtime-state-mutation-startup-gate.py",
"src/lib/onboard/runtime-provider/docker-state-mutation.ts",
]);
const HERMES_STARTUP_RUNTIME_FILES = new Set([
"agents/hermes/runtime-config-guard.py",
"agents/hermes/start.sh",
]);
const OPENCLAW_STARTUP_RUNTIME_FILES = new Set(["scripts/nemoclaw-start.sh"]);
const MANAGED_IMAGE_PROTECTED_RUNTIME_ACTIVATION =
"ci/protected-managed-image-runtime-activation-v1.json";
const MANAGED_IMAGE_PROTECTED_RUNTIME_JOB_ID = "managed-image-protected-runtime" as const;
Expand Down Expand Up @@ -340,6 +390,41 @@ export function focusedPrE2eJobsForChangedFiles(
isRuntimeRelevant(file),
),
);
const messagingRuntimeFiles = stableUnique(
changedFiles.filter(
(file) =>
(MESSAGING_RUNTIME_FILES.has(file) ||
MESSAGING_RUNTIME_PREFIXES.some((prefix) => file.startsWith(prefix))) &&
isRuntimeRelevant(file),
),
);
const hermesMessagingRuntimeFiles = stableUnique(
changedFiles.filter(
(file) => HERMES_STARTUP_RUNTIME_FILES.has(file) && isRuntimeRelevant(file),
),
);
const openClawMessagingRuntimeFiles = stableUnique(
changedFiles.filter(
(file) => OPENCLAW_STARTUP_RUNTIME_FILES.has(file) && isRuntimeRelevant(file),
),
);
const sharedShieldsRuntimeFiles = stableUnique(
changedFiles.filter(
(file) =>
(file.startsWith("src/lib/shields/") || SHARED_SHIELDS_RUNTIME_FILES.has(file)) &&
isRuntimeRelevant(file),
),
);
const hermesShieldsRuntimeFiles = stableUnique(
changedFiles.filter(
(file) => HERMES_STARTUP_RUNTIME_FILES.has(file) && isRuntimeRelevant(file),
),
);
const openClawShieldsRuntimeFiles = stableUnique(
changedFiles.filter(
(file) => OPENCLAW_STARTUP_RUNTIME_FILES.has(file) && isRuntimeRelevant(file),
),
);
return [
...(journaledRecreateResumeFiles.length > 0
? [
Expand All @@ -365,6 +450,30 @@ export function focusedPrE2eJobsForChangedFiles(
id,
matchedFiles: hermesManagedPolicyFiles,
})),
...SHARED_MESSAGING_RUNTIME_E2E_JOB_IDS.map((id) => ({
id,
matchedFiles: messagingRuntimeFiles,
})),
...HERMES_MESSAGING_RUNTIME_E2E_JOB_IDS.map((id) => ({
id,
matchedFiles: hermesMessagingRuntimeFiles,
})),
...OPENCLAW_MESSAGING_RUNTIME_E2E_JOB_IDS.map((id) => ({
id,
matchedFiles: openClawMessagingRuntimeFiles,
})),
...SHARED_SHIELDS_E2E_JOB_IDS.map((id) => ({
id,
matchedFiles: sharedShieldsRuntimeFiles,
})),
{
id: "hermes-shields-config",
matchedFiles: hermesShieldsRuntimeFiles,
},
{
id: "shields-config",
matchedFiles: openClawShieldsRuntimeFiles,
},
].filter((selection) => selection.matchedFiles.length > 0);
}

Expand Down
Loading