fix(ci): repair Quick Checks + Check Types after PROD-5 land

patrykkopycinski · patrykkopycinski · commit 639941fffbfd · 2026-05-15T10:10:14.000+02:00
The post-cleanup CI build (b441998) on PR elastic#269019 surfaced four real breakages introduced by recent commits + the cleanup itself: 1. **TS typo** in validate_rule_tool.ts:515 — concurrencyResult exposes `inflightScenarioFingerprint`, not `inflightFingerprint`. The route file already uses the right name; only the tool path had drifted. 2. **TS typing** in run_command_tools.test.ts — the parameterized `it.each(tools)` table mixes per-family schemas with different `command` literal unions; destructuring `parameters` failed because one entry didn't have it, and `getConfirmation`'s `toolParams` intersected to `never` across the four families. Made `parameters` a uniform widened type and cast `getConfirmation` through `unknown` to a single shape — runtime contract unchanged, all 62 tests still pass. 3. **TS-projects linter** rejected validate_rule.spec.ts as a stranded file (excluded from the security_solution tsconfigs but not part of any other TS project). The canonical fix — matching every other eval suite in the repo (kbn-evals-suite-pci-compliance, etc.) — is to extract the spec + its dataset into a sibling devOnly `functional-tests` package: `@kbn/evals-suite-detection-emulation` under `x-pack/solutions/security/packages/`. This keeps the production plugin clean of the devOnly `@kbn/evals` reference and gives the spec a real owning tsconfig. CODEOWNERS updated to point at the new path. 4. **Moon project regen** — running `node scripts/regenerate_moon_projects.js --update` registered the new package in `package.json`, `tsconfig.base.json`, and `yarn.lock`. Auto-generated; included. Verification (local): - jest: 88/88 pass across run_command_tools.test.ts (62) + validate_rule_tool.test.ts (6) + concurrency_gate.test.ts (10) + validate_rule/route.test.ts (10). - eslint --fix: clean on all touched files. - type_check: clean on the new evals suite package (~70s) AND on the full security_solution plugin (~6m). No behavioural changes; this is purely a CI-repair commit.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -3198,7 +3198,7 @@ x-pack/platform/test/automatic_import_api_integration @elastic/integration-exper
 /x-pack/solutions/security/plugins/security_solution/common/detection_emulation @elastic/security-detection-engine @elastic/security-defend-workflows
 /x-pack/solutions/security/plugins/security_solution/public/detections/components/emulation @elastic/security-detection-engine @elastic/security-defend-workflows
 /x-pack/solutions/security/plugins/security_solution/server/agent_builder/skills/detection_emulation @elastic/security-detection-engine @elastic/security-defend-workflows
-/x-pack/solutions/security/plugins/security_solution/server/agent_builder/skills/detection_emulation/evals @elastic/security-detection-engine @elastic/security-defend-workflows
+/x-pack/solutions/security/packages/kbn-evals-suite-detection-emulation @elastic/security-detection-engine @elastic/security-defend-workflows
 /x-pack/solutions/security/plugins/security_solution/public/detections/components/osquery @elastic/security-defend-workflows
 
 # Cloud Security Posture team
diff --git a/package.json b/package.json
@@ -1740,6 +1740,7 @@
     "@kbn/evals-phoenix-executor": "link:x-pack/platform/packages/shared/kbn-evals-phoenix-executor",
     "@kbn/evals-suite-agent-builder": "link:x-pack/platform/packages/shared/agent-builder/kbn-evals-suite-agent-builder",
     "@kbn/evals-suite-attack-discovery": "link:x-pack/solutions/security/packages/kbn-evals-suite-attack-discovery",
+    "@kbn/evals-suite-detection-emulation": "link:x-pack/solutions/security/packages/kbn-evals-suite-detection-emulation",
     "@kbn/evals-suite-endpoint": "link:x-pack/solutions/security/packages/kbn-evals-suite-endpoint",
     "@kbn/evals-suite-entity-analytics": "link:x-pack/solutions/security/packages/kbn-evals-suite-entity-analytics",
     "@kbn/evals-suite-llm-tasks": "link:x-pack/platform/packages/shared/ai-infra/kbn-evals-suite-llm-tasks",
diff --git a/tsconfig.base.json b/tsconfig.base.json
@@ -1198,6 +1198,8 @@
       "@kbn/evals-suite-agent-builder/*": ["x-pack/platform/packages/shared/agent-builder/kbn-evals-suite-agent-builder/*"],
       "@kbn/evals-suite-attack-discovery": ["x-pack/solutions/security/packages/kbn-evals-suite-attack-discovery"],
       "@kbn/evals-suite-attack-discovery/*": ["x-pack/solutions/security/packages/kbn-evals-suite-attack-discovery/*"],
+      "@kbn/evals-suite-detection-emulation": ["x-pack/solutions/security/packages/kbn-evals-suite-detection-emulation"],
+      "@kbn/evals-suite-detection-emulation/*": ["x-pack/solutions/security/packages/kbn-evals-suite-detection-emulation/*"],
       "@kbn/evals-suite-endpoint": ["x-pack/solutions/security/packages/kbn-evals-suite-endpoint"],
       "@kbn/evals-suite-endpoint/*": ["x-pack/solutions/security/packages/kbn-evals-suite-endpoint/*"],
       "@kbn/evals-suite-entity-analytics": ["x-pack/solutions/security/packages/kbn-evals-suite-entity-analytics"],
diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-detection-emulation/evals/validate_rule.spec.ts b/x-pack/solutions/security/packages/kbn-evals-suite-detection-emulation/evals/validate_rule.spec.ts
@@ -30,7 +30,9 @@
  *                            additional cost per example.
  *
  * NOT a Jest test — run via the @kbn/evals Playwright eval runner.
- * Excluded from the plugin's main tsconfig; requires @kbn/evals (devOnly).
+ * Lives in `@kbn/evals-suite-detection-emulation` (devOnly functional-tests
+ * package) so the production `security_solution` plugin doesn't need to
+ * pull in the devOnly `@kbn/evals` package as a kbn_reference.
  */
 
 import type { Client as EsClient } from '@elastic/elasticsearch';
diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-detection-emulation/evals/validate_rule_dataset.ts b/x-pack/solutions/security/packages/kbn-evals-suite-detection-emulation/evals/validate_rule_dataset.ts
diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-detection-emulation/kibana.jsonc b/x-pack/solutions/security/packages/kbn-evals-suite-detection-emulation/kibana.jsonc
@@ -0,0 +1,7 @@
+{
+  "type": "functional-tests",
+  "id": "@kbn/evals-suite-detection-emulation",
+  "owner": "@elastic/security-defend-workflows",
+  "group": "security",
+  "visibility": "private"
+}
diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-detection-emulation/moon.yml b/x-pack/solutions/security/packages/kbn-evals-suite-detection-emulation/moon.yml
@@ -0,0 +1,34 @@
+# This file is generated by the @kbn/moon package. Any manual edits will be erased!
+#  To extend this, write your extensions/overrides to 'moon.extend.yml'
+#  then regenerate this file with: 'node scripts/regenerate_moon_projects.js --update --filter @kbn/evals-suite-detection-emulation'
+
+$schema: https://moonrepo.dev/schemas/project.json
+id: '@kbn/evals-suite-detection-emulation'
+layer: unknown
+owners:
+  defaultOwner: '@elastic/security-defend-workflows'
+toolchains:
+  default: node
+language: typescript
+project:
+  title: '@kbn/evals-suite-detection-emulation'
+  description: Moon project for @kbn/evals-suite-detection-emulation
+  channel: ''
+  owner: '@elastic/security-defend-workflows'
+  sourceRoot: x-pack/solutions/security/packages/kbn-evals-suite-detection-emulation
+dependsOn:
+  - '@kbn/evals'
+  - '@kbn/agent-builder-common'
+  - '@kbn/tooling-log'
+  - '@kbn/core-http-browser'
+tags:
+  - functional-tests
+  - package
+  - prod
+  - group-security
+  - private
+fileGroups:
+  src:
+    - '**/*.ts'
+    - '!target/**/*'
+tasks: {}
diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-detection-emulation/package.json b/x-pack/solutions/security/packages/kbn-evals-suite-detection-emulation/package.json
@@ -0,0 +1,6 @@
+{
+  "name": "@kbn/evals-suite-detection-emulation",
+  "private": true,
+  "version": "1.0.0",
+  "license": "Elastic License 2.0"
+}
diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-detection-emulation/playwright.config.ts b/x-pack/solutions/security/packages/kbn-evals-suite-detection-emulation/playwright.config.ts
@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { createPlaywrightEvalsConfig } from '@kbn/evals';
+
+export default createPlaywrightEvalsConfig({
+  testDir: `${__dirname}/evals`,
+  timeout: 30 * 60_000,
+});
diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-detection-emulation/tsconfig.json b/x-pack/solutions/security/packages/kbn-evals-suite-detection-emulation/tsconfig.json
@@ -0,0 +1,14 @@
+{
+  "extends": "@kbn/tsconfig-base/tsconfig.json",
+  "compilerOptions": {
+    "outDir": "target/types"
+  },
+  "include": ["**/*.ts"],
+  "exclude": ["target/**/*"],
+  "kbn_references": [
+    "@kbn/evals",
+    "@kbn/agent-builder-common",
+    "@kbn/tooling-log",
+    "@kbn/core-http-browser"
+  ]
+}
diff --git a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/skills/detection_emulation/evals/.eslintrc.js b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/skills/detection_emulation/evals/.eslintrc.js
diff --git a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/skills/detection_emulation/run_command_tools.test.ts b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/skills/detection_emulation/run_command_tools.test.ts
@@ -211,14 +211,15 @@ describe('per-family runEmulationCommand tools', () => {
         name: 'runNetworkCommand',
         schema: createRunNetworkCommandTool(createMockDeps()).schema,
         command: 'isolate',
+        parameters: undefined,
       },
       {
         name: 'runExecutionCommand',
         schema: createRunExecutionCommandTool(createMockDeps()).schema,
         command: 'execute',
-        parameters: { command: 'whoami' },
+        parameters: { command: 'whoami' } as Record<string, unknown> | undefined,
       },
-    ] as const;
+    ];
 
     it.each(tools)(
       '$name accepts exactly MAX_ENDPOINT_FANOUT endpointIds',
@@ -308,17 +309,29 @@ describe('per-family runEmulationCommand tools', () => {
       }
     );
 
+    // The four per-family tools each have a discriminated `command` literal
+    // union, so `tool.confirmation.getConfirmation` is typed per-family. When
+    // `it.each` iterates the heterogenous `cases` union, TS intersects the
+    // four signatures and `toolParams` collapses to `never`. Cast the callback
+    // through `unknown` to a single shape that accepts any toolParams object;
+    // the runtime contract is identical (all four implementations accept the
+    // same property set, only the `command` enum differs per-family).
+    type GetConfirmationFn = (args: {
+      toolParams: Record<string, unknown>;
+    }) => Promise<{ color: string; title: string; message: string }>;
+
     it.each(cases)(
       '$name renders a destructive-coloured confirmation for [$destructiveCommand]',
       async ({ tool, destructiveCommand }) => {
-        const result = await tool.confirmation!.getConfirmation!({
+        const getConfirmation = tool.confirmation!.getConfirmation as unknown as GetConfirmationFn;
+        const result = await getConfirmation({
           toolParams: {
             emulationId: 'em-42',
             agentType: 'endpoint',
             endpointIds: ['host-a'],
             command: destructiveCommand,
             parameters: {},
-          } as Record<string, unknown>,
+          },
         });
         expect(result.color).toBe('danger');
         expect(result.title).toContain(destructiveCommand);
@@ -330,14 +343,15 @@ describe('per-family runEmulationCommand tools', () => {
     it.each(cases)(
       '$name renders a warning-coloured confirmation for the non-destructive [$nonDestructiveCommand]',
       async ({ tool, nonDestructiveCommand }) => {
-        const result = await tool.confirmation!.getConfirmation!({
+        const getConfirmation = tool.confirmation!.getConfirmation as unknown as GetConfirmationFn;
+        const result = await getConfirmation({
           toolParams: {
             emulationId: 'em-42',
             agentType: 'endpoint',
             endpointIds: ['host-a', 'host-b'],
             command: nonDestructiveCommand,
             parameters: {},
-          } as Record<string, unknown>,
+          },
         });
         expect(result.color).toBe('warning');
         expect(result.title).toContain(nonDestructiveCommand);
diff --git a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/skills/detection_emulation/validate_rule_tool.ts b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/skills/detection_emulation/validate_rule_tool.ts
@@ -512,7 +512,7 @@ Use this tool when the user asks to validate, test, score, or confirm whether a
                       'Another real_execution scenario is already in flight for this Kibana space. Concurrent real_execution scenarios are not allowed.',
                     reason: 'concurrency_exceeded',
                     rule_id: ruleId,
-                    inflight_scenario_fingerprint: concurrencyResult.inflightFingerprint,
+                    inflight_scenario_fingerprint: concurrencyResult.inflightScenarioFingerprint,
                     retry_after_seconds: concurrencyResult.retryAfterSeconds,
                     likely_cause:
                       'Another real_execution scenario is currently in flight for this space. Wait for it to complete or retry after the suggested interval.',
diff --git a/x-pack/solutions/security/plugins/security_solution/tsconfig.json b/x-pack/solutions/security/plugins/security_solution/tsconfig.json
@@ -21,8 +21,7 @@
   "exclude": [
     "target/**/*",
     "**/cypress/**",
-    "public/management/cypress.config.ts",
-    "server/agent_builder/skills/detection_emulation/evals/*.spec.ts"
+    "public/management/cypress.config.ts"
   ],
   "kbn_references": [
     "@kbn/setup-node-env",
diff --git a/yarn.lock b/yarn.lock
@@ -6906,6 +6906,10 @@
   version "0.0.0"
   uid ""
 
+"@kbn/evals-suite-detection-emulation@link:x-pack/solutions/security/packages/kbn-evals-suite-detection-emulation":
+  version "0.0.0"
+  uid ""
+
 "@kbn/evals-suite-endpoint@link:x-pack/solutions/security/packages/kbn-evals-suite-endpoint":
   version "0.0.0"
   uid ""
