From 3059eb650c3938d829540356b1424a618e9a405f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20W=C3=A5lstedt?= Date: Sun, 24 May 2026 22:00:59 +0200 Subject: [PATCH 01/14] feat(security): bulk add alerts to chat Adds a "Add to chat" bulk action to the Kibana alerts table toolbar, allowing analysts to select multiple security alerts and send them to the Agent Builder AI chat in a single click. - security.alerts attachment type with server-side ES fetch (space-scoped) - alertsToAttachmentGroup helper chunks selections into batches of 20 - Bulk action wired across all 4 surfaces: Alerts page, Cases, Rule Details, and Attack Discovery - disableOnQuery: true prevents use with "Select all N" query mode - alert_count EBT telemetry event - Scout E2E test and kbn-evals-suite-security-alert-triage eval package Co-Authored-By: Claude Sonnet 4.6 --- .buildkite/pipelines/evals/evals.suites.json | 8 + .github/CODEOWNERS | 1 + package.json | 1 + tsconfig.base.json | 2 + .../components/alerts_data_grid.tsx | 12 +- .../hooks/use_bulk_actions.test.tsx | 94 ++++- .../alerts-table/hooks/use_bulk_actions.ts | 61 ++- .../response-ops/alerts-table/translations.ts | 4 + .../shared/response-ops/alerts-table/types.ts | 38 ++ .../.eslintrc.js | 21 + .../.gitignore | 1 + .../evals/alert_triage_quality.spec.ts | 266 +++++++++++++ .../evals/bulk_alerts_attachment_read.spec.ts | 215 +++++++++++ .../kibana.jsonc | 7 + .../moon.yml | 35 ++ .../package.json | 6 + .../playwright.config.ts | 13 + .../src/evaluate.ts | 8 + .../src/synthetic_alerts.ts | 361 ++++++++++++++++++ .../tsconfig.json | 16 + .../test/page_objects/alerts_table.ts | 6 + .../security_solution/common/constants.ts | 2 + .../plugins/security_solution/moon.yml | 1 + .../agent_builder/attachment_types/index.ts | 21 + .../agent_builder/components/prompts.ts | 2 + .../public/agent_builder/helpers.test.tsx | 73 +++- .../public/agent_builder/helpers.tsx | 37 +- .../hooks/use_bulk_add_to_chat_config.ts | 34 ++ .../hooks/use_report_add_to_chat.test.ts | 25 ++ .../hooks/use_report_add_to_chat.ts | 13 +- .../tabs/alerts_tab/ease/table.tsx | 21 +- .../ease/table_bulk_add_to_chat.test.tsx | 99 +++++ .../public/cases/.eslintrc.js | 7 + .../public/cases/components/ease/table.tsx | 21 +- .../ease/table_bulk_add_to_chat.test.tsx | 96 +++++ .../components/alert_summary/table/table.tsx | 19 +- .../table/table_bulk_add_to_chat.test.tsx | 94 +++++ .../components/alerts_table/index.tsx | 23 +- .../table_bulk_add_to_chat.test.tsx | 180 +++++++++ .../agent_builder/attachments/alerts.test.ts | 261 +++++++++++++ .../agent_builder/attachments/alerts.ts | 129 +++++++ .../attachments/register_attachments.ts | 10 +- .../entity_risk_score_tool.ts | 40 +- .../tools/get_alerts_by_id.test.ts | 93 +++++ .../agent_builder/tools/get_alerts_by_id.ts | 51 +++ .../security_solution/server/plugin.ts | 4 +- .../test/scout/ui/fixtures/index.ts | 67 ++++ .../bulk_add_alerts_to_chat.spec.ts | 201 ++++++++++ .../plugins/security_solution/tsconfig.json | 1 + yarn.lock | 4 + 50 files changed, 2749 insertions(+), 56 deletions(-) create mode 100644 x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/.eslintrc.js create mode 100644 x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/.gitignore create mode 100644 x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/evals/alert_triage_quality.spec.ts create mode 100644 x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/evals/bulk_alerts_attachment_read.spec.ts create mode 100644 x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/kibana.jsonc create mode 100644 x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/moon.yml create mode 100644 x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/package.json create mode 100644 x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/playwright.config.ts create mode 100644 x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/src/evaluate.ts create mode 100644 x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/src/synthetic_alerts.ts create mode 100644 x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/tsconfig.json create mode 100644 x-pack/solutions/security/plugins/security_solution/public/agent_builder/hooks/use_bulk_add_to_chat_config.ts create mode 100644 x-pack/solutions/security/plugins/security_solution/public/attack_discovery/pages/results/attack_discovery_panel/tabs/alerts_tab/ease/table_bulk_add_to_chat.test.tsx create mode 100644 x-pack/solutions/security/plugins/security_solution/public/cases/components/ease/table_bulk_add_to_chat.test.tsx create mode 100644 x-pack/solutions/security/plugins/security_solution/public/detections/components/alert_summary/table/table_bulk_add_to_chat.test.tsx create mode 100644 x-pack/solutions/security/plugins/security_solution/public/detections/components/alerts_table/table_bulk_add_to_chat.test.tsx create mode 100644 x-pack/solutions/security/plugins/security_solution/server/agent_builder/attachments/alerts.test.ts create mode 100644 x-pack/solutions/security/plugins/security_solution/server/agent_builder/attachments/alerts.ts create mode 100644 x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/get_alerts_by_id.test.ts create mode 100644 x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/get_alerts_by_id.ts create mode 100644 x-pack/solutions/security/plugins/security_solution/test/scout/ui/fixtures/index.ts create mode 100644 x-pack/solutions/security/plugins/security_solution/test/scout/ui/parallel_tests/investigations/bulk_add_alerts_to_chat.spec.ts diff --git a/.buildkite/pipelines/evals/evals.suites.json b/.buildkite/pipelines/evals/evals.suites.json index 6396660b2e8f0..49b9c5f727c18 100644 --- a/.buildkite/pipelines/evals/evals.suites.json +++ b/.buildkite/pipelines/evals/evals.suites.json @@ -251,6 +251,14 @@ "eis/openai-gpt-5.2", "eis/openai-gpt-5.4" ] + }, + { + "id": "security-alert-triage", + "name": "Security Alert Triage", + "slackChannel": "#security-generative-ai-evals", + "configPath": "x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/playwright.config.ts", + "tags": ["security", "alert-triage"], + "ciLabels": ["evals:security-alert-triage"] } ] } diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 4e288a602193d..0c892f679350f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1341,6 +1341,7 @@ x-pack/solutions/security/packages/kbn-evals-suite-entity-analytics @elastic/sec x-pack/solutions/security/packages/kbn-evals-suite-lead-generation @elastic/security-entity-analytics x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance @elastic/security-defend-workflows x-pack/solutions/security/packages/kbn-evals-suite-security-ai-rules @elastic/security-detection-engine +x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage @elastic/security-threat-hunting x-pack/solutions/security/packages/kbn-evals-suite-security-automatic-migrations @elastic/security-threat-hunting x-pack/solutions/security/packages/kbn-evals-suite-security-esql-generation-regression @elastic/security-detection-platform x-pack/solutions/security/packages/kbn-scout-security @elastic/appex-qa @elastic/security-engineering-productivity diff --git a/package.json b/package.json index f8cf9bbd01738..3485576a2445d 100644 --- a/package.json +++ b/package.json @@ -1756,6 +1756,7 @@ "@kbn/evals-suite-observability-ai": "link:x-pack/solutions/observability/packages/kbn-evals-suite-observability-ai", "@kbn/evals-suite-pci-compliance": "link:x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance", "@kbn/evals-suite-security-ai-rules": "link:x-pack/solutions/security/packages/kbn-evals-suite-security-ai-rules", + "@kbn/evals-suite-security-alert-triage": "link:x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage", "@kbn/evals-suite-security-automatic-migrations": "link:x-pack/solutions/security/packages/kbn-evals-suite-security-automatic-migrations", "@kbn/evals-suite-security-esql-generation-regression": "link:x-pack/solutions/security/packages/kbn-evals-suite-security-esql-generation-regression", "@kbn/evals-suite-significant-events": "link:x-pack/platform/packages/shared/kbn-evals-suite-significant-events", diff --git a/tsconfig.base.json b/tsconfig.base.json index 2c4654b964e01..a23a172a244b2 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -1240,6 +1240,8 @@ "@kbn/evals-suite-pci-compliance/*": ["x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance/*"], "@kbn/evals-suite-security-ai-rules": ["x-pack/solutions/security/packages/kbn-evals-suite-security-ai-rules"], "@kbn/evals-suite-security-ai-rules/*": ["x-pack/solutions/security/packages/kbn-evals-suite-security-ai-rules/*"], + "@kbn/evals-suite-security-alert-triage": ["x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage"], + "@kbn/evals-suite-security-alert-triage/*": ["x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/*"], "@kbn/evals-suite-security-automatic-migrations": ["x-pack/solutions/security/packages/kbn-evals-suite-security-automatic-migrations"], "@kbn/evals-suite-security-automatic-migrations/*": ["x-pack/solutions/security/packages/kbn-evals-suite-security-automatic-migrations/*"], "@kbn/evals-suite-security-esql-generation-regression": ["x-pack/solutions/security/packages/kbn-evals-suite-security-esql-generation-regression"], diff --git a/x-pack/platform/packages/shared/response-ops/alerts-table/components/alerts_data_grid.tsx b/x-pack/platform/packages/shared/response-ops/alerts-table/components/alerts_data_grid.tsx index aa09676100076..9662d56e60f87 100644 --- a/x-pack/platform/packages/shared/response-ops/alerts-table/components/alerts_data_grid.tsx +++ b/x-pack/platform/packages/shared/response-ops/alerts-table/components/alerts_data_grid.tsx @@ -80,6 +80,7 @@ export const AlertsDataGrid = typedMemo( cellActionsOptions, pageSizeOptions = DEFAULT_PAGE_SIZE_OPTIONS, height, + bulkAddToChatConfig, ...euiDataGridProps } = props; const { @@ -98,7 +99,14 @@ export const AlertsDataGrid = typedMemo( refresh: refreshQueries, columns, dataGridRef, - services: { http, notifications, application, cases: casesService, settings }, + services: { + http, + notifications, + application, + cases: casesService, + agentBuilder: agentBuilderService, + settings, + }, } = renderContext; const { colorMode, euiTheme } = useEuiTheme(); @@ -126,6 +134,8 @@ export const AlertsDataGrid = typedMemo( notifications, application, casesService, + agentBuilderService, + bulkAddToChatConfig, }); const refresh = useCallback(() => { diff --git a/x-pack/platform/packages/shared/response-ops/alerts-table/hooks/use_bulk_actions.test.tsx b/x-pack/platform/packages/shared/response-ops/alerts-table/hooks/use_bulk_actions.test.tsx index 5fda08070ad11..fab5a941a9692 100644 --- a/x-pack/platform/packages/shared/response-ops/alerts-table/hooks/use_bulk_actions.test.tsx +++ b/x-pack/platform/packages/shared/response-ops/alerts-table/hooks/use_bulk_actions.test.tsx @@ -15,12 +15,13 @@ import { AlertsQueryContext } from '@kbn/alerts-ui-shared/src/common/contexts/al import { useBulkActions, useBulkAddToCaseActions, + useBulkAddToChatActions, useBulkUntrackActions, useBulkMuteActions, } from './use_bulk_actions'; import { createCasesServiceMock } from '../mocks/cases.mock'; import { BulkActionsVerbs, type PublicAlertsDataGridProps } from '../types'; -import type { AdditionalContext, RenderContext } from '../types'; +import type { AdditionalContext, OpenChatService, RenderContext, TimelineItem } from '../types'; import { useAlertsTableContext } from '../contexts/alerts_table_context'; import { createPartialObjectMock, testQueryClientConfig } from '../utils/test'; import { applicationServiceMock } from '@kbn/core-application-browser-mocks'; @@ -427,6 +428,97 @@ describe('bulk action hooks', () => { }); }); + describe('useBulkAddToChatActions', () => { + const mockOpenChat = jest.fn(); + const agentBuilderService: OpenChatService = { openChat: mockOpenChat }; + const mockAttachments = [{ type: 'security.alerts', data: { alertIds: ['id1'] } }]; + const convertAlertToAttachment = jest.fn().mockReturnValue(mockAttachments); + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('returns empty array when agentBuilderService is not provided', () => { + const { result } = renderHook( + () => + useBulkAddToChatActions({ + bulkAddToChatConfig: { convertAlertToAttachment }, + }), + { wrapper } + ); + + expect(result.current).toEqual([]); + }); + + it('returns empty array when bulkAddToChatConfig is not provided', () => { + const { result } = renderHook(() => useBulkAddToChatActions({ agentBuilderService }), { + wrapper, + }); + + expect(result.current).toEqual([]); + }); + + it('returns the add-to-chat action when both service and config are provided', () => { + const { result } = renderHook( + () => + useBulkAddToChatActions({ + agentBuilderService, + bulkAddToChatConfig: { convertAlertToAttachment }, + }), + { wrapper } + ); + + expect(result.current).toHaveLength(1); + expect(result.current[0].key).toBe('bulk-add-to-chat'); + expect(result.current[0]['data-test-subj']).toBe('bulk-add-to-chat'); + }); + + it('calls openChat with converted attachments when the action is clicked', () => { + const alerts: TimelineItem[] = [ + { _id: 'id1', _index: 'idx', data: [], ecs: { _id: 'id1', _index: 'idx' } }, + ]; + + const { result } = renderHook( + () => + useBulkAddToChatActions({ + agentBuilderService, + bulkAddToChatConfig: { convertAlertToAttachment }, + }), + { wrapper } + ); + + result.current[0].onClick(alerts); + + expect(convertAlertToAttachment).toHaveBeenCalledWith(alerts); + expect(mockOpenChat).toHaveBeenCalledWith({ + autoSendInitialMessage: false, + newConversation: true, + initialMessage: undefined, + attachments: mockAttachments, + }); + }); + + it('passes initialMessage to openChat', () => { + const { result } = renderHook( + () => + useBulkAddToChatActions({ + agentBuilderService, + bulkAddToChatConfig: { + convertAlertToAttachment, + initialMessage: 'Please triage these alerts.', + }, + }), + { wrapper } + ); + + result.current[0].onClick([]); + + expect(mockOpenChat).toHaveBeenCalledWith( + expect.objectContaining({ initialMessage: 'Please triage these alerts.' }) + ); + }); + }); + describe('useBulkUntrackActions', () => { beforeEach(() => { jest.clearAllMocks(); diff --git a/x-pack/platform/packages/shared/response-ops/alerts-table/hooks/use_bulk_actions.ts b/x-pack/platform/packages/shared/response-ops/alerts-table/hooks/use_bulk_actions.ts index 39fe32a9c20da..2129d3d6a8077 100644 --- a/x-pack/platform/packages/shared/response-ops/alerts-table/hooks/use_bulk_actions.ts +++ b/x-pack/platform/packages/shared/response-ops/alerts-table/hooks/use_bulk_actions.ts @@ -23,12 +23,15 @@ import type { BulkActionsReducerAction, TimelineItem, BulkEditTagsFlyoutState, + BulkAddToChatConfig, + OpenChatService, } from '../types'; import { BulkActionsVerbs } from '../types'; import type { CasesService, PublicAlertsDataGridProps } from '../types'; import { ADD_TO_EXISTING_CASE, ADD_TO_NEW_CASE, + ADD_TO_CHAT, ALERTS_ALREADY_ATTACHED_TO_CASE, EDIT_TAGS, MARK_AS_UNTRACKED, @@ -49,6 +52,8 @@ interface BulkActionsProps { hideBulkActions?: boolean; application: ApplicationStart; casesService?: CasesService; + agentBuilderService?: OpenChatService; + bulkAddToChatConfig?: BulkAddToChatConfig; http: HttpStart; notifications: NotificationsStart; } @@ -408,6 +413,45 @@ export const useBulkMuteActions = ({ ); }; +export const useBulkAddToChatActions = ({ + agentBuilderService, + bulkAddToChatConfig, +}: { + agentBuilderService?: OpenChatService; + bulkAddToChatConfig?: BulkAddToChatConfig; +}) => { + const { convertAlertToAttachment, initialMessage, onAddedToChat } = bulkAddToChatConfig ?? {}; + + const onAddToChatClick = useCallback( + (alerts?: TimelineItem[]) => { + if (!agentBuilderService || !convertAlertToAttachment) return; + const items = alerts ?? []; + agentBuilderService.openChat({ + autoSendInitialMessage: false, + newConversation: true, + initialMessage, + attachments: convertAlertToAttachment(items), + }); + onAddedToChat?.(items.length); + }, + [agentBuilderService, convertAlertToAttachment, initialMessage, onAddedToChat] + ); + + return useMemo(() => { + if (!agentBuilderService || !convertAlertToAttachment) return []; + return [ + { + label: ADD_TO_CHAT, + key: 'bulk-add-to-chat', + disableOnQuery: true, + disabledLabel: ADD_TO_CHAT, + 'data-test-subj': 'bulk-add-to-chat', + onClick: onAddToChatClick, + }, + ]; + }, [agentBuilderService, convertAlertToAttachment, onAddToChatClick]); +}; + const EMPTY_BULK_ACTIONS_CONFIG: BulkActionsPanelConfig[] = []; export function useBulkActions({ @@ -422,6 +466,8 @@ export function useBulkActions({ notifications, application, casesService, + agentBuilderService, + bulkAddToChatConfig, }: BulkActionsProps): UseBulkActions { const { bulkActionsStore: [bulkActionsState, updateBulkActionsState], @@ -491,16 +537,29 @@ export function useBulkActions({ }, ]; }, [tagsAction, application?.capabilities]); + const addToChatActions = useBulkAddToChatActions({ + agentBuilderService, + bulkAddToChatConfig, + }); const initialItems = useMemo(() => { const isSiem = ruleTypeIds?.some(isSiemRuleType); return [ ...caseBulkActions, + ...(agentBuilderService ? addToChatActions : []), ...(isSiem ? [] : untrackBulkActions), ...(isSiem ? [] : tagsBulkActions), ...(isSiem ? [] : muteBulkActions), ]; - }, [caseBulkActions, ruleTypeIds, untrackBulkActions, tagsBulkActions, muteBulkActions]); + }, [ + caseBulkActions, + ruleTypeIds, + untrackBulkActions, + tagsBulkActions, + muteBulkActions, + addToChatActions, + agentBuilderService, + ]); const bulkActions = useMemo(() => { if (hideBulkActions) { diff --git a/x-pack/platform/packages/shared/response-ops/alerts-table/translations.ts b/x-pack/platform/packages/shared/response-ops/alerts-table/translations.ts index b6fa2cf898044..dd4ebe19614b9 100644 --- a/x-pack/platform/packages/shared/response-ops/alerts-table/translations.ts +++ b/x-pack/platform/packages/shared/response-ops/alerts-table/translations.ts @@ -251,6 +251,10 @@ export const ADD_TO_EXISTING_CASE = i18n.translate( } ); +export const ADD_TO_CHAT = i18n.translate('xpack.responseOpsAlertsTable.actions.addChat', { + defaultMessage: 'Add to chat', +}); + export const ADD_TO_NEW_CASE = i18n.translate('xpack.responseOpsAlertsTable.actions.addToNewCase', { defaultMessage: 'Add to new case', }); diff --git a/x-pack/platform/packages/shared/response-ops/alerts-table/types.ts b/x-pack/platform/packages/shared/response-ops/alerts-table/types.ts index 88be90e464716..7b186ec2884e4 100644 --- a/x-pack/platform/packages/shared/response-ops/alerts-table/types.ts +++ b/x-pack/platform/packages/shared/response-ops/alerts-table/types.ts @@ -43,6 +43,34 @@ import type { SetRequired } from 'type-fest'; import type { MaintenanceWindow } from '@kbn/maintenance-windows-plugin/common'; import type { FieldFormatsStart } from '@kbn/field-formats-plugin/public'; import type { DataPublicPluginStart } from '@kbn/data-plugin/public'; + +/** + * A single conversation attachment or a group of attachments. + * Defined structurally here to avoid a compile-time dependency on agent-builder packages. + */ +export type ConversationAttachmentInput = + | { type: string; data?: unknown; hidden?: boolean } + | { type: 'group'; id: string; label: string; items: Array<{ type: string; data?: unknown }> }; + +/** + * Minimal structural interface for the chat service required by the alerts table. + * Using a local structural type avoids a compile-time dependency on agent-builder packages + * from this shared platform package. + */ +export interface OpenChatService { + openChat(options?: { + attachments?: ConversationAttachmentInput[]; + newConversation?: boolean; + initialMessage?: string; + autoSendInitialMessage?: boolean; + }): void; +} + +export interface BulkAddToChatConfig { + convertAlertToAttachment: (alerts: TimelineItem[]) => ConversationAttachmentInput[]; + initialMessage?: string; + onAddedToChat?: (itemCount: number) => void; +} import type { FieldBrowserOptions } from '@kbn/response-ops-alerts-fields-browser'; import type { MutedAlerts } from '@kbn/response-ops-alerts-apis/types'; import type { NotificationsStart } from '@kbn/core-notifications-browser'; @@ -411,6 +439,14 @@ export interface AlertsTableProps { renderContext: RenderContext; + bulkAddToChatConfig?: BulkAddToChatConfig; additionalToolbarControls?: ReactNode; pageSizeOptions?: number[]; leadingControlColumns?: EuiDataGridControlColumn[]; diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/.eslintrc.js b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/.eslintrc.js new file mode 100644 index 0000000000000..f4b09c92e63e9 --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/.eslintrc.js @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** @type {import('eslint').Linter.Config} */ +module.exports = { + overrides: [ + // This package is Node-only (Playwright eval suite). Allow Node.js builtins. + { + files: ['**/*.{js,mjs,ts,tsx}'], + rules: { + 'import/no-nodejs-modules': 'off', + // functional-tests packages intentionally import from test-helper packages (e.g. @kbn/evals, @kbn/scout) + '@kbn/imports/no_boundary_crossing': 'off', + }, + }, + ], +}; diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/.gitignore b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/.gitignore new file mode 100644 index 0000000000000..51511d1f8f36f --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/.gitignore @@ -0,0 +1 @@ +test-results/ diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/evals/alert_triage_quality.spec.ts b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/evals/alert_triage_quality.spec.ts new file mode 100644 index 0000000000000..4bf1179f6a960 --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/evals/alert_triage_quality.spec.ts @@ -0,0 +1,266 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * Quality eval: does the LLM produce a useful triage given real alert data? + * + * Two scenarios run in inline mode (1 batch each → ≤5 attachments → LLM sees + * all data directly without needing attachment_read). This isolates output + * quality from the attachment-read compliance already covered by + * bulk_alerts_attachment_read.spec.ts. + * + * Synthetic alerts are indexed into .alerts-security.alerts-default in + * beforeAll and cleaned up in afterAll so no external snapshot is required. + */ + +import { tags } from '@kbn/scout'; +import type { EsClient } from '@kbn/scout'; +import { + selectEvaluators, + type DefaultEvaluators, + type EvalsExecutorClient, + type ExperimentTask, + type TaskOutput, +} from '@kbn/evals'; +import { agentBuilderDefaultAgentId } from '@kbn/agent-builder-common'; +import type { ToolingLog } from '@kbn/tooling-log'; +import type { HttpHandler } from '@kbn/core/public'; +import { evaluate as base } from '../src/evaluate'; +import { + ALL_TRIAGE_EVAL_ALERTS, + ENTITY_CORRELATION_IDS, + PRIORITY_TRIAGE_IDS, +} from '../src/synthetic_alerts'; + +// ── Constants ───────────────────────────────────────────────────────────────── + +const ALERTS_INDEX = '.alerts-security.alerts-default'; +const ALERTS_BATCH_MAX_SIZE = 20; + +const toAlertAttachments = (ids: string[]) => { + const batches = []; + for (let i = 0; i < ids.length; i += ALERTS_BATCH_MAX_SIZE) { + batches.push({ + type: 'security.alerts', + data: { alertIds: ids.slice(i, i + ALERTS_BATCH_MAX_SIZE) }, + }); + } + return batches; +}; + +// ── Types ───────────────────────────────────────────────────────────────────── + +interface TriageEvalExample { + input: { question: string }; + output: { expected: string }; + metadata?: { + attachments?: Array<{ type: string; data?: unknown }>; + }; +} + +// ── Fixture factory ─────────────────────────────────────────────────────────── + +function createEvaluateTriageQuality({ + fetch, + connector, + evaluators, + executorClient, + log, +}: { + fetch: HttpHandler; + connector: { id: string }; + evaluators: DefaultEvaluators; + executorClient: EvalsExecutorClient; + log: ToolingLog; +}) { + return async function evaluateTriageQuality({ + dataset, + criteria, + }: { + dataset: { name: string; description: string; examples: TriageEvalExample[] }; + criteria: string[]; + }) { + const task: ExperimentTask = async ({ input, metadata }) => { + log.info(`Running triage eval task: "${input.question.slice(0, 80)}..."`); + + const { attachments = [] } = metadata ?? {}; + + const raw = (await fetch('/api/agent_builder/converse', { + method: 'POST', + version: '2023-10-31', + body: JSON.stringify({ + agent_id: agentBuilderDefaultAgentId, + connector_id: connector.id, + input: input.question, + attachments, + }), + })) as { + conversation_id: string; + trace_id?: string; + steps: unknown[]; + response: { message: string }; + }; + + return { + errors: [], + messages: [{ message: input.question }, raw.response], + steps: raw.steps ?? [], + traceId: raw.trace_id, + }; + }; + + const selectedEvaluators = selectEvaluators([ + evaluators.criteria(criteria), + ...Object.values(evaluators.traceBasedEvaluators), + ]); + + await executorClient.runExperiment( + { + dataset: { + name: dataset.name, + description: dataset.description, + examples: dataset.examples, + }, + task, + }, + selectedEvaluators + ); + }; +} + +// ── Evaluate fixture extension ───────────────────────────────────────────────── + +type EvaluateTriageQuality = ReturnType; + +const evaluate = base.extend<{ evaluateTriageQuality: EvaluateTriageQuality }, {}>({ + evaluateTriageQuality: [ + ({ fetch, connector, evaluators, executorClient, log }, use) => { + use(createEvaluateTriageQuality({ fetch, connector, evaluators, executorClient, log })); + }, + { scope: 'test' }, + ], +}); + +// ── Eval spec ───────────────────────────────────────────────────────────────── + +evaluate.describe( + 'Security Alert Triage — output quality', + { tag: [...tags.serverless.security.complete, ...tags.serverless.security.ease] }, + () => { + evaluate.beforeAll(async ({ esClient, log }: { esClient: EsClient; log: ToolingLog }) => { + log.info( + `Indexing ${ALL_TRIAGE_EVAL_ALERTS.length} synthetic triage eval alerts into ${ALERTS_INDEX}` + ); + + const response = await esClient.bulk({ + refresh: 'wait_for', + operations: ALL_TRIAGE_EVAL_ALERTS.flatMap(({ id, doc }) => [ + { index: { _index: ALERTS_INDEX, _id: id } }, + doc, + ]), + }); + + if (response.errors) { + const failed = response.items.filter((item) => item.index?.error); + throw new Error( + `Failed to set up triage eval alerts: ${JSON.stringify( + failed.map((f) => f.index?.error) + )}` + ); + } + + log.info(`Successfully indexed ${ALL_TRIAGE_EVAL_ALERTS.length} synthetic alerts`); + }); + + evaluate.afterAll(async ({ esClient, log }: { esClient: EsClient; log: ToolingLog }) => { + const idsToDelete = ALL_TRIAGE_EVAL_ALERTS.map((alert) => alert.id); + log.info(`Deleting ${idsToDelete.length} synthetic triage eval alerts`); + + await esClient.deleteByQuery({ + index: ALERTS_INDEX, + query: { ids: { values: idsToDelete } }, + }); + }); + + evaluate( + 'Priority triage — structured output with severity prioritisation', + async ({ evaluateTriageQuality }) => { + await evaluateTriageQuality({ + dataset: { + name: 'agent builder: security-alert-triage-priority', + description: + 'Tests that the LLM prioritises critical and high severity alerts, mentions ' + + 'specific host names from the alert data, and provides actionable recommendations. ' + + '5 alerts on distinct hosts with severities: critical, high, high, medium, medium.', + examples: [ + { + input: { + question: + 'Triage these security alerts. Identify the most urgent issues, ' + + 'note key entities (hosts, users), and recommend immediate actions.', + }, + output: { + expected: + 'The response should prioritise the critical PowerShell alert on workstation-42, ' + + 'address the high-severity credential-dumping and network-scanning alerts, ' + + 'list the affected host names, and provide specific remediation steps for each.', + }, + metadata: { + attachments: toAlertAttachments(PRIORITY_TRIAGE_IDS), + }, + }, + ], + }, + criteria: [ + 'The response treats the critical severity alert as the highest priority item', + 'The response mentions specific host names present in the alert data', + 'The response includes concrete remediation steps, not just a description of the alerts', + 'The response distinguishes between urgent and lower-priority items', + ], + }); + } + ); + + evaluate( + 'Entity correlation — identifies a host targeted by multiple alerts', + async ({ evaluateTriageQuality }) => { + await evaluateTriageQuality({ + dataset: { + name: 'agent builder: security-alert-triage-entity-correlation', + description: + 'Tests that the LLM identifies the host appearing in 4 of 8 alerts (web-server-01) ' + + 'as a correlated or higher-risk target. The 4 web-server-01 alerts show a clear ' + + 'attack progression: SQL injection → web shell → reverse shell → privilege escalation.', + examples: [ + { + input: { + question: + 'Review these security alerts. Are any hosts or users targeted ' + + 'repeatedly? What does that pattern suggest about the threat?', + }, + output: { + expected: + 'The response identifies web-server-01 as the host appearing in multiple alerts ' + + 'and flags the sequence of alerts as evidence of an active intrusion or attack ' + + 'progression on that host, recommending immediate investigation or containment.', + }, + metadata: { + attachments: toAlertAttachments(ENTITY_CORRELATION_IDS), + }, + }, + ], + }, + criteria: [ + 'The response identifies that one specific host appears in multiple alerts', + 'The response names web-server-01 as the repeatedly targeted host', + 'The response treats the repeated targeting of a single host as a higher-risk indicator than the single-alert hosts', + ], + }); + } + ); + } +); diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/evals/bulk_alerts_attachment_read.spec.ts b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/evals/bulk_alerts_attachment_read.spec.ts new file mode 100644 index 0000000000000..980e6b7784862 --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/evals/bulk_alerts_attachment_read.spec.ts @@ -0,0 +1,215 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * Deterministic eval: attachment_read compliance for bulk alert batches. + * + * When >5 attachments are present the framework switches to summary mode, showing + * only metadata and instructing the LLM to call `attachment_read(attachment_id)` + * per batch before answering. This eval verifies the LLM follows that instruction. + * + * No real alert data is required. Synthetic IDs trigger summary mode and the + * format() function returns "not found" for each — enough for the LLM to see + * the attachment_read prompt and act on it. + */ + +import { tags } from '@kbn/scout'; +import { + getToolCallSteps, + selectEvaluators, + type DefaultEvaluators, + type EvalsExecutorClient, + type ExperimentTask, + type TaskOutput, +} from '@kbn/evals'; +import { attachmentTools, agentBuilderDefaultAgentId } from '@kbn/agent-builder-common'; +import type { EsClient } from '@kbn/scout'; +import type { ToolingLog } from '@kbn/tooling-log'; +import type { HttpHandler } from '@kbn/core/public'; +import { evaluate as base } from '../src/evaluate'; + +// ── Dataset constants ────────────────────────────────────────────────────────── + +const BATCH_SIZE = 20; +// 6 batches > framework inline threshold of 5 → summary mode +const BATCH_COUNT = 6; + +const alertBatches: Array<{ alertIds: string[] }> = Array.from( + { length: BATCH_COUNT }, + (_, batchIndex) => ({ + alertIds: Array.from( + { length: BATCH_SIZE }, + (__, alertIndex) => + `eval-alert-b${batchIndex.toString().padStart(2, '0')}-${alertIndex + .toString() + .padStart(2, '0')}` + ), + }) +); + +// ── Types ────────────────────────────────────────────────────────────────────── + +interface AlertEvalExample { + input: { question: string }; + output: { expected: string }; + metadata?: { + attachments?: Array<{ type: string; data?: unknown }>; + expectedAttachmentReads?: number; + }; +} + +// ── Fixture factory ──────────────────────────────────────────────────────────── + +function createEvaluateAlertBatches({ + fetch, + connector, + evaluators, + executorClient, + traceEsClient, + log, +}: { + fetch: HttpHandler; + connector: { id: string }; + evaluators: DefaultEvaluators; + executorClient: EvalsExecutorClient; + traceEsClient: EsClient; + log: ToolingLog; +}) { + return async function evaluateAlertBatches({ + dataset: { name, description, examples }, + }: { + dataset: { name: string; description: string; examples: AlertEvalExample[] }; + }) { + const task: ExperimentTask = async ({ input, metadata }) => { + log.info('Calling converse with alert batch attachments'); + + const attachments = metadata?.attachments ?? []; + + const raw = (await fetch('/api/agent_builder/converse', { + method: 'POST', + version: '2023-10-31', + body: JSON.stringify({ + agent_id: agentBuilderDefaultAgentId, + connector_id: connector.id, + input: input.question, + attachments, + }), + })) as { + conversation_id: string; + trace_id?: string; + steps: unknown[]; + response: { message: string }; + }; + + return { + errors: [], + messages: [{ message: input.question }, raw.response], + steps: raw.steps ?? [], + traceId: raw.trace_id, + }; + }; + + const selectedEvaluators = selectEvaluators([ + { + name: 'AttachmentReadCompliance', + kind: 'CODE' as const, + evaluate: async ({ output, metadata }) => { + const expected = + typeof metadata?.expectedAttachmentReads === 'number' + ? metadata.expectedAttachmentReads + : 0; + + if (!expected) { + return { score: 1 }; + } + + const toolCalls = getToolCallSteps(output as TaskOutput); + const readCalls = toolCalls.filter((t) => t.tool_id === attachmentTools.read); + + return { + score: Math.min(1, readCalls.length / expected), + metadata: { readCallCount: readCalls.length, expected }, + }; + }, + }, + ...Object.values(evaluators.traceBasedEvaluators), + ]); + + await executorClient.runExperiment( + { dataset: { name, description, examples }, task }, + selectedEvaluators + ); + }; +} + +// ── Evaluate fixture extension ───────────────────────────────────────────────── + +type EvaluateAlertBatches = ReturnType; + +const evaluate = base.extend<{ evaluateAlertBatches: EvaluateAlertBatches }, {}>({ + evaluateAlertBatches: [ + ({ fetch, connector, evaluators, executorClient, traceEsClient, log }, use) => { + use( + createEvaluateAlertBatches({ + fetch, + connector, + evaluators, + executorClient, + traceEsClient, + log, + }) + ); + }, + { scope: 'test' }, + ], +}); + +// ── Eval spec ───────────────────────────────────────────────────────────────── + +evaluate.describe( + 'Security Alerts — attachment_read compliance', + { tag: [...tags.serverless.security.complete, ...tags.serverless.security.ease] }, + () => { + evaluate( + `LLM calls attachment_read for all ${BATCH_COUNT} batches when in summary mode`, + async ({ evaluateAlertBatches }) => { + await evaluateAlertBatches({ + dataset: { + name: 'agent builder: security-bulk-alerts-attachment-read', + description: + `Validates the LLM calls attachment_read for each of ${BATCH_COUNT} alert batches ` + + 'when the framework is in summary mode (>5 active attachments). ' + + 'Synthetic alert IDs are used — format() returns "not found" per ID, which is ' + + 'sufficient to populate the attachment and trigger summary mode. ' + + 'Score = attachment_read call count / expected batch count.', + examples: [ + { + input: { + question: + 'I have added security alerts for your review. Please triage them — ' + + 'identify patterns, shared entities, and escalation indicators, then recommend actions.', + }, + output: { + expected: + `I will call attachment_read ${BATCH_COUNT} times to read each alert batch, ` + + 'then synthesize patterns, identify shared entities, and recommend triage actions.', + }, + metadata: { + attachments: alertBatches.map(({ alertIds }) => ({ + type: 'security.alerts', + data: { alertIds }, + })), + expectedAttachmentReads: BATCH_COUNT, + }, + }, + ], + }, + }); + } + ); + } +); diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/kibana.jsonc b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/kibana.jsonc new file mode 100644 index 0000000000000..0eafc009ed836 --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/kibana.jsonc @@ -0,0 +1,7 @@ +{ + "type": "functional-tests", + "id": "@kbn/evals-suite-security-alert-triage", + "owner": "@elastic/security-threat-hunting", + "group": "security", + "visibility": "private" +} diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/moon.yml b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/moon.yml new file mode 100644 index 0000000000000..1fb706dfb8ab2 --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/moon.yml @@ -0,0 +1,35 @@ +# This file is generated by the @kbn/moon package. Any manual edits will be erased! +# To extend this, write your extensions/overrides to 'moon.extend.yml' +# then regenerate this file with: 'node scripts/regenerate_moon_projects.js --update --filter @kbn/evals-suite-security-alert-triage' + +$schema: https://moonrepo.dev/schemas/project.json +id: '@kbn/evals-suite-security-alert-triage' +layer: unknown +owners: + defaultOwner: '@elastic/security-threat-hunting' +toolchains: + default: node +language: typescript +project: + title: '@kbn/evals-suite-security-alert-triage' + description: Moon project for @kbn/evals-suite-security-alert-triage + channel: '' + owner: '@elastic/security-threat-hunting' + sourceRoot: x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage +dependsOn: + - '@kbn/evals' + - '@kbn/scout' + - '@kbn/tooling-log' + - '@kbn/agent-builder-common' + - '@kbn/core' +tags: + - functional-tests + - package + - prod + - group-security + - private +fileGroups: + src: + - '**/*.ts' + - '!target/**/*' +tasks: {} diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/package.json b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/package.json new file mode 100644 index 0000000000000..287b74645c92f --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/package.json @@ -0,0 +1,6 @@ +{ + "name": "@kbn/evals-suite-security-alert-triage", + "private": true, + "version": "1.0.0", + "license": "Elastic License 2.0" +} diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/playwright.config.ts b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/playwright.config.ts new file mode 100644 index 0000000000000..137c2134aff28 --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/playwright.config.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createPlaywrightEvalsConfig } from '@kbn/evals'; + +export default createPlaywrightEvalsConfig({ + testDir: `${__dirname}/evals`, + timeout: 30 * 60_000, +}); diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/src/evaluate.ts b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/src/evaluate.ts new file mode 100644 index 0000000000000..d2c5362248548 --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/src/evaluate.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { evaluate, tags, selectEvaluators } from '@kbn/evals'; diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/src/synthetic_alerts.ts b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/src/synthetic_alerts.ts new file mode 100644 index 0000000000000..59cddd38f81fd --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/src/synthetic_alerts.ts @@ -0,0 +1,361 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * Synthetic alert documents for triage quality evals. + * + * Two groups of 100 alerts each (5 batches of 20 → inline mode): + * + * PRIORITY_TRIAGE (100 alerts) + * - 1 critical at index 49 (buried mid-list) + * - 10 high (indices 0–9) + * - 30 medium (indices 10–39) + * - 59 low (indices 40–99 excluding 49) + * Tests whether the LLM finds the critical needle and prioritises correctly. + * + * ENTITY_CORRELATION (100 alerts, 17 hosts) + * - 20 alerts on web-server-01 (showing an attack progression) + * - 80 alerts spread 5-per-host across 16 other hosts + * Tests whether the LLM identifies the most targeted host. + * + * Fields are a subset of ESSENTIAL_ALERT_FIELDS (common/constants.ts:585) — the + * same fields the attachment agent description instructs the LLM to extract. + */ + +interface SyntheticAlert { + id: string; + doc: Record; +} + +const ts = (offsetMinutes: number): string => { + const d = new Date('2024-06-15T09:00:00Z'); + d.setMinutes(d.getMinutes() + offsetMinutes); + return d.toISOString(); +}; + +// ── Lookup tables ───────────────────────────────────────────────────────────── + +const RULES_BY_SEVERITY: Record< + string, + Array<{ rule: string; tactic: string; technique: string }> +> = { + critical: [ + { + rule: 'Suspicious PowerShell with Encoded Commands', + tactic: 'Execution', + technique: 'PowerShell', + }, + { + rule: 'Memory Injection via Process Hollowing', + tactic: 'Defense Evasion', + technique: 'Process Injection', + }, + { + rule: 'Ransomware File Encryption Activity', + tactic: 'Impact', + technique: 'Data Encrypted for Impact', + }, + ], + high: [ + { + rule: 'Credential Dumping via LSASS Access', + tactic: 'Credential Access', + technique: 'OS Credential Dumping', + }, + { + rule: 'Lateral Movement via Admin Share', + tactic: 'Lateral Movement', + technique: 'SMB/Windows Admin Shares', + }, + { + rule: 'Kerberoasting Detected', + tactic: 'Credential Access', + technique: 'Steal or Forge Kerberos Tickets', + }, + { + rule: 'Web Shell File Created', + tactic: 'Persistence', + technique: 'Server Software Component', + }, + { + rule: 'Privilege Escalation via SUID Binary', + tactic: 'Privilege Escalation', + technique: 'Abuse Elevation Control Mechanism', + }, + ], + medium: [ + { + rule: 'Scheduled Task in Unusual Location', + tactic: 'Persistence', + technique: 'Scheduled Task/Job', + }, + { + rule: 'Suspicious Registry Key Modified', + tactic: 'Defense Evasion', + technique: 'Modify Registry', + }, + { + rule: 'Network Scanning Activity Detected', + tactic: 'Discovery', + technique: 'Network Service Discovery', + }, + { + rule: 'Unusual DNS Query Volume', + tactic: 'Command and Control', + technique: 'Application Layer Protocol', + }, + { + rule: 'Outbound Connection to Threat Intelligence Blocklist IP', + tactic: 'Command and Control', + technique: 'Application Layer Protocol', + }, + ], + low: [ + { + rule: 'Software Installed by Non-Admin User', + tactic: 'Persistence', + technique: 'Boot or Logon Autostart Execution', + }, + { + rule: 'USB Drive Inserted on Endpoint', + tactic: 'Initial Access', + technique: 'Replication Through Removable Media', + }, + { rule: 'Browser Extension Installed', tactic: 'Persistence', technique: 'Browser Extensions' }, + { rule: 'Phishing Link Clicked', tactic: 'Initial Access', technique: 'Phishing' }, + { + rule: 'Screensaver Executable Modified', + tactic: 'Persistence', + technique: 'Boot or Logon Autostart Execution', + }, + ], +}; + +const RISK_SCORE_BY_SEVERITY: Record = { + critical: 99, + high: 73, + medium: 47, + low: 18, +}; + +const OTHER_HOSTS = [ + 'workstation-42', + 'laptop-finance-07', + 'desktop-hr-03', + 'server-prod-01', + 'workstation-dev-11', + 'laptop-legal-02', + 'desktop-exec-01', + 'server-backup-01', + 'workstation-support-08', + 'laptop-remote-06', + 'desktop-reception-04', + 'db-server-02', + 'fileserver-01', + 'workstation-mktg-05', + 'laptop-sales-09', + 'app-server-03', +]; + +const USERS = [ + 'alice.smith', + 'bob.jones', + 'carol.white', + 'david.brown', + 'eve.chen', + 'frank.miller', + 'grace.lee', + 'henry.wilson', + 'iris.wang', + 'james.taylor', +]; + +// ── Priority triage group (100 alerts) ─────────────────────────────────────── + +export const PRIORITY_TRIAGE_IDS: string[] = Array.from( + { length: 100 }, + (_, i) => `triage-eval-priority-${i.toString().padStart(3, '0')}` +); + +// Severity distribution: 1 critical (at index 49), 9 high, 30 medium, 60 low. +// The critical alert is intentionally buried mid-list to test whether the LLM +// scans all alerts rather than stopping after the first few. +const severityAt = (i: number): string => { + if (i === 49) return 'critical'; + if (i < 10) return 'high'; + if (i < 40) return 'medium'; + return 'low'; +}; + +const PRIORITY_TRIAGE_ALERTS: SyntheticAlert[] = PRIORITY_TRIAGE_IDS.map((id, i) => { + const severity = severityAt(i); + const rules = RULES_BY_SEVERITY[severity]; + const { rule, tactic, technique } = rules[i % rules.length]; + const host = OTHER_HOSTS[i % OTHER_HOSTS.length]; + const user = USERS[i % USERS.length]; + return { + id, + doc: { + '@timestamp': ts(i), + 'kibana.alert.rule.name': rule, + 'kibana.alert.severity': severity, + 'kibana.alert.risk_score': RISK_SCORE_BY_SEVERITY[severity], + 'kibana.alert.uuid': id, + 'kibana.alert.workflow_status': 'open', + 'kibana.alert.rule.category': 'Custom Query', + 'kibana.alert.reason': `${rule} triggered on ${host}`, + 'host.name': host, + 'user.name': user, + 'threat.tactic.name': tactic, + 'threat.technique.name': technique, + }, + }; +}); + +// ── Entity correlation group (100 alerts) ───────────────────────────────────── + +export const ENTITY_CORRELATION_IDS: string[] = Array.from( + { length: 100 }, + (_, i) => `triage-eval-correlation-${i.toString().padStart(3, '0')}` +); + +// web-server-01 attack progression (20 alerts, indices chosen to scatter them +// throughout the list so the LLM must synthesise across the full batch set). +const WEB_SERVER_INDICES = new Set([ + 2, 7, 12, 17, 22, 27, 32, 37, 42, 47, 52, 57, 62, 67, 72, 77, 82, 87, 92, 97, +]); + +const WEB_SERVER_ATTACKS = [ + { + rule: 'SQL Injection Attempt Detected', + tactic: 'Initial Access', + technique: 'Exploit Public-Facing Application', + user: 'www-data', + }, + { + rule: 'Web Shell File Created', + tactic: 'Persistence', + technique: 'Server Software Component', + user: 'www-data', + }, + { + rule: 'Unusual Outbound Data Transfer from Web Process', + tactic: 'Exfiltration', + technique: 'Exfiltration Over C2 Channel', + user: 'www-data', + }, + { + rule: 'Reverse Shell Spawned from Web Process', + tactic: 'Execution', + technique: 'Command and Scripting Interpreter', + user: 'root', + }, + { + rule: 'Privilege Escalation via SUID Binary', + tactic: 'Privilege Escalation', + technique: 'Abuse Elevation Control Mechanism', + user: 'root', + }, + { + rule: 'Credential Dumping via /etc/shadow Read', + tactic: 'Credential Access', + technique: 'OS Credential Dumping', + user: 'root', + }, + { + rule: 'Lateral Movement Attempt from Web Server', + tactic: 'Lateral Movement', + technique: 'Remote Services', + user: 'root', + }, + { + rule: 'Suspicious Cron Job Installed', + tactic: 'Persistence', + technique: 'Scheduled Task/Job', + user: 'root', + }, + { + rule: 'Port Scanning from Compromised Web Server', + tactic: 'Discovery', + technique: 'Network Service Discovery', + user: 'root', + }, + { + rule: 'Data Staged in Temp Directory', + tactic: 'Collection', + technique: 'Data Staged', + user: 'root', + }, +]; + +let webServerAttackIndex = 0; + +const ENTITY_CORRELATION_ALERTS: SyntheticAlert[] = ENTITY_CORRELATION_IDS.map((id, i) => { + const isWebServer = WEB_SERVER_INDICES.has(i); + const otherHostIndex = + Math.floor((i - (WEB_SERVER_INDICES.size > 0 ? 0 : 0)) / 1) % OTHER_HOSTS.length; + + if (isWebServer) { + const attack = WEB_SERVER_ATTACKS[webServerAttackIndex % WEB_SERVER_ATTACKS.length]; + webServerAttackIndex++; + const severity = + webServerAttackIndex % 5 === 0 + ? 'critical' + : webServerAttackIndex % 3 === 0 + ? 'high' + : 'high'; + return { + id, + doc: { + '@timestamp': ts(i), + 'kibana.alert.rule.name': attack.rule, + 'kibana.alert.severity': severity, + 'kibana.alert.risk_score': RISK_SCORE_BY_SEVERITY[severity], + 'kibana.alert.uuid': id, + 'kibana.alert.workflow_status': 'open', + 'kibana.alert.rule.category': 'Custom Query', + 'kibana.alert.reason': `${attack.rule} detected on web-server-01`, + 'host.name': 'web-server-01', + 'user.name': attack.user, + 'threat.tactic.name': attack.tactic, + 'threat.technique.name': attack.technique, + }, + }; + } + + const severity = i % 7 === 0 ? 'high' : i % 3 === 0 ? 'medium' : 'low'; + const rules = RULES_BY_SEVERITY[severity]; + const { rule, tactic, technique } = rules[i % rules.length]; + const host = OTHER_HOSTS[otherHostIndex % OTHER_HOSTS.length]; + const user = USERS[i % USERS.length]; + return { + id, + doc: { + '@timestamp': ts(i), + 'kibana.alert.rule.name': rule, + 'kibana.alert.severity': severity, + 'kibana.alert.risk_score': RISK_SCORE_BY_SEVERITY[severity], + 'kibana.alert.uuid': id, + 'kibana.alert.workflow_status': 'open', + 'kibana.alert.rule.category': 'Custom Query', + 'kibana.alert.reason': `${rule} triggered on ${host}`, + 'host.name': host, + 'user.name': user, + 'threat.tactic.name': tactic, + 'threat.technique.name': technique, + }, + }; +}); + +// ── Combined exports ────────────────────────────────────────────────────────── + +export const ALL_TRIAGE_EVAL_IDS: string[] = [...PRIORITY_TRIAGE_IDS, ...ENTITY_CORRELATION_IDS]; + +export const ALL_TRIAGE_EVAL_ALERTS: SyntheticAlert[] = [ + ...PRIORITY_TRIAGE_ALERTS, + ...ENTITY_CORRELATION_ALERTS, +]; diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/tsconfig.json b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/tsconfig.json new file mode 100644 index 0000000000000..7e9867c6dff1e --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/tsconfig.json @@ -0,0 +1,16 @@ +{ + "extends": "@kbn/tsconfig-base/tsconfig.json", + "compilerOptions": { + "outDir": "target/types", + "types": ["jest", "node"] + }, + "include": ["**/*.ts"], + "exclude": ["target/**/*"], + "kbn_references": [ + "@kbn/evals", + "@kbn/scout", + "@kbn/tooling-log", + "@kbn/agent-builder-common", + "@kbn/core" + ] +} diff --git a/x-pack/solutions/security/packages/kbn-scout-security/src/playwright/fixtures/test/page_objects/alerts_table.ts b/x-pack/solutions/security/packages/kbn-scout-security/src/playwright/fixtures/test/page_objects/alerts_table.ts index 07f3668a65bfe..701a3903f429c 100644 --- a/x-pack/solutions/security/packages/kbn-scout-security/src/playwright/fixtures/test/page_objects/alerts_table.ts +++ b/x-pack/solutions/security/packages/kbn-scout-security/src/playwright/fixtures/test/page_objects/alerts_table.ts @@ -22,6 +22,9 @@ export class AlertsTablePage { public bulkRunWorkflowMenuItem: Locator; public bulkWorkflowPanel: Locator; public selectedShowBulkActionsButton: Locator; + public bulkAddToChatMenuItem: Locator; + public selectAllAlertsButton: Locator; + public bulkActionsHeaderCheckbox: Locator; constructor(private readonly page: ScoutPage) { this.detectionsAlertsWrapper = this.page.testSubj.locator('alerts-by-rule-table'); @@ -37,6 +40,9 @@ export class AlertsTablePage { this.selectedShowBulkActionsButton = this.page.testSubj.locator( 'selectedShowBulkActionsButton' ); + this.bulkAddToChatMenuItem = this.page.testSubj.locator('bulk-add-to-chat'); + this.selectAllAlertsButton = this.page.testSubj.locator('selectAllAlertsButton'); + this.bulkActionsHeaderCheckbox = this.page.testSubj.locator('bulk-actions-header'); } async navigate() { diff --git a/x-pack/solutions/security/plugins/security_solution/common/constants.ts b/x-pack/solutions/security/plugins/security_solution/common/constants.ts index 6f96505793076..82b6292871b50 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/constants.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/constants.ts @@ -55,6 +55,7 @@ export const DEFAULT_REFRESH_RATE_INTERVAL = 'timepicker:refreshIntervalDefaults export const DEFAULT_APP_TIME_RANGE = 'securitySolution:timeDefaults' as const; export const DEFAULT_APP_REFRESH_INTERVAL = 'securitySolution:refreshIntervalDefaults' as const; export const DEFAULT_ALERTS_INDEX = '.alerts-security.alerts' as const; +export const ALERTS_BATCH_MAX_SIZE = 20 as const; export const DEFAULT_SIGNALS_INDEX = '.siem-signals' as const; export const DEFAULT_PREVIEW_INDEX = '.preview.alerts-security.alerts' as const; export const DEFAULT_LISTS_INDEX = '.lists' as const; @@ -724,6 +725,7 @@ export const ESSENTIAL_ALERT_FIELDS: string[] = [ export enum SecurityAgentBuilderAttachments { alert = 'security.alert', + alerts = 'security.alerts', entity = 'security.entity', entityAnalyticsDashboard = 'security.entity_analytics_dashboard', rule = 'security.rule', diff --git a/x-pack/solutions/security/plugins/security_solution/moon.yml b/x-pack/solutions/security/plugins/security_solution/moon.yml index a910398fa59d2..84a9282e75eae 100644 --- a/x-pack/solutions/security/plugins/security_solution/moon.yml +++ b/x-pack/solutions/security/plugins/security_solution/moon.yml @@ -237,6 +237,7 @@ dependsOn: - '@kbn/inference-common' - '@kbn/inference-langchain' - '@kbn/scout-security' + - '@kbn/ftr-llm-proxy' - '@kbn/custom-icons' - '@kbn/security-plugin-types-common' - '@kbn/management-settings-ids' diff --git a/x-pack/solutions/security/plugins/security_solution/public/agent_builder/attachment_types/index.ts b/x-pack/solutions/security/plugins/security_solution/public/agent_builder/attachment_types/index.ts index a9505c83d0f9b..1a65dd1003905 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/agent_builder/attachment_types/index.ts +++ b/x-pack/solutions/security/plugins/security_solution/public/agent_builder/attachment_types/index.ts @@ -41,6 +41,11 @@ const ALERT_ATTACHMENT_CONFIG: AttachmentTypeConfig = { icon: 'bell', }; +const ALERTS_DEFAULT_LABEL = i18n.translate( + 'xpack.securitySolution.agentBuilder.attachments.alerts.label', + { defaultMessage: 'Security Alerts' } +); + const createAttachmentTypeConfig = (defaultLabel: string, icon: string) => ({ getLabel: (attachment: UnknownAttachmentWithLabel) => { const attachmentLabel = attachment?.data?.attachmentLabel; @@ -63,6 +68,22 @@ export const registerAttachmentUiDefinitions = (attachments: AttachmentServiceSt ALERT_ATTACHMENT_CONFIG.type, createAttachmentTypeConfig(ALERT_ATTACHMENT_CONFIG.label, ALERT_ATTACHMENT_CONFIG.icon) ); + + attachments.addAttachmentType>( + SecurityAgentBuilderAttachments.alerts, + { + getLabel: (attachment) => { + const count = attachment.data?.alertIds?.length ?? 0; + return count > 0 + ? i18n.translate('xpack.securitySolution.agentBuilder.attachments.alerts.countLabel', { + defaultMessage: '{count} {count, plural, one {Alert} other {Alerts}}', + values: { count }, + }) + : ALERTS_DEFAULT_LABEL; + }, + getIcon: () => 'bell', + } + ); }; /** diff --git a/x-pack/solutions/security/plugins/security_solution/public/agent_builder/components/prompts.ts b/x-pack/solutions/security/plugins/security_solution/public/agent_builder/components/prompts.ts index b2f26376b1751..e67d48385b821 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/agent_builder/components/prompts.ts +++ b/x-pack/solutions/security/plugins/security_solution/public/agent_builder/components/prompts.ts @@ -130,3 +130,5 @@ When Suggesting Improvements: - Put each suggested field value in a separate, copyable code block with a clear label - If you need more context, ask concise follow-up questions `; + +export const BULK_ALERTS_ATTACHMENT_PROMPT = `Triage and prioritize these security alerts`; diff --git a/x-pack/solutions/security/plugins/security_solution/public/agent_builder/helpers.test.tsx b/x-pack/solutions/security/plugins/security_solution/public/agent_builder/helpers.test.tsx index 5189d56149489..5cf294c1df9c5 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/agent_builder/helpers.test.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/agent_builder/helpers.test.tsx @@ -6,7 +6,78 @@ */ import { ESSENTIAL_ALERT_FIELDS } from '../../common'; -import { stringifyEssentialAlertData } from './helpers'; +import { SecurityAgentBuilderAttachments } from '../../common/constants'; +import { alertsToAttachmentGroup, stringifyEssentialAlertData } from './helpers'; + +const makeItem = (id: string) => + ({ _id: id, data: [], ecs: { _id: id, _index: '' } } as unknown as Parameters< + typeof alertsToAttachmentGroup + >[0][number]); + +describe('alertsToAttachmentGroup', () => { + it('returns an AttachmentGroup with type "group"', () => { + const items = [makeItem('a')]; + const result = alertsToAttachmentGroup(items); + expect(result.type).toBe('group'); + expect(result.id).toBeDefined(); + expect(typeof result.id).toBe('string'); + }); + + it('returns a single item in items for ≤20 alerts', () => { + const items = Array.from({ length: 5 }, (_, i) => makeItem(`id-${i}`)); + const result = alertsToAttachmentGroup(items); + + expect(result.items).toHaveLength(1); + expect(result.items[0].hidden).toBeFalsy(); + expect(result.items[0].type).toBe(SecurityAgentBuilderAttachments.alerts); + }); + + it('returns N batches in items for >20 alerts', () => { + const items = Array.from({ length: 50 }, (_, i) => makeItem(`id-${i}`)); + const result = alertsToAttachmentGroup(items); + + expect(result.items).toHaveLength(3); + result.items.forEach((item) => { + expect(item.hidden).toBeFalsy(); + }); + }); + + it('uses plural label for multiple alerts', () => { + const items = Array.from({ length: 5 }, (_, i) => makeItem(`id-${i}`)); + expect(alertsToAttachmentGroup(items).label).toBe('5 Alerts'); + }); + + it('uses singular label for 1 alert', () => { + expect(alertsToAttachmentGroup([makeItem('x')]).label).toBe('1 Alert'); + }); + + it('each batch contains the correct alert IDs', () => { + const items = Array.from({ length: 25 }, (_, i) => makeItem(`id-${i}`)); + const result = alertsToAttachmentGroup(items); + + expect(result.items[0].data).toEqual({ alertIds: items.slice(0, 20).map((i) => i._id) }); + expect(result.items[1].data).toEqual({ alertIds: items.slice(20).map((i) => i._id) }); + }); + + it('items do not pre-stamp group_id or description (stamped by flattenAttachments)', () => { + const items = Array.from({ length: 25 }, (_, i) => makeItem(`id-${i}`)); + const result = alertsToAttachmentGroup(items); + + result.items.forEach((item) => { + expect(item).not.toHaveProperty('group_id'); + expect(item).not.toHaveProperty('description'); + expect(item.hidden).toBeFalsy(); + }); + }); + + it('group id differs between separate calls', () => { + const items = Array.from({ length: 5 }, (_, i) => makeItem(`id-${i}`)); + const first = alertsToAttachmentGroup(items); + const second = alertsToAttachmentGroup(items); + + expect(first.id).not.toBe(second.id); + }); +}); describe('stringifyEssentialAlertData', () => { it('filters to essential fields only', () => { diff --git a/x-pack/solutions/security/plugins/security_solution/public/agent_builder/helpers.tsx b/x-pack/solutions/security/plugins/security_solution/public/agent_builder/helpers.tsx index c5aeb306d502e..b8a97d2103078 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/agent_builder/helpers.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/agent_builder/helpers.tsx @@ -6,13 +6,40 @@ */ import { pick } from 'lodash'; +import { v4 as uuidv4 } from 'uuid'; +import type { AttachmentInput, AttachmentGroup } from '@kbn/agent-builder-common/attachments'; +import type { TimelineItem } from '@kbn/response-ops-alerts-table/types'; import { ESSENTIAL_ALERT_FIELDS } from '../../common'; +import { ALERTS_BATCH_MAX_SIZE, SecurityAgentBuilderAttachments } from '../../common/constants'; + +export type BulkAlertsAttachmentInput = AttachmentInput< + typeof SecurityAgentBuilderAttachments.alerts, + { alertIds: string[] } +>; + +const chunkAlerts = (alertItems: TimelineItem[]): BulkAlertsAttachmentInput[] => { + const batches: BulkAlertsAttachmentInput[] = []; + for (let i = 0; i < alertItems.length; i += ALERTS_BATCH_MAX_SIZE) { + batches.push({ + type: SecurityAgentBuilderAttachments.alerts, + data: { alertIds: alertItems.slice(i, i + ALERTS_BATCH_MAX_SIZE).map((a) => a._id) }, + }); + } + return batches; +}; /** - * Filters raw alert data to only include essential fields and stringifies the result. - * This reduces context window usage by keeping only the most relevant information. + * Converts alert items into a single AttachmentGroup whose items are the chunked batches. + * The group renders as one chip in the UI and is flattened to AttachmentInput[] at the + * serialization boundary before being sent to the server. */ -export const stringifyEssentialAlertData = (rawData: Record): string => { - return JSON.stringify(pick(rawData, ESSENTIAL_ALERT_FIELDS)); -}; +export const alertsToAttachmentGroup = (alertItems: TimelineItem[]): AttachmentGroup => ({ + type: 'group', + id: uuidv4(), + label: `${alertItems.length} Alert${alertItems.length !== 1 ? 's' : ''}`, + items: chunkAlerts(alertItems), +}); + +export const stringifyEssentialAlertData = (rawData: Record): string => + JSON.stringify(pick(rawData, ESSENTIAL_ALERT_FIELDS)); diff --git a/x-pack/solutions/security/plugins/security_solution/public/agent_builder/hooks/use_bulk_add_to_chat_config.ts b/x-pack/solutions/security/plugins/security_solution/public/agent_builder/hooks/use_bulk_add_to_chat_config.ts new file mode 100644 index 0000000000000..7647be4eba2cf --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/public/agent_builder/hooks/use_bulk_add_to_chat_config.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useCallback } from 'react'; +import type { BulkAddToChatConfig, TimelineItem } from '@kbn/response-ops-alerts-table/types'; +import { alertsToAttachmentGroup } from '../helpers'; +import { BULK_ALERTS_ATTACHMENT_PROMPT } from '../components/prompts'; +import { type BulkAlertPathway, useReportAddToChat } from './use_report_add_to_chat'; + +export const useBulkAddToChatConfig = (pathway: BulkAlertPathway): BulkAddToChatConfig => { + const reportAddToChat = useReportAddToChat(); + + const convertAlertToAttachment = useCallback( + (alertItems: TimelineItem[]) => [alertsToAttachmentGroup(alertItems)], + [] + ); + + const onAddedToChat = useCallback( + (itemCount: number) => { + reportAddToChat({ pathway, attachments: ['alert'], item_count: itemCount }); + }, + [pathway, reportAddToChat] + ); + + return { + convertAlertToAttachment, + initialMessage: BULK_ALERTS_ATTACHMENT_PROMPT, + onAddedToChat, + }; +}; diff --git a/x-pack/solutions/security/plugins/security_solution/public/agent_builder/hooks/use_report_add_to_chat.test.ts b/x-pack/solutions/security/plugins/security_solution/public/agent_builder/hooks/use_report_add_to_chat.test.ts index caebf87239fc3..ccfdb3cfed5bd 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/agent_builder/hooks/use_report_add_to_chat.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/public/agent_builder/hooks/use_report_add_to_chat.test.ts @@ -43,4 +43,29 @@ describe('useReportAddToChat', () => { expect(reportEvent).toHaveBeenCalledWith(AGENT_BUILDER_EVENT_TYPES.AddToChatClicked, payload); }); + + it('passes item_count through to reportEvent for bulk pathways', () => { + const reportEvent = jest.fn(); + mockUseKibana.mockReturnValue({ + services: { + telemetry: { + reportEvent, + }, + }, + }); + + const payload: AgentBuilderAddToChatTelemetry = { + pathway: 'bulk_alerts_alerts_page', + attachments: ['alert'], + item_count: 5, + }; + + const { result } = renderHook(() => useReportAddToChat()); + + act(() => { + result.current(payload); + }); + + expect(reportEvent).toHaveBeenCalledWith(AGENT_BUILDER_EVENT_TYPES.AddToChatClicked, payload); + }); }); diff --git a/x-pack/solutions/security/plugins/security_solution/public/agent_builder/hooks/use_report_add_to_chat.ts b/x-pack/solutions/security/plugins/security_solution/public/agent_builder/hooks/use_report_add_to_chat.ts index 487a3b5b7d65e..9c9c30a6e6eb2 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/agent_builder/hooks/use_report_add_to_chat.ts +++ b/x-pack/solutions/security/plugins/security_solution/public/agent_builder/hooks/use_report_add_to_chat.ts @@ -8,6 +8,14 @@ import { AGENT_BUILDER_EVENT_TYPES } from '@kbn/agent-builder-common'; import { useCallback } from 'react'; import { useKibana } from '../../common/lib/kibana'; + +export type BulkAlertPathway = + | 'bulk_alerts_alerts_page' + | 'bulk_alerts_rule_details' + | 'bulk_alerts_alert_summary' + | 'bulk_alerts_cases' + | 'bulk_alerts_attack_discovery'; + export interface AgentBuilderAddToChatTelemetry { /** * Pathway where "Add to Chat" was clicked @@ -29,11 +37,14 @@ export interface AgentBuilderAddToChatTelemetry { | 'attack_discovery_bottom' | 'attacks_page_group_summary' | 'attacks_page_group_take_action' - | 'attacks_page_flyout_take_action'; + | 'attacks_page_flyout_take_action' + | BulkAlertPathway; /** * Attachment type */ attachments?: Array<'alert' | 'entity' | 'rule'>; + /** Number of items added (for bulk add-to-chat actions) */ + item_count?: number; } /** diff --git a/x-pack/solutions/security/plugins/security_solution/public/attack_discovery/pages/results/attack_discovery_panel/tabs/alerts_tab/ease/table.tsx b/x-pack/solutions/security/plugins/security_solution/public/attack_discovery/pages/results/attack_discovery_panel/tabs/alerts_tab/ease/table.tsx index 91daa0792def3..2a4513d9674b7 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/attack_discovery/pages/results/attack_discovery_panel/tabs/alerts_tab/ease/table.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/attack_discovery/pages/results/attack_discovery_panel/tabs/alerts_tab/ease/table.tsx @@ -27,6 +27,7 @@ import { } from '../../../../../../../detections/components/alert_summary/table/table'; import { ActionsCell } from '../../../../../../../detections/components/alert_summary/table/actions_cell'; import { useKibana } from '../../../../../../../common/lib/kibana'; +import { useBulkAddToChatConfig } from '../../../../../../../agent_builder/hooks/use_bulk_add_to_chat_config'; import { CellValue } from '../../../../../../../detections/components/alert_summary/table/render_cell'; import { useAdditionalBulkActions } from '../../../../../../../detections/hooks/alert_summary/use_additional_bulk_actions'; @@ -56,6 +57,7 @@ export interface TableProps { export const Table = memo(({ dataView, id, packages, query }: TableProps) => { const { services: { + agentBuilder, application, cases, data, @@ -69,19 +71,33 @@ export const Table = memo(({ dataView, id, packages, query }: TableProps) => { } = useKibana(); const services = useMemo( () => ({ + agentBuilder, + application, cases, data, http, notifications, rendering, fieldFormats, - application, licensing, settings, }), - [application, cases, data, fieldFormats, http, licensing, notifications, rendering, settings] + [ + agentBuilder, + application, + cases, + data, + fieldFormats, + http, + licensing, + notifications, + rendering, + settings, + ] ); + const bulkAddToChatConfig = useBulkAddToChatConfig('bulk_alerts_attack_discovery'); + const browserFields = useBrowserFields(PageScope.alerts, dataView); const additionalContext: AdditionalTableContext = useMemo( @@ -122,6 +138,7 @@ export const Table = memo(({ dataView, id, packages, query }: TableProps) => { ruleTypeIds={RULE_TYPE_IDS} services={services} toolbarVisibility={TOOLBAR_VISIBILITY} + bulkAddToChatConfig={bulkAddToChatConfig} /> ); diff --git a/x-pack/solutions/security/plugins/security_solution/public/attack_discovery/pages/results/attack_discovery_panel/tabs/alerts_tab/ease/table_bulk_add_to_chat.test.tsx b/x-pack/solutions/security/plugins/security_solution/public/attack_discovery/pages/results/attack_discovery_panel/tabs/alerts_tab/ease/table_bulk_add_to_chat.test.tsx new file mode 100644 index 0000000000000..864f5b7488a6e --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/public/attack_discovery/pages/results/attack_discovery_panel/tabs/alerts_tab/ease/table_bulk_add_to_chat.test.tsx @@ -0,0 +1,99 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { render } from '@testing-library/react'; +import type { DataView } from '@kbn/data-views-plugin/common'; +import { createStubDataView } from '@kbn/data-views-plugin/common/data_views/data_view.stub'; +import type { PackageListItem } from '@kbn/fleet-plugin/common'; +import { installationStatuses } from '@kbn/fleet-plugin/common/constants'; +import { AlertsTable } from '@kbn/response-ops-alerts-table'; +import type { TimelineItem } from '@kbn/response-ops-alerts-table/types'; +import { TestProviders } from '../../../../../../../common/mock'; +import { BULK_ALERTS_ATTACHMENT_PROMPT } from '../../../../../../../agent_builder/components/prompts'; +import { alertsToAttachmentGroup } from '../../../../../../../agent_builder/helpers'; +import { useReportAddToChat } from '../../../../../../../agent_builder/hooks/use_report_add_to_chat'; +import { Table } from './table'; + +jest.mock('@kbn/response-ops-alerts-table', () => ({ + AlertsTable: jest.fn(() => null), +})); +jest.mock('../../../../../../../agent_builder/hooks/use_report_add_to_chat'); +jest.mock('../../../../../../../agent_builder/helpers', () => ({ + alertsToAttachmentGroup: jest.fn(() => []), +})); +jest.mock('../../../../../../../data_view_manager/hooks/use_browser_fields', () => ({ + useBrowserFields: jest.fn(() => ({})), +})); +jest.mock( + '../../../../../../../detections/hooks/alert_summary/use_additional_bulk_actions', + () => ({ + useAdditionalBulkActions: jest.fn(() => []), + }) +); + +const makeItem = (id: string): TimelineItem => + ({ _id: id, data: [], ecs: { _id: id, _index: '' } } as unknown as TimelineItem); + +const dataView: DataView = createStubDataView({ spec: {} }); +const packages: PackageListItem[] = [ + { + id: 'splunk', + icons: [{ src: 'icon.svg', path: 'mypath/icon.svg', type: 'image/svg+xml' }], + name: 'splunk', + status: installationStatuses.NotInstalled, + title: 'Splunk', + version: '0.1.0', + }, +]; +const id = 'attack-discovery-table-test'; +const query = { ids: { values: ['abc'] } }; + +describe('Attack Discovery Table — bulkAddToChatConfig', () => { + let mockReportAddToChat: jest.Mock; + + beforeEach(() => { + jest.clearAllMocks(); + mockReportAddToChat = jest.fn(); + (useReportAddToChat as jest.Mock).mockReturnValue(mockReportAddToChat); + }); + + const renderAndGetBulkConfig = () => { + render( + + + + ); + return (AlertsTable as jest.Mock).mock.calls[0][0].bulkAddToChatConfig; + }; + + it('passes BULK_ALERTS_ATTACHMENT_PROMPT as initialMessage', () => { + const { initialMessage } = renderAndGetBulkConfig(); + expect(initialMessage).toBe(BULK_ALERTS_ATTACHMENT_PROMPT); + }); + + it('calls reportAddToChat with bulk_alerts_attack_discovery pathway and alert_count', () => { + const { convertAlertToAttachment } = renderAndGetBulkConfig(); + const items = [makeItem('a'), makeItem('b')]; + convertAlertToAttachment(items); + expect(mockReportAddToChat).toHaveBeenCalledWith({ + pathway: 'bulk_alerts_attack_discovery', + attachments: ['alert'], + alert_count: 2, + }); + }); + + it('delegates to alertsToAttachmentGroup and returns its result', () => { + const mockGroup = { type: 'group', id: 'x', label: '1 Alert', items: [] }; + (alertsToAttachmentGroup as jest.Mock).mockReturnValueOnce(mockGroup); + const { convertAlertToAttachment } = renderAndGetBulkConfig(); + const items = [makeItem('a')]; + const result = convertAlertToAttachment(items); + expect(alertsToAttachmentGroup).toHaveBeenCalledWith(items); + expect(result).toEqual([mockGroup]); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/public/cases/.eslintrc.js b/x-pack/solutions/security/plugins/security_solution/public/cases/.eslintrc.js index 594d744e6422f..2ba444c3af3a8 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/cases/.eslintrc.js +++ b/x-pack/solutions/security/plugins/security_solution/public/cases/.eslintrc.js @@ -81,9 +81,16 @@ const RESTRICTED_IMPORTS_PATHS = [ }, ]; +// Strip GIT_DIR / GIT_WORK_TREE so the pre-commit hook's env vars don't +// cause `show-toplevel` to return __dirname instead of the repo root. +const gitEnv = { ...process.env }; +delete gitEnv.GIT_DIR; +delete gitEnv.GIT_WORK_TREE; + const ROOT_DIR = execSync('git rev-parse --show-toplevel', { encoding: 'utf8', cwd: __dirname, + env: gitEnv, }).trim(); const ROOT_CLIMB_STRING = path.relative(__dirname, ROOT_DIR); // i.e. '../../..' diff --git a/x-pack/solutions/security/plugins/security_solution/public/cases/components/ease/table.tsx b/x-pack/solutions/security/plugins/security_solution/public/cases/components/ease/table.tsx index 92500107fe6fe..11e49d6c6b713 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/cases/components/ease/table.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/cases/components/ease/table.tsx @@ -15,6 +15,7 @@ import type { } from '@kbn/response-ops-alerts-table/types'; import React, { memo, useCallback, useMemo, useRef } from 'react'; import { useKibana } from '../../../common/lib/kibana'; +import { useBulkAddToChatConfig } from '../../../agent_builder/hooks/use_bulk_add_to_chat_config'; import { ActionsCell } from '../../../detections/components/alert_summary/table/actions_cell'; import { CellValue } from '../../../detections/components/alert_summary/table/render_cell'; import { useBrowserFields } from '../../../data_view_manager/hooks/use_browser_fields'; @@ -63,6 +64,7 @@ export interface TableProps { export const Table = memo(({ dataView, id, onLoaded, packages, query }: TableProps) => { const { services: { + agentBuilder, application, cases, data, @@ -76,19 +78,33 @@ export const Table = memo(({ dataView, id, onLoaded, packages, query }: TablePro } = useKibana(); const services = useMemo( () => ({ + agentBuilder, + application, cases, data, http, notifications, rendering, fieldFormats, - application, licensing, settings, }), - [application, cases, data, fieldFormats, http, licensing, notifications, rendering, settings] + [ + agentBuilder, + application, + cases, + data, + fieldFormats, + http, + licensing, + notifications, + rendering, + settings, + ] ); + const bulkAddToChatConfig = useBulkAddToChatConfig('bulk_alerts_cases'); + const browserFields = useBrowserFields(PageScope.alerts, dataView); const additionalContext: AdditionalTableContext = useMemo(() => ({ packages }), [packages]); @@ -125,6 +141,7 @@ export const Table = memo(({ dataView, id, onLoaded, packages, query }: TablePro runtimeMappings={runtimeMappings} services={services} toolbarVisibility={TOOLBAR_VISIBILITY} + bulkAddToChatConfig={bulkAddToChatConfig} /> ); diff --git a/x-pack/solutions/security/plugins/security_solution/public/cases/components/ease/table_bulk_add_to_chat.test.tsx b/x-pack/solutions/security/plugins/security_solution/public/cases/components/ease/table_bulk_add_to_chat.test.tsx new file mode 100644 index 0000000000000..1ca2bef6f897e --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/public/cases/components/ease/table_bulk_add_to_chat.test.tsx @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { render } from '@testing-library/react'; +import type { DataView } from '@kbn/data-views-plugin/common'; +import { createStubDataView } from '@kbn/data-views-plugin/common/data_views/data_view.stub'; +import type { PackageListItem } from '@kbn/fleet-plugin/common'; +import { installationStatuses } from '@kbn/fleet-plugin/common/constants'; +import { AlertsTable } from '@kbn/response-ops-alerts-table'; +import type { TimelineItem } from '@kbn/response-ops-alerts-table/types'; +import { TestProviders } from '../../../common/mock'; +import { BULK_ALERTS_ATTACHMENT_PROMPT } from '../../../agent_builder/components/prompts'; +import { alertsToAttachmentGroup } from '../../../agent_builder/helpers'; +import { useReportAddToChat } from '../../../agent_builder/hooks/use_report_add_to_chat'; +import { Table } from './table'; + +jest.mock('@kbn/response-ops-alerts-table', () => ({ + AlertsTable: jest.fn(() => null), +})); +jest.mock('../../../agent_builder/hooks/use_report_add_to_chat'); +jest.mock('../../../agent_builder/helpers', () => ({ + alertsToAttachmentGroup: jest.fn(() => []), +})); +jest.mock('../../../data_view_manager/hooks/use_browser_fields', () => ({ + useBrowserFields: jest.fn(() => ({})), +})); +jest.mock('../../../detections/hooks/alert_summary/use_additional_bulk_actions', () => ({ + useAdditionalBulkActions: jest.fn(() => []), +})); + +const makeItem = (id: string): TimelineItem => + ({ _id: id, data: [], ecs: { _id: id, _index: '' } } as unknown as TimelineItem); + +const dataView: DataView = createStubDataView({ spec: {} }); +const packages: PackageListItem[] = [ + { + id: 'splunk', + icons: [{ src: 'icon.svg', path: 'mypath/icon.svg', type: 'image/svg+xml' }], + name: 'splunk', + status: installationStatuses.NotInstalled, + title: 'Splunk', + version: '0.1.0', + }, +]; +const id = 'cases-table-test'; +const query = { ids: { values: ['abc'] } }; + +describe('Cases Table — bulkAddToChatConfig', () => { + let mockReportAddToChat: jest.Mock; + + beforeEach(() => { + jest.clearAllMocks(); + mockReportAddToChat = jest.fn(); + (useReportAddToChat as jest.Mock).mockReturnValue(mockReportAddToChat); + }); + + const renderAndGetBulkConfig = () => { + render( + +
+ + ); + return (AlertsTable as jest.Mock).mock.calls[0][0].bulkAddToChatConfig; + }; + + it('passes BULK_ALERTS_ATTACHMENT_PROMPT as initialMessage', () => { + const { initialMessage } = renderAndGetBulkConfig(); + expect(initialMessage).toBe(BULK_ALERTS_ATTACHMENT_PROMPT); + }); + + it('calls reportAddToChat with bulk_alerts_cases pathway and alert_count', () => { + const { convertAlertToAttachment } = renderAndGetBulkConfig(); + const items = [makeItem('a'), makeItem('b')]; + convertAlertToAttachment(items); + expect(mockReportAddToChat).toHaveBeenCalledWith({ + pathway: 'bulk_alerts_cases', + attachments: ['alert'], + alert_count: 2, + }); + }); + + it('delegates to alertsToAttachmentGroup and returns its result', () => { + const mockGroup = { type: 'group', id: 'x', label: '1 Alert', items: [] }; + (alertsToAttachmentGroup as jest.Mock).mockReturnValueOnce(mockGroup); + const { convertAlertToAttachment } = renderAndGetBulkConfig(); + const items = [makeItem('a')]; + const result = convertAlertToAttachment(items); + expect(alertsToAttachmentGroup).toHaveBeenCalledWith(items); + expect(result).toEqual([mockGroup]); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/public/detections/components/alert_summary/table/table.tsx b/x-pack/solutions/security/plugins/security_solution/public/detections/components/alert_summary/table/table.tsx index 2fd7ecb7267af..7e332e3ababdf 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detections/components/alert_summary/table/table.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/detections/components/alert_summary/table/table.tsx @@ -37,6 +37,7 @@ import { useDeepEqualSelector } from '../../../../common/hooks/use_selector'; import { combineQueries } from '../../../../common/lib/kuery'; import { useKibana } from '../../../../common/lib/kibana'; import { CellValue } from './render_cell'; +import { useBulkAddToChatConfig } from '../../../../agent_builder/hooks/use_bulk_add_to_chat_config'; import { buildTimeRangeFilter } from '../../alerts_table/helpers'; import { useGlobalTime } from '../../../../common/containers/use_global_time'; @@ -134,6 +135,7 @@ export const Table = memo(({ dataView, groupingFilters, packages }: TableProps) const { services: { application, + agentBuilder, cases, data, fieldFormats, @@ -156,10 +158,24 @@ export const Table = memo(({ dataView, groupingFilters, packages }: TableProps) application, licensing, settings, + agentBuilder, }), - [application, cases, data, fieldFormats, http, licensing, notifications, rendering, settings] + [ + agentBuilder, + application, + cases, + data, + fieldFormats, + http, + licensing, + notifications, + rendering, + settings, + ] ); + const bulkAddToChatConfig = useBulkAddToChatConfig('bulk_alerts_alert_summary'); + const getGlobalFiltersSelector = useMemo(() => inputsSelectors.globalFiltersQuerySelector(), []); const globalFilters = useDeepEqualSelector(getGlobalFiltersSelector); @@ -245,6 +261,7 @@ export const Table = memo(({ dataView, groupingFilters, packages }: TableProps) ruleTypeIds={RULE_TYPE_IDS} services={services} toolbarVisibility={TOOLBAR_VISIBILITY} + bulkAddToChatConfig={bulkAddToChatConfig} /> ); diff --git a/x-pack/solutions/security/plugins/security_solution/public/detections/components/alert_summary/table/table_bulk_add_to_chat.test.tsx b/x-pack/solutions/security/plugins/security_solution/public/detections/components/alert_summary/table/table_bulk_add_to_chat.test.tsx new file mode 100644 index 0000000000000..182e70cc7c38c --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/public/detections/components/alert_summary/table/table_bulk_add_to_chat.test.tsx @@ -0,0 +1,94 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { render } from '@testing-library/react'; +import type { DataView } from '@kbn/data-views-plugin/common'; +import { createStubDataView } from '@kbn/data-views-plugin/common/data_views/data_view.stub'; +import type { PackageListItem } from '@kbn/fleet-plugin/common'; +import { installationStatuses } from '@kbn/fleet-plugin/common/constants'; +import { AlertsTable } from '@kbn/response-ops-alerts-table'; +import type { TimelineItem } from '@kbn/response-ops-alerts-table/types'; +import { TestProviders } from '../../../../common/mock'; +import { BULK_ALERTS_ATTACHMENT_PROMPT } from '../../../../agent_builder/components/prompts'; +import { alertsToAttachmentGroup } from '../../../../agent_builder/helpers'; +import { useReportAddToChat } from '../../../../agent_builder/hooks/use_report_add_to_chat'; +import { Table } from './table'; + +jest.mock('@kbn/response-ops-alerts-table', () => ({ + AlertsTable: jest.fn(() => null), +})); +jest.mock('../../../../agent_builder/hooks/use_report_add_to_chat'); +jest.mock('../../../../agent_builder/helpers', () => ({ + alertsToAttachmentGroup: jest.fn(() => []), +})); +jest.mock('../../../../data_view_manager/hooks/use_browser_fields', () => ({ + useBrowserFields: jest.fn(() => ({})), +})); +jest.mock('../../../hooks/alert_summary/use_additional_bulk_actions', () => ({ + useAdditionalBulkActions: jest.fn(() => []), +})); + +const makeItem = (id: string): TimelineItem => + ({ _id: id, data: [], ecs: { _id: id, _index: '' } } as unknown as TimelineItem); + +const dataView: DataView = createStubDataView({ spec: {} }); +const packages: PackageListItem[] = [ + { + id: 'splunk', + icons: [{ src: 'icon.svg', path: 'mypath/icon.svg', type: 'image/svg+xml' }], + name: 'splunk', + status: installationStatuses.NotInstalled, + title: 'Splunk', + version: '0.1.0', + }, +]; + +describe('Alert Summary Table — bulkAddToChatConfig', () => { + let mockReportAddToChat: jest.Mock; + + beforeEach(() => { + jest.clearAllMocks(); + mockReportAddToChat = jest.fn(); + (useReportAddToChat as jest.Mock).mockReturnValue(mockReportAddToChat); + }); + + const renderAndGetBulkConfig = () => { + render( + +
+ + ); + return (AlertsTable as jest.Mock).mock.calls[0][0].bulkAddToChatConfig; + }; + + it('passes BULK_ALERTS_ATTACHMENT_PROMPT as initialMessage', () => { + const { initialMessage } = renderAndGetBulkConfig(); + expect(initialMessage).toBe(BULK_ALERTS_ATTACHMENT_PROMPT); + }); + + it('calls reportAddToChat with bulk_alerts_alert_summary pathway and alert_count', () => { + const { convertAlertToAttachment } = renderAndGetBulkConfig(); + const items = [makeItem('a'), makeItem('b'), makeItem('c')]; + convertAlertToAttachment(items); + expect(mockReportAddToChat).toHaveBeenCalledWith({ + pathway: 'bulk_alerts_alert_summary', + attachments: ['alert'], + alert_count: 3, + }); + }); + + it('delegates to alertsToAttachmentGroup and returns its result', () => { + const mockGroup = { type: 'group', id: 'x', label: '1 Alert', items: [] }; + (alertsToAttachmentGroup as jest.Mock).mockReturnValueOnce(mockGroup); + const { convertAlertToAttachment } = renderAndGetBulkConfig(); + const items = [makeItem('a')]; + const result = convertAlertToAttachment(items); + expect(alertsToAttachmentGroup).toHaveBeenCalledWith(items); + expect(result).toEqual([mockGroup]); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/public/detections/components/alerts_table/index.tsx b/x-pack/solutions/security/plugins/security_solution/public/detections/components/alerts_table/index.tsx index 3b651accf5009..e432e4a5371cb 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detections/components/alerts_table/index.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/detections/components/alerts_table/index.tsx @@ -38,6 +38,7 @@ import { ActionsCell } from './actions_cell'; import { useGlobalTime } from '../../../common/containers/use_global_time'; import { useLicense } from '../../../common/hooks/use_license'; import { APP_ID, CASES_FEATURE_ID, VIEW_SELECTION } from '../../../../common/constants'; +import { useBulkAddToChatConfig } from '../../../agent_builder/hooks/use_bulk_add_to_chat_config'; import { DEFAULT_COLUMN_MIN_WIDTH } from '../../../timelines/components/timeline/body/constants'; import { defaultRowRenderers } from '../../../timelines/components/timeline/body/renderers'; import { eventsDefaultModel } from '../../../common/components/events_viewer/default_model'; @@ -171,6 +172,7 @@ const AlertsTableComponent: FC onLoad(alerts), [onLoad]); + const pathway = + tableType === TableId.alertsOnRuleDetailsPage + ? ('bulk_alerts_rule_details' as const) + : ('bulk_alerts_alerts_page' as const); + const bulkAddToChatConfig = useBulkAddToChatConfig(pathway); + /** * We want to hide additional controls (like grouping) if the table is being rendered * in the cases page OR if the user of the table explicitly set `disableAdditionalToolbarControls` @@ -503,6 +523,7 @@ const AlertsTableComponent: FC diff --git a/x-pack/solutions/security/plugins/security_solution/public/detections/components/alerts_table/table_bulk_add_to_chat.test.tsx b/x-pack/solutions/security/plugins/security_solution/public/detections/components/alerts_table/table_bulk_add_to_chat.test.tsx new file mode 100644 index 0000000000000..037029d042269 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/public/detections/components/alerts_table/table_bulk_add_to_chat.test.tsx @@ -0,0 +1,180 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { render } from '@testing-library/react'; +import { AlertsTable as ResponseOpsAlertsTable } from '@kbn/response-ops-alerts-table'; +import type { TimelineItem } from '@kbn/response-ops-alerts-table/types'; +import { TableId } from '@kbn/securitysolution-data-table'; +import { TestProviders } from '../../../common/mock'; +import { BULK_ALERTS_ATTACHMENT_PROMPT } from '../../../agent_builder/components/prompts'; +import { alertsToAttachmentGroup } from '../../../agent_builder/helpers'; +import { useReportAddToChat } from '../../../agent_builder/hooks/use_report_add_to_chat'; +import { AlertsTable } from '.'; + +jest.mock('@kbn/response-ops-alerts-table', () => ({ + AlertsTable: jest.fn(() => null), +})); +jest.mock('../../../agent_builder/hooks/use_report_add_to_chat'); +jest.mock('../../../agent_builder/helpers', () => ({ + alertsToAttachmentGroup: jest.fn(), +})); +jest.mock('../../../common/lib/kibana', () => ({ + useKibana: jest.fn(() => ({ + services: { + data: {}, + http: {}, + notifications: {}, + rendering: {}, + fieldFormats: {}, + application: {}, + licensing: {}, + uiSettings: { get: jest.fn() }, + settings: {}, + cases: {}, + agentBuilder: {}, + }, + })), + KibanaContextProvider: ({ children }: { children: React.ReactNode }) => <>{children}, +})); +jest.mock('../../../common/containers/use_global_time', () => ({ + useGlobalTime: jest.fn(() => ({ + from: '2020-01-01T00:00:00Z', + to: '2020-01-02T00:00:00Z', + setQuery: jest.fn(), + deleteQuery: jest.fn(), + })), +})); +jest.mock('../../../sourcerer/containers', () => ({ + useSourcererDataView: jest.fn(() => ({ + browserFields: {}, + sourcererDataView: { runtimeFieldMap: {} }, + loading: false, + indicesExist: true, + })), +})); +jest.mock('../../../common/hooks/use_experimental_features', () => ({ + useIsExperimentalFeatureEnabled: jest.fn(() => false), +})); +jest.mock('../../../data_view_manager/hooks/use_data_view', () => ({ + useDataView: jest.fn(() => ({ + dataView: { getRuntimeMappings: jest.fn(() => ({})) }, + status: 'ready', + })), +})); +jest.mock('../../../data_view_manager/hooks/use_browser_fields', () => ({ + useBrowserFields: jest.fn(() => ({})), +})); +jest.mock('../../../common/hooks/use_license', () => ({ + useLicense: jest.fn(() => ({ + isEnterprise: jest.fn(() => false), + isPlatinumPlus: jest.fn(() => false), + isGold: jest.fn(() => false), + getType: jest.fn(() => 'basic'), + })), +})); +jest.mock('../../../common/hooks/use_selector', () => ({ + useDeepEqualSelector: jest.fn(() => []), + useShallowEqualSelector: jest.fn(() => ({})), +})); +jest.mock('../../hooks/trigger_actions_alert_table/use_bulk_actions', () => ({ + useBulkActionsByTableType: jest.fn(() => []), +})); +jest.mock('../../../common/components/user_privileges', () => ({ + useUserPrivileges: jest.fn(() => ({ + timelinePrivileges: { read: true }, + notesPrivileges: { read: true }, + kibanaSecuritySolutionsPrivileges: { crud: true, read: true }, + })), +})); +jest.mock('../../../notes/hooks/use_fetch_notes', () => ({ + useFetchNotes: jest.fn(() => ({ onLoad: jest.fn() })), +})); +jest.mock('../../configurations/security_solution_detections/fetch_page_context', () => ({ + useFetchUserProfilesFromAlerts: jest.fn(() => new Map()), +})); +jest.mock('../../hooks/trigger_actions_alert_table/use_cell_actions', () => ({ + useCellActionsOptions: jest.fn(() => undefined), +})); +jest.mock( + '../../hooks/trigger_actions_alert_table/use_trigger_actions_browser_fields_options', + () => ({ + useAlertsTableFieldsBrowserOptions: jest.fn(() => undefined), + }) +); +jest.mock('../../../common/hooks/use_invalid_filter_query', () => ({ + useInvalidFilterQuery: jest.fn(), +})); +jest.mock('../../../common/lib/kuery', () => ({ + combineQueries: jest.fn(() => null), +})); +jest.mock('../../configurations/security_solution_detections', () => ({ + CellValue: () => null, + getColumns: jest.fn(() => []), +})); +jest.mock('../../../timelines/components/timeline/body/control_columns', () => ({ + getDefaultControlColumn: jest.fn(() => [{ width: 124 }]), +})); + +const makeItem = (id: string): TimelineItem => + ({ _id: id, data: [], ecs: { _id: id, _index: '' } } as unknown as TimelineItem); + +describe('Alerts Page Table — bulkAddToChatConfig', () => { + let mockReportAddToChat: jest.Mock; + + beforeEach(() => { + jest.clearAllMocks(); + mockReportAddToChat = jest.fn(); + (useReportAddToChat as jest.Mock).mockReturnValue(mockReportAddToChat); + }); + + const renderAndGetBulkConfig = (tableType?: TableId) => { + render( + + + + ); + return (ResponseOpsAlertsTable as jest.Mock).mock.calls[0][0].bulkAddToChatConfig; + }; + + it('passes BULK_ALERTS_ATTACHMENT_PROMPT as initialMessage', () => { + const { initialMessage } = renderAndGetBulkConfig(); + expect(initialMessage).toBe(BULK_ALERTS_ATTACHMENT_PROMPT); + }); + + it('uses bulk_alerts_alerts_page pathway for the default alerts page table', () => { + const { convertAlertToAttachment } = renderAndGetBulkConfig(TableId.alertsOnAlertsPage); + const items = [makeItem('a'), makeItem('b')]; + convertAlertToAttachment(items); + expect(mockReportAddToChat).toHaveBeenCalledWith({ + pathway: 'bulk_alerts_alerts_page', + attachments: ['alert'], + alert_count: 2, + }); + }); + + it('uses bulk_alerts_rule_details pathway for the rule details table', () => { + const { convertAlertToAttachment } = renderAndGetBulkConfig(TableId.alertsOnRuleDetailsPage); + const items = [makeItem('a')]; + convertAlertToAttachment(items); + expect(mockReportAddToChat).toHaveBeenCalledWith({ + pathway: 'bulk_alerts_rule_details', + attachments: ['alert'], + alert_count: 1, + }); + }); + + it('delegates to alertsToAttachmentGroup and returns its result', () => { + const mockGroup = { type: 'group', id: 'x', label: '1 Alert', items: [] }; + (alertsToAttachmentGroup as jest.Mock).mockReturnValueOnce(mockGroup); + const { convertAlertToAttachment } = renderAndGetBulkConfig(); + const items = [makeItem('a')]; + const result = convertAlertToAttachment(items); + expect(alertsToAttachmentGroup).toHaveBeenCalledWith(items); + expect(result).toEqual([mockGroup]); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/attachments/alerts.test.ts b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/attachments/alerts.test.ts new file mode 100644 index 0000000000000..29fc83494d7ff --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/attachments/alerts.test.ts @@ -0,0 +1,261 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Attachment } from '@kbn/agent-builder-common/attachments'; +import { platformCoreTools } from '@kbn/agent-builder-common'; +import { agentBuilderMocks } from '@kbn/agent-builder-plugin/server/mocks'; +import { coreMock } from '@kbn/core/server/mocks'; +import { loggerMock } from '@kbn/logging-mocks'; +import type { SecuritySolutionPluginCoreSetupDependencies } from '../../plugin_contract'; +import { SecurityAgentBuilderAttachments } from '../../../common/constants'; +import { + SECURITY_ENTITY_RISK_SCORE_TOOL_ID, + SECURITY_ATTACK_DISCOVERY_SEARCH_TOOL_ID, + SECURITY_LABS_SEARCH_TOOL_ID, + SECURITY_ALERTS_TOOL_ID, +} from '../tools'; +import { createBulkAlertsAttachmentType } from './alerts'; + +const mockAlertSource = { + 'kibana.alert.rule.name': 'Test Rule', + 'kibana.alert.severity': 'critical', + 'kibana.alert.risk_score': 75, + 'host.name': 'test-host', + 'user.name': 'test-user', +}; + +const mockLogger = loggerMock.create(); + +const buildCoreMock = (hits: Array<{ _id: string; _source: unknown }> = []) => { + const core = coreMock.createSetup(); + const scopedClient = { + asCurrentUser: { + search: jest.fn().mockResolvedValue({ + hits: { hits: hits.map((h) => ({ _id: h._id, _source: h._source })) }, + }), + }, + }; + const elasticsearchMock = { + client: { asScoped: jest.fn().mockReturnValue(scopedClient) }, + }; + (core.getStartServices as jest.Mock).mockResolvedValue([{ elasticsearch: elasticsearchMock }]); + return core as unknown as SecuritySolutionPluginCoreSetupDependencies; +}; + +describe('createBulkAlertsAttachmentType', () => { + const formatContext = agentBuilderMocks.attachments.createFormatContextMock(); + + describe('validate', () => { + const attachmentType = createBulkAlertsAttachmentType(buildCoreMock(), mockLogger); + + it('returns valid when alertIds array is present', async () => { + const input = { alertIds: ['abc123', 'def456'] }; + + const result = await attachmentType.validate(input); + + expect(result.valid).toBe(true); + if (result.valid) { + expect(result.data).toEqual(input); + } + }); + + it('returns invalid with an empty alertIds array', async () => { + const result = await attachmentType.validate({ alertIds: [] }); + + expect(result.valid).toBe(false); + }); + + it('returns invalid when alertIds exceeds 20 entries', async () => { + const result = await attachmentType.validate({ + alertIds: Array.from({ length: 21 }, (_, i) => `id-${i}`), + }); + + expect(result.valid).toBe(false); + }); + + it('returns invalid when alertIds field is missing', async () => { + const result = await attachmentType.validate({}); + + expect(result.valid).toBe(false); + if (!result.valid) { + expect(result.error).toBeDefined(); + } + }); + + it('returns invalid when alertIds field is not an array', async () => { + const result = await attachmentType.validate({ alertIds: 'not-an-array' }); + + expect(result.valid).toBe(false); + }); + + it('returns invalid when an alertIds entry is not a string', async () => { + const result = await attachmentType.validate({ alertIds: [123] }); + + expect(result.valid).toBe(false); + }); + }); + + describe('format', () => { + it('fetches alerts from ES and returns inline JSON representation', async () => { + const core = buildCoreMock([ + { _id: 'abc123', _source: mockAlertSource }, + { _id: 'def456', _source: { ...mockAlertSource, 'host.name': 'other-host' } }, + ]); + const attachmentType = createBulkAlertsAttachmentType(core, mockLogger); + + const attachment: Attachment = { + id: 'id-fetch-test', + type: SecurityAgentBuilderAttachments.alerts, + data: { alertIds: ['abc123', 'def456'] }, + }; + + const formatted = await attachmentType.format(attachment, formatContext); + const representation = formatted.getRepresentation + ? await formatted.getRepresentation() + : undefined; + + expect(representation?.type).toBe('text'); + expect(representation?.value).toContain('2 security alerts'); + expect(representation?.value).toContain('Alert 1:'); + expect(representation?.value).toContain('Alert 2:'); + expect(representation?.value).toContain('"_id": "abc123"'); + expect(representation?.value).toContain('"_id": "def456"'); + }); + + it('marks missing alerts with an error placeholder', async () => { + const core = buildCoreMock([{ _id: 'abc123', _source: mockAlertSource }]); + const attachmentType = createBulkAlertsAttachmentType(core, mockLogger); + + const attachment: Attachment = { + id: 'id-missing-test', + type: SecurityAgentBuilderAttachments.alerts, + data: { alertIds: ['abc123', 'missing-id'] }, + }; + + const formatted = await attachmentType.format(attachment, formatContext); + const representation = formatted.getRepresentation + ? await formatted.getRepresentation() + : undefined; + + expect(representation?.value).toContain('"_id": "missing-id"'); + expect(representation?.value).toContain('"error": "not found"'); + }); + + it('logs a warn and returns placeholders when ES search throws', async () => { + const core = coreMock.createSetup(); + const scopedClient = { + asCurrentUser: { search: jest.fn().mockRejectedValue(new Error('index not available')) }, + }; + (core.getStartServices as jest.Mock).mockResolvedValue([ + { elasticsearch: { client: { asScoped: jest.fn().mockReturnValue(scopedClient) } } }, + ]); + const logger = loggerMock.create(); + const attachmentType = createBulkAlertsAttachmentType( + core as unknown as SecuritySolutionPluginCoreSetupDependencies, + logger + ); + + const attachment: Attachment = { + id: 'id-error-test', + type: SecurityAgentBuilderAttachments.alerts, + data: { alertIds: ['abc123'] }, + }; + + const formatted = await attachmentType.format(attachment, formatContext); + await formatted.getRepresentation?.(); + + expect(logger.warn).toHaveBeenCalledTimes(1); + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('Failed to fetch')); + }); + + it('logs a warn when ES returns no results', async () => { + const core = buildCoreMock([]); + const logger = loggerMock.create(); + const attachmentType = createBulkAlertsAttachmentType(core, logger); + + const attachment: Attachment = { + id: 'id-no-results-test', + type: SecurityAgentBuilderAttachments.alerts, + data: { alertIds: ['abc123'] }, + }; + + const formatted = await attachmentType.format(attachment, formatContext); + await formatted.getRepresentation?.(); + + expect(logger.warn).toHaveBeenCalledTimes(1); + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('no results')); + }); + + it('returns cached representation on repeated reads without re-fetching from ES', async () => { + const rawCore = coreMock.createSetup(); + const scopedClient = { + asCurrentUser: { + search: jest.fn().mockResolvedValue({ + hits: { hits: [{ _id: 'abc123', _source: mockAlertSource }] }, + }), + }, + }; + (rawCore.getStartServices as jest.Mock).mockResolvedValue([ + { elasticsearch: { client: { asScoped: jest.fn().mockReturnValue(scopedClient) } } }, + ]); + const core = rawCore as unknown as SecuritySolutionPluginCoreSetupDependencies; + const attachmentType = createBulkAlertsAttachmentType(core, mockLogger); + + const attachment: Attachment = { + id: 'id-cache-test', + type: SecurityAgentBuilderAttachments.alerts, + data: { alertIds: ['abc123'] }, + }; + + const formatted = await attachmentType.format(attachment, formatContext); + const first = await formatted.getRepresentation?.(); + + // Second read with the same attachment ID should return cached value without hitting ES + (rawCore.getStartServices as jest.Mock).mockClear(); + const formatted2 = await attachmentType.format(attachment, formatContext); + const second = await formatted2.getRepresentation?.(); + + expect(second?.value).toBe(first?.value); + expect(rawCore.getStartServices).not.toHaveBeenCalled(); + }); + }); + + describe('getTools', () => { + const attachmentType = createBulkAlertsAttachmentType(buildCoreMock(), mockLogger); + + it('returns enrichment tool IDs but not the ES|QL alerts search tool', () => { + const tools = attachmentType.getTools?.(); + + expect(tools).toBeDefined(); + if (tools) { + expect(tools).toContain(SECURITY_ENTITY_RISK_SCORE_TOOL_ID); + expect(tools).toContain(SECURITY_ATTACK_DISCOVERY_SEARCH_TOOL_ID); + expect(tools).toContain(SECURITY_LABS_SEARCH_TOOL_ID); + expect(tools).toContain(platformCoreTools.cases); + expect(tools).toContain(platformCoreTools.generateEsql); + expect(tools).toContain(platformCoreTools.productDocumentation); + expect(tools).not.toContain(SECURITY_ALERTS_TOOL_ID); + } + }); + }); + + describe('getAgentDescription', () => { + const attachmentType = createBulkAlertsAttachmentType(buildCoreMock(), mockLogger); + + it('instructs the agent to process batches and synthesize across all', () => { + const description = attachmentType.getAgentDescription?.(); + + expect(description).toContain('batch'); + expect(description).toContain('After processing all batches'); + }); + }); + + it('is marked readonly so the LLM cannot modify alert data', () => { + const attachmentType = createBulkAlertsAttachmentType(buildCoreMock(), mockLogger); + expect(attachmentType.isReadonly).toBe(true); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/attachments/alerts.ts b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/attachments/alerts.ts new file mode 100644 index 0000000000000..16ce348937f55 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/attachments/alerts.ts @@ -0,0 +1,129 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from '@kbn/zod/v4'; +import type { Logger } from '@kbn/core/server'; +import type { AttachmentTypeDefinition } from '@kbn/agent-builder-server/attachments'; +import type { Attachment } from '@kbn/agent-builder-common/attachments'; +import { platformCoreTools } from '@kbn/agent-builder-common'; +import { ALERTS_BATCH_MAX_SIZE, SecurityAgentBuilderAttachments } from '../../../common/constants'; +import { getAlertsIndex } from '../../../common/entity_analytics/utils'; +import { + SECURITY_ENTITY_RISK_SCORE_TOOL_ID, + SECURITY_ATTACK_DISCOVERY_SEARCH_TOOL_ID, + SECURITY_LABS_SEARCH_TOOL_ID, +} from '../tools'; +import type { SecuritySolutionPluginCoreSetupDependencies } from '../../plugin_contract'; +import { getAlertsById } from '../tools/get_alerts_by_id'; +import { securityAttachmentDataSchema } from './security_attachment_data_schema'; + +export const bulkAlertsAttachmentDataSchema = securityAttachmentDataSchema.extend({ + alertIds: z.array(z.string()).min(1).max(ALERTS_BATCH_MAX_SIZE), +}); + +export type BulkAlertsAttachmentData = z.infer; + +// Bounded cache to avoid re-fetching alert data from ES on every LLM attachment_read call. +// Keyed on attachment.id (UUIDs), evicts oldest entry when full. +const REPRESENTATION_CACHE_MAX_SIZE = 500; +const representationCache = new Map(); + +export const createBulkAlertsAttachmentType = ( + core: SecuritySolutionPluginCoreSetupDependencies, + logger: Logger +): AttachmentTypeDefinition => ({ + id: SecurityAgentBuilderAttachments.alerts, + isReadonly: true, + maxContentLength: 50_000, + validate: (input) => { + const result = bulkAlertsAttachmentDataSchema.safeParse(input); + return result.success + ? { valid: true, data: result.data } + : { valid: false, error: result.error.message }; + }, + format: (attachment, context) => { + // framework guarantees validate() ran first, so data is BulkAlertsAttachmentData + const { data } = attachment as Attachment; + return { + getRepresentation: async () => { + const cached = representationCache.get(attachment.id); + if (cached !== undefined) { + return { type: 'text' as const, value: cached }; + } + + const [coreStart] = await core.getStartServices(); + const esClient = coreStart.elasticsearch.client.asScoped(context.request).asCurrentUser; + const index = getAlertsIndex(context.spaceId); + + let alertsById: Record | undefined; + try { + alertsById = await getAlertsById({ esClient, index, ids: data.alertIds }); + } catch (err) { + logger.warn( + `Failed to fetch ${data.alertIds.length} alert(s) from index ${index}: ${ + err?.message ?? err + }` + ); + } + + if ( + alertsById !== undefined && + Object.keys(alertsById).length === 0 && + data.alertIds.length > 0 + ) { + logger.warn( + `ES returned no results for ${data.alertIds.length} alert ID(s) from index ${index}` + ); + } + + const resolvedAlerts = alertsById ?? {}; + const entries = data.alertIds.map((id, i) => { + const alert = resolvedAlerts[id] ?? { error: 'not found' }; + return `Alert ${i + 1}:\n${JSON.stringify({ _id: id, ...(alert as object) }, null, 2)}`; + }); + + const value = `${data.alertIds.length} security alert${ + data.alertIds.length !== 1 ? 's' : '' + }:\n\n${entries.join('\n\n---\n\n')}`; + + if (representationCache.size >= REPRESENTATION_CACHE_MAX_SIZE) { + const oldest = representationCache.keys().next().value; + if (oldest !== undefined) { + representationCache.delete(oldest); + } + } + representationCache.set(attachment.id, value); + + return { type: 'text' as const, value }; + }, + }; + }, + getTools: () => [ + SECURITY_ENTITY_RISK_SCORE_TOOL_ID, + SECURITY_ATTACK_DISCOVERY_SEARCH_TOOL_ID, + SECURITY_LABS_SEARCH_TOOL_ID, + platformCoreTools.cases, + platformCoreTools.generateEsql, + platformCoreTools.productDocumentation, + ], + getAgentDescription: () => + `You have access to security alerts provided in one or more batches. Each attachment contains a batch of up to 20 alerts with full field data. + +When the conversation has many batches (summary mode), the system shows metadata only. Read each batch before answering: +- Each batch appears as in the XML +- Call the attachment read tool (attachment_read) and pass the attachment_id value directly as the "attachment_id" parameter +- Read ALL batches before forming conclusions + +Process each batch in order: +1. Extract _id, kibana.alert.rule.name, severity, risk_score, host.name, user.name, and MITRE fields. +2. Note entities (hosts, users) and patterns within the batch. + +After processing all batches: +3. Identify shared entities, escalation indicators, and patterns across the full alert set. +4. Use the enrichment tools (entity risk score, security labs, attack discovery) for collective context. +5. Produce a structured summary with triage steps and recommended actions.`, +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/attachments/register_attachments.ts b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/attachments/register_attachments.ts index a8889f9792b88..3e2d8d7fa7cc3 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/attachments/register_attachments.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/attachments/register_attachments.ts @@ -5,9 +5,12 @@ * 2.0. */ +import type { Logger } from '@kbn/core/server'; import type { AgentBuilderPluginSetup } from '@kbn/agent-builder-server'; +import type { SecuritySolutionPluginCoreSetupDependencies } from '../../plugin_contract'; import { createRuleAttachmentType } from './rule'; import { createAlertAttachmentType } from './alert'; +import { createBulkAlertsAttachmentType } from './alerts'; import { createEntityAttachmentType } from './entity'; import { createEntityAnalyticsDashboardAttachmentType } from './entity_analytics_dashboard'; import { createSiemReadinessAttachmentType } from './siem_readiness'; @@ -15,8 +18,13 @@ import { createSiemReadinessAttachmentType } from './siem_readiness'; /** * Registers all security agent builder attachments with the agentBuilder plugin */ -export const registerAttachments = async (agentBuilder: AgentBuilderPluginSetup) => { +export const registerAttachments = async ( + agentBuilder: AgentBuilderPluginSetup, + core: SecuritySolutionPluginCoreSetupDependencies, + logger: Logger +) => { agentBuilder.attachments.registerType(createAlertAttachmentType()); + agentBuilder.attachments.registerType(createBulkAlertsAttachmentType(core, logger)); agentBuilder.attachments.registerType(createEntityAttachmentType()); agentBuilder.attachments.registerType(createEntityAnalyticsDashboardAttachmentType()); agentBuilder.attachments.registerType(createRuleAttachmentType()); diff --git a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/entity_analytics/entity_risk_score_tool.ts b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/entity_analytics/entity_risk_score_tool.ts index ebc7c9596e9d5..3458d075c055f 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/entity_analytics/entity_risk_score_tool.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/entity_analytics/entity_risk_score_tool.ts @@ -16,9 +16,10 @@ import type { SecuritySolutionPluginCoreSetupDependencies } from '../../../plugi import type { EntityRiskScoreRecord } from '../../../../common/api/entity_analytics/common'; import { createGetRiskScores } from '../../../lib/entity_analytics/risk_score/get_risk_score'; import type { EntityType } from '../../../../common/entity_analytics/types'; -import { DEFAULT_ALERTS_INDEX, ESSENTIAL_ALERT_FIELDS } from '../../../../common/constants'; +import { DEFAULT_ALERTS_INDEX } from '../../../../common/constants'; import { getRiskIndex } from '../../../../common/search_strategy/security_solution/risk_score/common'; import { securityTool } from '../constants'; +import { getAlertsById } from '../get_alerts_by_id'; const entityRiskScoreSchema = z.object({ identifierType: z @@ -88,43 +89,6 @@ const queryRiskIndexForWildcard = async ({ .filter((risk): risk is EntityRiskScoreRecord => risk !== undefined); }; -/** - * Fetches alerts by their IDs, returning only essential fields for risk score context - */ -const getAlertsById = async ({ - esClient, - index, - ids, -}: { - esClient: ElasticsearchClient; - index: string; - ids: string[]; -}): Promise> => { - if (ids.length === 0) { - return {}; - } - - const response = await esClient.search({ - index, - ignore_unavailable: true, - allow_no_indices: true, - size: ids.length, - _source: ESSENTIAL_ALERT_FIELDS, - query: { - bool: { - filter: [{ terms: { _id: ids } }], - }, - }, - }); - - return response.hits.hits.reduce>((acc, hit) => { - if (hit._source && hit._id) { - acc[hit._id] = hit._source; - } - return acc; - }, {}); -}; - export const entityRiskScoreTool = ( core: SecuritySolutionPluginCoreSetupDependencies, logger: Logger diff --git a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/get_alerts_by_id.test.ts b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/get_alerts_by_id.test.ts new file mode 100644 index 0000000000000..16da986527595 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/get_alerts_by_id.test.ts @@ -0,0 +1,93 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient } from '@kbn/core/server'; +import { ALERTS_BATCH_MAX_SIZE, ESSENTIAL_ALERT_FIELDS } from '../../../common/constants'; +import { getAlertsById } from './get_alerts_by_id'; + +const makeClient = (hits: Array<{ _id?: string; _source?: unknown }>) => + ({ + search: jest.fn().mockResolvedValue({ hits: { hits } }), + } as unknown as ElasticsearchClient); + +describe('getAlertsById', () => { + const index = '.alerts-security.alerts-default'; + + it('returns an empty object without calling ES when ids is empty', async () => { + const esClient = makeClient([]); + + const result = await getAlertsById({ esClient, index, ids: [] }); + + expect(result).toEqual({}); + expect(esClient.search).not.toHaveBeenCalled(); + }); + + it('throws when more than ALERTS_BATCH_MAX_SIZE ids are provided', async () => { + const esClient = makeClient([]); + const ids = Array.from({ length: ALERTS_BATCH_MAX_SIZE + 1 }, (_, i) => `id-${i}`); + + await expect(getAlertsById({ esClient, index, ids })).rejects.toThrow( + `getAlertsById: ids.length (${ + ALERTS_BATCH_MAX_SIZE + 1 + }) exceeds the maximum of ${ALERTS_BATCH_MAX_SIZE}` + ); + expect(esClient.search).not.toHaveBeenCalled(); + }); + + it('returns hits keyed by _id with _source as the value', async () => { + const source1 = { 'kibana.alert.rule.name': 'Rule A' }; + const source2 = { 'kibana.alert.rule.name': 'Rule B' }; + const esClient = makeClient([ + { _id: 'abc', _source: source1 }, + { _id: 'def', _source: source2 }, + ]); + + const result = await getAlertsById({ esClient, index, ids: ['abc', 'def'] }); + + expect(result).toEqual({ abc: source1, def: source2 }); + }); + + it('skips hits that are missing _source', async () => { + const esClient = makeClient([ + { _id: 'abc', _source: { 'kibana.alert.rule.name': 'Rule A' } }, + { _id: 'def' }, + ]); + + const result = await getAlertsById({ esClient, index, ids: ['abc', 'def'] }); + + expect(result).toEqual({ abc: { 'kibana.alert.rule.name': 'Rule A' } }); + expect(result).not.toHaveProperty('def'); + }); + + it('skips hits that are missing _id', async () => { + const esClient = makeClient([ + { _source: { 'kibana.alert.rule.name': 'Rule A' } }, + { _id: 'def', _source: { 'kibana.alert.rule.name': 'Rule B' } }, + ]); + + const result = await getAlertsById({ esClient, index, ids: ['def'] }); + + expect(Object.keys(result)).toEqual(['def']); + }); + + it('queries with the correct index, source fields, and id filter', async () => { + const esClient = makeClient([]); + + await getAlertsById({ esClient, index, ids: ['abc', 'def'] }); + + expect(esClient.search).toHaveBeenCalledWith( + expect.objectContaining({ + index, + ignore_unavailable: true, + allow_no_indices: true, + size: 2, + _source: ESSENTIAL_ALERT_FIELDS, + query: { bool: { filter: [{ terms: { _id: ['abc', 'def'] } }] } }, + }) + ); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/get_alerts_by_id.ts b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/get_alerts_by_id.ts new file mode 100644 index 0000000000000..3cb481674a7e6 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/get_alerts_by_id.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient } from '@kbn/core/server'; +import { ALERTS_BATCH_MAX_SIZE, ESSENTIAL_ALERT_FIELDS } from '../../../common/constants'; + +/** + * Fetches alerts by their _id values from Elasticsearch, returning only essential fields. + */ +export const getAlertsById = async ({ + esClient, + index, + ids, +}: { + esClient: ElasticsearchClient; + index: string; + ids: string[]; +}): Promise> => { + if (ids.length === 0) { + return {}; + } + if (ids.length > ALERTS_BATCH_MAX_SIZE) { + throw new Error( + `getAlertsById: ids.length (${ids.length}) exceeds the maximum of ${ALERTS_BATCH_MAX_SIZE}` + ); + } + + const response = await esClient.search({ + index, + ignore_unavailable: true, + allow_no_indices: true, + size: ids.length, + _source: ESSENTIAL_ALERT_FIELDS, + query: { + bool: { + filter: [{ terms: { _id: ids } }], + }, + }, + }); + + return response.hits.hits.reduce>((acc, hit) => { + if (hit._source && hit._id) { + acc[hit._id] = hit._source; + } + return acc; + }, {}); +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/plugin.ts b/x-pack/solutions/security/plugins/security_solution/server/plugin.ts index ee97ece000e98..a5b7a2c771958 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/plugin.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/plugin.ts @@ -274,9 +274,11 @@ export class Plugin implements ISecuritySolutionPlugin { this.logger.error(`Error registering security tools: ${error}`); } ); - registerAttachments(agentBuilder).catch((error) => { + + registerAttachments(agentBuilder, core, logger).catch((error) => { this.logger.error(`Error registering security attachments: ${error}`); }); + registerSkills({ agentBuilder, experimentalFeatures, diff --git a/x-pack/solutions/security/plugins/security_solution/test/scout/ui/fixtures/index.ts b/x-pack/solutions/security/plugins/security_solution/test/scout/ui/fixtures/index.ts new file mode 100644 index 0000000000000..33ee8d23401d2 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/test/scout/ui/fixtures/index.ts @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { spaceTest as baseSpaceTest } from '@kbn/scout-security'; +import type { ScoutWorkerFixtures } from '@kbn/scout-security'; +import { createLlmProxy, type LlmProxy } from '@kbn/ftr-llm-proxy'; + +const XSRF = { 'kbn-xsrf': 'scout-security-solution' }; + +interface SecuritySolutionWorkerFixtures extends ScoutWorkerFixtures { + llmProxy: LlmProxy; +} + +export const spaceTest = baseSpaceTest.extend<{}, SecuritySolutionWorkerFixtures>({ + llmProxy: [ + // scoutSpace is declared first so Playwright sets up the test space before the proxy, + // avoiding concurrent initialization that can delay Sourcerer setup. + async ({ log, kbnClient, scoutSpace }, use) => { + const proxy = await createLlmProxy(log); + + // Create the connector inside the test space so EmbeddableAccessBoundary finds it. + await kbnClient.request({ + method: 'POST', + path: `/s/${scoutSpace.id}/api/actions/connector`, + headers: XSRF, + body: { + name: 'scout-llm-proxy', + config: { + apiProvider: 'OpenAI', + apiUrl: `http://localhost:${proxy.getPort()}`, + defaultModel: 'gpt-4', + }, + secrets: { apiKey: 'test-key' }, + connector_type_id: '.gen-ai', + }, + }); + + await use(proxy); + + proxy.close(); + + const list = await kbnClient.request>({ + method: 'GET', + path: `/s/${scoutSpace.id}/api/actions/connectors`, + }); + const connectors = Array.isArray(list.data) ? list.data : []; + await Promise.all( + connectors + .filter((c) => c.name === 'scout-llm-proxy') + .map((c) => + kbnClient.request({ + method: 'DELETE', + path: `/s/${scoutSpace.id}/api/actions/connector/${encodeURIComponent(c.id)}`, + headers: XSRF, + }) + ) + ); + }, + { scope: 'worker', auto: false }, + ], +}); + +export { tags } from '@kbn/scout-security'; diff --git a/x-pack/solutions/security/plugins/security_solution/test/scout/ui/parallel_tests/investigations/bulk_add_alerts_to_chat.spec.ts b/x-pack/solutions/security/plugins/security_solution/test/scout/ui/parallel_tests/investigations/bulk_add_alerts_to_chat.spec.ts new file mode 100644 index 0000000000000..ae2fa758c69e4 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/test/scout/ui/parallel_tests/investigations/bulk_add_alerts_to_chat.spec.ts @@ -0,0 +1,201 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { KibanaRole } from '@kbn/scout-security'; +import { createToolCallMessage } from '@kbn/ftr-llm-proxy'; +import { expect } from '@kbn/scout-security/ui'; +import { CUSTOM_QUERY_RULE } from '@kbn/scout-security/src/playwright/constants/detection_rules'; +import { spaceTest, tags } from '../../fixtures'; + +const ALERT_COUNT = 3; + +/** + * A role with security alert index privileges and full Kibana access, + * which includes the agentBuilder feature needed for the "Add to chat" bulk action. + */ +const AGENT_BUILDER_ROLE: KibanaRole = { + elasticsearch: { + cluster: ['manage'], + indices: [ + { + names: [ + '.alerts-security*', + '.internal.alerts-security*', + '.siem-signals-*', + 'apm-*-transaction*', + 'traces-apm*', + 'auditbeat-*', + 'endgame-*', + 'filebeat-*', + 'logs-*', + 'packetbeat-*', + 'winlogbeat-*', + 'logstash-*', + '.lists*', + '.items*', + ], + privileges: ['read', 'write'], + }, + ], + }, + kibana: [ + { + base: ['all'], + feature: {}, + spaces: ['*'], + }, + ], +}; + +spaceTest.describe('Bulk add alerts to chat', { tag: [...tags.stateful.classic] }, () => { + // One rule name per created rule; the first is used as the anchor to wait for page readiness. + const ruleNames: string[] = []; + + spaceTest.beforeEach(async ({ browserAuth, apiServices, scoutSpace }) => { + ruleNames.length = 0; + for (let i = 0; i < ALERT_COUNT; i++) { + const name = `${CUSTOM_QUERY_RULE.name}_${scoutSpace.id}_${Date.now()}_${i}`; + ruleNames.push(name); + await apiServices.detectionRule.createCustomQueryRule({ + ...CUSTOM_QUERY_RULE, + name, + rule_id: `scout-bulk-chat-${scoutSpace.id}-${i}`, + }); + } + await browserAuth.loginWithCustomRole(AGENT_BUILDER_ROLE); + }); + + spaceTest.afterEach(async ({ apiServices }) => { + await apiServices.detectionRule.deleteAll(); + await apiServices.detectionAlerts.deleteAll(); + }); + + spaceTest( + 'should disable the "Add to chat" bulk action when all alerts are selected via the query button', + // _llmProxy is declared so the worker fixture provisions the AI connector in the test space; + // without it the bulk action would not appear at all. + async ({ pageObjects, llmProxy: _llmProxy }) => { + const { alertsTablePage } = pageObjects; + + await alertsTablePage.navigate(); + + const firstRuleNameCell = alertsTablePage.alertsTable + .getByTestId('ruleName') + .filter({ hasText: ruleNames[0] }); + await expect(firstRuleNameCell).toBeVisible({ timeout: 60_000 }); + + await spaceTest.step('check one row to activate the bulk toolbar', async () => { + const alertCheckbox = firstRuleNameCell + .locator('xpath=ancestor::div[contains(@class,"euiDataGridRow")]') + .locator('.euiCheckbox__input'); + await alertCheckbox.check(); + }); + + await spaceTest.step('click "Select all" to switch to query-based selection', async () => { + await alertsTablePage.selectAllAlertsButton.click(); + }); + + await spaceTest.step( + 'verify "Add to chat" is present but disabled in the bulk actions menu', + async () => { + await alertsTablePage.selectedShowBulkActionsButton.click(); + await expect(alertsTablePage.bulkAddToChatMenuItem).toBeVisible(); + await expect(alertsTablePage.bulkAddToChatMenuItem).toBeDisabled(); + } + ); + } + ); + + spaceTest( + 'should open agent builder conversation with attachment chip and pre-filled prompt, and send multiple alerts to the LLM', + async ({ pageObjects, page, llmProxy }) => { + const { alertsTablePage } = pageObjects; + + // Set up LLM mock interceptors before triggering the chat so no call goes unanswered. + // The agent builder makes three sequential calls: title generation, handover, and final answer. + void llmProxy + .intercept({ + name: 'set_title', + when: (body) => { + const sys = body.messages.find((m) => m.role === 'system'); + return String(sys?.content ?? '').includes('You are a title-generation utility'); + }, + responseMock: createToolCallMessage('set_title', { title: 'Alert triage' }), + }) + .completeAfterIntercept(); + + void llmProxy + .intercept({ + name: 'handover-to-answer', + when: (body) => { + const sys = body.messages.find((m) => m.role === 'system'); + return String(sys?.content ?? '').includes( + 'This response will serve as a handover note for the answering agent' + ); + }, + responseMock: 'ready to answer', + }) + .completeAfterIntercept(); + + void llmProxy + .intercept({ + name: 'final-answer', + when: () => true, + responseMock: 'Here is the alert triage summary.', + }) + .completeAfterIntercept(); + + await alertsTablePage.navigate(); + + // Wait for the first rule's alert to appear — this proves Sourcerer has initialized + // and the alerts table is ready. Longer timeout covers the full initialization window. + const firstRuleNameCell = alertsTablePage.alertsTable + .getByTestId('ruleName') + .filter({ hasText: ruleNames[0] }); + await expect(firstRuleNameCell).toBeVisible({ timeout: 60_000 }); + + await spaceTest.step('select all visible alerts via header checkbox', async () => { + // The header checkbox uses selectCurrentPage (not selectAll/query mode), so + // disableOnQuery actions like bulk-add-to-chat remain enabled. + await alertsTablePage.bulkActionsHeaderCheckbox.check(); + }); + + await spaceTest.step('open bulk actions and click Add to chat', async () => { + await alertsTablePage.selectedShowBulkActionsButton.click(); + await alertsTablePage.bulkAddToChatMenuItem.click(); + }); + + await spaceTest.step( + 'verify conversation opens with attachment chip and pre-filled prompt', + async () => { + await expect(page.testSubj.locator('agentBuilderConversation')).toBeVisible(); + await expect(page.testSubj.locator('agentBuilderAttachmentPillsRow')).toBeVisible(); + await expect(page.testSubj.locator('agentBuilderConversationInputEditor')).toContainText( + 'Triage and prioritize these security alerts' + ); + } + ); + + await spaceTest.step('send message and verify LLM received multiple alerts', async () => { + await page.testSubj.locator('agentBuilderConversationInputSubmitButton').click(); + await llmProxy.waitForAllInterceptorsToHaveBeenCalled(); + + const handoverRequest = llmProxy.interceptedRequests.find( + (r) => r.matchingInterceptorName === 'handover-to-answer' + )?.requestBody; + + expect(handoverRequest).toBeDefined(); + const messages: Array<{ content?: unknown }> = handoverRequest?.messages ?? []; + const allContent = messages.map((m) => String(m.content ?? '')).join('\n'); + + // Verify multiple alerts were delivered: each alert block contains the rule name field. + const alertFieldOccurrences = allContent.split('kibana.alert.rule.name').length - 1; + expect(alertFieldOccurrences).toBeGreaterThanOrEqual(ALERT_COUNT); + }); + } + ); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/tsconfig.json b/x-pack/solutions/security/plugins/security_solution/tsconfig.json index f753ba9b7a2e6..4aec8f14fae23 100644 --- a/x-pack/solutions/security/plugins/security_solution/tsconfig.json +++ b/x-pack/solutions/security/plugins/security_solution/tsconfig.json @@ -240,6 +240,7 @@ "@kbn/inference-common", "@kbn/inference-langchain", "@kbn/scout-security", + "@kbn/ftr-llm-proxy", "@kbn/custom-icons", "@kbn/security-plugin-types-common", "@kbn/management-settings-ids", diff --git a/yarn.lock b/yarn.lock index de3361f18319d..15577e7f8270f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6927,6 +6927,10 @@ version "0.0.0" uid "" +"@kbn/evals-suite-security-alert-triage@link:x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage": + version "0.0.0" + uid "" + "@kbn/evals-suite-security-automatic-migrations@link:x-pack/solutions/security/packages/kbn-evals-suite-security-automatic-migrations": version "0.0.0" uid "" From 1007aab089fe00543ba7452e1f109c93ac318c50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20W=C3=A5lstedt?= Date: Tue, 26 May 2026 11:16:56 +0200 Subject: [PATCH 02/14] test(security_solution): fix failing tests --- .../hooks/use_bulk_add_to_chat_config.ts | 11 +++-------- .../tabs/alerts_tab/ease/table.tsx | 5 ++++- .../public/cases/components/ease/table.tsx | 5 ++++- .../components/alert_summary/table/table.tsx | 5 ++++- .../table/table_bulk_add_to_chat.test.tsx | 12 ++++++++++-- .../detections/components/alerts_table/index.tsx | 5 ++++- .../alerts_table/table_bulk_add_to_chat.test.tsx | 12 ++++++++++-- 7 files changed, 39 insertions(+), 16 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/public/agent_builder/hooks/use_bulk_add_to_chat_config.ts b/x-pack/solutions/security/plugins/security_solution/public/agent_builder/hooks/use_bulk_add_to_chat_config.ts index 7647be4eba2cf..f218462ff79d0 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/agent_builder/hooks/use_bulk_add_to_chat_config.ts +++ b/x-pack/solutions/security/plugins/security_solution/public/agent_builder/hooks/use_bulk_add_to_chat_config.ts @@ -15,13 +15,9 @@ export const useBulkAddToChatConfig = (pathway: BulkAlertPathway): BulkAddToChat const reportAddToChat = useReportAddToChat(); const convertAlertToAttachment = useCallback( - (alertItems: TimelineItem[]) => [alertsToAttachmentGroup(alertItems)], - [] - ); - - const onAddedToChat = useCallback( - (itemCount: number) => { - reportAddToChat({ pathway, attachments: ['alert'], item_count: itemCount }); + (alertItems: TimelineItem[]) => { + reportAddToChat({ pathway, attachments: ['alert'], item_count: alertItems.length }); + return [alertsToAttachmentGroup(alertItems)]; }, [pathway, reportAddToChat] ); @@ -29,6 +25,5 @@ export const useBulkAddToChatConfig = (pathway: BulkAlertPathway): BulkAddToChat return { convertAlertToAttachment, initialMessage: BULK_ALERTS_ATTACHMENT_PROMPT, - onAddedToChat, }; }; diff --git a/x-pack/solutions/security/plugins/security_solution/public/attack_discovery/pages/results/attack_discovery_panel/tabs/alerts_tab/ease/table.tsx b/x-pack/solutions/security/plugins/security_solution/public/attack_discovery/pages/results/attack_discovery_panel/tabs/alerts_tab/ease/table.tsx index 2a4513d9674b7..b234bfe38043b 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/attack_discovery/pages/results/attack_discovery_panel/tabs/alerts_tab/ease/table.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/attack_discovery/pages/results/attack_discovery_panel/tabs/alerts_tab/ease/table.tsx @@ -28,6 +28,7 @@ import { import { ActionsCell } from '../../../../../../../detections/components/alert_summary/table/actions_cell'; import { useKibana } from '../../../../../../../common/lib/kibana'; import { useBulkAddToChatConfig } from '../../../../../../../agent_builder/hooks/use_bulk_add_to_chat_config'; +import { useAgentBuilderAvailability } from '../../../../../../../agent_builder/hooks/use_agent_builder_availability'; import { CellValue } from '../../../../../../../detections/components/alert_summary/table/render_cell'; import { useAdditionalBulkActions } from '../../../../../../../detections/hooks/alert_summary/use_additional_bulk_actions'; @@ -96,7 +97,9 @@ export const Table = memo(({ dataView, id, packages, query }: TableProps) => { ] ); + const { isAgentBuilderEnabled } = useAgentBuilderAvailability(); const bulkAddToChatConfig = useBulkAddToChatConfig('bulk_alerts_attack_discovery'); + const maybeBulkAddToChatConfig = isAgentBuilderEnabled ? bulkAddToChatConfig : undefined; const browserFields = useBrowserFields(PageScope.alerts, dataView); @@ -138,7 +141,7 @@ export const Table = memo(({ dataView, id, packages, query }: TableProps) => { ruleTypeIds={RULE_TYPE_IDS} services={services} toolbarVisibility={TOOLBAR_VISIBILITY} - bulkAddToChatConfig={bulkAddToChatConfig} + bulkAddToChatConfig={maybeBulkAddToChatConfig} /> ); diff --git a/x-pack/solutions/security/plugins/security_solution/public/cases/components/ease/table.tsx b/x-pack/solutions/security/plugins/security_solution/public/cases/components/ease/table.tsx index 11e49d6c6b713..056f0a59819b1 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/cases/components/ease/table.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/cases/components/ease/table.tsx @@ -16,6 +16,7 @@ import type { import React, { memo, useCallback, useMemo, useRef } from 'react'; import { useKibana } from '../../../common/lib/kibana'; import { useBulkAddToChatConfig } from '../../../agent_builder/hooks/use_bulk_add_to_chat_config'; +import { useAgentBuilderAvailability } from '../../../agent_builder/hooks/use_agent_builder_availability'; import { ActionsCell } from '../../../detections/components/alert_summary/table/actions_cell'; import { CellValue } from '../../../detections/components/alert_summary/table/render_cell'; import { useBrowserFields } from '../../../data_view_manager/hooks/use_browser_fields'; @@ -103,7 +104,9 @@ export const Table = memo(({ dataView, id, onLoaded, packages, query }: TablePro ] ); + const { isAgentBuilderEnabled } = useAgentBuilderAvailability(); const bulkAddToChatConfig = useBulkAddToChatConfig('bulk_alerts_cases'); + const maybeBulkAddToChatConfig = isAgentBuilderEnabled ? bulkAddToChatConfig : undefined; const browserFields = useBrowserFields(PageScope.alerts, dataView); @@ -141,7 +144,7 @@ export const Table = memo(({ dataView, id, onLoaded, packages, query }: TablePro runtimeMappings={runtimeMappings} services={services} toolbarVisibility={TOOLBAR_VISIBILITY} - bulkAddToChatConfig={bulkAddToChatConfig} + bulkAddToChatConfig={maybeBulkAddToChatConfig} /> ); diff --git a/x-pack/solutions/security/plugins/security_solution/public/detections/components/alert_summary/table/table.tsx b/x-pack/solutions/security/plugins/security_solution/public/detections/components/alert_summary/table/table.tsx index 7e332e3ababdf..5b189b36c4d68 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detections/components/alert_summary/table/table.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/detections/components/alert_summary/table/table.tsx @@ -38,6 +38,7 @@ import { combineQueries } from '../../../../common/lib/kuery'; import { useKibana } from '../../../../common/lib/kibana'; import { CellValue } from './render_cell'; import { useBulkAddToChatConfig } from '../../../../agent_builder/hooks/use_bulk_add_to_chat_config'; +import { useAgentBuilderAvailability } from '../../../../agent_builder/hooks/use_agent_builder_availability'; import { buildTimeRangeFilter } from '../../alerts_table/helpers'; import { useGlobalTime } from '../../../../common/containers/use_global_time'; @@ -174,7 +175,9 @@ export const Table = memo(({ dataView, groupingFilters, packages }: TableProps) ] ); + const { isAgentBuilderEnabled } = useAgentBuilderAvailability(); const bulkAddToChatConfig = useBulkAddToChatConfig('bulk_alerts_alert_summary'); + const maybeBulkAddToChatConfig = isAgentBuilderEnabled ? bulkAddToChatConfig : undefined; const getGlobalFiltersSelector = useMemo(() => inputsSelectors.globalFiltersQuerySelector(), []); const globalFilters = useDeepEqualSelector(getGlobalFiltersSelector); @@ -261,7 +264,7 @@ export const Table = memo(({ dataView, groupingFilters, packages }: TableProps) ruleTypeIds={RULE_TYPE_IDS} services={services} toolbarVisibility={TOOLBAR_VISIBILITY} - bulkAddToChatConfig={bulkAddToChatConfig} + bulkAddToChatConfig={maybeBulkAddToChatConfig} /> ); diff --git a/x-pack/solutions/security/plugins/security_solution/public/detections/components/alert_summary/table/table_bulk_add_to_chat.test.tsx b/x-pack/solutions/security/plugins/security_solution/public/detections/components/alert_summary/table/table_bulk_add_to_chat.test.tsx index 182e70cc7c38c..9357e0ccdf4f2 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detections/components/alert_summary/table/table_bulk_add_to_chat.test.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/detections/components/alert_summary/table/table_bulk_add_to_chat.test.tsx @@ -23,6 +23,14 @@ jest.mock('@kbn/response-ops-alerts-table', () => ({ AlertsTable: jest.fn(() => null), })); jest.mock('../../../../agent_builder/hooks/use_report_add_to_chat'); +jest.mock('../../../../agent_builder/hooks/use_agent_builder_availability', () => ({ + useAgentBuilderAvailability: jest.fn(() => ({ + isAgentBuilderEnabled: true, + hasAgentBuilderPrivilege: true, + isAgentChatExperienceEnabled: true, + hasValidAgentBuilderLicense: false, + })), +})); jest.mock('../../../../agent_builder/helpers', () => ({ alertsToAttachmentGroup: jest.fn(() => []), })); @@ -71,14 +79,14 @@ describe('Alert Summary Table — bulkAddToChatConfig', () => { expect(initialMessage).toBe(BULK_ALERTS_ATTACHMENT_PROMPT); }); - it('calls reportAddToChat with bulk_alerts_alert_summary pathway and alert_count', () => { + it('calls reportAddToChat with bulk_alerts_alert_summary pathway and item_count', () => { const { convertAlertToAttachment } = renderAndGetBulkConfig(); const items = [makeItem('a'), makeItem('b'), makeItem('c')]; convertAlertToAttachment(items); expect(mockReportAddToChat).toHaveBeenCalledWith({ pathway: 'bulk_alerts_alert_summary', attachments: ['alert'], - alert_count: 3, + item_count: 3, }); }); diff --git a/x-pack/solutions/security/plugins/security_solution/public/detections/components/alerts_table/index.tsx b/x-pack/solutions/security/plugins/security_solution/public/detections/components/alerts_table/index.tsx index e432e4a5371cb..be2b30b58f88d 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detections/components/alerts_table/index.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/detections/components/alerts_table/index.tsx @@ -39,6 +39,7 @@ import { useGlobalTime } from '../../../common/containers/use_global_time'; import { useLicense } from '../../../common/hooks/use_license'; import { APP_ID, CASES_FEATURE_ID, VIEW_SELECTION } from '../../../../common/constants'; import { useBulkAddToChatConfig } from '../../../agent_builder/hooks/use_bulk_add_to_chat_config'; +import { useAgentBuilderAvailability } from '../../../agent_builder/hooks/use_agent_builder_availability'; import { DEFAULT_COLUMN_MIN_WIDTH } from '../../../timelines/components/timeline/body/constants'; import { defaultRowRenderers } from '../../../timelines/components/timeline/body/renderers'; import { eventsDefaultModel } from '../../../common/components/events_viewer/default_model'; @@ -460,11 +461,13 @@ const AlertsTableComponent: FC onLoad(alerts), [onLoad]); + const { isAgentBuilderEnabled } = useAgentBuilderAvailability(); const pathway = tableType === TableId.alertsOnRuleDetailsPage ? ('bulk_alerts_rule_details' as const) : ('bulk_alerts_alerts_page' as const); const bulkAddToChatConfig = useBulkAddToChatConfig(pathway); + const maybeBulkAddToChatConfig = isAgentBuilderEnabled ? bulkAddToChatConfig : undefined; /** * We want to hide additional controls (like grouping) if the table is being rendered @@ -523,7 +526,7 @@ const AlertsTableComponent: FC diff --git a/x-pack/solutions/security/plugins/security_solution/public/detections/components/alerts_table/table_bulk_add_to_chat.test.tsx b/x-pack/solutions/security/plugins/security_solution/public/detections/components/alerts_table/table_bulk_add_to_chat.test.tsx index 037029d042269..f8cdfecb5c261 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detections/components/alerts_table/table_bulk_add_to_chat.test.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/detections/components/alerts_table/table_bulk_add_to_chat.test.tsx @@ -20,6 +20,14 @@ jest.mock('@kbn/response-ops-alerts-table', () => ({ AlertsTable: jest.fn(() => null), })); jest.mock('../../../agent_builder/hooks/use_report_add_to_chat'); +jest.mock('../../../agent_builder/hooks/use_agent_builder_availability', () => ({ + useAgentBuilderAvailability: jest.fn(() => ({ + isAgentBuilderEnabled: true, + hasAgentBuilderPrivilege: true, + isAgentChatExperienceEnabled: true, + hasValidAgentBuilderLicense: false, + })), +})); jest.mock('../../../agent_builder/helpers', () => ({ alertsToAttachmentGroup: jest.fn(), })); @@ -153,7 +161,7 @@ describe('Alerts Page Table — bulkAddToChatConfig', () => { expect(mockReportAddToChat).toHaveBeenCalledWith({ pathway: 'bulk_alerts_alerts_page', attachments: ['alert'], - alert_count: 2, + item_count: 2, }); }); @@ -164,7 +172,7 @@ describe('Alerts Page Table — bulkAddToChatConfig', () => { expect(mockReportAddToChat).toHaveBeenCalledWith({ pathway: 'bulk_alerts_rule_details', attachments: ['alert'], - alert_count: 1, + item_count: 1, }); }); From e4883c755c49ae8229d35023cd41ec78257be1ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20W=C3=A5lstedt?= Date: Tue, 26 May 2026 11:17:26 +0200 Subject: [PATCH 03/14] add buildkite clie (bk) config to gitignore file --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 9b567a49dd169..7b339513dca09 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .aws-config.json +.bk.yaml .signing-config.json .ackrc /.es From bc8bdbf919903ea375a063a3f6c7e8af0d996e81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20W=C3=A5lstedt?= Date: Tue, 26 May 2026 13:27:51 +0200 Subject: [PATCH 04/14] test(security_solution): fix failing tests for bulk adding alerts to chat --- .../alerts_tab/ease/table_bulk_add_to_chat.test.tsx | 12 ++++++++++-- .../components/ease/table_bulk_add_to_chat.test.tsx | 12 ++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/public/attack_discovery/pages/results/attack_discovery_panel/tabs/alerts_tab/ease/table_bulk_add_to_chat.test.tsx b/x-pack/solutions/security/plugins/security_solution/public/attack_discovery/pages/results/attack_discovery_panel/tabs/alerts_tab/ease/table_bulk_add_to_chat.test.tsx index 864f5b7488a6e..85e7d078790cd 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/attack_discovery/pages/results/attack_discovery_panel/tabs/alerts_tab/ease/table_bulk_add_to_chat.test.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/attack_discovery/pages/results/attack_discovery_panel/tabs/alerts_tab/ease/table_bulk_add_to_chat.test.tsx @@ -23,6 +23,14 @@ jest.mock('@kbn/response-ops-alerts-table', () => ({ AlertsTable: jest.fn(() => null), })); jest.mock('../../../../../../../agent_builder/hooks/use_report_add_to_chat'); +jest.mock('../../../../../../../agent_builder/hooks/use_agent_builder_availability', () => ({ + useAgentBuilderAvailability: jest.fn(() => ({ + isAgentBuilderEnabled: true, + hasAgentBuilderPrivilege: true, + isAgentChatExperienceEnabled: true, + hasValidAgentBuilderLicense: false, + })), +})); jest.mock('../../../../../../../agent_builder/helpers', () => ({ alertsToAttachmentGroup: jest.fn(() => []), })); @@ -76,14 +84,14 @@ describe('Attack Discovery Table — bulkAddToChatConfig', () => { expect(initialMessage).toBe(BULK_ALERTS_ATTACHMENT_PROMPT); }); - it('calls reportAddToChat with bulk_alerts_attack_discovery pathway and alert_count', () => { + it('calls reportAddToChat with bulk_alerts_attack_discovery pathway and item_count', () => { const { convertAlertToAttachment } = renderAndGetBulkConfig(); const items = [makeItem('a'), makeItem('b')]; convertAlertToAttachment(items); expect(mockReportAddToChat).toHaveBeenCalledWith({ pathway: 'bulk_alerts_attack_discovery', attachments: ['alert'], - alert_count: 2, + item_count: 2, }); }); diff --git a/x-pack/solutions/security/plugins/security_solution/public/cases/components/ease/table_bulk_add_to_chat.test.tsx b/x-pack/solutions/security/plugins/security_solution/public/cases/components/ease/table_bulk_add_to_chat.test.tsx index 1ca2bef6f897e..a5304d128afaa 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/cases/components/ease/table_bulk_add_to_chat.test.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/cases/components/ease/table_bulk_add_to_chat.test.tsx @@ -23,6 +23,14 @@ jest.mock('@kbn/response-ops-alerts-table', () => ({ AlertsTable: jest.fn(() => null), })); jest.mock('../../../agent_builder/hooks/use_report_add_to_chat'); +jest.mock('../../../agent_builder/hooks/use_agent_builder_availability', () => ({ + useAgentBuilderAvailability: jest.fn(() => ({ + isAgentBuilderEnabled: true, + hasAgentBuilderPrivilege: true, + isAgentChatExperienceEnabled: true, + hasValidAgentBuilderLicense: false, + })), +})); jest.mock('../../../agent_builder/helpers', () => ({ alertsToAttachmentGroup: jest.fn(() => []), })); @@ -73,14 +81,14 @@ describe('Cases Table — bulkAddToChatConfig', () => { expect(initialMessage).toBe(BULK_ALERTS_ATTACHMENT_PROMPT); }); - it('calls reportAddToChat with bulk_alerts_cases pathway and alert_count', () => { + it('calls reportAddToChat with bulk_alerts_cases pathway and item_count', () => { const { convertAlertToAttachment } = renderAndGetBulkConfig(); const items = [makeItem('a'), makeItem('b')]; convertAlertToAttachment(items); expect(mockReportAddToChat).toHaveBeenCalledWith({ pathway: 'bulk_alerts_cases', attachments: ['alert'], - alert_count: 2, + item_count: 2, }); }); From e0a191be9d4db9dedb8ad3f0495624df4552e446 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20W=C3=A5lstedt?= Date: Tue, 26 May 2026 21:58:12 +0200 Subject: [PATCH 05/14] fix(security): add max length constraint to alertIds string schema Addresses CodeQL unbounded-string alert: adds .max(512) to the z.string() inside the alertIds array schema (ES _id values are at most 512 bytes). Co-Authored-By: Claude Sonnet 4.6 --- .../server/agent_builder/attachments/alerts.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/attachments/alerts.ts b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/attachments/alerts.ts index 16ce348937f55..75ca5663a9608 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/attachments/alerts.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/attachments/alerts.ts @@ -22,7 +22,7 @@ import { getAlertsById } from '../tools/get_alerts_by_id'; import { securityAttachmentDataSchema } from './security_attachment_data_schema'; export const bulkAlertsAttachmentDataSchema = securityAttachmentDataSchema.extend({ - alertIds: z.array(z.string()).min(1).max(ALERTS_BATCH_MAX_SIZE), + alertIds: z.array(z.string().max(512)).min(1).max(ALERTS_BATCH_MAX_SIZE), }); export type BulkAlertsAttachmentData = z.infer; From 966056f0cd4e730b848025194abdb48c1d73d58d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20W=C3=A5lstedt?= Date: Thu, 28 May 2026 13:36:45 +0200 Subject: [PATCH 06/14] Update x-pack/solutions/security/plugins/security_solution/public/agent_builder/attachment_types/index.ts Update casing in copy Co-authored-by: Florent LB --- .../public/agent_builder/attachment_types/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/solutions/security/plugins/security_solution/public/agent_builder/attachment_types/index.ts b/x-pack/solutions/security/plugins/security_solution/public/agent_builder/attachment_types/index.ts index 1a65dd1003905..521ed9845bf60 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/agent_builder/attachment_types/index.ts +++ b/x-pack/solutions/security/plugins/security_solution/public/agent_builder/attachment_types/index.ts @@ -43,7 +43,7 @@ const ALERT_ATTACHMENT_CONFIG: AttachmentTypeConfig = { const ALERTS_DEFAULT_LABEL = i18n.translate( 'xpack.securitySolution.agentBuilder.attachments.alerts.label', - { defaultMessage: 'Security Alerts' } + { defaultMessage: 'Security alerts' } ); const createAttachmentTypeConfig = (defaultLabel: string, icon: string) => ({ From de26f65d113d60ff5c21a2ecb6ccf72f49abd5bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20W=C3=A5lstedt?= Date: Thu, 28 May 2026 13:37:19 +0200 Subject: [PATCH 07/14] Update x-pack/solutions/security/plugins/security_solution/public/agent_builder/attachment_types/index.ts Update casing in copy Co-authored-by: Florent LB --- .../public/agent_builder/attachment_types/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/solutions/security/plugins/security_solution/public/agent_builder/attachment_types/index.ts b/x-pack/solutions/security/plugins/security_solution/public/agent_builder/attachment_types/index.ts index 521ed9845bf60..b60a2cb2ad37c 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/agent_builder/attachment_types/index.ts +++ b/x-pack/solutions/security/plugins/security_solution/public/agent_builder/attachment_types/index.ts @@ -76,7 +76,7 @@ export const registerAttachmentUiDefinitions = (attachments: AttachmentServiceSt const count = attachment.data?.alertIds?.length ?? 0; return count > 0 ? i18n.translate('xpack.securitySolution.agentBuilder.attachments.alerts.countLabel', { - defaultMessage: '{count} {count, plural, one {Alert} other {Alerts}}', + defaultMessage: '{count} {count, plural, one {alert} other {alerts}}', values: { count }, }) : ALERTS_DEFAULT_LABEL; From 8f019e180419aa3970de78c91ff04765b00910e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20W=C3=A5lstedt?= Date: Fri, 29 May 2026 11:56:20 +0200 Subject: [PATCH 08/14] test(kbn-evals-suite-security-alert-triage): address PR feedback --- .../evals/alert_triage_quality.spec.ts | 110 +++++++++++++----- .../evals/bulk_alerts_attachment_read.spec.ts | 65 ++--------- .../src/converse_task.ts | 51 ++++++++ .../src/evaluators.ts | 37 ++++++ 4 files changed, 177 insertions(+), 86 deletions(-) create mode 100644 x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/src/converse_task.ts create mode 100644 x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/src/evaluators.ts diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/evals/alert_triage_quality.spec.ts b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/evals/alert_triage_quality.spec.ts index 4bf1179f6a960..88e5309dda510 100644 --- a/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/evals/alert_triage_quality.spec.ts +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/evals/alert_triage_quality.spec.ts @@ -8,10 +8,18 @@ /** * Quality eval: does the LLM produce a useful triage given real alert data? * - * Two scenarios run in inline mode (1 batch each → ≤5 attachments → LLM sees - * all data directly without needing attachment_read). This isolates output - * quality from the attachment-read compliance already covered by - * bulk_alerts_attachment_read.spec.ts. + * Three scenarios: + * 1. Priority triage (inline mode, 100 alerts / 5 batches ≤ threshold): + * LLM sees all data directly; tests severity prioritisation. + * 2. Entity correlation (inline mode, 100 alerts / 5 batches): + * Tests whether the LLM spots the host targeted by multiple alerts. + * 3. End-to-end / summary mode (200 alerts / 10 batches > threshold): + * Uses both alert groups so attachment_read is required before + * the LLM can reason over the data. Covers the full shipped path: + * bulk selection → summary mode → attachment_read → triage real alerts. + * + * All scenarios include a grounding criterion that fails when the LLM + * invents hosts or rule names not present in the attached alert data. * * Synthetic alerts are indexed into .alerts-security.alerts-default in * beforeAll and cleaned up in afterAll so no external snapshot is required. @@ -26,12 +34,14 @@ import { type ExperimentTask, type TaskOutput, } from '@kbn/evals'; -import { agentBuilderDefaultAgentId } from '@kbn/agent-builder-common'; import type { ToolingLog } from '@kbn/tooling-log'; import type { HttpHandler } from '@kbn/core/public'; import { evaluate as base } from '../src/evaluate'; +import { callConverse } from '../src/converse_task'; +import { attachmentReadCompliance } from '../src/evaluators'; import { ALL_TRIAGE_EVAL_ALERTS, + ALL_TRIAGE_EVAL_IDS, ENTITY_CORRELATION_IDS, PRIORITY_TRIAGE_IDS, } from '../src/synthetic_alerts'; @@ -59,6 +69,7 @@ interface TriageEvalExample { output: { expected: string }; metadata?: { attachments?: Array<{ type: string; data?: unknown }>; + expectedAttachmentReads?: number; }; } @@ -85,36 +96,19 @@ function createEvaluateTriageQuality({ criteria: string[]; }) { const task: ExperimentTask = async ({ input, metadata }) => { - log.info(`Running triage eval task: "${input.question.slice(0, 80)}..."`); - const { attachments = [] } = metadata ?? {}; - - const raw = (await fetch('/api/agent_builder/converse', { - method: 'POST', - version: '2023-10-31', - body: JSON.stringify({ - agent_id: agentBuilderDefaultAgentId, - connector_id: connector.id, - input: input.question, - attachments, - }), - })) as { - conversation_id: string; - trace_id?: string; - steps: unknown[]; - response: { message: string }; - }; - - return { - errors: [], - messages: [{ message: input.question }, raw.response], - steps: raw.steps ?? [], - traceId: raw.trace_id, - }; + return callConverse({ + fetch, + connectorId: connector.id, + question: input.question, + attachments, + log, + }); }; const selectedEvaluators = selectEvaluators([ evaluators.criteria(criteria), + attachmentReadCompliance, ...Object.values(evaluators.traceBasedEvaluators), ]); @@ -205,8 +199,8 @@ evaluate.describe( }, output: { expected: - 'The response should prioritise the critical PowerShell alert on workstation-42, ' + - 'address the high-severity credential-dumping and network-scanning alerts, ' + + 'The response should prioritise the critical Memory Injection via Process Hollowing alert on laptop-finance-07, ' + + 'address the high-severity credential-dumping and lateral-movement alerts, ' + 'list the affected host names, and provide specific remediation steps for each.', }, metadata: { @@ -220,6 +214,7 @@ evaluate.describe( 'The response mentions specific host names present in the alert data', 'The response includes concrete remediation steps, not just a description of the alerts', 'The response distinguishes between urgent and lower-priority items', + 'The response does not reference any hostname, username, or rule name that is not present in the attached alert data', ], }); } @@ -258,6 +253,57 @@ evaluate.describe( 'The response identifies that one specific host appears in multiple alerts', 'The response names web-server-01 as the repeatedly targeted host', 'The response treats the repeated targeting of a single host as a higher-risk indicator than the single-alert hosts', + 'The response does not reference any hostname, username, or rule name that is not present in the attached alert data', + ], + }); + } + ); + + evaluate( + 'End-to-end — summary mode with real alert data exercises attachment_read', + async ({ evaluateTriageQuality }) => { + // ALL_TRIAGE_EVAL_IDS = 200 alerts → 10 batches > inline threshold of 5 + // → framework switches to summary mode → LLM must call attachment_read + // to read each batch before it can reason over real alert content. + // This is the only scenario that covers the full shipped path. + const E2E_BATCH_COUNT = Math.ceil(ALL_TRIAGE_EVAL_IDS.length / ALERTS_BATCH_MAX_SIZE); + await evaluateTriageQuality({ + dataset: { + name: 'agent builder: security-alert-triage-e2e-summary-mode', + description: + `End-to-end coverage for the full bulk-add path: ${ALL_TRIAGE_EVAL_IDS.length} real ` + + `alerts produce ${E2E_BATCH_COUNT} batches, triggering summary mode. The LLM must ` + + 'call attachment_read for each batch before reasoning. Combines both the priority-' + + 'triage group (1 critical alert buried at index 49) and the entity-correlation group ' + + '(web-server-01 targeted by 20 alerts showing an attack progression).', + examples: [ + { + input: { + question: + 'Review all these security alerts. Identify the most critical threats, ' + + 'note any hosts or users that appear across multiple alerts, and recommend ' + + 'prioritized actions.', + }, + output: { + expected: + `The response calls attachment_read ${E2E_BATCH_COUNT} times to read each alert ` + + 'batch, identifies the critical Memory Injection via Process Hollowing alert on ' + + 'laptop-finance-07 as the highest priority, identifies web-server-01 as the most ' + + 'frequently targeted host with an attack progression, and provides prioritized ' + + 'remediation steps without referencing hosts or rules not in the data.', + }, + metadata: { + attachments: toAlertAttachments(ALL_TRIAGE_EVAL_IDS), + expectedAttachmentReads: E2E_BATCH_COUNT, + }, + }, + ], + }, + criteria: [ + 'The response treats the critical severity alert as the highest priority item', + 'The response names web-server-01 as a host that appears in multiple alerts', + 'The response provides prioritized remediation steps', + 'The response does not reference any hostname, username, or rule name that is not present in the attached alert data', ], }); } diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/evals/bulk_alerts_attachment_read.spec.ts b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/evals/bulk_alerts_attachment_read.spec.ts index 980e6b7784862..11450c141e1e8 100644 --- a/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/evals/bulk_alerts_attachment_read.spec.ts +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/evals/bulk_alerts_attachment_read.spec.ts @@ -19,18 +19,17 @@ import { tags } from '@kbn/scout'; import { - getToolCallSteps, selectEvaluators, type DefaultEvaluators, type EvalsExecutorClient, type ExperimentTask, type TaskOutput, } from '@kbn/evals'; -import { attachmentTools, agentBuilderDefaultAgentId } from '@kbn/agent-builder-common'; -import type { EsClient } from '@kbn/scout'; import type { ToolingLog } from '@kbn/tooling-log'; import type { HttpHandler } from '@kbn/core/public'; import { evaluate as base } from '../src/evaluate'; +import { callConverse } from '../src/converse_task'; +import { attachmentReadCompliance } from '../src/evaluators'; // ── Dataset constants ────────────────────────────────────────────────────────── @@ -69,14 +68,12 @@ function createEvaluateAlertBatches({ connector, evaluators, executorClient, - traceEsClient, log, }: { fetch: HttpHandler; connector: { id: string }; evaluators: DefaultEvaluators; executorClient: EvalsExecutorClient; - traceEsClient: EsClient; log: ToolingLog; }) { return async function evaluateAlertBatches({ @@ -85,57 +82,18 @@ function createEvaluateAlertBatches({ dataset: { name: string; description: string; examples: AlertEvalExample[] }; }) { const task: ExperimentTask = async ({ input, metadata }) => { - log.info('Calling converse with alert batch attachments'); - const attachments = metadata?.attachments ?? []; - - const raw = (await fetch('/api/agent_builder/converse', { - method: 'POST', - version: '2023-10-31', - body: JSON.stringify({ - agent_id: agentBuilderDefaultAgentId, - connector_id: connector.id, - input: input.question, - attachments, - }), - })) as { - conversation_id: string; - trace_id?: string; - steps: unknown[]; - response: { message: string }; - }; - - return { - errors: [], - messages: [{ message: input.question }, raw.response], - steps: raw.steps ?? [], - traceId: raw.trace_id, - }; + return callConverse({ + fetch, + connectorId: connector.id, + question: input.question, + attachments, + log, + }); }; const selectedEvaluators = selectEvaluators([ - { - name: 'AttachmentReadCompliance', - kind: 'CODE' as const, - evaluate: async ({ output, metadata }) => { - const expected = - typeof metadata?.expectedAttachmentReads === 'number' - ? metadata.expectedAttachmentReads - : 0; - - if (!expected) { - return { score: 1 }; - } - - const toolCalls = getToolCallSteps(output as TaskOutput); - const readCalls = toolCalls.filter((t) => t.tool_id === attachmentTools.read); - - return { - score: Math.min(1, readCalls.length / expected), - metadata: { readCallCount: readCalls.length, expected }, - }; - }, - }, + attachmentReadCompliance, ...Object.values(evaluators.traceBasedEvaluators), ]); @@ -152,14 +110,13 @@ type EvaluateAlertBatches = ReturnType; const evaluate = base.extend<{ evaluateAlertBatches: EvaluateAlertBatches }, {}>({ evaluateAlertBatches: [ - ({ fetch, connector, evaluators, executorClient, traceEsClient, log }, use) => { + ({ fetch, connector, evaluators, executorClient, log }, use) => { use( createEvaluateAlertBatches({ fetch, connector, evaluators, executorClient, - traceEsClient, log, }) ); diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/src/converse_task.ts b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/src/converse_task.ts new file mode 100644 index 0000000000000..a58a11578cb8b --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/src/converse_task.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { agentBuilderDefaultAgentId } from '@kbn/agent-builder-common'; +import type { HttpHandler } from '@kbn/core/public'; +import type { ToolingLog } from '@kbn/tooling-log'; + +interface ConverseResponse { + conversation_id: string; + trace_id?: string; + steps: unknown[]; + response: { message: string }; +} + +export const callConverse = async ({ + fetch, + connectorId, + question, + attachments, + log, +}: { + fetch: HttpHandler; + connectorId: string; + question: string; + attachments: Array<{ type: string; data?: unknown }>; + log: ToolingLog; +}) => { + log.info(`Calling converse: "${question.slice(0, 80)}..."`); + + const raw = (await fetch('/api/agent_builder/converse', { + method: 'POST', + version: '2023-10-31', + body: JSON.stringify({ + agent_id: agentBuilderDefaultAgentId, + connector_id: connectorId, + input: question, + attachments, + }), + })) as ConverseResponse; + + return { + errors: [], + messages: [{ message: question }, raw.response], + steps: raw.steps ?? [], + traceId: raw.trace_id, + }; +}; diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/src/evaluators.ts b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/src/evaluators.ts new file mode 100644 index 0000000000000..ce06cfc82756e --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/src/evaluators.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getToolCallSteps, type TaskOutput } from '@kbn/evals'; +import { attachmentTools } from '@kbn/agent-builder-common'; + +/** + * CODE evaluator: scores whether the LLM called attachment_read the expected + * number of times. Returns score=1 (pass) when the example metadata does not + * set expectedAttachmentReads, so it can be included in eval suites that mix + * inline-mode (no reads expected) and summary-mode (reads required) scenarios. + */ +export const attachmentReadCompliance = { + name: 'AttachmentReadCompliance', + kind: 'CODE' as const, + evaluate: async ({ output, metadata }: { output: unknown; metadata: unknown }) => { + const expectedReads = (metadata as { expectedAttachmentReads?: number } | undefined) + ?.expectedAttachmentReads; + const expected = typeof expectedReads === 'number' ? expectedReads : 0; + + if (!expected) { + return { score: 1 }; + } + + const toolCalls = getToolCallSteps(output as TaskOutput); + const readCalls = toolCalls.filter((t) => t.tool_id === attachmentTools.read); + + return { + score: Math.min(1, readCalls.length / expected), + metadata: { readCallCount: readCalls.length, expected }, + }; + }, +}; From b061b0822b772bedc2be08bd5a3a993f03cf1199 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20W=C3=A5lstedt?= Date: Fri, 29 May 2026 12:43:17 +0200 Subject: [PATCH 09/14] refactor(security/scout): address js-jankisalvi and MadameSheema PR feedback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit js-jankisalvi: - Move ConversationAttachmentInput, OpenChatService, BulkAddToChatConfig exports below the last import in types.ts so all imports are contiguous - Simplify ConversationAttachmentInput to interface { type: string } — the payload is only passed through to openChat so response-ops doesn't need to mirror the attachment schema; Record & { type: string } was tried but breaks TypeScript index-signature compatibility for concrete types like AttachmentGroup - Drop the redundant agentBuilderService guard in useBulkActions; useBulkAddToChatActions already returns [] when the service is absent MadameSheema (Scout best-practices review): - Add waitForRuleAlert() and checkAlertRowCheckbox() to AlertsTablePage, eliminating inline locator chains and the .euiCheckbox__input class selector from the spec - Create AgentBuilderPage page object with conversation, attachmentPillsRow, inputEditor, and submitButton locators; register it in SecurityPageObjects - Export CUSTOM_QUERY_RULE from @kbn/scout-security public barrel; both specs now import from '@kbn/scout-security' instead of the internal subpath - Extract FULL_KIBANA_SECURITY_ROLE to test/scout/ui/common/roles.ts and import it in both specs, replacing the duplicate role definitions - Add defensive pre-test cleanup at the top of beforeEach and scoutSpace.savedObjects.cleanStandardList() to afterEach - Add comment explaining serverless tag exclusion - Scope connector name to scoutSpace.id in the llmProxy fixture Refs #17496 --- .../alerts-table/hooks/use_bulk_actions.ts | 3 +- .../shared/response-ops/alerts-table/types.ts | 39 +- .../packages/kbn-scout-security/index.ts | 3 + .../test/page_objects/agent_builder.ts | 22 ++ .../test/page_objects/alerts_table.ts | 12 + .../fixtures/test/page_objects/index.ts | 3 + .../test/scout/ui/common/roles.ts | 47 +++ .../test/scout/ui/fixtures/index.ts | 4 +- .../bulk_add_alerts_to_chat.spec.ts | 335 ++++++++---------- .../run_workflow_action.spec.ts | 46 +-- 10 files changed, 262 insertions(+), 252 deletions(-) create mode 100644 x-pack/solutions/security/packages/kbn-scout-security/src/playwright/fixtures/test/page_objects/agent_builder.ts create mode 100644 x-pack/solutions/security/plugins/security_solution/test/scout/ui/common/roles.ts diff --git a/x-pack/platform/packages/shared/response-ops/alerts-table/hooks/use_bulk_actions.ts b/x-pack/platform/packages/shared/response-ops/alerts-table/hooks/use_bulk_actions.ts index 2129d3d6a8077..55316cbfaa4ba 100644 --- a/x-pack/platform/packages/shared/response-ops/alerts-table/hooks/use_bulk_actions.ts +++ b/x-pack/platform/packages/shared/response-ops/alerts-table/hooks/use_bulk_actions.ts @@ -546,7 +546,7 @@ export function useBulkActions({ const isSiem = ruleTypeIds?.some(isSiemRuleType); return [ ...caseBulkActions, - ...(agentBuilderService ? addToChatActions : []), + ...addToChatActions, ...(isSiem ? [] : untrackBulkActions), ...(isSiem ? [] : tagsBulkActions), ...(isSiem ? [] : muteBulkActions), @@ -558,7 +558,6 @@ export function useBulkActions({ tagsBulkActions, muteBulkActions, addToChatActions, - agentBuilderService, ]); const bulkActions = useMemo(() => { diff --git a/x-pack/platform/packages/shared/response-ops/alerts-table/types.ts b/x-pack/platform/packages/shared/response-ops/alerts-table/types.ts index 7b186ec2884e4..ae395005acc05 100644 --- a/x-pack/platform/packages/shared/response-ops/alerts-table/types.ts +++ b/x-pack/platform/packages/shared/response-ops/alerts-table/types.ts @@ -43,14 +43,29 @@ import type { SetRequired } from 'type-fest'; import type { MaintenanceWindow } from '@kbn/maintenance-windows-plugin/common'; import type { FieldFormatsStart } from '@kbn/field-formats-plugin/public'; import type { DataPublicPluginStart } from '@kbn/data-plugin/public'; +import type { FieldBrowserOptions } from '@kbn/response-ops-alerts-fields-browser'; +import type { MutedAlerts } from '@kbn/response-ops-alerts-apis/types'; +import type { NotificationsStart } from '@kbn/core-notifications-browser'; +import type { LicensingPluginStart } from '@kbn/licensing-plugin/public'; +import type { ApplicationStart } from '@kbn/core-application-browser'; +import type { SettingsStart } from '@kbn/core-ui-settings-browser'; +import type { RenderingService } from '@kbn/core-rendering-browser'; +import type { IStorageWrapper } from '@kbn/kibana-utils-plugin/public'; +import type { ProjectRouting } from '@kbn/es-query'; +import type { EuiDataGridCellValueElementProps } from '@elastic/eui/src/components/datagrid/data_grid_types'; +import type { EuiContextMenuPanelId } from '@elastic/eui/src/components/context_menu/context_menu'; +import type { AlertFormatter } from '@kbn/alerts-ui-shared/src/common/types'; +import type { Case } from './apis/bulk_get_cases'; +import type { ItemsSelectionState } from './components/tags/items/types'; /** - * A single conversation attachment or a group of attachments. - * Defined structurally here to avoid a compile-time dependency on agent-builder packages. + * A single conversation attachment or a group of attachments. Defined structurally + * here to avoid a compile-time dependency on agent-builder packages. The payload is + * passed through to openChat without inspection, so a broad structural type is enough. */ -export type ConversationAttachmentInput = - | { type: string; data?: unknown; hidden?: boolean } - | { type: 'group'; id: string; label: string; items: Array<{ type: string; data?: unknown }> }; +export interface ConversationAttachmentInput { + type: string; +} /** * Minimal structural interface for the chat service required by the alerts table. @@ -71,20 +86,6 @@ export interface BulkAddToChatConfig { initialMessage?: string; onAddedToChat?: (itemCount: number) => void; } -import type { FieldBrowserOptions } from '@kbn/response-ops-alerts-fields-browser'; -import type { MutedAlerts } from '@kbn/response-ops-alerts-apis/types'; -import type { NotificationsStart } from '@kbn/core-notifications-browser'; -import type { LicensingPluginStart } from '@kbn/licensing-plugin/public'; -import type { ApplicationStart } from '@kbn/core-application-browser'; -import type { SettingsStart } from '@kbn/core-ui-settings-browser'; -import type { RenderingService } from '@kbn/core-rendering-browser'; -import type { IStorageWrapper } from '@kbn/kibana-utils-plugin/public'; -import type { ProjectRouting } from '@kbn/es-query'; -import type { EuiDataGridCellValueElementProps } from '@elastic/eui/src/components/datagrid/data_grid_types'; -import type { EuiContextMenuPanelId } from '@elastic/eui/src/components/context_menu/context_menu'; -import type { AlertFormatter } from '@kbn/alerts-ui-shared/src/common/types'; -import type { Case } from './apis/bulk_get_cases'; -import type { ItemsSelectionState } from './components/tags/items/types'; export interface Consumer { id: AlertConsumers; diff --git a/x-pack/solutions/security/packages/kbn-scout-security/index.ts b/x-pack/solutions/security/packages/kbn-scout-security/index.ts index 8c92fb77cb635..f01f09e46ac70 100644 --- a/x-pack/solutions/security/packages/kbn-scout-security/index.ts +++ b/x-pack/solutions/security/packages/kbn-scout-security/index.ts @@ -8,6 +8,9 @@ // Security-specific test framework export { test, spaceTest } from './src/playwright'; +// Security-specific test constants +export { CUSTOM_QUERY_RULE } from './src/playwright/constants/detection_rules'; + // re-exported test framework from @kbn/scout export { lighthouseTest, apiTest, globalSetupHook, globalTeardownHook, tags } from '@kbn/scout'; diff --git a/x-pack/solutions/security/packages/kbn-scout-security/src/playwright/fixtures/test/page_objects/agent_builder.ts b/x-pack/solutions/security/packages/kbn-scout-security/src/playwright/fixtures/test/page_objects/agent_builder.ts new file mode 100644 index 0000000000000..a9e3b129b9a22 --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-scout-security/src/playwright/fixtures/test/page_objects/agent_builder.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ScoutPage, Locator } from '@kbn/scout'; + +export class AgentBuilderPage { + public conversation: Locator; + public attachmentPillsRow: Locator; + public inputEditor: Locator; + public submitButton: Locator; + + constructor(private readonly page: ScoutPage) { + this.conversation = this.page.testSubj.locator('agentBuilderConversation'); + this.attachmentPillsRow = this.page.testSubj.locator('agentBuilderAttachmentPillsRow'); + this.inputEditor = this.page.testSubj.locator('agentBuilderConversationInputEditor'); + this.submitButton = this.page.testSubj.locator('agentBuilderConversationInputSubmitButton'); + } +} diff --git a/x-pack/solutions/security/packages/kbn-scout-security/src/playwright/fixtures/test/page_objects/alerts_table.ts b/x-pack/solutions/security/packages/kbn-scout-security/src/playwright/fixtures/test/page_objects/alerts_table.ts index 701a3903f429c..f9d248eac9908 100644 --- a/x-pack/solutions/security/packages/kbn-scout-security/src/playwright/fixtures/test/page_objects/alerts_table.ts +++ b/x-pack/solutions/security/packages/kbn-scout-security/src/playwright/fixtures/test/page_objects/alerts_table.ts @@ -83,4 +83,16 @@ export class AlertsTablePage { // Increased timeout to 20 seconds because this page sometimes takes longer to load return this.detectionsAlertsWrapper.waitFor({ state: 'visible', timeout: 20_000 }); } + + async waitForRuleAlert(ruleName: string) { + const cell = this.alertsTable.getByTestId('ruleName').filter({ hasText: ruleName }); + await expect(cell).toBeVisible({ timeout: 60_000 }); + return cell; + } + + async checkAlertRowCheckbox(ruleName: string) { + const cell = this.alertsTable.getByTestId('ruleName').filter({ hasText: ruleName }); + const row = cell.locator('xpath=ancestor::div[contains(@class,"euiDataGridRow")]'); + await row.getByRole('checkbox').check(); + } } diff --git a/x-pack/solutions/security/packages/kbn-scout-security/src/playwright/fixtures/test/page_objects/index.ts b/x-pack/solutions/security/packages/kbn-scout-security/src/playwright/fixtures/test/page_objects/index.ts index 88ab8943f2a7b..7ed5d0f4bc935 100644 --- a/x-pack/solutions/security/packages/kbn-scout-security/src/playwright/fixtures/test/page_objects/index.ts +++ b/x-pack/solutions/security/packages/kbn-scout-security/src/playwright/fixtures/test/page_objects/index.ts @@ -8,6 +8,7 @@ import type { PageObjects, ScoutPage, ScoutTestConfig } from '@kbn/scout'; import { createLazyPageObject } from '@kbn/scout'; import { AlertsTablePage } from './alerts_table'; +import { AgentBuilderPage } from './agent_builder'; import { AlertDetailsRightPanelPage } from './alert_details_right_panel'; import { EntityAnalyticsDashboardsPage } from './entity_analytics_dashboards'; import { EntityAnalyticsManagementPage } from './entity_analytics_management'; @@ -22,6 +23,7 @@ export type { ThreatMatchRuleCreatePage } from './threat_match_rule_create_page' export interface SecurityPageObjects extends PageObjects { alertsTablePage: AlertsTablePage; + agentBuilderPage: AgentBuilderPage; alertDetailsRightPanelPage: AlertDetailsRightPanelPage; entityAnalyticsDashboardsPage: EntityAnalyticsDashboardsPage; entityAnalyticsManagementPage: EntityAnalyticsManagementPage; @@ -42,6 +44,7 @@ export function extendPageObjects( return { ...pageObjects, alertsTablePage: createLazyPageObject(AlertsTablePage, page), + agentBuilderPage: createLazyPageObject(AgentBuilderPage, page), alertDetailsRightPanelPage: createLazyPageObject(AlertDetailsRightPanelPage, page), entityAnalyticsDashboardsPage: createLazyPageObject(EntityAnalyticsDashboardsPage, page), entityAnalyticsManagementPage: createLazyPageObject(EntityAnalyticsManagementPage, page), diff --git a/x-pack/solutions/security/plugins/security_solution/test/scout/ui/common/roles.ts b/x-pack/solutions/security/plugins/security_solution/test/scout/ui/common/roles.ts new file mode 100644 index 0000000000000..72b51b20f77cc --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/test/scout/ui/common/roles.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { KibanaRole } from '@kbn/scout-security'; + +/** + * Role combining security alert index privileges with full Kibana access. + * Shared by tests that need both alert visibility and Kibana-level features + * (e.g. agent builder, workflow management). + */ +export const FULL_KIBANA_SECURITY_ROLE: KibanaRole = { + elasticsearch: { + cluster: ['manage'], + indices: [ + { + names: [ + '.alerts-security*', + '.internal.alerts-security*', + '.siem-signals-*', + 'apm-*-transaction*', + 'traces-apm*', + 'auditbeat-*', + 'endgame-*', + 'filebeat-*', + 'logs-*', + 'packetbeat-*', + 'winlogbeat-*', + 'logstash-*', + '.lists*', + '.items*', + ], + privileges: ['read', 'write'], + }, + ], + }, + kibana: [ + { + base: ['all'], + feature: {}, + spaces: ['*'], + }, + ], +}; diff --git a/x-pack/solutions/security/plugins/security_solution/test/scout/ui/fixtures/index.ts b/x-pack/solutions/security/plugins/security_solution/test/scout/ui/fixtures/index.ts index 33ee8d23401d2..8ec9445a3172a 100644 --- a/x-pack/solutions/security/plugins/security_solution/test/scout/ui/fixtures/index.ts +++ b/x-pack/solutions/security/plugins/security_solution/test/scout/ui/fixtures/index.ts @@ -28,7 +28,7 @@ export const spaceTest = baseSpaceTest.extend<{}, SecuritySolutionWorkerFixtures path: `/s/${scoutSpace.id}/api/actions/connector`, headers: XSRF, body: { - name: 'scout-llm-proxy', + name: `scout-llm-proxy-${scoutSpace.id}`, config: { apiProvider: 'OpenAI', apiUrl: `http://localhost:${proxy.getPort()}`, @@ -50,7 +50,7 @@ export const spaceTest = baseSpaceTest.extend<{}, SecuritySolutionWorkerFixtures const connectors = Array.isArray(list.data) ? list.data : []; await Promise.all( connectors - .filter((c) => c.name === 'scout-llm-proxy') + .filter((c) => c.name === `scout-llm-proxy-${scoutSpace.id}`) .map((c) => kbnClient.request({ method: 'DELETE', diff --git a/x-pack/solutions/security/plugins/security_solution/test/scout/ui/parallel_tests/investigations/bulk_add_alerts_to_chat.spec.ts b/x-pack/solutions/security/plugins/security_solution/test/scout/ui/parallel_tests/investigations/bulk_add_alerts_to_chat.spec.ts index ae2fa758c69e4..39fd3f44777ec 100644 --- a/x-pack/solutions/security/plugins/security_solution/test/scout/ui/parallel_tests/investigations/bulk_add_alerts_to_chat.spec.ts +++ b/x-pack/solutions/security/plugins/security_solution/test/scout/ui/parallel_tests/investigations/bulk_add_alerts_to_chat.spec.ts @@ -5,197 +5,160 @@ * 2.0. */ -import type { KibanaRole } from '@kbn/scout-security'; import { createToolCallMessage } from '@kbn/ftr-llm-proxy'; import { expect } from '@kbn/scout-security/ui'; -import { CUSTOM_QUERY_RULE } from '@kbn/scout-security/src/playwright/constants/detection_rules'; +import { CUSTOM_QUERY_RULE } from '@kbn/scout-security'; import { spaceTest, tags } from '../../fixtures'; +import { FULL_KIBANA_SECURITY_ROLE } from '../../common/roles'; const ALERT_COUNT = 3; -/** - * A role with security alert index privileges and full Kibana access, - * which includes the agentBuilder feature needed for the "Add to chat" bulk action. - */ -const AGENT_BUILDER_ROLE: KibanaRole = { - elasticsearch: { - cluster: ['manage'], - indices: [ - { - names: [ - '.alerts-security*', - '.internal.alerts-security*', - '.siem-signals-*', - 'apm-*-transaction*', - 'traces-apm*', - 'auditbeat-*', - 'endgame-*', - 'filebeat-*', - 'logs-*', - 'packetbeat-*', - 'winlogbeat-*', - 'logstash-*', - '.lists*', - '.items*', - ], - privileges: ['read', 'write'], - }, - ], - }, - kibana: [ - { - base: ['all'], - feature: {}, - spaces: ['*'], - }, - ], -}; - -spaceTest.describe('Bulk add alerts to chat', { tag: [...tags.stateful.classic] }, () => { - // One rule name per created rule; the first is used as the anchor to wait for page readiness. - const ruleNames: string[] = []; - - spaceTest.beforeEach(async ({ browserAuth, apiServices, scoutSpace }) => { - ruleNames.length = 0; - for (let i = 0; i < ALERT_COUNT; i++) { - const name = `${CUSTOM_QUERY_RULE.name}_${scoutSpace.id}_${Date.now()}_${i}`; - ruleNames.push(name); - await apiServices.detectionRule.createCustomQueryRule({ - ...CUSTOM_QUERY_RULE, - name, - rule_id: `scout-bulk-chat-${scoutSpace.id}-${i}`, - }); - } - await browserAuth.loginWithCustomRole(AGENT_BUILDER_ROLE); - }); - - spaceTest.afterEach(async ({ apiServices }) => { - await apiServices.detectionRule.deleteAll(); - await apiServices.detectionAlerts.deleteAll(); - }); - - spaceTest( - 'should disable the "Add to chat" bulk action when all alerts are selected via the query button', - // _llmProxy is declared so the worker fixture provisions the AI connector in the test space; - // without it the bulk action would not appear at all. - async ({ pageObjects, llmProxy: _llmProxy }) => { - const { alertsTablePage } = pageObjects; - - await alertsTablePage.navigate(); - - const firstRuleNameCell = alertsTablePage.alertsTable - .getByTestId('ruleName') - .filter({ hasText: ruleNames[0] }); - await expect(firstRuleNameCell).toBeVisible({ timeout: 60_000 }); - - await spaceTest.step('check one row to activate the bulk toolbar', async () => { - const alertCheckbox = firstRuleNameCell - .locator('xpath=ancestor::div[contains(@class,"euiDataGridRow")]') - .locator('.euiCheckbox__input'); - await alertCheckbox.check(); - }); - - await spaceTest.step('click "Select all" to switch to query-based selection', async () => { - await alertsTablePage.selectAllAlertsButton.click(); - }); - - await spaceTest.step( - 'verify "Add to chat" is present but disabled in the bulk actions menu', - async () => { +spaceTest.describe( + 'Bulk add alerts to chat', + // Serverless excluded: agent builder feature not yet available on serverless + { tag: [...tags.stateful.classic] }, + () => { + // One rule name per created rule; the first is used as the anchor to wait for page readiness. + const ruleNames: string[] = []; + + spaceTest.beforeEach(async ({ browserAuth, apiServices, scoutSpace }) => { + // Defensive cleanup from any prior failed run + await apiServices.detectionRule.deleteAll(); + await apiServices.detectionAlerts.deleteAll(); + await scoutSpace.savedObjects.cleanStandardList(); + + ruleNames.length = 0; + for (let i = 0; i < ALERT_COUNT; i++) { + const name = `${CUSTOM_QUERY_RULE.name}_${scoutSpace.id}_${Date.now()}_${i}`; + ruleNames.push(name); + await apiServices.detectionRule.createCustomQueryRule({ + ...CUSTOM_QUERY_RULE, + name, + rule_id: `scout-bulk-chat-${scoutSpace.id}-${i}`, + }); + } + await browserAuth.loginWithCustomRole(FULL_KIBANA_SECURITY_ROLE); + }); + + spaceTest.afterEach(async ({ apiServices, scoutSpace }) => { + await apiServices.detectionRule.deleteAll(); + await apiServices.detectionAlerts.deleteAll(); + await scoutSpace.savedObjects.cleanStandardList(); + }); + + spaceTest( + 'should disable the "Add to chat" bulk action when all alerts are selected via the query button', + // _llmProxy is declared so the worker fixture provisions the AI connector in the test space; + // without it the bulk action would not appear at all. + async ({ pageObjects, llmProxy: _llmProxy }) => { + const { alertsTablePage } = pageObjects; + + await alertsTablePage.navigate(); + await alertsTablePage.waitForRuleAlert(ruleNames[0]); + + await spaceTest.step('check one row to activate the bulk toolbar', async () => { + await alertsTablePage.checkAlertRowCheckbox(ruleNames[0]); + }); + + await spaceTest.step('click "Select all" to switch to query-based selection', async () => { + await alertsTablePage.selectAllAlertsButton.click(); + }); + + await spaceTest.step( + 'verify "Add to chat" is present but disabled in the bulk actions menu', + async () => { + await alertsTablePage.selectedShowBulkActionsButton.click(); + await expect(alertsTablePage.bulkAddToChatMenuItem).toBeVisible(); + await expect(alertsTablePage.bulkAddToChatMenuItem).toBeDisabled(); + } + ); + } + ); + + spaceTest( + 'should open agent builder conversation with attachment chip and pre-filled prompt, and send multiple alerts to the LLM', + async ({ pageObjects, llmProxy }) => { + const { alertsTablePage, agentBuilderPage } = pageObjects; + + // Set up LLM mock interceptors before triggering the chat so no call goes unanswered. + // The agent builder makes three sequential calls: title generation, handover, and final answer. + void llmProxy + .intercept({ + name: 'set_title', + when: (body) => { + const sys = body.messages.find((m) => m.role === 'system'); + return String(sys?.content ?? '').includes('You are a title-generation utility'); + }, + responseMock: createToolCallMessage('set_title', { title: 'Alert triage' }), + }) + .completeAfterIntercept(); + + void llmProxy + .intercept({ + name: 'handover-to-answer', + when: (body) => { + const sys = body.messages.find((m) => m.role === 'system'); + return String(sys?.content ?? '').includes( + 'This response will serve as a handover note for the answering agent' + ); + }, + responseMock: 'ready to answer', + }) + .completeAfterIntercept(); + + void llmProxy + .intercept({ + name: 'final-answer', + when: () => true, + responseMock: 'Here is the alert triage summary.', + }) + .completeAfterIntercept(); + + await alertsTablePage.navigate(); + + // Wait for the first rule's alert to appear — this proves Sourcerer has initialized + // and the alerts table is ready. Longer timeout covers the full initialization window. + await alertsTablePage.waitForRuleAlert(ruleNames[0]); + + await spaceTest.step('select all visible alerts via header checkbox', async () => { + // The header checkbox uses selectCurrentPage (not selectAll/query mode), so + // disableOnQuery actions like bulk-add-to-chat remain enabled. + await alertsTablePage.bulkActionsHeaderCheckbox.check(); + }); + + await spaceTest.step('open bulk actions and click Add to chat', async () => { await alertsTablePage.selectedShowBulkActionsButton.click(); - await expect(alertsTablePage.bulkAddToChatMenuItem).toBeVisible(); - await expect(alertsTablePage.bulkAddToChatMenuItem).toBeDisabled(); - } - ); - } - ); - - spaceTest( - 'should open agent builder conversation with attachment chip and pre-filled prompt, and send multiple alerts to the LLM', - async ({ pageObjects, page, llmProxy }) => { - const { alertsTablePage } = pageObjects; - - // Set up LLM mock interceptors before triggering the chat so no call goes unanswered. - // The agent builder makes three sequential calls: title generation, handover, and final answer. - void llmProxy - .intercept({ - name: 'set_title', - when: (body) => { - const sys = body.messages.find((m) => m.role === 'system'); - return String(sys?.content ?? '').includes('You are a title-generation utility'); - }, - responseMock: createToolCallMessage('set_title', { title: 'Alert triage' }), - }) - .completeAfterIntercept(); - - void llmProxy - .intercept({ - name: 'handover-to-answer', - when: (body) => { - const sys = body.messages.find((m) => m.role === 'system'); - return String(sys?.content ?? '').includes( - 'This response will serve as a handover note for the answering agent' + await alertsTablePage.bulkAddToChatMenuItem.click(); + }); + + await spaceTest.step( + 'verify conversation opens with attachment chip and pre-filled prompt', + async () => { + await expect(agentBuilderPage.conversation).toBeVisible(); + await expect(agentBuilderPage.attachmentPillsRow).toBeVisible(); + await expect(agentBuilderPage.inputEditor).toContainText( + 'Triage and prioritize these security alerts' ); - }, - responseMock: 'ready to answer', - }) - .completeAfterIntercept(); - - void llmProxy - .intercept({ - name: 'final-answer', - when: () => true, - responseMock: 'Here is the alert triage summary.', - }) - .completeAfterIntercept(); - - await alertsTablePage.navigate(); - - // Wait for the first rule's alert to appear — this proves Sourcerer has initialized - // and the alerts table is ready. Longer timeout covers the full initialization window. - const firstRuleNameCell = alertsTablePage.alertsTable - .getByTestId('ruleName') - .filter({ hasText: ruleNames[0] }); - await expect(firstRuleNameCell).toBeVisible({ timeout: 60_000 }); - - await spaceTest.step('select all visible alerts via header checkbox', async () => { - // The header checkbox uses selectCurrentPage (not selectAll/query mode), so - // disableOnQuery actions like bulk-add-to-chat remain enabled. - await alertsTablePage.bulkActionsHeaderCheckbox.check(); - }); - - await spaceTest.step('open bulk actions and click Add to chat', async () => { - await alertsTablePage.selectedShowBulkActionsButton.click(); - await alertsTablePage.bulkAddToChatMenuItem.click(); - }); - - await spaceTest.step( - 'verify conversation opens with attachment chip and pre-filled prompt', - async () => { - await expect(page.testSubj.locator('agentBuilderConversation')).toBeVisible(); - await expect(page.testSubj.locator('agentBuilderAttachmentPillsRow')).toBeVisible(); - await expect(page.testSubj.locator('agentBuilderConversationInputEditor')).toContainText( - 'Triage and prioritize these security alerts' - ); - } - ); - - await spaceTest.step('send message and verify LLM received multiple alerts', async () => { - await page.testSubj.locator('agentBuilderConversationInputSubmitButton').click(); - await llmProxy.waitForAllInterceptorsToHaveBeenCalled(); - - const handoverRequest = llmProxy.interceptedRequests.find( - (r) => r.matchingInterceptorName === 'handover-to-answer' - )?.requestBody; - - expect(handoverRequest).toBeDefined(); - const messages: Array<{ content?: unknown }> = handoverRequest?.messages ?? []; - const allContent = messages.map((m) => String(m.content ?? '')).join('\n'); - - // Verify multiple alerts were delivered: each alert block contains the rule name field. - const alertFieldOccurrences = allContent.split('kibana.alert.rule.name').length - 1; - expect(alertFieldOccurrences).toBeGreaterThanOrEqual(ALERT_COUNT); - }); - } - ); -}); + } + ); + + await spaceTest.step('send message and verify LLM received multiple alerts', async () => { + await agentBuilderPage.submitButton.click(); + await llmProxy.waitForAllInterceptorsToHaveBeenCalled(); + + const handoverRequest = llmProxy.interceptedRequests.find( + (r) => r.matchingInterceptorName === 'handover-to-answer' + )?.requestBody; + + expect(handoverRequest).toBeDefined(); + const messages: Array<{ content?: unknown }> = handoverRequest?.messages ?? []; + const allContent = messages.map((m) => String(m.content ?? '')).join('\n'); + + // Verify multiple alerts were delivered: each alert block contains the rule name field. + const alertFieldOccurrences = allContent.split('kibana.alert.rule.name').length - 1; + expect(alertFieldOccurrences).toBeGreaterThanOrEqual(ALERT_COUNT); + }); + } + ); + } +); diff --git a/x-pack/solutions/security/plugins/security_solution/test/scout/ui/parallel_tests/investigations/run_workflow_action.spec.ts b/x-pack/solutions/security/plugins/security_solution/test/scout/ui/parallel_tests/investigations/run_workflow_action.spec.ts index 3311d2c16e75b..abf89c810b7f0 100644 --- a/x-pack/solutions/security/plugins/security_solution/test/scout/ui/parallel_tests/investigations/run_workflow_action.spec.ts +++ b/x-pack/solutions/security/plugins/security_solution/test/scout/ui/parallel_tests/investigations/run_workflow_action.spec.ts @@ -5,49 +5,9 @@ * 2.0. */ -import type { KibanaRole } from '@kbn/scout-security'; -import { spaceTest, tags } from '@kbn/scout-security'; +import { spaceTest, tags, CUSTOM_QUERY_RULE } from '@kbn/scout-security'; import { expect } from '@kbn/scout-security/ui'; -import { CUSTOM_QUERY_RULE } from '@kbn/scout-security/src/playwright/constants/detection_rules'; - -/** - * A role combining the security alert index privileges of platform_engineer with - * full Kibana access (including workflowsManagement) required to see and use the - * "Run workflow" alert action. - */ -const WORKFLOW_ENABLED_ROLE: KibanaRole = { - elasticsearch: { - cluster: ['manage'], - indices: [ - { - names: [ - '.alerts-security*', - '.internal.alerts-security*', - '.siem-signals-*', - 'apm-*-transaction*', - 'traces-apm*', - 'auditbeat-*', - 'endgame-*', - 'filebeat-*', - 'logs-*', - 'packetbeat-*', - 'winlogbeat-*', - 'logstash-*', - '.lists*', - '.items*', - ], - privileges: ['read', 'write'], - }, - ], - }, - kibana: [ - { - base: ['all'], - feature: {}, - spaces: ['*'], - }, - ], -}; +import { FULL_KIBANA_SECURITY_ROLE } from '../../common/roles'; // Failing: See https://github.com/elastic/kibana/issues/261392 spaceTest.describe.skip('Run workflow alert action', { tag: [...tags.stateful.classic] }, () => { @@ -66,7 +26,7 @@ spaceTest.describe.skip('Run workflow alert action', { tag: [...tags.stateful.cl }); // Use a custom role that includes workflowsManagement privileges (canExecuteWorkflow) // in addition to the security index privileges needed to view alerts - await browserAuth.loginWithCustomRole(WORKFLOW_ENABLED_ROLE); + await browserAuth.loginWithCustomRole(FULL_KIBANA_SECURITY_ROLE); }); spaceTest.afterEach(async ({ apiServices }) => { From d45a58ae237a8bad8b68192105ff8f120790e667 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20W=C3=A5lstedt?= Date: Fri, 29 May 2026 14:05:32 +0200 Subject: [PATCH 10/14] fix(security/scout): increase attachmentPillsRow assertion timeout to 30s The attachment pills row renders one React context propagation cycle after the outer conversation div becomes visible, so the default 10s timeout is not reliable in slower CI environments. Adds waitForAttachmentPillsRow() to AgentBuilderPage with an explicit 30s timeout. Refs #17496 --- .../fixtures/test/page_objects/agent_builder.ts | 11 +++++++++++ .../investigations/bulk_add_alerts_to_chat.spec.ts | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/x-pack/solutions/security/packages/kbn-scout-security/src/playwright/fixtures/test/page_objects/agent_builder.ts b/x-pack/solutions/security/packages/kbn-scout-security/src/playwright/fixtures/test/page_objects/agent_builder.ts index a9e3b129b9a22..fafa87d44d5c6 100644 --- a/x-pack/solutions/security/packages/kbn-scout-security/src/playwright/fixtures/test/page_objects/agent_builder.ts +++ b/x-pack/solutions/security/packages/kbn-scout-security/src/playwright/fixtures/test/page_objects/agent_builder.ts @@ -6,6 +6,7 @@ */ import type { ScoutPage, Locator } from '@kbn/scout'; +import { expect } from '../../../../../ui'; export class AgentBuilderPage { public conversation: Locator; @@ -19,4 +20,14 @@ export class AgentBuilderPage { this.inputEditor = this.page.testSubj.locator('agentBuilderConversationInputEditor'); this.submitButton = this.page.testSubj.locator('agentBuilderConversationInputSubmitButton'); } + + /** + * Waits for the attachment pills row to become visible. Uses a longer timeout + * than the default because the pills row renders after the sidebar's React context + * propagates the attachments — one extra render cycle past when the outer + * conversation panel first becomes visible. + */ + async waitForAttachmentPillsRow() { + await expect(this.attachmentPillsRow).toBeVisible({ timeout: 30_000 }); + } } diff --git a/x-pack/solutions/security/plugins/security_solution/test/scout/ui/parallel_tests/investigations/bulk_add_alerts_to_chat.spec.ts b/x-pack/solutions/security/plugins/security_solution/test/scout/ui/parallel_tests/investigations/bulk_add_alerts_to_chat.spec.ts index 39fd3f44777ec..82792160e5569 100644 --- a/x-pack/solutions/security/plugins/security_solution/test/scout/ui/parallel_tests/investigations/bulk_add_alerts_to_chat.spec.ts +++ b/x-pack/solutions/security/plugins/security_solution/test/scout/ui/parallel_tests/investigations/bulk_add_alerts_to_chat.spec.ts @@ -135,7 +135,7 @@ spaceTest.describe( 'verify conversation opens with attachment chip and pre-filled prompt', async () => { await expect(agentBuilderPage.conversation).toBeVisible(); - await expect(agentBuilderPage.attachmentPillsRow).toBeVisible(); + await agentBuilderPage.waitForAttachmentPillsRow(); await expect(agentBuilderPage.inputEditor).toContainText( 'Triage and prioritize these security alerts' ); From 0f58edb86bd5daeb47fc048671a02034dd05970a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20W=C3=A5lstedt?= Date: Fri, 29 May 2026 15:57:01 +0200 Subject: [PATCH 11/14] fix(security/scout): register LLM interceptors immediately before submit The kbn-ftr-llm-proxy 30s timeout starts at intercept() call time. When interceptors were registered at test start, slow CI steps (waitForRuleAlert up to 60s, waitForAttachmentPillsRow up to 30s) could exhaust the timeout before the submit button was ever clicked, producing a spurious "Interceptor set_title timed out" failure. Moving the registration to immediately before submitButton.click() is safe because autoSendInitialMessage: false guarantees no LLM call fires before the user submits. Refs #17496 --- .../bulk_add_alerts_to_chat.spec.ts | 69 ++++++++++--------- 1 file changed, 35 insertions(+), 34 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/test/scout/ui/parallel_tests/investigations/bulk_add_alerts_to_chat.spec.ts b/x-pack/solutions/security/plugins/security_solution/test/scout/ui/parallel_tests/investigations/bulk_add_alerts_to_chat.spec.ts index 82792160e5569..3d80a8e80ca9a 100644 --- a/x-pack/solutions/security/plugins/security_solution/test/scout/ui/parallel_tests/investigations/bulk_add_alerts_to_chat.spec.ts +++ b/x-pack/solutions/security/plugins/security_solution/test/scout/ui/parallel_tests/investigations/bulk_add_alerts_to_chat.spec.ts @@ -80,40 +80,6 @@ spaceTest.describe( async ({ pageObjects, llmProxy }) => { const { alertsTablePage, agentBuilderPage } = pageObjects; - // Set up LLM mock interceptors before triggering the chat so no call goes unanswered. - // The agent builder makes three sequential calls: title generation, handover, and final answer. - void llmProxy - .intercept({ - name: 'set_title', - when: (body) => { - const sys = body.messages.find((m) => m.role === 'system'); - return String(sys?.content ?? '').includes('You are a title-generation utility'); - }, - responseMock: createToolCallMessage('set_title', { title: 'Alert triage' }), - }) - .completeAfterIntercept(); - - void llmProxy - .intercept({ - name: 'handover-to-answer', - when: (body) => { - const sys = body.messages.find((m) => m.role === 'system'); - return String(sys?.content ?? '').includes( - 'This response will serve as a handover note for the answering agent' - ); - }, - responseMock: 'ready to answer', - }) - .completeAfterIntercept(); - - void llmProxy - .intercept({ - name: 'final-answer', - when: () => true, - responseMock: 'Here is the alert triage summary.', - }) - .completeAfterIntercept(); - await alertsTablePage.navigate(); // Wait for the first rule's alert to appear — this proves Sourcerer has initialized @@ -143,6 +109,41 @@ spaceTest.describe( ); await spaceTest.step('send message and verify LLM received multiple alerts', async () => { + // Register interceptors here — immediately before submit — so the 30s proxy + // timeout doesn't expire during the earlier waitForRuleAlert / waitForAttachmentPillsRow + // steps. autoSendInitialMessage: false guarantees no LLM call fires before this point. + void llmProxy + .intercept({ + name: 'set_title', + when: (body) => { + const sys = body.messages.find((m) => m.role === 'system'); + return String(sys?.content ?? '').includes('You are a title-generation utility'); + }, + responseMock: createToolCallMessage('set_title', { title: 'Alert triage' }), + }) + .completeAfterIntercept(); + + void llmProxy + .intercept({ + name: 'handover-to-answer', + when: (body) => { + const sys = body.messages.find((m) => m.role === 'system'); + return String(sys?.content ?? '').includes( + 'This response will serve as a handover note for the answering agent' + ); + }, + responseMock: 'ready to answer', + }) + .completeAfterIntercept(); + + void llmProxy + .intercept({ + name: 'final-answer', + when: () => true, + responseMock: 'Here is the alert triage summary.', + }) + .completeAfterIntercept(); + await agentBuilderPage.submitButton.click(); await llmProxy.waitForAllInterceptorsToHaveBeenCalled(); From 9dd51cb2c2a737d08b72544b0a8ef07213abcdb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20W=C3=A5lstedt?= Date: Fri, 29 May 2026 16:15:25 +0200 Subject: [PATCH 12/14] chore(evals): add security-alert-triage to weekly LLM eval schedule Registers the suite alongside the other security evals so it runs automatically in the weekly pipeline against the core model matrix. Refs #17496 --- .buildkite/pipelines/evals/llm_evals.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.buildkite/pipelines/evals/llm_evals.yml b/.buildkite/pipelines/evals/llm_evals.yml index 4237bc04eebed..5e2ff84fbf8a8 100644 --- a/.buildkite/pipelines/evals/llm_evals.yml +++ b/.buildkite/pipelines/evals/llm_evals.yml @@ -398,3 +398,25 @@ steps: automatic: - exit_status: '-1' limit: 3 + + - label: 'Evals: Security Alert Triage' + key: kbn-evals-weekly-security-alert-triage + command: bash .buildkite/scripts/steps/evals/run_suite.sh + env: + KBN_EVALS: '1' + FTR_EIS_CCM: '1' + EVAL_SUITE_ID: 'security-alert-triage' + EVAL_FANOUT: '1' + EVAL_INCLUDE_EIS_MODELS: '1' + EVAL_MODEL_GROUPS: *weekly_eis_core_models + timeout_in_minutes: 60 + agents: + image: family/kibana-ubuntu-2404 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-8 + preemptible: true + retry: + automatic: + - exit_status: '-1' + limit: 3 From 8589d1088813c67d370c75d8b3922323ca82f415 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20W=C3=A5lstedt?= Date: Fri, 29 May 2026 17:39:10 +0200 Subject: [PATCH 13/14] fix(security/scout): remove cleanStandardList to preserve llmProxy connector cleanStandardList() deletes saved objects of type 'action', which includes the .gen-ai connector created by the worker-scoped llmProxy fixture. Calling it in beforeEach destroyed the connector before the test ran, making the bulk 'Add to chat' action invisible and causing every subsequent step to fail. The connector lifecycle is owned by llmProxy; this test only needs targeted deleteAll() calls for the objects it creates. Refs #17496 --- .../investigations/bulk_add_alerts_to_chat.spec.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/test/scout/ui/parallel_tests/investigations/bulk_add_alerts_to_chat.spec.ts b/x-pack/solutions/security/plugins/security_solution/test/scout/ui/parallel_tests/investigations/bulk_add_alerts_to_chat.spec.ts index 3d80a8e80ca9a..f298bd84a382a 100644 --- a/x-pack/solutions/security/plugins/security_solution/test/scout/ui/parallel_tests/investigations/bulk_add_alerts_to_chat.spec.ts +++ b/x-pack/solutions/security/plugins/security_solution/test/scout/ui/parallel_tests/investigations/bulk_add_alerts_to_chat.spec.ts @@ -22,10 +22,12 @@ spaceTest.describe( const ruleNames: string[] = []; spaceTest.beforeEach(async ({ browserAuth, apiServices, scoutSpace }) => { - // Defensive cleanup from any prior failed run + // Defensive cleanup from any prior failed run. + // Note: cleanStandardList() is intentionally omitted here — it deletes saved objects + // of type 'action', which would destroy the .gen-ai connector created by the + // worker-scoped llmProxy fixture and make the bulk "Add to chat" action disappear. await apiServices.detectionRule.deleteAll(); await apiServices.detectionAlerts.deleteAll(); - await scoutSpace.savedObjects.cleanStandardList(); ruleNames.length = 0; for (let i = 0; i < ALERT_COUNT; i++) { @@ -40,10 +42,9 @@ spaceTest.describe( await browserAuth.loginWithCustomRole(FULL_KIBANA_SECURITY_ROLE); }); - spaceTest.afterEach(async ({ apiServices, scoutSpace }) => { + spaceTest.afterEach(async ({ apiServices }) => { await apiServices.detectionRule.deleteAll(); await apiServices.detectionAlerts.deleteAll(); - await scoutSpace.savedObjects.cleanStandardList(); }); spaceTest( From 18156b158aeb0572f7e388a66ac6e7152c4c0985 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=20W=C3=A5lstedt?= Date: Mon, 1 Jun 2026 10:43:58 +0200 Subject: [PATCH 14/14] fix(kbn-evals-suite-security-alert-triage): update for kbn-evals API changes @kbn/evals updated its API: runExperiment now takes `datasets: T[]` (plural array) instead of `dataset: T` (singular), and Example.input is now optional. Two spec files needed updating: - Switch from `dataset: {...}` to `datasets: [{...} satisfies EvaluationDataset]` - Move task definition inline so TypeScript infers the parameter type from TEvaluationDataset rather than falling back to Example defaults, which triggered a contravariance error on the typed ExperimentTask - Add `extends Example` to TriageEvalExample and AlertEvalExample so they satisfy EvaluationDataset's TExample constraint - Drop ExperimentTask/TaskOutput imports (no longer needed in specs) - Add EvaluationDataset import Refs #17496 --- .../evals/alert_triage_quality.spec.ts | 38 +++++++++---------- .../evals/bulk_alerts_attachment_read.spec.ts | 31 +++++++-------- 2 files changed, 35 insertions(+), 34 deletions(-) diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/evals/alert_triage_quality.spec.ts b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/evals/alert_triage_quality.spec.ts index 88e5309dda510..7a1a382b58193 100644 --- a/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/evals/alert_triage_quality.spec.ts +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/evals/alert_triage_quality.spec.ts @@ -30,9 +30,9 @@ import type { EsClient } from '@kbn/scout'; import { selectEvaluators, type DefaultEvaluators, + type EvaluationDataset, type EvalsExecutorClient, - type ExperimentTask, - type TaskOutput, + type Example, } from '@kbn/evals'; import type { ToolingLog } from '@kbn/tooling-log'; import type { HttpHandler } from '@kbn/core/public'; @@ -64,7 +64,7 @@ const toAlertAttachments = (ids: string[]) => { // ── Types ───────────────────────────────────────────────────────────────────── -interface TriageEvalExample { +interface TriageEvalExample extends Example { input: { question: string }; output: { expected: string }; metadata?: { @@ -95,17 +95,6 @@ function createEvaluateTriageQuality({ dataset: { name: string; description: string; examples: TriageEvalExample[] }; criteria: string[]; }) { - const task: ExperimentTask = async ({ input, metadata }) => { - const { attachments = [] } = metadata ?? {}; - return callConverse({ - fetch, - connectorId: connector.id, - question: input.question, - attachments, - log, - }); - }; - const selectedEvaluators = selectEvaluators([ evaluators.criteria(criteria), attachmentReadCompliance, @@ -114,12 +103,23 @@ function createEvaluateTriageQuality({ await executorClient.runExperiment( { - dataset: { - name: dataset.name, - description: dataset.description, - examples: dataset.examples, + datasets: [ + { + name: dataset.name, + description: dataset.description, + examples: dataset.examples, + } satisfies EvaluationDataset, + ], + task: async ({ input, metadata }) => { + const { attachments = [] } = metadata ?? {}; + return callConverse({ + fetch, + connectorId: connector.id, + question: input.question, + attachments, + log, + }); }, - task, }, selectedEvaluators ); diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/evals/bulk_alerts_attachment_read.spec.ts b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/evals/bulk_alerts_attachment_read.spec.ts index 11450c141e1e8..d1cba6ba32981 100644 --- a/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/evals/bulk_alerts_attachment_read.spec.ts +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/evals/bulk_alerts_attachment_read.spec.ts @@ -21,9 +21,9 @@ import { tags } from '@kbn/scout'; import { selectEvaluators, type DefaultEvaluators, + type EvaluationDataset, type EvalsExecutorClient, - type ExperimentTask, - type TaskOutput, + type Example, } from '@kbn/evals'; import type { ToolingLog } from '@kbn/tooling-log'; import type { HttpHandler } from '@kbn/core/public'; @@ -52,7 +52,7 @@ const alertBatches: Array<{ alertIds: string[] }> = Array.from( // ── Types ────────────────────────────────────────────────────────────────────── -interface AlertEvalExample { +interface AlertEvalExample extends Example { input: { question: string }; output: { expected: string }; metadata?: { @@ -81,24 +81,25 @@ function createEvaluateAlertBatches({ }: { dataset: { name: string; description: string; examples: AlertEvalExample[] }; }) { - const task: ExperimentTask = async ({ input, metadata }) => { - const attachments = metadata?.attachments ?? []; - return callConverse({ - fetch, - connectorId: connector.id, - question: input.question, - attachments, - log, - }); - }; - const selectedEvaluators = selectEvaluators([ attachmentReadCompliance, ...Object.values(evaluators.traceBasedEvaluators), ]); await executorClient.runExperiment( - { dataset: { name, description, examples }, task }, + { + datasets: [{ name, description, examples } satisfies EvaluationDataset], + task: async ({ input, metadata }) => { + const attachments = metadata?.attachments ?? []; + return callConverse({ + fetch, + connectorId: connector.id, + question: input.question, + attachments, + log, + }); + }, + }, selectedEvaluators ); };