diff --git a/.buildkite/pipelines/evals/evals.suites.json b/.buildkite/pipelines/evals/evals.suites.json index 6396660b2e8f0..49b9c5f727c18 100644 --- a/.buildkite/pipelines/evals/evals.suites.json +++ b/.buildkite/pipelines/evals/evals.suites.json @@ -251,6 +251,14 @@ "eis/openai-gpt-5.2", "eis/openai-gpt-5.4" ] + }, + { + "id": "security-alert-triage", + "name": "Security Alert Triage", + "slackChannel": "#security-generative-ai-evals", + "configPath": "x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/playwright.config.ts", + "tags": ["security", "alert-triage"], + "ciLabels": ["evals:security-alert-triage"] } ] } diff --git a/.buildkite/pipelines/evals/llm_evals.yml b/.buildkite/pipelines/evals/llm_evals.yml index 4237bc04eebed..5e2ff84fbf8a8 100644 --- a/.buildkite/pipelines/evals/llm_evals.yml +++ b/.buildkite/pipelines/evals/llm_evals.yml @@ -398,3 +398,25 @@ steps: automatic: - exit_status: '-1' limit: 3 + + - label: 'Evals: Security Alert Triage' + key: kbn-evals-weekly-security-alert-triage + command: bash .buildkite/scripts/steps/evals/run_suite.sh + env: + KBN_EVALS: '1' + FTR_EIS_CCM: '1' + EVAL_SUITE_ID: 'security-alert-triage' + EVAL_FANOUT: '1' + EVAL_INCLUDE_EIS_MODELS: '1' + EVAL_MODEL_GROUPS: *weekly_eis_core_models + timeout_in_minutes: 60 + agents: + image: family/kibana-ubuntu-2404 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-8 + preemptible: true + retry: + automatic: + - exit_status: '-1' + limit: 3 diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 4e288a602193d..0c892f679350f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1341,6 +1341,7 @@ x-pack/solutions/security/packages/kbn-evals-suite-entity-analytics @elastic/sec x-pack/solutions/security/packages/kbn-evals-suite-lead-generation @elastic/security-entity-analytics x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance @elastic/security-defend-workflows x-pack/solutions/security/packages/kbn-evals-suite-security-ai-rules @elastic/security-detection-engine +x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage @elastic/security-threat-hunting x-pack/solutions/security/packages/kbn-evals-suite-security-automatic-migrations @elastic/security-threat-hunting x-pack/solutions/security/packages/kbn-evals-suite-security-esql-generation-regression @elastic/security-detection-platform x-pack/solutions/security/packages/kbn-scout-security @elastic/appex-qa @elastic/security-engineering-productivity diff --git a/.gitignore b/.gitignore index 9b567a49dd169..7b339513dca09 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .aws-config.json +.bk.yaml .signing-config.json .ackrc /.es diff --git a/package.json b/package.json index f8cf9bbd01738..3485576a2445d 100644 --- a/package.json +++ b/package.json @@ -1756,6 +1756,7 @@ "@kbn/evals-suite-observability-ai": "link:x-pack/solutions/observability/packages/kbn-evals-suite-observability-ai", "@kbn/evals-suite-pci-compliance": "link:x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance", "@kbn/evals-suite-security-ai-rules": "link:x-pack/solutions/security/packages/kbn-evals-suite-security-ai-rules", + "@kbn/evals-suite-security-alert-triage": "link:x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage", "@kbn/evals-suite-security-automatic-migrations": "link:x-pack/solutions/security/packages/kbn-evals-suite-security-automatic-migrations", "@kbn/evals-suite-security-esql-generation-regression": "link:x-pack/solutions/security/packages/kbn-evals-suite-security-esql-generation-regression", "@kbn/evals-suite-significant-events": "link:x-pack/platform/packages/shared/kbn-evals-suite-significant-events", diff --git a/tsconfig.base.json b/tsconfig.base.json index 2c4654b964e01..a23a172a244b2 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -1240,6 +1240,8 @@ "@kbn/evals-suite-pci-compliance/*": ["x-pack/solutions/security/packages/kbn-evals-suite-pci-compliance/*"], "@kbn/evals-suite-security-ai-rules": ["x-pack/solutions/security/packages/kbn-evals-suite-security-ai-rules"], "@kbn/evals-suite-security-ai-rules/*": ["x-pack/solutions/security/packages/kbn-evals-suite-security-ai-rules/*"], + "@kbn/evals-suite-security-alert-triage": ["x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage"], + "@kbn/evals-suite-security-alert-triage/*": ["x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/*"], "@kbn/evals-suite-security-automatic-migrations": ["x-pack/solutions/security/packages/kbn-evals-suite-security-automatic-migrations"], "@kbn/evals-suite-security-automatic-migrations/*": ["x-pack/solutions/security/packages/kbn-evals-suite-security-automatic-migrations/*"], "@kbn/evals-suite-security-esql-generation-regression": ["x-pack/solutions/security/packages/kbn-evals-suite-security-esql-generation-regression"], diff --git a/x-pack/platform/packages/shared/response-ops/alerts-table/components/alerts_data_grid.tsx b/x-pack/platform/packages/shared/response-ops/alerts-table/components/alerts_data_grid.tsx index aa09676100076..9662d56e60f87 100644 --- a/x-pack/platform/packages/shared/response-ops/alerts-table/components/alerts_data_grid.tsx +++ b/x-pack/platform/packages/shared/response-ops/alerts-table/components/alerts_data_grid.tsx @@ -80,6 +80,7 @@ export const AlertsDataGrid = typedMemo( cellActionsOptions, pageSizeOptions = DEFAULT_PAGE_SIZE_OPTIONS, height, + bulkAddToChatConfig, ...euiDataGridProps } = props; const { @@ -98,7 +99,14 @@ export const AlertsDataGrid = typedMemo( refresh: refreshQueries, columns, dataGridRef, - services: { http, notifications, application, cases: casesService, settings }, + services: { + http, + notifications, + application, + cases: casesService, + agentBuilder: agentBuilderService, + settings, + }, } = renderContext; const { colorMode, euiTheme } = useEuiTheme(); @@ -126,6 +134,8 @@ export const AlertsDataGrid = typedMemo( notifications, application, casesService, + agentBuilderService, + bulkAddToChatConfig, }); const refresh = useCallback(() => { diff --git a/x-pack/platform/packages/shared/response-ops/alerts-table/hooks/use_bulk_actions.test.tsx b/x-pack/platform/packages/shared/response-ops/alerts-table/hooks/use_bulk_actions.test.tsx index 5fda08070ad11..fab5a941a9692 100644 --- a/x-pack/platform/packages/shared/response-ops/alerts-table/hooks/use_bulk_actions.test.tsx +++ b/x-pack/platform/packages/shared/response-ops/alerts-table/hooks/use_bulk_actions.test.tsx @@ -15,12 +15,13 @@ import { AlertsQueryContext } from '@kbn/alerts-ui-shared/src/common/contexts/al import { useBulkActions, useBulkAddToCaseActions, + useBulkAddToChatActions, useBulkUntrackActions, useBulkMuteActions, } from './use_bulk_actions'; import { createCasesServiceMock } from '../mocks/cases.mock'; import { BulkActionsVerbs, type PublicAlertsDataGridProps } from '../types'; -import type { AdditionalContext, RenderContext } from '../types'; +import type { AdditionalContext, OpenChatService, RenderContext, TimelineItem } from '../types'; import { useAlertsTableContext } from '../contexts/alerts_table_context'; import { createPartialObjectMock, testQueryClientConfig } from '../utils/test'; import { applicationServiceMock } from '@kbn/core-application-browser-mocks'; @@ -427,6 +428,97 @@ describe('bulk action hooks', () => { }); }); + describe('useBulkAddToChatActions', () => { + const mockOpenChat = jest.fn(); + const agentBuilderService: OpenChatService = { openChat: mockOpenChat }; + const mockAttachments = [{ type: 'security.alerts', data: { alertIds: ['id1'] } }]; + const convertAlertToAttachment = jest.fn().mockReturnValue(mockAttachments); + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('returns empty array when agentBuilderService is not provided', () => { + const { result } = renderHook( + () => + useBulkAddToChatActions({ + bulkAddToChatConfig: { convertAlertToAttachment }, + }), + { wrapper } + ); + + expect(result.current).toEqual([]); + }); + + it('returns empty array when bulkAddToChatConfig is not provided', () => { + const { result } = renderHook(() => useBulkAddToChatActions({ agentBuilderService }), { + wrapper, + }); + + expect(result.current).toEqual([]); + }); + + it('returns the add-to-chat action when both service and config are provided', () => { + const { result } = renderHook( + () => + useBulkAddToChatActions({ + agentBuilderService, + bulkAddToChatConfig: { convertAlertToAttachment }, + }), + { wrapper } + ); + + expect(result.current).toHaveLength(1); + expect(result.current[0].key).toBe('bulk-add-to-chat'); + expect(result.current[0]['data-test-subj']).toBe('bulk-add-to-chat'); + }); + + it('calls openChat with converted attachments when the action is clicked', () => { + const alerts: TimelineItem[] = [ + { _id: 'id1', _index: 'idx', data: [], ecs: { _id: 'id1', _index: 'idx' } }, + ]; + + const { result } = renderHook( + () => + useBulkAddToChatActions({ + agentBuilderService, + bulkAddToChatConfig: { convertAlertToAttachment }, + }), + { wrapper } + ); + + result.current[0].onClick(alerts); + + expect(convertAlertToAttachment).toHaveBeenCalledWith(alerts); + expect(mockOpenChat).toHaveBeenCalledWith({ + autoSendInitialMessage: false, + newConversation: true, + initialMessage: undefined, + attachments: mockAttachments, + }); + }); + + it('passes initialMessage to openChat', () => { + const { result } = renderHook( + () => + useBulkAddToChatActions({ + agentBuilderService, + bulkAddToChatConfig: { + convertAlertToAttachment, + initialMessage: 'Please triage these alerts.', + }, + }), + { wrapper } + ); + + result.current[0].onClick([]); + + expect(mockOpenChat).toHaveBeenCalledWith( + expect.objectContaining({ initialMessage: 'Please triage these alerts.' }) + ); + }); + }); + describe('useBulkUntrackActions', () => { beforeEach(() => { jest.clearAllMocks(); diff --git a/x-pack/platform/packages/shared/response-ops/alerts-table/hooks/use_bulk_actions.ts b/x-pack/platform/packages/shared/response-ops/alerts-table/hooks/use_bulk_actions.ts index 39fe32a9c20da..55316cbfaa4ba 100644 --- a/x-pack/platform/packages/shared/response-ops/alerts-table/hooks/use_bulk_actions.ts +++ b/x-pack/platform/packages/shared/response-ops/alerts-table/hooks/use_bulk_actions.ts @@ -23,12 +23,15 @@ import type { BulkActionsReducerAction, TimelineItem, BulkEditTagsFlyoutState, + BulkAddToChatConfig, + OpenChatService, } from '../types'; import { BulkActionsVerbs } from '../types'; import type { CasesService, PublicAlertsDataGridProps } from '../types'; import { ADD_TO_EXISTING_CASE, ADD_TO_NEW_CASE, + ADD_TO_CHAT, ALERTS_ALREADY_ATTACHED_TO_CASE, EDIT_TAGS, MARK_AS_UNTRACKED, @@ -49,6 +52,8 @@ interface BulkActionsProps { hideBulkActions?: boolean; application: ApplicationStart; casesService?: CasesService; + agentBuilderService?: OpenChatService; + bulkAddToChatConfig?: BulkAddToChatConfig; http: HttpStart; notifications: NotificationsStart; } @@ -408,6 +413,45 @@ export const useBulkMuteActions = ({ ); }; +export const useBulkAddToChatActions = ({ + agentBuilderService, + bulkAddToChatConfig, +}: { + agentBuilderService?: OpenChatService; + bulkAddToChatConfig?: BulkAddToChatConfig; +}) => { + const { convertAlertToAttachment, initialMessage, onAddedToChat } = bulkAddToChatConfig ?? {}; + + const onAddToChatClick = useCallback( + (alerts?: TimelineItem[]) => { + if (!agentBuilderService || !convertAlertToAttachment) return; + const items = alerts ?? []; + agentBuilderService.openChat({ + autoSendInitialMessage: false, + newConversation: true, + initialMessage, + attachments: convertAlertToAttachment(items), + }); + onAddedToChat?.(items.length); + }, + [agentBuilderService, convertAlertToAttachment, initialMessage, onAddedToChat] + ); + + return useMemo(() => { + if (!agentBuilderService || !convertAlertToAttachment) return []; + return [ + { + label: ADD_TO_CHAT, + key: 'bulk-add-to-chat', + disableOnQuery: true, + disabledLabel: ADD_TO_CHAT, + 'data-test-subj': 'bulk-add-to-chat', + onClick: onAddToChatClick, + }, + ]; + }, [agentBuilderService, convertAlertToAttachment, onAddToChatClick]); +}; + const EMPTY_BULK_ACTIONS_CONFIG: BulkActionsPanelConfig[] = []; export function useBulkActions({ @@ -422,6 +466,8 @@ export function useBulkActions({ notifications, application, casesService, + agentBuilderService, + bulkAddToChatConfig, }: BulkActionsProps): UseBulkActions { const { bulkActionsStore: [bulkActionsState, updateBulkActionsState], @@ -491,16 +537,28 @@ export function useBulkActions({ }, ]; }, [tagsAction, application?.capabilities]); + const addToChatActions = useBulkAddToChatActions({ + agentBuilderService, + bulkAddToChatConfig, + }); const initialItems = useMemo(() => { const isSiem = ruleTypeIds?.some(isSiemRuleType); return [ ...caseBulkActions, + ...addToChatActions, ...(isSiem ? [] : untrackBulkActions), ...(isSiem ? [] : tagsBulkActions), ...(isSiem ? [] : muteBulkActions), ]; - }, [caseBulkActions, ruleTypeIds, untrackBulkActions, tagsBulkActions, muteBulkActions]); + }, [ + caseBulkActions, + ruleTypeIds, + untrackBulkActions, + tagsBulkActions, + muteBulkActions, + addToChatActions, + ]); const bulkActions = useMemo(() => { if (hideBulkActions) { diff --git a/x-pack/platform/packages/shared/response-ops/alerts-table/translations.ts b/x-pack/platform/packages/shared/response-ops/alerts-table/translations.ts index b6fa2cf898044..dd4ebe19614b9 100644 --- a/x-pack/platform/packages/shared/response-ops/alerts-table/translations.ts +++ b/x-pack/platform/packages/shared/response-ops/alerts-table/translations.ts @@ -251,6 +251,10 @@ export const ADD_TO_EXISTING_CASE = i18n.translate( } ); +export const ADD_TO_CHAT = i18n.translate('xpack.responseOpsAlertsTable.actions.addChat', { + defaultMessage: 'Add to chat', +}); + export const ADD_TO_NEW_CASE = i18n.translate('xpack.responseOpsAlertsTable.actions.addToNewCase', { defaultMessage: 'Add to new case', }); diff --git a/x-pack/platform/packages/shared/response-ops/alerts-table/types.ts b/x-pack/platform/packages/shared/response-ops/alerts-table/types.ts index 88be90e464716..ae395005acc05 100644 --- a/x-pack/platform/packages/shared/response-ops/alerts-table/types.ts +++ b/x-pack/platform/packages/shared/response-ops/alerts-table/types.ts @@ -58,6 +58,35 @@ import type { AlertFormatter } from '@kbn/alerts-ui-shared/src/common/types'; import type { Case } from './apis/bulk_get_cases'; import type { ItemsSelectionState } from './components/tags/items/types'; +/** + * A single conversation attachment or a group of attachments. Defined structurally + * here to avoid a compile-time dependency on agent-builder packages. The payload is + * passed through to openChat without inspection, so a broad structural type is enough. + */ +export interface ConversationAttachmentInput { + type: string; +} + +/** + * Minimal structural interface for the chat service required by the alerts table. + * Using a local structural type avoids a compile-time dependency on agent-builder packages + * from this shared platform package. + */ +export interface OpenChatService { + openChat(options?: { + attachments?: ConversationAttachmentInput[]; + newConversation?: boolean; + initialMessage?: string; + autoSendInitialMessage?: boolean; + }): void; +} + +export interface BulkAddToChatConfig { + convertAlertToAttachment: (alerts: TimelineItem[]) => ConversationAttachmentInput[]; + initialMessage?: string; + onAddedToChat?: (itemCount: number) => void; +} + export interface Consumer { id: AlertConsumers; name: string; @@ -411,6 +440,14 @@ export interface AlertsTableProps { renderContext: RenderContext; + bulkAddToChatConfig?: BulkAddToChatConfig; additionalToolbarControls?: ReactNode; pageSizeOptions?: number[]; leadingControlColumns?: EuiDataGridControlColumn[]; diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/.eslintrc.js b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/.eslintrc.js new file mode 100644 index 0000000000000..f4b09c92e63e9 --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/.eslintrc.js @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** @type {import('eslint').Linter.Config} */ +module.exports = { + overrides: [ + // This package is Node-only (Playwright eval suite). Allow Node.js builtins. + { + files: ['**/*.{js,mjs,ts,tsx}'], + rules: { + 'import/no-nodejs-modules': 'off', + // functional-tests packages intentionally import from test-helper packages (e.g. @kbn/evals, @kbn/scout) + '@kbn/imports/no_boundary_crossing': 'off', + }, + }, + ], +}; diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/.gitignore b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/.gitignore new file mode 100644 index 0000000000000..51511d1f8f36f --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/.gitignore @@ -0,0 +1 @@ +test-results/ diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/evals/alert_triage_quality.spec.ts b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/evals/alert_triage_quality.spec.ts new file mode 100644 index 0000000000000..7a1a382b58193 --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/evals/alert_triage_quality.spec.ts @@ -0,0 +1,312 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * Quality eval: does the LLM produce a useful triage given real alert data? + * + * Three scenarios: + * 1. Priority triage (inline mode, 100 alerts / 5 batches ≤ threshold): + * LLM sees all data directly; tests severity prioritisation. + * 2. Entity correlation (inline mode, 100 alerts / 5 batches): + * Tests whether the LLM spots the host targeted by multiple alerts. + * 3. End-to-end / summary mode (200 alerts / 10 batches > threshold): + * Uses both alert groups so attachment_read is required before + * the LLM can reason over the data. Covers the full shipped path: + * bulk selection → summary mode → attachment_read → triage real alerts. + * + * All scenarios include a grounding criterion that fails when the LLM + * invents hosts or rule names not present in the attached alert data. + * + * Synthetic alerts are indexed into .alerts-security.alerts-default in + * beforeAll and cleaned up in afterAll so no external snapshot is required. + */ + +import { tags } from '@kbn/scout'; +import type { EsClient } from '@kbn/scout'; +import { + selectEvaluators, + type DefaultEvaluators, + type EvaluationDataset, + type EvalsExecutorClient, + type Example, +} from '@kbn/evals'; +import type { ToolingLog } from '@kbn/tooling-log'; +import type { HttpHandler } from '@kbn/core/public'; +import { evaluate as base } from '../src/evaluate'; +import { callConverse } from '../src/converse_task'; +import { attachmentReadCompliance } from '../src/evaluators'; +import { + ALL_TRIAGE_EVAL_ALERTS, + ALL_TRIAGE_EVAL_IDS, + ENTITY_CORRELATION_IDS, + PRIORITY_TRIAGE_IDS, +} from '../src/synthetic_alerts'; + +// ── Constants ───────────────────────────────────────────────────────────────── + +const ALERTS_INDEX = '.alerts-security.alerts-default'; +const ALERTS_BATCH_MAX_SIZE = 20; + +const toAlertAttachments = (ids: string[]) => { + const batches = []; + for (let i = 0; i < ids.length; i += ALERTS_BATCH_MAX_SIZE) { + batches.push({ + type: 'security.alerts', + data: { alertIds: ids.slice(i, i + ALERTS_BATCH_MAX_SIZE) }, + }); + } + return batches; +}; + +// ── Types ───────────────────────────────────────────────────────────────────── + +interface TriageEvalExample extends Example { + input: { question: string }; + output: { expected: string }; + metadata?: { + attachments?: Array<{ type: string; data?: unknown }>; + expectedAttachmentReads?: number; + }; +} + +// ── Fixture factory ─────────────────────────────────────────────────────────── + +function createEvaluateTriageQuality({ + fetch, + connector, + evaluators, + executorClient, + log, +}: { + fetch: HttpHandler; + connector: { id: string }; + evaluators: DefaultEvaluators; + executorClient: EvalsExecutorClient; + log: ToolingLog; +}) { + return async function evaluateTriageQuality({ + dataset, + criteria, + }: { + dataset: { name: string; description: string; examples: TriageEvalExample[] }; + criteria: string[]; + }) { + const selectedEvaluators = selectEvaluators([ + evaluators.criteria(criteria), + attachmentReadCompliance, + ...Object.values(evaluators.traceBasedEvaluators), + ]); + + await executorClient.runExperiment( + { + datasets: [ + { + name: dataset.name, + description: dataset.description, + examples: dataset.examples, + } satisfies EvaluationDataset, + ], + task: async ({ input, metadata }) => { + const { attachments = [] } = metadata ?? {}; + return callConverse({ + fetch, + connectorId: connector.id, + question: input.question, + attachments, + log, + }); + }, + }, + selectedEvaluators + ); + }; +} + +// ── Evaluate fixture extension ───────────────────────────────────────────────── + +type EvaluateTriageQuality = ReturnType; + +const evaluate = base.extend<{ evaluateTriageQuality: EvaluateTriageQuality }, {}>({ + evaluateTriageQuality: [ + ({ fetch, connector, evaluators, executorClient, log }, use) => { + use(createEvaluateTriageQuality({ fetch, connector, evaluators, executorClient, log })); + }, + { scope: 'test' }, + ], +}); + +// ── Eval spec ───────────────────────────────────────────────────────────────── + +evaluate.describe( + 'Security Alert Triage — output quality', + { tag: [...tags.serverless.security.complete, ...tags.serverless.security.ease] }, + () => { + evaluate.beforeAll(async ({ esClient, log }: { esClient: EsClient; log: ToolingLog }) => { + log.info( + `Indexing ${ALL_TRIAGE_EVAL_ALERTS.length} synthetic triage eval alerts into ${ALERTS_INDEX}` + ); + + const response = await esClient.bulk({ + refresh: 'wait_for', + operations: ALL_TRIAGE_EVAL_ALERTS.flatMap(({ id, doc }) => [ + { index: { _index: ALERTS_INDEX, _id: id } }, + doc, + ]), + }); + + if (response.errors) { + const failed = response.items.filter((item) => item.index?.error); + throw new Error( + `Failed to set up triage eval alerts: ${JSON.stringify( + failed.map((f) => f.index?.error) + )}` + ); + } + + log.info(`Successfully indexed ${ALL_TRIAGE_EVAL_ALERTS.length} synthetic alerts`); + }); + + evaluate.afterAll(async ({ esClient, log }: { esClient: EsClient; log: ToolingLog }) => { + const idsToDelete = ALL_TRIAGE_EVAL_ALERTS.map((alert) => alert.id); + log.info(`Deleting ${idsToDelete.length} synthetic triage eval alerts`); + + await esClient.deleteByQuery({ + index: ALERTS_INDEX, + query: { ids: { values: idsToDelete } }, + }); + }); + + evaluate( + 'Priority triage — structured output with severity prioritisation', + async ({ evaluateTriageQuality }) => { + await evaluateTriageQuality({ + dataset: { + name: 'agent builder: security-alert-triage-priority', + description: + 'Tests that the LLM prioritises critical and high severity alerts, mentions ' + + 'specific host names from the alert data, and provides actionable recommendations. ' + + '5 alerts on distinct hosts with severities: critical, high, high, medium, medium.', + examples: [ + { + input: { + question: + 'Triage these security alerts. Identify the most urgent issues, ' + + 'note key entities (hosts, users), and recommend immediate actions.', + }, + output: { + expected: + 'The response should prioritise the critical Memory Injection via Process Hollowing alert on laptop-finance-07, ' + + 'address the high-severity credential-dumping and lateral-movement alerts, ' + + 'list the affected host names, and provide specific remediation steps for each.', + }, + metadata: { + attachments: toAlertAttachments(PRIORITY_TRIAGE_IDS), + }, + }, + ], + }, + criteria: [ + 'The response treats the critical severity alert as the highest priority item', + 'The response mentions specific host names present in the alert data', + 'The response includes concrete remediation steps, not just a description of the alerts', + 'The response distinguishes between urgent and lower-priority items', + 'The response does not reference any hostname, username, or rule name that is not present in the attached alert data', + ], + }); + } + ); + + evaluate( + 'Entity correlation — identifies a host targeted by multiple alerts', + async ({ evaluateTriageQuality }) => { + await evaluateTriageQuality({ + dataset: { + name: 'agent builder: security-alert-triage-entity-correlation', + description: + 'Tests that the LLM identifies the host appearing in 4 of 8 alerts (web-server-01) ' + + 'as a correlated or higher-risk target. The 4 web-server-01 alerts show a clear ' + + 'attack progression: SQL injection → web shell → reverse shell → privilege escalation.', + examples: [ + { + input: { + question: + 'Review these security alerts. Are any hosts or users targeted ' + + 'repeatedly? What does that pattern suggest about the threat?', + }, + output: { + expected: + 'The response identifies web-server-01 as the host appearing in multiple alerts ' + + 'and flags the sequence of alerts as evidence of an active intrusion or attack ' + + 'progression on that host, recommending immediate investigation or containment.', + }, + metadata: { + attachments: toAlertAttachments(ENTITY_CORRELATION_IDS), + }, + }, + ], + }, + criteria: [ + 'The response identifies that one specific host appears in multiple alerts', + 'The response names web-server-01 as the repeatedly targeted host', + 'The response treats the repeated targeting of a single host as a higher-risk indicator than the single-alert hosts', + 'The response does not reference any hostname, username, or rule name that is not present in the attached alert data', + ], + }); + } + ); + + evaluate( + 'End-to-end — summary mode with real alert data exercises attachment_read', + async ({ evaluateTriageQuality }) => { + // ALL_TRIAGE_EVAL_IDS = 200 alerts → 10 batches > inline threshold of 5 + // → framework switches to summary mode → LLM must call attachment_read + // to read each batch before it can reason over real alert content. + // This is the only scenario that covers the full shipped path. + const E2E_BATCH_COUNT = Math.ceil(ALL_TRIAGE_EVAL_IDS.length / ALERTS_BATCH_MAX_SIZE); + await evaluateTriageQuality({ + dataset: { + name: 'agent builder: security-alert-triage-e2e-summary-mode', + description: + `End-to-end coverage for the full bulk-add path: ${ALL_TRIAGE_EVAL_IDS.length} real ` + + `alerts produce ${E2E_BATCH_COUNT} batches, triggering summary mode. The LLM must ` + + 'call attachment_read for each batch before reasoning. Combines both the priority-' + + 'triage group (1 critical alert buried at index 49) and the entity-correlation group ' + + '(web-server-01 targeted by 20 alerts showing an attack progression).', + examples: [ + { + input: { + question: + 'Review all these security alerts. Identify the most critical threats, ' + + 'note any hosts or users that appear across multiple alerts, and recommend ' + + 'prioritized actions.', + }, + output: { + expected: + `The response calls attachment_read ${E2E_BATCH_COUNT} times to read each alert ` + + 'batch, identifies the critical Memory Injection via Process Hollowing alert on ' + + 'laptop-finance-07 as the highest priority, identifies web-server-01 as the most ' + + 'frequently targeted host with an attack progression, and provides prioritized ' + + 'remediation steps without referencing hosts or rules not in the data.', + }, + metadata: { + attachments: toAlertAttachments(ALL_TRIAGE_EVAL_IDS), + expectedAttachmentReads: E2E_BATCH_COUNT, + }, + }, + ], + }, + criteria: [ + 'The response treats the critical severity alert as the highest priority item', + 'The response names web-server-01 as a host that appears in multiple alerts', + 'The response provides prioritized remediation steps', + 'The response does not reference any hostname, username, or rule name that is not present in the attached alert data', + ], + }); + } + ); + } +); diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/evals/bulk_alerts_attachment_read.spec.ts b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/evals/bulk_alerts_attachment_read.spec.ts new file mode 100644 index 0000000000000..d1cba6ba32981 --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/evals/bulk_alerts_attachment_read.spec.ts @@ -0,0 +1,173 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * Deterministic eval: attachment_read compliance for bulk alert batches. + * + * When >5 attachments are present the framework switches to summary mode, showing + * only metadata and instructing the LLM to call `attachment_read(attachment_id)` + * per batch before answering. This eval verifies the LLM follows that instruction. + * + * No real alert data is required. Synthetic IDs trigger summary mode and the + * format() function returns "not found" for each — enough for the LLM to see + * the attachment_read prompt and act on it. + */ + +import { tags } from '@kbn/scout'; +import { + selectEvaluators, + type DefaultEvaluators, + type EvaluationDataset, + type EvalsExecutorClient, + type Example, +} from '@kbn/evals'; +import type { ToolingLog } from '@kbn/tooling-log'; +import type { HttpHandler } from '@kbn/core/public'; +import { evaluate as base } from '../src/evaluate'; +import { callConverse } from '../src/converse_task'; +import { attachmentReadCompliance } from '../src/evaluators'; + +// ── Dataset constants ────────────────────────────────────────────────────────── + +const BATCH_SIZE = 20; +// 6 batches > framework inline threshold of 5 → summary mode +const BATCH_COUNT = 6; + +const alertBatches: Array<{ alertIds: string[] }> = Array.from( + { length: BATCH_COUNT }, + (_, batchIndex) => ({ + alertIds: Array.from( + { length: BATCH_SIZE }, + (__, alertIndex) => + `eval-alert-b${batchIndex.toString().padStart(2, '0')}-${alertIndex + .toString() + .padStart(2, '0')}` + ), + }) +); + +// ── Types ────────────────────────────────────────────────────────────────────── + +interface AlertEvalExample extends Example { + input: { question: string }; + output: { expected: string }; + metadata?: { + attachments?: Array<{ type: string; data?: unknown }>; + expectedAttachmentReads?: number; + }; +} + +// ── Fixture factory ──────────────────────────────────────────────────────────── + +function createEvaluateAlertBatches({ + fetch, + connector, + evaluators, + executorClient, + log, +}: { + fetch: HttpHandler; + connector: { id: string }; + evaluators: DefaultEvaluators; + executorClient: EvalsExecutorClient; + log: ToolingLog; +}) { + return async function evaluateAlertBatches({ + dataset: { name, description, examples }, + }: { + dataset: { name: string; description: string; examples: AlertEvalExample[] }; + }) { + const selectedEvaluators = selectEvaluators([ + attachmentReadCompliance, + ...Object.values(evaluators.traceBasedEvaluators), + ]); + + await executorClient.runExperiment( + { + datasets: [{ name, description, examples } satisfies EvaluationDataset], + task: async ({ input, metadata }) => { + const attachments = metadata?.attachments ?? []; + return callConverse({ + fetch, + connectorId: connector.id, + question: input.question, + attachments, + log, + }); + }, + }, + selectedEvaluators + ); + }; +} + +// ── Evaluate fixture extension ───────────────────────────────────────────────── + +type EvaluateAlertBatches = ReturnType; + +const evaluate = base.extend<{ evaluateAlertBatches: EvaluateAlertBatches }, {}>({ + evaluateAlertBatches: [ + ({ fetch, connector, evaluators, executorClient, log }, use) => { + use( + createEvaluateAlertBatches({ + fetch, + connector, + evaluators, + executorClient, + log, + }) + ); + }, + { scope: 'test' }, + ], +}); + +// ── Eval spec ───────────────────────────────────────────────────────────────── + +evaluate.describe( + 'Security Alerts — attachment_read compliance', + { tag: [...tags.serverless.security.complete, ...tags.serverless.security.ease] }, + () => { + evaluate( + `LLM calls attachment_read for all ${BATCH_COUNT} batches when in summary mode`, + async ({ evaluateAlertBatches }) => { + await evaluateAlertBatches({ + dataset: { + name: 'agent builder: security-bulk-alerts-attachment-read', + description: + `Validates the LLM calls attachment_read for each of ${BATCH_COUNT} alert batches ` + + 'when the framework is in summary mode (>5 active attachments). ' + + 'Synthetic alert IDs are used — format() returns "not found" per ID, which is ' + + 'sufficient to populate the attachment and trigger summary mode. ' + + 'Score = attachment_read call count / expected batch count.', + examples: [ + { + input: { + question: + 'I have added security alerts for your review. Please triage them — ' + + 'identify patterns, shared entities, and escalation indicators, then recommend actions.', + }, + output: { + expected: + `I will call attachment_read ${BATCH_COUNT} times to read each alert batch, ` + + 'then synthesize patterns, identify shared entities, and recommend triage actions.', + }, + metadata: { + attachments: alertBatches.map(({ alertIds }) => ({ + type: 'security.alerts', + data: { alertIds }, + })), + expectedAttachmentReads: BATCH_COUNT, + }, + }, + ], + }, + }); + } + ); + } +); diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/kibana.jsonc b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/kibana.jsonc new file mode 100644 index 0000000000000..0eafc009ed836 --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/kibana.jsonc @@ -0,0 +1,7 @@ +{ + "type": "functional-tests", + "id": "@kbn/evals-suite-security-alert-triage", + "owner": "@elastic/security-threat-hunting", + "group": "security", + "visibility": "private" +} diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/moon.yml b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/moon.yml new file mode 100644 index 0000000000000..1fb706dfb8ab2 --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/moon.yml @@ -0,0 +1,35 @@ +# This file is generated by the @kbn/moon package. Any manual edits will be erased! +# To extend this, write your extensions/overrides to 'moon.extend.yml' +# then regenerate this file with: 'node scripts/regenerate_moon_projects.js --update --filter @kbn/evals-suite-security-alert-triage' + +$schema: https://moonrepo.dev/schemas/project.json +id: '@kbn/evals-suite-security-alert-triage' +layer: unknown +owners: + defaultOwner: '@elastic/security-threat-hunting' +toolchains: + default: node +language: typescript +project: + title: '@kbn/evals-suite-security-alert-triage' + description: Moon project for @kbn/evals-suite-security-alert-triage + channel: '' + owner: '@elastic/security-threat-hunting' + sourceRoot: x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage +dependsOn: + - '@kbn/evals' + - '@kbn/scout' + - '@kbn/tooling-log' + - '@kbn/agent-builder-common' + - '@kbn/core' +tags: + - functional-tests + - package + - prod + - group-security + - private +fileGroups: + src: + - '**/*.ts' + - '!target/**/*' +tasks: {} diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/package.json b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/package.json new file mode 100644 index 0000000000000..287b74645c92f --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/package.json @@ -0,0 +1,6 @@ +{ + "name": "@kbn/evals-suite-security-alert-triage", + "private": true, + "version": "1.0.0", + "license": "Elastic License 2.0" +} diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/playwright.config.ts b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/playwright.config.ts new file mode 100644 index 0000000000000..137c2134aff28 --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/playwright.config.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createPlaywrightEvalsConfig } from '@kbn/evals'; + +export default createPlaywrightEvalsConfig({ + testDir: `${__dirname}/evals`, + timeout: 30 * 60_000, +}); diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/src/converse_task.ts b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/src/converse_task.ts new file mode 100644 index 0000000000000..a58a11578cb8b --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/src/converse_task.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { agentBuilderDefaultAgentId } from '@kbn/agent-builder-common'; +import type { HttpHandler } from '@kbn/core/public'; +import type { ToolingLog } from '@kbn/tooling-log'; + +interface ConverseResponse { + conversation_id: string; + trace_id?: string; + steps: unknown[]; + response: { message: string }; +} + +export const callConverse = async ({ + fetch, + connectorId, + question, + attachments, + log, +}: { + fetch: HttpHandler; + connectorId: string; + question: string; + attachments: Array<{ type: string; data?: unknown }>; + log: ToolingLog; +}) => { + log.info(`Calling converse: "${question.slice(0, 80)}..."`); + + const raw = (await fetch('/api/agent_builder/converse', { + method: 'POST', + version: '2023-10-31', + body: JSON.stringify({ + agent_id: agentBuilderDefaultAgentId, + connector_id: connectorId, + input: question, + attachments, + }), + })) as ConverseResponse; + + return { + errors: [], + messages: [{ message: question }, raw.response], + steps: raw.steps ?? [], + traceId: raw.trace_id, + }; +}; diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/src/evaluate.ts b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/src/evaluate.ts new file mode 100644 index 0000000000000..d2c5362248548 --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/src/evaluate.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { evaluate, tags, selectEvaluators } from '@kbn/evals'; diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/src/evaluators.ts b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/src/evaluators.ts new file mode 100644 index 0000000000000..ce06cfc82756e --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/src/evaluators.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getToolCallSteps, type TaskOutput } from '@kbn/evals'; +import { attachmentTools } from '@kbn/agent-builder-common'; + +/** + * CODE evaluator: scores whether the LLM called attachment_read the expected + * number of times. Returns score=1 (pass) when the example metadata does not + * set expectedAttachmentReads, so it can be included in eval suites that mix + * inline-mode (no reads expected) and summary-mode (reads required) scenarios. + */ +export const attachmentReadCompliance = { + name: 'AttachmentReadCompliance', + kind: 'CODE' as const, + evaluate: async ({ output, metadata }: { output: unknown; metadata: unknown }) => { + const expectedReads = (metadata as { expectedAttachmentReads?: number } | undefined) + ?.expectedAttachmentReads; + const expected = typeof expectedReads === 'number' ? expectedReads : 0; + + if (!expected) { + return { score: 1 }; + } + + const toolCalls = getToolCallSteps(output as TaskOutput); + const readCalls = toolCalls.filter((t) => t.tool_id === attachmentTools.read); + + return { + score: Math.min(1, readCalls.length / expected), + metadata: { readCallCount: readCalls.length, expected }, + }; + }, +}; diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/src/synthetic_alerts.ts b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/src/synthetic_alerts.ts new file mode 100644 index 0000000000000..59cddd38f81fd --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/src/synthetic_alerts.ts @@ -0,0 +1,361 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * Synthetic alert documents for triage quality evals. + * + * Two groups of 100 alerts each (5 batches of 20 → inline mode): + * + * PRIORITY_TRIAGE (100 alerts) + * - 1 critical at index 49 (buried mid-list) + * - 10 high (indices 0–9) + * - 30 medium (indices 10–39) + * - 59 low (indices 40–99 excluding 49) + * Tests whether the LLM finds the critical needle and prioritises correctly. + * + * ENTITY_CORRELATION (100 alerts, 17 hosts) + * - 20 alerts on web-server-01 (showing an attack progression) + * - 80 alerts spread 5-per-host across 16 other hosts + * Tests whether the LLM identifies the most targeted host. + * + * Fields are a subset of ESSENTIAL_ALERT_FIELDS (common/constants.ts:585) — the + * same fields the attachment agent description instructs the LLM to extract. + */ + +interface SyntheticAlert { + id: string; + doc: Record; +} + +const ts = (offsetMinutes: number): string => { + const d = new Date('2024-06-15T09:00:00Z'); + d.setMinutes(d.getMinutes() + offsetMinutes); + return d.toISOString(); +}; + +// ── Lookup tables ───────────────────────────────────────────────────────────── + +const RULES_BY_SEVERITY: Record< + string, + Array<{ rule: string; tactic: string; technique: string }> +> = { + critical: [ + { + rule: 'Suspicious PowerShell with Encoded Commands', + tactic: 'Execution', + technique: 'PowerShell', + }, + { + rule: 'Memory Injection via Process Hollowing', + tactic: 'Defense Evasion', + technique: 'Process Injection', + }, + { + rule: 'Ransomware File Encryption Activity', + tactic: 'Impact', + technique: 'Data Encrypted for Impact', + }, + ], + high: [ + { + rule: 'Credential Dumping via LSASS Access', + tactic: 'Credential Access', + technique: 'OS Credential Dumping', + }, + { + rule: 'Lateral Movement via Admin Share', + tactic: 'Lateral Movement', + technique: 'SMB/Windows Admin Shares', + }, + { + rule: 'Kerberoasting Detected', + tactic: 'Credential Access', + technique: 'Steal or Forge Kerberos Tickets', + }, + { + rule: 'Web Shell File Created', + tactic: 'Persistence', + technique: 'Server Software Component', + }, + { + rule: 'Privilege Escalation via SUID Binary', + tactic: 'Privilege Escalation', + technique: 'Abuse Elevation Control Mechanism', + }, + ], + medium: [ + { + rule: 'Scheduled Task in Unusual Location', + tactic: 'Persistence', + technique: 'Scheduled Task/Job', + }, + { + rule: 'Suspicious Registry Key Modified', + tactic: 'Defense Evasion', + technique: 'Modify Registry', + }, + { + rule: 'Network Scanning Activity Detected', + tactic: 'Discovery', + technique: 'Network Service Discovery', + }, + { + rule: 'Unusual DNS Query Volume', + tactic: 'Command and Control', + technique: 'Application Layer Protocol', + }, + { + rule: 'Outbound Connection to Threat Intelligence Blocklist IP', + tactic: 'Command and Control', + technique: 'Application Layer Protocol', + }, + ], + low: [ + { + rule: 'Software Installed by Non-Admin User', + tactic: 'Persistence', + technique: 'Boot or Logon Autostart Execution', + }, + { + rule: 'USB Drive Inserted on Endpoint', + tactic: 'Initial Access', + technique: 'Replication Through Removable Media', + }, + { rule: 'Browser Extension Installed', tactic: 'Persistence', technique: 'Browser Extensions' }, + { rule: 'Phishing Link Clicked', tactic: 'Initial Access', technique: 'Phishing' }, + { + rule: 'Screensaver Executable Modified', + tactic: 'Persistence', + technique: 'Boot or Logon Autostart Execution', + }, + ], +}; + +const RISK_SCORE_BY_SEVERITY: Record = { + critical: 99, + high: 73, + medium: 47, + low: 18, +}; + +const OTHER_HOSTS = [ + 'workstation-42', + 'laptop-finance-07', + 'desktop-hr-03', + 'server-prod-01', + 'workstation-dev-11', + 'laptop-legal-02', + 'desktop-exec-01', + 'server-backup-01', + 'workstation-support-08', + 'laptop-remote-06', + 'desktop-reception-04', + 'db-server-02', + 'fileserver-01', + 'workstation-mktg-05', + 'laptop-sales-09', + 'app-server-03', +]; + +const USERS = [ + 'alice.smith', + 'bob.jones', + 'carol.white', + 'david.brown', + 'eve.chen', + 'frank.miller', + 'grace.lee', + 'henry.wilson', + 'iris.wang', + 'james.taylor', +]; + +// ── Priority triage group (100 alerts) ─────────────────────────────────────── + +export const PRIORITY_TRIAGE_IDS: string[] = Array.from( + { length: 100 }, + (_, i) => `triage-eval-priority-${i.toString().padStart(3, '0')}` +); + +// Severity distribution: 1 critical (at index 49), 9 high, 30 medium, 60 low. +// The critical alert is intentionally buried mid-list to test whether the LLM +// scans all alerts rather than stopping after the first few. +const severityAt = (i: number): string => { + if (i === 49) return 'critical'; + if (i < 10) return 'high'; + if (i < 40) return 'medium'; + return 'low'; +}; + +const PRIORITY_TRIAGE_ALERTS: SyntheticAlert[] = PRIORITY_TRIAGE_IDS.map((id, i) => { + const severity = severityAt(i); + const rules = RULES_BY_SEVERITY[severity]; + const { rule, tactic, technique } = rules[i % rules.length]; + const host = OTHER_HOSTS[i % OTHER_HOSTS.length]; + const user = USERS[i % USERS.length]; + return { + id, + doc: { + '@timestamp': ts(i), + 'kibana.alert.rule.name': rule, + 'kibana.alert.severity': severity, + 'kibana.alert.risk_score': RISK_SCORE_BY_SEVERITY[severity], + 'kibana.alert.uuid': id, + 'kibana.alert.workflow_status': 'open', + 'kibana.alert.rule.category': 'Custom Query', + 'kibana.alert.reason': `${rule} triggered on ${host}`, + 'host.name': host, + 'user.name': user, + 'threat.tactic.name': tactic, + 'threat.technique.name': technique, + }, + }; +}); + +// ── Entity correlation group (100 alerts) ───────────────────────────────────── + +export const ENTITY_CORRELATION_IDS: string[] = Array.from( + { length: 100 }, + (_, i) => `triage-eval-correlation-${i.toString().padStart(3, '0')}` +); + +// web-server-01 attack progression (20 alerts, indices chosen to scatter them +// throughout the list so the LLM must synthesise across the full batch set). +const WEB_SERVER_INDICES = new Set([ + 2, 7, 12, 17, 22, 27, 32, 37, 42, 47, 52, 57, 62, 67, 72, 77, 82, 87, 92, 97, +]); + +const WEB_SERVER_ATTACKS = [ + { + rule: 'SQL Injection Attempt Detected', + tactic: 'Initial Access', + technique: 'Exploit Public-Facing Application', + user: 'www-data', + }, + { + rule: 'Web Shell File Created', + tactic: 'Persistence', + technique: 'Server Software Component', + user: 'www-data', + }, + { + rule: 'Unusual Outbound Data Transfer from Web Process', + tactic: 'Exfiltration', + technique: 'Exfiltration Over C2 Channel', + user: 'www-data', + }, + { + rule: 'Reverse Shell Spawned from Web Process', + tactic: 'Execution', + technique: 'Command and Scripting Interpreter', + user: 'root', + }, + { + rule: 'Privilege Escalation via SUID Binary', + tactic: 'Privilege Escalation', + technique: 'Abuse Elevation Control Mechanism', + user: 'root', + }, + { + rule: 'Credential Dumping via /etc/shadow Read', + tactic: 'Credential Access', + technique: 'OS Credential Dumping', + user: 'root', + }, + { + rule: 'Lateral Movement Attempt from Web Server', + tactic: 'Lateral Movement', + technique: 'Remote Services', + user: 'root', + }, + { + rule: 'Suspicious Cron Job Installed', + tactic: 'Persistence', + technique: 'Scheduled Task/Job', + user: 'root', + }, + { + rule: 'Port Scanning from Compromised Web Server', + tactic: 'Discovery', + technique: 'Network Service Discovery', + user: 'root', + }, + { + rule: 'Data Staged in Temp Directory', + tactic: 'Collection', + technique: 'Data Staged', + user: 'root', + }, +]; + +let webServerAttackIndex = 0; + +const ENTITY_CORRELATION_ALERTS: SyntheticAlert[] = ENTITY_CORRELATION_IDS.map((id, i) => { + const isWebServer = WEB_SERVER_INDICES.has(i); + const otherHostIndex = + Math.floor((i - (WEB_SERVER_INDICES.size > 0 ? 0 : 0)) / 1) % OTHER_HOSTS.length; + + if (isWebServer) { + const attack = WEB_SERVER_ATTACKS[webServerAttackIndex % WEB_SERVER_ATTACKS.length]; + webServerAttackIndex++; + const severity = + webServerAttackIndex % 5 === 0 + ? 'critical' + : webServerAttackIndex % 3 === 0 + ? 'high' + : 'high'; + return { + id, + doc: { + '@timestamp': ts(i), + 'kibana.alert.rule.name': attack.rule, + 'kibana.alert.severity': severity, + 'kibana.alert.risk_score': RISK_SCORE_BY_SEVERITY[severity], + 'kibana.alert.uuid': id, + 'kibana.alert.workflow_status': 'open', + 'kibana.alert.rule.category': 'Custom Query', + 'kibana.alert.reason': `${attack.rule} detected on web-server-01`, + 'host.name': 'web-server-01', + 'user.name': attack.user, + 'threat.tactic.name': attack.tactic, + 'threat.technique.name': attack.technique, + }, + }; + } + + const severity = i % 7 === 0 ? 'high' : i % 3 === 0 ? 'medium' : 'low'; + const rules = RULES_BY_SEVERITY[severity]; + const { rule, tactic, technique } = rules[i % rules.length]; + const host = OTHER_HOSTS[otherHostIndex % OTHER_HOSTS.length]; + const user = USERS[i % USERS.length]; + return { + id, + doc: { + '@timestamp': ts(i), + 'kibana.alert.rule.name': rule, + 'kibana.alert.severity': severity, + 'kibana.alert.risk_score': RISK_SCORE_BY_SEVERITY[severity], + 'kibana.alert.uuid': id, + 'kibana.alert.workflow_status': 'open', + 'kibana.alert.rule.category': 'Custom Query', + 'kibana.alert.reason': `${rule} triggered on ${host}`, + 'host.name': host, + 'user.name': user, + 'threat.tactic.name': tactic, + 'threat.technique.name': technique, + }, + }; +}); + +// ── Combined exports ────────────────────────────────────────────────────────── + +export const ALL_TRIAGE_EVAL_IDS: string[] = [...PRIORITY_TRIAGE_IDS, ...ENTITY_CORRELATION_IDS]; + +export const ALL_TRIAGE_EVAL_ALERTS: SyntheticAlert[] = [ + ...PRIORITY_TRIAGE_ALERTS, + ...ENTITY_CORRELATION_ALERTS, +]; diff --git a/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/tsconfig.json b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/tsconfig.json new file mode 100644 index 0000000000000..7e9867c6dff1e --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage/tsconfig.json @@ -0,0 +1,16 @@ +{ + "extends": "@kbn/tsconfig-base/tsconfig.json", + "compilerOptions": { + "outDir": "target/types", + "types": ["jest", "node"] + }, + "include": ["**/*.ts"], + "exclude": ["target/**/*"], + "kbn_references": [ + "@kbn/evals", + "@kbn/scout", + "@kbn/tooling-log", + "@kbn/agent-builder-common", + "@kbn/core" + ] +} diff --git a/x-pack/solutions/security/packages/kbn-scout-security/index.ts b/x-pack/solutions/security/packages/kbn-scout-security/index.ts index 8c92fb77cb635..f01f09e46ac70 100644 --- a/x-pack/solutions/security/packages/kbn-scout-security/index.ts +++ b/x-pack/solutions/security/packages/kbn-scout-security/index.ts @@ -8,6 +8,9 @@ // Security-specific test framework export { test, spaceTest } from './src/playwright'; +// Security-specific test constants +export { CUSTOM_QUERY_RULE } from './src/playwright/constants/detection_rules'; + // re-exported test framework from @kbn/scout export { lighthouseTest, apiTest, globalSetupHook, globalTeardownHook, tags } from '@kbn/scout'; diff --git a/x-pack/solutions/security/packages/kbn-scout-security/src/playwright/fixtures/test/page_objects/agent_builder.ts b/x-pack/solutions/security/packages/kbn-scout-security/src/playwright/fixtures/test/page_objects/agent_builder.ts new file mode 100644 index 0000000000000..fafa87d44d5c6 --- /dev/null +++ b/x-pack/solutions/security/packages/kbn-scout-security/src/playwright/fixtures/test/page_objects/agent_builder.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ScoutPage, Locator } from '@kbn/scout'; +import { expect } from '../../../../../ui'; + +export class AgentBuilderPage { + public conversation: Locator; + public attachmentPillsRow: Locator; + public inputEditor: Locator; + public submitButton: Locator; + + constructor(private readonly page: ScoutPage) { + this.conversation = this.page.testSubj.locator('agentBuilderConversation'); + this.attachmentPillsRow = this.page.testSubj.locator('agentBuilderAttachmentPillsRow'); + this.inputEditor = this.page.testSubj.locator('agentBuilderConversationInputEditor'); + this.submitButton = this.page.testSubj.locator('agentBuilderConversationInputSubmitButton'); + } + + /** + * Waits for the attachment pills row to become visible. Uses a longer timeout + * than the default because the pills row renders after the sidebar's React context + * propagates the attachments — one extra render cycle past when the outer + * conversation panel first becomes visible. + */ + async waitForAttachmentPillsRow() { + await expect(this.attachmentPillsRow).toBeVisible({ timeout: 30_000 }); + } +} diff --git a/x-pack/solutions/security/packages/kbn-scout-security/src/playwright/fixtures/test/page_objects/alerts_table.ts b/x-pack/solutions/security/packages/kbn-scout-security/src/playwright/fixtures/test/page_objects/alerts_table.ts index 07f3668a65bfe..f9d248eac9908 100644 --- a/x-pack/solutions/security/packages/kbn-scout-security/src/playwright/fixtures/test/page_objects/alerts_table.ts +++ b/x-pack/solutions/security/packages/kbn-scout-security/src/playwright/fixtures/test/page_objects/alerts_table.ts @@ -22,6 +22,9 @@ export class AlertsTablePage { public bulkRunWorkflowMenuItem: Locator; public bulkWorkflowPanel: Locator; public selectedShowBulkActionsButton: Locator; + public bulkAddToChatMenuItem: Locator; + public selectAllAlertsButton: Locator; + public bulkActionsHeaderCheckbox: Locator; constructor(private readonly page: ScoutPage) { this.detectionsAlertsWrapper = this.page.testSubj.locator('alerts-by-rule-table'); @@ -37,6 +40,9 @@ export class AlertsTablePage { this.selectedShowBulkActionsButton = this.page.testSubj.locator( 'selectedShowBulkActionsButton' ); + this.bulkAddToChatMenuItem = this.page.testSubj.locator('bulk-add-to-chat'); + this.selectAllAlertsButton = this.page.testSubj.locator('selectAllAlertsButton'); + this.bulkActionsHeaderCheckbox = this.page.testSubj.locator('bulk-actions-header'); } async navigate() { @@ -77,4 +83,16 @@ export class AlertsTablePage { // Increased timeout to 20 seconds because this page sometimes takes longer to load return this.detectionsAlertsWrapper.waitFor({ state: 'visible', timeout: 20_000 }); } + + async waitForRuleAlert(ruleName: string) { + const cell = this.alertsTable.getByTestId('ruleName').filter({ hasText: ruleName }); + await expect(cell).toBeVisible({ timeout: 60_000 }); + return cell; + } + + async checkAlertRowCheckbox(ruleName: string) { + const cell = this.alertsTable.getByTestId('ruleName').filter({ hasText: ruleName }); + const row = cell.locator('xpath=ancestor::div[contains(@class,"euiDataGridRow")]'); + await row.getByRole('checkbox').check(); + } } diff --git a/x-pack/solutions/security/packages/kbn-scout-security/src/playwright/fixtures/test/page_objects/index.ts b/x-pack/solutions/security/packages/kbn-scout-security/src/playwright/fixtures/test/page_objects/index.ts index 88ab8943f2a7b..7ed5d0f4bc935 100644 --- a/x-pack/solutions/security/packages/kbn-scout-security/src/playwright/fixtures/test/page_objects/index.ts +++ b/x-pack/solutions/security/packages/kbn-scout-security/src/playwright/fixtures/test/page_objects/index.ts @@ -8,6 +8,7 @@ import type { PageObjects, ScoutPage, ScoutTestConfig } from '@kbn/scout'; import { createLazyPageObject } from '@kbn/scout'; import { AlertsTablePage } from './alerts_table'; +import { AgentBuilderPage } from './agent_builder'; import { AlertDetailsRightPanelPage } from './alert_details_right_panel'; import { EntityAnalyticsDashboardsPage } from './entity_analytics_dashboards'; import { EntityAnalyticsManagementPage } from './entity_analytics_management'; @@ -22,6 +23,7 @@ export type { ThreatMatchRuleCreatePage } from './threat_match_rule_create_page' export interface SecurityPageObjects extends PageObjects { alertsTablePage: AlertsTablePage; + agentBuilderPage: AgentBuilderPage; alertDetailsRightPanelPage: AlertDetailsRightPanelPage; entityAnalyticsDashboardsPage: EntityAnalyticsDashboardsPage; entityAnalyticsManagementPage: EntityAnalyticsManagementPage; @@ -42,6 +44,7 @@ export function extendPageObjects( return { ...pageObjects, alertsTablePage: createLazyPageObject(AlertsTablePage, page), + agentBuilderPage: createLazyPageObject(AgentBuilderPage, page), alertDetailsRightPanelPage: createLazyPageObject(AlertDetailsRightPanelPage, page), entityAnalyticsDashboardsPage: createLazyPageObject(EntityAnalyticsDashboardsPage, page), entityAnalyticsManagementPage: createLazyPageObject(EntityAnalyticsManagementPage, page), diff --git a/x-pack/solutions/security/plugins/security_solution/common/constants.ts b/x-pack/solutions/security/plugins/security_solution/common/constants.ts index 6f96505793076..82b6292871b50 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/constants.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/constants.ts @@ -55,6 +55,7 @@ export const DEFAULT_REFRESH_RATE_INTERVAL = 'timepicker:refreshIntervalDefaults export const DEFAULT_APP_TIME_RANGE = 'securitySolution:timeDefaults' as const; export const DEFAULT_APP_REFRESH_INTERVAL = 'securitySolution:refreshIntervalDefaults' as const; export const DEFAULT_ALERTS_INDEX = '.alerts-security.alerts' as const; +export const ALERTS_BATCH_MAX_SIZE = 20 as const; export const DEFAULT_SIGNALS_INDEX = '.siem-signals' as const; export const DEFAULT_PREVIEW_INDEX = '.preview.alerts-security.alerts' as const; export const DEFAULT_LISTS_INDEX = '.lists' as const; @@ -724,6 +725,7 @@ export const ESSENTIAL_ALERT_FIELDS: string[] = [ export enum SecurityAgentBuilderAttachments { alert = 'security.alert', + alerts = 'security.alerts', entity = 'security.entity', entityAnalyticsDashboard = 'security.entity_analytics_dashboard', rule = 'security.rule', diff --git a/x-pack/solutions/security/plugins/security_solution/moon.yml b/x-pack/solutions/security/plugins/security_solution/moon.yml index a910398fa59d2..84a9282e75eae 100644 --- a/x-pack/solutions/security/plugins/security_solution/moon.yml +++ b/x-pack/solutions/security/plugins/security_solution/moon.yml @@ -237,6 +237,7 @@ dependsOn: - '@kbn/inference-common' - '@kbn/inference-langchain' - '@kbn/scout-security' + - '@kbn/ftr-llm-proxy' - '@kbn/custom-icons' - '@kbn/security-plugin-types-common' - '@kbn/management-settings-ids' diff --git a/x-pack/solutions/security/plugins/security_solution/public/agent_builder/attachment_types/index.ts b/x-pack/solutions/security/plugins/security_solution/public/agent_builder/attachment_types/index.ts index a9505c83d0f9b..b60a2cb2ad37c 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/agent_builder/attachment_types/index.ts +++ b/x-pack/solutions/security/plugins/security_solution/public/agent_builder/attachment_types/index.ts @@ -41,6 +41,11 @@ const ALERT_ATTACHMENT_CONFIG: AttachmentTypeConfig = { icon: 'bell', }; +const ALERTS_DEFAULT_LABEL = i18n.translate( + 'xpack.securitySolution.agentBuilder.attachments.alerts.label', + { defaultMessage: 'Security alerts' } +); + const createAttachmentTypeConfig = (defaultLabel: string, icon: string) => ({ getLabel: (attachment: UnknownAttachmentWithLabel) => { const attachmentLabel = attachment?.data?.attachmentLabel; @@ -63,6 +68,22 @@ export const registerAttachmentUiDefinitions = (attachments: AttachmentServiceSt ALERT_ATTACHMENT_CONFIG.type, createAttachmentTypeConfig(ALERT_ATTACHMENT_CONFIG.label, ALERT_ATTACHMENT_CONFIG.icon) ); + + attachments.addAttachmentType>( + SecurityAgentBuilderAttachments.alerts, + { + getLabel: (attachment) => { + const count = attachment.data?.alertIds?.length ?? 0; + return count > 0 + ? i18n.translate('xpack.securitySolution.agentBuilder.attachments.alerts.countLabel', { + defaultMessage: '{count} {count, plural, one {alert} other {alerts}}', + values: { count }, + }) + : ALERTS_DEFAULT_LABEL; + }, + getIcon: () => 'bell', + } + ); }; /** diff --git a/x-pack/solutions/security/plugins/security_solution/public/agent_builder/components/prompts.ts b/x-pack/solutions/security/plugins/security_solution/public/agent_builder/components/prompts.ts index b2f26376b1751..e67d48385b821 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/agent_builder/components/prompts.ts +++ b/x-pack/solutions/security/plugins/security_solution/public/agent_builder/components/prompts.ts @@ -130,3 +130,5 @@ When Suggesting Improvements: - Put each suggested field value in a separate, copyable code block with a clear label - If you need more context, ask concise follow-up questions `; + +export const BULK_ALERTS_ATTACHMENT_PROMPT = `Triage and prioritize these security alerts`; diff --git a/x-pack/solutions/security/plugins/security_solution/public/agent_builder/helpers.test.tsx b/x-pack/solutions/security/plugins/security_solution/public/agent_builder/helpers.test.tsx index 5189d56149489..5cf294c1df9c5 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/agent_builder/helpers.test.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/agent_builder/helpers.test.tsx @@ -6,7 +6,78 @@ */ import { ESSENTIAL_ALERT_FIELDS } from '../../common'; -import { stringifyEssentialAlertData } from './helpers'; +import { SecurityAgentBuilderAttachments } from '../../common/constants'; +import { alertsToAttachmentGroup, stringifyEssentialAlertData } from './helpers'; + +const makeItem = (id: string) => + ({ _id: id, data: [], ecs: { _id: id, _index: '' } } as unknown as Parameters< + typeof alertsToAttachmentGroup + >[0][number]); + +describe('alertsToAttachmentGroup', () => { + it('returns an AttachmentGroup with type "group"', () => { + const items = [makeItem('a')]; + const result = alertsToAttachmentGroup(items); + expect(result.type).toBe('group'); + expect(result.id).toBeDefined(); + expect(typeof result.id).toBe('string'); + }); + + it('returns a single item in items for ≤20 alerts', () => { + const items = Array.from({ length: 5 }, (_, i) => makeItem(`id-${i}`)); + const result = alertsToAttachmentGroup(items); + + expect(result.items).toHaveLength(1); + expect(result.items[0].hidden).toBeFalsy(); + expect(result.items[0].type).toBe(SecurityAgentBuilderAttachments.alerts); + }); + + it('returns N batches in items for >20 alerts', () => { + const items = Array.from({ length: 50 }, (_, i) => makeItem(`id-${i}`)); + const result = alertsToAttachmentGroup(items); + + expect(result.items).toHaveLength(3); + result.items.forEach((item) => { + expect(item.hidden).toBeFalsy(); + }); + }); + + it('uses plural label for multiple alerts', () => { + const items = Array.from({ length: 5 }, (_, i) => makeItem(`id-${i}`)); + expect(alertsToAttachmentGroup(items).label).toBe('5 Alerts'); + }); + + it('uses singular label for 1 alert', () => { + expect(alertsToAttachmentGroup([makeItem('x')]).label).toBe('1 Alert'); + }); + + it('each batch contains the correct alert IDs', () => { + const items = Array.from({ length: 25 }, (_, i) => makeItem(`id-${i}`)); + const result = alertsToAttachmentGroup(items); + + expect(result.items[0].data).toEqual({ alertIds: items.slice(0, 20).map((i) => i._id) }); + expect(result.items[1].data).toEqual({ alertIds: items.slice(20).map((i) => i._id) }); + }); + + it('items do not pre-stamp group_id or description (stamped by flattenAttachments)', () => { + const items = Array.from({ length: 25 }, (_, i) => makeItem(`id-${i}`)); + const result = alertsToAttachmentGroup(items); + + result.items.forEach((item) => { + expect(item).not.toHaveProperty('group_id'); + expect(item).not.toHaveProperty('description'); + expect(item.hidden).toBeFalsy(); + }); + }); + + it('group id differs between separate calls', () => { + const items = Array.from({ length: 5 }, (_, i) => makeItem(`id-${i}`)); + const first = alertsToAttachmentGroup(items); + const second = alertsToAttachmentGroup(items); + + expect(first.id).not.toBe(second.id); + }); +}); describe('stringifyEssentialAlertData', () => { it('filters to essential fields only', () => { diff --git a/x-pack/solutions/security/plugins/security_solution/public/agent_builder/helpers.tsx b/x-pack/solutions/security/plugins/security_solution/public/agent_builder/helpers.tsx index c5aeb306d502e..b8a97d2103078 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/agent_builder/helpers.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/agent_builder/helpers.tsx @@ -6,13 +6,40 @@ */ import { pick } from 'lodash'; +import { v4 as uuidv4 } from 'uuid'; +import type { AttachmentInput, AttachmentGroup } from '@kbn/agent-builder-common/attachments'; +import type { TimelineItem } from '@kbn/response-ops-alerts-table/types'; import { ESSENTIAL_ALERT_FIELDS } from '../../common'; +import { ALERTS_BATCH_MAX_SIZE, SecurityAgentBuilderAttachments } from '../../common/constants'; + +export type BulkAlertsAttachmentInput = AttachmentInput< + typeof SecurityAgentBuilderAttachments.alerts, + { alertIds: string[] } +>; + +const chunkAlerts = (alertItems: TimelineItem[]): BulkAlertsAttachmentInput[] => { + const batches: BulkAlertsAttachmentInput[] = []; + for (let i = 0; i < alertItems.length; i += ALERTS_BATCH_MAX_SIZE) { + batches.push({ + type: SecurityAgentBuilderAttachments.alerts, + data: { alertIds: alertItems.slice(i, i + ALERTS_BATCH_MAX_SIZE).map((a) => a._id) }, + }); + } + return batches; +}; /** - * Filters raw alert data to only include essential fields and stringifies the result. - * This reduces context window usage by keeping only the most relevant information. + * Converts alert items into a single AttachmentGroup whose items are the chunked batches. + * The group renders as one chip in the UI and is flattened to AttachmentInput[] at the + * serialization boundary before being sent to the server. */ -export const stringifyEssentialAlertData = (rawData: Record): string => { - return JSON.stringify(pick(rawData, ESSENTIAL_ALERT_FIELDS)); -}; +export const alertsToAttachmentGroup = (alertItems: TimelineItem[]): AttachmentGroup => ({ + type: 'group', + id: uuidv4(), + label: `${alertItems.length} Alert${alertItems.length !== 1 ? 's' : ''}`, + items: chunkAlerts(alertItems), +}); + +export const stringifyEssentialAlertData = (rawData: Record): string => + JSON.stringify(pick(rawData, ESSENTIAL_ALERT_FIELDS)); diff --git a/x-pack/solutions/security/plugins/security_solution/public/agent_builder/hooks/use_bulk_add_to_chat_config.ts b/x-pack/solutions/security/plugins/security_solution/public/agent_builder/hooks/use_bulk_add_to_chat_config.ts new file mode 100644 index 0000000000000..f218462ff79d0 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/public/agent_builder/hooks/use_bulk_add_to_chat_config.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useCallback } from 'react'; +import type { BulkAddToChatConfig, TimelineItem } from '@kbn/response-ops-alerts-table/types'; +import { alertsToAttachmentGroup } from '../helpers'; +import { BULK_ALERTS_ATTACHMENT_PROMPT } from '../components/prompts'; +import { type BulkAlertPathway, useReportAddToChat } from './use_report_add_to_chat'; + +export const useBulkAddToChatConfig = (pathway: BulkAlertPathway): BulkAddToChatConfig => { + const reportAddToChat = useReportAddToChat(); + + const convertAlertToAttachment = useCallback( + (alertItems: TimelineItem[]) => { + reportAddToChat({ pathway, attachments: ['alert'], item_count: alertItems.length }); + return [alertsToAttachmentGroup(alertItems)]; + }, + [pathway, reportAddToChat] + ); + + return { + convertAlertToAttachment, + initialMessage: BULK_ALERTS_ATTACHMENT_PROMPT, + }; +}; diff --git a/x-pack/solutions/security/plugins/security_solution/public/agent_builder/hooks/use_report_add_to_chat.test.ts b/x-pack/solutions/security/plugins/security_solution/public/agent_builder/hooks/use_report_add_to_chat.test.ts index caebf87239fc3..ccfdb3cfed5bd 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/agent_builder/hooks/use_report_add_to_chat.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/public/agent_builder/hooks/use_report_add_to_chat.test.ts @@ -43,4 +43,29 @@ describe('useReportAddToChat', () => { expect(reportEvent).toHaveBeenCalledWith(AGENT_BUILDER_EVENT_TYPES.AddToChatClicked, payload); }); + + it('passes item_count through to reportEvent for bulk pathways', () => { + const reportEvent = jest.fn(); + mockUseKibana.mockReturnValue({ + services: { + telemetry: { + reportEvent, + }, + }, + }); + + const payload: AgentBuilderAddToChatTelemetry = { + pathway: 'bulk_alerts_alerts_page', + attachments: ['alert'], + item_count: 5, + }; + + const { result } = renderHook(() => useReportAddToChat()); + + act(() => { + result.current(payload); + }); + + expect(reportEvent).toHaveBeenCalledWith(AGENT_BUILDER_EVENT_TYPES.AddToChatClicked, payload); + }); }); diff --git a/x-pack/solutions/security/plugins/security_solution/public/agent_builder/hooks/use_report_add_to_chat.ts b/x-pack/solutions/security/plugins/security_solution/public/agent_builder/hooks/use_report_add_to_chat.ts index 487a3b5b7d65e..9c9c30a6e6eb2 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/agent_builder/hooks/use_report_add_to_chat.ts +++ b/x-pack/solutions/security/plugins/security_solution/public/agent_builder/hooks/use_report_add_to_chat.ts @@ -8,6 +8,14 @@ import { AGENT_BUILDER_EVENT_TYPES } from '@kbn/agent-builder-common'; import { useCallback } from 'react'; import { useKibana } from '../../common/lib/kibana'; + +export type BulkAlertPathway = + | 'bulk_alerts_alerts_page' + | 'bulk_alerts_rule_details' + | 'bulk_alerts_alert_summary' + | 'bulk_alerts_cases' + | 'bulk_alerts_attack_discovery'; + export interface AgentBuilderAddToChatTelemetry { /** * Pathway where "Add to Chat" was clicked @@ -29,11 +37,14 @@ export interface AgentBuilderAddToChatTelemetry { | 'attack_discovery_bottom' | 'attacks_page_group_summary' | 'attacks_page_group_take_action' - | 'attacks_page_flyout_take_action'; + | 'attacks_page_flyout_take_action' + | BulkAlertPathway; /** * Attachment type */ attachments?: Array<'alert' | 'entity' | 'rule'>; + /** Number of items added (for bulk add-to-chat actions) */ + item_count?: number; } /** diff --git a/x-pack/solutions/security/plugins/security_solution/public/attack_discovery/pages/results/attack_discovery_panel/tabs/alerts_tab/ease/table.tsx b/x-pack/solutions/security/plugins/security_solution/public/attack_discovery/pages/results/attack_discovery_panel/tabs/alerts_tab/ease/table.tsx index 91daa0792def3..b234bfe38043b 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/attack_discovery/pages/results/attack_discovery_panel/tabs/alerts_tab/ease/table.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/attack_discovery/pages/results/attack_discovery_panel/tabs/alerts_tab/ease/table.tsx @@ -27,6 +27,8 @@ import { } from '../../../../../../../detections/components/alert_summary/table/table'; import { ActionsCell } from '../../../../../../../detections/components/alert_summary/table/actions_cell'; import { useKibana } from '../../../../../../../common/lib/kibana'; +import { useBulkAddToChatConfig } from '../../../../../../../agent_builder/hooks/use_bulk_add_to_chat_config'; +import { useAgentBuilderAvailability } from '../../../../../../../agent_builder/hooks/use_agent_builder_availability'; import { CellValue } from '../../../../../../../detections/components/alert_summary/table/render_cell'; import { useAdditionalBulkActions } from '../../../../../../../detections/hooks/alert_summary/use_additional_bulk_actions'; @@ -56,6 +58,7 @@ export interface TableProps { export const Table = memo(({ dataView, id, packages, query }: TableProps) => { const { services: { + agentBuilder, application, cases, data, @@ -69,19 +72,35 @@ export const Table = memo(({ dataView, id, packages, query }: TableProps) => { } = useKibana(); const services = useMemo( () => ({ + agentBuilder, + application, cases, data, http, notifications, rendering, fieldFormats, - application, licensing, settings, }), - [application, cases, data, fieldFormats, http, licensing, notifications, rendering, settings] + [ + agentBuilder, + application, + cases, + data, + fieldFormats, + http, + licensing, + notifications, + rendering, + settings, + ] ); + const { isAgentBuilderEnabled } = useAgentBuilderAvailability(); + const bulkAddToChatConfig = useBulkAddToChatConfig('bulk_alerts_attack_discovery'); + const maybeBulkAddToChatConfig = isAgentBuilderEnabled ? bulkAddToChatConfig : undefined; + const browserFields = useBrowserFields(PageScope.alerts, dataView); const additionalContext: AdditionalTableContext = useMemo( @@ -122,6 +141,7 @@ export const Table = memo(({ dataView, id, packages, query }: TableProps) => { ruleTypeIds={RULE_TYPE_IDS} services={services} toolbarVisibility={TOOLBAR_VISIBILITY} + bulkAddToChatConfig={maybeBulkAddToChatConfig} /> ); diff --git a/x-pack/solutions/security/plugins/security_solution/public/attack_discovery/pages/results/attack_discovery_panel/tabs/alerts_tab/ease/table_bulk_add_to_chat.test.tsx b/x-pack/solutions/security/plugins/security_solution/public/attack_discovery/pages/results/attack_discovery_panel/tabs/alerts_tab/ease/table_bulk_add_to_chat.test.tsx new file mode 100644 index 0000000000000..85e7d078790cd --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/public/attack_discovery/pages/results/attack_discovery_panel/tabs/alerts_tab/ease/table_bulk_add_to_chat.test.tsx @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { render } from '@testing-library/react'; +import type { DataView } from '@kbn/data-views-plugin/common'; +import { createStubDataView } from '@kbn/data-views-plugin/common/data_views/data_view.stub'; +import type { PackageListItem } from '@kbn/fleet-plugin/common'; +import { installationStatuses } from '@kbn/fleet-plugin/common/constants'; +import { AlertsTable } from '@kbn/response-ops-alerts-table'; +import type { TimelineItem } from '@kbn/response-ops-alerts-table/types'; +import { TestProviders } from '../../../../../../../common/mock'; +import { BULK_ALERTS_ATTACHMENT_PROMPT } from '../../../../../../../agent_builder/components/prompts'; +import { alertsToAttachmentGroup } from '../../../../../../../agent_builder/helpers'; +import { useReportAddToChat } from '../../../../../../../agent_builder/hooks/use_report_add_to_chat'; +import { Table } from './table'; + +jest.mock('@kbn/response-ops-alerts-table', () => ({ + AlertsTable: jest.fn(() => null), +})); +jest.mock('../../../../../../../agent_builder/hooks/use_report_add_to_chat'); +jest.mock('../../../../../../../agent_builder/hooks/use_agent_builder_availability', () => ({ + useAgentBuilderAvailability: jest.fn(() => ({ + isAgentBuilderEnabled: true, + hasAgentBuilderPrivilege: true, + isAgentChatExperienceEnabled: true, + hasValidAgentBuilderLicense: false, + })), +})); +jest.mock('../../../../../../../agent_builder/helpers', () => ({ + alertsToAttachmentGroup: jest.fn(() => []), +})); +jest.mock('../../../../../../../data_view_manager/hooks/use_browser_fields', () => ({ + useBrowserFields: jest.fn(() => ({})), +})); +jest.mock( + '../../../../../../../detections/hooks/alert_summary/use_additional_bulk_actions', + () => ({ + useAdditionalBulkActions: jest.fn(() => []), + }) +); + +const makeItem = (id: string): TimelineItem => + ({ _id: id, data: [], ecs: { _id: id, _index: '' } } as unknown as TimelineItem); + +const dataView: DataView = createStubDataView({ spec: {} }); +const packages: PackageListItem[] = [ + { + id: 'splunk', + icons: [{ src: 'icon.svg', path: 'mypath/icon.svg', type: 'image/svg+xml' }], + name: 'splunk', + status: installationStatuses.NotInstalled, + title: 'Splunk', + version: '0.1.0', + }, +]; +const id = 'attack-discovery-table-test'; +const query = { ids: { values: ['abc'] } }; + +describe('Attack Discovery Table — bulkAddToChatConfig', () => { + let mockReportAddToChat: jest.Mock; + + beforeEach(() => { + jest.clearAllMocks(); + mockReportAddToChat = jest.fn(); + (useReportAddToChat as jest.Mock).mockReturnValue(mockReportAddToChat); + }); + + const renderAndGetBulkConfig = () => { + render( + + + + ); + return (AlertsTable as jest.Mock).mock.calls[0][0].bulkAddToChatConfig; + }; + + it('passes BULK_ALERTS_ATTACHMENT_PROMPT as initialMessage', () => { + const { initialMessage } = renderAndGetBulkConfig(); + expect(initialMessage).toBe(BULK_ALERTS_ATTACHMENT_PROMPT); + }); + + it('calls reportAddToChat with bulk_alerts_attack_discovery pathway and item_count', () => { + const { convertAlertToAttachment } = renderAndGetBulkConfig(); + const items = [makeItem('a'), makeItem('b')]; + convertAlertToAttachment(items); + expect(mockReportAddToChat).toHaveBeenCalledWith({ + pathway: 'bulk_alerts_attack_discovery', + attachments: ['alert'], + item_count: 2, + }); + }); + + it('delegates to alertsToAttachmentGroup and returns its result', () => { + const mockGroup = { type: 'group', id: 'x', label: '1 Alert', items: [] }; + (alertsToAttachmentGroup as jest.Mock).mockReturnValueOnce(mockGroup); + const { convertAlertToAttachment } = renderAndGetBulkConfig(); + const items = [makeItem('a')]; + const result = convertAlertToAttachment(items); + expect(alertsToAttachmentGroup).toHaveBeenCalledWith(items); + expect(result).toEqual([mockGroup]); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/public/cases/.eslintrc.js b/x-pack/solutions/security/plugins/security_solution/public/cases/.eslintrc.js index 594d744e6422f..2ba444c3af3a8 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/cases/.eslintrc.js +++ b/x-pack/solutions/security/plugins/security_solution/public/cases/.eslintrc.js @@ -81,9 +81,16 @@ const RESTRICTED_IMPORTS_PATHS = [ }, ]; +// Strip GIT_DIR / GIT_WORK_TREE so the pre-commit hook's env vars don't +// cause `show-toplevel` to return __dirname instead of the repo root. +const gitEnv = { ...process.env }; +delete gitEnv.GIT_DIR; +delete gitEnv.GIT_WORK_TREE; + const ROOT_DIR = execSync('git rev-parse --show-toplevel', { encoding: 'utf8', cwd: __dirname, + env: gitEnv, }).trim(); const ROOT_CLIMB_STRING = path.relative(__dirname, ROOT_DIR); // i.e. '../../..' diff --git a/x-pack/solutions/security/plugins/security_solution/public/cases/components/ease/table.tsx b/x-pack/solutions/security/plugins/security_solution/public/cases/components/ease/table.tsx index 92500107fe6fe..056f0a59819b1 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/cases/components/ease/table.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/cases/components/ease/table.tsx @@ -15,6 +15,8 @@ import type { } from '@kbn/response-ops-alerts-table/types'; import React, { memo, useCallback, useMemo, useRef } from 'react'; import { useKibana } from '../../../common/lib/kibana'; +import { useBulkAddToChatConfig } from '../../../agent_builder/hooks/use_bulk_add_to_chat_config'; +import { useAgentBuilderAvailability } from '../../../agent_builder/hooks/use_agent_builder_availability'; import { ActionsCell } from '../../../detections/components/alert_summary/table/actions_cell'; import { CellValue } from '../../../detections/components/alert_summary/table/render_cell'; import { useBrowserFields } from '../../../data_view_manager/hooks/use_browser_fields'; @@ -63,6 +65,7 @@ export interface TableProps { export const Table = memo(({ dataView, id, onLoaded, packages, query }: TableProps) => { const { services: { + agentBuilder, application, cases, data, @@ -76,19 +79,35 @@ export const Table = memo(({ dataView, id, onLoaded, packages, query }: TablePro } = useKibana(); const services = useMemo( () => ({ + agentBuilder, + application, cases, data, http, notifications, rendering, fieldFormats, - application, licensing, settings, }), - [application, cases, data, fieldFormats, http, licensing, notifications, rendering, settings] + [ + agentBuilder, + application, + cases, + data, + fieldFormats, + http, + licensing, + notifications, + rendering, + settings, + ] ); + const { isAgentBuilderEnabled } = useAgentBuilderAvailability(); + const bulkAddToChatConfig = useBulkAddToChatConfig('bulk_alerts_cases'); + const maybeBulkAddToChatConfig = isAgentBuilderEnabled ? bulkAddToChatConfig : undefined; + const browserFields = useBrowserFields(PageScope.alerts, dataView); const additionalContext: AdditionalTableContext = useMemo(() => ({ packages }), [packages]); @@ -125,6 +144,7 @@ export const Table = memo(({ dataView, id, onLoaded, packages, query }: TablePro runtimeMappings={runtimeMappings} services={services} toolbarVisibility={TOOLBAR_VISIBILITY} + bulkAddToChatConfig={maybeBulkAddToChatConfig} /> ); diff --git a/x-pack/solutions/security/plugins/security_solution/public/cases/components/ease/table_bulk_add_to_chat.test.tsx b/x-pack/solutions/security/plugins/security_solution/public/cases/components/ease/table_bulk_add_to_chat.test.tsx new file mode 100644 index 0000000000000..a5304d128afaa --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/public/cases/components/ease/table_bulk_add_to_chat.test.tsx @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { render } from '@testing-library/react'; +import type { DataView } from '@kbn/data-views-plugin/common'; +import { createStubDataView } from '@kbn/data-views-plugin/common/data_views/data_view.stub'; +import type { PackageListItem } from '@kbn/fleet-plugin/common'; +import { installationStatuses } from '@kbn/fleet-plugin/common/constants'; +import { AlertsTable } from '@kbn/response-ops-alerts-table'; +import type { TimelineItem } from '@kbn/response-ops-alerts-table/types'; +import { TestProviders } from '../../../common/mock'; +import { BULK_ALERTS_ATTACHMENT_PROMPT } from '../../../agent_builder/components/prompts'; +import { alertsToAttachmentGroup } from '../../../agent_builder/helpers'; +import { useReportAddToChat } from '../../../agent_builder/hooks/use_report_add_to_chat'; +import { Table } from './table'; + +jest.mock('@kbn/response-ops-alerts-table', () => ({ + AlertsTable: jest.fn(() => null), +})); +jest.mock('../../../agent_builder/hooks/use_report_add_to_chat'); +jest.mock('../../../agent_builder/hooks/use_agent_builder_availability', () => ({ + useAgentBuilderAvailability: jest.fn(() => ({ + isAgentBuilderEnabled: true, + hasAgentBuilderPrivilege: true, + isAgentChatExperienceEnabled: true, + hasValidAgentBuilderLicense: false, + })), +})); +jest.mock('../../../agent_builder/helpers', () => ({ + alertsToAttachmentGroup: jest.fn(() => []), +})); +jest.mock('../../../data_view_manager/hooks/use_browser_fields', () => ({ + useBrowserFields: jest.fn(() => ({})), +})); +jest.mock('../../../detections/hooks/alert_summary/use_additional_bulk_actions', () => ({ + useAdditionalBulkActions: jest.fn(() => []), +})); + +const makeItem = (id: string): TimelineItem => + ({ _id: id, data: [], ecs: { _id: id, _index: '' } } as unknown as TimelineItem); + +const dataView: DataView = createStubDataView({ spec: {} }); +const packages: PackageListItem[] = [ + { + id: 'splunk', + icons: [{ src: 'icon.svg', path: 'mypath/icon.svg', type: 'image/svg+xml' }], + name: 'splunk', + status: installationStatuses.NotInstalled, + title: 'Splunk', + version: '0.1.0', + }, +]; +const id = 'cases-table-test'; +const query = { ids: { values: ['abc'] } }; + +describe('Cases Table — bulkAddToChatConfig', () => { + let mockReportAddToChat: jest.Mock; + + beforeEach(() => { + jest.clearAllMocks(); + mockReportAddToChat = jest.fn(); + (useReportAddToChat as jest.Mock).mockReturnValue(mockReportAddToChat); + }); + + const renderAndGetBulkConfig = () => { + render( + +
+ + ); + return (AlertsTable as jest.Mock).mock.calls[0][0].bulkAddToChatConfig; + }; + + it('passes BULK_ALERTS_ATTACHMENT_PROMPT as initialMessage', () => { + const { initialMessage } = renderAndGetBulkConfig(); + expect(initialMessage).toBe(BULK_ALERTS_ATTACHMENT_PROMPT); + }); + + it('calls reportAddToChat with bulk_alerts_cases pathway and item_count', () => { + const { convertAlertToAttachment } = renderAndGetBulkConfig(); + const items = [makeItem('a'), makeItem('b')]; + convertAlertToAttachment(items); + expect(mockReportAddToChat).toHaveBeenCalledWith({ + pathway: 'bulk_alerts_cases', + attachments: ['alert'], + item_count: 2, + }); + }); + + it('delegates to alertsToAttachmentGroup and returns its result', () => { + const mockGroup = { type: 'group', id: 'x', label: '1 Alert', items: [] }; + (alertsToAttachmentGroup as jest.Mock).mockReturnValueOnce(mockGroup); + const { convertAlertToAttachment } = renderAndGetBulkConfig(); + const items = [makeItem('a')]; + const result = convertAlertToAttachment(items); + expect(alertsToAttachmentGroup).toHaveBeenCalledWith(items); + expect(result).toEqual([mockGroup]); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/public/detections/components/alert_summary/table/table.tsx b/x-pack/solutions/security/plugins/security_solution/public/detections/components/alert_summary/table/table.tsx index 2fd7ecb7267af..5b189b36c4d68 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detections/components/alert_summary/table/table.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/detections/components/alert_summary/table/table.tsx @@ -37,6 +37,8 @@ import { useDeepEqualSelector } from '../../../../common/hooks/use_selector'; import { combineQueries } from '../../../../common/lib/kuery'; import { useKibana } from '../../../../common/lib/kibana'; import { CellValue } from './render_cell'; +import { useBulkAddToChatConfig } from '../../../../agent_builder/hooks/use_bulk_add_to_chat_config'; +import { useAgentBuilderAvailability } from '../../../../agent_builder/hooks/use_agent_builder_availability'; import { buildTimeRangeFilter } from '../../alerts_table/helpers'; import { useGlobalTime } from '../../../../common/containers/use_global_time'; @@ -134,6 +136,7 @@ export const Table = memo(({ dataView, groupingFilters, packages }: TableProps) const { services: { application, + agentBuilder, cases, data, fieldFormats, @@ -156,10 +159,26 @@ export const Table = memo(({ dataView, groupingFilters, packages }: TableProps) application, licensing, settings, + agentBuilder, }), - [application, cases, data, fieldFormats, http, licensing, notifications, rendering, settings] + [ + agentBuilder, + application, + cases, + data, + fieldFormats, + http, + licensing, + notifications, + rendering, + settings, + ] ); + const { isAgentBuilderEnabled } = useAgentBuilderAvailability(); + const bulkAddToChatConfig = useBulkAddToChatConfig('bulk_alerts_alert_summary'); + const maybeBulkAddToChatConfig = isAgentBuilderEnabled ? bulkAddToChatConfig : undefined; + const getGlobalFiltersSelector = useMemo(() => inputsSelectors.globalFiltersQuerySelector(), []); const globalFilters = useDeepEqualSelector(getGlobalFiltersSelector); @@ -245,6 +264,7 @@ export const Table = memo(({ dataView, groupingFilters, packages }: TableProps) ruleTypeIds={RULE_TYPE_IDS} services={services} toolbarVisibility={TOOLBAR_VISIBILITY} + bulkAddToChatConfig={maybeBulkAddToChatConfig} /> ); diff --git a/x-pack/solutions/security/plugins/security_solution/public/detections/components/alert_summary/table/table_bulk_add_to_chat.test.tsx b/x-pack/solutions/security/plugins/security_solution/public/detections/components/alert_summary/table/table_bulk_add_to_chat.test.tsx new file mode 100644 index 0000000000000..9357e0ccdf4f2 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/public/detections/components/alert_summary/table/table_bulk_add_to_chat.test.tsx @@ -0,0 +1,102 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { render } from '@testing-library/react'; +import type { DataView } from '@kbn/data-views-plugin/common'; +import { createStubDataView } from '@kbn/data-views-plugin/common/data_views/data_view.stub'; +import type { PackageListItem } from '@kbn/fleet-plugin/common'; +import { installationStatuses } from '@kbn/fleet-plugin/common/constants'; +import { AlertsTable } from '@kbn/response-ops-alerts-table'; +import type { TimelineItem } from '@kbn/response-ops-alerts-table/types'; +import { TestProviders } from '../../../../common/mock'; +import { BULK_ALERTS_ATTACHMENT_PROMPT } from '../../../../agent_builder/components/prompts'; +import { alertsToAttachmentGroup } from '../../../../agent_builder/helpers'; +import { useReportAddToChat } from '../../../../agent_builder/hooks/use_report_add_to_chat'; +import { Table } from './table'; + +jest.mock('@kbn/response-ops-alerts-table', () => ({ + AlertsTable: jest.fn(() => null), +})); +jest.mock('../../../../agent_builder/hooks/use_report_add_to_chat'); +jest.mock('../../../../agent_builder/hooks/use_agent_builder_availability', () => ({ + useAgentBuilderAvailability: jest.fn(() => ({ + isAgentBuilderEnabled: true, + hasAgentBuilderPrivilege: true, + isAgentChatExperienceEnabled: true, + hasValidAgentBuilderLicense: false, + })), +})); +jest.mock('../../../../agent_builder/helpers', () => ({ + alertsToAttachmentGroup: jest.fn(() => []), +})); +jest.mock('../../../../data_view_manager/hooks/use_browser_fields', () => ({ + useBrowserFields: jest.fn(() => ({})), +})); +jest.mock('../../../hooks/alert_summary/use_additional_bulk_actions', () => ({ + useAdditionalBulkActions: jest.fn(() => []), +})); + +const makeItem = (id: string): TimelineItem => + ({ _id: id, data: [], ecs: { _id: id, _index: '' } } as unknown as TimelineItem); + +const dataView: DataView = createStubDataView({ spec: {} }); +const packages: PackageListItem[] = [ + { + id: 'splunk', + icons: [{ src: 'icon.svg', path: 'mypath/icon.svg', type: 'image/svg+xml' }], + name: 'splunk', + status: installationStatuses.NotInstalled, + title: 'Splunk', + version: '0.1.0', + }, +]; + +describe('Alert Summary Table — bulkAddToChatConfig', () => { + let mockReportAddToChat: jest.Mock; + + beforeEach(() => { + jest.clearAllMocks(); + mockReportAddToChat = jest.fn(); + (useReportAddToChat as jest.Mock).mockReturnValue(mockReportAddToChat); + }); + + const renderAndGetBulkConfig = () => { + render( + +
+ + ); + return (AlertsTable as jest.Mock).mock.calls[0][0].bulkAddToChatConfig; + }; + + it('passes BULK_ALERTS_ATTACHMENT_PROMPT as initialMessage', () => { + const { initialMessage } = renderAndGetBulkConfig(); + expect(initialMessage).toBe(BULK_ALERTS_ATTACHMENT_PROMPT); + }); + + it('calls reportAddToChat with bulk_alerts_alert_summary pathway and item_count', () => { + const { convertAlertToAttachment } = renderAndGetBulkConfig(); + const items = [makeItem('a'), makeItem('b'), makeItem('c')]; + convertAlertToAttachment(items); + expect(mockReportAddToChat).toHaveBeenCalledWith({ + pathway: 'bulk_alerts_alert_summary', + attachments: ['alert'], + item_count: 3, + }); + }); + + it('delegates to alertsToAttachmentGroup and returns its result', () => { + const mockGroup = { type: 'group', id: 'x', label: '1 Alert', items: [] }; + (alertsToAttachmentGroup as jest.Mock).mockReturnValueOnce(mockGroup); + const { convertAlertToAttachment } = renderAndGetBulkConfig(); + const items = [makeItem('a')]; + const result = convertAlertToAttachment(items); + expect(alertsToAttachmentGroup).toHaveBeenCalledWith(items); + expect(result).toEqual([mockGroup]); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/public/detections/components/alerts_table/index.tsx b/x-pack/solutions/security/plugins/security_solution/public/detections/components/alerts_table/index.tsx index 3b651accf5009..be2b30b58f88d 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detections/components/alerts_table/index.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/detections/components/alerts_table/index.tsx @@ -38,6 +38,8 @@ import { ActionsCell } from './actions_cell'; import { useGlobalTime } from '../../../common/containers/use_global_time'; import { useLicense } from '../../../common/hooks/use_license'; import { APP_ID, CASES_FEATURE_ID, VIEW_SELECTION } from '../../../../common/constants'; +import { useBulkAddToChatConfig } from '../../../agent_builder/hooks/use_bulk_add_to_chat_config'; +import { useAgentBuilderAvailability } from '../../../agent_builder/hooks/use_agent_builder_availability'; import { DEFAULT_COLUMN_MIN_WIDTH } from '../../../timelines/components/timeline/body/constants'; import { defaultRowRenderers } from '../../../timelines/components/timeline/body/renderers'; import { eventsDefaultModel } from '../../../common/components/events_viewer/default_model'; @@ -171,6 +173,7 @@ const AlertsTableComponent: FC onLoad(alerts), [onLoad]); + const { isAgentBuilderEnabled } = useAgentBuilderAvailability(); + const pathway = + tableType === TableId.alertsOnRuleDetailsPage + ? ('bulk_alerts_rule_details' as const) + : ('bulk_alerts_alerts_page' as const); + const bulkAddToChatConfig = useBulkAddToChatConfig(pathway); + const maybeBulkAddToChatConfig = isAgentBuilderEnabled ? bulkAddToChatConfig : undefined; + /** * We want to hide additional controls (like grouping) if the table is being rendered * in the cases page OR if the user of the table explicitly set `disableAdditionalToolbarControls` @@ -503,6 +526,7 @@ const AlertsTableComponent: FC diff --git a/x-pack/solutions/security/plugins/security_solution/public/detections/components/alerts_table/table_bulk_add_to_chat.test.tsx b/x-pack/solutions/security/plugins/security_solution/public/detections/components/alerts_table/table_bulk_add_to_chat.test.tsx new file mode 100644 index 0000000000000..f8cdfecb5c261 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/public/detections/components/alerts_table/table_bulk_add_to_chat.test.tsx @@ -0,0 +1,188 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { render } from '@testing-library/react'; +import { AlertsTable as ResponseOpsAlertsTable } from '@kbn/response-ops-alerts-table'; +import type { TimelineItem } from '@kbn/response-ops-alerts-table/types'; +import { TableId } from '@kbn/securitysolution-data-table'; +import { TestProviders } from '../../../common/mock'; +import { BULK_ALERTS_ATTACHMENT_PROMPT } from '../../../agent_builder/components/prompts'; +import { alertsToAttachmentGroup } from '../../../agent_builder/helpers'; +import { useReportAddToChat } from '../../../agent_builder/hooks/use_report_add_to_chat'; +import { AlertsTable } from '.'; + +jest.mock('@kbn/response-ops-alerts-table', () => ({ + AlertsTable: jest.fn(() => null), +})); +jest.mock('../../../agent_builder/hooks/use_report_add_to_chat'); +jest.mock('../../../agent_builder/hooks/use_agent_builder_availability', () => ({ + useAgentBuilderAvailability: jest.fn(() => ({ + isAgentBuilderEnabled: true, + hasAgentBuilderPrivilege: true, + isAgentChatExperienceEnabled: true, + hasValidAgentBuilderLicense: false, + })), +})); +jest.mock('../../../agent_builder/helpers', () => ({ + alertsToAttachmentGroup: jest.fn(), +})); +jest.mock('../../../common/lib/kibana', () => ({ + useKibana: jest.fn(() => ({ + services: { + data: {}, + http: {}, + notifications: {}, + rendering: {}, + fieldFormats: {}, + application: {}, + licensing: {}, + uiSettings: { get: jest.fn() }, + settings: {}, + cases: {}, + agentBuilder: {}, + }, + })), + KibanaContextProvider: ({ children }: { children: React.ReactNode }) => <>{children}, +})); +jest.mock('../../../common/containers/use_global_time', () => ({ + useGlobalTime: jest.fn(() => ({ + from: '2020-01-01T00:00:00Z', + to: '2020-01-02T00:00:00Z', + setQuery: jest.fn(), + deleteQuery: jest.fn(), + })), +})); +jest.mock('../../../sourcerer/containers', () => ({ + useSourcererDataView: jest.fn(() => ({ + browserFields: {}, + sourcererDataView: { runtimeFieldMap: {} }, + loading: false, + indicesExist: true, + })), +})); +jest.mock('../../../common/hooks/use_experimental_features', () => ({ + useIsExperimentalFeatureEnabled: jest.fn(() => false), +})); +jest.mock('../../../data_view_manager/hooks/use_data_view', () => ({ + useDataView: jest.fn(() => ({ + dataView: { getRuntimeMappings: jest.fn(() => ({})) }, + status: 'ready', + })), +})); +jest.mock('../../../data_view_manager/hooks/use_browser_fields', () => ({ + useBrowserFields: jest.fn(() => ({})), +})); +jest.mock('../../../common/hooks/use_license', () => ({ + useLicense: jest.fn(() => ({ + isEnterprise: jest.fn(() => false), + isPlatinumPlus: jest.fn(() => false), + isGold: jest.fn(() => false), + getType: jest.fn(() => 'basic'), + })), +})); +jest.mock('../../../common/hooks/use_selector', () => ({ + useDeepEqualSelector: jest.fn(() => []), + useShallowEqualSelector: jest.fn(() => ({})), +})); +jest.mock('../../hooks/trigger_actions_alert_table/use_bulk_actions', () => ({ + useBulkActionsByTableType: jest.fn(() => []), +})); +jest.mock('../../../common/components/user_privileges', () => ({ + useUserPrivileges: jest.fn(() => ({ + timelinePrivileges: { read: true }, + notesPrivileges: { read: true }, + kibanaSecuritySolutionsPrivileges: { crud: true, read: true }, + })), +})); +jest.mock('../../../notes/hooks/use_fetch_notes', () => ({ + useFetchNotes: jest.fn(() => ({ onLoad: jest.fn() })), +})); +jest.mock('../../configurations/security_solution_detections/fetch_page_context', () => ({ + useFetchUserProfilesFromAlerts: jest.fn(() => new Map()), +})); +jest.mock('../../hooks/trigger_actions_alert_table/use_cell_actions', () => ({ + useCellActionsOptions: jest.fn(() => undefined), +})); +jest.mock( + '../../hooks/trigger_actions_alert_table/use_trigger_actions_browser_fields_options', + () => ({ + useAlertsTableFieldsBrowserOptions: jest.fn(() => undefined), + }) +); +jest.mock('../../../common/hooks/use_invalid_filter_query', () => ({ + useInvalidFilterQuery: jest.fn(), +})); +jest.mock('../../../common/lib/kuery', () => ({ + combineQueries: jest.fn(() => null), +})); +jest.mock('../../configurations/security_solution_detections', () => ({ + CellValue: () => null, + getColumns: jest.fn(() => []), +})); +jest.mock('../../../timelines/components/timeline/body/control_columns', () => ({ + getDefaultControlColumn: jest.fn(() => [{ width: 124 }]), +})); + +const makeItem = (id: string): TimelineItem => + ({ _id: id, data: [], ecs: { _id: id, _index: '' } } as unknown as TimelineItem); + +describe('Alerts Page Table — bulkAddToChatConfig', () => { + let mockReportAddToChat: jest.Mock; + + beforeEach(() => { + jest.clearAllMocks(); + mockReportAddToChat = jest.fn(); + (useReportAddToChat as jest.Mock).mockReturnValue(mockReportAddToChat); + }); + + const renderAndGetBulkConfig = (tableType?: TableId) => { + render( + + + + ); + return (ResponseOpsAlertsTable as jest.Mock).mock.calls[0][0].bulkAddToChatConfig; + }; + + it('passes BULK_ALERTS_ATTACHMENT_PROMPT as initialMessage', () => { + const { initialMessage } = renderAndGetBulkConfig(); + expect(initialMessage).toBe(BULK_ALERTS_ATTACHMENT_PROMPT); + }); + + it('uses bulk_alerts_alerts_page pathway for the default alerts page table', () => { + const { convertAlertToAttachment } = renderAndGetBulkConfig(TableId.alertsOnAlertsPage); + const items = [makeItem('a'), makeItem('b')]; + convertAlertToAttachment(items); + expect(mockReportAddToChat).toHaveBeenCalledWith({ + pathway: 'bulk_alerts_alerts_page', + attachments: ['alert'], + item_count: 2, + }); + }); + + it('uses bulk_alerts_rule_details pathway for the rule details table', () => { + const { convertAlertToAttachment } = renderAndGetBulkConfig(TableId.alertsOnRuleDetailsPage); + const items = [makeItem('a')]; + convertAlertToAttachment(items); + expect(mockReportAddToChat).toHaveBeenCalledWith({ + pathway: 'bulk_alerts_rule_details', + attachments: ['alert'], + item_count: 1, + }); + }); + + it('delegates to alertsToAttachmentGroup and returns its result', () => { + const mockGroup = { type: 'group', id: 'x', label: '1 Alert', items: [] }; + (alertsToAttachmentGroup as jest.Mock).mockReturnValueOnce(mockGroup); + const { convertAlertToAttachment } = renderAndGetBulkConfig(); + const items = [makeItem('a')]; + const result = convertAlertToAttachment(items); + expect(alertsToAttachmentGroup).toHaveBeenCalledWith(items); + expect(result).toEqual([mockGroup]); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/attachments/alerts.test.ts b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/attachments/alerts.test.ts new file mode 100644 index 0000000000000..29fc83494d7ff --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/attachments/alerts.test.ts @@ -0,0 +1,261 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Attachment } from '@kbn/agent-builder-common/attachments'; +import { platformCoreTools } from '@kbn/agent-builder-common'; +import { agentBuilderMocks } from '@kbn/agent-builder-plugin/server/mocks'; +import { coreMock } from '@kbn/core/server/mocks'; +import { loggerMock } from '@kbn/logging-mocks'; +import type { SecuritySolutionPluginCoreSetupDependencies } from '../../plugin_contract'; +import { SecurityAgentBuilderAttachments } from '../../../common/constants'; +import { + SECURITY_ENTITY_RISK_SCORE_TOOL_ID, + SECURITY_ATTACK_DISCOVERY_SEARCH_TOOL_ID, + SECURITY_LABS_SEARCH_TOOL_ID, + SECURITY_ALERTS_TOOL_ID, +} from '../tools'; +import { createBulkAlertsAttachmentType } from './alerts'; + +const mockAlertSource = { + 'kibana.alert.rule.name': 'Test Rule', + 'kibana.alert.severity': 'critical', + 'kibana.alert.risk_score': 75, + 'host.name': 'test-host', + 'user.name': 'test-user', +}; + +const mockLogger = loggerMock.create(); + +const buildCoreMock = (hits: Array<{ _id: string; _source: unknown }> = []) => { + const core = coreMock.createSetup(); + const scopedClient = { + asCurrentUser: { + search: jest.fn().mockResolvedValue({ + hits: { hits: hits.map((h) => ({ _id: h._id, _source: h._source })) }, + }), + }, + }; + const elasticsearchMock = { + client: { asScoped: jest.fn().mockReturnValue(scopedClient) }, + }; + (core.getStartServices as jest.Mock).mockResolvedValue([{ elasticsearch: elasticsearchMock }]); + return core as unknown as SecuritySolutionPluginCoreSetupDependencies; +}; + +describe('createBulkAlertsAttachmentType', () => { + const formatContext = agentBuilderMocks.attachments.createFormatContextMock(); + + describe('validate', () => { + const attachmentType = createBulkAlertsAttachmentType(buildCoreMock(), mockLogger); + + it('returns valid when alertIds array is present', async () => { + const input = { alertIds: ['abc123', 'def456'] }; + + const result = await attachmentType.validate(input); + + expect(result.valid).toBe(true); + if (result.valid) { + expect(result.data).toEqual(input); + } + }); + + it('returns invalid with an empty alertIds array', async () => { + const result = await attachmentType.validate({ alertIds: [] }); + + expect(result.valid).toBe(false); + }); + + it('returns invalid when alertIds exceeds 20 entries', async () => { + const result = await attachmentType.validate({ + alertIds: Array.from({ length: 21 }, (_, i) => `id-${i}`), + }); + + expect(result.valid).toBe(false); + }); + + it('returns invalid when alertIds field is missing', async () => { + const result = await attachmentType.validate({}); + + expect(result.valid).toBe(false); + if (!result.valid) { + expect(result.error).toBeDefined(); + } + }); + + it('returns invalid when alertIds field is not an array', async () => { + const result = await attachmentType.validate({ alertIds: 'not-an-array' }); + + expect(result.valid).toBe(false); + }); + + it('returns invalid when an alertIds entry is not a string', async () => { + const result = await attachmentType.validate({ alertIds: [123] }); + + expect(result.valid).toBe(false); + }); + }); + + describe('format', () => { + it('fetches alerts from ES and returns inline JSON representation', async () => { + const core = buildCoreMock([ + { _id: 'abc123', _source: mockAlertSource }, + { _id: 'def456', _source: { ...mockAlertSource, 'host.name': 'other-host' } }, + ]); + const attachmentType = createBulkAlertsAttachmentType(core, mockLogger); + + const attachment: Attachment = { + id: 'id-fetch-test', + type: SecurityAgentBuilderAttachments.alerts, + data: { alertIds: ['abc123', 'def456'] }, + }; + + const formatted = await attachmentType.format(attachment, formatContext); + const representation = formatted.getRepresentation + ? await formatted.getRepresentation() + : undefined; + + expect(representation?.type).toBe('text'); + expect(representation?.value).toContain('2 security alerts'); + expect(representation?.value).toContain('Alert 1:'); + expect(representation?.value).toContain('Alert 2:'); + expect(representation?.value).toContain('"_id": "abc123"'); + expect(representation?.value).toContain('"_id": "def456"'); + }); + + it('marks missing alerts with an error placeholder', async () => { + const core = buildCoreMock([{ _id: 'abc123', _source: mockAlertSource }]); + const attachmentType = createBulkAlertsAttachmentType(core, mockLogger); + + const attachment: Attachment = { + id: 'id-missing-test', + type: SecurityAgentBuilderAttachments.alerts, + data: { alertIds: ['abc123', 'missing-id'] }, + }; + + const formatted = await attachmentType.format(attachment, formatContext); + const representation = formatted.getRepresentation + ? await formatted.getRepresentation() + : undefined; + + expect(representation?.value).toContain('"_id": "missing-id"'); + expect(representation?.value).toContain('"error": "not found"'); + }); + + it('logs a warn and returns placeholders when ES search throws', async () => { + const core = coreMock.createSetup(); + const scopedClient = { + asCurrentUser: { search: jest.fn().mockRejectedValue(new Error('index not available')) }, + }; + (core.getStartServices as jest.Mock).mockResolvedValue([ + { elasticsearch: { client: { asScoped: jest.fn().mockReturnValue(scopedClient) } } }, + ]); + const logger = loggerMock.create(); + const attachmentType = createBulkAlertsAttachmentType( + core as unknown as SecuritySolutionPluginCoreSetupDependencies, + logger + ); + + const attachment: Attachment = { + id: 'id-error-test', + type: SecurityAgentBuilderAttachments.alerts, + data: { alertIds: ['abc123'] }, + }; + + const formatted = await attachmentType.format(attachment, formatContext); + await formatted.getRepresentation?.(); + + expect(logger.warn).toHaveBeenCalledTimes(1); + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('Failed to fetch')); + }); + + it('logs a warn when ES returns no results', async () => { + const core = buildCoreMock([]); + const logger = loggerMock.create(); + const attachmentType = createBulkAlertsAttachmentType(core, logger); + + const attachment: Attachment = { + id: 'id-no-results-test', + type: SecurityAgentBuilderAttachments.alerts, + data: { alertIds: ['abc123'] }, + }; + + const formatted = await attachmentType.format(attachment, formatContext); + await formatted.getRepresentation?.(); + + expect(logger.warn).toHaveBeenCalledTimes(1); + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('no results')); + }); + + it('returns cached representation on repeated reads without re-fetching from ES', async () => { + const rawCore = coreMock.createSetup(); + const scopedClient = { + asCurrentUser: { + search: jest.fn().mockResolvedValue({ + hits: { hits: [{ _id: 'abc123', _source: mockAlertSource }] }, + }), + }, + }; + (rawCore.getStartServices as jest.Mock).mockResolvedValue([ + { elasticsearch: { client: { asScoped: jest.fn().mockReturnValue(scopedClient) } } }, + ]); + const core = rawCore as unknown as SecuritySolutionPluginCoreSetupDependencies; + const attachmentType = createBulkAlertsAttachmentType(core, mockLogger); + + const attachment: Attachment = { + id: 'id-cache-test', + type: SecurityAgentBuilderAttachments.alerts, + data: { alertIds: ['abc123'] }, + }; + + const formatted = await attachmentType.format(attachment, formatContext); + const first = await formatted.getRepresentation?.(); + + // Second read with the same attachment ID should return cached value without hitting ES + (rawCore.getStartServices as jest.Mock).mockClear(); + const formatted2 = await attachmentType.format(attachment, formatContext); + const second = await formatted2.getRepresentation?.(); + + expect(second?.value).toBe(first?.value); + expect(rawCore.getStartServices).not.toHaveBeenCalled(); + }); + }); + + describe('getTools', () => { + const attachmentType = createBulkAlertsAttachmentType(buildCoreMock(), mockLogger); + + it('returns enrichment tool IDs but not the ES|QL alerts search tool', () => { + const tools = attachmentType.getTools?.(); + + expect(tools).toBeDefined(); + if (tools) { + expect(tools).toContain(SECURITY_ENTITY_RISK_SCORE_TOOL_ID); + expect(tools).toContain(SECURITY_ATTACK_DISCOVERY_SEARCH_TOOL_ID); + expect(tools).toContain(SECURITY_LABS_SEARCH_TOOL_ID); + expect(tools).toContain(platformCoreTools.cases); + expect(tools).toContain(platformCoreTools.generateEsql); + expect(tools).toContain(platformCoreTools.productDocumentation); + expect(tools).not.toContain(SECURITY_ALERTS_TOOL_ID); + } + }); + }); + + describe('getAgentDescription', () => { + const attachmentType = createBulkAlertsAttachmentType(buildCoreMock(), mockLogger); + + it('instructs the agent to process batches and synthesize across all', () => { + const description = attachmentType.getAgentDescription?.(); + + expect(description).toContain('batch'); + expect(description).toContain('After processing all batches'); + }); + }); + + it('is marked readonly so the LLM cannot modify alert data', () => { + const attachmentType = createBulkAlertsAttachmentType(buildCoreMock(), mockLogger); + expect(attachmentType.isReadonly).toBe(true); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/attachments/alerts.ts b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/attachments/alerts.ts new file mode 100644 index 0000000000000..75ca5663a9608 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/attachments/alerts.ts @@ -0,0 +1,129 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from '@kbn/zod/v4'; +import type { Logger } from '@kbn/core/server'; +import type { AttachmentTypeDefinition } from '@kbn/agent-builder-server/attachments'; +import type { Attachment } from '@kbn/agent-builder-common/attachments'; +import { platformCoreTools } from '@kbn/agent-builder-common'; +import { ALERTS_BATCH_MAX_SIZE, SecurityAgentBuilderAttachments } from '../../../common/constants'; +import { getAlertsIndex } from '../../../common/entity_analytics/utils'; +import { + SECURITY_ENTITY_RISK_SCORE_TOOL_ID, + SECURITY_ATTACK_DISCOVERY_SEARCH_TOOL_ID, + SECURITY_LABS_SEARCH_TOOL_ID, +} from '../tools'; +import type { SecuritySolutionPluginCoreSetupDependencies } from '../../plugin_contract'; +import { getAlertsById } from '../tools/get_alerts_by_id'; +import { securityAttachmentDataSchema } from './security_attachment_data_schema'; + +export const bulkAlertsAttachmentDataSchema = securityAttachmentDataSchema.extend({ + alertIds: z.array(z.string().max(512)).min(1).max(ALERTS_BATCH_MAX_SIZE), +}); + +export type BulkAlertsAttachmentData = z.infer; + +// Bounded cache to avoid re-fetching alert data from ES on every LLM attachment_read call. +// Keyed on attachment.id (UUIDs), evicts oldest entry when full. +const REPRESENTATION_CACHE_MAX_SIZE = 500; +const representationCache = new Map(); + +export const createBulkAlertsAttachmentType = ( + core: SecuritySolutionPluginCoreSetupDependencies, + logger: Logger +): AttachmentTypeDefinition => ({ + id: SecurityAgentBuilderAttachments.alerts, + isReadonly: true, + maxContentLength: 50_000, + validate: (input) => { + const result = bulkAlertsAttachmentDataSchema.safeParse(input); + return result.success + ? { valid: true, data: result.data } + : { valid: false, error: result.error.message }; + }, + format: (attachment, context) => { + // framework guarantees validate() ran first, so data is BulkAlertsAttachmentData + const { data } = attachment as Attachment; + return { + getRepresentation: async () => { + const cached = representationCache.get(attachment.id); + if (cached !== undefined) { + return { type: 'text' as const, value: cached }; + } + + const [coreStart] = await core.getStartServices(); + const esClient = coreStart.elasticsearch.client.asScoped(context.request).asCurrentUser; + const index = getAlertsIndex(context.spaceId); + + let alertsById: Record | undefined; + try { + alertsById = await getAlertsById({ esClient, index, ids: data.alertIds }); + } catch (err) { + logger.warn( + `Failed to fetch ${data.alertIds.length} alert(s) from index ${index}: ${ + err?.message ?? err + }` + ); + } + + if ( + alertsById !== undefined && + Object.keys(alertsById).length === 0 && + data.alertIds.length > 0 + ) { + logger.warn( + `ES returned no results for ${data.alertIds.length} alert ID(s) from index ${index}` + ); + } + + const resolvedAlerts = alertsById ?? {}; + const entries = data.alertIds.map((id, i) => { + const alert = resolvedAlerts[id] ?? { error: 'not found' }; + return `Alert ${i + 1}:\n${JSON.stringify({ _id: id, ...(alert as object) }, null, 2)}`; + }); + + const value = `${data.alertIds.length} security alert${ + data.alertIds.length !== 1 ? 's' : '' + }:\n\n${entries.join('\n\n---\n\n')}`; + + if (representationCache.size >= REPRESENTATION_CACHE_MAX_SIZE) { + const oldest = representationCache.keys().next().value; + if (oldest !== undefined) { + representationCache.delete(oldest); + } + } + representationCache.set(attachment.id, value); + + return { type: 'text' as const, value }; + }, + }; + }, + getTools: () => [ + SECURITY_ENTITY_RISK_SCORE_TOOL_ID, + SECURITY_ATTACK_DISCOVERY_SEARCH_TOOL_ID, + SECURITY_LABS_SEARCH_TOOL_ID, + platformCoreTools.cases, + platformCoreTools.generateEsql, + platformCoreTools.productDocumentation, + ], + getAgentDescription: () => + `You have access to security alerts provided in one or more batches. Each attachment contains a batch of up to 20 alerts with full field data. + +When the conversation has many batches (summary mode), the system shows metadata only. Read each batch before answering: +- Each batch appears as in the XML +- Call the attachment read tool (attachment_read) and pass the attachment_id value directly as the "attachment_id" parameter +- Read ALL batches before forming conclusions + +Process each batch in order: +1. Extract _id, kibana.alert.rule.name, severity, risk_score, host.name, user.name, and MITRE fields. +2. Note entities (hosts, users) and patterns within the batch. + +After processing all batches: +3. Identify shared entities, escalation indicators, and patterns across the full alert set. +4. Use the enrichment tools (entity risk score, security labs, attack discovery) for collective context. +5. Produce a structured summary with triage steps and recommended actions.`, +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/attachments/register_attachments.ts b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/attachments/register_attachments.ts index a8889f9792b88..3e2d8d7fa7cc3 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/attachments/register_attachments.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/attachments/register_attachments.ts @@ -5,9 +5,12 @@ * 2.0. */ +import type { Logger } from '@kbn/core/server'; import type { AgentBuilderPluginSetup } from '@kbn/agent-builder-server'; +import type { SecuritySolutionPluginCoreSetupDependencies } from '../../plugin_contract'; import { createRuleAttachmentType } from './rule'; import { createAlertAttachmentType } from './alert'; +import { createBulkAlertsAttachmentType } from './alerts'; import { createEntityAttachmentType } from './entity'; import { createEntityAnalyticsDashboardAttachmentType } from './entity_analytics_dashboard'; import { createSiemReadinessAttachmentType } from './siem_readiness'; @@ -15,8 +18,13 @@ import { createSiemReadinessAttachmentType } from './siem_readiness'; /** * Registers all security agent builder attachments with the agentBuilder plugin */ -export const registerAttachments = async (agentBuilder: AgentBuilderPluginSetup) => { +export const registerAttachments = async ( + agentBuilder: AgentBuilderPluginSetup, + core: SecuritySolutionPluginCoreSetupDependencies, + logger: Logger +) => { agentBuilder.attachments.registerType(createAlertAttachmentType()); + agentBuilder.attachments.registerType(createBulkAlertsAttachmentType(core, logger)); agentBuilder.attachments.registerType(createEntityAttachmentType()); agentBuilder.attachments.registerType(createEntityAnalyticsDashboardAttachmentType()); agentBuilder.attachments.registerType(createRuleAttachmentType()); diff --git a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/entity_analytics/entity_risk_score_tool.ts b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/entity_analytics/entity_risk_score_tool.ts index ebc7c9596e9d5..3458d075c055f 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/entity_analytics/entity_risk_score_tool.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/entity_analytics/entity_risk_score_tool.ts @@ -16,9 +16,10 @@ import type { SecuritySolutionPluginCoreSetupDependencies } from '../../../plugi import type { EntityRiskScoreRecord } from '../../../../common/api/entity_analytics/common'; import { createGetRiskScores } from '../../../lib/entity_analytics/risk_score/get_risk_score'; import type { EntityType } from '../../../../common/entity_analytics/types'; -import { DEFAULT_ALERTS_INDEX, ESSENTIAL_ALERT_FIELDS } from '../../../../common/constants'; +import { DEFAULT_ALERTS_INDEX } from '../../../../common/constants'; import { getRiskIndex } from '../../../../common/search_strategy/security_solution/risk_score/common'; import { securityTool } from '../constants'; +import { getAlertsById } from '../get_alerts_by_id'; const entityRiskScoreSchema = z.object({ identifierType: z @@ -88,43 +89,6 @@ const queryRiskIndexForWildcard = async ({ .filter((risk): risk is EntityRiskScoreRecord => risk !== undefined); }; -/** - * Fetches alerts by their IDs, returning only essential fields for risk score context - */ -const getAlertsById = async ({ - esClient, - index, - ids, -}: { - esClient: ElasticsearchClient; - index: string; - ids: string[]; -}): Promise> => { - if (ids.length === 0) { - return {}; - } - - const response = await esClient.search({ - index, - ignore_unavailable: true, - allow_no_indices: true, - size: ids.length, - _source: ESSENTIAL_ALERT_FIELDS, - query: { - bool: { - filter: [{ terms: { _id: ids } }], - }, - }, - }); - - return response.hits.hits.reduce>((acc, hit) => { - if (hit._source && hit._id) { - acc[hit._id] = hit._source; - } - return acc; - }, {}); -}; - export const entityRiskScoreTool = ( core: SecuritySolutionPluginCoreSetupDependencies, logger: Logger diff --git a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/get_alerts_by_id.test.ts b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/get_alerts_by_id.test.ts new file mode 100644 index 0000000000000..16da986527595 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/get_alerts_by_id.test.ts @@ -0,0 +1,93 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient } from '@kbn/core/server'; +import { ALERTS_BATCH_MAX_SIZE, ESSENTIAL_ALERT_FIELDS } from '../../../common/constants'; +import { getAlertsById } from './get_alerts_by_id'; + +const makeClient = (hits: Array<{ _id?: string; _source?: unknown }>) => + ({ + search: jest.fn().mockResolvedValue({ hits: { hits } }), + } as unknown as ElasticsearchClient); + +describe('getAlertsById', () => { + const index = '.alerts-security.alerts-default'; + + it('returns an empty object without calling ES when ids is empty', async () => { + const esClient = makeClient([]); + + const result = await getAlertsById({ esClient, index, ids: [] }); + + expect(result).toEqual({}); + expect(esClient.search).not.toHaveBeenCalled(); + }); + + it('throws when more than ALERTS_BATCH_MAX_SIZE ids are provided', async () => { + const esClient = makeClient([]); + const ids = Array.from({ length: ALERTS_BATCH_MAX_SIZE + 1 }, (_, i) => `id-${i}`); + + await expect(getAlertsById({ esClient, index, ids })).rejects.toThrow( + `getAlertsById: ids.length (${ + ALERTS_BATCH_MAX_SIZE + 1 + }) exceeds the maximum of ${ALERTS_BATCH_MAX_SIZE}` + ); + expect(esClient.search).not.toHaveBeenCalled(); + }); + + it('returns hits keyed by _id with _source as the value', async () => { + const source1 = { 'kibana.alert.rule.name': 'Rule A' }; + const source2 = { 'kibana.alert.rule.name': 'Rule B' }; + const esClient = makeClient([ + { _id: 'abc', _source: source1 }, + { _id: 'def', _source: source2 }, + ]); + + const result = await getAlertsById({ esClient, index, ids: ['abc', 'def'] }); + + expect(result).toEqual({ abc: source1, def: source2 }); + }); + + it('skips hits that are missing _source', async () => { + const esClient = makeClient([ + { _id: 'abc', _source: { 'kibana.alert.rule.name': 'Rule A' } }, + { _id: 'def' }, + ]); + + const result = await getAlertsById({ esClient, index, ids: ['abc', 'def'] }); + + expect(result).toEqual({ abc: { 'kibana.alert.rule.name': 'Rule A' } }); + expect(result).not.toHaveProperty('def'); + }); + + it('skips hits that are missing _id', async () => { + const esClient = makeClient([ + { _source: { 'kibana.alert.rule.name': 'Rule A' } }, + { _id: 'def', _source: { 'kibana.alert.rule.name': 'Rule B' } }, + ]); + + const result = await getAlertsById({ esClient, index, ids: ['def'] }); + + expect(Object.keys(result)).toEqual(['def']); + }); + + it('queries with the correct index, source fields, and id filter', async () => { + const esClient = makeClient([]); + + await getAlertsById({ esClient, index, ids: ['abc', 'def'] }); + + expect(esClient.search).toHaveBeenCalledWith( + expect.objectContaining({ + index, + ignore_unavailable: true, + allow_no_indices: true, + size: 2, + _source: ESSENTIAL_ALERT_FIELDS, + query: { bool: { filter: [{ terms: { _id: ['abc', 'def'] } }] } }, + }) + ); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/get_alerts_by_id.ts b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/get_alerts_by_id.ts new file mode 100644 index 0000000000000..3cb481674a7e6 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/agent_builder/tools/get_alerts_by_id.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient } from '@kbn/core/server'; +import { ALERTS_BATCH_MAX_SIZE, ESSENTIAL_ALERT_FIELDS } from '../../../common/constants'; + +/** + * Fetches alerts by their _id values from Elasticsearch, returning only essential fields. + */ +export const getAlertsById = async ({ + esClient, + index, + ids, +}: { + esClient: ElasticsearchClient; + index: string; + ids: string[]; +}): Promise> => { + if (ids.length === 0) { + return {}; + } + if (ids.length > ALERTS_BATCH_MAX_SIZE) { + throw new Error( + `getAlertsById: ids.length (${ids.length}) exceeds the maximum of ${ALERTS_BATCH_MAX_SIZE}` + ); + } + + const response = await esClient.search({ + index, + ignore_unavailable: true, + allow_no_indices: true, + size: ids.length, + _source: ESSENTIAL_ALERT_FIELDS, + query: { + bool: { + filter: [{ terms: { _id: ids } }], + }, + }, + }); + + return response.hits.hits.reduce>((acc, hit) => { + if (hit._source && hit._id) { + acc[hit._id] = hit._source; + } + return acc; + }, {}); +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/plugin.ts b/x-pack/solutions/security/plugins/security_solution/server/plugin.ts index ee97ece000e98..a5b7a2c771958 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/plugin.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/plugin.ts @@ -274,9 +274,11 @@ export class Plugin implements ISecuritySolutionPlugin { this.logger.error(`Error registering security tools: ${error}`); } ); - registerAttachments(agentBuilder).catch((error) => { + + registerAttachments(agentBuilder, core, logger).catch((error) => { this.logger.error(`Error registering security attachments: ${error}`); }); + registerSkills({ agentBuilder, experimentalFeatures, diff --git a/x-pack/solutions/security/plugins/security_solution/test/scout/ui/common/roles.ts b/x-pack/solutions/security/plugins/security_solution/test/scout/ui/common/roles.ts new file mode 100644 index 0000000000000..72b51b20f77cc --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/test/scout/ui/common/roles.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { KibanaRole } from '@kbn/scout-security'; + +/** + * Role combining security alert index privileges with full Kibana access. + * Shared by tests that need both alert visibility and Kibana-level features + * (e.g. agent builder, workflow management). + */ +export const FULL_KIBANA_SECURITY_ROLE: KibanaRole = { + elasticsearch: { + cluster: ['manage'], + indices: [ + { + names: [ + '.alerts-security*', + '.internal.alerts-security*', + '.siem-signals-*', + 'apm-*-transaction*', + 'traces-apm*', + 'auditbeat-*', + 'endgame-*', + 'filebeat-*', + 'logs-*', + 'packetbeat-*', + 'winlogbeat-*', + 'logstash-*', + '.lists*', + '.items*', + ], + privileges: ['read', 'write'], + }, + ], + }, + kibana: [ + { + base: ['all'], + feature: {}, + spaces: ['*'], + }, + ], +}; diff --git a/x-pack/solutions/security/plugins/security_solution/test/scout/ui/fixtures/index.ts b/x-pack/solutions/security/plugins/security_solution/test/scout/ui/fixtures/index.ts new file mode 100644 index 0000000000000..8ec9445a3172a --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/test/scout/ui/fixtures/index.ts @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { spaceTest as baseSpaceTest } from '@kbn/scout-security'; +import type { ScoutWorkerFixtures } from '@kbn/scout-security'; +import { createLlmProxy, type LlmProxy } from '@kbn/ftr-llm-proxy'; + +const XSRF = { 'kbn-xsrf': 'scout-security-solution' }; + +interface SecuritySolutionWorkerFixtures extends ScoutWorkerFixtures { + llmProxy: LlmProxy; +} + +export const spaceTest = baseSpaceTest.extend<{}, SecuritySolutionWorkerFixtures>({ + llmProxy: [ + // scoutSpace is declared first so Playwright sets up the test space before the proxy, + // avoiding concurrent initialization that can delay Sourcerer setup. + async ({ log, kbnClient, scoutSpace }, use) => { + const proxy = await createLlmProxy(log); + + // Create the connector inside the test space so EmbeddableAccessBoundary finds it. + await kbnClient.request({ + method: 'POST', + path: `/s/${scoutSpace.id}/api/actions/connector`, + headers: XSRF, + body: { + name: `scout-llm-proxy-${scoutSpace.id}`, + config: { + apiProvider: 'OpenAI', + apiUrl: `http://localhost:${proxy.getPort()}`, + defaultModel: 'gpt-4', + }, + secrets: { apiKey: 'test-key' }, + connector_type_id: '.gen-ai', + }, + }); + + await use(proxy); + + proxy.close(); + + const list = await kbnClient.request>({ + method: 'GET', + path: `/s/${scoutSpace.id}/api/actions/connectors`, + }); + const connectors = Array.isArray(list.data) ? list.data : []; + await Promise.all( + connectors + .filter((c) => c.name === `scout-llm-proxy-${scoutSpace.id}`) + .map((c) => + kbnClient.request({ + method: 'DELETE', + path: `/s/${scoutSpace.id}/api/actions/connector/${encodeURIComponent(c.id)}`, + headers: XSRF, + }) + ) + ); + }, + { scope: 'worker', auto: false }, + ], +}); + +export { tags } from '@kbn/scout-security'; diff --git a/x-pack/solutions/security/plugins/security_solution/test/scout/ui/parallel_tests/investigations/bulk_add_alerts_to_chat.spec.ts b/x-pack/solutions/security/plugins/security_solution/test/scout/ui/parallel_tests/investigations/bulk_add_alerts_to_chat.spec.ts new file mode 100644 index 0000000000000..f298bd84a382a --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/test/scout/ui/parallel_tests/investigations/bulk_add_alerts_to_chat.spec.ts @@ -0,0 +1,166 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createToolCallMessage } from '@kbn/ftr-llm-proxy'; +import { expect } from '@kbn/scout-security/ui'; +import { CUSTOM_QUERY_RULE } from '@kbn/scout-security'; +import { spaceTest, tags } from '../../fixtures'; +import { FULL_KIBANA_SECURITY_ROLE } from '../../common/roles'; + +const ALERT_COUNT = 3; + +spaceTest.describe( + 'Bulk add alerts to chat', + // Serverless excluded: agent builder feature not yet available on serverless + { tag: [...tags.stateful.classic] }, + () => { + // One rule name per created rule; the first is used as the anchor to wait for page readiness. + const ruleNames: string[] = []; + + spaceTest.beforeEach(async ({ browserAuth, apiServices, scoutSpace }) => { + // Defensive cleanup from any prior failed run. + // Note: cleanStandardList() is intentionally omitted here — it deletes saved objects + // of type 'action', which would destroy the .gen-ai connector created by the + // worker-scoped llmProxy fixture and make the bulk "Add to chat" action disappear. + await apiServices.detectionRule.deleteAll(); + await apiServices.detectionAlerts.deleteAll(); + + ruleNames.length = 0; + for (let i = 0; i < ALERT_COUNT; i++) { + const name = `${CUSTOM_QUERY_RULE.name}_${scoutSpace.id}_${Date.now()}_${i}`; + ruleNames.push(name); + await apiServices.detectionRule.createCustomQueryRule({ + ...CUSTOM_QUERY_RULE, + name, + rule_id: `scout-bulk-chat-${scoutSpace.id}-${i}`, + }); + } + await browserAuth.loginWithCustomRole(FULL_KIBANA_SECURITY_ROLE); + }); + + spaceTest.afterEach(async ({ apiServices }) => { + await apiServices.detectionRule.deleteAll(); + await apiServices.detectionAlerts.deleteAll(); + }); + + spaceTest( + 'should disable the "Add to chat" bulk action when all alerts are selected via the query button', + // _llmProxy is declared so the worker fixture provisions the AI connector in the test space; + // without it the bulk action would not appear at all. + async ({ pageObjects, llmProxy: _llmProxy }) => { + const { alertsTablePage } = pageObjects; + + await alertsTablePage.navigate(); + await alertsTablePage.waitForRuleAlert(ruleNames[0]); + + await spaceTest.step('check one row to activate the bulk toolbar', async () => { + await alertsTablePage.checkAlertRowCheckbox(ruleNames[0]); + }); + + await spaceTest.step('click "Select all" to switch to query-based selection', async () => { + await alertsTablePage.selectAllAlertsButton.click(); + }); + + await spaceTest.step( + 'verify "Add to chat" is present but disabled in the bulk actions menu', + async () => { + await alertsTablePage.selectedShowBulkActionsButton.click(); + await expect(alertsTablePage.bulkAddToChatMenuItem).toBeVisible(); + await expect(alertsTablePage.bulkAddToChatMenuItem).toBeDisabled(); + } + ); + } + ); + + spaceTest( + 'should open agent builder conversation with attachment chip and pre-filled prompt, and send multiple alerts to the LLM', + async ({ pageObjects, llmProxy }) => { + const { alertsTablePage, agentBuilderPage } = pageObjects; + + await alertsTablePage.navigate(); + + // Wait for the first rule's alert to appear — this proves Sourcerer has initialized + // and the alerts table is ready. Longer timeout covers the full initialization window. + await alertsTablePage.waitForRuleAlert(ruleNames[0]); + + await spaceTest.step('select all visible alerts via header checkbox', async () => { + // The header checkbox uses selectCurrentPage (not selectAll/query mode), so + // disableOnQuery actions like bulk-add-to-chat remain enabled. + await alertsTablePage.bulkActionsHeaderCheckbox.check(); + }); + + await spaceTest.step('open bulk actions and click Add to chat', async () => { + await alertsTablePage.selectedShowBulkActionsButton.click(); + await alertsTablePage.bulkAddToChatMenuItem.click(); + }); + + await spaceTest.step( + 'verify conversation opens with attachment chip and pre-filled prompt', + async () => { + await expect(agentBuilderPage.conversation).toBeVisible(); + await agentBuilderPage.waitForAttachmentPillsRow(); + await expect(agentBuilderPage.inputEditor).toContainText( + 'Triage and prioritize these security alerts' + ); + } + ); + + await spaceTest.step('send message and verify LLM received multiple alerts', async () => { + // Register interceptors here — immediately before submit — so the 30s proxy + // timeout doesn't expire during the earlier waitForRuleAlert / waitForAttachmentPillsRow + // steps. autoSendInitialMessage: false guarantees no LLM call fires before this point. + void llmProxy + .intercept({ + name: 'set_title', + when: (body) => { + const sys = body.messages.find((m) => m.role === 'system'); + return String(sys?.content ?? '').includes('You are a title-generation utility'); + }, + responseMock: createToolCallMessage('set_title', { title: 'Alert triage' }), + }) + .completeAfterIntercept(); + + void llmProxy + .intercept({ + name: 'handover-to-answer', + when: (body) => { + const sys = body.messages.find((m) => m.role === 'system'); + return String(sys?.content ?? '').includes( + 'This response will serve as a handover note for the answering agent' + ); + }, + responseMock: 'ready to answer', + }) + .completeAfterIntercept(); + + void llmProxy + .intercept({ + name: 'final-answer', + when: () => true, + responseMock: 'Here is the alert triage summary.', + }) + .completeAfterIntercept(); + + await agentBuilderPage.submitButton.click(); + await llmProxy.waitForAllInterceptorsToHaveBeenCalled(); + + const handoverRequest = llmProxy.interceptedRequests.find( + (r) => r.matchingInterceptorName === 'handover-to-answer' + )?.requestBody; + + expect(handoverRequest).toBeDefined(); + const messages: Array<{ content?: unknown }> = handoverRequest?.messages ?? []; + const allContent = messages.map((m) => String(m.content ?? '')).join('\n'); + + // Verify multiple alerts were delivered: each alert block contains the rule name field. + const alertFieldOccurrences = allContent.split('kibana.alert.rule.name').length - 1; + expect(alertFieldOccurrences).toBeGreaterThanOrEqual(ALERT_COUNT); + }); + } + ); + } +); diff --git a/x-pack/solutions/security/plugins/security_solution/test/scout/ui/parallel_tests/investigations/run_workflow_action.spec.ts b/x-pack/solutions/security/plugins/security_solution/test/scout/ui/parallel_tests/investigations/run_workflow_action.spec.ts index 3311d2c16e75b..abf89c810b7f0 100644 --- a/x-pack/solutions/security/plugins/security_solution/test/scout/ui/parallel_tests/investigations/run_workflow_action.spec.ts +++ b/x-pack/solutions/security/plugins/security_solution/test/scout/ui/parallel_tests/investigations/run_workflow_action.spec.ts @@ -5,49 +5,9 @@ * 2.0. */ -import type { KibanaRole } from '@kbn/scout-security'; -import { spaceTest, tags } from '@kbn/scout-security'; +import { spaceTest, tags, CUSTOM_QUERY_RULE } from '@kbn/scout-security'; import { expect } from '@kbn/scout-security/ui'; -import { CUSTOM_QUERY_RULE } from '@kbn/scout-security/src/playwright/constants/detection_rules'; - -/** - * A role combining the security alert index privileges of platform_engineer with - * full Kibana access (including workflowsManagement) required to see and use the - * "Run workflow" alert action. - */ -const WORKFLOW_ENABLED_ROLE: KibanaRole = { - elasticsearch: { - cluster: ['manage'], - indices: [ - { - names: [ - '.alerts-security*', - '.internal.alerts-security*', - '.siem-signals-*', - 'apm-*-transaction*', - 'traces-apm*', - 'auditbeat-*', - 'endgame-*', - 'filebeat-*', - 'logs-*', - 'packetbeat-*', - 'winlogbeat-*', - 'logstash-*', - '.lists*', - '.items*', - ], - privileges: ['read', 'write'], - }, - ], - }, - kibana: [ - { - base: ['all'], - feature: {}, - spaces: ['*'], - }, - ], -}; +import { FULL_KIBANA_SECURITY_ROLE } from '../../common/roles'; // Failing: See https://github.com/elastic/kibana/issues/261392 spaceTest.describe.skip('Run workflow alert action', { tag: [...tags.stateful.classic] }, () => { @@ -66,7 +26,7 @@ spaceTest.describe.skip('Run workflow alert action', { tag: [...tags.stateful.cl }); // Use a custom role that includes workflowsManagement privileges (canExecuteWorkflow) // in addition to the security index privileges needed to view alerts - await browserAuth.loginWithCustomRole(WORKFLOW_ENABLED_ROLE); + await browserAuth.loginWithCustomRole(FULL_KIBANA_SECURITY_ROLE); }); spaceTest.afterEach(async ({ apiServices }) => { diff --git a/x-pack/solutions/security/plugins/security_solution/tsconfig.json b/x-pack/solutions/security/plugins/security_solution/tsconfig.json index f753ba9b7a2e6..4aec8f14fae23 100644 --- a/x-pack/solutions/security/plugins/security_solution/tsconfig.json +++ b/x-pack/solutions/security/plugins/security_solution/tsconfig.json @@ -240,6 +240,7 @@ "@kbn/inference-common", "@kbn/inference-langchain", "@kbn/scout-security", + "@kbn/ftr-llm-proxy", "@kbn/custom-icons", "@kbn/security-plugin-types-common", "@kbn/management-settings-ids", diff --git a/yarn.lock b/yarn.lock index de3361f18319d..15577e7f8270f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6927,6 +6927,10 @@ version "0.0.0" uid "" +"@kbn/evals-suite-security-alert-triage@link:x-pack/solutions/security/packages/kbn-evals-suite-security-alert-triage": + version "0.0.0" + uid "" + "@kbn/evals-suite-security-automatic-migrations@link:x-pack/solutions/security/packages/kbn-evals-suite-security-automatic-migrations": version "0.0.0" uid ""