Skip to content

Commit 9c80944

Browse files
committed
Add wiring for security detection rules
1 parent d9badae commit 9c80944

17 files changed

Lines changed: 1189 additions & 46 deletions

File tree

x-pack/platform/plugins/shared/alerting/server/application/rule/methods/bulk_create/bulk_create_rules.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -418,7 +418,6 @@ async function runBatch<Params extends RuleParams>({
418418
})
419419
);
420420

421-
const createTime = Date.now();
422421
let bulkResponse;
423422
try {
424423
bulkResponse = await withSpan(
@@ -456,6 +455,7 @@ async function runBatch<Params extends RuleParams>({
456455
// Phase B4 per-row outcomes.
457456
const batchSuccessfulIds: string[] = [];
458457
const taskIdsToCleanUp: string[] = [];
458+
const createTime = Date.now();
459459
const successfulSavedObjects: Array<SavedObject<RawRule>> = [];
460460
let perRowFailureOccurred = false;
461461

x-pack/solutions/security/plugins/security_solution/common/experimental_features.ts

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,16 @@ export const allowedExperimentalValues = Object.freeze({
2222
*/
2323
previewTelemetryUrlEnabled: false,
2424

25+
/**
26+
* When enabled, prebuilt rule installation (POST .../prebuilt_rules/installation/_perform)
27+
* and rule import (POST .../rules/_import) use the new alerting `rulesClient.bulkCreateRules`
28+
* path, which handles both disabled and enabled rules (API key minting + task scheduling)
29+
* in a single bulk call instead of the per-rule create loop.
30+
*
31+
* Release: TBD
32+
*/
33+
bulkCreateRulesEnabled: false,
34+
2535
/**
2636
* Enables extended rule execution logging to Event Log. When this setting is enabled:
2737
* - Rules write their console error, info, debug, and trace messages to Event Log,

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/perform_rule_installation/perform_rule_installation_handler.ts

Lines changed: 26 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -109,31 +109,40 @@ export const performRuleInstallationHandler = async (
109109
);
110110
ruleInstallQueue.push(...(await excludeLicenseRestrictedRules(allInstallableRules, mlAuthz)));
111111
}
112-
113112
const changeTracking = {
114113
metadata: {
115114
bulkCount: ruleInstallQueue.length,
116115
},
117116
};
118117

119-
const BATCH_SIZE = 100;
120-
while (ruleInstallQueue.length > 0) {
121-
const rulesToInstall = ruleInstallQueue.splice(0, BATCH_SIZE);
122-
const ruleAssets = await ruleAssetsClient.fetchAssetsByVersion(rulesToInstall);
123-
124-
const { results, errors } = await createPrebuiltRules(
125-
detectionRulesClient,
126-
ruleAssets,
127-
changeTracking,
128-
logger
129-
);
130-
131-
const batchInstalledRules = results.map(({ result: rule }) =>
132-
pick(rule, ['id', 'rule_id', 'version'])
118+
const { bulkCreateRulesEnabled } = ctx.securitySolution.getConfig().experimentalFeatures;
119+
if (bulkCreateRulesEnabled) {
120+
const ruleAssets = await ruleAssetsClient.fetchAssetsByVersion(ruleInstallQueue);
121+
const { results, errors } = await detectionRulesClient.bulkCreatePrebuiltRules({
122+
rules: ruleAssets,
123+
});
124+
installedRules.push(
125+
...results.map(({ result: rule }) => pick(rule, ['id', 'rule_id', 'version']))
133126
);
134-
135-
installedRules.push(...batchInstalledRules);
136127
ruleErrors.push(...errors);
128+
} else {
129+
const BATCH_SIZE = 100;
130+
while (ruleInstallQueue.length > 0) {
131+
const rulesToInstall = ruleInstallQueue.splice(0, BATCH_SIZE);
132+
const ruleAssets = await ruleAssetsClient.fetchAssetsByVersion(rulesToInstall);
133+
134+
const { results, errors } = await createPrebuiltRules(
135+
detectionRulesClient,
136+
ruleAssets,
137+
changeTracking,
138+
logger
139+
);
140+
141+
installedRules.push(
142+
...results.map(({ result: rule }) => pick(rule, ['id', 'rule_id', 'version']))
143+
);
144+
ruleErrors.push(...errors);
145+
}
137146
}
138147

139148
const { error: timelineInstallationError } = await performTimelinesInstallation(

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/api/timeouts.ts renamed to x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/api/constants.ts

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,3 +13,6 @@ export const RULE_MANAGEMENT_BULK_ACTION_SOCKET_TIMEOUT_MS = 3600000 as const;
1313
* 1 hour = 3600000 ms = 60 minutes * 60 seconds * 1000 ms
1414
*/
1515
export const RULE_MANAGEMENT_IMPORT_EXPORT_SOCKET_TIMEOUT_MS = 3600000 as const;
16+
17+
/** Cap concurrent rule imports at 1 to bound heap; mirrors PREBUILT_RULES_OPERATION_CONCURRENCY. */
18+
export const RULE_MANAGEMENT_IMPORT_CONCURRENCY = 1;

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/bulk_actions/route.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ import {
4444
validateBulkDuplicateRule,
4545
} from '../../../logic/bulk_actions/validations';
4646
import { getExportByObjectIds } from '../../../logic/export/get_export_by_object_ids';
47-
import { RULE_MANAGEMENT_BULK_ACTION_SOCKET_TIMEOUT_MS } from '../../timeouts';
47+
import { RULE_MANAGEMENT_BULK_ACTION_SOCKET_TIMEOUT_MS } from '../../constants';
4848
import type { BulkActionError } from './bulk_actions_response';
4949
import { buildBulkResponse } from './bulk_actions_response';
5050
import { bulkEnableDisableRules } from './bulk_enable_disable_rules';

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/export_rules/route.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ import { getRulesCount } from '../../../logic/search/get_existing_prepackaged_ru
2020
import { getExportByObjectIds } from '../../../logic/export/get_export_by_object_ids';
2121
import { getExportAll } from '../../../logic/export/get_export_all';
2222
import { buildSiemResponse } from '../../../../routes/utils';
23-
import { RULE_MANAGEMENT_IMPORT_EXPORT_SOCKET_TIMEOUT_MS } from '../../timeouts';
23+
import { RULE_MANAGEMENT_IMPORT_EXPORT_SOCKET_TIMEOUT_MS } from '../../constants';
2424

2525
export const exportRulesRoute = (
2626
router: SecuritySolutionPluginRouter,

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/import_rules/route.ts

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,11 @@ import {
3939
getTupleDuplicateErrorsAndUniqueRules,
4040
migrateLegacyActionsIds,
4141
} from '../../../utils/utils';
42-
import { RULE_MANAGEMENT_IMPORT_EXPORT_SOCKET_TIMEOUT_MS } from '../../timeouts';
42+
import {
43+
RULE_MANAGEMENT_IMPORT_CONCURRENCY,
44+
RULE_MANAGEMENT_IMPORT_EXPORT_SOCKET_TIMEOUT_MS,
45+
} from '../../constants';
46+
import { routeLimitedConcurrencyTag } from '../../../../../../utils/route_limited_concurrency_tag';
4347
import { createPrebuiltRuleObjectsClient } from '../../../../prebuilt_rules/logic/rule_objects/prebuilt_rule_objects_client';
4448

4549
const CHUNK_PARSED_OBJECT_SIZE = 50;
@@ -63,6 +67,7 @@ export const importRulesRoute = (
6367
maxBytes: config.maxRuleImportPayloadBytes,
6468
output: 'stream',
6569
},
70+
tags: [routeLimitedConcurrencyTag(RULE_MANAGEMENT_IMPORT_CONCURRENCY)],
6671
timeout: {
6772
idleSocket: RULE_MANAGEMENT_IMPORT_EXPORT_SOCKET_TIMEOUT_MS,
6873
},
@@ -187,6 +192,7 @@ export const importRulesRoute = (
187192
ctx.securitySolution.getCheckOsqueryResponseActionAuthz(),
188193
});
189194

195+
const experimentalFeatures = ctx.securitySolution.getConfig().experimentalFeatures;
190196
const ruleChunks = chunk(CHUNK_PARSED_OBJECT_SIZE, validatedResponseActionsRules);
191197

192198
const importRuleResponse = await importRules({
@@ -200,6 +206,7 @@ export const importRulesRoute = (
200206
allowMissingConnectorSecrets: !!actionConnectors.length,
201207
ruleSourceImporter,
202208
detectionRulesClient,
209+
experimentalFeatures,
203210
});
204211

205212
const parseErrors = parsedRuleErrors.map((error) =>

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/__mocks__/detection_rules_client.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ const createDetectionRulesClientMock = () => {
1313
const mocked: DetectionRulesClientMock = {
1414
createCustomRule: jest.fn(),
1515
createPrebuiltRule: jest.fn(),
16+
bulkCreatePrebuiltRules: jest.fn().mockResolvedValue({ results: [], errors: [] }),
1617
updateRule: jest.fn(),
1718
patchRule: jest.fn(),
1819
deleteRule: jest.fn(),
@@ -21,6 +22,7 @@ const createDetectionRulesClientMock = () => {
2122
revertPrebuiltRule: jest.fn(),
2223
importRule: jest.fn(),
2324
importRules: jest.fn(),
25+
bulkImportRules: jest.fn().mockResolvedValue({ responses: [] }),
2426
getRuleCustomizationStatus: jest.fn(),
2527
getHistoryForRule: jest.fn(),
2628
};
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
import { rulesClientMock } from '@kbn/alerting-plugin/server/mocks';
9+
import type { ActionsClient } from '@kbn/actions-plugin/server';
10+
import { savedObjectsClientMock } from '@kbn/core/server/mocks';
11+
import { licenseMock } from '@kbn/licensing-plugin/common/licensing.mock';
12+
13+
import { SecurityRuleChangeTrackingAction } from '../../../../../../common/detection_engine/rule_management/rule_change_tracking';
14+
import { getCreateRulesSchemaMock } from '../../../../../../common/api/detection_engine/model/rule_schema/mocks';
15+
import { buildMlAuthz } from '../../../../machine_learning/authz';
16+
import { throwAuthzError } from '../../../../machine_learning/validation';
17+
import { createDetectionRulesClient } from './detection_rules_client';
18+
import type { IDetectionRulesClient } from './detection_rules_client_interface';
19+
import { createProductFeaturesServiceMock } from '../../../../product_features_service/mocks';
20+
import { getMockRulesAuthz } from '../../__mocks__/authz';
21+
22+
jest.mock('../../../../machine_learning/authz');
23+
jest.mock('../../../../machine_learning/validation');
24+
25+
describe('DetectionRulesClient.bulkCreatePrebuiltRules', () => {
26+
let rulesClient: ReturnType<typeof rulesClientMock.create>;
27+
let detectionRulesClient: IDetectionRulesClient;
28+
29+
const mlAuthz = (buildMlAuthz as jest.Mock)();
30+
const rulesAuthz = getMockRulesAuthz();
31+
const actionsClient: jest.Mocked<ActionsClient> = {
32+
isSystemAction: jest.fn(),
33+
} as unknown as jest.Mocked<ActionsClient>;
34+
35+
beforeEach(() => {
36+
rulesClient = rulesClientMock.create();
37+
(throwAuthzError as jest.Mock).mockReset();
38+
detectionRulesClient = createDetectionRulesClient({
39+
actionsClient,
40+
rulesClient,
41+
mlAuthz,
42+
rulesAuthz,
43+
savedObjectsClient: savedObjectsClientMock.create(),
44+
license: licenseMock.createLicenseMock(),
45+
productFeaturesService: createProductFeaturesServiceMock(),
46+
});
47+
});
48+
49+
it('issues a single bulkCreateRules call with disabled, immutable rules and emits { id, rule_id, version } pairs', async () => {
50+
const asset1 = { ...getCreateRulesSchemaMock(), version: 1, rule_id: 'rule-1' };
51+
const asset2 = { ...getCreateRulesSchemaMock(), version: 2, rule_id: 'rule-2' };
52+
53+
rulesClient.bulkCreateRules.mockImplementationOnce(async (args) => {
54+
const ids = args.rules.map((r) => (r.options as { id: string }).id);
55+
return { successfulIds: ids, errors: [], total: ids.length };
56+
});
57+
58+
const { results, errors } = await detectionRulesClient.bulkCreatePrebuiltRules({
59+
rules: [asset1, asset2],
60+
});
61+
62+
expect(rulesClient.bulkCreateRules).toHaveBeenCalledTimes(1);
63+
const callArgs = rulesClient.bulkCreateRules.mock.calls[0][0];
64+
expect(callArgs.rules).toHaveLength(2);
65+
expect(callArgs.rules.every((r) => r.data.enabled === false)).toBe(true);
66+
expect(callArgs.rules.every((r) => r.data.params.immutable === true)).toBe(true);
67+
expect(errors).toEqual([]);
68+
expect(results).toHaveLength(2);
69+
const callIds = callArgs.rules.map((r) => (r.options as { id: string }).id);
70+
expect(results[0].result).toEqual({ id: callIds[0], rule_id: 'rule-1', version: 1 });
71+
expect(results[1].result).toEqual({ id: callIds[1], rule_id: 'rule-2', version: 2 });
72+
});
73+
74+
it('issues a single bulkCreateRules call regardless of input size (alerting batches internally)', async () => {
75+
const assets = Array.from({ length: 250 }, (_, i) => ({
76+
...getCreateRulesSchemaMock(),
77+
version: 1,
78+
rule_id: `rule-${i}`,
79+
}));
80+
81+
rulesClient.bulkCreateRules.mockImplementationOnce(async (args) => ({
82+
successfulIds: args.rules.map((r) => (r.options as { id: string }).id),
83+
errors: [],
84+
total: args.rules.length,
85+
}));
86+
87+
const { results, errors } = await detectionRulesClient.bulkCreatePrebuiltRules({
88+
rules: assets,
89+
});
90+
91+
expect(rulesClient.bulkCreateRules).toHaveBeenCalledTimes(1);
92+
expect(results).toHaveLength(250);
93+
expect(errors).toEqual([]);
94+
});
95+
96+
it('reports per-rule errors from the alerting layer', async () => {
97+
const asset = { ...getCreateRulesSchemaMock(), version: 1, rule_id: 'rule-1' };
98+
99+
rulesClient.bulkCreateRules.mockImplementationOnce(async (args) => {
100+
const id = (args.rules[0].options as { id: string }).id;
101+
return {
102+
successfulIds: [],
103+
errors: [{ message: 'boom', status: 500, rule: { id, name: asset.name } }],
104+
total: 1,
105+
};
106+
});
107+
108+
const { results, errors } = await detectionRulesClient.bulkCreatePrebuiltRules({
109+
rules: [asset],
110+
});
111+
112+
expect(results).toEqual([]);
113+
expect(errors).toHaveLength(1);
114+
expect(errors[0].item).toBe(asset);
115+
expect(errors[0].error.message).toBe('boom');
116+
});
117+
118+
it('returns ML-auth failures as per-rule errors without calling bulkCreateRules', async () => {
119+
const asset = { ...getCreateRulesSchemaMock(), version: 1, rule_id: 'rule-1' };
120+
(throwAuthzError as jest.Mock).mockImplementation(() => {
121+
throw new Error('ML auth denied');
122+
});
123+
124+
const { results, errors } = await detectionRulesClient.bulkCreatePrebuiltRules({
125+
rules: [asset],
126+
});
127+
128+
expect(rulesClient.bulkCreateRules).not.toHaveBeenCalled();
129+
expect(results).toEqual([]);
130+
expect(errors[0].error.message).toBe('ML auth denied');
131+
});
132+
133+
it('returns empty result for empty input', async () => {
134+
const result = await detectionRulesClient.bulkCreatePrebuiltRules({ rules: [] });
135+
expect(result).toEqual({ results: [], errors: [] });
136+
expect(rulesClient.bulkCreateRules).not.toHaveBeenCalled();
137+
});
138+
139+
it('forwards ruleInstall action and rules.length as bulkCount to rulesClient.bulkCreateRules', async () => {
140+
const assets = Array.from({ length: 3 }, (_, i) => ({
141+
...getCreateRulesSchemaMock(),
142+
version: 1,
143+
rule_id: `rule-${i}`,
144+
}));
145+
146+
rulesClient.bulkCreateRules.mockResolvedValueOnce({
147+
successfulIds: [],
148+
errors: [],
149+
total: 0,
150+
});
151+
152+
await detectionRulesClient.bulkCreatePrebuiltRules({ rules: assets });
153+
154+
expect(rulesClient.bulkCreateRules).toHaveBeenCalledWith(
155+
expect.objectContaining({
156+
changeTracking: {
157+
action: SecurityRuleChangeTrackingAction.ruleInstall,
158+
metadata: { bulkCount: assets.length },
159+
},
160+
})
161+
);
162+
});
163+
});

0 commit comments

Comments
 (0)