Skip to content

Commit 6d315e4

Browse files
maximpnmaximpn
authored
[Security Solution] Add alerts_suppressed_count metrics tests for all rule types (elastic#262662)
**Epic:** elastic/security-team#15617 (internal) **Relates to:** elastic#257203 ## Summary Adds `alerts_suppressed_count` rule execution metric functional tests to make sure we log this metric correctly. ## Changes - Add integration test coverage for the `alerts_suppressed_count` execution metric across all detection rule types (EQL, ES|QL, indicator match, ML, new terms, custom query, threshold) - Each rule type gets two tests: verifying the count is 0 when no suppression is configured and verifying the correct count when alerts are suppressed - Refactor ML metrics tests to use inline document indexing instead of heavy esArchiver/ML module setup - Update `getLatestSecurityRuleExecutionMetricsFromEventLog` to accept an options object for better extensibility ## Flaky Test Runs - ✅ [EQL ESS + Serverless x100](https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/11567) - 🟡 [ES|QL ESS + Serverless x100](https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/11569) (one of the Serverless runs failed during setup with `index_not_found_exception: no such index [.kibana_streams]`. It looks like a flakiness in the test setup and not in the tests themselves). - ✅ [Machine Learning ESS + Serverless x100](https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/11549) - ✅ [Indicator Match ESS + Serverless x100](https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/11561) - ✅ [New Terms ESS + Serverless x100](https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/11559) - ✅ [Threshold ESS + Serverless x100](https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/11557) - ✅ [Custom Query ESS + Serverless x100](https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/11553) 🤖 Generated with [Claude Code](https://claude.com/claude-code)
1 parent c4510ea commit 6d315e4

8 files changed

Lines changed: 552 additions & 33 deletions

File tree

‎x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_metrics.ts‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -226,6 +226,74 @@ export default ({ getService }: FtrProviderContext) => {
226226
expect(alerts_candidate_count).toBe(2);
227227
});
228228
});
229+
230+
describe('alerts_suppressed_count', () => {
231+
it('records alerts_suppressed_count as 0 when no suppression is configured', async () => {
232+
const timestamp = new Date().toISOString();
233+
const document = {
234+
'@timestamp': timestamp,
235+
host: {
236+
name: 'test',
237+
},
238+
};
239+
const rule = getEqlRuleParams({
240+
index: ['test-data-1'],
241+
query: 'any where true',
242+
from: 'now-35m',
243+
interval: '30m',
244+
enabled: true,
245+
});
246+
247+
await indexListOfSourceDocuments([document]);
248+
249+
const createdRule = await createRule(supertest, log, rule);
250+
const alerts = await getOpenAlerts(supertest, log, es, createdRule);
251+
252+
expect(alerts.hits.hits).toHaveLength(1);
253+
254+
const { alerts_suppressed_count } =
255+
await getLatestSecurityRuleExecutionMetricsFromEventLog(es, log, createdRule.id);
256+
257+
expect(alerts_suppressed_count).toBe(0);
258+
});
259+
260+
it('records alerts_suppressed_count when alerts are suppressed', async () => {
261+
const timestamp = new Date().toISOString();
262+
const document = {
263+
'@timestamp': timestamp,
264+
host: {
265+
name: 'host-suppression-metrics',
266+
},
267+
};
268+
const rule = getEqlRuleParams({
269+
index: ['test-data-1'],
270+
query: 'any where true',
271+
alert_suppression: {
272+
group_by: ['host.name'],
273+
duration: {
274+
value: 300,
275+
unit: 'm',
276+
},
277+
missing_fields_strategy: 'suppress',
278+
},
279+
from: 'now-35m',
280+
interval: '30m',
281+
enabled: true,
282+
});
283+
284+
await indexListOfSourceDocuments([document, document]);
285+
286+
const createdRule = await createRule(supertest, log, rule);
287+
const alerts = await getOpenAlerts(supertest, log, es, createdRule);
288+
289+
expect(alerts.hits.hits).toHaveLength(1);
290+
291+
const { alerts_suppressed_count } =
292+
await getLatestSecurityRuleExecutionMetricsFromEventLog(es, log, createdRule.id);
293+
294+
expect(alerts_suppressed_count).toBe(1);
295+
});
296+
});
229297
});
230298
});
231299
};

‎x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/esql/trial_license_complete_tier/esql_metrics.ts‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,68 @@ export default ({ getService }: FtrProviderContext) => {
199199
expect(alerts_candidate_count).toBe(2);
200200
});
201201
});
202+
203+
describe('alerts_suppressed_count', () => {
204+
it('records alerts_suppressed_count as 0 when no suppression is configured', async () => {
205+
const timestamp = new Date().toISOString();
206+
const document = {
207+
'@timestamp': timestamp,
208+
host: { name: 'test-1' },
209+
};
210+
const rule = getEsqlRuleParams({
211+
query: 'from test-data-1 metadata _id, _index, _version',
212+
from: 'now-35m',
213+
interval: '30m',
214+
enabled: true,
215+
});
216+
217+
await indexListOfSourceDocuments([document]);
218+
219+
const createdRule = await createRule(supertest, log, rule);
220+
const alerts = await getOpenAlerts(supertest, log, es, createdRule);
221+
222+
expect(alerts.hits.hits).toHaveLength(1);
223+
224+
const { alerts_suppressed_count } =
225+
await getLatestSecurityRuleExecutionMetricsFromEventLog(es, log, createdRule.id);
226+
227+
expect(alerts_suppressed_count).toBe(0);
228+
});
229+
230+
it('records alerts_suppressed_count when alerts are suppressed', async () => {
231+
const timestamp = new Date().toISOString();
232+
const document = {
233+
'@timestamp': timestamp,
234+
host: { name: 'test-1' },
235+
};
236+
const rule = getEsqlRuleParams({
237+
query: 'from test-data-1 metadata _id, _index, _version',
238+
alert_suppression: {
239+
group_by: ['host.name'],
240+
duration: {
241+
value: 300,
242+
unit: 'm',
243+
},
244+
missing_fields_strategy: 'suppress',
245+
},
246+
from: 'now-35m',
247+
interval: '30m',
248+
enabled: true,
249+
});
250+
251+
await indexListOfSourceDocuments([document, document]);
252+
253+
const createdRule = await createRule(supertest, log, rule);
254+
const alerts = await getOpenAlerts(supertest, log, es, createdRule);
255+
256+
expect(alerts.hits.hits).toHaveLength(1);
257+
258+
const { alerts_suppressed_count } =
259+
await getLatestSecurityRuleExecutionMetricsFromEventLog(es, log, createdRule.id);
260+
261+
expect(alerts_suppressed_count).toBe(1);
262+
});
263+
});
202264
});
203265
});
204266
};

‎x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/indicator_match/trial_license_complete_tier/indicator_match_metrics.ts‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -255,6 +255,84 @@ export default ({ getService }: FtrProviderContext) => {
255255
expect(alerts_candidate_count).toBe(2);
256256
});
257257
});
258+
259+
describe('alerts_suppressed_count', () => {
260+
it('records alerts_suppressed_count as 0 when no suppression is configured', async () => {
261+
const timestamp = new Date().toISOString();
262+
const threatIndicatorDocument = {
263+
'@timestamp': timestamp,
264+
host: { name: 'test-1' },
265+
};
266+
const document = {
267+
'@timestamp': timestamp,
268+
host: { name: 'test-1' },
269+
};
270+
const rule = getThreatMatchRuleParams({
271+
threat_query: '*:*',
272+
threat_index: ['ti_test_1'],
273+
query: '*:*',
274+
index: ['test-data-1'],
275+
from: 'now-35m',
276+
interval: '30m',
277+
enabled: true,
278+
});
279+
280+
await indexThreatIndicatorDocuments([threatIndicatorDocument]);
281+
await indexListOfSourceDocuments([document]);
282+
283+
const createdRule = await createRule(supertest, log, rule);
284+
const alerts = await getOpenAlerts(supertest, log, es, createdRule);
285+
286+
expect(alerts.hits.hits).toHaveLength(1);
287+
288+
const { alerts_suppressed_count } =
289+
await getLatestSecurityRuleExecutionMetricsFromEventLog(es, log, createdRule.id);
290+
291+
expect(alerts_suppressed_count).toBe(0);
292+
});
293+
294+
it('records alerts_suppressed_count when alerts are suppressed', async () => {
295+
const timestamp = new Date().toISOString();
296+
const threatIndicatorDocument = {
297+
'@timestamp': timestamp,
298+
host: { name: 'test-1' },
299+
};
300+
const document = {
301+
'@timestamp': timestamp,
302+
host: { name: 'test-1' },
303+
};
304+
const rule = getThreatMatchRuleParams({
305+
threat_query: '*:*',
306+
threat_index: ['ti_test_1'],
307+
query: '*:*',
308+
index: ['test-data-1'],
309+
from: 'now-35m',
310+
interval: '30m',
311+
alert_suppression: {
312+
group_by: ['host.name'],
313+
duration: {
314+
value: 300,
315+
unit: 'm',
316+
},
317+
missing_fields_strategy: 'suppress',
318+
},
319+
enabled: true,
320+
});
321+
322+
await indexThreatIndicatorDocuments([threatIndicatorDocument]);
323+
await indexListOfSourceDocuments([document, document]);
324+
325+
const createdRule = await createRule(supertest, log, rule);
326+
const alerts = await getOpenAlerts(supertest, log, es, createdRule);
327+
328+
expect(alerts.hits.hits).toHaveLength(1);
329+
330+
const { alerts_suppressed_count } =
331+
await getLatestSecurityRuleExecutionMetricsFromEventLog(es, log, createdRule.id);
332+
333+
expect(alerts_suppressed_count).toBe(1);
334+
});
335+
});
258336
});
259337
});
260338
};

‎x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/machine_learning/trial_license_complete_tier/machine_learning_metrics.ts‎

Lines changed: 75 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -11,32 +11,22 @@ import type { Anomaly } from '@kbn/security-solution-plugin/server/lib/machine_l
1111
import {
1212
getMLRuleParams,
1313
dataGeneratorFactory,
14-
forceStartDatafeeds,
1514
getLatestSecurityRuleExecutionMetricsFromEventLog,
1615
getOpenAlerts,
17-
setupMlModulesWithRetry,
1816
} from '../../../../utils';
1917
import type { FtrProviderContext } from '../../../../../../ftr_provider_context';
20-
import { EsArchivePathBuilder } from '../../../../../../es_archive_path_builder';
2118

2219
export default ({ getService }: FtrProviderContext) => {
2320
const supertest = getService('supertest');
24-
const esArchiver = getService('esArchiver');
2521
const es = getService('es');
2622
const log = getService('log');
27-
const config = getService('config');
28-
const retry = getService('retry');
2923

30-
const isServerless = config.get('serverless');
31-
const dataPathBuilder = new EsArchivePathBuilder(isServerless);
32-
const auditPath = dataPathBuilder.getPath('auditbeat/hosts');
33-
34-
const siemModule = 'security_linux_v3';
3524
const mlJobId = 'v3_linux_anomalous_network_activity_ea';
25+
const sourceEventsIndexName = '.ml-anomalies-custom-v3_linux_anomalous_network_activity_ea';
3626

3727
const { indexListOfDocuments } = dataGeneratorFactory({
3828
es,
39-
index: '.ml-anomalies-custom-v3_linux_anomalous_network_activity_ea',
29+
index: sourceEventsIndexName,
4030
log,
4131
});
4232

@@ -58,25 +48,12 @@ export default ({ getService }: FtrProviderContext) => {
5848
};
5949

6050
describe('@ess @serverless Rule execution metrics for Machine Learning rules', () => {
61-
before(async () => {
62-
await esArchiver.load(auditPath);
63-
await setupMlModulesWithRetry({ module: siemModule, supertest, retry });
64-
await forceStartDatafeeds({ jobId: mlJobId, rspCode: 200, supertest });
65-
await esArchiver.load(
66-
'x-pack/solutions/security/test/fixtures/es_archives/security_solution/anomalies'
67-
);
68-
});
69-
70-
after(async () => {
71-
await esArchiver.unload(auditPath);
72-
await esArchiver.unload(
73-
'x-pack/solutions/security/test/fixtures/es_archives/security_solution/anomalies'
74-
);
75-
await deleteAllAlerts(supertest, log, es);
76-
await deleteAllRules(supertest, log);
77-
});
78-
7951
beforeEach(async () => {
52+
await es.indices.delete({
53+
index: sourceEventsIndexName,
54+
ignore_unavailable: true,
55+
});
56+
8057
await deleteAllAlerts(supertest, log, es);
8158
await deleteAllRules(supertest, log);
8259
});
@@ -108,13 +85,19 @@ export default ({ getService }: FtrProviderContext) => {
10885

10986
describe('alerts_candidate_count', () => {
11087
it('records alerts_candidate_count value', async () => {
88+
const timestamp = new Date().toISOString();
89+
const anomaly = {
90+
...baseAnomaly,
91+
timestamp,
92+
};
93+
await indexListOfDocuments([anomaly]);
94+
11195
const createdRule = await createRule(
11296
supertest,
11397
log,
11498
getMLRuleParams({
11599
...sharedMlRuleRewrites,
116100
anomaly_threshold: 30,
117-
from: '1900-01-01T00:00:00.000Z',
118101
enabled: true,
119102
})
120103
);
@@ -160,6 +143,67 @@ export default ({ getService }: FtrProviderContext) => {
160143
expect(alerts_candidate_count).toBe(2);
161144
});
162145
});
146+
147+
describe('alerts_suppressed_count', () => {
148+
it('records alerts_suppressed_count as 0 when no suppression is configured', async () => {
149+
const timestamp = new Date().toISOString();
150+
const anomaly = {
151+
...baseAnomaly,
152+
timestamp,
153+
};
154+
await indexListOfDocuments([anomaly]);
155+
156+
const createdRule = await createRule(
157+
supertest,
158+
log,
159+
getMLRuleParams({
160+
...sharedMlRuleRewrites,
161+
anomaly_threshold: 30,
162+
enabled: true,
163+
})
164+
);
165+
const alerts = await getOpenAlerts(supertest, log, es, createdRule);
166+
167+
expect(alerts.hits.hits).toHaveLength(1);
168+
169+
const { alerts_suppressed_count } =
170+
await getLatestSecurityRuleExecutionMetricsFromEventLog(es, log, createdRule.id);
171+
172+
expect(alerts_suppressed_count).toBe(0);
173+
});
174+
175+
it('records alerts_suppressed_count when alerts are suppressed', async () => {
176+
const timestamp = new Date().toISOString();
177+
const anomaly = {
178+
...baseAnomaly,
179+
timestamp,
180+
};
181+
await indexListOfDocuments([anomaly, anomaly]);
182+
183+
const createdRule = await createRule(
184+
supertest,
185+
log,
186+
getMLRuleParams({
187+
...sharedMlRuleRewrites,
188+
anomaly_threshold: 40,
189+
enabled: true,
190+
alert_suppression: {
191+
group_by: ['user.name'],
192+
missing_fields_strategy: 'suppress',
193+
},
194+
from: timestamp,
195+
})
196+
);
197+
const alerts = await getOpenAlerts(supertest, log, es, createdRule);
198+
199+
expect(alerts.hits.hits).toHaveLength(1);
200+
201+
const { alerts_suppressed_count } =
202+
await getLatestSecurityRuleExecutionMetricsFromEventLog(es, log, createdRule.id);
203+
204+
expect(alerts_suppressed_count).toBe(1);
205+
});
206+
});
163207
});
164208
});
165209
};

0 commit comments

Comments
 (0)