From d8cf4cf583405fbd5313b3f04e29123a46f76b7a Mon Sep 17 00:00:00 2001 From: Nikita Khristinin Date: Thu, 21 Nov 2024 20:15:15 +0100 Subject: [PATCH 01/17] structure --- .../rule/methods/fing_gaps/find_gaps.ts | 30 +++++ .../lib/rule_gaps/find_gaps_by_rule_id.ts | 26 ++++ .../get_total_unfilled_gap_duration.ts | 39 ++++++ .../rule_gaps/process_gaps_after_execution.ts | 119 ++++++++++++++++++ .../rule_gaps/api/find/fing_gaps_route.ts | 47 +++++++ .../server/rules_client/rules_client.ts | 4 + .../server/task_runner/ad_hoc_task_runner.ts | 16 +++ .../server/task_runner/task_runner.ts | 25 +++- 8 files changed, 304 insertions(+), 2 deletions(-) create mode 100644 x-pack/plugins/alerting/server/application/rule/methods/fing_gaps/find_gaps.ts create mode 100644 x-pack/plugins/alerting/server/lib/rule_gaps/find_gaps_by_rule_id.ts create mode 100644 x-pack/plugins/alerting/server/lib/rule_gaps/get_total_unfilled_gap_duration.ts create mode 100644 x-pack/plugins/alerting/server/lib/rule_gaps/process_gaps_after_execution.ts create mode 100644 x-pack/plugins/alerting/server/routes/rule_gaps/api/find/fing_gaps_route.ts diff --git a/x-pack/plugins/alerting/server/application/rule/methods/fing_gaps/find_gaps.ts b/x-pack/plugins/alerting/server/application/rule/methods/fing_gaps/find_gaps.ts new file mode 100644 index 0000000000000..891218692cc83 --- /dev/null +++ b/x-pack/plugins/alerting/server/application/rule/methods/fing_gaps/find_gaps.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import Boom from '@hapi/boom'; + +import { RulesClientContext } from '../../../../rules_client'; + +import { findGapsByRuleId } from '../../../../lib/rule_gaps/find_gaps_by_rule_id'; + +export async function findBackfill(context: RulesClientContext, params: FindGapsParams) { + try { + const gaps = await findGapsByRuleId({ + ruleId: params.ruleId, + eventLog: context.eventLogger, + logger: context.logger, + }); + + // transform gaps + + return gaps; + } catch (err) { + const errorMessage = `Failed to find gaps`; + context.logger.error(`${errorMessage} - ${err}`); + throw Boom.boomify(err, { message: errorMessage }); + } +} diff --git a/x-pack/plugins/alerting/server/lib/rule_gaps/find_gaps_by_rule_id.ts b/x-pack/plugins/alerting/server/lib/rule_gaps/find_gaps_by_rule_id.ts new file mode 100644 index 0000000000000..4fe17708ca2d5 --- /dev/null +++ b/x-pack/plugins/alerting/server/lib/rule_gaps/find_gaps_by_rule_id.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IEventLogger } from '@kbn/event-log-plugin/server'; +import { Logger } from '@kbn/core/server'; +export async function findGapsByRuleId(params: { + ruleId: string; + timeRange?: { from: string; to: string }; + eventLog: IEventLogger; + logger: Logger; +}): Promise<[]> { + const { ruleId, timeRange, eventLog, logger } = params; + + try { + // return await eventLogeventLog.findEvents({ + // }); + return []; + } catch (err) { + logger.error(`Failed to find gaps for rule ${ruleId}: ${err.message}`); + throw err; + } +} diff --git a/x-pack/plugins/alerting/server/lib/rule_gaps/get_total_unfilled_gap_duration.ts b/x-pack/plugins/alerting/server/lib/rule_gaps/get_total_unfilled_gap_duration.ts new file mode 100644 index 0000000000000..5d4eaeaba4c78 --- /dev/null +++ b/x-pack/plugins/alerting/server/lib/rule_gaps/get_total_unfilled_gap_duration.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IEventLogger } from '@kbn/event-log-plugin/server'; +import { Logger } from '@kbn/core/server'; + +export async function getTotalUnfilledGapDuration(params: { + ruleId: string; + timeRange?: { from: string; to: string }; + eventLog: IEventLogger; + logger: Logger; +}): Promise<{ + unfiled_gap_duration_ms: { + "1d": number; + "3d": number; + "7d": number; + }; +}> { + const { ruleId, timeRange, eventLog, logger } = params; + + try { + + const aggs = // eventLog... + return { + unfiled_gap_duration_ms: { + "1d": 0, + "3d": 0, + "7d": 0, + }, + }; + } catch (err) { + logger.error(`Failed to find unffiled gap duration for rule ${ruleId}: ${err.message}`); + throw err; + } +} diff --git a/x-pack/plugins/alerting/server/lib/rule_gaps/process_gaps_after_execution.ts b/x-pack/plugins/alerting/server/lib/rule_gaps/process_gaps_after_execution.ts new file mode 100644 index 0000000000000..12e6ef121c817 --- /dev/null +++ b/x-pack/plugins/alerting/server/lib/rule_gaps/process_gaps_after_execution.ts @@ -0,0 +1,119 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ISavedObjectsRepository, Logger } from '@kbn/core/server'; +import { IEventLogger } from '@kbn/event-log-plugin/server'; +import { AD_HOC_RUN_SAVED_OBJECT_TYPE, RULE_SAVED_OBJECT_TYPE } from '../../saved_objects'; +import { AlertingEventLogger } from '../alerting_event_logger/alerting_event_logger'; +import { findGapsByRuleId } from './find_gaps_by_rule_id'; +import { RuleGap } from './types'; +import { AdHocRunSO } from '../../data/ad_hoc_run/types'; + +/** + * Fill the gap with the interval + */ +const fillGap = async ({ + gap, +}: { + gap: RuleGap; +}) => { + + gap.status = // .. + gap.filled_intervals = // ... + gap.in_progress_intervals = // ... + gap.unfilled_intervals = // ... + + return gap; +}; + + + +/** + * Find all overlapping backfill tasks and update the gap status accordingly + */ +const updateGapStatus = async ({ + gap, + savedObjectsClient, + ruleId, +}: { + gap: RuleGap; + savedObjectsClient: ISavedObjectsRepository; + ruleId: string; +}) => { + + + const backfillResults = await savedObjectsClient.find({ + type: AD_HOC_RUN_SAVED_OBJECT_TYPE, + hasReference: { + type: RULE_SAVED_OBJECT_TYPE, + id: ruleId, + }, + // Filter for backfills that overlap with our interval + filter: ` + ad_hoc_run_params.attributes.start <= "${gap.range.lte}" and + ad_hoc_run_params.attributes.end >= "${gap.range.gte}" + `, + page: 1, + perPage: 100, + }); + + const hasPendingOverlapingTasks = // backfillResults.......; + + if (hasPendingOverlapingTasks && gap.status !== "FILLED") { + gap.status = "IN_PROGRESS"; + gap.in_progress_intervals = //..[] + gap.unfilled_intervals = //... [] + } + + + return gap; + +} +export async function processGapsAfterExecution(params: { + ruleId: string; + executedInterval: { from: Date; to: Date }; + alertingEventLogger: AlertingEventLogger; + eventLog: IEventLogger; + savedObjectsClient: ISavedObjectsRepository; + logger: Logger; + needToFillGaps: boolean; +}): Promise { + const { + ruleId, + executedInterval, + alertingEventLogger, + logger, + savedObjectsClient, + eventLog, + needToFillGaps, + } = params; + + try { + const allGaps = await findGapsByRuleId({ + ruleId, + timeRange: executedInterval, + eventLog, + logger, + }); + + for (const gap of allGaps) { + let newGap = needToFillGaps ? fillGap(gap) : gap; + + const updatedGap = await updateGapStatus({ + gap: newGap, + savedObjectsClient, + ruleId, + }); + + await alertingEventLogger.updateGap(updatedGap); + } + + } catch (err) { + logger.error(`Failed to process gaps for rule ${ruleId}: ${err.message}`); + throw err; + } +} diff --git a/x-pack/plugins/alerting/server/routes/rule_gaps/api/find/fing_gaps_route.ts b/x-pack/plugins/alerting/server/routes/rule_gaps/api/find/fing_gaps_route.ts new file mode 100644 index 0000000000000..4f34383c819b1 --- /dev/null +++ b/x-pack/plugins/alerting/server/routes/rule_gaps/api/find/fing_gaps_route.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { IRouter } from '@kbn/core/server'; +import { + findQuerySchemaV1, + FindGapsRequestQueryV1, + FindGapsResponseV1, +} from '../../../../../common/routes/gaps/apis/find'; +import { ILicenseState } from '../../../../lib'; +import { verifyAccessAndContext } from '../../../lib'; +import { + AlertingRequestHandlerContext, + INTERNAL_ALERTING_GAPS_FIND_API_PATH, +} from '../../../../types'; + +export const findGapsRoute = ( + router: IRouter, + licenseState: ILicenseState +) => { + router.post( + { + path: `${INTERNAL_ALERTING_GAPS_FIND_API_PATH}`, + validate: { + query: findQuerySchemaV1, + }, + options: { + access: 'internal', + }, + }, + router.handleLegacyErrors( + verifyAccessAndContext(licenseState, async function (context, req, res) { + const rulesClient = (await context.alerting).getRulesClient(); + const query: FindGapsRequestQueryV1 = req.query; + + const result = await rulesClient.findGaps(transformRequestV1(query)); + const response: FindGapsResponseV1 = { + body: transformResponseV1(result), + }; + return res.ok(response); + }) + ) + ); +}; diff --git a/x-pack/plugins/alerting/server/rules_client/rules_client.ts b/x-pack/plugins/alerting/server/rules_client/rules_client.ts index 4c86469f11a29..71f408af1c6bb 100644 --- a/x-pack/plugins/alerting/server/rules_client/rules_client.ts +++ b/x-pack/plugins/alerting/server/rules_client/rules_client.ts @@ -76,6 +76,8 @@ import { deleteBackfill } from '../application/backfill/methods/delete'; import { FindBackfillParams } from '../application/backfill/methods/find/types'; import { DisableRuleParams } from '../application/rule/methods/disable'; import { EnableRuleParams } from '../application/rule/methods/enable_rule'; +import { findGaps } from '../application/rule/methods/find_gaps'; + export type ConstructorOptions = Omit< RulesClientContext, @@ -207,4 +209,6 @@ export class RulesClient { public getTags = (params: RuleTagsParams) => getRuleTags(this.context, params); public getScheduleFrequency = () => getScheduleFrequency(this.context); + + public findGaps = (params: FindGapsParams) => findGaps(this.context, params); } diff --git a/x-pack/plugins/alerting/server/task_runner/ad_hoc_task_runner.ts b/x-pack/plugins/alerting/server/task_runner/ad_hoc_task_runner.ts index d126151030672..2fe019938fa80 100644 --- a/x-pack/plugins/alerting/server/task_runner/ad_hoc_task_runner.ts +++ b/x-pack/plugins/alerting/server/task_runner/ad_hoc_task_runner.ts @@ -52,6 +52,7 @@ import { import { RuleRunMetrics, RuleRunMetricsStore } from '../lib/rule_run_metrics_store'; import { getEsErrorMessage } from '../lib/errors'; import { Result, isOk, asOk, asErr } from '../lib/result_type'; +import { processGapsAfterExecution } from '../lib/rule_gaps/process_gaps_after_execution'; interface ConstructorParams { context: TaskRunnerContext; @@ -486,6 +487,21 @@ export class AdHocTaskRunner implements CancellableTask { ...(this.scheduleToRunIndex > -1 ? { schedule: this.adHocRunSchedule } : {}), }); + // TODO: calculate end time of the interval + const end = new Date(); + // TODO: check if we need to fill gaps + const needToFillGaps = true; + + processGapsAfterExecution({ + ruleId: this.ruleId, + executedInterval: { from: start, to: end }, + alertingEventLogger: this.alertingEventLogger, + eventLogClient: this.context.eventLogger, + savedObjectsClient: this.internalSavedObjectsRepository, + logger: this.logger, + needToFillGaps, + }); + if (startedAt) { // Capture how long it took for the rule to run after being claimed this.timer.setDuration(TaskRunnerTimerSpan.TotalRunDuration, startedAt); diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner.ts b/x-pack/plugins/alerting/server/task_runner/task_runner.ts index 5fa7f2b303951..d7b38bf5d06d9 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner.ts @@ -72,6 +72,7 @@ import { processRunResults, clearExpiredSnoozes, } from './lib'; +import { getTotalUnfilledGapDuration } from '../lib/rule_gaps/get_total_unfilled_gap_duration'; const FALLBACK_RETRY_INTERVAL = '5m'; const CONNECTIVITY_RETRY_INTERVAL = '5m'; @@ -611,7 +612,7 @@ export class TaskRunner< gap, }); this.ruleMonitoring.getLastRunMetricsSetters().setLastRunMetricsGapRange(null); - } + if (!this.cancelled) { this.inMemoryMetrics.increment(IN_MEMORY_METRICS.RULE_EXECUTIONS); @@ -625,11 +626,31 @@ export class TaskRunner< )} - ${JSON.stringify(lastRun)}` ); } + + const unfiledGapDuration = await getTotalUnfilledGapDuration({ + ruleId, + eventLog: this.context.eventLogger, + logger: this.logger, + }); + + const monitoring = { + ...this.ruleMonitoring.getMonitoring(), + run: { + ...this.ruleMonitoring.getMonitoring()?.run, + last_run: { + ...this.ruleMonitoring.getMonitoring()?.run?.last_run, + metrics: { + ...this.ruleMonitoring.getMonitoring()?.run?.last_run?.metrics, + unfilled_gap_duration_ms: unfiledGapDuration.unfiled_gap_duration_ms, + }, + }, + }, + }; await this.updateRuleSavedObjectPostRun(ruleId, { executionStatus: ruleExecutionStatusToRaw(executionStatus), nextRun, lastRun: lastRunToRaw(lastRun), - monitoring: this.ruleMonitoring.getMonitoring() as RawRuleMonitoring, + monitoring, }); } From ee4f9b06f46b2256992dc705767c2df921e5ba34 Mon Sep 17 00:00:00 2001 From: Nikita Khristinin Date: Thu, 5 Dec 2024 12:01:23 +0100 Subject: [PATCH 02/17] Update gaps POC --- .../alerting/common/constants/gap_status.ts | 14 + .../alerting/common/constants/index.ts | 1 + .../methods/schedule/schedule_backfill.ts | 3 + .../{fing_gaps => find_gaps}/find_gaps.ts | 11 +- .../rule/methods/find_gaps/index.ts | 1 + .../server/backfill_client/backfill_client.ts | 31 ++- .../alerting_event_logger.ts | 13 + .../server/lib/rule_gaps/find_gaps.ts | 94 +++++++ .../lib/rule_gaps/find_gaps_by_rule_id.ts | 26 -- .../get_total_unfilled_gap_duration.ts | 15 +- .../rule_gaps/process_gaps_after_execution.ts | 119 -------- .../server/lib/rule_gaps/schemas/index.ts | 62 +++++ .../rule_gaps/transforms/transformToGap.ts | 30 ++ .../server/lib/rule_gaps/types/index.ts | 12 + .../server/lib/rule_gaps/update_gaps.ts | 261 ++++++++++++++++++ x-pack/plugins/alerting/server/plugin.ts | 2 + .../server/task_runner/ad_hoc_task_runner.ts | 31 ++- .../server/task_runner/task_runner.ts | 7 +- .../alerting/server/task_runner/types.ts | 3 +- .../server/es/cluster_client_adapter.ts | 58 +++- .../event_log/server/event_log_client.ts | 44 ++- .../server/event_log_start_service.ts | 22 +- .../plugins/event_log/server/event_logger.ts | 24 ++ x-pack/plugins/event_log/server/types.ts | 3 +- 24 files changed, 688 insertions(+), 199 deletions(-) create mode 100644 x-pack/plugins/alerting/common/constants/gap_status.ts rename x-pack/plugins/alerting/server/application/rule/methods/{fing_gaps => find_gaps}/find_gaps.ts (76%) create mode 100644 x-pack/plugins/alerting/server/application/rule/methods/find_gaps/index.ts create mode 100644 x-pack/plugins/alerting/server/lib/rule_gaps/find_gaps.ts delete mode 100644 x-pack/plugins/alerting/server/lib/rule_gaps/find_gaps_by_rule_id.ts delete mode 100644 x-pack/plugins/alerting/server/lib/rule_gaps/process_gaps_after_execution.ts create mode 100644 x-pack/plugins/alerting/server/lib/rule_gaps/schemas/index.ts create mode 100644 x-pack/plugins/alerting/server/lib/rule_gaps/transforms/transformToGap.ts create mode 100644 x-pack/plugins/alerting/server/lib/rule_gaps/types/index.ts create mode 100644 x-pack/plugins/alerting/server/lib/rule_gaps/update_gaps.ts diff --git a/x-pack/plugins/alerting/common/constants/gap_status.ts b/x-pack/plugins/alerting/common/constants/gap_status.ts new file mode 100644 index 0000000000000..e4b1995312dc7 --- /dev/null +++ b/x-pack/plugins/alerting/common/constants/gap_status.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const gapStatus = { + UNFILLED: 'unfilled', + FILLED: 'filled', + PARTIALLY_FILLED: 'partially_filled', +} as const; + +export type GapStatus = (typeof gapStatus)[keyof typeof gapStatus]; diff --git a/x-pack/plugins/alerting/common/constants/index.ts b/x-pack/plugins/alerting/common/constants/index.ts index 0acc25785d194..2a0c6bf3dcd5d 100644 --- a/x-pack/plugins/alerting/common/constants/index.ts +++ b/x-pack/plugins/alerting/common/constants/index.ts @@ -13,3 +13,4 @@ export { MAX_SCHEDULE_BACKFILL_LOOKBACK_WINDOW_MS, } from './backfill'; export { PLUGIN } from './plugin'; +export { gapStatus } from './gap_status'; diff --git a/x-pack/plugins/alerting/server/application/backfill/methods/schedule/schedule_backfill.ts b/x-pack/plugins/alerting/server/application/backfill/methods/schedule/schedule_backfill.ts index 534262aa31c31..baa5c705712b3 100644 --- a/x-pack/plugins/alerting/server/application/backfill/methods/schedule/schedule_backfill.ts +++ b/x-pack/plugins/alerting/server/application/backfill/methods/schedule/schedule_backfill.ts @@ -164,5 +164,8 @@ export async function scheduleBackfill( ruleTypeRegistry: context.ruleTypeRegistry, spaceId: context.spaceId, unsecuredSavedObjectsClient: context.unsecuredSavedObjectsClient, + eventLogClient: await context.getEventLogClient(), + internalSavedObjectsRepository: context.internalSavedObjectsRepository, + eventLogger: context.eventLogger, }); } diff --git a/x-pack/plugins/alerting/server/application/rule/methods/fing_gaps/find_gaps.ts b/x-pack/plugins/alerting/server/application/rule/methods/find_gaps/find_gaps.ts similarity index 76% rename from x-pack/plugins/alerting/server/application/rule/methods/fing_gaps/find_gaps.ts rename to x-pack/plugins/alerting/server/application/rule/methods/find_gaps/find_gaps.ts index 891218692cc83..d9aac3b8c975a 100644 --- a/x-pack/plugins/alerting/server/application/rule/methods/fing_gaps/find_gaps.ts +++ b/x-pack/plugins/alerting/server/application/rule/methods/find_gaps/find_gaps.ts @@ -9,13 +9,18 @@ import Boom from '@hapi/boom'; import { RulesClientContext } from '../../../../rules_client'; -import { findGapsByRuleId } from '../../../../lib/rule_gaps/find_gaps_by_rule_id'; +import { findGapsByRuleId } from '../../../../lib/rule_gaps/find_gaps'; -export async function findBackfill(context: RulesClientContext, params: FindGapsParams) { +interface FindGapsParams { + ruleId: string; +} + +export async function findGaps(context: RulesClientContext, params: FindGapsParams) { try { + const eventLogClient = await context.getEventLogClient(); const gaps = await findGapsByRuleId({ ruleId: params.ruleId, - eventLog: context.eventLogger, + eventLogClient, logger: context.logger, }); diff --git a/x-pack/plugins/alerting/server/application/rule/methods/find_gaps/index.ts b/x-pack/plugins/alerting/server/application/rule/methods/find_gaps/index.ts new file mode 100644 index 0000000000000..38a6e16dce35e --- /dev/null +++ b/x-pack/plugins/alerting/server/application/rule/methods/find_gaps/index.ts @@ -0,0 +1 @@ +export { findGaps } from './find_gaps'; \ No newline at end of file diff --git a/x-pack/plugins/alerting/server/backfill_client/backfill_client.ts b/x-pack/plugins/alerting/server/backfill_client/backfill_client.ts index 48b5e49c428c0..b7e4e6667ffc7 100644 --- a/x-pack/plugins/alerting/server/backfill_client/backfill_client.ts +++ b/x-pack/plugins/alerting/server/backfill_client/backfill_client.ts @@ -42,6 +42,7 @@ import { AD_HOC_RUN_SAVED_OBJECT_TYPE, RULE_SAVED_OBJECT_TYPE } from '../saved_o import { TaskRunnerFactory } from '../task_runner'; import { RuleTypeRegistry } from '../types'; import { createBackfillError } from './lib'; +import { updateGaps } from '../lib/rule_gaps/update_gaps'; export const BACKFILL_TASK_TYPE = 'ad_hoc_run-backfill'; @@ -92,6 +93,9 @@ export class BackfillClient { ruleTypeRegistry, spaceId, unsecuredSavedObjectsClient, + eventLogClient, + internalSavedObjectsRepository, + eventLogger, }: BulkQueueOpts): Promise { const adHocSOsToCreate: Array> = []; @@ -211,10 +215,11 @@ export class BackfillClient { // Build array of tasks to schedule const adHocTasksToSchedule: TaskInstance[] = []; + const backfullsToSchedule: Backfill[] = []; createSOResult.forEach((result: ScheduleBackfillResult) => { if (!(result as ScheduleBackfillError).error) { const createdSO = result as Backfill; - + backfullsToSchedule.push(createdSO); const ruleTypeTimeout = ruleTypeRegistry.get(createdSO.rule.alertTypeId).ruleTaskTimeout; adHocTasksToSchedule.push({ id: createdSO.id, @@ -234,6 +239,30 @@ export class BackfillClient { await taskManager.bulkSchedule(adHocTasksToSchedule); } + try { + // TODO: make it parallel + + backfullsToSchedule.forEach((backfill) => { + console.log('backfill client update', JSON.stringify(backfill)); + updateGaps({ + ruleId: backfill.rule.id, + start: new Date(backfill.start), + end: backfill?.end ? new Date(backfill.end) : new Date(), + eventLogger, + eventLogClient, + savedObjectsClient: internalSavedObjectsRepository, + logger: this.logger, + needToFillGaps: false, + }); + }); + } catch { + this.logger.warn( + `Error updating gaps ƒor backfill jobs: ${backfullsToSchedule + .map((backfill) => backfill.id) + .join(', ')}` + ); + } + return createSOResult; } diff --git a/x-pack/plugins/alerting/server/lib/alerting_event_logger/alerting_event_logger.ts b/x-pack/plugins/alerting/server/lib/alerting_event_logger/alerting_event_logger.ts index 72c239fff9e0a..3d73c2bfc83c1 100644 --- a/x-pack/plugins/alerting/server/lib/alerting_event_logger/alerting_event_logger.ts +++ b/x-pack/plugins/alerting/server/lib/alerting_event_logger/alerting_event_logger.ts @@ -416,6 +416,19 @@ export class AlertingEventLogger { }) ); } + + public updateGap({ _id, gap }: { _id: string; gap: {} }): void { + // TODO: Extract function to form gap document + this.eventLogger.updateEvent(_id, { + kibana: { + alert: { + rule: { + gap, + }, + }, + }, + }); + } } export function createAlertRecord( diff --git a/x-pack/plugins/alerting/server/lib/rule_gaps/find_gaps.ts b/x-pack/plugins/alerting/server/lib/rule_gaps/find_gaps.ts new file mode 100644 index 0000000000000..8b56c18ab822c --- /dev/null +++ b/x-pack/plugins/alerting/server/lib/rule_gaps/find_gaps.ts @@ -0,0 +1,94 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IEventLogClient } from '@kbn/event-log-plugin/server'; +import { Logger } from '@kbn/core/server'; +import { RULE_SAVED_OBJECT_TYPE } from '../../saved_objects'; +import { Gap, FindGapsParams } from './types'; +import { transformToGap } from './transforms/transformToGap'; + +export const findGaps = async ({ + eventLogClient, + logger, + params, +}: { + eventLogClient: IEventLogClient; + logger: Logger; + params: FindGapsParams; +}): Promise<{ + total: number; + data: Gap[]; + page: number; + perPage: number; +}> => { + const { ruleIds, start, end, page, perPage } = params; + try { + const gapsResponse = await eventLogClient.findEventsBySavedObjectIds( + RULE_SAVED_OBJECT_TYPE, + ruleIds, + { + filter: `event.action: gap AND event.provider: alerting and (kibana.alert.rule.gap.range <= "${end}" AND kibana.alert.rule.gap.range >= "${start}")`, + sort: [{ sort_field: '@timestamp', sort_order: 'desc' }], + page, + per_page: perPage, + } + ); + + return { + total: gapsResponse.total, + data: transformToGap(gapsResponse), + page: gapsResponse.page, + perPage: gapsResponse.per_page, + }; + } catch (err) { + logger.error(`Failed to find gaps for rule ${ruleIds.toString()}: ${err.message}`); + throw err; + } +}; + +export const findAllGaps = async ({ + eventLogClient, + logger, + params, +}: { + eventLogClient: IEventLogClient; + logger: Logger; + params: { + ruleIds: string[]; + start: Date; + end: Date; + }; +}): Promise => { + const { ruleIds, start, end } = params; + const allGaps: Gap[] = []; + let currentPage = 1; + const perPage = 10000; + + while (true) { + const { data } = await findGaps({ + eventLogClient, + logger, + params: { + ruleIds, + start: start.toISOString(), + end: end.toISOString(), + page: currentPage, + perPage, + }, + }); + + allGaps.push(...data); + + if (data.length === 0 || data.length < perPage) { + break; + } + + currentPage++; + } + + return allGaps; +}; diff --git a/x-pack/plugins/alerting/server/lib/rule_gaps/find_gaps_by_rule_id.ts b/x-pack/plugins/alerting/server/lib/rule_gaps/find_gaps_by_rule_id.ts deleted file mode 100644 index 4fe17708ca2d5..0000000000000 --- a/x-pack/plugins/alerting/server/lib/rule_gaps/find_gaps_by_rule_id.ts +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { IEventLogger } from '@kbn/event-log-plugin/server'; -import { Logger } from '@kbn/core/server'; -export async function findGapsByRuleId(params: { - ruleId: string; - timeRange?: { from: string; to: string }; - eventLog: IEventLogger; - logger: Logger; -}): Promise<[]> { - const { ruleId, timeRange, eventLog, logger } = params; - - try { - // return await eventLogeventLog.findEvents({ - // }); - return []; - } catch (err) { - logger.error(`Failed to find gaps for rule ${ruleId}: ${err.message}`); - throw err; - } -} diff --git a/x-pack/plugins/alerting/server/lib/rule_gaps/get_total_unfilled_gap_duration.ts b/x-pack/plugins/alerting/server/lib/rule_gaps/get_total_unfilled_gap_duration.ts index 5d4eaeaba4c78..ce2950c41072c 100644 --- a/x-pack/plugins/alerting/server/lib/rule_gaps/get_total_unfilled_gap_duration.ts +++ b/x-pack/plugins/alerting/server/lib/rule_gaps/get_total_unfilled_gap_duration.ts @@ -15,21 +15,20 @@ export async function getTotalUnfilledGapDuration(params: { logger: Logger; }): Promise<{ unfiled_gap_duration_ms: { - "1d": number; - "3d": number; - "7d": number; + '1d': number; + '3d': number; + '7d': number; }; }> { const { ruleId, timeRange, eventLog, logger } = params; try { - - const aggs = // eventLog... + const aggs = {}; // eventLog... return { unfiled_gap_duration_ms: { - "1d": 0, - "3d": 0, - "7d": 0, + '1d': 0, + '3d': 0, + '7d': 0, }, }; } catch (err) { diff --git a/x-pack/plugins/alerting/server/lib/rule_gaps/process_gaps_after_execution.ts b/x-pack/plugins/alerting/server/lib/rule_gaps/process_gaps_after_execution.ts deleted file mode 100644 index 12e6ef121c817..0000000000000 --- a/x-pack/plugins/alerting/server/lib/rule_gaps/process_gaps_after_execution.ts +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ISavedObjectsRepository, Logger } from '@kbn/core/server'; -import { IEventLogger } from '@kbn/event-log-plugin/server'; -import { AD_HOC_RUN_SAVED_OBJECT_TYPE, RULE_SAVED_OBJECT_TYPE } from '../../saved_objects'; -import { AlertingEventLogger } from '../alerting_event_logger/alerting_event_logger'; -import { findGapsByRuleId } from './find_gaps_by_rule_id'; -import { RuleGap } from './types'; -import { AdHocRunSO } from '../../data/ad_hoc_run/types'; - -/** - * Fill the gap with the interval - */ -const fillGap = async ({ - gap, -}: { - gap: RuleGap; -}) => { - - gap.status = // .. - gap.filled_intervals = // ... - gap.in_progress_intervals = // ... - gap.unfilled_intervals = // ... - - return gap; -}; - - - -/** - * Find all overlapping backfill tasks and update the gap status accordingly - */ -const updateGapStatus = async ({ - gap, - savedObjectsClient, - ruleId, -}: { - gap: RuleGap; - savedObjectsClient: ISavedObjectsRepository; - ruleId: string; -}) => { - - - const backfillResults = await savedObjectsClient.find({ - type: AD_HOC_RUN_SAVED_OBJECT_TYPE, - hasReference: { - type: RULE_SAVED_OBJECT_TYPE, - id: ruleId, - }, - // Filter for backfills that overlap with our interval - filter: ` - ad_hoc_run_params.attributes.start <= "${gap.range.lte}" and - ad_hoc_run_params.attributes.end >= "${gap.range.gte}" - `, - page: 1, - perPage: 100, - }); - - const hasPendingOverlapingTasks = // backfillResults.......; - - if (hasPendingOverlapingTasks && gap.status !== "FILLED") { - gap.status = "IN_PROGRESS"; - gap.in_progress_intervals = //..[] - gap.unfilled_intervals = //... [] - } - - - return gap; - -} -export async function processGapsAfterExecution(params: { - ruleId: string; - executedInterval: { from: Date; to: Date }; - alertingEventLogger: AlertingEventLogger; - eventLog: IEventLogger; - savedObjectsClient: ISavedObjectsRepository; - logger: Logger; - needToFillGaps: boolean; -}): Promise { - const { - ruleId, - executedInterval, - alertingEventLogger, - logger, - savedObjectsClient, - eventLog, - needToFillGaps, - } = params; - - try { - const allGaps = await findGapsByRuleId({ - ruleId, - timeRange: executedInterval, - eventLog, - logger, - }); - - for (const gap of allGaps) { - let newGap = needToFillGaps ? fillGap(gap) : gap; - - const updatedGap = await updateGapStatus({ - gap: newGap, - savedObjectsClient, - ruleId, - }); - - await alertingEventLogger.updateGap(updatedGap); - } - - } catch (err) { - logger.error(`Failed to process gaps for rule ${ruleId}: ${err.message}`); - throw err; - } -} diff --git a/x-pack/plugins/alerting/server/lib/rule_gaps/schemas/index.ts b/x-pack/plugins/alerting/server/lib/rule_gaps/schemas/index.ts new file mode 100644 index 0000000000000..0a6b23f769fb5 --- /dev/null +++ b/x-pack/plugins/alerting/server/lib/rule_gaps/schemas/index.ts @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { schema } from '@kbn/config-schema'; +import { gapStatus } from '../../../../common/constants'; + +export const gapStatusSchema = schema.oneOf([ + schema.literal(gapStatus.UNFILLED), + schema.literal(gapStatus.FILLED), + schema.literal(gapStatus.PARTIALLY_FILLED), +]); + +const rangeSchema = schema.object({ + lte: schema.string(), + gte: schema.string(), +}); + +export const gapSchema = schema.object({ + _id: schema.string(), + eventId: schema.string(), + status: gapStatusSchema, + range: rangeSchema, + inProgressIntervals: schema.arrayOf(rangeSchema), + filledIntervals: schema.arrayOf(rangeSchema), + unfilledIntervals: schema.arrayOf(rangeSchema), + totalGapDurationMs: schema.number(), + filledDurationMs: schema.number(), + unfilledDurationMs: schema.number(), + inProgressDurationMs: schema.number(), +}); + +export const findGapsParamsSchema = schema.object( + { + end: schema.maybe(schema.string()), + page: schema.number({ defaultValue: 1, min: 1 }), + perPage: schema.number({ defaultValue: 10, min: 0 }), + ruleIds: schema.arrayOf(schema.string()), + start: schema.maybe(schema.string()), + sortField: schema.maybe(schema.oneOf([schema.literal('@timestamp')])), + sortOrder: schema.maybe(schema.oneOf([schema.literal('asc'), schema.literal('desc')])), + }, + { + validate({ start, end }) { + if (start) { + const parsedStart = Date.parse(start); + if (isNaN(parsedStart)) { + return `[start]: query start must be valid date`; + } + } + if (end) { + const parsedEnd = Date.parse(end); + if (isNaN(parsedEnd)) { + return `[end]: query end must be valid date`; + } + } + }, + } +); diff --git a/x-pack/plugins/alerting/server/lib/rule_gaps/transforms/transformToGap.ts b/x-pack/plugins/alerting/server/lib/rule_gaps/transforms/transformToGap.ts new file mode 100644 index 0000000000000..b5537e81a120e --- /dev/null +++ b/x-pack/plugins/alerting/server/lib/rule_gaps/transforms/transformToGap.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { QueryEventsBySavedObjectResult } from '@kbn/event-log-plugin/server'; +import { Gap } from '../types'; + +export const transformToGap = (events: QueryEventsBySavedObjectResult): Gap[] => { + return events?.data + ?.map((doc) => { + const gap = doc?.kibana?.alert?.rule?.gap; + if (!gap) return null; + return { + // eventId: doc.event.id, + status: gap.status, + range: gap.range, + inProgressIntervals: gap.in_progress_intervals ?? [], + filledIntervals: gap.filled_intervals ?? [], + unfilledIntervals: gap.unfilled_intervals ?? [], + totalGapDurationMs: Number(gap.total_gap_duration_ms), + filledDurationMs: Number(gap.filled_duration_ms), + unfilledDurationMs: Number(gap.unfilled_duration_ms), + inProgressDurationMs: Number(gap.in_progress_duration_ms), + _id: doc._id, + }; + }) + .filter((gap): gap is Gap => gap !== null); +}; diff --git a/x-pack/plugins/alerting/server/lib/rule_gaps/types/index.ts b/x-pack/plugins/alerting/server/lib/rule_gaps/types/index.ts new file mode 100644 index 0000000000000..1eec48080ec83 --- /dev/null +++ b/x-pack/plugins/alerting/server/lib/rule_gaps/types/index.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { TypeOf } from '@kbn/config-schema'; +import { gapSchema, findGapsParamsSchema } from '../schemas'; + +export type Gap = TypeOf; +export type FindGapsParams = TypeOf; diff --git a/x-pack/plugins/alerting/server/lib/rule_gaps/update_gaps.ts b/x-pack/plugins/alerting/server/lib/rule_gaps/update_gaps.ts new file mode 100644 index 0000000000000..4d3b3e9bf9dc9 --- /dev/null +++ b/x-pack/plugins/alerting/server/lib/rule_gaps/update_gaps.ts @@ -0,0 +1,261 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ISavedObjectsRepository, Logger } from '@kbn/core/server'; +import { IEventLogClient, IEventLogger } from '@kbn/event-log-plugin/server'; +import { AD_HOC_RUN_SAVED_OBJECT_TYPE, RULE_SAVED_OBJECT_TYPE } from '../../saved_objects'; +import { AlertingEventLogger } from '../alerting_event_logger/alerting_event_logger'; +import { findAllGaps } from './find_gaps'; +import { Gap } from './types'; +import { AdHocRunSO } from '../../data/ad_hoc_run/types'; +import { parseDuration } from '../../../common'; +import { transformAdHocRunToBackfillResult } from '../../application/backfill/transforms'; + +interface Interval { + lte: string; + gte: string; +} + +const overlapIntervals = (interval1: Interval, interval2: Interval): Interval | null => { + const start1 = new Date(interval1.gte).getTime(); + const end1 = new Date(interval1.lte).getTime(); + const start2 = new Date(interval2.gte).getTime(); + const end2 = new Date(interval2.lte).getTime(); + const maxStart = Math.max(start1, start2); + const minEnd = Math.min(end1, end2); + if (maxStart < minEnd) { + return { + gte: new Date(maxStart).toISOString(), + lte: new Date(minEnd).toISOString(), + }; + } else { + return null; + } +}; + +/** + * Merges overlapping intervals in a list. + */ +const mergeIntervals = (intervals: Interval[]): Interval[] => { + if (!intervals.length) return []; + intervals.sort((a, b) => new Date(a.gte).getTime() - new Date(b.gte).getTime()); + const merged: Interval[] = [intervals[0]]; + for (const current of intervals.slice(1)) { + const last = merged[merged.length - 1]; + if (new Date(last.lte) >= new Date(current.gte)) { + last.lte = new Date( + Math.max(new Date(last.lte).getTime(), new Date(current.lte).getTime()) + ).toISOString(); + } else { + merged.push(current); + } + } + return merged; +}; + +/** + * Subtracts a list of intervals from a base interval. + */ +const subtractIntervals = (interval: Interval, intervalsToSubtract: Interval[]): Interval[] => { + let intervals: Interval[] = [interval]; + for (const subtractInterval of intervalsToSubtract) { + intervals = intervals.flatMap((current) => { + const overlap = overlapIntervals(current, subtractInterval); + if (!overlap) return [current]; + const results: Interval[] = []; + if (new Date(current.gte) < new Date(overlap.gte)) { + results.push({ gte: current.gte, lte: overlap.gte }); + } + if (new Date(current.lte) > new Date(overlap.lte)) { + results.push({ gte: overlap.lte, lte: current.lte }); + } + return results; + }); + } + return intervals; +}; + +/** + * Subtracts a list of intervals from another list of intervals. + */ +const subtractIntervalsFromIntervals = ( + intervals: Interval[], + intervalsToSubtract: Interval[] +): Interval[] => { + const result: Interval[] = []; + for (const interval of intervals) { + const subtracted = subtractIntervals(interval, intervalsToSubtract); + result.push(...subtracted); + } + return mergeIntervals(result); +}; + +/** + * Fill the gap with the interval + */ +const fillGap = (gap: Gap, interval: { from: Date; to: Date }): Gap => { + const newGap = { + ...gap, + }; + newGap.status = 'filled'; // .. + newGap.filledIntervals = []; // ... + newGap.inProgressIntervals = []; // ... + newGap.unfilledIntervals = []; // ... + + return newGap; +}; + +/** + * Find all overlapping backfill tasks and update the gap status accordingly + */ +const updateGapStatus = async ({ + gap, + savedObjectsClient, + ruleId, +}: { + gap: Gap; + savedObjectsClient: ISavedObjectsRepository; + ruleId: string; +}): Promise => { + // TODO: get all backfill, not first page + const { saved_objects: backfillSOs } = await savedObjectsClient.find({ + type: AD_HOC_RUN_SAVED_OBJECT_TYPE, + hasReference: { + type: RULE_SAVED_OBJECT_TYPE, + id: ruleId, + }, + // Filter for backfills that overlap with our interval + filter: ` + ad_hoc_run_params.attributes.start <= "${gap.range.lte}" and + ad_hoc_run_params.attributes.end >= "${gap.range.gte}" + `, + page: 1, + perPage: 100, + }); + + // TODO: Extract backfill transform to another function, to reuse in API + const transformedBackfills = backfillSOs.map((data) => transformAdHocRunToBackfillResult(data)); + + console.log('backfillResults', JSON.stringify(backfillSOs, null, 2)); + console.log('transformedBackfills', JSON.stringify(transformedBackfills, null, 2)); + + const potenialInProgressIntervals = []; + + // Process each backfill's schedule + for (const backfill of transformedBackfills) { + if ('error' in backfill) { + break; + } + const backfillScheduleIntervals = backfill?.schedule ?? []; + for (const scheduleItem of backfillScheduleIntervals) { + const runAt = new Date(scheduleItem.runAt).getTime(); + const intervalDuration = parseDuration(scheduleItem.interval); + const from = runAt - intervalDuration; + const to = runAt; + const scheduleInterval = { + gte: new Date(from).toISOString(), + lte: new Date(to).toISOString(), + }; + // Check overlap with gap.range + const overlap = overlapIntervals(scheduleInterval, gap.range); + if (overlap) { + potenialInProgressIntervals.push(overlap); + } + } + } + + // Merge overlapping intervals in potentialInProgress + const mergedPotentialInProgress = mergeIntervals(potenialInProgressIntervals); + + const inProgressIntervals = subtractIntervalsFromIntervals( + mergedPotentialInProgress, + gap.filledIntervals + ); + + // unfilledIntervals = gap.range excluding filledIntervals and inProgressIntervals + const unfilledIntervals = subtractIntervals( + gap.range, + mergeIntervals([...gap.filledIntervals, ...inProgressIntervals]) + ); + + const newGap = { ...gap, inProgressIntervals, unfilledIntervals }; + + console.log('newGap', JSON.stringify(newGap)); + + const hasPendingOverlapingTasks = true; // backfillResults.......; + + if (hasPendingOverlapingTasks && newGap.status !== 'filled') { + newGap.status = 'filled'; + newGap.inProgressIntervals = []; // ..[] + } + + return newGap; +}; + +export async function updateGaps(params: { + ruleId: string; + start: Date; + end: Date; + eventLogger: IEventLogger; + eventLogClient: IEventLogClient; + savedObjectsClient: ISavedObjectsRepository; + logger: Logger; + needToFillGaps: boolean; +}): Promise { + const { + ruleId, + start, + end, + logger, + savedObjectsClient, + eventLogClient, + needToFillGaps, + eventLogger, + } = params; + + const alertingEventLogger = new AlertingEventLogger(eventLogger); + + try { + const allGaps = await findAllGaps({ + eventLogClient, + logger, + params: { + ruleIds: [ruleId], + start, + end, + }, + }); + + console.log('allGaps', JSON.stringify(allGaps, null, 2)); + + for (const gap of allGaps) { + const newGap = needToFillGaps + ? fillGap(gap, { + from: start, + to: end, + }) + : gap; + + const updatedGap = await updateGapStatus({ + gap: newGap, + savedObjectsClient, + ruleId, + }); + + const { _id, ...gapBody } = updatedGap; + console.log('updatedGap', JSON.stringify(updatedGap)); + + await alertingEventLogger.updateGap({ + _id, + gap: { ...gapBody, unfilled_intervals: gapBody.unfilledIntervals }, + }); + } + } catch (err) { + logger.error(`Failed to process gaps for rule ${ruleId}: ${err.message}`); + throw err; + } +} diff --git a/x-pack/plugins/alerting/server/plugin.ts b/x-pack/plugins/alerting/server/plugin.ts index fa562cfa10d91..f198f2159212b 100644 --- a/x-pack/plugins/alerting/server/plugin.ts +++ b/x-pack/plugins/alerting/server/plugin.ts @@ -537,6 +537,7 @@ export class AlertingPlugin { securityPluginStart: plugins.security, internalSavedObjectsRepository: core.savedObjects.createInternalRepository([ RULE_SAVED_OBJECT_TYPE, + AD_HOC_RUN_SAVED_OBJECT_TYPE, ]), encryptedSavedObjectsClient, spaceIdToNamespace, @@ -629,6 +630,7 @@ export class AlertingPlugin { supportsEphemeralTasks: plugins.taskManager.supportsEphemeralTasks(), uiSettings: core.uiSettings, usageCounter: this.usageCounter, + getEventLogClient: (spaceId: string) => plugins.eventLog.getClient(spaceId), }); this.eventLogService!.registerSavedObjectProvider(RULE_SAVED_OBJECT_TYPE, (request) => { diff --git a/x-pack/plugins/alerting/server/task_runner/ad_hoc_task_runner.ts b/x-pack/plugins/alerting/server/task_runner/ad_hoc_task_runner.ts index 2fe019938fa80..b99bd42885357 100644 --- a/x-pack/plugins/alerting/server/task_runner/ad_hoc_task_runner.ts +++ b/x-pack/plugins/alerting/server/task_runner/ad_hoc_task_runner.ts @@ -24,7 +24,7 @@ import { CancellableTask, RunResult } from '@kbn/task-manager-plugin/server/task import { AdHocRunStatus, adHocRunStatus } from '../../common/constants'; import { RuleRunnerErrorStackTraceLog, RuleTaskStateAndMetrics, TaskRunnerContext } from './types'; import { getExecutorServices } from './get_executor_services'; -import { ErrorWithReason, validateRuleTypeParams } from '../lib'; +import { ErrorWithReason, parseDuration, validateRuleTypeParams } from '../lib'; import { AlertInstanceContext, AlertInstanceState, @@ -52,7 +52,7 @@ import { import { RuleRunMetrics, RuleRunMetricsStore } from '../lib/rule_run_metrics_store'; import { getEsErrorMessage } from '../lib/errors'; import { Result, isOk, asOk, asErr } from '../lib/result_type'; -import { processGapsAfterExecution } from '../lib/rule_gaps/process_gaps_after_execution'; +import { updateGaps } from '../lib/rule_gaps/update_gaps'; interface ConstructorParams { context: TaskRunnerContext; @@ -487,22 +487,27 @@ export class AdHocTaskRunner implements CancellableTask { ...(this.scheduleToRunIndex > -1 ? { schedule: this.adHocRunSchedule } : {}), }); - // TODO: calculate end time of the interval - const end = new Date(); // TODO: check if we need to fill gaps const needToFillGaps = true; - processGapsAfterExecution({ - ruleId: this.ruleId, - executedInterval: { from: start, to: end }, - alertingEventLogger: this.alertingEventLogger, - eventLogClient: this.context.eventLogger, - savedObjectsClient: this.internalSavedObjectsRepository, - logger: this.logger, - needToFillGaps, - }); + const eventLogClient = await this.context.getEventLogClient(spaceId); if (startedAt) { + const intervalInMs = parseDuration( + this.adHocRunSchedule[this.scheduleToRunIndex].interval + ); + const endGapsRange = new Date(this.adHocRunSchedule[this.scheduleToRunIndex].runAt); + const startGapsRange = new Date(endGapsRange.getTime() - intervalInMs); + await updateGaps({ + ruleId: this.ruleId, + start: startGapsRange, + end: endGapsRange, + eventLogger: this.context.eventLogger, + eventLogClient, + savedObjectsClient: this.internalSavedObjectsRepository, + logger: this.logger, + needToFillGaps, + }); // Capture how long it took for the rule to run after being claimed this.timer.setDuration(TaskRunnerTimerSpan.TotalRunDuration, startedAt); } diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner.ts b/x-pack/plugins/alerting/server/task_runner/task_runner.ts index d7b38bf5d06d9..25609d8af887d 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner.ts @@ -612,7 +612,7 @@ export class TaskRunner< gap, }); this.ruleMonitoring.getLastRunMetricsSetters().setLastRunMetricsGapRange(null); - + } if (!this.cancelled) { this.inMemoryMetrics.increment(IN_MEMORY_METRICS.RULE_EXECUTIONS); @@ -627,13 +627,14 @@ export class TaskRunner< ); } + // TODO: it's not working yet const unfiledGapDuration = await getTotalUnfilledGapDuration({ ruleId, eventLog: this.context.eventLogger, logger: this.logger, }); - const monitoring = { + const monitoring = { ...this.ruleMonitoring.getMonitoring(), run: { ...this.ruleMonitoring.getMonitoring()?.run, @@ -641,7 +642,7 @@ export class TaskRunner< ...this.ruleMonitoring.getMonitoring()?.run?.last_run, metrics: { ...this.ruleMonitoring.getMonitoring()?.run?.last_run?.metrics, - unfilled_gap_duration_ms: unfiledGapDuration.unfiled_gap_duration_ms, + // unfilled_gap_duration_ms: unfiledGapDuration.unfiled_gap_duration_ms, }, }, }, diff --git a/x-pack/plugins/alerting/server/task_runner/types.ts b/x-pack/plugins/alerting/server/task_runner/types.ts index a92ee2a89d654..4aaa2ae7591fe 100644 --- a/x-pack/plugins/alerting/server/task_runner/types.ts +++ b/x-pack/plugins/alerting/server/task_runner/types.ts @@ -21,7 +21,7 @@ import { ActionsClient } from '@kbn/actions-plugin/server/actions_client'; import { PluginStart as DataPluginStart } from '@kbn/data-plugin/server'; import { PluginStart as DataViewsPluginStart } from '@kbn/data-views-plugin/server'; import { EncryptedSavedObjectsClient } from '@kbn/encrypted-saved-objects-plugin/server'; -import { IEventLogger } from '@kbn/event-log-plugin/server'; +import { IEventLogClient, IEventLogger } from '@kbn/event-log-plugin/server'; import { SharePluginStart } from '@kbn/share-plugin/server'; import { UsageCounter } from '@kbn/usage-collection-plugin/server'; import { IAlertsClient } from '../alerts_client/types'; @@ -180,4 +180,5 @@ export interface TaskRunnerContext { supportsEphemeralTasks: boolean; uiSettings: UiSettingsServiceStart; usageCounter?: UsageCounter; + getEventLogClient: (spaceId: string) => IEventLogClient; } diff --git a/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts b/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts index 7076336c0c760..52ee5f9071e56 100644 --- a/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts +++ b/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts @@ -26,6 +26,7 @@ export type IClusterClientAdapter = PublicMethodsOf; export interface Doc { index: string; body: IEvent; + id?: string; } type Wait = () => Promise; @@ -98,7 +99,9 @@ type AliasAny = any; const LEGACY_ID_CUTOFF_VERSION = '8.0.0'; -export class ClusterClientAdapter { +export class ClusterClientAdapter< + TDoc extends { body: AliasAny; index: string; id?: string } = Doc +> { private readonly logger: Logger; private readonly elasticsearchClientPromise: Promise; private readonly docBuffer$: Subject; @@ -136,6 +139,10 @@ export class ClusterClientAdapter) { + this.docBuffer$.next(doc); + } + public indexDocument(doc: TDoc): void { this.docBuffer$.next(doc); } @@ -153,17 +160,50 @@ export class ClusterClientAdapter> = []; - for (const doc of docs) { + const docsToUpdate = docs.filter((doc) => doc.id && doc.body); + const docsToCreate = docs.filter((doc) => !doc.id); + + for (const doc of docsToCreate) { if (doc.body === undefined) continue; - bulkBody.push({ create: {} }); + bulkBody.push({ create: { _index: this.esNames.dataStream } }); bulkBody.push(doc.body); } try { const esClient = await this.elasticsearchClientPromise; - const response = await esClient.bulk({ + const originalDocsBeforeUpdate = await esClient.search({ index: this.esNames.dataStream, + seq_no_primary_term: true, + query: { + terms: { + _id: docsToUpdate.map((doc) => doc.id), + }, + }, + }); + + for (const originalDoc of originalDocsBeforeUpdate?.hits?.hits ?? []) { + const fieldsToUpdate = docsToUpdate.find( + (docToUpdate) => docToUpdate.id === originalDoc._id + ); + if (!fieldsToUpdate || !originalDoc._source) continue; + + bulkBody.push({ + index: { + _index: originalDoc._index, + _id: originalDoc._id, + if_seq_no: originalDoc._seq_no, + if_primary_term: originalDoc._primary_term, + }, + }); + bulkBody.push({ + ...originalDoc._source, + ...fieldsToUpdate.body, + }); + } + + const response = await esClient.bulk({ + // index: this.esNames.dataStream, body: bulkBody, }); @@ -421,7 +461,10 @@ export class ClusterClientAdapter hit._source), + data: hits.map((hit) => ({ + ...hit._source, + _id: hit._id, + })), }; } catch (err) { throw new Error( @@ -465,7 +508,10 @@ export class ClusterClientAdapter hit._source), + data: hits.map((hit) => ({ + ...hit._source, + _id: hit._id, + })), }; } catch (err) { throw new Error( diff --git a/x-pack/plugins/event_log/server/event_log_client.ts b/x-pack/plugins/event_log/server/event_log_client.ts index 4ffcfb6cef84e..86a27a0adfbcc 100644 --- a/x-pack/plugins/event_log/server/event_log_client.ts +++ b/x-pack/plugins/event_log/server/event_log_client.ts @@ -70,25 +70,42 @@ export type AggregateOptionsType = Pick, 'filt aggs: Record; }; -interface EventLogServiceCtorParams { +interface EventLogServiceCtorBaseParams { esContext: EsContext; - savedObjectGetter: SavedObjectBulkGetterResult; spacesService?: SpacesServiceStart; - request: KibanaRequest; } +type EventLogServiceCtorParams = + | (EventLogServiceCtorBaseParams & { + request: KibanaRequest; + savedObjectGetter: SavedObjectBulkGetterResult; + spaceId?: string; + }) + | (EventLogServiceCtorBaseParams & { + spaceId: string; + request?: KibanaRequest; + savedObjectGetter?: SavedObjectBulkGetterResult; + }); // note that clusterClient may be null, indicating we can't write to ES export class EventLogClient implements IEventLogClient { private esContext: EsContext; - private savedObjectGetter: SavedObjectBulkGetterResult; + private savedObjectGetter?: SavedObjectBulkGetterResult; private spacesService?: SpacesServiceStart; - private request: KibanaRequest; - - constructor({ esContext, savedObjectGetter, spacesService, request }: EventLogServiceCtorParams) { + private request?: KibanaRequest; + private spaceId?: string; + + constructor({ + esContext, + savedObjectGetter, + spacesService, + request, + spaceId, + }: EventLogServiceCtorParams) { this.esContext = esContext; this.savedObjectGetter = savedObjectGetter; this.spacesService = spacesService; this.request = request; + this.spaceId = spaceId; } public async findEventsBySavedObjectIds( @@ -100,7 +117,7 @@ export class EventLogClient implements IEventLogClient { const findOptions = queryOptionsSchema.validate(options ?? {}); // verify the user has the required permissions to view this saved object - await this.savedObjectGetter(type, ids); + await this.savedObjectGetter?.(type, ids); return await this.esContext.esAdapter.queryEventsBySavedObjects({ index: this.esContext.esNames.indexPattern, @@ -152,7 +169,7 @@ export class EventLogClient implements IEventLogClient { const aggregateOptions = queryOptionsSchema.validate(omit(options, 'aggs') ?? {}); // verify the user has the required permissions to view this saved object - await this.savedObjectGetter(type, ids); + await this.savedObjectGetter?.(type, ids); return await this.esContext.esAdapter.aggregateEventsBySavedObjects({ index: this.esContext.esNames.indexPattern, @@ -194,7 +211,12 @@ export class EventLogClient implements IEventLogClient { } private async getNamespace() { - const space = await this.spacesService?.getActiveSpace(this.request); - return space && this.spacesService?.spaceIdToNamespace(space.id); + if (this.spaceId) { + return this.spacesService?.spaceIdToNamespace(this.spaceId); + } else if (this.request) { + const space = await this.spacesService?.getActiveSpace(this.request); + return space && this.spacesService?.spaceIdToNamespace(space.id); + } + return undefined; } } diff --git a/x-pack/plugins/event_log/server/event_log_start_service.ts b/x-pack/plugins/event_log/server/event_log_start_service.ts index 94962c59f4dfc..2757460f79d4d 100644 --- a/x-pack/plugins/event_log/server/event_log_start_service.ts +++ b/x-pack/plugins/event_log/server/event_log_start_service.ts @@ -38,12 +38,20 @@ export class EventLogClientService implements IEventLogClientService { this.spacesService = spacesService; } - getClient(request: KibanaRequest) { - return new EventLogClient({ - esContext: this.esContext, - savedObjectGetter: this.savedObjectProviderRegistry.getProvidersClient(request), - spacesService: this.spacesService, - request, - }); + getClient(requestOrSpaceId: KibanaRequest | string) { + if (typeof requestOrSpaceId === 'string') { + return new EventLogClient({ + esContext: this.esContext, + spacesService: this.spacesService, + spaceId: requestOrSpaceId, + }); + } else { + return new EventLogClient({ + esContext: this.esContext, + savedObjectGetter: this.savedObjectProviderRegistry.getProvidersClient(requestOrSpaceId), + spacesService: this.spacesService, + request: requestOrSpaceId, + }); + } } } diff --git a/x-pack/plugins/event_log/server/event_logger.ts b/x-pack/plugins/event_log/server/event_logger.ts index 0c674ab805a6c..757da6e848ece 100644 --- a/x-pack/plugins/event_log/server/event_logger.ts +++ b/x-pack/plugins/event_log/server/event_logger.ts @@ -107,6 +107,22 @@ export class EventLogger implements IEventLogger { logEventDoc(this.systemLogger, doc); } } + + updateEvent(id: string, event: IEvent): void { + const doc: Required = { + index: this.esContext.esNames.dataStream, + body: event, + id, + }; + + if (this.eventLogService.isIndexingEntries()) { + updateEventDoc(this.esContext, doc); + } + + if (this.eventLogService.isLoggingEntries()) { + logUpdateEventDoc(this.systemLogger, doc); + } + } } // return the epoch millis of the start date, or null; may be NaN if garbage @@ -161,6 +177,14 @@ function logEventDoc(logger: Logger, doc: Doc): void { logger.info(`event logged: ${JSON.stringify(doc.body)}`); } +function logUpdateEventDoc(logger: Logger, doc: Doc): void { + logger.info(`event updated: ${JSON.stringify(doc.body)}`); +} + function indexEventDoc(esContext: EsContext, doc: Doc): void { esContext.esAdapter.indexDocument(doc); } + +function updateEventDoc(esContext: EsContext, doc: Required): void { + esContext.esAdapter.updateDocument(doc); +} diff --git a/x-pack/plugins/event_log/server/types.ts b/x-pack/plugins/event_log/server/types.ts index 7287da0c0fd6e..8820c2e1bfd04 100644 --- a/x-pack/plugins/event_log/server/types.ts +++ b/x-pack/plugins/event_log/server/types.ts @@ -47,7 +47,7 @@ export interface IEventLogService { } export interface IEventLogClientService { - getClient(request: KibanaRequest): IEventLogClient; + getClient(request: KibanaRequest | string): IEventLogClient; } export interface IEventLogClient { @@ -83,4 +83,5 @@ export interface IEventLogger { logEvent(properties: IEvent): void; startTiming(event: IEvent, startTime?: Date): void; stopTiming(event: IEvent): void; + updateEvent(id: string, event: IEvent): void; } From e2b09686615781c3125e4c07f5f0a966e07af802 Mon Sep 17 00:00:00 2001 From: Nikita Khristinin Date: Mon, 9 Dec 2024 16:53:25 +0100 Subject: [PATCH 03/17] add Gap class --- .../alerting_event_logger.ts | 4 +- .../server/lib/rule_gaps/find_gaps.ts | 17 +- .../alerting/server/lib/rule_gaps/gap.ts | 148 +++++++++++++ .../server/lib/rule_gaps/schemas/index.ts | 14 +- .../rule_gaps/transforms/transformToGap.ts | 49 +++-- .../server/lib/rule_gaps/types/index.ts | 10 + .../server/lib/rule_gaps/update_gaps.ts | 194 ++++------------- .../server/lib/rule_gaps/utils/intervals.ts | 203 ++++++++++++++++++ .../server/es/cluster_client_adapter.ts | 81 ++++--- .../plugins/event_log/server/event_logger.ts | 18 +- x-pack/plugins/event_log/server/types.ts | 2 +- 11 files changed, 523 insertions(+), 217 deletions(-) create mode 100644 x-pack/plugins/alerting/server/lib/rule_gaps/gap.ts create mode 100644 x-pack/plugins/alerting/server/lib/rule_gaps/utils/intervals.ts diff --git a/x-pack/plugins/alerting/server/lib/alerting_event_logger/alerting_event_logger.ts b/x-pack/plugins/alerting/server/lib/alerting_event_logger/alerting_event_logger.ts index 3d73c2bfc83c1..fef8b6441b45d 100644 --- a/x-pack/plugins/alerting/server/lib/alerting_event_logger/alerting_event_logger.ts +++ b/x-pack/plugins/alerting/server/lib/alerting_event_logger/alerting_event_logger.ts @@ -417,9 +417,9 @@ export class AlertingEventLogger { ); } - public updateGap({ _id, gap }: { _id: string; gap: {} }): void { + public async updateGap({ meta, gap }: { meta: {}; gap: {} }): Promise { // TODO: Extract function to form gap document - this.eventLogger.updateEvent(_id, { + return this.eventLogger.updateEvent(meta, { kibana: { alert: { rule: { diff --git a/x-pack/plugins/alerting/server/lib/rule_gaps/find_gaps.ts b/x-pack/plugins/alerting/server/lib/rule_gaps/find_gaps.ts index 8b56c18ab822c..6bb983f87e48b 100644 --- a/x-pack/plugins/alerting/server/lib/rule_gaps/find_gaps.ts +++ b/x-pack/plugins/alerting/server/lib/rule_gaps/find_gaps.ts @@ -8,8 +8,10 @@ import { IEventLogClient } from '@kbn/event-log-plugin/server'; import { Logger } from '@kbn/core/server'; import { RULE_SAVED_OBJECT_TYPE } from '../../saved_objects'; -import { Gap, FindGapsParams } from './types'; +import { FindGapsParams } from './types'; +import { Gap } from './gap'; import { transformToGap } from './transforms/transformToGap'; +import { GapStatus } from '../../../common/constants/gap_status'; export const findGaps = async ({ eventLogClient, @@ -25,13 +27,18 @@ export const findGaps = async ({ page: number; perPage: number; }> => { - const { ruleIds, start, end, page, perPage } = params; + const { ruleIds, start, end, page, perPage, statuses } = params; try { + const statusesFilter = statuses + ?.map((status) => `kibana.alert.rule.gap.status : ${status}`) + .join(' OR '); const gapsResponse = await eventLogClient.findEventsBySavedObjectIds( RULE_SAVED_OBJECT_TYPE, ruleIds, { - filter: `event.action: gap AND event.provider: alerting and (kibana.alert.rule.gap.range <= "${end}" AND kibana.alert.rule.gap.range >= "${start}")`, + filter: `event.action: gap AND event.provider: alerting AND (kibana.alert.rule.gap.range <= "${end}" AND kibana.alert.rule.gap.range >= "${start}") ${ + statusesFilter ? `AND (${statusesFilter})` : '' + }`, sort: [{ sort_field: '@timestamp', sort_order: 'desc' }], page, per_page: perPage, @@ -61,9 +68,10 @@ export const findAllGaps = async ({ ruleIds: string[]; start: Date; end: Date; + statuses?: GapStatus[]; }; }): Promise => { - const { ruleIds, start, end } = params; + const { ruleIds, start, end, statuses } = params; const allGaps: Gap[] = []; let currentPage = 1; const perPage = 10000; @@ -78,6 +86,7 @@ export const findAllGaps = async ({ end: end.toISOString(), page: currentPage, perPage, + statuses, }, }); diff --git a/x-pack/plugins/alerting/server/lib/rule_gaps/gap.ts b/x-pack/plugins/alerting/server/lib/rule_gaps/gap.ts new file mode 100644 index 0000000000000..3ccff1fa308db --- /dev/null +++ b/x-pack/plugins/alerting/server/lib/rule_gaps/gap.ts @@ -0,0 +1,148 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { GapStatus, gapStatus } from '../../../common/constants'; + +import { Gap as GapType, Interval, StringInterval } from './types'; + +import { + mergeIntervals, + subtractAllIntervals, + sumIntervalsDuration, + intervalDuration, + subtractIntervals, + normalizeInterval, + denormalizeInterval, +} from './utils/intervals'; + +interface GapConstructorParams { + range: StringInterval; + filledIntervals?: StringInterval[]; + inProgressIntervals?: StringInterval[]; + meta?: { + _id: string; + _index: string; + _seq_no: string; + _primary_term: string; + }; +} + +export class Gap { + private _range: Interval; + private _filledIntervals: Interval[]; + private _inProgressIntervals: Interval[]; + private _meta?: { + _id: string; + _index: string; + _seq_no: string; + _primary_term: string; + }; + + constructor({ + range, + filledIntervals = [], + inProgressIntervals = [], + meta, + }: GapConstructorParams) { + this._range = normalizeInterval(range); + this._filledIntervals = mergeIntervals(filledIntervals.map(normalizeInterval)); + this._inProgressIntervals = mergeIntervals(inProgressIntervals.map(normalizeInterval)); + if (meta) { + this._meta = meta; + } + } + + public fillGap(interval: Interval): void { + this._filledIntervals = mergeIntervals([...this.filledIntervals, interval]); + } + + public addInProgress(interval: StringInterval): void { + const normalized = normalizeInterval(interval); + const sanitized = subtractAllIntervals([normalized], this.filledIntervals); + this._inProgressIntervals = mergeIntervals([...this.inProgressIntervals, ...sanitized]); + } + + public get range() { + return this._range; + } + + public get filledIntervals() { + return this._filledIntervals; + } + + public get inProgressIntervals() { + return this._inProgressIntervals; + } + + /** + * unfilled = range - (filled + inProgress) + */ + public get unfilledIntervals(): Interval[] { + const combined = mergeIntervals([...this.filledIntervals, ...this.inProgressIntervals]); + return subtractIntervals(this.range, combined); + } + + public get totalGapDurationMs(): number { + return intervalDuration(this.range); + } + + public get filledGapDurationMs(): number { + return sumIntervalsDuration(this.filledIntervals); + } + + public get unfilledGapDurationMs(): number { + return sumIntervalsDuration(this.unfilledIntervals); + } + + public get inProgressGapDurationMs(): number { + return sumIntervalsDuration(this.inProgressIntervals); + } + + public get status(): GapStatus { + if (this.unfilledGapDurationMs === this.totalGapDurationMs) { + return gapStatus.UNFILLED; + } else if (this.unfilledGapDurationMs === 0 && this.inProgressGapDurationMs === 0) { + return gapStatus.FILLED; + } else { + return gapStatus.PARTIALLY_FILLED; + } + } + + public get meta() { + return this._meta; + } + + public getState(): GapType { + return { + range: denormalizeInterval(this.range), + filledIntervals: this.filledIntervals.map(denormalizeInterval), + inProgressIntervals: this.inProgressIntervals.map(denormalizeInterval), + unfilledIntervals: this.unfilledIntervals.map(denormalizeInterval), + status: this.status, + totalGapDurationMs: this.totalGapDurationMs, + filledDurationMs: this.filledGapDurationMs, + unfilledDurationMs: this.unfilledGapDurationMs, + inProgressDurationMs: this.inProgressGapDurationMs, + }; + } + + /** + * Returns the gap object for es + */ + public getEsObject() { + return { + range: denormalizeInterval(this.range), + filled_intervals: this.filledIntervals.map(denormalizeInterval), + in_progress_intervals: this.inProgressIntervals.map(denormalizeInterval), + unfilled_intervals: this.unfilledIntervals.map(denormalizeInterval), + status: this.status, + total_gap_duration_ms: this.totalGapDurationMs, + filled_duration_ms: this.filledGapDurationMs, + unfilled_duration_ms: this.unfilledGapDurationMs, + in_progress_duration_ms: this.inProgressGapDurationMs, + }; + } +} diff --git a/x-pack/plugins/alerting/server/lib/rule_gaps/schemas/index.ts b/x-pack/plugins/alerting/server/lib/rule_gaps/schemas/index.ts index 0a6b23f769fb5..624ed7cb0cae6 100644 --- a/x-pack/plugins/alerting/server/lib/rule_gaps/schemas/index.ts +++ b/x-pack/plugins/alerting/server/lib/rule_gaps/schemas/index.ts @@ -14,19 +14,20 @@ export const gapStatusSchema = schema.oneOf([ schema.literal(gapStatus.PARTIALLY_FILLED), ]); -const rangeSchema = schema.object({ +export const rangeSchema = schema.object({ lte: schema.string(), gte: schema.string(), }); +export const rangeListSchema = schema.arrayOf(rangeSchema); + export const gapSchema = schema.object({ - _id: schema.string(), - eventId: schema.string(), + _id: schema.maybe(schema.string()), status: gapStatusSchema, range: rangeSchema, - inProgressIntervals: schema.arrayOf(rangeSchema), - filledIntervals: schema.arrayOf(rangeSchema), - unfilledIntervals: schema.arrayOf(rangeSchema), + inProgressIntervals: rangeListSchema, + filledIntervals: rangeListSchema, + unfilledIntervals: rangeListSchema, totalGapDurationMs: schema.number(), filledDurationMs: schema.number(), unfilledDurationMs: schema.number(), @@ -42,6 +43,7 @@ export const findGapsParamsSchema = schema.object( start: schema.maybe(schema.string()), sortField: schema.maybe(schema.oneOf([schema.literal('@timestamp')])), sortOrder: schema.maybe(schema.oneOf([schema.literal('asc'), schema.literal('desc')])), + statuses: schema.maybe(schema.arrayOf(gapStatusSchema)), }, { validate({ start, end }) { diff --git a/x-pack/plugins/alerting/server/lib/rule_gaps/transforms/transformToGap.ts b/x-pack/plugins/alerting/server/lib/rule_gaps/transforms/transformToGap.ts index b5537e81a120e..2713e2237067c 100644 --- a/x-pack/plugins/alerting/server/lib/rule_gaps/transforms/transformToGap.ts +++ b/x-pack/plugins/alerting/server/lib/rule_gaps/transforms/transformToGap.ts @@ -5,26 +5,47 @@ * 2.0. */ import { QueryEventsBySavedObjectResult } from '@kbn/event-log-plugin/server'; -import { Gap } from '../types'; +import { Gap } from '../gap'; +import { StringInterval } from '../types'; + +type PotenialInterval = { lte?: string; gte?: string } | undefined; +const validateInterval = (interval: PotenialInterval): StringInterval | null => { + if (!interval?.gte || !interval?.lte) return null; + + return { + lte: interval.lte, + gte: interval.gte, + }; +}; + +const validateIntervals = (intervals: PotenialInterval[] | undefined): StringInterval[] => + (intervals?.map(validateInterval)?.filter((interval) => interval !== null) as StringInterval[]) ?? + []; export const transformToGap = (events: QueryEventsBySavedObjectResult): Gap[] => { return events?.data ?.map((doc) => { const gap = doc?.kibana?.alert?.rule?.gap; if (!gap) return null; - return { - // eventId: doc.event.id, - status: gap.status, - range: gap.range, - inProgressIntervals: gap.in_progress_intervals ?? [], - filledIntervals: gap.filled_intervals ?? [], - unfilledIntervals: gap.unfilled_intervals ?? [], - totalGapDurationMs: Number(gap.total_gap_duration_ms), - filledDurationMs: Number(gap.filled_duration_ms), - unfilledDurationMs: Number(gap.unfilled_duration_ms), - inProgressDurationMs: Number(gap.in_progress_duration_ms), - _id: doc._id, - }; + + const range = validateInterval(gap.range); + + if (!range) return null; + + const filledIntervals = validateIntervals(gap?.filled_intervals); + const inProgressIntervals = validateIntervals(gap?.in_progress_intervals); + + return new Gap({ + range, + filledIntervals, + inProgressIntervals, + meta: { + _id: doc._id, + _index: doc._index, + _seq_no: doc._seq_no, + _primary_term: doc._primary_term, + }, + }); }) .filter((gap): gap is Gap => gap !== null); }; diff --git a/x-pack/plugins/alerting/server/lib/rule_gaps/types/index.ts b/x-pack/plugins/alerting/server/lib/rule_gaps/types/index.ts index 1eec48080ec83..821c41267125b 100644 --- a/x-pack/plugins/alerting/server/lib/rule_gaps/types/index.ts +++ b/x-pack/plugins/alerting/server/lib/rule_gaps/types/index.ts @@ -10,3 +10,13 @@ import { gapSchema, findGapsParamsSchema } from '../schemas'; export type Gap = TypeOf; export type FindGapsParams = TypeOf; + +export interface Interval { + gte: Date; + lte: Date; +} + +export interface StringInterval { + gte: string; + lte: string; +} diff --git a/x-pack/plugins/alerting/server/lib/rule_gaps/update_gaps.ts b/x-pack/plugins/alerting/server/lib/rule_gaps/update_gaps.ts index 4d3b3e9bf9dc9..f8f8b0536d34e 100644 --- a/x-pack/plugins/alerting/server/lib/rule_gaps/update_gaps.ts +++ b/x-pack/plugins/alerting/server/lib/rule_gaps/update_gaps.ts @@ -10,105 +10,12 @@ import { IEventLogClient, IEventLogger } from '@kbn/event-log-plugin/server'; import { AD_HOC_RUN_SAVED_OBJECT_TYPE, RULE_SAVED_OBJECT_TYPE } from '../../saved_objects'; import { AlertingEventLogger } from '../alerting_event_logger/alerting_event_logger'; import { findAllGaps } from './find_gaps'; -import { Gap } from './types'; +import { Gap } from './gap'; import { AdHocRunSO } from '../../data/ad_hoc_run/types'; +import { adHocRunStatus, gapStatus } from '../../../common/constants'; import { parseDuration } from '../../../common'; import { transformAdHocRunToBackfillResult } from '../../application/backfill/transforms'; -interface Interval { - lte: string; - gte: string; -} - -const overlapIntervals = (interval1: Interval, interval2: Interval): Interval | null => { - const start1 = new Date(interval1.gte).getTime(); - const end1 = new Date(interval1.lte).getTime(); - const start2 = new Date(interval2.gte).getTime(); - const end2 = new Date(interval2.lte).getTime(); - const maxStart = Math.max(start1, start2); - const minEnd = Math.min(end1, end2); - if (maxStart < minEnd) { - return { - gte: new Date(maxStart).toISOString(), - lte: new Date(minEnd).toISOString(), - }; - } else { - return null; - } -}; - -/** - * Merges overlapping intervals in a list. - */ -const mergeIntervals = (intervals: Interval[]): Interval[] => { - if (!intervals.length) return []; - intervals.sort((a, b) => new Date(a.gte).getTime() - new Date(b.gte).getTime()); - const merged: Interval[] = [intervals[0]]; - for (const current of intervals.slice(1)) { - const last = merged[merged.length - 1]; - if (new Date(last.lte) >= new Date(current.gte)) { - last.lte = new Date( - Math.max(new Date(last.lte).getTime(), new Date(current.lte).getTime()) - ).toISOString(); - } else { - merged.push(current); - } - } - return merged; -}; - -/** - * Subtracts a list of intervals from a base interval. - */ -const subtractIntervals = (interval: Interval, intervalsToSubtract: Interval[]): Interval[] => { - let intervals: Interval[] = [interval]; - for (const subtractInterval of intervalsToSubtract) { - intervals = intervals.flatMap((current) => { - const overlap = overlapIntervals(current, subtractInterval); - if (!overlap) return [current]; - const results: Interval[] = []; - if (new Date(current.gte) < new Date(overlap.gte)) { - results.push({ gte: current.gte, lte: overlap.gte }); - } - if (new Date(current.lte) > new Date(overlap.lte)) { - results.push({ gte: overlap.lte, lte: current.lte }); - } - return results; - }); - } - return intervals; -}; - -/** - * Subtracts a list of intervals from another list of intervals. - */ -const subtractIntervalsFromIntervals = ( - intervals: Interval[], - intervalsToSubtract: Interval[] -): Interval[] => { - const result: Interval[] = []; - for (const interval of intervals) { - const subtracted = subtractIntervals(interval, intervalsToSubtract); - result.push(...subtracted); - } - return mergeIntervals(result); -}; - -/** - * Fill the gap with the interval - */ -const fillGap = (gap: Gap, interval: { from: Date; to: Date }): Gap => { - const newGap = { - ...gap, - }; - newGap.status = 'filled'; // .. - newGap.filledIntervals = []; // ... - newGap.inProgressIntervals = []; // ... - newGap.unfilledIntervals = []; // ... - - return newGap; -}; - /** * Find all overlapping backfill tasks and update the gap status accordingly */ @@ -130,22 +37,16 @@ const updateGapStatus = async ({ }, // Filter for backfills that overlap with our interval filter: ` - ad_hoc_run_params.attributes.start <= "${gap.range.lte}" and - ad_hoc_run_params.attributes.end >= "${gap.range.gte}" + ad_hoc_run_params.attributes.start <= "${gap.range.lte.toISOString()}" and + ad_hoc_run_params.attributes.end >= "${gap.range.gte.toISOString()}" `, page: 1, - perPage: 100, + perPage: 10000, }); // TODO: Extract backfill transform to another function, to reuse in API const transformedBackfills = backfillSOs.map((data) => transformAdHocRunToBackfillResult(data)); - console.log('backfillResults', JSON.stringify(backfillSOs, null, 2)); - console.log('transformedBackfills', JSON.stringify(transformedBackfills, null, 2)); - - const potenialInProgressIntervals = []; - - // Process each backfill's schedule for (const backfill of transformedBackfills) { if ('error' in backfill) { break; @@ -160,40 +61,16 @@ const updateGapStatus = async ({ gte: new Date(from).toISOString(), lte: new Date(to).toISOString(), }; - // Check overlap with gap.range - const overlap = overlapIntervals(scheduleInterval, gap.range); - if (overlap) { - potenialInProgressIntervals.push(overlap); + if ( + scheduleItem.status === adHocRunStatus.PENDING || + scheduleItem.status === adHocRunStatus.RUNNING + ) { + gap.addInProgress(scheduleInterval); } } } - // Merge overlapping intervals in potentialInProgress - const mergedPotentialInProgress = mergeIntervals(potenialInProgressIntervals); - - const inProgressIntervals = subtractIntervalsFromIntervals( - mergedPotentialInProgress, - gap.filledIntervals - ); - - // unfilledIntervals = gap.range excluding filledIntervals and inProgressIntervals - const unfilledIntervals = subtractIntervals( - gap.range, - mergeIntervals([...gap.filledIntervals, ...inProgressIntervals]) - ); - - const newGap = { ...gap, inProgressIntervals, unfilledIntervals }; - - console.log('newGap', JSON.stringify(newGap)); - - const hasPendingOverlapingTasks = true; // backfillResults.......; - - if (hasPendingOverlapingTasks && newGap.status !== 'filled') { - newGap.status = 'filled'; - newGap.inProgressIntervals = []; // ..[] - } - - return newGap; + return gap; }; export async function updateGaps(params: { @@ -227,32 +104,49 @@ export async function updateGaps(params: { ruleIds: [ruleId], start, end, + statuses: [gapStatus.PARTIALLY_FILLED, gapStatus.UNFILLED], }, }); - console.log('allGaps', JSON.stringify(allGaps, null, 2)); + console.log('start/endn', start, end); + console.log('needTofilleGap', needToFillGaps); + console.log('allGaps', allGaps.length); + console.log('gaps', JSON.stringify(allGaps)); for (const gap of allGaps) { - const newGap = needToFillGaps - ? fillGap(gap, { - from: start, - to: end, - }) - : gap; + if (needToFillGaps) { + gap.fillGap({ + gte: start, + lte: end, + }); + } - const updatedGap = await updateGapStatus({ - gap: newGap, + updateGapStatus({ + gap, savedObjectsClient, ruleId, }); - const { _id, ...gapBody } = updatedGap; - console.log('updatedGap', JSON.stringify(updatedGap)); - - await alertingEventLogger.updateGap({ - _id, - gap: { ...gapBody, unfilled_intervals: gapBody.unfilledIntervals }, - }); + console.log(gap); + + const esGap = gap.getEsObject(); + const meta = gap.meta; + + if (meta) { + try { + await alertingEventLogger.updateGap({ + meta, + gap: esGap, + }); + } catch (e) { + // TODO version mismatch -> + // refetch gap + // check status + // retry + + logger.error('failed update'); + } + } } } catch (err) { logger.error(`Failed to process gaps for rule ${ruleId}: ${err.message}`); diff --git a/x-pack/plugins/alerting/server/lib/rule_gaps/utils/intervals.ts b/x-pack/plugins/alerting/server/lib/rule_gaps/utils/intervals.ts new file mode 100644 index 0000000000000..7ac8a60d05071 --- /dev/null +++ b/x-pack/plugins/alerting/server/lib/rule_gaps/utils/intervals.ts @@ -0,0 +1,203 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Interval, StringInterval } from '../types'; + +/** + * Finds the overlapping portion of two intervals, if any. + * + * @param interval1 - The first interval. + * @param interval2 - The second interval. + * @returns The overlapping interval if it exists, otherwise null. + * + * @example + * const i1: Interval = { gte: new Date('2023-10-19T12:00:00Z'), lte: new Date('2023-10-19T12:30:00Z') }; + * const i2: Interval = { gte: new Date('2023-10-19T12:15:00Z'), lte: new Date('2023-10-19T12:45:00Z') }; + * Overlap: 12:15 - 12:30 + * const overlap = getOverlap(i1, i2); + * overlap = { gte: 2023-10-19T12:15:00Z, lte: 2023-10-19T12:30:00Z } + */ +export const getOverlap = (interval1: Interval, interval2: Interval): Interval | null => { + const start = new Date(Math.max(interval1.gte.getTime(), interval2.gte.getTime())); + const end = new Date(Math.min(interval1.lte.getTime(), interval2.lte.getTime())); + return start < end ? { gte: start, lte: end } : null; +}; + +/** + * Merges a list of possibly overlapping intervals into a minimal set of non-overlapping intervals. + * + * @param intervals - An array of intervals to merge. + * @returns A new array of merged intervals with no overlaps. + * + * @example + * const intervals: Interval[] = [ + * { gte: new Date('2023-10-19T12:00:00Z'), lte: new Date('2023-10-19T12:10:00Z') }, + * { gte: new Date('2023-10-19T12:05:00Z'), lte: new Date('2023-10-19T12:15:00Z') } + * ]; + * // Merge into one interval: 12:00 - 12:15 + * const merged = mergeIntervals(intervals); + * // merged = [{ gte: 2023-10-19T12:00:00Z, lte: 2023-10-19T12:15:00Z }] + */ +export const mergeIntervals = (intervals: Interval[]): Interval[] => { + if (!intervals.length) return []; + + const sorted = [...intervals].sort((a, b) => a.gte.getTime() - b.gte.getTime()); + + return sorted.reduce((merged, current, index) => { + if (index === 0) { + merged.push(current); + return merged; + } + + const last = merged[merged.length - 1]; + if (last.lte.getTime() >= current.gte.getTime()) { + // Overlapping, merge intervals by extending the last interval's end time + last.lte = new Date(Math.max(last.lte.getTime(), current.lte.getTime())); + } else { + // No overlap, just add the current interval + merged.push(current); + } + return merged; + }, [] as Interval[]); +}; + +/** + * Subtracts a list of intervals from a single base interval, returning any remaining parts that do not overlap. + * + * @param base - The base interval. + * @param intervalsToSubtract - Intervals to remove from the base. + * @returns An array of intervals representing the remainder of the base interval. + * + * @example + * const base: Interval = { gte: new Date('2023-10-19T12:00:00Z'), lte: new Date('2023-10-19T13:00:00Z') }; + * const toSubtract: Interval[] = [{ gte: new Date('2023-10-19T12:20:00Z'), lte: new Date('2023-10-19T12:30:00Z') }]; + * // Result splits the base into two: 12:00-12:20 and 12:30-13:00 + * const result = subtractIntervals(base, toSubtract); + * // result = [ + * // { gte: 2023-10-19T12:00:00Z, lte: 2023-10-19T12:20:00Z }, + * // { gte: 2023-10-19T12:30:00Z, lte: 2023-10-19T13:00:00Z } + * // ] + */ +export const subtractIntervals = (base: Interval, intervalsToSubtract: Interval[]): Interval[] => { + let result: Interval[] = [base]; + for (const toSubtract of intervalsToSubtract) { + result = result.flatMap((current) => { + const overlap = getOverlap(current, toSubtract); + if (!overlap) return [current]; + + const remainder: Interval[] = []; + const currentStart = current.gte.getTime(); + const currentEnd = current.lte.getTime(); + const overlapStart = overlap.gte.getTime(); + const overlapEnd = overlap.lte.getTime(); + + if (overlapStart > currentStart) { + remainder.push({ gte: new Date(currentStart), lte: new Date(overlapStart) }); + } + + if (overlapEnd < currentEnd) { + remainder.push({ gte: new Date(overlapEnd), lte: new Date(currentEnd) }); + } + + return remainder; + }); + } + return result; +}; + +/** + * Applies subtractIntervals to each interval in an array and merges the result, + * effectively removing intervalsToSubtract from all given intervals. + * + * @param intervals - The original intervals from which to subtract. + * @param intervalsToSubtract - The intervals to remove. + * @returns An array of intervals representing what remains after all subtractions. + * + * @example + * const intervals: Interval[] = [ + * { gte: new Date('2023-10-19T12:00:00Z'), lte: new Date('2023-10-19T12:30:00Z') }, + * { gte: new Date('2023-10-19T12:40:00Z'), lte: new Date('2023-10-19T13:00:00Z') } + * ]; + * const toSubtract: Interval[] = [{ gte: new Date('2023-10-19T12:20:00Z'), lte: new Date('2023-10-19T12:45:00Z') }]; + * // For the first interval: removing 12:20-12:30 leaves 12:00-12:20 + * // For the second interval: removing 12:20-12:45 overlaps partially, leaving 12:45-13:00 + * // After merging, result: + * // [ + * // { gte: 2023-10-19T12:00:00Z, lte: 2023-10-19T12:20:00Z }, + * // { gte: 2023-10-19T12:45:00Z, lte: 2023-10-19T13:00:00Z } + * // ] + * const result = subtractAllIntervals(intervals, toSubtract); + */ +export const subtractAllIntervals = ( + intervals: Interval[], + intervalsToSubtract: Interval[] +): Interval[] => { + const result: Interval[] = []; + for (const interval of intervals) { + const subtracted = subtractIntervals(interval, intervalsToSubtract); + result.push(...subtracted); + } + return mergeIntervals(result); +}; + +/** + * Calculates the duration of a single interval in milliseconds. + * + * @param interval - The interval. + * @returns The duration in milliseconds. + * + * @example + * const i: Interval = { gte: new Date('2023-10-19T12:00:00Z'), lte: new Date('2023-10-19T12:10:00Z') }; + * const duration = intervalDuration(i); + * // duration = 600000 ms (10 minutes) + */ +export const intervalDuration = (interval: Interval): number => { + return Math.max(0, interval.lte.getTime() - interval.gte.getTime()); +}; + +/** + * Sums the durations of multiple intervals in milliseconds. + * + * @param intervals - An array of intervals. + * @returns The total combined duration in milliseconds. + * + * @example + * const intervals: Interval[] = [ + * { gte: new Date('2023-10-19T12:00:00Z'), lte: new Date('2023-10-19T12:10:00Z') }, + * { gte: new Date('2023-10-19T12:15:00Z'), lte: new Date('2023-10-19T12:20:00Z') } + * ]; + * // First interval: 10 min, Second interval: 5 min + * // Total: 15 min = 900000 ms + * const totalDuration = sumIntervalsDuration(intervals); + */ +export const sumIntervalsDuration = (intervals: Interval[]): number => { + return intervals.reduce((sum, interval) => sum + intervalDuration(interval), 0); +}; + +/** + * Converts a String-based interval into a date-based interval + * @param interval + * @returns + */ +export const normalizeInterval = (interval: StringInterval): Interval => { + const gte = new Date(interval.gte); + const lte = new Date(interval.lte); + return { gte, lte }; +}; + +/** + * Converts a Date-based interval into a string-based interval. + * + * @param interval - An Interval with gte and lte as Date objects. + * @returns A ConstructorInterval with gte and lte as ISO strings. + */ +export const denormalizeInterval = (interval: Interval): StringInterval => { + return { + gte: interval.gte.toISOString(), + lte: interval.lte.toISOString(), + }; +}; diff --git a/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts b/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts index 52ee5f9071e56..54647990889f9 100644 --- a/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts +++ b/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts @@ -7,7 +7,7 @@ import { Subject } from 'rxjs'; import { bufferTime, filter as rxFilter, concatMap } from 'rxjs'; -import { reject, isUndefined, isNumber, pick, isEmpty, get } from 'lodash'; +import { reject, isUndefined, isNumber, pick, isEmpty, get, mergeWith } from 'lodash'; import type { PublicMethodsOf } from '@kbn/utility-types'; import { Logger, ElasticsearchClient } from '@kbn/core/server'; import util from 'util'; @@ -26,7 +26,7 @@ export type IClusterClientAdapter = PublicMethodsOf; export interface Doc { index: string; body: IEvent; - id?: string; + meta?: {}; } type Wait = () => Promise; @@ -139,8 +139,20 @@ export class ClusterClientAdapter< await this.docsBufferedFlushed; } - public updateDocument(doc: Required) { - this.docBuffer$.next(doc); + public async updateDocument(doc: Required) { + const esClient = await this.elasticsearchClientPromise; + try { + const result = await esClient.update({ + doc: doc.body, + id: doc.meta._id, + index: doc.meta._index, + if_primary_term: doc.meta._primary_term, + if_seq_no: doc.meta._seq_no, + }); + } catch (e) { + this.logger.error(`error update event: "${e.message}"; docs: ${JSON.stringify(doc)}`); + throw e; + } } public indexDocument(doc: TDoc): void { @@ -172,35 +184,32 @@ export class ClusterClientAdapter< try { const esClient = await this.elasticsearchClientPromise; - const originalDocsBeforeUpdate = await esClient.search({ - index: this.esNames.dataStream, - seq_no_primary_term: true, - query: { - terms: { - _id: docsToUpdate.map((doc) => doc.id), - }, - }, - }); - for (const originalDoc of originalDocsBeforeUpdate?.hits?.hits ?? []) { - const fieldsToUpdate = docsToUpdate.find( - (docToUpdate) => docToUpdate.id === originalDoc._id - ); - if (!fieldsToUpdate || !originalDoc._source) continue; - - bulkBody.push({ - index: { - _index: originalDoc._index, - _id: originalDoc._id, - if_seq_no: originalDoc._seq_no, - if_primary_term: originalDoc._primary_term, - }, - }); - bulkBody.push({ - ...originalDoc._source, - ...fieldsToUpdate.body, - }); - } + // for (const docToUpdate of docsToUpdate) { + // bulkBody.push({ + // update: { + // _index: docToUpdate.meta._index, + // _id: docToUpdate.meta._id, + // if_seq_no: docToUpdate.meta._seq_no, + // if_primary_term: docToUpdate.meta._primary_term, + // }, + // }); + + // // const updatedDoc = mergeWith( + // // {}, + // // originalDoc._source, + // // fieldsToUpdate.body, + // // (objValue, srcValue) => { + // // if (Array.isArray(srcValue)) { + // // return srcValue; + // // } + // // } + // // ); + + // // console.log('updatedDoc', JSON.stringify(updatedDoc)); + + // bulkBody.push({ doc: docToUpdate.body }); + // } const response = await esClient.bulk({ // index: this.esNames.dataStream, @@ -455,6 +464,7 @@ export class ClusterClientAdapter< } = await esClient.search({ index, track_total_hits: true, + seq_no_primary_term: true, body, }); return { @@ -464,6 +474,9 @@ export class ClusterClientAdapter< data: hits.map((hit) => ({ ...hit._source, _id: hit._id, + _index: hit._index, + _seq_no: hit._seq_no, + _primary_term: hit._primary_term, })), }; } catch (err) { @@ -503,6 +516,7 @@ export class ClusterClientAdapter< index, track_total_hits: true, body, + seq_no_primary_term: true, }); return { page, @@ -511,6 +525,9 @@ export class ClusterClientAdapter< data: hits.map((hit) => ({ ...hit._source, _id: hit._id, + _index: hit._index, + if_seq_no: hit._seq_no, + if_primary_term: hit._primary_term, })), }; } catch (err) { diff --git a/x-pack/plugins/event_log/server/event_logger.ts b/x-pack/plugins/event_log/server/event_logger.ts index 757da6e848ece..0424f44d28679 100644 --- a/x-pack/plugins/event_log/server/event_logger.ts +++ b/x-pack/plugins/event_log/server/event_logger.ts @@ -108,19 +108,21 @@ export class EventLogger implements IEventLogger { } } - updateEvent(id: string, event: IEvent): void { + async updateEvent(meta: {}, event: IEvent): Promise { const doc: Required = { index: this.esContext.esNames.dataStream, body: event, - id, + meta, }; if (this.eventLogService.isIndexingEntries()) { - updateEventDoc(this.esContext, doc); - } + const result = updateEventDoc(this.esContext, doc); - if (this.eventLogService.isLoggingEntries()) { - logUpdateEventDoc(this.systemLogger, doc); + if (this.eventLogService.isLoggingEntries()) { + logUpdateEventDoc(this.systemLogger, doc); + } + + return result; } } } @@ -185,6 +187,6 @@ function indexEventDoc(esContext: EsContext, doc: Doc): void { esContext.esAdapter.indexDocument(doc); } -function updateEventDoc(esContext: EsContext, doc: Required): void { - esContext.esAdapter.updateDocument(doc); +async function updateEventDoc(esContext: EsContext, doc: Required): Promise { + return esContext.esAdapter.updateDocument(doc); } diff --git a/x-pack/plugins/event_log/server/types.ts b/x-pack/plugins/event_log/server/types.ts index 8820c2e1bfd04..e72921d1c4553 100644 --- a/x-pack/plugins/event_log/server/types.ts +++ b/x-pack/plugins/event_log/server/types.ts @@ -83,5 +83,5 @@ export interface IEventLogger { logEvent(properties: IEvent): void; startTiming(event: IEvent, startTime?: Date): void; stopTiming(event: IEvent): void; - updateEvent(id: string, event: IEvent): void; + updateEvent(meta: {}, event: IEvent): void; } From c47be0ccb0c1707fdab11d056ec499801c396414 Mon Sep 17 00:00:00 2001 From: Nikita Khristinin Date: Sun, 15 Dec 2024 22:38:11 +0100 Subject: [PATCH 04/17] UI + update gaps --- .../alerting/common/constants/index.ts | 1 + x-pack/plugins/alerting/common/index.ts | 12 + .../common/routes/gaps/apis/fill/index.ts | 13 + .../routes/gaps/apis/fill/schemas/latest.ts | 8 + .../routes/gaps/apis/fill/schemas/v1.ts | 12 + .../routes/gaps/apis/fill/types/latest.ts | 8 + .../common/routes/gaps/apis/fill/types/v1.ts | 11 + .../common/routes/gaps/apis/find/index.ts | 19 + .../routes/gaps/apis/find/schemas/latest.ts | 8 + .../routes/gaps/apis/find/schemas/v1.ts | 43 ++ .../routes/gaps/apis/find/types/latest.ts | 8 + .../common/routes/gaps/apis/find/types/v1.ts | 16 + .../gaps/apis/get_rules_with_gaps/index.ts | 24 ++ .../get_rules_with_gaps/schemas/latest.ts | 8 + .../apis/get_rules_with_gaps/schemas/v1.ts | 36 ++ .../apis/get_rules_with_gaps/types/latest.ts | 8 + .../gaps/apis/get_rules_with_gaps/types/v1.ts | 16 + .../common/routes/gaps/response/index.ts | 16 + .../routes/gaps/response/schemas/latest.ts | 8 + .../common/routes/gaps/response/schemas/v1.ts | 43 ++ .../routes/gaps/response/types/latest.ts | 8 + .../common/routes/gaps/response/types/v1.ts | 12 + .../methods/delete/delete_backfill.ts | 28 +- .../methods/fill_gap_by_id/fill_gap_by_id.ts | 44 ++ .../rule/methods/fill_gap_by_id/index.ts | 8 + .../rule/methods/find_gaps/find_gaps.ts | 13 +- .../rule/methods/find_gaps/index.ts | 9 +- .../get_rules_with_gaps.ts | 93 +++++ .../rule/methods/get_rules_with_gaps/index.ts | 8 + .../schemas/get_rules_with_gaps.ts | 19 + .../get_rules_with_gaps/schemas/index.ts | 8 + .../types/get_rule_params.ts | 12 + .../get_rules_with_gaps/types/index.ts | 8 + .../server/backfill_client/backfill_client.ts | 32 +- .../alerting_event_logger.ts | 20 +- .../server/lib/rule_gaps/find_gap_by_id.ts | 43 ++ .../server/lib/rule_gaps/find_gaps.ts | 18 +- .../server/lib/rule_gaps/gap/index.test.ts | 218 ++++++++++ .../lib/rule_gaps/{gap.ts => gap/index.ts} | 54 ++- .../intervals.ts => gap/interval_utils.ts} | 26 +- .../get_total_unfilled_gap_duration.ts | 38 -- .../server/lib/rule_gaps/schemas/index.ts | 24 +- .../server/lib/rule_gaps/types/index.ts | 5 +- .../update/add_filled_interval_to_gaps.ts | 76 ++++ .../add_in_progress_intervals_to_gaps.ts | 98 +++++ ...lculate_in_progress_intervals_for_gaps.ts} | 82 ++-- x-pack/plugins/alerting/server/plugin.ts | 2 +- .../gaps/apis/fill/fill_gap_by_id_route.ts | 50 +++ .../routes/gaps/apis/fill/transforms/index.ts | 10 + .../transforms/transform_request/latest.ts | 8 + .../fill/transforms/transform_request/v1.ts | 18 + .../apis/find/find_gaps_route.ts} | 3 +- .../routes/gaps/apis/find/transforms/index.ts | 12 + .../transforms/transform_request/latest.ts | 8 + .../find/transforms/transform_request/v1.ts | 28 ++ .../transforms/transform_response/latest.ts | 8 + .../find/transforms/transform_response/v1.ts | 30 ++ .../get_rules_with_gaps_route.ts | 47 +++ .../plugins/alerting/server/routes/index.ts | 10 + .../server/rules_client/rules_client.ts | 10 +- .../server/task_runner/ad_hoc_task_runner.ts | 86 ++-- .../server/task_runner/task_runner.ts | 23 +- .../alerting/server/task_runner/types.ts | 2 +- .../server/es/cluster_client_adapter.ts | 54 +-- .../event_log/server/event_log_client.ts | 44 +- .../server/event_log_start_service.ts | 22 +- .../event_log/server/event_logger.mock.ts | 1 + .../plugins/event_log/server/event_logger.ts | 6 +- x-pack/plugins/event_log/server/index.ts | 1 + x-pack/plugins/event_log/server/types.ts | 4 +- .../rule_management/rule_filtering.ts | 10 + .../pages/rule_details/index.tsx | 3 + .../rule_gaps/components/rule_gaps/index.tsx | 379 ++++++++++++++++++ .../components/rule_gaps/translations.tsx | 112 ++++++ .../rules_with_gaps_overview_panel/index.tsx | 237 +++++++++++ .../translations.tsx | 50 +++ .../rule_management/logic/types.ts | 1 + .../rules_table/rules_table_context.tsx | 2 + .../components/rules_table/rules_tables.tsx | 16 +- .../rule_types/utils/utils.ts | 2 +- 80 files changed, 2320 insertions(+), 301 deletions(-) create mode 100644 x-pack/plugins/alerting/common/routes/gaps/apis/fill/index.ts create mode 100644 x-pack/plugins/alerting/common/routes/gaps/apis/fill/schemas/latest.ts create mode 100644 x-pack/plugins/alerting/common/routes/gaps/apis/fill/schemas/v1.ts create mode 100644 x-pack/plugins/alerting/common/routes/gaps/apis/fill/types/latest.ts create mode 100644 x-pack/plugins/alerting/common/routes/gaps/apis/fill/types/v1.ts create mode 100644 x-pack/plugins/alerting/common/routes/gaps/apis/find/index.ts create mode 100644 x-pack/plugins/alerting/common/routes/gaps/apis/find/schemas/latest.ts create mode 100644 x-pack/plugins/alerting/common/routes/gaps/apis/find/schemas/v1.ts create mode 100644 x-pack/plugins/alerting/common/routes/gaps/apis/find/types/latest.ts create mode 100644 x-pack/plugins/alerting/common/routes/gaps/apis/find/types/v1.ts create mode 100644 x-pack/plugins/alerting/common/routes/gaps/apis/get_rules_with_gaps/index.ts create mode 100644 x-pack/plugins/alerting/common/routes/gaps/apis/get_rules_with_gaps/schemas/latest.ts create mode 100644 x-pack/plugins/alerting/common/routes/gaps/apis/get_rules_with_gaps/schemas/v1.ts create mode 100644 x-pack/plugins/alerting/common/routes/gaps/apis/get_rules_with_gaps/types/latest.ts create mode 100644 x-pack/plugins/alerting/common/routes/gaps/apis/get_rules_with_gaps/types/v1.ts create mode 100644 x-pack/plugins/alerting/common/routes/gaps/response/index.ts create mode 100644 x-pack/plugins/alerting/common/routes/gaps/response/schemas/latest.ts create mode 100644 x-pack/plugins/alerting/common/routes/gaps/response/schemas/v1.ts create mode 100644 x-pack/plugins/alerting/common/routes/gaps/response/types/latest.ts create mode 100644 x-pack/plugins/alerting/common/routes/gaps/response/types/v1.ts create mode 100644 x-pack/plugins/alerting/server/application/rule/methods/fill_gap_by_id/fill_gap_by_id.ts create mode 100644 x-pack/plugins/alerting/server/application/rule/methods/fill_gap_by_id/index.ts create mode 100644 x-pack/plugins/alerting/server/application/rule/methods/get_rules_with_gaps/get_rules_with_gaps.ts create mode 100644 x-pack/plugins/alerting/server/application/rule/methods/get_rules_with_gaps/index.ts create mode 100644 x-pack/plugins/alerting/server/application/rule/methods/get_rules_with_gaps/schemas/get_rules_with_gaps.ts create mode 100644 x-pack/plugins/alerting/server/application/rule/methods/get_rules_with_gaps/schemas/index.ts create mode 100644 x-pack/plugins/alerting/server/application/rule/methods/get_rules_with_gaps/types/get_rule_params.ts create mode 100644 x-pack/plugins/alerting/server/application/rule/methods/get_rules_with_gaps/types/index.ts create mode 100644 x-pack/plugins/alerting/server/lib/rule_gaps/find_gap_by_id.ts create mode 100644 x-pack/plugins/alerting/server/lib/rule_gaps/gap/index.test.ts rename x-pack/plugins/alerting/server/lib/rule_gaps/{gap.ts => gap/index.ts} (73%) rename x-pack/plugins/alerting/server/lib/rule_gaps/{utils/intervals.ts => gap/interval_utils.ts} (90%) delete mode 100644 x-pack/plugins/alerting/server/lib/rule_gaps/get_total_unfilled_gap_duration.ts create mode 100644 x-pack/plugins/alerting/server/lib/rule_gaps/update/add_filled_interval_to_gaps.ts create mode 100644 x-pack/plugins/alerting/server/lib/rule_gaps/update/add_in_progress_intervals_to_gaps.ts rename x-pack/plugins/alerting/server/lib/rule_gaps/{update_gaps.ts => update/caclculate_in_progress_intervals_for_gaps.ts} (65%) create mode 100644 x-pack/plugins/alerting/server/routes/gaps/apis/fill/fill_gap_by_id_route.ts create mode 100644 x-pack/plugins/alerting/server/routes/gaps/apis/fill/transforms/index.ts create mode 100644 x-pack/plugins/alerting/server/routes/gaps/apis/fill/transforms/transform_request/latest.ts create mode 100644 x-pack/plugins/alerting/server/routes/gaps/apis/fill/transforms/transform_request/v1.ts rename x-pack/plugins/alerting/server/routes/{rule_gaps/api/find/fing_gaps_route.ts => gaps/apis/find/find_gaps_route.ts} (94%) create mode 100644 x-pack/plugins/alerting/server/routes/gaps/apis/find/transforms/index.ts create mode 100644 x-pack/plugins/alerting/server/routes/gaps/apis/find/transforms/transform_request/latest.ts create mode 100644 x-pack/plugins/alerting/server/routes/gaps/apis/find/transforms/transform_request/v1.ts create mode 100644 x-pack/plugins/alerting/server/routes/gaps/apis/find/transforms/transform_response/latest.ts create mode 100644 x-pack/plugins/alerting/server/routes/gaps/apis/find/transforms/transform_response/v1.ts create mode 100644 x-pack/plugins/alerting/server/routes/gaps/apis/get_rules_with_gaps/get_rules_with_gaps_route.ts create mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_gaps/components/rule_gaps/index.tsx create mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_gaps/components/rule_gaps/translations.tsx create mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_gaps/components/rules_with_gaps_overview_panel/index.tsx create mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_gaps/components/rules_with_gaps_overview_panel/translations.tsx diff --git a/x-pack/plugins/alerting/common/constants/index.ts b/x-pack/plugins/alerting/common/constants/index.ts index 2a0c6bf3dcd5d..afdfcd60d18c1 100644 --- a/x-pack/plugins/alerting/common/constants/index.ts +++ b/x-pack/plugins/alerting/common/constants/index.ts @@ -14,3 +14,4 @@ export { } from './backfill'; export { PLUGIN } from './plugin'; export { gapStatus } from './gap_status'; +export type { GapStatus } from './gap_status'; diff --git a/x-pack/plugins/alerting/common/index.ts b/x-pack/plugins/alerting/common/index.ts index 8e5a258f15e6c..44894d90fc415 100644 --- a/x-pack/plugins/alerting/common/index.ts +++ b/x-pack/plugins/alerting/common/index.ts @@ -90,6 +90,18 @@ export const INTERNAL_ALERTING_BACKFILL_FIND_API_PATH = export const INTERNAL_ALERTING_BACKFILL_SCHEDULE_API_PATH = `${INTERNAL_ALERTING_BACKFILL_API_PATH}/_schedule` as const; +export const INTERNAL_ALERTING_GAPS_API_PATH = + `${INTERNAL_BASE_ALERTING_API_PATH}/rules/gaps` as const; + +export const INTERNAL_ALERTING_GAPS_FIND_API_PATH = + `${INTERNAL_ALERTING_GAPS_API_PATH}/_find` as const; + +export const INTERNAL_ALERTING_GAPS_GET_RULES_API_PATH = + `${INTERNAL_ALERTING_GAPS_API_PATH}/_get_rules` as const; + +export const INTERNAL_ALERTING_GAPS_FILL_BY_ID_API_PATH = + `${INTERNAL_ALERTING_GAPS_API_PATH}/_fill_by_id` as const; + export const ALERTING_FEATURE_ID = 'alerts'; export const MONITORING_HISTORY_LIMIT = 200; export const ENABLE_MAINTENANCE_WINDOWS = true; diff --git a/x-pack/plugins/alerting/common/routes/gaps/apis/fill/index.ts b/x-pack/plugins/alerting/common/routes/gaps/apis/fill/index.ts new file mode 100644 index 0000000000000..3fb3dde4ed2b8 --- /dev/null +++ b/x-pack/plugins/alerting/common/routes/gaps/apis/fill/index.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { fillGapByIdQuerySchema } from './schemas/latest'; + +export { fillGapByIdQuerySchema as fillGapByIdQuerySchemaV1 } from './schemas/v1'; + +export type { FillGapByIdQuery as FillGapByIdQueryV1 } from './types/v1'; +export type { ScheduleBackfillRequestBodyV1 as FillGapByIdResponseV1 } from '../../../backfill/apis/schedule'; diff --git a/x-pack/plugins/alerting/common/routes/gaps/apis/fill/schemas/latest.ts b/x-pack/plugins/alerting/common/routes/gaps/apis/fill/schemas/latest.ts new file mode 100644 index 0000000000000..25300c97a6d2e --- /dev/null +++ b/x-pack/plugins/alerting/common/routes/gaps/apis/fill/schemas/latest.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './v1'; diff --git a/x-pack/plugins/alerting/common/routes/gaps/apis/fill/schemas/v1.ts b/x-pack/plugins/alerting/common/routes/gaps/apis/fill/schemas/v1.ts new file mode 100644 index 0000000000000..122cd35a498db --- /dev/null +++ b/x-pack/plugins/alerting/common/routes/gaps/apis/fill/schemas/v1.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { schema } from '@kbn/config-schema'; + +export const fillGapByIdQuerySchema = schema.object({ + rule_id: schema.string(), + gap_id: schema.string(), +}); diff --git a/x-pack/plugins/alerting/common/routes/gaps/apis/fill/types/latest.ts b/x-pack/plugins/alerting/common/routes/gaps/apis/fill/types/latest.ts new file mode 100644 index 0000000000000..25300c97a6d2e --- /dev/null +++ b/x-pack/plugins/alerting/common/routes/gaps/apis/fill/types/latest.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './v1'; diff --git a/x-pack/plugins/alerting/common/routes/gaps/apis/fill/types/v1.ts b/x-pack/plugins/alerting/common/routes/gaps/apis/fill/types/v1.ts new file mode 100644 index 0000000000000..64bee71b53643 --- /dev/null +++ b/x-pack/plugins/alerting/common/routes/gaps/apis/fill/types/v1.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { TypeOf } from '@kbn/config-schema'; +import { fillGapByIdQuerySchemaV1 } from '..'; + +export type FillGapByIdQuery = TypeOf; diff --git a/x-pack/plugins/alerting/common/routes/gaps/apis/find/index.ts b/x-pack/plugins/alerting/common/routes/gaps/apis/find/index.ts new file mode 100644 index 0000000000000..43ab071202721 --- /dev/null +++ b/x-pack/plugins/alerting/common/routes/gaps/apis/find/index.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { findQuerySchema, findResponseSchema } from './schemas/latest'; +export type { FindGapsRequestQuery, FindGapsResponseBody, FindGapsResponse } from './types/latest'; + +export { + findQuerySchema as findQuerySchemaV1, + findResponseSchema as findResponseSchemaV1, +} from './schemas/v1'; +export type { + FindGapsRequestQuery as FindGapsRequestQueryV1, + FindGapsResponseBody as FindGapsResponseBodyV1, + FindGapsResponse as FindGapsResponseV1, +} from './types/v1'; diff --git a/x-pack/plugins/alerting/common/routes/gaps/apis/find/schemas/latest.ts b/x-pack/plugins/alerting/common/routes/gaps/apis/find/schemas/latest.ts new file mode 100644 index 0000000000000..25300c97a6d2e --- /dev/null +++ b/x-pack/plugins/alerting/common/routes/gaps/apis/find/schemas/latest.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './v1'; diff --git a/x-pack/plugins/alerting/common/routes/gaps/apis/find/schemas/v1.ts b/x-pack/plugins/alerting/common/routes/gaps/apis/find/schemas/v1.ts new file mode 100644 index 0000000000000..905a877edb856 --- /dev/null +++ b/x-pack/plugins/alerting/common/routes/gaps/apis/find/schemas/v1.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { schema } from '@kbn/config-schema'; +import { gapsResponseSchemaV1 } from '../../../response'; + +export const findQuerySchema = schema.object( + { + end: schema.maybe(schema.string()), + page: schema.number({ defaultValue: 1, min: 1 }), + per_page: schema.number({ defaultValue: 10, min: 0 }), + rule_id: schema.maybe(schema.string()), + start: schema.maybe(schema.string()), + sort_field: schema.maybe(schema.oneOf([schema.literal('createdAt'), schema.literal('start')])), + sort_order: schema.maybe(schema.oneOf([schema.literal('asc'), schema.literal('desc')])), + }, + { + validate({ start, end }) { + if (start) { + const parsedStart = Date.parse(start); + if (isNaN(parsedStart)) { + return `[start]: query start must be valid date`; + } + } + if (end) { + const parsedEnd = Date.parse(end); + if (isNaN(parsedEnd)) { + return `[end]: query end must be valid date`; + } + } + }, + } +); + +export const findResponseSchema = schema.object({ + page: schema.number(), + per_page: schema.number(), + total: schema.number(), + data: schema.arrayOf(gapsResponseSchemaV1), +}); diff --git a/x-pack/plugins/alerting/common/routes/gaps/apis/find/types/latest.ts b/x-pack/plugins/alerting/common/routes/gaps/apis/find/types/latest.ts new file mode 100644 index 0000000000000..25300c97a6d2e --- /dev/null +++ b/x-pack/plugins/alerting/common/routes/gaps/apis/find/types/latest.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './v1'; diff --git a/x-pack/plugins/alerting/common/routes/gaps/apis/find/types/v1.ts b/x-pack/plugins/alerting/common/routes/gaps/apis/find/types/v1.ts new file mode 100644 index 0000000000000..7834443652869 --- /dev/null +++ b/x-pack/plugins/alerting/common/routes/gaps/apis/find/types/v1.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { TypeOf } from '@kbn/config-schema'; +import { findQuerySchemaV1, findResponseSchemaV1 } from '..'; + +export type FindGapsRequestQuery = TypeOf; +export type FindGapsResponseBody = TypeOf; + +export interface FindGapsResponse { + body: FindGapsResponseBody; +} diff --git a/x-pack/plugins/alerting/common/routes/gaps/apis/get_rules_with_gaps/index.ts b/x-pack/plugins/alerting/common/routes/gaps/apis/get_rules_with_gaps/index.ts new file mode 100644 index 0000000000000..9f3b190999fb7 --- /dev/null +++ b/x-pack/plugins/alerting/common/routes/gaps/apis/get_rules_with_gaps/index.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { getRulesWithGapQuerySchema, getRulesWithGapResponseSchema } from './schemas/latest'; +export type { + GetRulesWithGapQuery, + GetRulesWithGapResponse, + GetRulesWithGapResponseBody, +} from './types/latest'; + +export { + getRulesWithGapQuerySchema as getRulesWithGapQuerySchemaV1, + getRulesWithGapResponseSchema as getRulesWithGapResponseSchemaV1, +} from './schemas/v1'; + +export type { + GetRulesWithGapQuery as GetRulesWithGapQueryV1, + GetRulesWithGapResponse as GetRulesWithGapResponseV1, + GetRulesWithGapResponseBody as GetRulesWithGapResponseBodyV1, +} from './types/v1'; diff --git a/x-pack/plugins/alerting/common/routes/gaps/apis/get_rules_with_gaps/schemas/latest.ts b/x-pack/plugins/alerting/common/routes/gaps/apis/get_rules_with_gaps/schemas/latest.ts new file mode 100644 index 0000000000000..25300c97a6d2e --- /dev/null +++ b/x-pack/plugins/alerting/common/routes/gaps/apis/get_rules_with_gaps/schemas/latest.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './v1'; diff --git a/x-pack/plugins/alerting/common/routes/gaps/apis/get_rules_with_gaps/schemas/v1.ts b/x-pack/plugins/alerting/common/routes/gaps/apis/get_rules_with_gaps/schemas/v1.ts new file mode 100644 index 0000000000000..f07e9ef93cd12 --- /dev/null +++ b/x-pack/plugins/alerting/common/routes/gaps/apis/get_rules_with_gaps/schemas/v1.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { schema } from '@kbn/config-schema'; + +export const getRulesWithGapQuerySchema = schema.object( + { + end: schema.string(), + start: schema.string(), + statuses: schema.maybe(schema.arrayOf(schema.string())), + }, + { + validate({ start, end }) { + if (start) { + const parsedStart = Date.parse(start); + if (isNaN(parsedStart)) { + return `[start]: query start must be valid date`; + } + } + if (end) { + const parsedEnd = Date.parse(end); + if (isNaN(parsedEnd)) { + return `[end]: query end must be valid date`; + } + } + }, + } +); + +export const getRulesWithGapResponseSchema = schema.object({ + total: schema.number(), + ruleIds: schema.arrayOf(schema.string()), +}); diff --git a/x-pack/plugins/alerting/common/routes/gaps/apis/get_rules_with_gaps/types/latest.ts b/x-pack/plugins/alerting/common/routes/gaps/apis/get_rules_with_gaps/types/latest.ts new file mode 100644 index 0000000000000..25300c97a6d2e --- /dev/null +++ b/x-pack/plugins/alerting/common/routes/gaps/apis/get_rules_with_gaps/types/latest.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './v1'; diff --git a/x-pack/plugins/alerting/common/routes/gaps/apis/get_rules_with_gaps/types/v1.ts b/x-pack/plugins/alerting/common/routes/gaps/apis/get_rules_with_gaps/types/v1.ts new file mode 100644 index 0000000000000..55cf952ba0cd4 --- /dev/null +++ b/x-pack/plugins/alerting/common/routes/gaps/apis/get_rules_with_gaps/types/v1.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { TypeOf } from '@kbn/config-schema'; +import { getRulesWithGapQuerySchemaV1, getRulesWithGapResponseSchemaV1 } from '..'; + +export type GetRulesWithGapQuery = TypeOf; +export type GetRulesWithGapResponseBody = TypeOf; + +export interface GetRulesWithGapResponse { + body: GetRulesWithGapResponseBody; +} diff --git a/x-pack/plugins/alerting/common/routes/gaps/response/index.ts b/x-pack/plugins/alerting/common/routes/gaps/response/index.ts new file mode 100644 index 0000000000000..ac7d9fe1e9163 --- /dev/null +++ b/x-pack/plugins/alerting/common/routes/gaps/response/index.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { gapsResponseSchema, errorResponseSchema } from './schemas/latest'; +export type { GapsResponse } from './types/latest'; + +export { + gapsResponseSchema as gapsResponseSchemaV1, + errorResponseSchema as errorResponseSchemaV1, +} from './schemas/v1'; + +export type { GapsResponse as GapsResponseV1, ErrorResponse as ErrorResponseV1 } from './types/v1'; diff --git a/x-pack/plugins/alerting/common/routes/gaps/response/schemas/latest.ts b/x-pack/plugins/alerting/common/routes/gaps/response/schemas/latest.ts new file mode 100644 index 0000000000000..25300c97a6d2e --- /dev/null +++ b/x-pack/plugins/alerting/common/routes/gaps/response/schemas/latest.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './v1'; diff --git a/x-pack/plugins/alerting/common/routes/gaps/response/schemas/v1.ts b/x-pack/plugins/alerting/common/routes/gaps/response/schemas/v1.ts new file mode 100644 index 0000000000000..33077f264a1a9 --- /dev/null +++ b/x-pack/plugins/alerting/common/routes/gaps/response/schemas/v1.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { schema } from '@kbn/config-schema'; + +import { gapStatus } from '../../../../constants'; + +export const gapStatusSchema = schema.oneOf([ + schema.literal(gapStatus.UNFILLED), + schema.literal(gapStatus.FILLED), + schema.literal(gapStatus.PARTIALLY_FILLED), +]); + +export const rangeSchema = schema.object({ + lte: schema.string(), + gte: schema.string(), +}); + +export const rangeListSchema = schema.arrayOf(rangeSchema); + +export const gapsResponseSchema = schema.object({ + _id: schema.string(), + status: gapStatusSchema, + range: rangeSchema, + in_progress_intervals: rangeListSchema, + filled_intervals: rangeListSchema, + unfilled_intervals: rangeListSchema, + total_gap_duration_ms: schema.number(), + filled_duration_ms: schema.number(), + unfilled_duration_ms: schema.number(), + in_progress_duration_ms: schema.number(), +}); + +export const errorResponseSchema = schema.object({ + error: schema.object({ + message: schema.string(), + status: schema.maybe(schema.number()), + }), +}); diff --git a/x-pack/plugins/alerting/common/routes/gaps/response/types/latest.ts b/x-pack/plugins/alerting/common/routes/gaps/response/types/latest.ts new file mode 100644 index 0000000000000..25300c97a6d2e --- /dev/null +++ b/x-pack/plugins/alerting/common/routes/gaps/response/types/latest.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './v1'; diff --git a/x-pack/plugins/alerting/common/routes/gaps/response/types/v1.ts b/x-pack/plugins/alerting/common/routes/gaps/response/types/v1.ts new file mode 100644 index 0000000000000..3b538a97b0136 --- /dev/null +++ b/x-pack/plugins/alerting/common/routes/gaps/response/types/v1.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { TypeOf } from '@kbn/config-schema'; +import { gapsResponseSchemaV1, errorResponseSchemaV1 } from '..'; + +export type GapsResponse = TypeOf; +export type ErrorResponse = TypeOf; diff --git a/x-pack/plugins/alerting/server/application/backfill/methods/delete/delete_backfill.ts b/x-pack/plugins/alerting/server/application/backfill/methods/delete/delete_backfill.ts index d223f944305c7..f565654eb5fd7 100644 --- a/x-pack/plugins/alerting/server/application/backfill/methods/delete/delete_backfill.ts +++ b/x-pack/plugins/alerting/server/application/backfill/methods/delete/delete_backfill.ts @@ -15,6 +15,8 @@ import { AdHocRunAuditAction, adHocRunAuditEvent, } from '../../../../rules_client/common/audit_events'; +import { transformAdHocRunToBackfillResult } from '../../transforms'; +import { calculateInProgressIntervalsForGaps } from '../../../../lib/rule_gaps/update/caclculate_in_progress_intervals_for_gaps'; export async function deleteBackfill(context: RulesClientContext, id: string): Promise<{}> { return await retryIfConflicts( @@ -82,12 +84,36 @@ async function deleteWithOCC(context: RulesClientContext, { id }: { id: string } }) ); + const backfill = await context.unsecuredSavedObjectsClient.get( + AD_HOC_RUN_SAVED_OBJECT_TYPE, + id + ); + + const backfillResult = transformAdHocRunToBackfillResult(backfill); + // delete the saved object const removeResult = await context.unsecuredSavedObjectsClient.delete( AD_HOC_RUN_SAVED_OBJECT_TYPE, - id + id, + { + refresh: 'wait_for', + } ); + if ('rule' in backfillResult) { + const eventLogClient = await context.getEventLogClient(); + + await calculateInProgressIntervalsForGaps({ + ruleId: backfillResult.rule.id, + start: new Date(backfillResult.start), + end: backfillResult.end ? new Date(backfillResult.end) : new Date(), + savedObjectsRepository: context.internalSavedObjectsRepository, + logger: context.logger, + eventLogClient, + eventLogger: context.eventLogger, + }); + } + // remove the associated task await context.taskManager.removeIfExists(id); diff --git a/x-pack/plugins/alerting/server/application/rule/methods/fill_gap_by_id/fill_gap_by_id.ts b/x-pack/plugins/alerting/server/application/rule/methods/fill_gap_by_id/fill_gap_by_id.ts new file mode 100644 index 0000000000000..f506a5c45e273 --- /dev/null +++ b/x-pack/plugins/alerting/server/application/rule/methods/fill_gap_by_id/fill_gap_by_id.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import Boom from '@hapi/boom'; + +import { RulesClientContext } from '../../../../rules_client'; + +import { findGapById as _findGapById } from '../../../../lib/rule_gaps/find_gap_by_id'; +import { FindGapByIdParams } from '../../../../lib/rule_gaps/types'; +import { scheduleBackfill } from '../../../backfill/methods/schedule'; + +export async function fillGapById(context: RulesClientContext, params: FindGapByIdParams) { + try { + const eventLogClient = await context.getEventLogClient(); + const gap = await _findGapById({ + params, + eventLogClient, + logger: context.logger, + }); + + if (!gap) { + throw Boom.notFound(`Gap not found for ruleId ${params.ruleId} and gapId ${params.gapId}`); + } + + const gapState = gap.getState(); + + const allGapsScheduled = + gapState.unfilledIntervals.map((interval) => ({ + ruleId: params.ruleId, + start: interval.gte, + end: interval.lte, + })) ?? []; + + return scheduleBackfill(context, allGapsScheduled); + } catch (err) { + const errorMessage = `Failed to find gap and schedule manual rule run`; + context.logger.error(`${errorMessage} - ${err}`); + throw Boom.boomify(err, { message: errorMessage }); + } +} diff --git a/x-pack/plugins/alerting/server/application/rule/methods/fill_gap_by_id/index.ts b/x-pack/plugins/alerting/server/application/rule/methods/fill_gap_by_id/index.ts new file mode 100644 index 0000000000000..51e0f74dd65c2 --- /dev/null +++ b/x-pack/plugins/alerting/server/application/rule/methods/fill_gap_by_id/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { fillGapById } from './fill_gap_by_id'; diff --git a/x-pack/plugins/alerting/server/application/rule/methods/find_gaps/find_gaps.ts b/x-pack/plugins/alerting/server/application/rule/methods/find_gaps/find_gaps.ts index d9aac3b8c975a..e7d0f43635a3b 100644 --- a/x-pack/plugins/alerting/server/application/rule/methods/find_gaps/find_gaps.ts +++ b/x-pack/plugins/alerting/server/application/rule/methods/find_gaps/find_gaps.ts @@ -9,23 +9,18 @@ import Boom from '@hapi/boom'; import { RulesClientContext } from '../../../../rules_client'; -import { findGapsByRuleId } from '../../../../lib/rule_gaps/find_gaps'; - -interface FindGapsParams { - ruleId: string; -} +import { findGaps as _findGaps } from '../../../../lib/rule_gaps/find_gaps'; +import { FindGapsParams } from '../../../../lib/rule_gaps/types'; export async function findGaps(context: RulesClientContext, params: FindGapsParams) { try { const eventLogClient = await context.getEventLogClient(); - const gaps = await findGapsByRuleId({ - ruleId: params.ruleId, + const gaps = await _findGaps({ + params, eventLogClient, logger: context.logger, }); - // transform gaps - return gaps; } catch (err) { const errorMessage = `Failed to find gaps`; diff --git a/x-pack/plugins/alerting/server/application/rule/methods/find_gaps/index.ts b/x-pack/plugins/alerting/server/application/rule/methods/find_gaps/index.ts index 38a6e16dce35e..54703c4abdcad 100644 --- a/x-pack/plugins/alerting/server/application/rule/methods/find_gaps/index.ts +++ b/x-pack/plugins/alerting/server/application/rule/methods/find_gaps/index.ts @@ -1 +1,8 @@ -export { findGaps } from './find_gaps'; \ No newline at end of file +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { findGaps } from './find_gaps'; diff --git a/x-pack/plugins/alerting/server/application/rule/methods/get_rules_with_gaps/get_rules_with_gaps.ts b/x-pack/plugins/alerting/server/application/rule/methods/get_rules_with_gaps/get_rules_with_gaps.ts new file mode 100644 index 0000000000000..f6516470b6513 --- /dev/null +++ b/x-pack/plugins/alerting/server/application/rule/methods/get_rules_with_gaps/get_rules_with_gaps.ts @@ -0,0 +1,93 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import Boom from '@hapi/boom'; +import { KueryNode } from '@kbn/es-query'; +import { + AlertingAuthorizationEntity, + AlertingAuthorizationFilterType, +} from '../../../../authorization'; +import { RulesClientContext } from '../../../../rules_client'; +import { GetRulesWithGapsParams, GetRulesWithGapsResponse } from './types'; + +export const RULE_SAVED_OBJECT_TYPE = 'alert'; + +export async function getRulesWithGaps( + context: RulesClientContext, + params: GetRulesWithGapsParams +) { + try { + let authorizationTuple; + try { + authorizationTuple = await context.authorization.getFindAuthorizationFilter( + AlertingAuthorizationEntity.Alert, + { + type: AlertingAuthorizationFilterType.KQL, + fieldNames: { + ruleTypeId: 'kibana.alert.rule.rule_type_id', + consumer: 'kibana.alert.rule.consumer', + }, + } + ); + } catch (error) { + // context.auditLogger?.log( + // ruleAuditEvent({ + // action: RuleAuditAction.GET_GLOBAL_EXECUTION_KPI, + // error, + // }) + // ); + throw error; + } + + const { start, end, statuses } = params; + const eventLogClient = await context.getEventLogClient(); + + const filter = 'kibana.alert.rule.gap: *'; + const statusFilter = statuses + ?.map((status) => `kibana.alert.rule.gap.status:${status}`) + .join(' OR '); + + const aggs = await eventLogClient.aggregateEventsWithAuthFilter( + RULE_SAVED_OBJECT_TYPE, + authorizationTuple.filter as KueryNode, + { + start, + end, + filter: `${filter} ${statusFilter ? `AND (${statusFilter})` : ''}`, + aggs: { + unique_rule_ids: { + terms: { + field: 'rule.id', + size: 10000, + }, + }, + }, + } + ); + + interface UniqueRuleIdsAgg { + buckets: Array<{ key: string }>; + } + + const uniqueRuleIdsAgg = aggs.aggregations?.unique_rule_ids as UniqueRuleIdsAgg; + + const resultBuckets = uniqueRuleIdsAgg?.buckets ?? []; + + const ruleIds = resultBuckets.map((bucket) => bucket.key) ?? []; + + const result: GetRulesWithGapsResponse = { + total: ruleIds?.length, + ruleIds, + }; + + return result; + } catch (err) { + const errorMessage = `Failed to find rules with gaps`; + context.logger.error(`${errorMessage} - ${err}`); + throw Boom.boomify(err, { message: errorMessage }); + } +} diff --git a/x-pack/plugins/alerting/server/application/rule/methods/get_rules_with_gaps/index.ts b/x-pack/plugins/alerting/server/application/rule/methods/get_rules_with_gaps/index.ts new file mode 100644 index 0000000000000..6130adad1b7a8 --- /dev/null +++ b/x-pack/plugins/alerting/server/application/rule/methods/get_rules_with_gaps/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { getRulesWithGaps } from './get_rules_with_gaps'; diff --git a/x-pack/plugins/alerting/server/application/rule/methods/get_rules_with_gaps/schemas/get_rules_with_gaps.ts b/x-pack/plugins/alerting/server/application/rule/methods/get_rules_with_gaps/schemas/get_rules_with_gaps.ts new file mode 100644 index 0000000000000..995826f6b11e4 --- /dev/null +++ b/x-pack/plugins/alerting/server/application/rule/methods/get_rules_with_gaps/schemas/get_rules_with_gaps.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { schema } from '@kbn/config-schema'; + +export const getRulesWithGapsParamsSchema = schema.object({ + start: schema.string(), + end: schema.string(), + statuses: schema.maybe(schema.arrayOf(schema.string())), +}); + +export const getRulesWithGapsResponseSchema = schema.object({ + total: schema.number(), + ruleIds: schema.arrayOf(schema.string()), +}); diff --git a/x-pack/plugins/alerting/server/application/rule/methods/get_rules_with_gaps/schemas/index.ts b/x-pack/plugins/alerting/server/application/rule/methods/get_rules_with_gaps/schemas/index.ts new file mode 100644 index 0000000000000..2b3b33f812c16 --- /dev/null +++ b/x-pack/plugins/alerting/server/application/rule/methods/get_rules_with_gaps/schemas/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './get_rules_with_gaps'; diff --git a/x-pack/plugins/alerting/server/application/rule/methods/get_rules_with_gaps/types/get_rule_params.ts b/x-pack/plugins/alerting/server/application/rule/methods/get_rules_with_gaps/types/get_rule_params.ts new file mode 100644 index 0000000000000..e801681e19764 --- /dev/null +++ b/x-pack/plugins/alerting/server/application/rule/methods/get_rules_with_gaps/types/get_rule_params.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { TypeOf } from '@kbn/config-schema'; +import { getRulesWithGapsParamsSchema, getRulesWithGapsResponseSchema } from '../schemas'; + +export type GetRulesWithGapsParams = TypeOf; +export type GetRulesWithGapsResponse = TypeOf; diff --git a/x-pack/plugins/alerting/server/application/rule/methods/get_rules_with_gaps/types/index.ts b/x-pack/plugins/alerting/server/application/rule/methods/get_rules_with_gaps/types/index.ts new file mode 100644 index 0000000000000..553499a994ff6 --- /dev/null +++ b/x-pack/plugins/alerting/server/application/rule/methods/get_rules_with_gaps/types/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './get_rule_params'; diff --git a/x-pack/plugins/alerting/server/backfill_client/backfill_client.ts b/x-pack/plugins/alerting/server/backfill_client/backfill_client.ts index b7e4e6667ffc7..be3abe91bd014 100644 --- a/x-pack/plugins/alerting/server/backfill_client/backfill_client.ts +++ b/x-pack/plugins/alerting/server/backfill_client/backfill_client.ts @@ -6,6 +6,7 @@ */ import { + ISavedObjectsRepository, Logger, SavedObject, SavedObjectReference, @@ -22,6 +23,7 @@ import { TaskManagerStartContract, TaskPriority, } from '@kbn/task-manager-plugin/server'; +import { IEventLogger, IEventLogClient } from '@kbn/event-log-plugin/server'; import { isNumber } from 'lodash'; import { ScheduleBackfillError, @@ -42,7 +44,7 @@ import { AD_HOC_RUN_SAVED_OBJECT_TYPE, RULE_SAVED_OBJECT_TYPE } from '../saved_o import { TaskRunnerFactory } from '../task_runner'; import { RuleTypeRegistry } from '../types'; import { createBackfillError } from './lib'; -import { updateGaps } from '../lib/rule_gaps/update_gaps'; +import { addInProgressIntervalsToGaps } from '../lib/rule_gaps/update/add_in_progress_intervals_to_gaps'; export const BACKFILL_TASK_TYPE = 'ad_hoc_run-backfill'; @@ -60,6 +62,9 @@ interface BulkQueueOpts { ruleTypeRegistry: RuleTypeRegistry; spaceId: string; unsecuredSavedObjectsClient: SavedObjectsClientContract; + eventLogClient: IEventLogClient; + internalSavedObjectsRepository: ISavedObjectsRepository; + eventLogger: IEventLogger | undefined; } interface DeleteBackfillForRulesOpts { @@ -234,27 +239,17 @@ export class BackfillClient { } }); - if (adHocTasksToSchedule.length > 0) { - const taskManager = await this.taskManagerStartPromise; - await taskManager.bulkSchedule(adHocTasksToSchedule); - } - try { // TODO: make it parallel - - backfullsToSchedule.forEach((backfill) => { - console.log('backfill client update', JSON.stringify(backfill)); - updateGaps({ - ruleId: backfill.rule.id, - start: new Date(backfill.start), - end: backfill?.end ? new Date(backfill.end) : new Date(), + for (const backfill of backfullsToSchedule) { + await addInProgressIntervalsToGaps({ + backfill, eventLogger, eventLogClient, - savedObjectsClient: internalSavedObjectsRepository, + savedObjectsRepository: internalSavedObjectsRepository, logger: this.logger, - needToFillGaps: false, }); - }); + } } catch { this.logger.warn( `Error updating gaps ƒor backfill jobs: ${backfullsToSchedule @@ -263,6 +258,11 @@ export class BackfillClient { ); } + if (adHocTasksToSchedule.length > 0) { + const taskManager = await this.taskManagerStartPromise; + await taskManager.bulkSchedule(adHocTasksToSchedule); + } + return createSOResult; } diff --git a/x-pack/plugins/alerting/server/lib/alerting_event_logger/alerting_event_logger.ts b/x-pack/plugins/alerting/server/lib/alerting_event_logger/alerting_event_logger.ts index fef8b6441b45d..45cfce1e828ed 100644 --- a/x-pack/plugins/alerting/server/lib/alerting_event_logger/alerting_event_logger.ts +++ b/x-pack/plugins/alerting/server/lib/alerting_event_logger/alerting_event_logger.ts @@ -10,6 +10,7 @@ import { IEventLogger, millisToNanos, SAVED_OBJECT_REL_PRIMARY, + DocMeta, } from '@kbn/event-log-plugin/server'; import { EVENT_LOG_ACTIONS } from '../../plugin'; import { UntypedNormalizedRuleType } from '../../rule_type_registry'; @@ -18,6 +19,8 @@ import { TaskRunnerTimings } from '../../task_runner/task_runner_timer'; import { AlertInstanceState, RuleExecutionStatus } from '../../types'; import { createAlertEventLogRecordObject } from '../create_alert_event_log_record_object'; import { RuleRunMetrics } from '../rule_run_metrics_store'; +import { Gap } from '../rule_gaps/gap'; +import { GapBase } from '../rule_gaps/types'; // 1,000,000 nanoseconds in 1 millisecond const Millis2Nanos = 1000 * 1000; @@ -409,16 +412,21 @@ export class AlertingEventLogger { throw new Error('AlertingEventLogger not initialized'); } + const gapToReport = new Gap({ + range: gap, + }); + this.eventLogger.logEvent( - createGapRecord(this.context, this.ruleData, this.relatedSavedObjects, { - status: 'unfilled', - range: gap, - }) + createGapRecord( + this.context, + this.ruleData, + this.relatedSavedObjects, + gapToReport.getEsObject() + ) ); } - public async updateGap({ meta, gap }: { meta: {}; gap: {} }): Promise { - // TODO: Extract function to form gap document + public async updateGap({ meta, gap }: { meta: DocMeta; gap: GapBase }): Promise { return this.eventLogger.updateEvent(meta, { kibana: { alert: { diff --git a/x-pack/plugins/alerting/server/lib/rule_gaps/find_gap_by_id.ts b/x-pack/plugins/alerting/server/lib/rule_gaps/find_gap_by_id.ts new file mode 100644 index 0000000000000..85b9864f91589 --- /dev/null +++ b/x-pack/plugins/alerting/server/lib/rule_gaps/find_gap_by_id.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IEventLogClient } from '@kbn/event-log-plugin/server'; +import { Logger } from '@kbn/core/server'; +import { RULE_SAVED_OBJECT_TYPE } from '../../saved_objects'; +import { FindGapByIdParams } from './types'; +import { Gap } from './gap'; +import { transformToGap } from './transforms/transformToGap'; + +export const findGapById = async ({ + eventLogClient, + logger, + params, +}: { + eventLogClient: IEventLogClient; + logger: Logger; + params: FindGapByIdParams; +}): Promise => { + const { gapId, ruleId } = params; + try { + const gapsResponse = await eventLogClient.findEventsBySavedObjectIds( + RULE_SAVED_OBJECT_TYPE, + [ruleId], + { + filter: `_id: ${gapId}`, + } + ); + + if (gapsResponse.total === 0) return null; + + const gap = transformToGap(gapsResponse)[0]; + + return gap; + } catch (err) { + logger.error(`Failed to find gap by id ${gapId} for rule ${ruleId.toString()}: ${err.message}`); + throw err; + } +}; diff --git a/x-pack/plugins/alerting/server/lib/rule_gaps/find_gaps.ts b/x-pack/plugins/alerting/server/lib/rule_gaps/find_gaps.ts index 6bb983f87e48b..315625ad03997 100644 --- a/x-pack/plugins/alerting/server/lib/rule_gaps/find_gaps.ts +++ b/x-pack/plugins/alerting/server/lib/rule_gaps/find_gaps.ts @@ -27,16 +27,20 @@ export const findGaps = async ({ page: number; perPage: number; }> => { - const { ruleIds, start, end, page, perPage, statuses } = params; + const { ruleId, start, end, page, perPage, statuses } = params; try { const statusesFilter = statuses ?.map((status) => `kibana.alert.rule.gap.status : ${status}`) .join(' OR '); + const rangeFilter = + end && start + ? `AND (kibana.alert.rule.gap.range <= "${end}" AND kibana.alert.rule.gap.range >= "${start}")` + : ''; const gapsResponse = await eventLogClient.findEventsBySavedObjectIds( RULE_SAVED_OBJECT_TYPE, - ruleIds, + [ruleId], { - filter: `event.action: gap AND event.provider: alerting AND (kibana.alert.rule.gap.range <= "${end}" AND kibana.alert.rule.gap.range >= "${start}") ${ + filter: `event.action: gap AND event.provider: alerting ${rangeFilter} ${ statusesFilter ? `AND (${statusesFilter})` : '' }`, sort: [{ sort_field: '@timestamp', sort_order: 'desc' }], @@ -52,7 +56,7 @@ export const findGaps = async ({ perPage: gapsResponse.per_page, }; } catch (err) { - logger.error(`Failed to find gaps for rule ${ruleIds.toString()}: ${err.message}`); + logger.error(`Failed to find gaps for rule ${ruleId.toString()}: ${err.message}`); throw err; } }; @@ -65,13 +69,13 @@ export const findAllGaps = async ({ eventLogClient: IEventLogClient; logger: Logger; params: { - ruleIds: string[]; + ruleId: string; start: Date; end: Date; statuses?: GapStatus[]; }; }): Promise => { - const { ruleIds, start, end, statuses } = params; + const { ruleId, start, end, statuses } = params; const allGaps: Gap[] = []; let currentPage = 1; const perPage = 10000; @@ -81,7 +85,7 @@ export const findAllGaps = async ({ eventLogClient, logger, params: { - ruleIds, + ruleId, start: start.toISOString(), end: end.toISOString(), page: currentPage, diff --git a/x-pack/plugins/alerting/server/lib/rule_gaps/gap/index.test.ts b/x-pack/plugins/alerting/server/lib/rule_gaps/gap/index.test.ts new file mode 100644 index 0000000000000..5a93b8146b44b --- /dev/null +++ b/x-pack/plugins/alerting/server/lib/rule_gaps/gap/index.test.ts @@ -0,0 +1,218 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Gap } from '.'; // Adjust this import path as needed +import { gapStatus } from '../../../../common/constants'; // Adjust as needed +import { Interval, StringInterval } from '../types'; // Adjust as needed + +// Helper function to create Interval objects from ISO strings +function toInterval(gte: string, lte: string): Interval { + return { + gte: new Date(gte), + lte: new Date(lte), + }; +} + +describe('Gap Class Tests', () => { + const HOUR_MS = 60 * 60 * 1000; + const baseRange: StringInterval = { + gte: '2024-01-01T00:00:00.000Z', + lte: '2024-01-01T01:00:00.000Z', + }; + + it('initializes with no filled or in-progress intervals', () => { + const gap = new Gap({ range: baseRange }); + expect(gap.status).toBe(gapStatus.UNFILLED); + expect(gap.filledIntervals).toHaveLength(0); + expect(gap.inProgressIntervals).toHaveLength(0); + expect(gap.unfilledIntervals).toHaveLength(1); + expect(gap.unfilledIntervals[0].gte.getTime()).toBe(new Date(baseRange.gte).getTime()); + expect(gap.unfilledIntervals[0].lte.getTime()).toBe(new Date(baseRange.lte).getTime()); + expect(gap.totalGapDurationMs).toBe(HOUR_MS); + expect(gap.unfilledGapDurationMs).toBe(HOUR_MS); + }); + + it('initializes fully filled gap', () => { + const gap = new Gap({ range: baseRange, filledIntervals: [baseRange] }); + expect(gap.filledIntervals).toHaveLength(1); + expect(gap.filledGapDurationMs).toBe(HOUR_MS); + expect(gap.unfilledGapDurationMs).toBe(0); + expect(gap.inProgressIntervals).toHaveLength(0); + expect(gap.status).toBe(gapStatus.FILLED); + }); + + it('initializes partially filled', () => { + const partialFill: StringInterval = { + gte: '2024-01-01T00:15:00.000Z', + lte: '2024-01-01T00:30:00.000Z', + }; + const gap = new Gap({ range: baseRange, filledIntervals: [partialFill] }); + const filledDuration = (30 - 15) * 60 * 1000; // 15 min + + expect(gap.filledGapDurationMs).toBe(filledDuration); + expect(gap.unfilledGapDurationMs).toBe(HOUR_MS - filledDuration); + expect(gap.inProgressGapDurationMs).toBe(0); + expect(gap.status).toBe(gapStatus.PARTIALLY_FILLED); + }); + + it('initializes with in-progress intervals only', () => { + const inProgress: StringInterval = { + gte: '2024-01-01T00:40:00.000Z', + lte: '2024-01-01T00:50:00.000Z', + }; + const gap = new Gap({ range: baseRange, inProgressIntervals: [inProgress] }); + + const inProgressDuration = (50 - 40) * 60 * 1000; // 10 min + expect(gap.inProgressGapDurationMs).toBe(inProgressDuration); + expect(gap.filledGapDurationMs).toBe(0); + expect(gap.unfilledGapDurationMs).toBe(HOUR_MS - inProgressDuration); + expect(gap.status).toBe(gapStatus.PARTIALLY_FILLED); + }); + + it('handles intervals that extend beyond the range', () => { + const extendedInterval: Interval = { + gte: new Date('2023-12-31T23:50:00.000Z'), + lte: new Date('2024-01-01T01:10:00.000Z'), + }; + const gap = new Gap({ range: baseRange }); + gap.fillGap(extendedInterval); + // Extended interval should effectively fill only the baseRange + expect(gap.filledGapDurationMs).toBe(HOUR_MS); + expect(gap.status).toBe(gapStatus.FILLED); + }); + + it('filling the gap after initialization updates the state', () => { + const gap = new Gap({ range: baseRange }); + const oneMinute = toInterval('2024-01-01T00:00:00.000Z', '2024-01-01T00:01:00.000Z'); + + gap.fillGap(oneMinute); + expect(gap.filledGapDurationMs).toBe(60 * 1000); + expect(gap.unfilledGapDurationMs).toBe(HOUR_MS - 60 * 1000); + expect(gap.status).toBe(gapStatus.PARTIALLY_FILLED); + }); + + it('addInProgress intervals do not overlap with already filled', () => { + const filled: StringInterval = { + gte: '2024-01-01T00:00:00.000Z', + lte: '2024-01-01T00:10:00.000Z', + }; + const gap = new Gap({ range: baseRange, filledIntervals: [filled] }); + const filledDuration = 10 * 60 * 1000; + + // Add in-progress that overlaps from 00:05 - 00:15 + const inProgressOverlap = toInterval('2024-01-01T00:05:00.000Z', '2024-01-01T00:15:00.000Z'); + gap.addInProgress(inProgressOverlap); + + // Should adjust to only non-filled portion: 00:10 - 00:15 (5 min) + expect(gap.inProgressIntervals).toHaveLength(1); + expect(gap.inProgressGapDurationMs).toBe(5 * 60 * 1000); + expect(gap.filledGapDurationMs).toBe(filledDuration); + expect(gap.status).toBe(gapStatus.PARTIALLY_FILLED); + }); + + it('subsequent fill operations remove portions of in-progress intervals', () => { + const gap = new Gap({ range: baseRange }); + // Add in-progress for 00:20 - 00:30 + gap.addInProgress(toInterval('2024-01-01T00:20:00.000Z', '2024-01-01T00:30:00.000Z')); + expect(gap.inProgressGapDurationMs).toBe(10 * 60 * 1000); + + // Fill 5 min of that (00:25 - 00:30) + gap.fillGap(toInterval('2024-01-01T00:25:00.000Z', '2024-01-01T00:30:00.000Z')); + expect(gap.filledGapDurationMs).toBe(5 * 60 * 1000); + // In-progress now only 00:20 - 00:25 + expect(gap.inProgressGapDurationMs).toBe(5 * 60 * 1000); + expect(gap.status).toBe(gapStatus.PARTIALLY_FILLED); + }); + + it('filling entire gap in multiple steps leads to FILLED status', () => { + const gap = new Gap({ range: baseRange }); + + // Fill first half + gap.fillGap(toInterval('2024-01-01T00:00:00.000Z', '2024-01-01T00:30:00.000Z')); + expect(gap.status).toBe(gapStatus.PARTIALLY_FILLED); + + // Fill second half + gap.fillGap(toInterval('2024-01-01T00:30:00.000Z', '2024-01-01T01:00:00.000Z')); + expect(gap.status).toBe(gapStatus.FILLED); + expect(gap.filledGapDurationMs).toBe(HOUR_MS); + }); + + it('returns correct state via getState()', () => { + const filled: StringInterval = { + gte: '2024-01-01T00:10:00.000Z', + lte: '2024-01-01T00:20:00.000Z', + }; + const inProgress: StringInterval = { + gte: '2024-01-01T00:20:00.000Z', + lte: '2024-01-01T00:30:00.000Z', + }; + const gap = new Gap({ + range: baseRange, + filledIntervals: [filled], + inProgressIntervals: [inProgress], + }); + + const state = gap.getState(); + expect(state.range).toEqual(baseRange); + expect(state.filledIntervals).toHaveLength(1); + expect(state.inProgressIntervals).toHaveLength(1); + expect(state.unfilledIntervals).not.toHaveLength(0); + expect(state.status).toBe(gapStatus.PARTIALLY_FILLED); + expect(state.totalGapDurationMs).toBe(HOUR_MS); + }); + + it('returns correct ES object via getEsObject()', () => { + const filled: StringInterval = { + gte: '2024-01-01T00:10:00.000Z', + lte: '2024-01-01T00:20:00.000Z', + }; + const gap = new Gap({ range: baseRange, filledIntervals: [filled] }); + + const esObject = gap.getEsObject(); + expect(esObject.range).toEqual(baseRange); + expect(esObject.filled_intervals).toHaveLength(1); + expect(esObject.in_progress_intervals).toHaveLength(0); + expect(esObject.unfilled_intervals).toHaveLength(2); + expect(esObject.status).toBe(gapStatus.PARTIALLY_FILLED); + expect(esObject.total_gap_duration_ms).toBe(HOUR_MS); + expect(esObject.filled_duration_ms).toBe(10 * 60 * 1000); + }); + + it('returns correct ES object via getEsObject() after filling', () => { + const gap = new Gap({ range: baseRange }); + + gap.addInProgress({ + gte: new Date('2024-01-01T00:10:00.000Z'), + lte: new Date('2024-01-01T00:20:00.000Z'), + }); + gap.fillGap({ + gte: new Date('2024-01-01T00:40:00.000Z'), + lte: new Date('2024-01-01T00:50:00.000Z'), + }); + + const esObject = gap.getEsObject(); + expect(esObject.range).toEqual(baseRange); + expect(esObject.filled_intervals).toHaveLength(1); + expect(esObject.in_progress_intervals).toHaveLength(1); + expect(esObject.unfilled_intervals).toHaveLength(3); + expect(esObject.status).toBe(gapStatus.PARTIALLY_FILLED); + expect(esObject.total_gap_duration_ms).toBe(HOUR_MS); + expect(esObject.in_progress_duration_ms).toBe(10 * 60 * 1000); + expect(esObject.filled_duration_ms).toBe(10 * 60 * 1000); + }); + + it('handles meta information if provided', () => { + const meta = { + _id: 'test_id', + _index: 'test_index', + _seq_no: '1', + _primary_term: '1', + }; + const gap = new Gap({ range: baseRange, meta }); + expect(gap.meta).toEqual(meta); + }); +}); diff --git a/x-pack/plugins/alerting/server/lib/rule_gaps/gap.ts b/x-pack/plugins/alerting/server/lib/rule_gaps/gap/index.ts similarity index 73% rename from x-pack/plugins/alerting/server/lib/rule_gaps/gap.ts rename to x-pack/plugins/alerting/server/lib/rule_gaps/gap/index.ts index 3ccff1fa308db..b946e72dedb78 100644 --- a/x-pack/plugins/alerting/server/lib/rule_gaps/gap.ts +++ b/x-pack/plugins/alerting/server/lib/rule_gaps/gap/index.ts @@ -4,9 +4,10 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { GapStatus, gapStatus } from '../../../common/constants'; +import { DocMeta } from '@kbn/event-log-plugin/server/es/cluster_client_adapter'; +import { GapStatus, gapStatus } from '../../../../common/constants'; -import { Gap as GapType, Interval, StringInterval } from './types'; +import { Interval, StringInterval, GapBase } from '../types'; import { mergeIntervals, @@ -16,30 +17,21 @@ import { subtractIntervals, normalizeInterval, denormalizeInterval, -} from './utils/intervals'; + clipInterval, +} from './interval_utils'; interface GapConstructorParams { range: StringInterval; filledIntervals?: StringInterval[]; inProgressIntervals?: StringInterval[]; - meta?: { - _id: string; - _index: string; - _seq_no: string; - _primary_term: string; - }; + meta?: DocMeta; } export class Gap { private _range: Interval; private _filledIntervals: Interval[]; private _inProgressIntervals: Interval[]; - private _meta?: { - _id: string; - _index: string; - _seq_no: string; - _primary_term: string; - }; + private _meta?: DocMeta; constructor({ range, @@ -56,13 +48,27 @@ export class Gap { } public fillGap(interval: Interval): void { - this._filledIntervals = mergeIntervals([...this.filledIntervals, interval]); + const clipedInterval = clipInterval(interval, this.range); + if (clipedInterval === null) return; + + const newFilledIntervals = mergeIntervals([...this.filledIntervals, clipedInterval]); + this._filledIntervals = newFilledIntervals; + const newInProgressIntervals = subtractAllIntervals( + this.inProgressIntervals, + newFilledIntervals + ); + this._inProgressIntervals = newInProgressIntervals; } - public addInProgress(interval: StringInterval): void { - const normalized = normalizeInterval(interval); - const sanitized = subtractAllIntervals([normalized], this.filledIntervals); - this._inProgressIntervals = mergeIntervals([...this.inProgressIntervals, ...sanitized]); + public addInProgress(interval: Interval): void { + const clipedInterval = clipInterval(interval, this.range); + if (clipedInterval === null) return; + + const inProgressIntervals = subtractAllIntervals([clipedInterval], this.filledIntervals); + this._inProgressIntervals = mergeIntervals([ + ...this.inProgressIntervals, + ...inProgressIntervals, + ]); } public get range() { @@ -111,11 +117,15 @@ export class Gap { } } + public resetInProgressIntervals(): void { + this._inProgressIntervals = []; + } + public get meta() { return this._meta; } - public getState(): GapType { + public getState() { return { range: denormalizeInterval(this.range), filledIntervals: this.filledIntervals.map(denormalizeInterval), @@ -132,7 +142,7 @@ export class Gap { /** * Returns the gap object for es */ - public getEsObject() { + public getEsObject(): GapBase { return { range: denormalizeInterval(this.range), filled_intervals: this.filledIntervals.map(denormalizeInterval), diff --git a/x-pack/plugins/alerting/server/lib/rule_gaps/utils/intervals.ts b/x-pack/plugins/alerting/server/lib/rule_gaps/gap/interval_utils.ts similarity index 90% rename from x-pack/plugins/alerting/server/lib/rule_gaps/utils/intervals.ts rename to x-pack/plugins/alerting/server/lib/rule_gaps/gap/interval_utils.ts index 7ac8a60d05071..fc80efc26a9fa 100644 --- a/x-pack/plugins/alerting/server/lib/rule_gaps/utils/intervals.ts +++ b/x-pack/plugins/alerting/server/lib/rule_gaps/gap/interval_utils.ts @@ -24,7 +24,7 @@ import { Interval, StringInterval } from '../types'; export const getOverlap = (interval1: Interval, interval2: Interval): Interval | null => { const start = new Date(Math.max(interval1.gte.getTime(), interval2.gte.getTime())); const end = new Date(Math.min(interval1.lte.getTime(), interval2.lte.getTime())); - return start < end ? { gte: start, lte: end } : null; + return start <= end ? { gte: start, lte: end } : null; }; /** @@ -45,7 +45,9 @@ export const getOverlap = (interval1: Interval, interval2: Interval): Interval | export const mergeIntervals = (intervals: Interval[]): Interval[] => { if (!intervals.length) return []; - const sorted = [...intervals].sort((a, b) => a.gte.getTime() - b.gte.getTime()); + const sorted = intervals + .map((interval) => ({ ...interval })) + .sort((a, b) => a.gte.getTime() - b.gte.getTime()); return sorted.reduce((merged, current, index) => { if (index === 0) { @@ -201,3 +203,23 @@ export const denormalizeInterval = (interval: Interval): StringInterval => { lte: interval.lte.toISOString(), }; }; + +/** + * Clips an interval to ensure it falls within the given boundary range. + * If there's no overlap, returns null. + */ +export const clipInterval = (interval: Interval, boundary: Interval): Interval | null => { + const gte = interval.gte.getTime(); + const lte = interval.lte.getTime(); + const boundaryGte = boundary.gte.getTime(); + const boundaryLte = boundary.lte.getTime(); + + const clippedGte = Math.max(gte, boundaryGte); + const clippedLte = Math.min(lte, boundaryLte); + + if (clippedGte >= clippedLte) { + return null; + } + + return { gte: new Date(clippedGte), lte: new Date(clippedLte) }; +}; diff --git a/x-pack/plugins/alerting/server/lib/rule_gaps/get_total_unfilled_gap_duration.ts b/x-pack/plugins/alerting/server/lib/rule_gaps/get_total_unfilled_gap_duration.ts deleted file mode 100644 index ce2950c41072c..0000000000000 --- a/x-pack/plugins/alerting/server/lib/rule_gaps/get_total_unfilled_gap_duration.ts +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { IEventLogger } from '@kbn/event-log-plugin/server'; -import { Logger } from '@kbn/core/server'; - -export async function getTotalUnfilledGapDuration(params: { - ruleId: string; - timeRange?: { from: string; to: string }; - eventLog: IEventLogger; - logger: Logger; -}): Promise<{ - unfiled_gap_duration_ms: { - '1d': number; - '3d': number; - '7d': number; - }; -}> { - const { ruleId, timeRange, eventLog, logger } = params; - - try { - const aggs = {}; // eventLog... - return { - unfiled_gap_duration_ms: { - '1d': 0, - '3d': 0, - '7d': 0, - }, - }; - } catch (err) { - logger.error(`Failed to find unffiled gap duration for rule ${ruleId}: ${err.message}`); - throw err; - } -} diff --git a/x-pack/plugins/alerting/server/lib/rule_gaps/schemas/index.ts b/x-pack/plugins/alerting/server/lib/rule_gaps/schemas/index.ts index 624ed7cb0cae6..62b4d5b210960 100644 --- a/x-pack/plugins/alerting/server/lib/rule_gaps/schemas/index.ts +++ b/x-pack/plugins/alerting/server/lib/rule_gaps/schemas/index.ts @@ -21,17 +21,16 @@ export const rangeSchema = schema.object({ export const rangeListSchema = schema.arrayOf(rangeSchema); -export const gapSchema = schema.object({ - _id: schema.maybe(schema.string()), +export const gapBaseSchema = schema.object({ status: gapStatusSchema, range: rangeSchema, - inProgressIntervals: rangeListSchema, - filledIntervals: rangeListSchema, - unfilledIntervals: rangeListSchema, - totalGapDurationMs: schema.number(), - filledDurationMs: schema.number(), - unfilledDurationMs: schema.number(), - inProgressDurationMs: schema.number(), + in_progress_intervals: rangeListSchema, + filled_intervals: rangeListSchema, + unfilled_intervals: rangeListSchema, + total_gap_duration_ms: schema.number(), + filled_duration_ms: schema.number(), + unfilled_duration_ms: schema.number(), + in_progress_duration_ms: schema.number(), }); export const findGapsParamsSchema = schema.object( @@ -39,7 +38,7 @@ export const findGapsParamsSchema = schema.object( end: schema.maybe(schema.string()), page: schema.number({ defaultValue: 1, min: 1 }), perPage: schema.number({ defaultValue: 10, min: 0 }), - ruleIds: schema.arrayOf(schema.string()), + ruleId: schema.string(), start: schema.maybe(schema.string()), sortField: schema.maybe(schema.oneOf([schema.literal('@timestamp')])), sortOrder: schema.maybe(schema.oneOf([schema.literal('asc'), schema.literal('desc')])), @@ -62,3 +61,8 @@ export const findGapsParamsSchema = schema.object( }, } ); + +export const findGapByIdParamsSchema = schema.object({ + gapId: schema.string(), + ruleId: schema.string(), +}); diff --git a/x-pack/plugins/alerting/server/lib/rule_gaps/types/index.ts b/x-pack/plugins/alerting/server/lib/rule_gaps/types/index.ts index 821c41267125b..53c837a5f37f0 100644 --- a/x-pack/plugins/alerting/server/lib/rule_gaps/types/index.ts +++ b/x-pack/plugins/alerting/server/lib/rule_gaps/types/index.ts @@ -6,10 +6,11 @@ */ import { TypeOf } from '@kbn/config-schema'; -import { gapSchema, findGapsParamsSchema } from '../schemas'; +import { gapBaseSchema, findGapsParamsSchema, findGapByIdParamsSchema } from '../schemas'; -export type Gap = TypeOf; +export type GapBase = TypeOf; export type FindGapsParams = TypeOf; +export type FindGapByIdParams = TypeOf; export interface Interval { gte: Date; diff --git a/x-pack/plugins/alerting/server/lib/rule_gaps/update/add_filled_interval_to_gaps.ts b/x-pack/plugins/alerting/server/lib/rule_gaps/update/add_filled_interval_to_gaps.ts new file mode 100644 index 0000000000000..9441cc3ac6873 --- /dev/null +++ b/x-pack/plugins/alerting/server/lib/rule_gaps/update/add_filled_interval_to_gaps.ts @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Logger } from '@kbn/core/server'; +import { IEventLogClient, IEventLogger } from '@kbn/event-log-plugin/server'; +import { AlertingEventLogger } from '../../alerting_event_logger/alerting_event_logger'; +import { findAllGaps } from '../find_gaps'; +import { gapStatus } from '../../../../common/constants'; + +/** + * Find all gaps for this date range and ruleId + * Then add the filled interval to the gaps + * Then update the gaps in the event log + */ +export const addFilledIntervalToGaps = async (params: { + ruleId: string; + start: Date; + end: Date; + eventLogger: IEventLogger; + eventLogClient: IEventLogClient; + logger: Logger; +}) => { + const { ruleId, start, end, logger, eventLogClient, eventLogger } = params; + + const alertingEventLogger = new AlertingEventLogger(eventLogger); + + try { + const allGaps = await findAllGaps({ + eventLogClient, + logger, + params: { + ruleId, + start, + end, + statuses: [gapStatus.PARTIALLY_FILLED, gapStatus.UNFILLED], + }, + }); + + for (const gap of allGaps) { + gap.fillGap({ + gte: start, + lte: end, + }); + + const esGap = gap.getEsObject(); + const meta = gap.meta; + + if (meta) { + try { + await alertingEventLogger.updateGap({ + meta, + gap: esGap, + }); + } catch (e) { + // TODO version mismatch -> + // refetch gap + // check status + // retry + + logger.error('Failed to fill gap, because of conflicting versions'); + } + } + } + } catch (err) { + logger.error( + `Failed to fill gaps for rule ${ruleId} from: ${start.toISOString()} to: ${end.toISOString()}: ${ + err.message + }` + ); + throw err; + } +}; diff --git a/x-pack/plugins/alerting/server/lib/rule_gaps/update/add_in_progress_intervals_to_gaps.ts b/x-pack/plugins/alerting/server/lib/rule_gaps/update/add_in_progress_intervals_to_gaps.ts new file mode 100644 index 0000000000000..cd3b530db8efc --- /dev/null +++ b/x-pack/plugins/alerting/server/lib/rule_gaps/update/add_in_progress_intervals_to_gaps.ts @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ISavedObjectsRepository, Logger } from '@kbn/core/server'; +import { IEventLogClient, IEventLogger } from '@kbn/event-log-plugin/server'; +import { AlertingEventLogger } from '../../alerting_event_logger/alerting_event_logger'; +import { findAllGaps } from '../find_gaps'; +import { adHocRunStatus, gapStatus } from '../../../../common/constants'; +import { Backfill } from '../../../application/backfill/result/types'; +import { parseDuration } from '../../../../common'; + +/** + * Find all gaps for this date range and ruleId + * Then add inProgressIntervals to the gaps + * Then update the gaps in the event log + */ +export const addInProgressIntervalsToGaps = async (params: { + backfill: Backfill; + eventLogger: IEventLogger | undefined; + eventLogClient: IEventLogClient; + savedObjectsRepository: ISavedObjectsRepository; + logger: Logger; +}) => { + const { backfill, logger, eventLogClient, eventLogger } = params; + + const ruleId = backfill.rule.id; + const start = new Date(backfill.start); + const end = backfill?.end ? new Date(backfill.end) : new Date(); + + try { + if (!eventLogger) { + throw new Error('Add in-progress intervals to gaps: Event logger is not defined'); + } + + const alertingEventLogger = new AlertingEventLogger(eventLogger); + const allGaps = await findAllGaps({ + eventLogClient, + logger, + params: { + ruleId, + start, + end, + statuses: [gapStatus.PARTIALLY_FILLED, gapStatus.UNFILLED], + }, + }); + + for (const gap of allGaps) { + // TODO: we shouldn't write gap into alertevent log if not real update + backfill.schedule.forEach((scheduleItem) => { + const runAt = new Date(scheduleItem.runAt).getTime(); + const intervalDuration = parseDuration(scheduleItem.interval); + const from = runAt - intervalDuration; + const to = runAt; + if ( + scheduleItem.status === adHocRunStatus.PENDING || + scheduleItem.status === adHocRunStatus.RUNNING + ) { + gap.addInProgress({ + gte: new Date(from), + lte: new Date(to), + }); + } + }); + + const esGap = gap.getEsObject(); + const meta = gap.meta; + + if (meta) { + try { + await alertingEventLogger.updateGap({ + meta, + gap: esGap, + }); + } catch (e) { + // TODO version mismatch -> + // refetch gap + // check status + // retry + + logger.error( + 'Failed to add in-progress intervals into gap, because of conflicting versions' + ); + } + } + } + } catch (err) { + logger.error( + `Failed to add in-progress for rule ${ruleId} from: ${start.toISOString()} to: ${end.toISOString()}: ${ + err.message + }` + ); + throw err; + } +}; diff --git a/x-pack/plugins/alerting/server/lib/rule_gaps/update_gaps.ts b/x-pack/plugins/alerting/server/lib/rule_gaps/update/caclculate_in_progress_intervals_for_gaps.ts similarity index 65% rename from x-pack/plugins/alerting/server/lib/rule_gaps/update_gaps.ts rename to x-pack/plugins/alerting/server/lib/rule_gaps/update/caclculate_in_progress_intervals_for_gaps.ts index f8f8b0536d34e..c6866f09ab253 100644 --- a/x-pack/plugins/alerting/server/lib/rule_gaps/update_gaps.ts +++ b/x-pack/plugins/alerting/server/lib/rule_gaps/update/caclculate_in_progress_intervals_for_gaps.ts @@ -7,29 +7,29 @@ import { ISavedObjectsRepository, Logger } from '@kbn/core/server'; import { IEventLogClient, IEventLogger } from '@kbn/event-log-plugin/server'; -import { AD_HOC_RUN_SAVED_OBJECT_TYPE, RULE_SAVED_OBJECT_TYPE } from '../../saved_objects'; -import { AlertingEventLogger } from '../alerting_event_logger/alerting_event_logger'; -import { findAllGaps } from './find_gaps'; -import { Gap } from './gap'; -import { AdHocRunSO } from '../../data/ad_hoc_run/types'; -import { adHocRunStatus, gapStatus } from '../../../common/constants'; -import { parseDuration } from '../../../common'; -import { transformAdHocRunToBackfillResult } from '../../application/backfill/transforms'; +import { AD_HOC_RUN_SAVED_OBJECT_TYPE, RULE_SAVED_OBJECT_TYPE } from '../../../saved_objects'; +import { AlertingEventLogger } from '../../alerting_event_logger/alerting_event_logger'; +import { findAllGaps } from '../find_gaps'; +import { AdHocRunSO } from '../../../data/ad_hoc_run/types'; +import { adHocRunStatus, gapStatus } from '../../../../common/constants'; +import { parseDuration } from '../../../../common'; +import { transformAdHocRunToBackfillResult } from '../../../application/backfill/transforms'; +import { Gap } from '../gap'; /** * Find all overlapping backfill tasks and update the gap status accordingly */ const updateGapStatus = async ({ gap, - savedObjectsClient, + savedObjectsRepository, ruleId, }: { gap: Gap; - savedObjectsClient: ISavedObjectsRepository; + savedObjectsRepository: ISavedObjectsRepository; ruleId: string; }): Promise => { // TODO: get all backfill, not first page - const { saved_objects: backfillSOs } = await savedObjectsClient.find({ + const { saved_objects: backfillSOs } = await savedObjectsRepository.find({ type: AD_HOC_RUN_SAVED_OBJECT_TYPE, hasReference: { type: RULE_SAVED_OBJECT_TYPE, @@ -47,6 +47,7 @@ const updateGapStatus = async ({ // TODO: Extract backfill transform to another function, to reuse in API const transformedBackfills = backfillSOs.map((data) => transformAdHocRunToBackfillResult(data)); + gap.resetInProgressIntervals(); for (const backfill of transformedBackfills) { if ('error' in backfill) { break; @@ -58,8 +59,8 @@ const updateGapStatus = async ({ const from = runAt - intervalDuration; const to = runAt; const scheduleInterval = { - gte: new Date(from).toISOString(), - lte: new Date(to).toISOString(), + gte: new Date(from), + lte: new Date(to), }; if ( scheduleItem.status === adHocRunStatus.PENDING || @@ -69,66 +70,51 @@ const updateGapStatus = async ({ } } } - return gap; }; -export async function updateGaps(params: { +/** + * Find all gaps for this date range and ruleId + * Then find all backfill tasks that overlap with these gaps + * Update the gap status accordingly, be reset and then calculate the new inProgressIntervals + */ +export async function calculateInProgressIntervalsForGaps(params: { ruleId: string; start: Date; end: Date; - eventLogger: IEventLogger; + eventLogger: IEventLogger | undefined; eventLogClient: IEventLogClient; - savedObjectsClient: ISavedObjectsRepository; + savedObjectsRepository: ISavedObjectsRepository; logger: Logger; - needToFillGaps: boolean; }): Promise { - const { - ruleId, - start, - end, - logger, - savedObjectsClient, - eventLogClient, - needToFillGaps, - eventLogger, - } = params; - - const alertingEventLogger = new AlertingEventLogger(eventLogger); + const { ruleId, start, end, logger, savedObjectsRepository, eventLogClient, eventLogger } = + params; try { + if (!eventLogger) { + throw new Error('Event logger is required'); + } + + const alertingEventLogger = new AlertingEventLogger(eventLogger); + const allGaps = await findAllGaps({ eventLogClient, logger, params: { - ruleIds: [ruleId], + ruleId, start, end, statuses: [gapStatus.PARTIALLY_FILLED, gapStatus.UNFILLED], }, }); - console.log('start/endn', start, end); - - console.log('needTofilleGap', needToFillGaps); - console.log('allGaps', allGaps.length); - console.log('gaps', JSON.stringify(allGaps)); for (const gap of allGaps) { - if (needToFillGaps) { - gap.fillGap({ - gte: start, - lte: end, - }); - } - - updateGapStatus({ + await updateGapStatus({ gap, - savedObjectsClient, + savedObjectsRepository, ruleId, }); - console.log(gap); - const esGap = gap.getEsObject(); const meta = gap.meta; @@ -139,7 +125,7 @@ export async function updateGaps(params: { gap: esGap, }); } catch (e) { - // TODO version mismatch -> + // TODO version mismatch -> // refetch gap // check status // retry diff --git a/x-pack/plugins/alerting/server/plugin.ts b/x-pack/plugins/alerting/server/plugin.ts index f198f2159212b..341e88fe8f84c 100644 --- a/x-pack/plugins/alerting/server/plugin.ts +++ b/x-pack/plugins/alerting/server/plugin.ts @@ -630,7 +630,7 @@ export class AlertingPlugin { supportsEphemeralTasks: plugins.taskManager.supportsEphemeralTasks(), uiSettings: core.uiSettings, usageCounter: this.usageCounter, - getEventLogClient: (spaceId: string) => plugins.eventLog.getClient(spaceId), + getEventLogClient: (request: KibanaRequest) => plugins.eventLog.getClient(request), }); this.eventLogService!.registerSavedObjectProvider(RULE_SAVED_OBJECT_TYPE, (request) => { diff --git a/x-pack/plugins/alerting/server/routes/gaps/apis/fill/fill_gap_by_id_route.ts b/x-pack/plugins/alerting/server/routes/gaps/apis/fill/fill_gap_by_id_route.ts new file mode 100644 index 0000000000000..8171841c2817b --- /dev/null +++ b/x-pack/plugins/alerting/server/routes/gaps/apis/fill/fill_gap_by_id_route.ts @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { IRouter } from '@kbn/core/server'; +import { + fillGapByIdQuerySchemaV1, + FillGapByIdQueryV1, +} from '../../../../../common/routes/gaps/apis/fill'; +import { ScheduleBackfillResponseV1 } from '../../../../../common/routes/backfill/apis/schedule'; +import { ILicenseState } from '../../../../lib'; +import { verifyAccessAndContext } from '../../../lib'; +import { + AlertingRequestHandlerContext, + INTERNAL_ALERTING_GAPS_FILL_BY_ID_API_PATH, +} from '../../../../types'; +import { transformRequestV1 } from './transforms'; +import { transformResponseV1 } from '../../../backfill/apis/schedule/transforms'; + +export const fillGapByIdRoute = ( + router: IRouter, + licenseState: ILicenseState +) => { + router.post( + { + path: `${INTERNAL_ALERTING_GAPS_FILL_BY_ID_API_PATH}`, + validate: { + query: fillGapByIdQuerySchemaV1, + }, + options: { + access: 'internal', + }, + }, + router.handleLegacyErrors( + verifyAccessAndContext(licenseState, async function (context, req, res) { + const rulesClient = (await context.alerting).getRulesClient(); + const query: FillGapByIdQueryV1 = req.query; + + const result = await rulesClient.fillGapById(transformRequestV1(query)); + + const response: ScheduleBackfillResponseV1 = { + body: transformResponseV1(result), + }; + return res.ok(response); + }) + ) + ); +}; diff --git a/x-pack/plugins/alerting/server/routes/gaps/apis/fill/transforms/index.ts b/x-pack/plugins/alerting/server/routes/gaps/apis/fill/transforms/index.ts new file mode 100644 index 0000000000000..8f53ad06bea41 --- /dev/null +++ b/x-pack/plugins/alerting/server/routes/gaps/apis/fill/transforms/index.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { transformRequest } from './transform_request/latest'; + +export { transformRequest as transformRequestV1 } from './transform_request/v1'; diff --git a/x-pack/plugins/alerting/server/routes/gaps/apis/fill/transforms/transform_request/latest.ts b/x-pack/plugins/alerting/server/routes/gaps/apis/fill/transforms/transform_request/latest.ts new file mode 100644 index 0000000000000..25300c97a6d2e --- /dev/null +++ b/x-pack/plugins/alerting/server/routes/gaps/apis/fill/transforms/transform_request/latest.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './v1'; diff --git a/x-pack/plugins/alerting/server/routes/gaps/apis/fill/transforms/transform_request/v1.ts b/x-pack/plugins/alerting/server/routes/gaps/apis/fill/transforms/transform_request/v1.ts new file mode 100644 index 0000000000000..8e57187c21d5c --- /dev/null +++ b/x-pack/plugins/alerting/server/routes/gaps/apis/fill/transforms/transform_request/v1.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +/* eslint-disable @typescript-eslint/naming-convention */ + +import { FillGapByIdQueryV1 } from '../../../../../../../common/routes/gaps/apis/fill'; +import { FindGapByIdParams } from '../../../../../../lib/rule_gaps/types'; + +export const transformRequest = ({ + rule_id, + gap_id, +}: FillGapByIdQueryV1): FindGapByIdParams => ({ + gapId: gap_id, + ruleId: rule_id, +}); diff --git a/x-pack/plugins/alerting/server/routes/rule_gaps/api/find/fing_gaps_route.ts b/x-pack/plugins/alerting/server/routes/gaps/apis/find/find_gaps_route.ts similarity index 94% rename from x-pack/plugins/alerting/server/routes/rule_gaps/api/find/fing_gaps_route.ts rename to x-pack/plugins/alerting/server/routes/gaps/apis/find/find_gaps_route.ts index 4f34383c819b1..fae1e9952a258 100644 --- a/x-pack/plugins/alerting/server/routes/rule_gaps/api/find/fing_gaps_route.ts +++ b/x-pack/plugins/alerting/server/routes/gaps/apis/find/find_gaps_route.ts @@ -16,12 +16,13 @@ import { AlertingRequestHandlerContext, INTERNAL_ALERTING_GAPS_FIND_API_PATH, } from '../../../../types'; +import { transformRequestV1, transformResponseV1 } from './transforms'; export const findGapsRoute = ( router: IRouter, licenseState: ILicenseState ) => { - router.post( + router.get( { path: `${INTERNAL_ALERTING_GAPS_FIND_API_PATH}`, validate: { diff --git a/x-pack/plugins/alerting/server/routes/gaps/apis/find/transforms/index.ts b/x-pack/plugins/alerting/server/routes/gaps/apis/find/transforms/index.ts new file mode 100644 index 0000000000000..2eab64276e020 --- /dev/null +++ b/x-pack/plugins/alerting/server/routes/gaps/apis/find/transforms/index.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { transformRequest } from './transform_request/latest'; +export { transformResponse } from './transform_response/latest'; + +export { transformRequest as transformRequestV1 } from './transform_request/v1'; +export { transformResponse as transformResponseV1 } from './transform_response/v1'; diff --git a/x-pack/plugins/alerting/server/routes/gaps/apis/find/transforms/transform_request/latest.ts b/x-pack/plugins/alerting/server/routes/gaps/apis/find/transforms/transform_request/latest.ts new file mode 100644 index 0000000000000..25300c97a6d2e --- /dev/null +++ b/x-pack/plugins/alerting/server/routes/gaps/apis/find/transforms/transform_request/latest.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './v1'; diff --git a/x-pack/plugins/alerting/server/routes/gaps/apis/find/transforms/transform_request/v1.ts b/x-pack/plugins/alerting/server/routes/gaps/apis/find/transforms/transform_request/v1.ts new file mode 100644 index 0000000000000..3e73322011695 --- /dev/null +++ b/x-pack/plugins/alerting/server/routes/gaps/apis/find/transforms/transform_request/v1.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +/* eslint-disable @typescript-eslint/naming-convention */ + +import { FindGapsRequestQueryV1 } from '../../../../../../../common/routes/gaps/apis/find'; +import { FindGapsParams } from '../../../../../../lib/rule_gaps/types'; + +export const transformRequest = ({ + end, + page, + per_page, + rule_id, + start, + sort_field, + sort_order, +}: FindGapsRequestQueryV1): FindGapsParams => ({ + end, + page, + perPage: per_page, + ruleId: rule_id, + start, + sortField: sort_field, + sortOrder: sort_order, +}); diff --git a/x-pack/plugins/alerting/server/routes/gaps/apis/find/transforms/transform_response/latest.ts b/x-pack/plugins/alerting/server/routes/gaps/apis/find/transforms/transform_response/latest.ts new file mode 100644 index 0000000000000..25300c97a6d2e --- /dev/null +++ b/x-pack/plugins/alerting/server/routes/gaps/apis/find/transforms/transform_response/latest.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './v1'; diff --git a/x-pack/plugins/alerting/server/routes/gaps/apis/find/transforms/transform_response/v1.ts b/x-pack/plugins/alerting/server/routes/gaps/apis/find/transforms/transform_response/v1.ts new file mode 100644 index 0000000000000..412a8c8e131a7 --- /dev/null +++ b/x-pack/plugins/alerting/server/routes/gaps/apis/find/transforms/transform_response/v1.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FindGapsResponseBodyV1 } from '../../../../../../../common/routes/gaps/apis/find'; + +import { Gap } from '../../../../../../lib/rule_gaps/gap'; + +export const transformResponse = ({ + page, + perPage, + total, + data: gapsData, +}: { + page: number; + perPage: number; + total: number; + data: Gap[]; +}): FindGapsResponseBodyV1 => ({ + page, + per_page: perPage, + total, + data: gapsData.map((gap) => ({ + _id: gap?.meta?._id, + ...gap.getEsObject(), + })), +}); diff --git a/x-pack/plugins/alerting/server/routes/gaps/apis/get_rules_with_gaps/get_rules_with_gaps_route.ts b/x-pack/plugins/alerting/server/routes/gaps/apis/get_rules_with_gaps/get_rules_with_gaps_route.ts new file mode 100644 index 0000000000000..4c05ded9cc88f --- /dev/null +++ b/x-pack/plugins/alerting/server/routes/gaps/apis/get_rules_with_gaps/get_rules_with_gaps_route.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { IRouter } from '@kbn/core/server'; +import { + getRulesWithGapQuerySchemaV1, + GetRulesWithGapQueryV1, + GetRulesWithGapResponseV1, +} from '../../../../../common/routes/gaps/apis/get_rules_with_gaps'; +import { ILicenseState } from '../../../../lib'; +import { verifyAccessAndContext } from '../../../lib'; +import { + AlertingRequestHandlerContext, + INTERNAL_ALERTING_GAPS_GET_RULES_API_PATH, +} from '../../../../types'; + +export const getRulesWithGapsRoute = ( + router: IRouter, + licenseState: ILicenseState +) => { + router.get( + { + path: `${INTERNAL_ALERTING_GAPS_GET_RULES_API_PATH}`, + validate: { + query: getRulesWithGapQuerySchemaV1, + }, + options: { + access: 'internal', + }, + }, + router.handleLegacyErrors( + verifyAccessAndContext(licenseState, async function (context, req, res) { + const rulesClient = (await context.alerting).getRulesClient(); + const query: GetRulesWithGapQueryV1 = req.query; + + const result = await rulesClient.getRulesWithGaps(query); + const response: GetRulesWithGapResponseV1 = { + body: result, + }; + return res.ok(response); + }) + ) + ); +}; diff --git a/x-pack/plugins/alerting/server/routes/index.ts b/x-pack/plugins/alerting/server/routes/index.ts index 1a274692cefe4..b7cba7676d9c2 100644 --- a/x-pack/plugins/alerting/server/routes/index.ts +++ b/x-pack/plugins/alerting/server/routes/index.ts @@ -73,6 +73,11 @@ import { getBackfillRoute } from './backfill/apis/get/get_backfill_route'; import { findBackfillRoute } from './backfill/apis/find/find_backfill_route'; import { deleteBackfillRoute } from './backfill/apis/delete/delete_backfill_route'; +// Gaps ApI +import { findGapsRoute } from './gaps/apis/find/find_gaps_route'; +import { fillGapByIdRoute } from './gaps/apis/fill/fill_gap_by_id_route'; +import { getRulesWithGapsRoute } from './gaps/apis/get_rules_with_gaps/get_rules_with_gaps_route'; + export interface RouteOptions { router: IRouter; licenseState: ILicenseState; @@ -153,4 +158,9 @@ export function defineRoutes(opts: RouteOptions) { getBackfillRoute(router, licenseState); findBackfillRoute(router, licenseState); deleteBackfillRoute(router, licenseState); + + // Gaps APIs + findGapsRoute(router, licenseState); + fillGapByIdRoute(router, licenseState); + getRulesWithGapsRoute(router, licenseState); } diff --git a/x-pack/plugins/alerting/server/rules_client/rules_client.ts b/x-pack/plugins/alerting/server/rules_client/rules_client.ts index 71f408af1c6bb..85a9db84b7052 100644 --- a/x-pack/plugins/alerting/server/rules_client/rules_client.ts +++ b/x-pack/plugins/alerting/server/rules_client/rules_client.ts @@ -77,7 +77,10 @@ import { FindBackfillParams } from '../application/backfill/methods/find/types'; import { DisableRuleParams } from '../application/rule/methods/disable'; import { EnableRuleParams } from '../application/rule/methods/enable_rule'; import { findGaps } from '../application/rule/methods/find_gaps'; - +import { fillGapById } from '../application/rule/methods/fill_gap_by_id'; +import { GetRulesWithGapsParams } from '../application/rule/methods/get_rules_with_gaps/types'; +import { getRulesWithGaps } from '../application/rule/methods/get_rules_with_gaps'; +import { FindGapsParams, FindGapByIdParams } from '../lib/rule_gaps/types'; export type ConstructorOptions = Omit< RulesClientContext, @@ -211,4 +214,9 @@ export class RulesClient { public getScheduleFrequency = () => getScheduleFrequency(this.context); public findGaps = (params: FindGapsParams) => findGaps(this.context, params); + + public fillGapById = (params: FindGapByIdParams) => fillGapById(this.context, params); + + public getRulesWithGaps = (params: GetRulesWithGapsParams) => + getRulesWithGaps(this.context, params); } diff --git a/x-pack/plugins/alerting/server/task_runner/ad_hoc_task_runner.ts b/x-pack/plugins/alerting/server/task_runner/ad_hoc_task_runner.ts index b99bd42885357..1bfe517e5bf0d 100644 --- a/x-pack/plugins/alerting/server/task_runner/ad_hoc_task_runner.ts +++ b/x-pack/plugins/alerting/server/task_runner/ad_hoc_task_runner.ts @@ -52,7 +52,8 @@ import { import { RuleRunMetrics, RuleRunMetricsStore } from '../lib/rule_run_metrics_store'; import { getEsErrorMessage } from '../lib/errors'; import { Result, isOk, asOk, asErr } from '../lib/result_type'; -import { updateGaps } from '../lib/rule_gaps/update_gaps'; +import { addFilledIntervalToGaps } from '../lib/rule_gaps/update/add_filled_interval_to_gaps'; +import { calculateInProgressIntervalsForGaps } from '../lib/rule_gaps/update/caclculate_in_progress_intervals_for_gaps'; interface ConstructorParams { context: TaskRunnerContext; @@ -99,6 +100,7 @@ export class AdHocTaskRunner implements CancellableTask { private stackTraceLog: RuleRunnerErrorStackTraceLog | null = null; private taskRunning: AdHocTaskRunningHandler; private timer: TaskRunnerTimer; + private apiKeyToUse: string | null = null; constructor({ context, internalSavedObjectsRepository, taskInstance }: ConstructorParams) { this.context = context; @@ -319,6 +321,7 @@ export class AdHocTaskRunner implements CancellableTask { } const { rule, apiKeyToUse, schedule } = adHocRunData; + this.apiKeyToUse = apiKeyToUse; let ruleType: UntypedNormalizedRuleType; try { @@ -482,35 +485,22 @@ export class AdHocTaskRunner implements CancellableTask { executionStatus?.error?.reason === RuleExecutionStatusErrorReasons.License || executionStatus?.error?.reason === RuleExecutionStatusErrorReasons.Validate); + if (startedAt) { + if (executionStatus.status === 'error') { + await this.calculateInProgressIntervalsForGaps(); + } else { + await this.addFilledIntervalToGaps(); + } + + // Capture how long it took for the rule to run after being claimed + this.timer.setDuration(TaskRunnerTimerSpan.TotalRunDuration, startedAt); + } + await this.updateAdHocRunSavedObjectPostRun(adHocRunParamsId, namespace, { ...(this.shouldDeleteTask ? { status: adHocRunStatus.ERROR } : {}), ...(this.scheduleToRunIndex > -1 ? { schedule: this.adHocRunSchedule } : {}), }); - // TODO: check if we need to fill gaps - const needToFillGaps = true; - - const eventLogClient = await this.context.getEventLogClient(spaceId); - - if (startedAt) { - const intervalInMs = parseDuration( - this.adHocRunSchedule[this.scheduleToRunIndex].interval - ); - const endGapsRange = new Date(this.adHocRunSchedule[this.scheduleToRunIndex].runAt); - const startGapsRange = new Date(endGapsRange.getTime() - intervalInMs); - await updateGaps({ - ruleId: this.ruleId, - start: startGapsRange, - end: endGapsRange, - eventLogger: this.context.eventLogger, - eventLogClient, - savedObjectsClient: this.internalSavedObjectsRepository, - logger: this.logger, - needToFillGaps, - }); - // Capture how long it took for the rule to run after being claimed - this.timer.setDuration(TaskRunnerTimerSpan.TotalRunDuration, startedAt); - } return { executionStatus, executionMetrics }; }); this.alertingEventLogger.done({ @@ -555,7 +545,6 @@ export class AdHocTaskRunner implements CancellableTask { await this.processAdHocRunResults(runMetrics); this.shouldDeleteTask = this.shouldDeleteTask || !this.hasAnyPendingRuns(); - return { state: {}, ...(this.shouldDeleteTask ? {} : { runAt: new Date() }), @@ -597,6 +586,8 @@ export class AdHocTaskRunner implements CancellableTask { }); this.shouldDeleteTask = !this.hasAnyPendingRuns(); + await this.calculateInProgressIntervalsForGaps(); + // cleanup function is not called for timed out tasks await this.cleanup(); } @@ -620,4 +611,47 @@ export class AdHocTaskRunner implements CancellableTask { ); } } + + private async getParamsForGapsUpdate() { + if (this.scheduleToRunIndex < 0) return null; + + const intervalInMs = parseDuration(this.adHocRunSchedule[this.scheduleToRunIndex].interval); + + const endGapsRange = new Date(this.adHocRunSchedule[this.scheduleToRunIndex].runAt); + const startGapsRange = new Date(endGapsRange.getTime() - intervalInMs); + const fakeRequest = getFakeKibanaRequest( + this.context, + this.taskInstance.params.spaceId, + this.apiKeyToUse + ); + const eventLogClient = await this.context.getEventLogClient(fakeRequest); + + return { + ruleId: this.ruleId, + start: startGapsRange, + end: endGapsRange, + eventLogger: this.context.eventLogger, + eventLogClient, + logger: this.logger, + }; + } + + private async calculateInProgressIntervalsForGaps() { + const params = await this.getParamsForGapsUpdate(); + if (params === null) return; + + return calculateInProgressIntervalsForGaps({ + ...params, + savedObjectsRepository: this.internalSavedObjectsRepository, + }); + } + + private async addFilledIntervalToGaps() { + const params = await this.getParamsForGapsUpdate(); + if (params === null) return; + + return addFilledIntervalToGaps({ + ...params, + }); + } } diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner.ts b/x-pack/plugins/alerting/server/task_runner/task_runner.ts index 25609d8af887d..d5e78032c4cc3 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner.ts @@ -72,7 +72,6 @@ import { processRunResults, clearExpiredSnoozes, } from './lib'; -import { getTotalUnfilledGapDuration } from '../lib/rule_gaps/get_total_unfilled_gap_duration'; const FALLBACK_RETRY_INTERVAL = '5m'; const CONNECTIVITY_RETRY_INTERVAL = '5m'; @@ -627,31 +626,11 @@ export class TaskRunner< ); } - // TODO: it's not working yet - const unfiledGapDuration = await getTotalUnfilledGapDuration({ - ruleId, - eventLog: this.context.eventLogger, - logger: this.logger, - }); - - const monitoring = { - ...this.ruleMonitoring.getMonitoring(), - run: { - ...this.ruleMonitoring.getMonitoring()?.run, - last_run: { - ...this.ruleMonitoring.getMonitoring()?.run?.last_run, - metrics: { - ...this.ruleMonitoring.getMonitoring()?.run?.last_run?.metrics, - // unfilled_gap_duration_ms: unfiledGapDuration.unfiled_gap_duration_ms, - }, - }, - }, - }; await this.updateRuleSavedObjectPostRun(ruleId, { executionStatus: ruleExecutionStatusToRaw(executionStatus), nextRun, lastRun: lastRunToRaw(lastRun), - monitoring, + monitoring: this.ruleMonitoring.getMonitoring() as RawRuleMonitoring, }); } diff --git a/x-pack/plugins/alerting/server/task_runner/types.ts b/x-pack/plugins/alerting/server/task_runner/types.ts index 4aaa2ae7591fe..c8fa90fe17f7a 100644 --- a/x-pack/plugins/alerting/server/task_runner/types.ts +++ b/x-pack/plugins/alerting/server/task_runner/types.ts @@ -180,5 +180,5 @@ export interface TaskRunnerContext { supportsEphemeralTasks: boolean; uiSettings: UiSettingsServiceStart; usageCounter?: UsageCounter; - getEventLogClient: (spaceId: string) => IEventLogClient; + getEventLogClient: (request: KibanaRequest) => IEventLogClient; } diff --git a/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts b/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts index 54647990889f9..8a99d0ddf02d3 100644 --- a/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts +++ b/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts @@ -7,7 +7,7 @@ import { Subject } from 'rxjs'; import { bufferTime, filter as rxFilter, concatMap } from 'rxjs'; -import { reject, isUndefined, isNumber, pick, isEmpty, get, mergeWith } from 'lodash'; +import { reject, isUndefined, isNumber, pick, isEmpty, get } from 'lodash'; import type { PublicMethodsOf } from '@kbn/utility-types'; import { Logger, ElasticsearchClient } from '@kbn/core/server'; import util from 'util'; @@ -23,10 +23,17 @@ export const EVENT_BUFFER_LENGTH = 100; export type IClusterClientAdapter = PublicMethodsOf; +export interface DocMeta { + _id: string; + _index: string; + _seq_no: number; + _primary_term: number; +} + export interface Doc { index: string; body: IEvent; - meta?: {}; + meta?: DocMeta; } type Wait = () => Promise; @@ -100,7 +107,12 @@ type AliasAny = any; const LEGACY_ID_CUTOFF_VERSION = '8.0.0'; export class ClusterClientAdapter< - TDoc extends { body: AliasAny; index: string; id?: string } = Doc + TDoc extends { + body: AliasAny; + index: string; + id?: string; + meta?: DocMeta; + } = Doc > { private readonly logger: Logger; private readonly elasticsearchClientPromise: Promise; @@ -142,12 +154,16 @@ export class ClusterClientAdapter< public async updateDocument(doc: Required) { const esClient = await this.elasticsearchClientPromise; try { + if (!doc.meta) { + return; + } const result = await esClient.update({ doc: doc.body, id: doc.meta._id, index: doc.meta._index, if_primary_term: doc.meta._primary_term, if_seq_no: doc.meta._seq_no, + refresh: 'wait_for', }); } catch (e) { this.logger.error(`error update event: "${e.message}"; docs: ${JSON.stringify(doc)}`); @@ -172,47 +188,19 @@ export class ClusterClientAdapter< const bulkBody: Array> = []; - const docsToUpdate = docs.filter((doc) => doc.id && doc.body); const docsToCreate = docs.filter((doc) => !doc.id); for (const doc of docsToCreate) { if (doc.body === undefined) continue; - bulkBody.push({ create: { _index: this.esNames.dataStream } }); + bulkBody.push({ create: {} }); bulkBody.push(doc.body); } try { const esClient = await this.elasticsearchClientPromise; - - // for (const docToUpdate of docsToUpdate) { - // bulkBody.push({ - // update: { - // _index: docToUpdate.meta._index, - // _id: docToUpdate.meta._id, - // if_seq_no: docToUpdate.meta._seq_no, - // if_primary_term: docToUpdate.meta._primary_term, - // }, - // }); - - // // const updatedDoc = mergeWith( - // // {}, - // // originalDoc._source, - // // fieldsToUpdate.body, - // // (objValue, srcValue) => { - // // if (Array.isArray(srcValue)) { - // // return srcValue; - // // } - // // } - // // ); - - // // console.log('updatedDoc', JSON.stringify(updatedDoc)); - - // bulkBody.push({ doc: docToUpdate.body }); - // } - const response = await esClient.bulk({ - // index: this.esNames.dataStream, + index: this.esNames.dataStream, body: bulkBody, }); diff --git a/x-pack/plugins/event_log/server/event_log_client.ts b/x-pack/plugins/event_log/server/event_log_client.ts index 86a27a0adfbcc..4ffcfb6cef84e 100644 --- a/x-pack/plugins/event_log/server/event_log_client.ts +++ b/x-pack/plugins/event_log/server/event_log_client.ts @@ -70,42 +70,25 @@ export type AggregateOptionsType = Pick, 'filt aggs: Record; }; -interface EventLogServiceCtorBaseParams { +interface EventLogServiceCtorParams { esContext: EsContext; + savedObjectGetter: SavedObjectBulkGetterResult; spacesService?: SpacesServiceStart; + request: KibanaRequest; } -type EventLogServiceCtorParams = - | (EventLogServiceCtorBaseParams & { - request: KibanaRequest; - savedObjectGetter: SavedObjectBulkGetterResult; - spaceId?: string; - }) - | (EventLogServiceCtorBaseParams & { - spaceId: string; - request?: KibanaRequest; - savedObjectGetter?: SavedObjectBulkGetterResult; - }); // note that clusterClient may be null, indicating we can't write to ES export class EventLogClient implements IEventLogClient { private esContext: EsContext; - private savedObjectGetter?: SavedObjectBulkGetterResult; + private savedObjectGetter: SavedObjectBulkGetterResult; private spacesService?: SpacesServiceStart; - private request?: KibanaRequest; - private spaceId?: string; - - constructor({ - esContext, - savedObjectGetter, - spacesService, - request, - spaceId, - }: EventLogServiceCtorParams) { + private request: KibanaRequest; + + constructor({ esContext, savedObjectGetter, spacesService, request }: EventLogServiceCtorParams) { this.esContext = esContext; this.savedObjectGetter = savedObjectGetter; this.spacesService = spacesService; this.request = request; - this.spaceId = spaceId; } public async findEventsBySavedObjectIds( @@ -117,7 +100,7 @@ export class EventLogClient implements IEventLogClient { const findOptions = queryOptionsSchema.validate(options ?? {}); // verify the user has the required permissions to view this saved object - await this.savedObjectGetter?.(type, ids); + await this.savedObjectGetter(type, ids); return await this.esContext.esAdapter.queryEventsBySavedObjects({ index: this.esContext.esNames.indexPattern, @@ -169,7 +152,7 @@ export class EventLogClient implements IEventLogClient { const aggregateOptions = queryOptionsSchema.validate(omit(options, 'aggs') ?? {}); // verify the user has the required permissions to view this saved object - await this.savedObjectGetter?.(type, ids); + await this.savedObjectGetter(type, ids); return await this.esContext.esAdapter.aggregateEventsBySavedObjects({ index: this.esContext.esNames.indexPattern, @@ -211,12 +194,7 @@ export class EventLogClient implements IEventLogClient { } private async getNamespace() { - if (this.spaceId) { - return this.spacesService?.spaceIdToNamespace(this.spaceId); - } else if (this.request) { - const space = await this.spacesService?.getActiveSpace(this.request); - return space && this.spacesService?.spaceIdToNamespace(space.id); - } - return undefined; + const space = await this.spacesService?.getActiveSpace(this.request); + return space && this.spacesService?.spaceIdToNamespace(space.id); } } diff --git a/x-pack/plugins/event_log/server/event_log_start_service.ts b/x-pack/plugins/event_log/server/event_log_start_service.ts index 2757460f79d4d..e850d5342b1cf 100644 --- a/x-pack/plugins/event_log/server/event_log_start_service.ts +++ b/x-pack/plugins/event_log/server/event_log_start_service.ts @@ -38,20 +38,12 @@ export class EventLogClientService implements IEventLogClientService { this.spacesService = spacesService; } - getClient(requestOrSpaceId: KibanaRequest | string) { - if (typeof requestOrSpaceId === 'string') { - return new EventLogClient({ - esContext: this.esContext, - spacesService: this.spacesService, - spaceId: requestOrSpaceId, - }); - } else { - return new EventLogClient({ - esContext: this.esContext, - savedObjectGetter: this.savedObjectProviderRegistry.getProvidersClient(requestOrSpaceId), - spacesService: this.spacesService, - request: requestOrSpaceId, - }); - } + getClient(requestOrSpaceId: KibanaRequest) { + return new EventLogClient({ + esContext: this.esContext, + savedObjectGetter: this.savedObjectProviderRegistry.getProvidersClient(requestOrSpaceId), + spacesService: this.spacesService, + request: requestOrSpaceId, + }); } } diff --git a/x-pack/plugins/event_log/server/event_logger.mock.ts b/x-pack/plugins/event_log/server/event_logger.mock.ts index 837eac7ec2b5e..ed243d98548f1 100644 --- a/x-pack/plugins/event_log/server/event_logger.mock.ts +++ b/x-pack/plugins/event_log/server/event_logger.mock.ts @@ -10,6 +10,7 @@ import { IEventLogger } from './types'; const createEventLoggerMock = () => { const mock: jest.Mocked = { logEvent: jest.fn(), + updateEvent: jest.fn(), startTiming: jest.fn(), stopTiming: jest.fn(), }; diff --git a/x-pack/plugins/event_log/server/event_logger.ts b/x-pack/plugins/event_log/server/event_logger.ts index 0424f44d28679..f38c9d8356259 100644 --- a/x-pack/plugins/event_log/server/event_logger.ts +++ b/x-pack/plugins/event_log/server/event_logger.ts @@ -23,7 +23,7 @@ import { EventSchema, } from './types'; import { SAVED_OBJECT_REL_PRIMARY } from './types'; -import { Doc } from './es/cluster_client_adapter'; +import { Doc, DocMeta } from './es/cluster_client_adapter'; type SystemLogger = Plugin['systemLogger']; @@ -108,7 +108,7 @@ export class EventLogger implements IEventLogger { } } - async updateEvent(meta: {}, event: IEvent): Promise { + async updateEvent(meta: DocMeta, event: IEvent): Promise { const doc: Required = { index: this.esContext.esNames.dataStream, body: event, @@ -116,7 +116,7 @@ export class EventLogger implements IEventLogger { }; if (this.eventLogService.isIndexingEntries()) { - const result = updateEventDoc(this.esContext, doc); + const result = await updateEventDoc(this.esContext, doc); if (this.eventLogService.isLoggingEntries()) { logUpdateEventDoc(this.systemLogger, doc); diff --git a/x-pack/plugins/event_log/server/index.ts b/x-pack/plugins/event_log/server/index.ts index e070f0cf0c940..0bfc725200e48 100644 --- a/x-pack/plugins/event_log/server/index.ts +++ b/x-pack/plugins/event_log/server/index.ts @@ -19,6 +19,7 @@ export type { IEventLogClient, QueryEventsBySavedObjectResult, AggregateEventsBySavedObjectResult, + DocMeta, } from './types'; export { SAVED_OBJECT_REL_PRIMARY } from './types'; diff --git a/x-pack/plugins/event_log/server/types.ts b/x-pack/plugins/event_log/server/types.ts index e72921d1c4553..16130a7b1a861 100644 --- a/x-pack/plugins/event_log/server/types.ts +++ b/x-pack/plugins/event_log/server/types.ts @@ -15,12 +15,14 @@ import { IEvent } from '../generated/schemas'; import { AggregateOptionsType, FindOptionsType } from './event_log_client'; import { AggregateEventsBySavedObjectResult, + DocMeta, QueryEventsBySavedObjectResult, } from './es/cluster_client_adapter'; export type { QueryEventsBySavedObjectResult, AggregateEventsBySavedObjectResult, + DocMeta, } from './es/cluster_client_adapter'; import { SavedObjectProvider } from './saved_object_provider_registry'; @@ -83,5 +85,5 @@ export interface IEventLogger { logEvent(properties: IEvent): void; startTiming(event: IEvent, startTime?: Date): void; stopTiming(event: IEvent): void; - updateEvent(meta: {}, event: IEvent): void; + updateEvent(meta: DocMeta, event: IEvent): Promise; } diff --git a/x-pack/plugins/security_solution/common/detection_engine/rule_management/rule_filtering.ts b/x-pack/plugins/security_solution/common/detection_engine/rule_management/rule_filtering.ts index bfc31df88ec97..eed3bc13818b6 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/rule_management/rule_filtering.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/rule_management/rule_filtering.ts @@ -32,6 +32,7 @@ interface RulesFilterOptions { tags: string[]; excludeRuleTypes: Type[]; ruleExecutionStatus: RuleExecutionStatus; + ruleIds: string[]; } /** @@ -49,6 +50,7 @@ export function convertRulesFilterToKQL({ tags, excludeRuleTypes = [], ruleExecutionStatus, + ruleIds, }: Partial): string { const kql: string[] = []; @@ -84,6 +86,10 @@ export function convertRulesFilterToKQL({ kql.push(`${LAST_RUN_OUTCOME_FIELD}: "failed"`); } + if (ruleIds?.length) { + kql.push(convertRuleIdsToKQL(ruleIds)); + } + return kql.join(' AND '); } @@ -112,3 +118,7 @@ export function convertRuleTagsToKQL(tags: string[]): string { export function convertRuleTypesToKQL(ruleTypes: Type[]): string { return `${PARAMS_TYPE_FIELD}: (${ruleTypes.map(prepareKQLStringParam).join(' OR ')})`; } + +export function convertRuleIdsToKQL(ruleIds: string[]): string { + return `(${ruleIds.map((ruleId) => `alert.id: ("alert:${ruleId}")`).join(' OR ')})`; +} diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_details_ui/pages/rule_details/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_details_ui/pages/rule_details/index.tsx index 851d219ad43d3..c5c9285581409 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_details_ui/pages/rule_details/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_details_ui/pages/rule_details/index.tsx @@ -114,6 +114,7 @@ import { import { ExecutionEventsTable } from '../../../rule_monitoring'; import { ExecutionLogTable } from './execution_log_table/execution_log_table'; import { RuleBackfillsInfo } from '../../../rule_gaps/components/rule_backfills_info'; +import { RuleGaps } from '../../../rule_gaps/components/rule_gaps'; import * as ruleI18n from '../../../../detections/pages/detection_engine/rules/translations'; @@ -796,6 +797,8 @@ const RuleDetailsPageComponent: React.FC = ({ theme={theme} /> + + diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_gaps/components/rule_gaps/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_gaps/components/rule_gaps/index.tsx new file mode 100644 index 0000000000000..ec615724356c6 --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_gaps/components/rule_gaps/index.tsx @@ -0,0 +1,379 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { UseMutationOptions, UseQueryOptions } from '@tanstack/react-query'; +import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query'; +import { + INTERNAL_ALERTING_GAPS_FIND_API_PATH, + INTERNAL_ALERTING_GAPS_FILL_BY_ID_API_PATH, +} from '@kbn/alerting-plugin/common'; +import React, { useCallback, useState } from 'react'; +import type { CriteriaWithPagination, EuiBasicTableColumn } from '@elastic/eui'; +import { + EuiButton, + EuiBasicTable, + EuiFlexGroup, + EuiFlexItem, + EuiPanel, + EuiBetaBadge, + EuiProgress, + EuiText, + EuiHealth, + EuiButtonEmpty, +} from '@elastic/eui'; + +import type { FindGapsResponseBody } from '@kbn/alerting-plugin/common/routes/gaps/apis/find'; + +import type { FillGapByIdResponseV1 } from '@kbn/alerting-plugin/common/routes/gaps/apis/fill'; +import type { IHttpFetchError } from '@kbn/core/public'; +import { useAppToasts } from '../../../../common/hooks/use_app_toasts'; +import { useUserData } from '../../../../detections/components/user_info'; +import { hasUserCRUDPermission } from '../../../../common/utils/privileges'; + +import { BETA, BETA_TOOLTIP } from '../../../../common/translations'; + +import { HeaderSection } from '../../../../common/components/header_section'; +import { TableHeaderTooltipCell } from '../../../rule_management_ui/components/rules_table/table_header_tooltip_cell'; + +import { FormattedDate } from '../../../../common/components/formatted_date'; +import { KibanaServices, useKibana } from '../../../../common/lib/kibana'; +import { getFormattedDuration } from '../../../rule_details_ui/pages/rule_details/execution_log_table/rule_duration_format'; +import * as i18n from './translations'; + +export type Gap = FindGapsResponseBody['data']['0']; +export type GapStatus = Gap['status']; + +const FIND_GAPS_FOR_RULE = 'FIND_GAP_FOR_RULE'; + +/** + * Find gaps for the given rule ID + * @param ruleIds string[] + * @param signal? AbortSignal + * @returns + */ +export const findGapsForRule = async ({ + ruleId, + page, + perPage, + signal, + sortField = 'createdAt', + sortOrder = 'desc', +}: { + ruleId: string; + page: number; + perPage: number; + signal?: AbortSignal; + sortField?: string; + sortOrder?: string; +}): Promise => + KibanaServices.get().http.fetch(INTERNAL_ALERTING_GAPS_FIND_API_PATH, { + method: 'GET', + query: { + rule_id: ruleId, + page, + per_page: perPage, + sort_field: sortField, + sort_order: sortOrder, + }, + signal, + }); + +/** + * Fill gap by Id for the given rule ID + * @param ruleIds string[] + * @param signal? AbortSignal + * @returns + */ +export const fillGapByIdForRule = async ({ + ruleId, + gapId, + signal, +}: FillGapQuery & { + signal?: AbortSignal; +}): Promise => + KibanaServices.get().http.fetch( + INTERNAL_ALERTING_GAPS_FILL_BY_ID_API_PATH, + { + method: 'POST', + query: { + rule_id: ruleId, + gap_id: gapId, + }, + signal, + } + ); + +export const useInvalidateFindGapsQuery = () => { + const queryClient = useQueryClient(); + + return useCallback(() => { + queryClient.invalidateQueries([FIND_GAPS_FOR_RULE], { + refetchType: 'active', + }); + }, [queryClient]); +}; + +export const useFindGapsForRule = ( + { + ruleId, + page, + perPage, + }: { + ruleId: string; + page: number; + perPage: number; + }, + options?: UseQueryOptions +) => { + return useQuery( + [FIND_GAPS_FOR_RULE, ruleId, page, perPage], + async ({ signal }) => { + const response = await findGapsForRule({ signal, ruleId, page, perPage }); + + return response; + }, + { + retry: 0, + keepPreviousData: true, + ...options, + } + ); +}; + +export const FILL_GAP_BY_ID_MUTATION_KEY = ['POST', INTERNAL_ALERTING_GAPS_FILL_BY_ID_API_PATH]; + +interface FillGapQuery { + ruleId: string; + gapId: string; +} +export const useFillGapMutation = ( + options?: UseMutationOptions, FillGapQuery> +) => { + const invalidateFindGapsQuery = useInvalidateFindGapsQuery(); + return useMutation((fillGapsOptions: FillGapQuery) => fillGapByIdForRule(fillGapsOptions), { + ...options, + onSettled: (...args) => { + invalidateFindGapsQuery(); + if (options?.onSettled) { + options.onSettled(...args); + } + }, + mutationKey: FILL_GAP_BY_ID_MUTATION_KEY, + }); +}; + +const FillGap = ({ ruleId, gap }: { ruleId: string; gap: Gap }) => { + const { addSuccess, addError } = useAppToasts(); + const fillGapMutation = useFillGapMutation({ + onSuccess: () => { + addSuccess(i18n.GAP_FILL_REQUEST_SUCCESS_MESSAGE, { + toastMessage: i18n.GAP_FILL_REQUEST_SUCCESS_MESSAGE_TOOLTIP, + }); + }, + onError: (error) => { + addError(error, { + title: i18n.GAP_FILL_REQUEST_ERROR_MESSAGE, + toastMessage: error?.body?.message ?? error.message, + }); + }, + }); + + if (gap.status === 'filled' || gap.unfilled_intervals.length === 0) { + return null; + } + + const title = + gap.in_progress_intervals.length > 0 || gap.filled_intervals.length > 0 + ? i18n.GAPS_TABLE_FILL_REMAINING_GAP_BUTTON_LABEL + : i18n.GAPS_TABLE_FILL_GAP_BUTTON_LABEL; + + return ( + <> + + fillGapMutation.mutate({ + ruleId, + gapId: gap._id, + }) + } + > + {title} + + + ); +}; + +const getStatusLabel = (status: string) => { + switch (status) { + case 'partially_filled': + return i18n.GAP_STATUS_PARTIALLY_FILLED; + case 'unfilled': + return i18n.GAP_STATUS_UNFILLED; + case 'filled': + return i18n.GAP_STATUS_FILLED; + } + return ''; +}; + +const getGapsTableColumns = (hasCRUDPermissions: boolean, ruleId: string) => { + const fillActions = { + name: i18n.GAPS_TABLE_ACTIONS_LABEL, + align: 'right' as const, + render: (gap: Gap) => , + width: '15%', + }; + + const columns: Array> = [ + { + field: 'status', + name: , + render: (value: string) => getStatusLabel(value), + width: '10%', + }, + { + field: 'in_progress_intervals', + name: ( + + ), + render: (value: Gap['in_progress_intervals']) => { + if (!value || !value.length) return null; + return {i18n.GAPS_TABLE_IN_PROGRESS_LABEL}; + }, + width: '10%', + }, + { + width: '10%', + align: 'right', + name: ( + + ), + render: (item: Gap) => { + if (!item) return null; + const value = Math.ceil((item.filled_duration_ms * 100) / item.total_gap_duration_ms); + return ( + + + +

+ {value} + {'%'} +

+ + + + + + + ); + }, + }, + { + field: 'range', + name: , + render: (value: Gap['range']) => ( + <> + + {' - '} + + + ), + width: '40%', + }, + { + field: 'total_gap_duration_ms', + name: ( + + ), + render: (value: Gap['total_gap_duration_ms']) => <>{getFormattedDuration(value)}, + width: '10%', + }, + ]; + + if (hasCRUDPermissions) { + columns.push(fillActions); + } + + return columns; +}; + +const DEFAULT_PAGE_SIZE = 10; + +export const RuleGaps = ({ ruleId }: { ruleId: string }) => { + const [pageIndex, setPageIndex] = useState(0); + const [pageSize, setPageSize] = useState(DEFAULT_PAGE_SIZE); + const [{ canUserCRUD }] = useUserData(); + const { timelines } = useKibana().services; + const hasCRUDPermissions = hasUserCRUDPermission(canUserCRUD); + + const { data, isLoading, isError, refetch, dataUpdatedAt } = useFindGapsForRule({ + ruleId, + page: pageIndex + 1, + perPage: pageSize, + }); + + const pagination = { + pageIndex, + pageSize, + totalItemCount: data?.total ?? 0, + }; + + const columns = getGapsTableColumns(hasCRUDPermissions, ruleId); + + const handleRefresh = () => { + refetch(); + }; + + const handleTableChange: (params: CriteriaWithPagination) => void = ({ page }) => { + if (page) { + setPageIndex(page.index); + setPageSize(page.size); + } + }; + + return ( + + + + + + + + + + + + {'Refresh'} + + + + + + {timelines.getLastUpdated({ + showUpdating: isLoading, + updatedAt: dataUpdatedAt, + })} + + + + + ); +}; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_gaps/components/rule_gaps/translations.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_gaps/components/rule_gaps/translations.tsx new file mode 100644 index 0000000000000..ee40f5e4221ef --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_gaps/components/rule_gaps/translations.tsx @@ -0,0 +1,112 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { i18n } from '@kbn/i18n'; + +export const GAPS_TABLE_STATUS_LABEL = i18n.translate( + 'xpack.securitySolution.gapsTable.statusLabel', + { + defaultMessage: 'Status', + } +); + +export const GAPS_TABLE_ACTIONS_LABEL = i18n.translate( + 'xpack.securitySolution.gapsTable.actionsLabel', + { + defaultMessage: 'Actions', + } +); + +export const GAP_STATUS_UNFILLED = i18n.translate( + 'xpack.securitySolution.gapsTable.gapStatus.unfilled', + { + defaultMessage: 'Unfilled', + } +); + +export const GAP_STATUS_PARTIALLY_FILLED = i18n.translate( + 'xpack.securitySolution.gapsTable.gapStatus.partiallyFilled', + { + defaultMessage: 'Partially filled', + } +); + +export const GAP_STATUS_FILLED = i18n.translate( + 'xpack.securitySolution.gapsTable.gapStatus.filled', + { + defaultMessage: 'Filled', + } +); + +export const GAPS_TABLE_MANUAL_FILL_TASKS_LABEL = i18n.translate( + 'xpack.securitySolution.gapsTable.manualFillTasksLabel', + { + defaultMessage: 'Manual fill tasks', + } +); + +export const GAPS_TABLE_IN_PROGRESS_LABEL = i18n.translate( + 'xpack.securitySolution.gapsTable.inProgressIntervalsLabel', + { + defaultMessage: 'In progress', + } +); + +export const GAPS_TABLE_EVENT_TIME_COVERED_LABEL = i18n.translate( + 'xpack.securitySolution.gapsTable.eventTimeCoveredLabel', + { + defaultMessage: 'Event time covered', + } +); + +export const GAPS_TABLE_GAP_RANGE_LABEL = i18n.translate( + 'xpack.securitySolution.gapsTable.gapRangeLabel', + { + defaultMessage: 'Range', + } +); + +export const GAPS_TABLE_GAP_DURATION_TOOLTIP = i18n.translate( + 'xpack.securitySolution.gapsTable.gapDurationTooltip', + { + defaultMessage: 'Total gap duration', + } +); + +export const GAPS_TABLE_FILL_GAP_BUTTON_LABEL = i18n.translate( + 'xpack.securitySolution.gapsTable.fillGapButtonLabel', + { + defaultMessage: 'Fill gap', + } +); + +export const GAPS_TABLE_FILL_REMAINING_GAP_BUTTON_LABEL = i18n.translate( + 'xpack.securitySolution.gapsTable.fillRemainingGapButtonLabel', + { + defaultMessage: 'Fill remaining gap', + } +); + +export const GAP_FILL_REQUEST_SUCCESS_MESSAGE = i18n.translate( + 'xpack.securitySolution.gapsTable.gapFillRequestSuccessMessage', + { + defaultMessage: 'Manual run requested', + } +); + +export const GAP_FILL_REQUEST_SUCCESS_MESSAGE_TOOLTIP = i18n.translate( + 'xpack.securitySolution.gapsTable.gapFillRequestSuccessMessageTooltip', + { + defaultMessage: 'Check status in rule execution logs. Actions for this execution will be run.', + } +); + +export const GAP_FILL_REQUEST_ERROR_MESSAGE = i18n.translate( + 'xpack.securitySolution.gapsTable.gapFillRequestErrorMessage', + { + defaultMessage: 'Failed to request manual run', + } +); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_gaps/components/rules_with_gaps_overview_panel/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_gaps/components/rules_with_gaps_overview_panel/index.tsx new file mode 100644 index 0000000000000..ef0ae224ff0f5 --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_gaps/components/rules_with_gaps_overview_panel/index.tsx @@ -0,0 +1,237 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { GetRulesWithGapResponseBody } from '@kbn/alerting-plugin/common/routes/gaps/apis/get_rules_with_gaps'; +import type { UseQueryOptions } from '@tanstack/react-query'; +import { useQuery } from '@tanstack/react-query'; +import { INTERNAL_ALERTING_GAPS_GET_RULES_API_PATH } from '@kbn/alerting-plugin/common'; +import React, { useEffect, useState } from 'react'; + +import { + EuiButton, + EuiFlexGroup, + EuiFlexItem, + EuiPanel, + EuiText, + EuiContextMenuPanel, + EuiPopover, + EuiContextMenuItem, + EuiBadge, + EuiFilterButton, + EuiFilterGroup, +} from '@elastic/eui'; + +import { KibanaServices } from '../../../../common/lib/kibana'; +import { useRulesTableContext } from '../../../rule_management_ui/components/rules_table/rules_table/rules_table_context'; +import * as i18n from './translations'; + +/** + * Find gaps for the given rule ID + * @param ruleIds string[] + * @param signal? AbortSignal + * @returns + */ +export const getRulesWithGaps = async ({ + signal, + start, + end, +}: { + start: string; + end: string; + signal?: AbortSignal; +}): Promise => + KibanaServices.get().http.fetch( + INTERNAL_ALERTING_GAPS_GET_RULES_API_PATH, + { + method: 'GET', + query: { + start, + end, + }, + signal, + } + ); + +const GET_RULES_WITH_GAPS = ['GET_RULES_WITH_GAPS']; +export const useGetRulesWithGaps = ( + { + start, + end, + }: { + start: string; + end: string; + }, + options?: UseQueryOptions +) => { + return useQuery( + [GET_RULES_WITH_GAPS, start, end], + async ({ signal }) => { + const response = await getRulesWithGaps({ signal, start, end }); + + return response; + }, + { + retry: 0, + keepPreviousData: true, + ...options, + } + ); +}; + +enum RangeValue { + LAST_24_H = 'last_24_h', + LAST_3_D = 'last_3_d', + LAST_7_D = 'last_7_d', +} + +const defaultRangeValue = RangeValue.LAST_24_H; +export const RulesWithGapsOverviewPanel = () => { + const [rangeValue, setRangeValue] = useState(defaultRangeValue); + const [showRulesWithGaps, setShowRulesWithGaps] = useState(false); + const [gapsInterval, setGapsInterval] = useState<{ start: string; end: string } | null>(null); + const { data } = useGetRulesWithGaps( + { + start: gapsInterval?.start ?? '', + end: gapsInterval?.end ?? '', + }, + { + enabled: Boolean(gapsInterval), + } + ); + const { + actions: { setFilterOptions }, + } = useRulesTableContext(); + const [isPopoverOpen, setPopover] = useState(false); + + const rangeValueToLabel = { + [RangeValue.LAST_24_H]: i18n.RULE_GAPS_OVERVIEW_PANEL_LAST_24_HOURS_LABEL, + [RangeValue.LAST_3_D]: i18n.RULE_GAPS_OVERVIEW_PANEL_LAST_3_DAYS_LABEL, + [RangeValue.LAST_7_D]: i18n.RULE_GAPS_OVERVIEW_PANEL_LAST_7_DAYS_LABEL, + }; + + useEffect(() => { + if (rangeValue) { + const now = new Date(); + const dayMs = 24 * 60 * 60 * 1000; + let amountOfDays = 1; + switch (rangeValue) { + case RangeValue.LAST_24_H: + amountOfDays = 1; + break; + case RangeValue.LAST_3_D: + amountOfDays = 3; + break; + case RangeValue.LAST_7_D: + amountOfDays = 7; + break; + } + + const start = new Date(now.getTime() - amountOfDays * dayMs).toISOString(); + const end = now.toISOString(); + setGapsInterval({ start, end }); + } + }, [rangeValue]); + + const onButtonClick = () => { + setPopover(!isPopoverOpen); + }; + + const closePopover = () => { + setPopover(false); + }; + + const items = Object.values(RangeValue).map((value) => ({ + value, + label: rangeValueToLabel[value], + })); + + const button = ( + + {rangeValueToLabel[rangeValue]} + + ); + + const handleShowRulesWithGapsFilterButtonClick = (value: boolean) => { + setShowRulesWithGaps(value); + if (!data) return; + if (value) { + setFilterOptions({ + ruleIds: data.ruleIds, + }); + } else { + setFilterOptions({ + ruleIds: [], + }); + } + }; + + return ( + + + + + ( + { + setRangeValue(item.value); + closePopover(); + }} + > + {item.label} + + ))} + /> + + + + + + + {i18n.RULE_GAPS_OVERVIEW_PANEL_LABEL} + + + + {data?.total} + + + + + + handleShowRulesWithGapsFilterButtonClick(false)} + > + {i18n.RULE_GAPS_OVERVIEW_PANEL_SHOW_ALL_RULES_LABEL} + + handleShowRulesWithGapsFilterButtonClick(true)} + > + {i18n.RULE_GAPS_OVERVIEW_PANEL_SHOW_RULES_WITH_GAPS_LABEL} + + + + + + ); +}; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_gaps/components/rules_with_gaps_overview_panel/translations.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_gaps/components/rules_with_gaps_overview_panel/translations.tsx new file mode 100644 index 0000000000000..fe9963eace3d1 --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_gaps/components/rules_with_gaps_overview_panel/translations.tsx @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; + +export const RULE_GAPS_OVERVIEW_PANEL_LABEL = i18n.translate( + 'xpack.securitySolution.ruleGapsOverviewPanel.label', + { + defaultMessage: 'Total rules with gaps:', + } +); + +export const RULE_GAPS_OVERVIEW_PANEL_SHOW_ALL_RULES_LABEL = i18n.translate( + 'xpack.securitySolution.ruleGapsOverviewPanel.showAllRulesLabel', + { + defaultMessage: 'Show all rules', + } +); + +export const RULE_GAPS_OVERVIEW_PANEL_SHOW_RULES_WITH_GAPS_LABEL = i18n.translate( + 'xpack.securitySolution.ruleGapsOverviewPanel.showRulesWithGapsLabel', + { + defaultMessage: 'Show rules with gaps', + } +); + +export const RULE_GAPS_OVERVIEW_PANEL_LAST_24_HOURS_LABEL = i18n.translate( + 'xpack.securitySolution.ruleGapsOverviewPanel.last24HoursLabel', + { + defaultMessage: 'Last 24 hours', + } +); + +export const RULE_GAPS_OVERVIEW_PANEL_LAST_3_DAYS_LABEL = i18n.translate( + 'xpack.securitySolution.ruleGapsOverviewPanel.last3DaysLabel', + { + defaultMessage: 'Last 3 days', + } +); + +export const RULE_GAPS_OVERVIEW_PANEL_LAST_7_DAYS_LABEL = i18n.translate( + 'xpack.securitySolution.ruleGapsOverviewPanel.last7DaysLabel', + { + defaultMessage: 'Last 7 days', + } +); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/types.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/types.ts index 59ac52d592bcd..620ea5c30d5df 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/types.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/types.ts @@ -100,6 +100,7 @@ export interface FilterOptions { enabled?: boolean; // undefined is to display all the rules ruleExecutionStatus?: RuleExecutionStatus; // undefined means "all" ruleSource?: RuleCustomizationEnum[]; // undefined is to display all the rules + ruleIds?: string[]; } export interface FetchRulesResponse { diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_table/rules_table_context.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_table/rules_table_context.tsx index 8f0cac0130083..143991d33bbad 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_table/rules_table_context.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_table/rules_table_context.tsx @@ -209,6 +209,7 @@ export const RulesTableContextProvider = ({ children }: RulesTableContextProvide enabled: savedFilter?.enabled, ruleExecutionStatus: savedFilter?.ruleExecutionStatus ?? DEFAULT_FILTER_OPTIONS.ruleExecutionStatus, + ruleIds: [], }); const [sortingOptions, setSortingOptions] = useState({ @@ -280,6 +281,7 @@ export const RulesTableContextProvider = ({ children }: RulesTableContextProvide } }, [selectedRuleIds, isRefreshOn]); + console.log('filterOptions', filterOptions); // Fetch rules const { data: { rules, total } = { rules: [], total: 0 }, diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_tables.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_tables.tsx index 3a9c5dfb8ee00..3fbaf1a6d4709 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_tables.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_tables.tsx @@ -5,7 +5,13 @@ * 2.0. */ -import { EuiBasicTable, EuiConfirmModal, EuiEmptyPrompt, EuiProgress } from '@elastic/eui'; +import { + EuiBasicTable, + EuiConfirmModal, + EuiEmptyPrompt, + EuiProgress, + EuiSpacer, +} from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n-react'; import React, { useCallback, useMemo, useRef } from 'react'; import { Loader } from '../../../../common/components/loader'; @@ -38,6 +44,7 @@ import { useIsUpgradingSecurityPackages } from '../../../rule_management/logic/u import { useManualRuleRunConfirmation } from '../../../rule_gaps/components/manual_rule_run/use_manual_rule_run_confirmation'; import { ManualRuleRunModal } from '../../../rule_gaps/components/manual_rule_run'; import { BulkManualRuleRunLimitErrorModal } from './bulk_actions/bulk_manual_rule_run_limit_error_modal'; +import { RulesWithGapsOverviewPanel } from '../../../rule_gaps/components/rules_with_gaps_overview_panel'; const INITIAL_SORT_FIELD = 'enabled'; @@ -313,6 +320,13 @@ export const RulesTables = React.memo(({ selectedTab }) => { )} {shouldShowRulesTable && ( <> + {selectedTab === AllRulesTabs.monitoring && ( + <> + + + + + )} Date: Mon, 16 Dec 2024 12:30:25 +0100 Subject: [PATCH 05/17] deleting all gaps for rules --- .../methods/bulk_delete/bulk_delete_rules.ts | 6 ++++ .../rule/methods/delete/delete_rule.ts | 6 ++++ .../alerting_event_logger.ts | 19 +++++++++++ .../server/lib/rule_gaps/delete_gaps.ts | 32 +++++++++++++++++++ .../server/es/cluster_client_adapter.ts | 9 ++++-- .../event_log/server/event_logger.mock.ts | 1 + .../plugins/event_log/server/event_logger.ts | 12 +++++++ x-pack/plugins/event_log/server/types.ts | 2 ++ .../rules_with_gaps_overview_panel/index.tsx | 2 +- 9 files changed, 85 insertions(+), 4 deletions(-) create mode 100644 x-pack/plugins/alerting/server/lib/rule_gaps/delete_gaps.ts diff --git a/x-pack/plugins/alerting/server/application/rule/methods/bulk_delete/bulk_delete_rules.ts b/x-pack/plugins/alerting/server/application/rule/methods/bulk_delete/bulk_delete_rules.ts index 0c1fa9a3fe1e9..cddeb0fad4f4a 100644 --- a/x-pack/plugins/alerting/server/application/rule/methods/bulk_delete/bulk_delete_rules.ts +++ b/x-pack/plugins/alerting/server/application/rule/methods/bulk_delete/bulk_delete_rules.ts @@ -37,6 +37,7 @@ import { ruleDomainSchema } from '../../schemas'; import type { RuleParams, RuleDomain } from '../../types'; import type { RawRule, SanitizedRule } from '../../../../types'; import { untrackRuleAlerts } from '../../../../rules_client/lib'; +import { deleteGaps } from '../../../../lib/rule_gaps/delete_gaps'; export const bulkDeleteRules = async ( context: RulesClientContext, @@ -89,6 +90,11 @@ export const bulkDeleteRules = async ( namespace: context.namespace, unsecuredSavedObjectsClient: context.unsecuredSavedObjectsClient, }), + deleteGaps({ + ruleIds: rules.map(({ id }) => id), + eventLogger: context.eventLogger, + logger: context.logger, + }), bulkMarkApiKeysForInvalidation( { apiKeys: apiKeysToInvalidate }, context.logger, diff --git a/x-pack/plugins/alerting/server/application/rule/methods/delete/delete_rule.ts b/x-pack/plugins/alerting/server/application/rule/methods/delete/delete_rule.ts index dd3aaf5e82f78..411e83d9d8872 100644 --- a/x-pack/plugins/alerting/server/application/rule/methods/delete/delete_rule.ts +++ b/x-pack/plugins/alerting/server/application/rule/methods/delete/delete_rule.ts @@ -18,6 +18,7 @@ import { RULE_SAVED_OBJECT_TYPE } from '../../../../saved_objects'; import { DeleteRuleParams } from './types'; import { deleteRuleParamsSchema } from './schemas'; import { deleteRuleSo, getDecryptedRuleSo, getRuleSo } from '../../../../data/rule'; +import { deleteGaps } from '../../../../lib/rule_gaps/delete_gaps'; export async function deleteRule(context: RulesClientContext, params: DeleteRuleParams) { try { @@ -118,6 +119,11 @@ async function deleteRuleWithOCC(context: RulesClientContext, { id }: { id: stri namespace: context.namespace, unsecuredSavedObjectsClient: context.unsecuredSavedObjectsClient, }), + deleteGaps({ + ruleIds: [id], + eventLogger: context.eventLogger, + logger: context.logger, + }), apiKeyToInvalidate && !apiKeyCreatedByUser ? bulkMarkApiKeysForInvalidation( { apiKeys: [apiKeyToInvalidate] }, diff --git a/x-pack/plugins/alerting/server/lib/alerting_event_logger/alerting_event_logger.ts b/x-pack/plugins/alerting/server/lib/alerting_event_logger/alerting_event_logger.ts index 45cfce1e828ed..d0105b391490b 100644 --- a/x-pack/plugins/alerting/server/lib/alerting_event_logger/alerting_event_logger.ts +++ b/x-pack/plugins/alerting/server/lib/alerting_event_logger/alerting_event_logger.ts @@ -437,6 +437,25 @@ export class AlertingEventLogger { }, }); } + + public async deleteGaps(ruleIds: string[]) { + return this.eventLogger.deleteEventsDocsByQuery({ + bool: { + must: [ + { + term: { + 'event.action': EVENT_LOG_ACTIONS.gap, + }, + }, + { + terms: { + 'rule.id': ruleIds, + }, + }, + ], + }, + }); + } } export function createAlertRecord( diff --git a/x-pack/plugins/alerting/server/lib/rule_gaps/delete_gaps.ts b/x-pack/plugins/alerting/server/lib/rule_gaps/delete_gaps.ts new file mode 100644 index 0000000000000..7808cd1ebf9ca --- /dev/null +++ b/x-pack/plugins/alerting/server/lib/rule_gaps/delete_gaps.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Logger } from '@kbn/core/server'; +import { IEventLogger } from '@kbn/event-log-plugin/server'; +import { AlertingEventLogger } from '../alerting_event_logger/alerting_event_logger'; + +/** + * Delete all gaps for this ruleId + */ +export const deleteGaps = async (params: { + ruleIds: string[]; + eventLogger: IEventLogger | undefined; + logger: Logger; +}) => { + const { ruleIds, logger, eventLogger } = params; + + try { + if (!eventLogger) { + throw new Error('DeleteGaps: EventLogger not initialized'); + } + const alertingEventLogger = new AlertingEventLogger(eventLogger); + await alertingEventLogger.deleteGaps(ruleIds); + } catch (err) { + logger.error(`Failed to delete gaps for rule ${ruleIds.join(', ')}: ${err.message}`); + throw err; + } +}; diff --git a/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts b/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts index 8a99d0ddf02d3..f4836dfb49d6b 100644 --- a/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts +++ b/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts @@ -188,9 +188,7 @@ export class ClusterClientAdapter< const bulkBody: Array> = []; - const docsToCreate = docs.filter((doc) => !doc.id); - - for (const doc of docsToCreate) { + for (const doc of docs) { if (doc.body === undefined) continue; bulkBody.push({ create: {} }); @@ -216,6 +214,11 @@ export class ClusterClientAdapter< } } + public async deleteByQueryDocs(query: estypes.QueryDslQueryContainer): Promise { + const esClient = await this.elasticsearchClientPromise; + await esClient.deleteByQuery({ index: this.esNames.dataStream, query }); + } + public async doesIndexTemplateExist(name: string): Promise { try { const esClient = await this.elasticsearchClientPromise; diff --git a/x-pack/plugins/event_log/server/event_logger.mock.ts b/x-pack/plugins/event_log/server/event_logger.mock.ts index ed243d98548f1..8b693ba6dcb17 100644 --- a/x-pack/plugins/event_log/server/event_logger.mock.ts +++ b/x-pack/plugins/event_log/server/event_logger.mock.ts @@ -13,6 +13,7 @@ const createEventLoggerMock = () => { updateEvent: jest.fn(), startTiming: jest.fn(), stopTiming: jest.fn(), + deleteEventsDocsByQuery: jest.fn(), }; return mock; }; diff --git a/x-pack/plugins/event_log/server/event_logger.ts b/x-pack/plugins/event_log/server/event_logger.ts index f38c9d8356259..64c02f64f14c5 100644 --- a/x-pack/plugins/event_log/server/event_logger.ts +++ b/x-pack/plugins/event_log/server/event_logger.ts @@ -7,6 +7,7 @@ import { schema } from '@kbn/config-schema'; import { Logger } from '@kbn/core/server'; +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { merge } from 'lodash'; import { coerce } from 'semver'; @@ -125,6 +126,10 @@ export class EventLogger implements IEventLogger { return result; } } + + async deleteEventsDocsByQuery(query: estypes.QueryDslQueryContainer): Promise { + return deleteByQuery(this.esContext, query); + } } // return the epoch millis of the start date, or null; may be NaN if garbage @@ -190,3 +195,10 @@ function indexEventDoc(esContext: EsContext, doc: Doc): void { async function updateEventDoc(esContext: EsContext, doc: Required): Promise { return esContext.esAdapter.updateDocument(doc); } + +async function deleteByQuery( + esContext: EsContext, + query: estypes.QueryDslQueryContainer +): Promise { + return esContext.esAdapter.deleteByQueryDocs(query); +} diff --git a/x-pack/plugins/event_log/server/types.ts b/x-pack/plugins/event_log/server/types.ts index 16130a7b1a861..618a1de18d8ea 100644 --- a/x-pack/plugins/event_log/server/types.ts +++ b/x-pack/plugins/event_log/server/types.ts @@ -8,6 +8,7 @@ import { schema, TypeOf } from '@kbn/config-schema'; import type { KibanaRequest } from '@kbn/core/server'; import { KueryNode } from '@kbn/es-query'; +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; export type { IEvent, IValidatedEvent } from '../generated/schemas'; export { EventSchema, ECS_VERSION } from '../generated/schemas'; @@ -86,4 +87,5 @@ export interface IEventLogger { startTiming(event: IEvent, startTime?: Date): void; stopTiming(event: IEvent): void; updateEvent(meta: DocMeta, event: IEvent): Promise; + deleteEventsDocsByQuery(query: estypes.QueryDslQueryContainer): Promise; } diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_gaps/components/rules_with_gaps_overview_panel/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_gaps/components/rules_with_gaps_overview_panel/index.tsx index ef0ae224ff0f5..7540d1451e645 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_gaps/components/rules_with_gaps_overview_panel/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_gaps/components/rules_with_gaps_overview_panel/index.tsx @@ -174,7 +174,7 @@ export const RulesWithGapsOverviewPanel = () => { From 1532aacb3bb5a6e6183f52709512e5407c903a28 Mon Sep 17 00:00:00 2001 From: Nikita Khristinin Date: Mon, 16 Dec 2024 12:34:04 +0100 Subject: [PATCH 06/17] remove unused logic --- .../plugins/event_log/server/es/cluster_client_adapter.ts | 1 - x-pack/plugins/event_log/server/event_log_start_service.ts | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts b/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts index f4836dfb49d6b..db014c9601bc9 100644 --- a/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts +++ b/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts @@ -110,7 +110,6 @@ export class ClusterClientAdapter< TDoc extends { body: AliasAny; index: string; - id?: string; meta?: DocMeta; } = Doc > { diff --git a/x-pack/plugins/event_log/server/event_log_start_service.ts b/x-pack/plugins/event_log/server/event_log_start_service.ts index e850d5342b1cf..94962c59f4dfc 100644 --- a/x-pack/plugins/event_log/server/event_log_start_service.ts +++ b/x-pack/plugins/event_log/server/event_log_start_service.ts @@ -38,12 +38,12 @@ export class EventLogClientService implements IEventLogClientService { this.spacesService = spacesService; } - getClient(requestOrSpaceId: KibanaRequest) { + getClient(request: KibanaRequest) { return new EventLogClient({ esContext: this.esContext, - savedObjectGetter: this.savedObjectProviderRegistry.getProvidersClient(requestOrSpaceId), + savedObjectGetter: this.savedObjectProviderRegistry.getProvidersClient(request), spacesService: this.spacesService, - request: requestOrSpaceId, + request, }); } } From 82d999c2265b91c3b08b781f46c78d7251d08818 Mon Sep 17 00:00:00 2001 From: Nikita Khristinin Date: Tue, 17 Dec 2024 20:17:25 +0100 Subject: [PATCH 07/17] add refresh --- .../rule_gaps/components/rule_gaps/index.tsx | 98 +++++++++++++++---- 1 file changed, 77 insertions(+), 21 deletions(-) diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_gaps/components/rule_gaps/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_gaps/components/rule_gaps/index.tsx index ec615724356c6..f5d6d32552427 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_gaps/components/rule_gaps/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_gaps/components/rule_gaps/index.tsx @@ -11,10 +11,11 @@ import { INTERNAL_ALERTING_GAPS_FIND_API_PATH, INTERNAL_ALERTING_GAPS_FILL_BY_ID_API_PATH, } from '@kbn/alerting-plugin/common'; +import styled from 'styled-components'; import React, { useCallback, useState } from 'react'; -import type { CriteriaWithPagination, EuiBasicTableColumn } from '@elastic/eui'; +import dateMath from '@kbn/datemath'; +import type { CriteriaWithPagination, EuiBasicTableColumn, OnTimeChangeProps } from '@elastic/eui'; import { - EuiButton, EuiBasicTable, EuiFlexGroup, EuiFlexItem, @@ -24,8 +25,13 @@ import { EuiText, EuiHealth, EuiButtonEmpty, + EuiSuperDatePicker, } from '@elastic/eui'; +const DatePickerEuiFlexItem = styled(EuiFlexItem)` + max-width: 582px; +`; + import type { FindGapsResponseBody } from '@kbn/alerting-plugin/common/routes/gaps/apis/find'; import type { FillGapByIdResponseV1 } from '@kbn/alerting-plugin/common/routes/gaps/apis/fill'; @@ -62,6 +68,8 @@ export const findGapsForRule = async ({ signal, sortField = 'createdAt', sortOrder = 'desc', + start, + end, }: { ruleId: string; page: number; @@ -69,18 +77,27 @@ export const findGapsForRule = async ({ signal?: AbortSignal; sortField?: string; sortOrder?: string; -}): Promise => - KibanaServices.get().http.fetch(INTERNAL_ALERTING_GAPS_FIND_API_PATH, { - method: 'GET', - query: { - rule_id: ruleId, - page, - per_page: perPage, - sort_field: sortField, - sort_order: sortOrder, - }, - signal, - }); +}): Promise => { + const startDate = dateMath.parse(start); + const endDate = dateMath.parse(end, { roundUp: true }); + + return KibanaServices.get().http.fetch( + INTERNAL_ALERTING_GAPS_FIND_API_PATH, + { + method: 'GET', + query: { + rule_id: ruleId, + page, + per_page: perPage, + sort_field: sortField, + sort_order: sortOrder, + start: startDate?.utc().toISOString(), + end: endDate?.utc().toISOString(), + }, + signal, + } + ); +}; /** * Fill gap by Id for the given rule ID @@ -122,17 +139,21 @@ export const useFindGapsForRule = ( ruleId, page, perPage, + start, + end, }: { ruleId: string; page: number; perPage: number; + start: string; + end: string; }, options?: UseQueryOptions ) => { return useQuery( [FIND_GAPS_FOR_RULE, ruleId, page, perPage], async ({ signal }) => { - const response = await findGapsForRule({ signal, ruleId, page, perPage }); + const response = await findGapsForRule({ signal, ruleId, page, perPage, start, end }); return response; }, @@ -312,14 +333,22 @@ const DEFAULT_PAGE_SIZE = 10; export const RuleGaps = ({ ruleId }: { ruleId: string }) => { const [pageIndex, setPageIndex] = useState(0); const [pageSize, setPageSize] = useState(DEFAULT_PAGE_SIZE); + const [dateRange, setDateRange] = useState<{ start: string; end: string }>({ + start: 'now-24h', + end: 'now', + }); const [{ canUserCRUD }] = useUserData(); const { timelines } = useKibana().services; const hasCRUDPermissions = hasUserCRUDPermission(canUserCRUD); + const [refreshInterval, setRefreshInterval] = useState(1000); + const [isPaused, setIsPaused] = useState(true); - const { data, isLoading, isError, refetch, dataUpdatedAt } = useFindGapsForRule({ + const { data, isLoading, isError, isFetching, refetch, dataUpdatedAt } = useFindGapsForRule({ ruleId, page: pageIndex + 1, perPage: pageSize, + start: dateRange.start, + end: dateRange.end, }); const pagination = { @@ -330,7 +359,7 @@ export const RuleGaps = ({ ruleId }: { ruleId: string }) => { const columns = getGapsTableColumns(hasCRUDPermissions, ruleId); - const handleRefresh = () => { + const onRefreshCallback = () => { refetch(); }; @@ -341,6 +370,23 @@ export const RuleGaps = ({ ruleId }: { ruleId: string }) => { } }; + const onTimeChangeCallback = useCallback( + (props: OnTimeChangeProps) => { + console.log('props', props); + setDateRange({ start: props.start, end: props.end }); + }, + [setDateRange] + ); + + const onRefreshChangeCallback = useCallback( + (props: OnRefreshChangeProps) => { + setIsPaused(props.isPaused); + // Only support auto-refresh >= 5s -- no current ability to limit within component + setRefreshInterval(props.refreshInterval > 5000 ? props.refreshInterval : 5000); + }, + [setIsPaused, setRefreshInterval] + ); + return ( @@ -351,10 +397,20 @@ export const RuleGaps = ({ ruleId }: { ruleId: string }) => { - - - {'Refresh'} - + + + + From d517c9e249caa5f5bb7b3c03ac11204f7c9c166e Mon Sep 17 00:00:00 2001 From: Nikita Khristinin Date: Mon, 23 Dec 2024 11:00:56 +0100 Subject: [PATCH 08/17] some fixes --- .../routes/gaps/apis/find/schemas/v1.ts | 17 ++- .../common/routes/gaps/response/schemas/v1.ts | 1 + .../server/lib/rule_gaps/find_gaps.ts | 20 ++- .../server/lib/rule_gaps/gap/index.ts | 8 + .../rule_gaps/transforms/transformToGap.ts | 1 + .../routes/gaps/apis/find/find_gaps_route.ts | 8 +- .../find/transforms/transform_request/v1.ts | 6 +- .../find/transforms/transform_response/v1.ts | 1 + .../server/es/cluster_client_adapter.ts | 2 + .../event_log/server/event_log_client.ts | 2 + .../rule_gaps/components/rule_gaps/index.tsx | 138 +++++++++++++----- .../components/rule_gaps/status_filter.tsx | 54 +++++++ .../components/rule_gaps/translations.tsx | 14 ++ .../rule_gaps/components/rule_gaps/utils.ts | 21 +++ .../detection_engine/rule_gaps/types.ts | 5 + 15 files changed, 249 insertions(+), 49 deletions(-) create mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_gaps/components/rule_gaps/status_filter.tsx create mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_gaps/components/rule_gaps/utils.ts diff --git a/x-pack/plugins/alerting/common/routes/gaps/apis/find/schemas/v1.ts b/x-pack/plugins/alerting/common/routes/gaps/apis/find/schemas/v1.ts index 905a877edb856..796531f963345 100644 --- a/x-pack/plugins/alerting/common/routes/gaps/apis/find/schemas/v1.ts +++ b/x-pack/plugins/alerting/common/routes/gaps/apis/find/schemas/v1.ts @@ -14,8 +14,23 @@ export const findQuerySchema = schema.object( per_page: schema.number({ defaultValue: 10, min: 0 }), rule_id: schema.maybe(schema.string()), start: schema.maybe(schema.string()), - sort_field: schema.maybe(schema.oneOf([schema.literal('createdAt'), schema.literal('start')])), + sort_field: schema.maybe( + schema.oneOf([ + schema.literal('@timestamp'), + schema.literal('status'), + schema.literal('total_gap_duration_ms'), + ]) + ), sort_order: schema.maybe(schema.oneOf([schema.literal('asc'), schema.literal('desc')])), + statuses: schema.maybe( + schema.arrayOf( + schema.oneOf([ + schema.literal('partially_filled'), + schema.literal('unfilled'), + schema.literal('filled'), + ]) + ) + ), }, { validate({ start, end }) { diff --git a/x-pack/plugins/alerting/common/routes/gaps/response/schemas/v1.ts b/x-pack/plugins/alerting/common/routes/gaps/response/schemas/v1.ts index 33077f264a1a9..def7d14c9055e 100644 --- a/x-pack/plugins/alerting/common/routes/gaps/response/schemas/v1.ts +++ b/x-pack/plugins/alerting/common/routes/gaps/response/schemas/v1.ts @@ -23,6 +23,7 @@ export const rangeSchema = schema.object({ export const rangeListSchema = schema.arrayOf(rangeSchema); export const gapsResponseSchema = schema.object({ + '@timestamp': schema.string(), _id: schema.string(), status: gapStatusSchema, range: rangeSchema, diff --git a/x-pack/plugins/alerting/server/lib/rule_gaps/find_gaps.ts b/x-pack/plugins/alerting/server/lib/rule_gaps/find_gaps.ts index 315625ad03997..fe55bb0c8dd19 100644 --- a/x-pack/plugins/alerting/server/lib/rule_gaps/find_gaps.ts +++ b/x-pack/plugins/alerting/server/lib/rule_gaps/find_gaps.ts @@ -27,7 +27,8 @@ export const findGaps = async ({ page: number; perPage: number; }> => { - const { ruleId, start, end, page, perPage, statuses } = params; + const { ruleId, start, end, page, perPage, statuses, sortField, sortOrder } = params; + console.log('-----------------statuses', statuses); try { const statusesFilter = statuses ?.map((status) => `kibana.alert.rule.gap.status : ${status}`) @@ -36,6 +37,14 @@ export const findGaps = async ({ end && start ? `AND (kibana.alert.rule.gap.range <= "${end}" AND kibana.alert.rule.gap.range >= "${start}")` : ''; + + const getField = (field?: string) => { + if (field === '@timestamp' || !field) { + return '@timestamp'; + } + return `kibana.alert.rule.gap.${field}`; + }; + const gapsResponse = await eventLogClient.findEventsBySavedObjectIds( RULE_SAVED_OBJECT_TYPE, [ruleId], @@ -43,7 +52,12 @@ export const findGaps = async ({ filter: `event.action: gap AND event.provider: alerting ${rangeFilter} ${ statusesFilter ? `AND (${statusesFilter})` : '' }`, - sort: [{ sort_field: '@timestamp', sort_order: 'desc' }], + sort: [ + { + sort_field: getField(sortField), + sort_order: sortOrder ?? 'desc', + }, + ], page, per_page: perPage, } @@ -56,7 +70,7 @@ export const findGaps = async ({ perPage: gapsResponse.per_page, }; } catch (err) { - logger.error(`Failed to find gaps for rule ${ruleId.toString()}: ${err.message}`); + logger.error(`Failed to find gaps for rule ${ruleId}: ${err.message}`); throw err; } }; diff --git a/x-pack/plugins/alerting/server/lib/rule_gaps/gap/index.ts b/x-pack/plugins/alerting/server/lib/rule_gaps/gap/index.ts index b946e72dedb78..7b4b559d85b6d 100644 --- a/x-pack/plugins/alerting/server/lib/rule_gaps/gap/index.ts +++ b/x-pack/plugins/alerting/server/lib/rule_gaps/gap/index.ts @@ -21,6 +21,7 @@ import { } from './interval_utils'; interface GapConstructorParams { + timestamp: string; range: StringInterval; filledIntervals?: StringInterval[]; inProgressIntervals?: StringInterval[]; @@ -32,8 +33,10 @@ export class Gap { private _filledIntervals: Interval[]; private _inProgressIntervals: Interval[]; private _meta?: DocMeta; + private _timestamp: string; constructor({ + timestamp, range, filledIntervals = [], inProgressIntervals = [], @@ -45,6 +48,7 @@ export class Gap { if (meta) { this._meta = meta; } + this._timestamp = timestamp; } public fillGap(interval: Interval): void { @@ -83,6 +87,10 @@ export class Gap { return this._inProgressIntervals; } + public get timestamp() { + return this._timestamp; + } + /** * unfilled = range - (filled + inProgress) */ diff --git a/x-pack/plugins/alerting/server/lib/rule_gaps/transforms/transformToGap.ts b/x-pack/plugins/alerting/server/lib/rule_gaps/transforms/transformToGap.ts index 2713e2237067c..d83f3a77f1870 100644 --- a/x-pack/plugins/alerting/server/lib/rule_gaps/transforms/transformToGap.ts +++ b/x-pack/plugins/alerting/server/lib/rule_gaps/transforms/transformToGap.ts @@ -36,6 +36,7 @@ export const transformToGap = (events: QueryEventsBySavedObjectResult): Gap[] => const inProgressIntervals = validateIntervals(gap?.in_progress_intervals); return new Gap({ + timestamp: doc['@timestamp'], range, filledIntervals, inProgressIntervals, diff --git a/x-pack/plugins/alerting/server/routes/gaps/apis/find/find_gaps_route.ts b/x-pack/plugins/alerting/server/routes/gaps/apis/find/find_gaps_route.ts index fae1e9952a258..c6bcbcb37ae30 100644 --- a/x-pack/plugins/alerting/server/routes/gaps/apis/find/find_gaps_route.ts +++ b/x-pack/plugins/alerting/server/routes/gaps/apis/find/find_gaps_route.ts @@ -22,11 +22,11 @@ export const findGapsRoute = ( router: IRouter, licenseState: ILicenseState ) => { - router.get( + router.post( { path: `${INTERNAL_ALERTING_GAPS_FIND_API_PATH}`, validate: { - query: findQuerySchemaV1, + body: findQuerySchemaV1, }, options: { access: 'internal', @@ -35,8 +35,8 @@ export const findGapsRoute = ( router.handleLegacyErrors( verifyAccessAndContext(licenseState, async function (context, req, res) { const rulesClient = (await context.alerting).getRulesClient(); - const query: FindGapsRequestQueryV1 = req.query; - + const query: FindGapsRequestQueryV1 = req.body; + console.log('-----------------query', query); const result = await rulesClient.findGaps(transformRequestV1(query)); const response: FindGapsResponseV1 = { body: transformResponseV1(result), diff --git a/x-pack/plugins/alerting/server/routes/gaps/apis/find/transforms/transform_request/v1.ts b/x-pack/plugins/alerting/server/routes/gaps/apis/find/transforms/transform_request/v1.ts index 3e73322011695..76d2c470b43cd 100644 --- a/x-pack/plugins/alerting/server/routes/gaps/apis/find/transforms/transform_request/v1.ts +++ b/x-pack/plugins/alerting/server/routes/gaps/apis/find/transforms/transform_request/v1.ts @@ -10,18 +10,20 @@ import { FindGapsRequestQueryV1 } from '../../../../../../../common/routes/gaps/ import { FindGapsParams } from '../../../../../../lib/rule_gaps/types'; export const transformRequest = ({ - end, page, per_page, rule_id, start, + end, sort_field, sort_order, + statuses, }: FindGapsRequestQueryV1): FindGapsParams => ({ + ruleId: rule_id, end, page, perPage: per_page, - ruleId: rule_id, + statuses, start, sortField: sort_field, sortOrder: sort_order, diff --git a/x-pack/plugins/alerting/server/routes/gaps/apis/find/transforms/transform_response/v1.ts b/x-pack/plugins/alerting/server/routes/gaps/apis/find/transforms/transform_response/v1.ts index 412a8c8e131a7..a3e9b3b5852d1 100644 --- a/x-pack/plugins/alerting/server/routes/gaps/apis/find/transforms/transform_response/v1.ts +++ b/x-pack/plugins/alerting/server/routes/gaps/apis/find/transforms/transform_response/v1.ts @@ -26,5 +26,6 @@ export const transformResponse = ({ data: gapsData.map((gap) => ({ _id: gap?.meta?._id, ...gap.getEsObject(), + '@timestamp': gap.timestamp, })), }); diff --git a/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts b/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts index db014c9601bc9..b8fb1fc1392a3 100644 --- a/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts +++ b/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts @@ -448,6 +448,8 @@ export class ClusterClientAdapter< : {}), }; + console.log('-----------------body', JSON.stringify(body, null, 2)); + try { const { hits: { hits, total }, diff --git a/x-pack/plugins/event_log/server/event_log_client.ts b/x-pack/plugins/event_log/server/event_log_client.ts index 4ffcfb6cef84e..79e20a98ecfd3 100644 --- a/x-pack/plugins/event_log/server/event_log_client.ts +++ b/x-pack/plugins/event_log/server/event_log_client.ts @@ -40,6 +40,8 @@ const sortSchema = schema.object({ schema.literal('event.duration'), schema.literal('event.action'), schema.literal('message'), + schema.literal('kibana.alert.rule.gap.status'), + schema.literal('kibana.alert.rule.gap.total_gap_duration_ms'), ]), sort_order: schema.oneOf([schema.literal('asc'), schema.literal('desc')]), }); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_gaps/components/rule_gaps/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_gaps/components/rule_gaps/index.tsx index f5d6d32552427..419b29687a825 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_gaps/components/rule_gaps/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_gaps/components/rule_gaps/index.tsx @@ -14,7 +14,12 @@ import { import styled from 'styled-components'; import React, { useCallback, useState } from 'react'; import dateMath from '@kbn/datemath'; -import type { CriteriaWithPagination, EuiBasicTableColumn, OnTimeChangeProps } from '@elastic/eui'; +import type { + CriteriaWithPagination, + EuiBasicTableColumn, + OnRefreshChangeProps, + OnTimeChangeProps, +} from '@elastic/eui'; import { EuiBasicTable, EuiFlexGroup, @@ -49,9 +54,9 @@ import { FormattedDate } from '../../../../common/components/formatted_date'; import { KibanaServices, useKibana } from '../../../../common/lib/kibana'; import { getFormattedDuration } from '../../../rule_details_ui/pages/rule_details/execution_log_table/rule_duration_format'; import * as i18n from './translations'; - -export type Gap = FindGapsResponseBody['data']['0']; -export type GapStatus = Gap['status']; +import type { Gap, GapStatus } from '../../types'; +import { getStatusLabel } from './utils'; +import { GapStatusFilter } from './status_filter'; const FIND_GAPS_FOR_RULE = 'FIND_GAP_FOR_RULE'; @@ -66,14 +71,18 @@ export const findGapsForRule = async ({ page, perPage, signal, - sortField = 'createdAt', + sortField = '@timestamp', sortOrder = 'desc', start, end, + statuses, }: { ruleId: string; page: number; perPage: number; + start: string; + end: string; + statuses: GapStatus[]; signal?: AbortSignal; sortField?: string; sortOrder?: string; @@ -84,8 +93,8 @@ export const findGapsForRule = async ({ return KibanaServices.get().http.fetch( INTERNAL_ALERTING_GAPS_FIND_API_PATH, { - method: 'GET', - query: { + method: 'POST', + body: JSON.stringify({ rule_id: ruleId, page, per_page: perPage, @@ -93,7 +102,8 @@ export const findGapsForRule = async ({ sort_order: sortOrder, start: startDate?.utc().toISOString(), end: endDate?.utc().toISOString(), - }, + statuses, + }), signal, } ); @@ -141,19 +151,35 @@ export const useFindGapsForRule = ( perPage, start, end, + statuses, + sortField, + sortOrder, }: { ruleId: string; page: number; perPage: number; start: string; end: string; + statuses: GapStatus[]; + sortField: keyof Gap; + sortOrder: string; }, options?: UseQueryOptions ) => { return useQuery( - [FIND_GAPS_FOR_RULE, ruleId, page, perPage], + [FIND_GAPS_FOR_RULE, ruleId, page, perPage, statuses?.join(','), sortField, sortOrder], async ({ signal }) => { - const response = await findGapsForRule({ signal, ruleId, page, perPage, start, end }); + const response = await findGapsForRule({ + signal, + ruleId, + page, + perPage, + start, + end, + statuses, + sortField, + sortOrder, + }); return response; }, @@ -232,18 +258,6 @@ const FillGap = ({ ruleId, gap }: { ruleId: string; gap: Gap }) => { ); }; -const getStatusLabel = (status: string) => { - switch (status) { - case 'partially_filled': - return i18n.GAP_STATUS_PARTIALLY_FILLED; - case 'unfilled': - return i18n.GAP_STATUS_UNFILLED; - case 'filled': - return i18n.GAP_STATUS_FILLED; - } - return ''; -}; - const getGapsTableColumns = (hasCRUDPermissions: boolean, ruleId: string) => { const fillActions = { name: i18n.GAPS_TABLE_ACTIONS_LABEL, @@ -255,10 +269,20 @@ const getGapsTableColumns = (hasCRUDPermissions: boolean, ruleId: string) => { const columns: Array> = [ { field: 'status', + sortable: true, name: , - render: (value: string) => getStatusLabel(value), + render: (value: GapStatus) => getStatusLabel(value), width: '10%', }, + { + field: '@timestamp', + sortable: true, + name: , + render: (value: Gap['@timestamp']) => ( + + ), + width: '15%', + }, { field: 'in_progress_intervals', name: ( @@ -313,6 +337,7 @@ const getGapsTableColumns = (hasCRUDPermissions: boolean, ruleId: string) => { }, { field: 'total_gap_duration_ms', + sortable: true, name: ( ), @@ -342,6 +367,11 @@ export const RuleGaps = ({ ruleId }: { ruleId: string }) => { const hasCRUDPermissions = hasUserCRUDPermission(canUserCRUD); const [refreshInterval, setRefreshInterval] = useState(1000); const [isPaused, setIsPaused] = useState(true); + const [selectedStatuses, setSelectedStatuses] = useState([]); + const [sort, setSort] = useState<{ field: keyof Gap; direction: 'desc' | 'asc' }>({ + field: '@timestamp', + direction: 'desc', + }); const { data, isLoading, isError, isFetching, refetch, dataUpdatedAt } = useFindGapsForRule({ ruleId, @@ -349,6 +379,9 @@ export const RuleGaps = ({ ruleId }: { ruleId: string }) => { perPage: pageSize, start: dateRange.start, end: dateRange.end, + statuses: selectedStatuses, + sortField: sort.field, + sortOrder: sort.direction, }); const pagination = { @@ -363,16 +396,21 @@ export const RuleGaps = ({ ruleId }: { ruleId: string }) => { refetch(); }; - const handleTableChange: (params: CriteriaWithPagination) => void = ({ page }) => { + const handleTableChange: (params: CriteriaWithPagination) => void = ({ + page, + sort: newSort, + }) => { if (page) { setPageIndex(page.index); setPageSize(page.size); } + if (newSort) { + setSort(newSort); + } }; const onTimeChangeCallback = useCallback( (props: OnTimeChangeProps) => { - console.log('props', props); setDateRange({ start: props.start, end: props.end }); }, [setDateRange] @@ -387,9 +425,21 @@ export const RuleGaps = ({ ruleId }: { ruleId: string }) => { [setIsPaused, setRefreshInterval] ); + const handleStatusChange = useCallback( + (statuses: GapStatus[]) => { + setSelectedStatuses(statuses); + }, + [setSelectedStatuses] + ); + return ( - + @@ -398,19 +448,26 @@ export const RuleGaps = ({ ruleId }: { ruleId: string }) => { - - - + + + + + + + + + + @@ -429,6 +486,9 @@ export const RuleGaps = ({ ruleId }: { ruleId: string }) => { error={isError ? 'error' : undefined} loading={isLoading} onChange={handleTableChange} + sorting={{ + sort, + }} /> ); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_gaps/components/rule_gaps/status_filter.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_gaps/components/rule_gaps/status_filter.tsx new file mode 100644 index 0000000000000..ebe90a9a5ace0 --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_gaps/components/rule_gaps/status_filter.tsx @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useCallback } from 'react'; +import { EuiFilterGroup } from '@elastic/eui'; +import { MultiselectFilter } from '../../../../common/components/multiselect_filter'; +import * as i18n from './translations'; +import type { GapStatus } from '../../types'; +import { getStatusLabel } from './utils'; + +interface GapStatusFilterComponent { + selectedItems: GapStatus[]; + onChange: (selectedItems: GapStatusItem[]) => void; +} + +const items: GapStatus[] = ['partially_filled', 'unfilled', 'filled']; + +const GapStatusFilterComponent: React.FC = ({ + selectedItems, + onChange, +}) => { + const renderItem = useCallback((status: GapStatus) => { + return getStatusLabel(status); + }, []); + + const handleSelectionChange = useCallback( + (statuses: GapStatus[]) => { + console.log('statuses', statuses); + onChange(statuses); + }, + [onChange] + ); + + return ( + + + data-test-subj="GapStatusTypeFilter" + title={i18n.GAP_STATUS_FILTER_TITLE} + items={items} + selectedItems={selectedItems} + onSelectionChange={handleSelectionChange} + renderItem={renderItem} + width={200} + /> + + ); +}; + +export const GapStatusFilter = React.memo(GapStatusFilterComponent); +GapStatusFilter.displayName = 'GapStatusFilter'; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_gaps/components/rule_gaps/translations.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_gaps/components/rule_gaps/translations.tsx index ee40f5e4221ef..5e14d78ab8874 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_gaps/components/rule_gaps/translations.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_gaps/components/rule_gaps/translations.tsx @@ -110,3 +110,17 @@ export const GAP_FILL_REQUEST_ERROR_MESSAGE = i18n.translate( defaultMessage: 'Failed to request manual run', } ); + +export const GAP_STATUS_FILTER_TITLE = i18n.translate( + 'xpack.securitySolution.gapsTable.gapStatusFilterTitle', + { + defaultMessage: 'Status', + } +); + +export const GAPS_TABLE_EVENT_TIME_LABEL = i18n.translate( + 'xpack.securitySolution.gapsTable.eventTimeLabel', + { + defaultMessage: 'Detected at', + } +); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_gaps/components/rule_gaps/utils.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_gaps/components/rule_gaps/utils.ts new file mode 100644 index 0000000000000..71ada001dfa33 --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_gaps/components/rule_gaps/utils.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { GapStatus } from '../../types'; +import * as i18n from './translations'; + +export const getStatusLabel = (status: GapStatus) => { + switch (status) { + case 'partially_filled': + return i18n.GAP_STATUS_PARTIALLY_FILLED; + case 'unfilled': + return i18n.GAP_STATUS_UNFILLED; + case 'filled': + return i18n.GAP_STATUS_FILLED; + } + return ''; +}; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_gaps/types.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_gaps/types.ts index 03b9ed4a4bea0..223c6ad97aac3 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_gaps/types.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_gaps/types.ts @@ -5,12 +5,17 @@ * 2.0. */ +import type { FindGapsResponseBody } from '@kbn/alerting-plugin/common/routes/gaps/apis/find'; + import type { FindBackfillResponseBody } from '@kbn/alerting-plugin/common/routes/backfill/apis/find'; export type Backfill = FindBackfillResponseBody['data']['0']; export type BackfillStatus = Backfill['status']; +export type Gap = FindGapsResponseBody['data']['0']; +export type GapStatus = Gap['status']; + export interface BackfillStats { total: number; complete: number; From 36c02be1ae1a807261b04272cd60de7b0b47d7fc Mon Sep 17 00:00:00 2001 From: Nikita Khristinin Date: Tue, 7 Jan 2025 16:30:36 +0100 Subject: [PATCH 09/17] clean, add tests and fix types --- .../feature_privilege_builder/alerting.ts | 2 + .../common/routes/gaps/apis/find/index.ts | 6 +- .../routes/gaps/apis/find/schemas/v1.ts | 4 +- .../methods/bulk_delete/bulk_delete_rules.ts | 6 - .../rule/methods/delete/delete_rule.ts | 6 - .../rule/methods/find_gaps/find_gaps.ts | 30 +- .../alerting/server/authorization/types.ts | 2 + .../alerting_event_logger.ts | 31 +- .../server/lib/rule_gaps/delete_gaps.ts | 32 -- .../server/lib/rule_gaps/find_gap_by_id.ts | 2 +- .../server/lib/rule_gaps/find_gaps.ts | 4 +- .../server/lib/rule_gaps/gap/index.ts | 24 +- .../transforms/transform_to_gap.test.ts | 155 +++++++++ ...{transformToGap.ts => transform_to_gap.ts} | 9 +- .../update/add_filled_interval_to_gaps.ts | 6 +- .../add_in_progress_intervals_to_gaps.ts | 6 +- ...clculate_in_progress_intervals_for_gaps.ts | 6 +- .../gaps/apis/find/find_gaps_route.test.ts | 106 ++++++ .../routes/gaps/apis/find/find_gaps_route.ts | 7 +- .../alerting/server/rules_client.mock.ts | 3 + .../rules_client/common/audit_events.ts | 7 + .../server/es/cluster_client_adapter.mock.ts | 1 + .../server/es/cluster_client_adapter.test.ts | 93 +++++- .../server/es/cluster_client_adapter.ts | 53 +-- .../event_log/server/event_logger.mock.ts | 1 - .../shared/event_log/server/event_logger.ts | 18 +- .../plugins/shared/event_log/server/index.ts | 4 +- .../plugins/shared/event_log/server/types.ts | 8 +- .../common/plugins/alerts/server/plugin.ts | 10 +- .../common/plugins/alerts/server/routes.ts | 144 ++++++++- .../group1/tests/alerting/gap/find.ts | 305 ++++++++++++++++++ .../group1/tests/alerting/gap/index.ts | 15 + .../group1/tests/alerting/index.ts | 1 + .../tests/alerting/group1/event_log.ts | 96 ++++++ 34 files changed, 1047 insertions(+), 156 deletions(-) delete mode 100644 x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/delete_gaps.ts create mode 100644 x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/transforms/transform_to_gap.test.ts rename x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/transforms/{transformToGap.ts => transform_to_gap.ts} (90%) create mode 100644 x-pack/platform/plugins/shared/alerting/server/routes/gaps/apis/find/find_gaps_route.test.ts create mode 100644 x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/gap/find.ts create mode 100644 x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/gap/index.ts diff --git a/x-pack/packages/security/authorization_core/src/privileges/feature_privilege_builder/alerting.ts b/x-pack/packages/security/authorization_core/src/privileges/feature_privilege_builder/alerting.ts index 65c330b94c462..cc94f400ddab1 100644 --- a/x-pack/packages/security/authorization_core/src/privileges/feature_privilege_builder/alerting.ts +++ b/x-pack/packages/security/authorization_core/src/privileges/feature_privilege_builder/alerting.ts @@ -28,6 +28,7 @@ const readOperations: Record = { 'getRuleExecutionKPI', 'getBackfill', 'findBackfill', + 'findGaps', ], alert: ['get', 'find', 'getAuthorizedAlertsIndices', 'getAlertSummary'], }; @@ -53,6 +54,7 @@ const writeOperations: Record = { 'runSoon', 'scheduleBackfill', 'deleteBackfill', + 'fillGaps', ], alert: ['update'], }; diff --git a/x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/find/index.ts b/x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/find/index.ts index 43ab071202721..e4127a01f535e 100644 --- a/x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/find/index.ts +++ b/x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/find/index.ts @@ -5,12 +5,12 @@ * 2.0. */ -export { findQuerySchema, findResponseSchema } from './schemas/latest'; +export { findGapsBodySchema, findGapsResponseSchema } from './schemas/latest'; export type { FindGapsRequestQuery, FindGapsResponseBody, FindGapsResponse } from './types/latest'; export { - findQuerySchema as findQuerySchemaV1, - findResponseSchema as findResponseSchemaV1, + findGapsBodySchema as findGapsBodySchemaV1, + findGapsResponseSchema as findGapsResponseSchemaV1, } from './schemas/v1'; export type { FindGapsRequestQuery as FindGapsRequestQueryV1, diff --git a/x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/find/schemas/v1.ts b/x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/find/schemas/v1.ts index 796531f963345..c69e5fc78dba7 100644 --- a/x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/find/schemas/v1.ts +++ b/x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/find/schemas/v1.ts @@ -7,7 +7,7 @@ import { schema } from '@kbn/config-schema'; import { gapsResponseSchemaV1 } from '../../../response'; -export const findQuerySchema = schema.object( +export const findGapsBodySchema = schema.object( { end: schema.maybe(schema.string()), page: schema.number({ defaultValue: 1, min: 1 }), @@ -50,7 +50,7 @@ export const findQuerySchema = schema.object( } ); -export const findResponseSchema = schema.object({ +export const findGapsResponseSchema = schema.object({ page: schema.number(), per_page: schema.number(), total: schema.number(), diff --git a/x-pack/platform/plugins/shared/alerting/server/application/rule/methods/bulk_delete/bulk_delete_rules.ts b/x-pack/platform/plugins/shared/alerting/server/application/rule/methods/bulk_delete/bulk_delete_rules.ts index cddeb0fad4f4a..0c1fa9a3fe1e9 100644 --- a/x-pack/platform/plugins/shared/alerting/server/application/rule/methods/bulk_delete/bulk_delete_rules.ts +++ b/x-pack/platform/plugins/shared/alerting/server/application/rule/methods/bulk_delete/bulk_delete_rules.ts @@ -37,7 +37,6 @@ import { ruleDomainSchema } from '../../schemas'; import type { RuleParams, RuleDomain } from '../../types'; import type { RawRule, SanitizedRule } from '../../../../types'; import { untrackRuleAlerts } from '../../../../rules_client/lib'; -import { deleteGaps } from '../../../../lib/rule_gaps/delete_gaps'; export const bulkDeleteRules = async ( context: RulesClientContext, @@ -90,11 +89,6 @@ export const bulkDeleteRules = async ( namespace: context.namespace, unsecuredSavedObjectsClient: context.unsecuredSavedObjectsClient, }), - deleteGaps({ - ruleIds: rules.map(({ id }) => id), - eventLogger: context.eventLogger, - logger: context.logger, - }), bulkMarkApiKeysForInvalidation( { apiKeys: apiKeysToInvalidate }, context.logger, diff --git a/x-pack/platform/plugins/shared/alerting/server/application/rule/methods/delete/delete_rule.ts b/x-pack/platform/plugins/shared/alerting/server/application/rule/methods/delete/delete_rule.ts index 411e83d9d8872..dd3aaf5e82f78 100644 --- a/x-pack/platform/plugins/shared/alerting/server/application/rule/methods/delete/delete_rule.ts +++ b/x-pack/platform/plugins/shared/alerting/server/application/rule/methods/delete/delete_rule.ts @@ -18,7 +18,6 @@ import { RULE_SAVED_OBJECT_TYPE } from '../../../../saved_objects'; import { DeleteRuleParams } from './types'; import { deleteRuleParamsSchema } from './schemas'; import { deleteRuleSo, getDecryptedRuleSo, getRuleSo } from '../../../../data/rule'; -import { deleteGaps } from '../../../../lib/rule_gaps/delete_gaps'; export async function deleteRule(context: RulesClientContext, params: DeleteRuleParams) { try { @@ -119,11 +118,6 @@ async function deleteRuleWithOCC(context: RulesClientContext, { id }: { id: stri namespace: context.namespace, unsecuredSavedObjectsClient: context.unsecuredSavedObjectsClient, }), - deleteGaps({ - ruleIds: [id], - eventLogger: context.eventLogger, - logger: context.logger, - }), apiKeyToInvalidate && !apiKeyCreatedByUser ? bulkMarkApiKeysForInvalidation( { apiKeys: [apiKeyToInvalidate] }, diff --git a/x-pack/platform/plugins/shared/alerting/server/application/rule/methods/find_gaps/find_gaps.ts b/x-pack/platform/plugins/shared/alerting/server/application/rule/methods/find_gaps/find_gaps.ts index e7d0f43635a3b..612c2452b9f88 100644 --- a/x-pack/platform/plugins/shared/alerting/server/application/rule/methods/find_gaps/find_gaps.ts +++ b/x-pack/platform/plugins/shared/alerting/server/application/rule/methods/find_gaps/find_gaps.ts @@ -7,13 +7,41 @@ import Boom from '@hapi/boom'; -import { RulesClientContext } from '../../../../rules_client'; +import { ReadOperations, AlertingAuthorizationEntity } from '../../../../authorization'; +import { RulesClientContext } from '../../../../rules_client'; import { findGaps as _findGaps } from '../../../../lib/rule_gaps/find_gaps'; import { FindGapsParams } from '../../../../lib/rule_gaps/types'; +import { getRule } from '../get/get_rule'; +import { SanitizedRuleWithLegacyId } from '../../../../types'; +import { ruleAuditEvent, RuleAuditAction } from '../../../../rules_client/common/audit_events'; +import { RULE_SAVED_OBJECT_TYPE } from '../../../../saved_objects'; export async function findGaps(context: RulesClientContext, params: FindGapsParams) { try { + const rule = (await getRule(context, { + id: params.ruleId, + includeLegacyId: true, + })) as SanitizedRuleWithLegacyId; + try { + // Make sure user has access to this rule + await context.authorization.ensureAuthorized({ + ruleTypeId: rule.alertTypeId, + consumer: rule.consumer, + operation: ReadOperations.FindGaps, + entity: AlertingAuthorizationEntity.Rule, + }); + } catch (error) { + context.auditLogger?.log( + ruleAuditEvent({ + action: RuleAuditAction.FIND_GAPS, + savedObject: { type: RULE_SAVED_OBJECT_TYPE, id: rule.id, name: rule.name }, + error, + }) + ); + throw error; + } + const eventLogClient = await context.getEventLogClient(); const gaps = await _findGaps({ params, diff --git a/x-pack/platform/plugins/shared/alerting/server/authorization/types.ts b/x-pack/platform/plugins/shared/alerting/server/authorization/types.ts index 77357456c74b6..8668884463667 100644 --- a/x-pack/platform/plugins/shared/alerting/server/authorization/types.ts +++ b/x-pack/platform/plugins/shared/alerting/server/authorization/types.ts @@ -21,6 +21,7 @@ export enum ReadOperations { GetRuleExecutionKPI = 'getRuleExecutionKPI', GetBackfill = 'getBackfill', FindBackfill = 'findBackfill', + FindGaps = 'findGaps', } export enum WriteOperations { @@ -43,4 +44,5 @@ export enum WriteOperations { RunSoon = 'runSoon', ScheduleBackfill = 'scheduleBackfill', DeleteBackfill = 'deleteBackfill', + FillGaps = 'fillGaps', } diff --git a/x-pack/platform/plugins/shared/alerting/server/lib/alerting_event_logger/alerting_event_logger.ts b/x-pack/platform/plugins/shared/alerting/server/lib/alerting_event_logger/alerting_event_logger.ts index f10692b951fa0..26304687843c2 100644 --- a/x-pack/platform/plugins/shared/alerting/server/lib/alerting_event_logger/alerting_event_logger.ts +++ b/x-pack/platform/plugins/shared/alerting/server/lib/alerting_event_logger/alerting_event_logger.ts @@ -10,7 +10,7 @@ import { IEventLogger, millisToNanos, SAVED_OBJECT_REL_PRIMARY, - DocMeta, + InternalFields, } from '@kbn/event-log-plugin/server'; import { EVENT_LOG_ACTIONS } from '../../plugin'; import { UntypedNormalizedRuleType } from '../../rule_type_registry'; @@ -425,8 +425,14 @@ export class AlertingEventLogger { ); } - public async updateGap({ meta, gap }: { meta: DocMeta; gap: GapBase }): Promise { - return this.eventLogger.updateEvent(meta, { + public async updateGap({ + internalFields, + gap, + }: { + internalFields: InternalFields; + gap: GapBase; + }): Promise { + return this.eventLogger.updateEvent(internalFields, { kibana: { alert: { rule: { @@ -436,25 +442,6 @@ export class AlertingEventLogger { }, }); } - - public async deleteGaps(ruleIds: string[]) { - return this.eventLogger.deleteEventsDocsByQuery({ - bool: { - must: [ - { - term: { - 'event.action': EVENT_LOG_ACTIONS.gap, - }, - }, - { - terms: { - 'rule.id': ruleIds, - }, - }, - ], - }, - }); - } } export function createAlertRecord( diff --git a/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/delete_gaps.ts b/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/delete_gaps.ts deleted file mode 100644 index 7808cd1ebf9ca..0000000000000 --- a/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/delete_gaps.ts +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { Logger } from '@kbn/core/server'; -import { IEventLogger } from '@kbn/event-log-plugin/server'; -import { AlertingEventLogger } from '../alerting_event_logger/alerting_event_logger'; - -/** - * Delete all gaps for this ruleId - */ -export const deleteGaps = async (params: { - ruleIds: string[]; - eventLogger: IEventLogger | undefined; - logger: Logger; -}) => { - const { ruleIds, logger, eventLogger } = params; - - try { - if (!eventLogger) { - throw new Error('DeleteGaps: EventLogger not initialized'); - } - const alertingEventLogger = new AlertingEventLogger(eventLogger); - await alertingEventLogger.deleteGaps(ruleIds); - } catch (err) { - logger.error(`Failed to delete gaps for rule ${ruleIds.join(', ')}: ${err.message}`); - throw err; - } -}; diff --git a/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/find_gap_by_id.ts b/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/find_gap_by_id.ts index 85b9864f91589..977d7cb9a6a9c 100644 --- a/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/find_gap_by_id.ts +++ b/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/find_gap_by_id.ts @@ -10,7 +10,7 @@ import { Logger } from '@kbn/core/server'; import { RULE_SAVED_OBJECT_TYPE } from '../../saved_objects'; import { FindGapByIdParams } from './types'; import { Gap } from './gap'; -import { transformToGap } from './transforms/transformToGap'; +import { transformToGap } from './transforms/transform_to_gap'; export const findGapById = async ({ eventLogClient, diff --git a/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/find_gaps.ts b/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/find_gaps.ts index fe55bb0c8dd19..c4de2b8087da7 100644 --- a/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/find_gaps.ts +++ b/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/find_gaps.ts @@ -10,7 +10,7 @@ import { Logger } from '@kbn/core/server'; import { RULE_SAVED_OBJECT_TYPE } from '../../saved_objects'; import { FindGapsParams } from './types'; import { Gap } from './gap'; -import { transformToGap } from './transforms/transformToGap'; +import { transformToGap } from './transforms/transform_to_gap'; import { GapStatus } from '../../../common/constants/gap_status'; export const findGaps = async ({ @@ -28,7 +28,7 @@ export const findGaps = async ({ perPage: number; }> => { const { ruleId, start, end, page, perPage, statuses, sortField, sortOrder } = params; - console.log('-----------------statuses', statuses); + try { const statusesFilter = statuses ?.map((status) => `kibana.alert.rule.gap.status : ${status}`) diff --git a/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/gap/index.ts b/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/gap/index.ts index 7b4b559d85b6d..766c7dc734c79 100644 --- a/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/gap/index.ts +++ b/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/gap/index.ts @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { DocMeta } from '@kbn/event-log-plugin/server/es/cluster_client_adapter'; +import { InternalFields } from '@kbn/event-log-plugin/server/es/cluster_client_adapter'; import { GapStatus, gapStatus } from '../../../../common/constants'; import { Interval, StringInterval, GapBase } from '../types'; @@ -21,34 +21,36 @@ import { } from './interval_utils'; interface GapConstructorParams { - timestamp: string; + timestamp?: string; range: StringInterval; filledIntervals?: StringInterval[]; inProgressIntervals?: StringInterval[]; - meta?: DocMeta; + internalFields?: InternalFields; } export class Gap { private _range: Interval; private _filledIntervals: Interval[]; private _inProgressIntervals: Interval[]; - private _meta?: DocMeta; - private _timestamp: string; + private _internalFields?: InternalFields; + private _timestamp?: string; constructor({ timestamp, range, filledIntervals = [], inProgressIntervals = [], - meta, + internalFields, }: GapConstructorParams) { this._range = normalizeInterval(range); this._filledIntervals = mergeIntervals(filledIntervals.map(normalizeInterval)); this._inProgressIntervals = mergeIntervals(inProgressIntervals.map(normalizeInterval)); - if (meta) { - this._meta = meta; + if (internalFields) { + this._internalFields = internalFields; + } + if (timestamp) { + this._timestamp = timestamp; } - this._timestamp = timestamp; } public fillGap(interval: Interval): void { @@ -129,8 +131,8 @@ export class Gap { this._inProgressIntervals = []; } - public get meta() { - return this._meta; + public get internalFields() { + return this._internalFields; } public getState() { diff --git a/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/transforms/transform_to_gap.test.ts b/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/transforms/transform_to_gap.test.ts new file mode 100644 index 0000000000000..79f05f2e4bced --- /dev/null +++ b/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/transforms/transform_to_gap.test.ts @@ -0,0 +1,155 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { QueryEventsBySavedObjectResult } from '@kbn/event-log-plugin/server'; +import { transformToGap } from './transform_to_gap'; +import { Gap } from '../gap'; + +describe('transformToGap', () => { + const timestamp = '2023-01-01T00:00:00.000Z'; + const validInterval = { + gte: '2023-01-01T00:00:00.000Z', + lte: '2023-01-01T01:00:00.000Z', + }; + + const createMockEvent = (overrides = {}): QueryEventsBySavedObjectResult => ({ + total: 1, + data: [ + { + '@timestamp': timestamp, + _id: 'test-id', + _index: 'test-index', + _seq_no: 1, + _primary_term: 1, + kibana: { + alert: { + rule: { + gap: { + range: validInterval, + filled_intervals: [validInterval], + in_progress_intervals: [validInterval], + }, + }, + }, + }, + ...overrides, + }, + ], + page: 1, + per_page: 10, + }); + + it('transforms valid event to Gap object', () => { + const events = createMockEvent(); + const result = transformToGap(events); + + expect(result).toHaveLength(1); + expect(result[0]).toBeInstanceOf(Gap); + expect(result[0]).toEqual( + new Gap({ + timestamp, + range: validInterval, + filledIntervals: [validInterval], + inProgressIntervals: [validInterval], + internalFields: { + _id: 'test-id', + _index: 'test-index', + _seq_no: 1, + _primary_term: 1, + }, + }) + ); + }); + + it('filters out invalid gaps (missing range)', () => { + const events = createMockEvent({ + kibana: { + alert: { + rule: { + gap: { + range: undefined, + }, + }, + }, + }, + }); + const result = transformToGap(events); + expect(result).toHaveLength(0); + }); + + it('filters out invalid gaps (invalid range)', () => { + const events = createMockEvent({ + kibana: { + alert: { + rule: { + gap: { + range: { gte: undefined, lte: '2023-01-01T01:00:00.000Z' }, + }, + }, + }, + }, + }); + const result = transformToGap(events); + expect(result).toHaveLength(0); + }); + + it('handles missing timestamp', () => { + const events = createMockEvent({ + '@timestamp': undefined, + }); + const result = transformToGap(events); + expect(result).toHaveLength(0); + }); + + it('handles missing intervals', () => { + const events = createMockEvent({ + kibana: { + alert: { + rule: { + gap: { + range: validInterval, + filled_intervals: undefined, + in_progress_intervals: undefined, + }, + }, + }, + }, + }); + const result = transformToGap(events); + + expect(result).toHaveLength(1); + expect(result[0].filledIntervals).toEqual([]); + expect(result[0].inProgressIntervals).toEqual([]); + }); + + it('filters out invalid intervals while keeping valid ones', () => { + const events = createMockEvent({ + kibana: { + alert: { + rule: { + gap: { + range: validInterval, + filled_intervals: [ + validInterval, + { gte: undefined, lte: '2023-01-01T01:00:00.000Z' }, + ], + in_progress_intervals: [ + validInterval, + { gte: '2023-01-01T00:00:00.000Z', lte: undefined }, + ], + }, + }, + }, + }, + }); + const result = transformToGap(events); + + expect(result).toHaveLength(1); + expect(result[0].filledIntervals).toHaveLength(1); + expect(result[0].inProgressIntervals).toHaveLength(1); + }); +}); diff --git a/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/transforms/transformToGap.ts b/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/transforms/transform_to_gap.ts similarity index 90% rename from x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/transforms/transformToGap.ts rename to x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/transforms/transform_to_gap.ts index d83f3a77f1870..f93963e34b4c0 100644 --- a/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/transforms/transformToGap.ts +++ b/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/transforms/transform_to_gap.ts @@ -9,6 +9,7 @@ import { Gap } from '../gap'; import { StringInterval } from '../types'; type PotenialInterval = { lte?: string; gte?: string } | undefined; + const validateInterval = (interval: PotenialInterval): StringInterval | null => { if (!interval?.gte || !interval?.lte) return null; @@ -22,6 +23,10 @@ const validateIntervals = (intervals: PotenialInterval[] | undefined): StringInt (intervals?.map(validateInterval)?.filter((interval) => interval !== null) as StringInterval[]) ?? []; +/** + * Transforms event log results into Gap objects + * Filters out invalid gaps/gaps intervals + */ export const transformToGap = (events: QueryEventsBySavedObjectResult): Gap[] => { return events?.data ?.map((doc) => { @@ -30,7 +35,7 @@ export const transformToGap = (events: QueryEventsBySavedObjectResult): Gap[] => const range = validateInterval(gap.range); - if (!range) return null; + if (!range || !doc['@timestamp']) return null; const filledIntervals = validateIntervals(gap?.filled_intervals); const inProgressIntervals = validateIntervals(gap?.in_progress_intervals); @@ -40,7 +45,7 @@ export const transformToGap = (events: QueryEventsBySavedObjectResult): Gap[] => range, filledIntervals, inProgressIntervals, - meta: { + internalFields: { _id: doc._id, _index: doc._index, _seq_no: doc._seq_no, diff --git a/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/add_filled_interval_to_gaps.ts b/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/add_filled_interval_to_gaps.ts index 9441cc3ac6873..3c95eaac19526 100644 --- a/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/add_filled_interval_to_gaps.ts +++ b/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/add_filled_interval_to_gaps.ts @@ -47,12 +47,12 @@ export const addFilledIntervalToGaps = async (params: { }); const esGap = gap.getEsObject(); - const meta = gap.meta; + const internalFields = gap.internalFields; - if (meta) { + if (internalFields) { try { await alertingEventLogger.updateGap({ - meta, + internalFields, gap: esGap, }); } catch (e) { diff --git a/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/add_in_progress_intervals_to_gaps.ts b/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/add_in_progress_intervals_to_gaps.ts index cd3b530db8efc..3e4f79c584644 100644 --- a/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/add_in_progress_intervals_to_gaps.ts +++ b/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/add_in_progress_intervals_to_gaps.ts @@ -67,12 +67,12 @@ export const addInProgressIntervalsToGaps = async (params: { }); const esGap = gap.getEsObject(); - const meta = gap.meta; + const internalFields = gap.internalFields; - if (meta) { + if (internalFields) { try { await alertingEventLogger.updateGap({ - meta, + internalFields, gap: esGap, }); } catch (e) { diff --git a/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/caclculate_in_progress_intervals_for_gaps.ts b/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/caclculate_in_progress_intervals_for_gaps.ts index c6866f09ab253..622c93762ed3f 100644 --- a/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/caclculate_in_progress_intervals_for_gaps.ts +++ b/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/caclculate_in_progress_intervals_for_gaps.ts @@ -116,12 +116,12 @@ export async function calculateInProgressIntervalsForGaps(params: { }); const esGap = gap.getEsObject(); - const meta = gap.meta; + const internalFields = gap.internalFields; - if (meta) { + if (internalFields) { try { await alertingEventLogger.updateGap({ - meta, + internalFields, gap: esGap, }); } catch (e) { diff --git a/x-pack/platform/plugins/shared/alerting/server/routes/gaps/apis/find/find_gaps_route.test.ts b/x-pack/platform/plugins/shared/alerting/server/routes/gaps/apis/find/find_gaps_route.test.ts new file mode 100644 index 0000000000000..94aa10e715f24 --- /dev/null +++ b/x-pack/platform/plugins/shared/alerting/server/routes/gaps/apis/find/find_gaps_route.test.ts @@ -0,0 +1,106 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { httpServiceMock } from '@kbn/core/server/mocks'; +import { licenseStateMock } from '../../../../lib/license_state.mock'; +import { verifyApiAccess } from '../../../../lib/license_api_access'; +import { mockHandlerArguments } from '../../../_mock_handler_arguments'; +import { transformRequestV1, transformResponseV1 } from './transforms'; +import { rulesClientMock } from '../../../../rules_client.mock'; +import { findGapsRoute } from './find_gaps_route'; +import { Gap } from '../../../../lib/rule_gaps/gap'; + +const rulesClient = rulesClientMock.create(); + +jest.mock('../../../../lib/license_api_access', () => ({ + verifyApiAccess: jest.fn(), +})); + +describe('findGapsRoute', () => { + beforeEach(() => { + jest.resetAllMocks(); + }); + + const mockFindOptions = { + rule_ids: ['abc'], + start: '2023-11-16T08:00:00.000Z', + end: '2023-11-17T08:00:00.000Z', + page: 1, + per_page: 10, + }; + + const createMockGap = () => { + const gap = new Gap({ + timestamp: '2024-01-30T00:00:00.000Z', + range: { + gte: '2023-11-16T08:00:00.000Z', + lte: '2023-11-16T20:00:00.000Z', + }, + meta: { + _id: 'gap-1', + _index: 'test-index', + _seq_no: 1, + _primary_term: 1, + }, + }); + return gap; + }; + + const mockFindResult = { + page: 1, + perPage: 10, + total: 1, + data: [createMockGap()], + }; + + test('should find gaps with the proper parameters', async () => { + const licenseState = licenseStateMock.create(); + const router = httpServiceMock.createRouter(); + + findGapsRoute(router, licenseState); + + rulesClient.findGaps.mockResolvedValueOnce(mockFindResult); + const [config, handler] = router.post.mock.calls[0]; + const [context, req, res] = mockHandlerArguments({ rulesClient }, { body: mockFindOptions }); + + expect(config.path).toEqual('/internal/alerting/rules/gaps/_find'); + + await handler(context, req, res); + + expect(rulesClient.findGaps).toHaveBeenLastCalledWith(transformRequestV1(mockFindOptions)); + expect(res.ok).toHaveBeenLastCalledWith({ + body: transformResponseV1(mockFindResult), + }); + }); + + test('ensures the license allows for finding gaps', async () => { + const licenseState = licenseStateMock.create(); + const router = httpServiceMock.createRouter(); + + findGapsRoute(router, licenseState); + + rulesClient.findGaps.mockResolvedValueOnce(mockFindResult); + const [, handler] = router.post.mock.calls[0]; + const [context, req, res] = mockHandlerArguments({ rulesClient }, { body: mockFindOptions }); + await handler(context, req, res); + expect(verifyApiAccess).toHaveBeenCalledWith(licenseState); + }); + + test('ensures the license check prevents finding gaps when appropriate', async () => { + const licenseState = licenseStateMock.create(); + const router = httpServiceMock.createRouter(); + + findGapsRoute(router, licenseState); + + (verifyApiAccess as jest.Mock).mockImplementation(() => { + throw new Error('Failure'); + }); + const [, handler] = router.post.mock.calls[0]; + const [context, req, res] = mockHandlerArguments({ rulesClient }, { body: mockFindOptions }); + await expect(handler(context, req, res)).rejects.toMatchInlineSnapshot(`[Error: Failure]`); + }); +}); diff --git a/x-pack/platform/plugins/shared/alerting/server/routes/gaps/apis/find/find_gaps_route.ts b/x-pack/platform/plugins/shared/alerting/server/routes/gaps/apis/find/find_gaps_route.ts index f6d8e361c140e..034e0944bd40a 100644 --- a/x-pack/platform/plugins/shared/alerting/server/routes/gaps/apis/find/find_gaps_route.ts +++ b/x-pack/platform/plugins/shared/alerting/server/routes/gaps/apis/find/find_gaps_route.ts @@ -6,7 +6,7 @@ */ import { IRouter } from '@kbn/core/server'; import { - findQuerySchemaV1, + findGapsBodySchemaV1, FindGapsRequestQueryV1, FindGapsResponseV1, } from '../../../../../common/routes/gaps/apis/find'; @@ -24,9 +24,9 @@ export const findGapsRoute = ( ) => { router.post( { - path: `${INTERNAL_ALERTING_GAPS_FIND_API_PATH}`, + path: INTERNAL_ALERTING_GAPS_FIND_API_PATH, validate: { - body: findQuerySchemaV1, + body: findGapsBodySchemaV1, }, options: { access: 'internal', @@ -37,7 +37,6 @@ export const findGapsRoute = ( const alertingContext = await context.alerting; const rulesClient = await alertingContext.getRulesClient(); const query: FindGapsRequestQueryV1 = req.body; - console.log('-----------------query', query); const result = await rulesClient.findGaps(transformRequestV1(query)); const response: FindGapsResponseV1 = { body: transformResponseV1(result), diff --git a/x-pack/platform/plugins/shared/alerting/server/rules_client.mock.ts b/x-pack/platform/plugins/shared/alerting/server/rules_client.mock.ts index c7577656306d0..b1e0b0d225fc4 100644 --- a/x-pack/platform/plugins/shared/alerting/server/rules_client.mock.ts +++ b/x-pack/platform/plugins/shared/alerting/server/rules_client.mock.ts @@ -56,6 +56,9 @@ const createRulesClientMock = () => { clone: jest.fn(), getScheduleFrequency: jest.fn(), bulkUntrackAlerts: jest.fn(), + findGaps: jest.fn(), + fillGapById: jest.fn(), + getRulesWithGaps: jest.fn(), }; return mocked; }; diff --git a/x-pack/platform/plugins/shared/alerting/server/rules_client/common/audit_events.ts b/x-pack/platform/plugins/shared/alerting/server/rules_client/common/audit_events.ts index 66bba16c2805c..ba74527a9f6b3 100644 --- a/x-pack/platform/plugins/shared/alerting/server/rules_client/common/audit_events.ts +++ b/x-pack/platform/plugins/shared/alerting/server/rules_client/common/audit_events.ts @@ -36,6 +36,8 @@ export enum RuleAuditAction { RUN_SOON = 'rule_run_soon', UNTRACK_ALERT = 'rule_alert_untrack', SCHEDULE_BACKFILL = 'rule_schedule_backfill', + FIND_GAPS = 'rule_find_gaps', + FILL_GAPS = 'rule_fill_gaps', } export enum AdHocRunAuditAction { @@ -45,6 +47,7 @@ export enum AdHocRunAuditAction { DELETE = 'ad_hoc_run_delete', } + type VerbsTuple = [string, string, string]; const ruleEventVerbs: Record = { @@ -97,6 +100,8 @@ const ruleEventVerbs: Record = { 'scheduling backfill for', 'scheduled backfill for', ], + rule_find_gaps: ['find gaps for', 'finding gaps for', 'found gaps for'], + rule_fill_gaps: ['fill gaps for', 'filling gaps for', 'filled gaps for'], }; const adHocRunEventVerbs: Record = { @@ -132,6 +137,8 @@ const ruleEventTypes: Record> = rule_get_global_execution_kpi: 'access', rule_alert_untrack: 'change', rule_schedule_backfill: 'access', + rule_find_gaps: 'access', + rule_fill_gaps: 'change', }; const adHocRunEventTypes: Record> = { diff --git a/x-pack/platform/plugins/shared/event_log/server/es/cluster_client_adapter.mock.ts b/x-pack/platform/plugins/shared/event_log/server/es/cluster_client_adapter.mock.ts index c416fcb0f7bf6..2f5d6da914d02 100644 --- a/x-pack/platform/plugins/shared/event_log/server/es/cluster_client_adapter.mock.ts +++ b/x-pack/platform/plugins/shared/event_log/server/es/cluster_client_adapter.mock.ts @@ -28,6 +28,7 @@ const createClusterClientMock = () => { aggregateEventsBySavedObjects: jest.fn(), aggregateEventsWithAuthFilter: jest.fn(), shutdown: jest.fn(), + updateDocument: jest.fn(), }; return mock; }; diff --git a/x-pack/platform/plugins/shared/event_log/server/es/cluster_client_adapter.test.ts b/x-pack/platform/plugins/shared/event_log/server/es/cluster_client_adapter.test.ts index 1d8105d2be49b..a8145261c10e6 100644 --- a/x-pack/platform/plugins/shared/event_log/server/es/cluster_client_adapter.test.ts +++ b/x-pack/platform/plugins/shared/event_log/server/es/cluster_client_adapter.test.ts @@ -15,6 +15,7 @@ import { AggregateEventsOptionsBySavedObjectFilter, AggregateEventsWithAuthFilter, getQueryBodyWithAuthFilter, + Doc, } from './cluster_client_adapter'; import { AggregateOptionsType, queryOptionsSchema } from '../event_log_client'; import { delay } from '../lib/delay'; @@ -733,7 +734,15 @@ describe('queryEventsBySavedObject', () => { test('should call cluster with correct options', async () => { clusterClient.search.mockResponse({ hits: { - hits: [{ _index: 'index-name-00001', _id: '1', _source: { foo: 'bar' } }], + hits: [ + { + _index: 'index-name-00001', + _id: '1', + _source: { foo: 'bar' }, + _seq_no: 1, + _primary_term: 1, + }, + ], total: { relation: 'eq', value: 1 }, }, took: 0, @@ -766,6 +775,7 @@ describe('queryEventsBySavedObject', () => { expect(query).toEqual({ index: 'index-name', track_total_hits: true, + seq_no_primary_term: true, body: { size: 6, from: 12, @@ -777,7 +787,7 @@ describe('queryEventsBySavedObject', () => { page: 3, per_page: 6, total: 1, - data: [{ foo: 'bar' }], + data: [{ foo: 'bar', _id: '1', _index: 'index-name-00001', _seq_no: 1, _primary_term: 1 }], }); }); }); @@ -2291,6 +2301,85 @@ describe('getQueryBodyWithAuthFilter', () => { }); }); +describe('updateDocument', () => { + test('should successfully update document with meta information', async () => { + const doc = { + body: { foo: 'updated' }, + index: 'test-index', + internalFields: { + _id: 'test-id', + _index: 'test-index', + _seq_no: 1, + _primary_term: 1, + }, + }; + + clusterClient.update.mockResponse({ + _index: 'test-index', + _id: 'test-id', + _version: 2, + result: 'updated', + _shards: { + total: 2, + successful: 1, + failed: 0, + }, + _seq_no: 2, + _primary_term: 1, + }); + + await clusterClientAdapter.updateDocument(doc as unknown as Required); + + expect(clusterClient.update).toHaveBeenCalledWith({ + doc: doc.body, + id: doc.internalFields._id, + index: doc.internalFields._index, + if_primary_term: doc.internalFields._primary_term, + if_seq_no: doc.internalFields._seq_no, + refresh: 'wait_for', + }); + }); + + test('should throw error if internal fields information is missing', async () => { + const doc = { + body: { foo: 'updated' }, + index: 'test-index', + }; + + await expect( + clusterClientAdapter.updateDocument(doc as unknown as Required) + ).rejects.toThrowErrorMatchingInlineSnapshot('"Internal fields are required"'); + + expect(logger.error).toHaveBeenCalledWith( + `error updating event: "Internal fields are required"; docs: ${JSON.stringify(doc)}` + ); + }); + + test('should throw error when update fails', async () => { + const doc = { + body: { foo: 'updated' }, + index: 'test-index', + internalFields: { + _id: 'test-id', + _index: 'test-index', + _seq_no: 1, + _primary_term: 1, + }, + }; + + const error = new Error('Update failed'); + clusterClient.update.mockRejectedValue(error); + + await expect( + clusterClientAdapter.updateDocument(doc as unknown as Required) + ).rejects.toThrowErrorMatchingInlineSnapshot(`"Update failed"`); + + expect(logger.error).toHaveBeenCalledWith( + `error updating event: "Update failed"; docs: ${JSON.stringify(doc)}` + ); + }); +}); + type RetryableFunction = () => boolean; const RETRY_UNTIL_DEFAULT_COUNT = 20; diff --git a/x-pack/platform/plugins/shared/event_log/server/es/cluster_client_adapter.ts b/x-pack/platform/plugins/shared/event_log/server/es/cluster_client_adapter.ts index be2ca6f31e55f..18dbc02690d37 100644 --- a/x-pack/platform/plugins/shared/event_log/server/es/cluster_client_adapter.ts +++ b/x-pack/platform/plugins/shared/event_log/server/es/cluster_client_adapter.ts @@ -13,6 +13,7 @@ import { Logger, ElasticsearchClient } from '@kbn/core/server'; import util from 'util'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { fromKueryExpression, toElasticsearchQuery, KueryNode, nodeBuilder } from '@kbn/es-query'; +import { long } from '@elastic/elasticsearch/lib/api/types'; import { IEvent, IValidatedEvent, SAVED_OBJECT_REL_PRIMARY } from '../types'; import { AggregateOptionsType, FindOptionsType, QueryOptionsType } from '../event_log_client'; import { ParsedIndexAlias } from './init'; @@ -23,7 +24,7 @@ export const EVENT_BUFFER_LENGTH = 100; export type IClusterClientAdapter = PublicMethodsOf; -export interface DocMeta { +export interface InternalFields { _id: string; _index: string; _seq_no: number; @@ -33,7 +34,7 @@ export interface DocMeta { export interface Doc { index: string; body: IEvent; - meta?: DocMeta; + internalFields?: InternalFields; } type Wait = () => Promise; @@ -45,11 +46,18 @@ export interface ConstructorOpts { wait: Wait; } +type IValidatedEventInternalDocInfo = IValidatedEvent & { + _id: estypes.Id; + _index: estypes.IndexName; + _seq_no: estypes.SequenceNumber; + _primary_term: long; +}; + export interface QueryEventsBySavedObjectResult { page: number; per_page: number; total: number; - data: IValidatedEvent[]; + data: IValidatedEventInternalDocInfo[]; } interface QueryOptionsEventsBySavedObjectFilter { @@ -109,7 +117,7 @@ export class ClusterClientAdapter< TDoc extends { body: AliasAny; index: string; - meta?: DocMeta; + internalFields?: InternalFields; } = Doc > { private readonly logger: Logger; @@ -152,19 +160,19 @@ export class ClusterClientAdapter< public async updateDocument(doc: Required) { const esClient = await this.elasticsearchClientPromise; try { - if (!doc.meta) { - return; + if (!doc.internalFields) { + throw new Error('Internal fields are required'); } - const result = await esClient.update({ + await esClient.update({ doc: doc.body, - id: doc.meta._id, - index: doc.meta._index, - if_primary_term: doc.meta._primary_term, - if_seq_no: doc.meta._seq_no, + id: doc.internalFields._id, + index: doc.internalFields._index, + if_primary_term: doc.internalFields._primary_term, + if_seq_no: doc.internalFields._seq_no, refresh: 'wait_for', }); } catch (e) { - this.logger.error(`error update event: "${e.message}"; docs: ${JSON.stringify(doc)}`); + this.logger.error(`error updating event: "${e.message}"; docs: ${JSON.stringify(doc)}`); throw e; } } @@ -212,11 +220,6 @@ export class ClusterClientAdapter< } } - public async deleteByQueryDocs(query: estypes.QueryDslQueryContainer): Promise { - const esClient = await this.elasticsearchClientPromise; - await esClient.deleteByQuery({ index: this.esNames.dataStream, query }); - } - public async doesIndexTemplateExist(name: string): Promise { try { const esClient = await this.elasticsearchClientPromise; @@ -446,9 +449,6 @@ export class ClusterClientAdapter< ? { sort: sort.map((s) => ({ [s.sort_field]: { order: s.sort_order } })) as estypes.Sort } : {}), }; - - console.log('-----------------body', JSON.stringify(body, null, 2)); - try { const { hits: { hits, total }, @@ -458,16 +458,17 @@ export class ClusterClientAdapter< seq_no_primary_term: true, body, }); + return { page, per_page: perPage, total: isNumber(total) ? total : total!.value, data: hits.map((hit) => ({ ...hit._source, - _id: hit._id, + _id: hit._id!, _index: hit._index, - _seq_no: hit._seq_no, - _primary_term: hit._primary_term, + _seq_no: hit._seq_no!, + _primary_term: hit._primary_term!, })), }; } catch (err) { @@ -515,10 +516,10 @@ export class ClusterClientAdapter< total: isNumber(total) ? total : total!.value, data: hits.map((hit) => ({ ...hit._source, - _id: hit._id, + _id: hit._id!, _index: hit._index, - if_seq_no: hit._seq_no, - if_primary_term: hit._primary_term, + _seq_no: hit._seq_no!, + _primary_term: hit._primary_term!, })), }; } catch (err) { diff --git a/x-pack/platform/plugins/shared/event_log/server/event_logger.mock.ts b/x-pack/platform/plugins/shared/event_log/server/event_logger.mock.ts index 8b693ba6dcb17..ed243d98548f1 100644 --- a/x-pack/platform/plugins/shared/event_log/server/event_logger.mock.ts +++ b/x-pack/platform/plugins/shared/event_log/server/event_logger.mock.ts @@ -13,7 +13,6 @@ const createEventLoggerMock = () => { updateEvent: jest.fn(), startTiming: jest.fn(), stopTiming: jest.fn(), - deleteEventsDocsByQuery: jest.fn(), }; return mock; }; diff --git a/x-pack/platform/plugins/shared/event_log/server/event_logger.ts b/x-pack/platform/plugins/shared/event_log/server/event_logger.ts index 64c02f64f14c5..249cc6b155eed 100644 --- a/x-pack/platform/plugins/shared/event_log/server/event_logger.ts +++ b/x-pack/platform/plugins/shared/event_log/server/event_logger.ts @@ -7,7 +7,6 @@ import { schema } from '@kbn/config-schema'; import { Logger } from '@kbn/core/server'; -import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { merge } from 'lodash'; import { coerce } from 'semver'; @@ -24,7 +23,7 @@ import { EventSchema, } from './types'; import { SAVED_OBJECT_REL_PRIMARY } from './types'; -import { Doc, DocMeta } from './es/cluster_client_adapter'; +import { Doc, InternalFields } from './es/cluster_client_adapter'; type SystemLogger = Plugin['systemLogger']; @@ -109,11 +108,11 @@ export class EventLogger implements IEventLogger { } } - async updateEvent(meta: DocMeta, event: IEvent): Promise { + async updateEvent(internalFields: InternalFields, event: IEvent): Promise { const doc: Required = { index: this.esContext.esNames.dataStream, body: event, - meta, + internalFields, }; if (this.eventLogService.isIndexingEntries()) { @@ -126,10 +125,6 @@ export class EventLogger implements IEventLogger { return result; } } - - async deleteEventsDocsByQuery(query: estypes.QueryDslQueryContainer): Promise { - return deleteByQuery(this.esContext, query); - } } // return the epoch millis of the start date, or null; may be NaN if garbage @@ -195,10 +190,3 @@ function indexEventDoc(esContext: EsContext, doc: Doc): void { async function updateEventDoc(esContext: EsContext, doc: Required): Promise { return esContext.esAdapter.updateDocument(doc); } - -async function deleteByQuery( - esContext: EsContext, - query: estypes.QueryDslQueryContainer -): Promise { - return esContext.esAdapter.deleteByQueryDocs(query); -} diff --git a/x-pack/platform/plugins/shared/event_log/server/index.ts b/x-pack/platform/plugins/shared/event_log/server/index.ts index 0bfc725200e48..b33ad15e2bc1c 100644 --- a/x-pack/platform/plugins/shared/event_log/server/index.ts +++ b/x-pack/platform/plugins/shared/event_log/server/index.ts @@ -19,12 +19,10 @@ export type { IEventLogClient, QueryEventsBySavedObjectResult, AggregateEventsBySavedObjectResult, - DocMeta, + InternalFields, } from './types'; export { SAVED_OBJECT_REL_PRIMARY } from './types'; -export { ClusterClientAdapter } from './es/cluster_client_adapter'; - export { createReadySignal } from './lib/ready_signal'; export const config: PluginConfigDescriptor = { diff --git a/x-pack/platform/plugins/shared/event_log/server/types.ts b/x-pack/platform/plugins/shared/event_log/server/types.ts index 618a1de18d8ea..bfabe1c9467ba 100644 --- a/x-pack/platform/plugins/shared/event_log/server/types.ts +++ b/x-pack/platform/plugins/shared/event_log/server/types.ts @@ -8,7 +8,6 @@ import { schema, TypeOf } from '@kbn/config-schema'; import type { KibanaRequest } from '@kbn/core/server'; import { KueryNode } from '@kbn/es-query'; -import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; export type { IEvent, IValidatedEvent } from '../generated/schemas'; export { EventSchema, ECS_VERSION } from '../generated/schemas'; @@ -16,14 +15,14 @@ import { IEvent } from '../generated/schemas'; import { AggregateOptionsType, FindOptionsType } from './event_log_client'; import { AggregateEventsBySavedObjectResult, - DocMeta, QueryEventsBySavedObjectResult, + InternalFields, } from './es/cluster_client_adapter'; export type { QueryEventsBySavedObjectResult, AggregateEventsBySavedObjectResult, - DocMeta, + InternalFields, } from './es/cluster_client_adapter'; import { SavedObjectProvider } from './saved_object_provider_registry'; @@ -86,6 +85,5 @@ export interface IEventLogger { logEvent(properties: IEvent): void; startTiming(event: IEvent, startTime?: Date): void; stopTiming(event: IEvent): void; - updateEvent(meta: DocMeta, event: IEvent): Promise; - deleteEventsDocsByQuery(query: estypes.QueryDslQueryContainer): Promise; + updateEvent(internalFields: InternalFields, event: IEvent): Promise; } diff --git a/x-pack/test/alerting_api_integration/common/plugins/alerts/server/plugin.ts b/x-pack/test/alerting_api_integration/common/plugins/alerts/server/plugin.ts index 62b6d6594b06c..7635950f23abc 100644 --- a/x-pack/test/alerting_api_integration/common/plugins/alerts/server/plugin.ts +++ b/x-pack/test/alerting_api_integration/common/plugins/alerts/server/plugin.ts @@ -92,7 +92,7 @@ export class FixturePlugin implements Plugin, - { features, actions, alerting, ruleRegistry }: FixtureSetupDeps + { features, actions, alerting, ruleRegistry, eventLog }: FixtureSetupDeps ) { features.registerKibanaFeature({ id: 'alertsFixture', @@ -134,7 +134,13 @@ export class FixturePlugin implements Plugin, taskManagerStart: Promise, notificationsStart: Promise, - { logger }: { logger: Logger } + { logger, eventLogger }: { logger: Logger; eventLogger: IEventLogger } ) { const router = core.http.createRouter(); router.get( @@ -589,4 +591,144 @@ export function defineRoutes( } } ); + + router.post( + { + path: '/_test/report_gap', + validate: { + body: schema.object({ + ruleId: schema.string(), + start: schema.string(), + end: schema.string(), + spaceId: schema.string(), + }), + }, + }, + async ( + context: RequestHandlerContext, + req: KibanaRequest, + res: KibanaResponseFactory + ): Promise> => { + const [, { eventLog }] = await core.getStartServices(); + + const eventLogClient = eventLog.getClient(req); + + const getAmountOfGaps = async () => { + try { + const gaps = await eventLogClient.findEventsBySavedObjectIds('alert', [req.body.ruleId], { + filter: 'event.action: gap', + }); + return gaps.total; + } catch (err) { + return 0; + } + }; + + const amountOfGaps = await getAmountOfGaps(); + + const alertingEventLogger = new AlertingEventLogger(eventLogger); + + alertingEventLogger.initialize({ + context: { + savedObjectId: req.body.ruleId, + spaceId: req.body.spaceId, + savedObjectType: 'alert', + executionId: '123', + taskScheduledAt: new Date(), + namespace: req.body.spaceId, + }, + runDate: new Date(), + ruleData: { + id: req.body.ruleId, + consumer: 'siem', + type: { + id: req.body.ruleId, + name: 'My test rule', + actionGroups: [], + defaultActionGroupId: 'default', + minimumLicenseRequired: 'basic', + isExportable: true, + executor: { + execute: async () => ({ state: {} }), + }, + category: 'test', + producer: 'alerts', + cancelAlertsOnRuleTimeout: true, + ruleTaskTimeout: '5m', + recoveryActionGroup: { + id: 'customRecovered', + name: 'Custom Recovered', + }, + autoRecoverAlerts: true, + validate: { + params: { validate: (params) => params }, + }, + alerts: { + context: 'test', + mappings: { fieldMap: { field: { type: 'keyword', required: false } } }, + }, + validLegacyConsumers: [], + }, + }, + }); + + await alertingEventLogger.reportGap({ + gap: { + lte: req.body.end, + gte: req.body.start, + }, + }); + + try { + await pRetry( + async () => { + const newAmountOfGaps = await getAmountOfGaps(); + if (newAmountOfGaps === amountOfGaps + 1) { + return; + } + await new Promise((resolve) => setTimeout(resolve, 5000)); + throw new Error('Amount of gaps did not increase'); + }, + { retries: 5 } + ); + return res.ok({ body: { ok: true } }); + } catch (err) { + return res.customError({ + statusCode: 500, + body: { message: 'Amount of gaps did not increase' }, + }); + } + } + ); + + router.post( + { + path: '/_test/event_log/update_document', + validate: { + body: schema.object({ + _id: schema.string(), + _index: schema.string(), + _seq_no: schema.number(), + _primary_term: schema.number(), + fieldsToUpdate: schema.any(), + }), + }, + }, + async ( + context: RequestHandlerContext, + req: KibanaRequest, + res: KibanaResponseFactory + ) => { + const result = await eventLogger.updateEvent( + { + _id: req.body._id, + _index: req.body._index, + _seq_no: req.body._seq_no, + _primary_term: req.body._primary_term, + }, + req.body.fieldsToUpdate + ); + return res.ok({ body: { ok: true, result } }); + } + ); } diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/gap/find.ts b/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/gap/find.ts new file mode 100644 index 0000000000000..df1c87d93e4d7 --- /dev/null +++ b/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/gap/find.ts @@ -0,0 +1,305 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import expect from '@kbn/expect'; +import { UserAtSpaceScenarios } from '../../../../scenarios'; +import { FtrProviderContext } from '../../../../../common/ftr_provider_context'; +import { + getUrlPrefix, + ObjectRemover, + getTestRuleData, + getUnauthorizedErrorMessage, +} from '../../../../../common/lib'; + +// eslint-disable-next-line import/no-default-export +export default function findGapsTests({ getService }: FtrProviderContext) { + const supertest = getService('supertest'); + const supertestWithoutAuth = getService('supertestWithoutAuth'); + + describe('find gaps', () => { + const objectRemover = new ObjectRemover(supertest); + const searchStart = '2024-01-01T00:00:00.000Z'; + const searchEnd = '2024-01-31T00:00:00.000Z'; + const gap1Start = '2024-01-05T00:00:00.000Z'; + const gap1End = '2024-01-06T00:00:00.000Z'; + const gap2Start = '2024-01-15T00:00:00.000Z'; + const gap2End = '2024-01-16T00:00:00.000Z'; + + afterEach(async () => { + await objectRemover.removeAll(); + }); + + function getRule(overwrites = {}) { + return getTestRuleData({ + rule_type_id: 'test.patternFiringAutoRecoverFalse', + params: { + pattern: { + instance: [true, false, true], + }, + }, + schedule: { interval: '12h' }, + ...overwrites, + }); + } + + for (const scenario of UserAtSpaceScenarios) { + const { user, space } = scenario; + describe(scenario.id, () => { + const apiOptions = { + spaceId: space.id, + username: user.username, + password: user.password, + }; + + describe('find gaps with request body', () => { + it('should handle finding gaps with various parameters', async () => { + // Create 2 rules + const rresponse1 = await supertest + .post(`${getUrlPrefix(apiOptions.spaceId)}/api/alerting/rule`) + .set('kbn-xsrf', 'foo') + .send(getRule()) + .expect(200); + const ruleId1 = rresponse1.body.id; + objectRemover.add(apiOptions.spaceId, ruleId1, 'rule', 'alerting'); + + const rresponse2 = await supertest + .post(`${getUrlPrefix(apiOptions.spaceId)}/api/alerting/rule`) + .set('kbn-xsrf', 'foo') + .send(getRule()) + .expect(200); + const ruleId2 = rresponse2.body.id; + objectRemover.add(apiOptions.spaceId, ruleId2, 'rule', 'alerting'); + + // Create gaps for both rules + await supertest + .post(`${getUrlPrefix(apiOptions.spaceId)}/_test/report_gap`) + .set('kbn-xsrf', 'foo') + .send({ + ruleId: ruleId1, + start: gap1Start, + end: gap1End, + spaceId: apiOptions.spaceId, + }); + + await supertest + .post(`${getUrlPrefix(apiOptions.spaceId)}/_test/report_gap`) + .set('kbn-xsrf', 'foo') + .send({ + ruleId: ruleId1, + start: gap2Start, + end: gap2End, + spaceId: apiOptions.spaceId, + }); + + await supertest + .post(`${getUrlPrefix(apiOptions.spaceId)}/_test/report_gap`) + .set('kbn-xsrf', 'foo') + .send({ + ruleId: ruleId2, + start: gap2Start, + end: gap2End, + spaceId: apiOptions.spaceId, + }); + + // Test cases for finding gaps + const testCases = [ + // Find gaps for rule 1 + { + body: { + rule_id: ruleId1, + start: searchStart, + end: searchEnd, + }, + expectedTotal: 2, + description: 'should find gaps for rule 1', + }, + // Find gaps for rule 2 + { + body: { + rule_id: ruleId2, + start: searchStart, + end: searchEnd, + }, + expectedTotal: 1, + description: 'should find gaps for rule 2', + }, + // Test pagination + { + body: { + rule_id: ruleId1, + start: searchStart, + end: searchEnd, + per_page: 1, + page: 1, + }, + expectedTotal: 2, + expectedPerPage: 1, + description: 'should return first page of gaps', + }, + // Test response schema and data validation + { + body: { + rule_id: ruleId1, + start: searchStart, + end: searchEnd, + }, + expectedTotal: 2, + validate: (response: any) => { + expect(response.body).to.have.keys(['page', 'per_page', 'total', 'data']); + const data = response.body.data; + expect(data[0].range.gte).to.eql(gap2Start); + expect(data[0].range.lte).to.eql(gap2End); + expect(data[0].filled_intervals).to.eql([]); + expect(data[0].in_progress_intervals).to.eql([]); + expect(data[0].unfilled_intervals).to.eql([ + { + start: gap2Start, + end: gap2End, + }, + ]); + expect(data[0].status).to.eql('unfilled'); + expect(data[0].total_gap_duration_ms).to.eql(86400000); + expect(data[0].filled_duration_ms).to.eql(0); + expect(data[0].unfilled_duration_ms).to.eql(86400000); + expect(data[0].in_progress_duration_ms).to.eql(0); + + expect(data[0]).to.have.keys('_id', '@timestamp'); + }, + description: 'should return correct response schema', + }, + // Test sorting order validation + { + body: { + rule_id: ruleId1, + start: searchStart, + end: searchEnd, + sort_field: '@timestamp', + sort_order: 'desc', + }, + expectedTotal: 2, + validate: (response: any) => { + const timestamps = response.body.data.map((gap: any) => gap['@timestamp']); + expect(timestamps[0]).to.be.greaterThan(timestamps[1]); + }, + description: 'should return gaps sorted by @timestamp in descending order', + }, + // Test non-overlapping date range + { + body: { + rule_id: ruleId1, + start: '2024-02-01T00:00:00.000Z', + end: '2024-02-28T00:00:00.000Z', + }, + expectedTotal: 0, + description: 'should return no gaps for non-overlapping date range', + }, + // Test partially overlapping ranges + { + body: { + rule_id: ruleId1, + start: '2024-01-05T11:30:00.000Z', + end: '2024-01-05T12:30:00.000Z', + }, + expectedTotal: 1, + validate: (response: any) => { + expect(response.body.data[0].range.gte).to.equal(gap1Start); + expect(response.body.data[0].range.lte).to.equal(gap1End); + }, + description: 'should return gap when search range partially overlaps', + }, + ]; + + for (const testCase of testCases) { + const response = await supertestWithoutAuth + .post(`${getUrlPrefix(apiOptions.spaceId)}/internal/alerting/rules/gaps/_find`) + .set('kbn-xsrf', 'foo') + .auth(apiOptions.username, apiOptions.password) + .send(testCase.body); + + switch (scenario.id) { + case 'no_kibana_privileges at space1': + case 'space_1_all at space2': + expect(response.statusCode).to.eql(403); + expect(response.body).to.eql({ + error: 'Forbidden', + message: `Failed to find gaps: ${getUnauthorizedErrorMessage( + 'get', + 'test.patternFiringAutoRecoverFalse', + 'alertsFixture' + )}`, + statusCode: 403, + }); + break; + + case 'global_read at space1': + case 'space_1_all_alerts_none_actions at space1': + case 'superuser at space1': + case 'space_1_all at space1': + case 'space_1_all_with_restricted_fixture at space1': + expect(response.statusCode).to.eql(200); + expect(response.body.total).to.eql(testCase.expectedTotal); + if (testCase.expectedPerPage) { + expect(response.body.per_page).to.eql(testCase.expectedPerPage); + } + // Execute additional validations if present + if (testCase.validate) { + testCase.validate(response); + } + break; + + default: + throw new Error(`Scenario untested: ${JSON.stringify(scenario)}`); + } + } + }); + }); + + describe('find gaps with invalid parameters', () => { + it('should handle invalid parameters appropriately', async () => { + const invalidBodies = [ + { + body: {}, + expectedError: + 'Failed to find gaps: Error validating get data - [id]: expected value of type [string] but got [undefined]', + }, + { + body: { + rule_id: '1', + start: 'invalid-date', + end: searchEnd, + }, + expectedError: '[request body]: [start]: query start must be valid date', + }, + { + body: { + rule_id: '1', + start: searchStart, + end: 'invalid-date', + }, + expectedError: '[request body]: [end]: query end must be valid date', + }, + ]; + + for (const { body, expectedError } of invalidBodies) { + const response = await supertestWithoutAuth + .post(`${getUrlPrefix(apiOptions.spaceId)}/internal/alerting/rules/gaps/_find`) + .set('kbn-xsrf', 'foo') + .auth(apiOptions.username, apiOptions.password) + .send(body); + + expect(response.statusCode).to.eql(400); + expect(response.body).to.eql({ + statusCode: 400, + error: 'Bad Request', + message: expectedError, + }); + } + }); + }); + }); + } + }); +} diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/gap/index.ts b/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/gap/index.ts new file mode 100644 index 0000000000000..9784827bbe3bc --- /dev/null +++ b/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/gap/index.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrProviderContext } from '../../../../../common/ftr_provider_context'; + +// eslint-disable-next-line import/no-default-export +export default function gapsTests({ loadTestFile }: FtrProviderContext) { + describe('rule gaps', () => { + loadTestFile(require.resolve('./find')); + }); +} diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/index.ts b/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/index.ts index 777479a631ec9..b1d66191d990f 100644 --- a/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/index.ts +++ b/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/index.ts @@ -23,6 +23,7 @@ export default function alertingTests({ loadTestFile, getService }: FtrProviderC loadTestFile(require.resolve('./backfill')); loadTestFile(require.resolve('./find')); loadTestFile(require.resolve('./find_internal')); + loadTestFile(require.resolve('./gap')); }); }); } diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group1/event_log.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group1/event_log.ts index 2aa82d4c5cce0..1c2ba31c98608 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group1/event_log.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group1/event_log.ts @@ -2026,6 +2026,102 @@ export default function eventLogTests({ getService }: FtrProviderContext) { expect(get(executeEvents[5], ACTION_PATH)).to.be(0); expect(get(executeEvents[5], DELAYED_PATH)).to.be(1); }); + + it('should update event log document fields', async () => { + const { body: createdAction } = await supertest + .post(`${getUrlPrefix(space.id)}/api/actions/connector`) + .set('kbn-xsrf', 'foo') + .send({ + name: 'MY action', + connector_type_id: 'test.noop', + config: {}, + secrets: {}, + }) + .expect(200); + + // Create a rule that will generate events + const response = await supertest + .post(`${getUrlPrefix(space.id)}/api/alerting/rule`) + .set('kbn-xsrf', 'foo') + .send( + getTestRuleData({ + rule_type_id: 'test.patternFiring', + schedule: { interval: '1s' }, + throttle: null, + params: { + pattern: { + instance: [true, false], + }, + }, + actions: [ + { + id: createdAction.id, + group: 'default', + params: {}, + }, + ], + }) + ); + + expect(response.status).to.eql(200); + const alertId = response.body.id; + objectRemover.add(space.id, alertId, 'rule', 'alerting'); + + // Get the events and find one to update + const events = await retry.try(async () => { + return await getEventLog({ + getService, + spaceId: space.id, + type: 'alert', + id: alertId, + provider: 'alerting', + actions: new Map([['execute', { gte: 1 }]]), + }); + }); + + expect(events.length).to.be.greaterThan(0); + const eventToUpdate = events[0]; + + // Prepare the update + const fieldsToUpdate = { + event: { kind: 'test_update' }, + new_field: 'Updated test message', + }; + + // Call the update API + const updateResponse = await supertest + .post(`${getUrlPrefix(space.id)}/_test/event_log/update_document`) + .set('kbn-xsrf', 'foo') + .send({ + _id: eventToUpdate._id, + _index: eventToUpdate._index, + _seq_no: eventToUpdate._seq_no, + _primary_term: eventToUpdate._primary_term, + fieldsToUpdate, + }) + .expect(200); + + expect(updateResponse.body.ok).to.be(true); + + // Verify the update by getting the event again + const updatedEvents = await retry.try(async () => { + const newResponse = await getEventLog({ + getService, + spaceId: space.id, + type: 'alert', + id: alertId, + provider: 'alerting', + actions: new Map([['execute', { gte: 1 }]]), + }); + + const updatedEvent = newResponse.find((event) => event._id === eventToUpdate._id); + expect(updatedEvent).to.be.ok(); + expect(updatedEvent.event.kind).to.be('test_update'); + expect(updatedEvent.new_field).to.be('Updated test message'); + + return response; + }); + }); }); } }); From ad60cdf6a0a00ddcc016f5e2afa9727e61397429 Mon Sep 17 00:00:00 2001 From: Nikita Khristinin Date: Thu, 9 Jan 2025 15:23:35 +0100 Subject: [PATCH 10/17] Add more tests --- .../lib/retry_transient_es_errors.ts | 15 +- .../methods/delete/delete_backfill.ts | 7 +- .../backfill_client/backfill_client.mock.ts | 1 + .../server/backfill_client/backfill_client.ts | 50 +- .../server/lib/rule_gaps/find_gaps.test.ts | 316 +++++++++++ .../server/lib/rule_gaps/find_gaps.ts | 35 +- .../server/lib/rule_gaps/gap/index.test.ts | 12 +- .../server/lib/rule_gaps/schemas/index.ts | 8 +- .../update/add_filled_interval_to_gaps.ts | 76 --- .../add_in_progress_intervals_to_gaps.ts | 98 ---- ...clculate_in_progress_intervals_for_gaps.ts | 141 ----- .../update/calculate_gaps_state.test.ts | 146 +++++ .../rule_gaps/update/calculate_gaps_state.ts | 46 ++ .../update/update_gap_from_schedule.test.ts | 154 ++++++ .../update/update_gap_from_schedule.ts | 40 ++ .../lib/rule_gaps/update/update_gaps.test.ts | 223 ++++++++ .../lib/rule_gaps/update/update_gaps.ts | 131 +++++ .../find/transforms/transform_response/v1.ts | 2 +- .../server/task_runner/ad_hoc_task_runner.ts | 57 +- .../group1/tests/alerting/gap/index.ts | 1 + .../group1/tests/alerting/gap/update_gaps.ts | 519 ++++++++++++++++++ 21 files changed, 1696 insertions(+), 382 deletions(-) create mode 100644 x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/find_gaps.test.ts delete mode 100644 x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/add_filled_interval_to_gaps.ts delete mode 100644 x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/add_in_progress_intervals_to_gaps.ts delete mode 100644 x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/caclculate_in_progress_intervals_for_gaps.ts create mode 100644 x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/calculate_gaps_state.test.ts create mode 100644 x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/calculate_gaps_state.ts create mode 100644 x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/update_gap_from_schedule.test.ts create mode 100644 x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/update_gap_from_schedule.ts create mode 100644 x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/update_gaps.test.ts create mode 100644 x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/update_gaps.ts create mode 100644 x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/gap/update_gaps.ts diff --git a/x-pack/platform/plugins/shared/alerting/server/alerts_service/lib/retry_transient_es_errors.ts b/x-pack/platform/plugins/shared/alerting/server/alerts_service/lib/retry_transient_es_errors.ts index 2df03e65690f1..04ae07078e944 100644 --- a/x-pack/platform/plugins/shared/alerting/server/alerts_service/lib/retry_transient_es_errors.ts +++ b/x-pack/platform/plugins/shared/alerting/server/alerts_service/lib/retry_transient_es_errors.ts @@ -16,11 +16,12 @@ const retryResponseStatuses = [ 410, // Gone ]; -const isRetryableError = (e: Error) => +const isRetryableError = (e: Error, additionalRetryableStatusCodes: number[]) => e instanceof EsErrors.NoLivingConnectionsError || e instanceof EsErrors.ConnectionError || e instanceof EsErrors.TimeoutError || - (e instanceof EsErrors.ResponseError && retryResponseStatuses.includes(e?.statusCode!)); + (e instanceof EsErrors.ResponseError && + [...retryResponseStatuses, ...additionalRetryableStatusCodes].includes(e?.statusCode!)); const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); @@ -29,15 +30,17 @@ export const retryTransientEsErrors = async ( { logger, attempt = 0, + additionalRetryableStatusCodes = [], }: { logger: Logger; attempt?: number; + additionalRetryableStatusCodes?: number[]; } ): Promise => { try { return await esCall(); } catch (e) { - if (attempt < MAX_ATTEMPTS && isRetryableError(e)) { + if (attempt < MAX_ATTEMPTS && isRetryableError(e, additionalRetryableStatusCodes)) { const retryCount = attempt + 1; const retryDelaySec: number = Math.min(Math.pow(2, retryCount), 30); // 2s, 4s, 8s, 16s, 30s, 30s, 30s... @@ -49,7 +52,11 @@ export const retryTransientEsErrors = async ( // delay with some randomness await delay(retryDelaySec * 1000 * Math.random()); - return retryTransientEsErrors(esCall, { logger, attempt: retryCount }); + return retryTransientEsErrors(esCall, { + logger, + attempt: retryCount, + additionalRetryableStatusCodes, + }); } throw e; diff --git a/x-pack/platform/plugins/shared/alerting/server/application/backfill/methods/delete/delete_backfill.ts b/x-pack/platform/plugins/shared/alerting/server/application/backfill/methods/delete/delete_backfill.ts index f565654eb5fd7..51b6b1de4c1f2 100644 --- a/x-pack/platform/plugins/shared/alerting/server/application/backfill/methods/delete/delete_backfill.ts +++ b/x-pack/platform/plugins/shared/alerting/server/application/backfill/methods/delete/delete_backfill.ts @@ -16,7 +16,7 @@ import { adHocRunAuditEvent, } from '../../../../rules_client/common/audit_events'; import { transformAdHocRunToBackfillResult } from '../../transforms'; -import { calculateInProgressIntervalsForGaps } from '../../../../lib/rule_gaps/update/caclculate_in_progress_intervals_for_gaps'; +import { updateGaps } from '../../../../lib/rule_gaps/update/update_gaps'; export async function deleteBackfill(context: RulesClientContext, id: string): Promise<{}> { return await retryIfConflicts( @@ -103,14 +103,17 @@ async function deleteWithOCC(context: RulesClientContext, { id }: { id: string } if ('rule' in backfillResult) { const eventLogClient = await context.getEventLogClient(); - await calculateInProgressIntervalsForGaps({ + await updateGaps({ ruleId: backfillResult.rule.id, start: new Date(backfillResult.start), end: backfillResult.end ? new Date(backfillResult.end) : new Date(), + backfillSchedule: backfillResult.schedule, savedObjectsRepository: context.internalSavedObjectsRepository, logger: context.logger, eventLogClient, eventLogger: context.eventLogger, + shouldRefetchAllBackfills: true, + backfillClient: context.backfillClient, }); } diff --git a/x-pack/platform/plugins/shared/alerting/server/backfill_client/backfill_client.mock.ts b/x-pack/platform/plugins/shared/alerting/server/backfill_client/backfill_client.mock.ts index 36919208b608d..2a4ea82eb54f8 100644 --- a/x-pack/platform/plugins/shared/alerting/server/backfill_client/backfill_client.mock.ts +++ b/x-pack/platform/plugins/shared/alerting/server/backfill_client/backfill_client.mock.ts @@ -9,6 +9,7 @@ const createBackfillClientMock = () => { return { bulkQueue: jest.fn(), deleteBackfillForRules: jest.fn(), + findOverlappingBackfills: jest.fn(), }; }); }; diff --git a/x-pack/platform/plugins/shared/alerting/server/backfill_client/backfill_client.ts b/x-pack/platform/plugins/shared/alerting/server/backfill_client/backfill_client.ts index be3abe91bd014..1747fd87e6814 100644 --- a/x-pack/platform/plugins/shared/alerting/server/backfill_client/backfill_client.ts +++ b/x-pack/platform/plugins/shared/alerting/server/backfill_client/backfill_client.ts @@ -44,7 +44,7 @@ import { AD_HOC_RUN_SAVED_OBJECT_TYPE, RULE_SAVED_OBJECT_TYPE } from '../saved_o import { TaskRunnerFactory } from '../task_runner'; import { RuleTypeRegistry } from '../types'; import { createBackfillError } from './lib'; -import { addInProgressIntervalsToGaps } from '../lib/rule_gaps/update/add_in_progress_intervals_to_gaps'; +import { updateGaps } from '../lib/rule_gaps/update/update_gaps'; export const BACKFILL_TASK_TYPE = 'ad_hoc_run-backfill'; @@ -240,19 +240,22 @@ export class BackfillClient { }); try { - // TODO: make it parallel for (const backfill of backfullsToSchedule) { - await addInProgressIntervalsToGaps({ - backfill, + await updateGaps({ + backfillSchedule: backfill.schedule, + ruleId: backfill.rule.id, + start: new Date(backfill.start), + end: backfill?.end ? new Date(backfill.end) : new Date(), eventLogger, eventLogClient, savedObjectsRepository: internalSavedObjectsRepository, logger: this.logger, + backfillClient: this, }); } } catch { this.logger.warn( - `Error updating gaps ƒor backfill jobs: ${backfullsToSchedule + `Error updating gaps for backfill jobs: ${backfullsToSchedule .map((backfill) => backfill.id) .join(', ')}` ); @@ -328,6 +331,43 @@ export class BackfillClient { ); } } + + public async findOverlappingBackfills({ + ruleId, + start, + end, + savedObjectsRepository, + }: { + ruleId: string; + start: Date; + end: Date; + savedObjectsRepository: ISavedObjectsRepository; + }) { + const adHocRuns: Array> = []; + + // Create a point in time finder for efficient pagination + const adHocRunFinder = await savedObjectsRepository.createPointInTimeFinder({ + type: AD_HOC_RUN_SAVED_OBJECT_TYPE, + perPage: 100, + hasReference: [{ id: ruleId, type: RULE_SAVED_OBJECT_TYPE }], + filter: ` + ad_hoc_run_params.attributes.start <= "${end.toISOString()}" and + ad_hoc_run_params.attributes.end >= "${start.toISOString()}" + `, + }); + + try { + // Collect all results using async iterator + for await (const response of adHocRunFinder.find()) { + adHocRuns.push(...response.saved_objects); + } + } finally { + // Make sure we always close the finder + await adHocRunFinder.close(); + } + + return adHocRuns.map((data) => transformAdHocRunToBackfillResult(data)); + } } function getRuleOrError( diff --git a/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/find_gaps.test.ts b/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/find_gaps.test.ts new file mode 100644 index 0000000000000..9a00c00d1bfc7 --- /dev/null +++ b/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/find_gaps.test.ts @@ -0,0 +1,316 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { buildGapsFilter, findGaps, findAllGaps } from './find_gaps'; +import { gapStatus } from '../../../common/constants/gap_status'; +import { loggerMock } from '@kbn/logging-mocks'; +import { eventLogClientMock } from '@kbn/event-log-plugin/server/event_log_client.mock'; +import { Gap } from './gap'; + +const createMockGapEvent = () => ({ + _id: 'test-id', + _index: 'test-index', + _seq_no: 1, + _primary_term: 1, + '@timestamp': '2024-01-01T00:00:00.000Z', + event: { + start: '2024-01-01', + end: '2024-01-02', + action: 'gap', + }, + kibana: { + alert: { + rule: { + gap: { + range: { gte: '2024-01-01', lte: '2024-01-02' }, + filled_intervals: [], + in_progress_intervals: [], + }, + }, + }, + }, +}); + +describe('buildGapsFilter', () => { + it('should build base filter when no params provided', () => { + expect(buildGapsFilter({})).toBe('event.action: gap AND event.provider: alerting'); + }); + + it('should build filter with range', () => { + expect(buildGapsFilter({ start: '2024-01-01', end: '2024-01-02' })).toBe( + 'event.action: gap AND event.provider: alerting AND ' + + 'kibana.alert.rule.gap.range <= "2024-01-02" AND kibana.alert.rule.gap.range >= "2024-01-01"' + ); + }); + + it('should build filter with statuses', () => { + expect(buildGapsFilter({ statuses: [gapStatus.UNFILLED] })).toBe( + 'event.action: gap AND event.provider: alerting AND ' + + '(kibana.alert.rule.gap.status : unfilled)' + ); + }); + + it('should build filter with range and statuses', () => { + expect( + buildGapsFilter({ + start: '2024-01-01', + end: '2024-01-02', + statuses: [gapStatus.UNFILLED, gapStatus.PARTIALLY_FILLED], + }) + ).toBe( + 'event.action: gap AND event.provider: alerting AND ' + + 'kibana.alert.rule.gap.range <= "2024-01-02" AND kibana.alert.rule.gap.range >= "2024-01-01" AND ' + + '(kibana.alert.rule.gap.status : unfilled OR kibana.alert.rule.gap.status : partially_filled)' + ); + }); +}); + +describe('findGaps', () => { + const mockLogger = loggerMock.create(); + const mockEventLogClient = eventLogClientMock.create(); + + beforeEach(() => { + jest.resetAllMocks(); + }); + + it('should call findEventsBySavedObjectIds with correct parameters', async () => { + mockEventLogClient.findEventsBySavedObjectIds.mockResolvedValue({ + total: 0, + data: [], + page: 1, + per_page: 10, + }); + + await findGaps({ + eventLogClient: mockEventLogClient, + logger: mockLogger, + params: { + ruleId: 'test-rule', + start: '2024-01-01', + end: '2024-01-02', + page: 1, + perPage: 10, + statuses: [gapStatus.UNFILLED], + }, + }); + + expect(mockEventLogClient.findEventsBySavedObjectIds).toHaveBeenCalledWith( + 'alert', + ['test-rule'], + expect.objectContaining({ + filter: expect.stringContaining('event.action: gap'), + sort: [{ sort_field: '@timestamp', sort_order: 'desc' }], + page: 1, + per_page: 10, + }) + ); + }); + + it('should handle custom sort field', async () => { + mockEventLogClient.findEventsBySavedObjectIds.mockResolvedValue({ + total: 0, + data: [], + page: 1, + per_page: 10, + }); + + await findGaps({ + eventLogClient: mockEventLogClient, + logger: mockLogger, + params: { + ruleId: 'test-rule', + page: 1, + perPage: 10, + sortField: 'total_gap_duration_ms', + sortOrder: 'asc', + }, + }); + + expect(mockEventLogClient.findEventsBySavedObjectIds).toHaveBeenCalledWith( + 'alert', + ['test-rule'], + expect.objectContaining({ + sort: [{ sort_field: 'kibana.alert.rule.gap.total_gap_duration_ms', sort_order: 'asc' }], + }) + ); + }); + + it('should transform response data to Gap objects', async () => { + const mockResponse = { + total: 1, + data: [createMockGapEvent()], + page: 1, + per_page: 10, + }; + + mockEventLogClient.findEventsBySavedObjectIds.mockResolvedValue(mockResponse); + + const result = await findGaps({ + eventLogClient: mockEventLogClient, + logger: mockLogger, + params: { ruleId: 'test-rule', page: 1, perPage: 10 }, + }); + + expect(result.data[0]).toBeInstanceOf(Gap); + expect(result.data[0].range).toEqual({ + gte: new Date('2024-01-01'), + lte: new Date('2024-01-02'), + }); + expect(result.data[0].filledIntervals).toEqual([]); + expect(result.data[0].inProgressIntervals).toEqual([]); + }); + + it('should handle errors and log them', async () => { + const error = new Error('Test error'); + mockEventLogClient.findEventsBySavedObjectIds.mockRejectedValue(error); + + await expect( + findGaps({ + eventLogClient: mockEventLogClient, + logger: mockLogger, + params: { ruleId: 'test-rule', page: 1, perPage: 10 }, + }) + ).rejects.toThrow(error); + + expect(mockLogger.error).toHaveBeenCalledWith( + expect.stringContaining('Failed to find gaps for rule test-rule') + ); + }); +}); + +describe('findAllGaps', () => { + const mockLogger = loggerMock.create(); + const mockEventLogClient = eventLogClientMock.create(); + + beforeEach(() => { + jest.resetAllMocks(); + }); + + it('should fetch all pages of gaps', async () => { + mockEventLogClient.findEventsBySavedObjectIds + .mockResolvedValueOnce({ + total: 15000, + data: Array(10000).fill(createMockGapEvent()), + page: 1, + per_page: 10000, + }) + .mockResolvedValueOnce({ + total: 15000, + data: Array(5000).fill(createMockGapEvent()), + page: 2, + per_page: 10000, + }); + + const result = await findAllGaps({ + eventLogClient: mockEventLogClient, + logger: mockLogger, + params: { + ruleId: 'test-rule', + start: new Date('2024-01-01'), + end: new Date('2024-01-02'), + }, + }); + + expect(result).toHaveLength(15000); + expect(mockEventLogClient.findEventsBySavedObjectIds).toHaveBeenCalledTimes(2); + }); + + it('should stop fetching when no more data', async () => { + const createMockGapEvent = () => ({ + _id: 'test-id', + _index: 'test-index', + _seq_no: 1, + _primary_term: 1, + '@timestamp': '2024-01-01T00:00:00.000Z', + event: { + start: '2024-01-01', + end: '2024-01-02', + action: 'gap', + }, + kibana: { + alert: { + rule: { + gap: { + range: { gte: '2024-01-01', lte: '2024-01-02' }, + filled_intervals: [], + in_progress_intervals: [], + }, + }, + }, + }, + }); + + mockEventLogClient.findEventsBySavedObjectIds.mockResolvedValue({ + total: 50, + data: Array(50).fill(createMockGapEvent()), + page: 1, + per_page: 10000, + }); + + const result = await findAllGaps({ + eventLogClient: mockEventLogClient, + logger: mockLogger, + params: { + ruleId: 'test-rule', + start: new Date('2024-01-01'), + end: new Date('2024-01-02'), + }, + }); + + expect(result).toHaveLength(50); + expect(mockEventLogClient.findEventsBySavedObjectIds).toHaveBeenCalledTimes(1); + }); + + it('should handle errors during pagination', async () => { + const createMockGapEvent = () => ({ + _id: 'test-id', + _index: 'test-index', + _seq_no: 1, + _primary_term: 1, + '@timestamp': '2024-01-01T00:00:00.000Z', + event: { + start: '2024-01-01', + end: '2024-01-02', + action: 'gap', + }, + kibana: { + alert: { + rule: { + gap: { + range: { gte: '2024-01-01', lte: '2024-01-02' }, + filled_intervals: [], + in_progress_intervals: [], + }, + }, + }, + }, + }); + + mockEventLogClient.findEventsBySavedObjectIds + .mockResolvedValueOnce({ + total: 15000, + data: Array(10000).fill(createMockGapEvent()), + page: 1, + per_page: 10000, + }) + .mockRejectedValueOnce(new Error('Pagination failed')); + + await expect( + findAllGaps({ + eventLogClient: mockEventLogClient, + logger: mockLogger, + params: { + ruleId: 'test-rule', + start: new Date('2024-01-01'), + end: new Date('2024-01-02'), + }, + }) + ).rejects.toThrow('Pagination failed'); + + expect(mockLogger.error).toHaveBeenCalled(); + }); +}); diff --git a/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/find_gaps.ts b/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/find_gaps.ts index c4de2b8087da7..e189c75674bc3 100644 --- a/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/find_gaps.ts +++ b/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/find_gaps.ts @@ -13,6 +13,29 @@ import { Gap } from './gap'; import { transformToGap } from './transforms/transform_to_gap'; import { GapStatus } from '../../../common/constants/gap_status'; +export const buildGapsFilter = ({ + start, + end, + statuses, +}: { + start?: string; + end?: string; + statuses?: GapStatus[]; +}) => { + const baseFilter = 'event.action: gap AND event.provider: alerting'; + + const rangeFilter = + end && start + ? `kibana.alert.rule.gap.range <= "${end}" AND kibana.alert.rule.gap.range >= "${start}"` + : null; + + const statusesFilter = statuses?.length + ? `(${statuses.map((status) => `kibana.alert.rule.gap.status : ${status}`).join(' OR ')})` + : null; + + return [baseFilter, rangeFilter, statusesFilter].filter(Boolean).join(' AND '); +}; + export const findGaps = async ({ eventLogClient, logger, @@ -30,13 +53,7 @@ export const findGaps = async ({ const { ruleId, start, end, page, perPage, statuses, sortField, sortOrder } = params; try { - const statusesFilter = statuses - ?.map((status) => `kibana.alert.rule.gap.status : ${status}`) - .join(' OR '); - const rangeFilter = - end && start - ? `AND (kibana.alert.rule.gap.range <= "${end}" AND kibana.alert.rule.gap.range >= "${start}")` - : ''; + const filter = buildGapsFilter({ start, end, statuses }); const getField = (field?: string) => { if (field === '@timestamp' || !field) { @@ -49,9 +66,7 @@ export const findGaps = async ({ RULE_SAVED_OBJECT_TYPE, [ruleId], { - filter: `event.action: gap AND event.provider: alerting ${rangeFilter} ${ - statusesFilter ? `AND (${statusesFilter})` : '' - }`, + filter, sort: [ { sort_field: getField(sortField), diff --git a/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/gap/index.test.ts b/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/gap/index.test.ts index 5a93b8146b44b..1c13a40de75b9 100644 --- a/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/gap/index.test.ts +++ b/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/gap/index.test.ts @@ -205,14 +205,14 @@ describe('Gap Class Tests', () => { expect(esObject.filled_duration_ms).toBe(10 * 60 * 1000); }); - it('handles meta information if provided', () => { - const meta = { + it('handles internalFields information if provided', () => { + const internalFields = { _id: 'test_id', _index: 'test_index', - _seq_no: '1', - _primary_term: '1', + _seq_no: 1, + _primary_term: 1, }; - const gap = new Gap({ range: baseRange, meta }); - expect(gap.meta).toEqual(meta); + const gap = new Gap({ range: baseRange, internalFields }); + expect(gap.internalFields).toEqual(internalFields); }); }); diff --git a/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/schemas/index.ts b/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/schemas/index.ts index 62b4d5b210960..ab85499d32dca 100644 --- a/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/schemas/index.ts +++ b/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/schemas/index.ts @@ -40,7 +40,13 @@ export const findGapsParamsSchema = schema.object( perPage: schema.number({ defaultValue: 10, min: 0 }), ruleId: schema.string(), start: schema.maybe(schema.string()), - sortField: schema.maybe(schema.oneOf([schema.literal('@timestamp')])), + sortField: schema.maybe( + schema.oneOf([ + schema.literal('@timestamp'), + schema.literal('total_gap_duration_ms'), + schema.literal('status'), + ]) + ), sortOrder: schema.maybe(schema.oneOf([schema.literal('asc'), schema.literal('desc')])), statuses: schema.maybe(schema.arrayOf(gapStatusSchema)), }, diff --git a/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/add_filled_interval_to_gaps.ts b/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/add_filled_interval_to_gaps.ts deleted file mode 100644 index 3c95eaac19526..0000000000000 --- a/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/add_filled_interval_to_gaps.ts +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { Logger } from '@kbn/core/server'; -import { IEventLogClient, IEventLogger } from '@kbn/event-log-plugin/server'; -import { AlertingEventLogger } from '../../alerting_event_logger/alerting_event_logger'; -import { findAllGaps } from '../find_gaps'; -import { gapStatus } from '../../../../common/constants'; - -/** - * Find all gaps for this date range and ruleId - * Then add the filled interval to the gaps - * Then update the gaps in the event log - */ -export const addFilledIntervalToGaps = async (params: { - ruleId: string; - start: Date; - end: Date; - eventLogger: IEventLogger; - eventLogClient: IEventLogClient; - logger: Logger; -}) => { - const { ruleId, start, end, logger, eventLogClient, eventLogger } = params; - - const alertingEventLogger = new AlertingEventLogger(eventLogger); - - try { - const allGaps = await findAllGaps({ - eventLogClient, - logger, - params: { - ruleId, - start, - end, - statuses: [gapStatus.PARTIALLY_FILLED, gapStatus.UNFILLED], - }, - }); - - for (const gap of allGaps) { - gap.fillGap({ - gte: start, - lte: end, - }); - - const esGap = gap.getEsObject(); - const internalFields = gap.internalFields; - - if (internalFields) { - try { - await alertingEventLogger.updateGap({ - internalFields, - gap: esGap, - }); - } catch (e) { - // TODO version mismatch -> - // refetch gap - // check status - // retry - - logger.error('Failed to fill gap, because of conflicting versions'); - } - } - } - } catch (err) { - logger.error( - `Failed to fill gaps for rule ${ruleId} from: ${start.toISOString()} to: ${end.toISOString()}: ${ - err.message - }` - ); - throw err; - } -}; diff --git a/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/add_in_progress_intervals_to_gaps.ts b/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/add_in_progress_intervals_to_gaps.ts deleted file mode 100644 index 3e4f79c584644..0000000000000 --- a/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/add_in_progress_intervals_to_gaps.ts +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ISavedObjectsRepository, Logger } from '@kbn/core/server'; -import { IEventLogClient, IEventLogger } from '@kbn/event-log-plugin/server'; -import { AlertingEventLogger } from '../../alerting_event_logger/alerting_event_logger'; -import { findAllGaps } from '../find_gaps'; -import { adHocRunStatus, gapStatus } from '../../../../common/constants'; -import { Backfill } from '../../../application/backfill/result/types'; -import { parseDuration } from '../../../../common'; - -/** - * Find all gaps for this date range and ruleId - * Then add inProgressIntervals to the gaps - * Then update the gaps in the event log - */ -export const addInProgressIntervalsToGaps = async (params: { - backfill: Backfill; - eventLogger: IEventLogger | undefined; - eventLogClient: IEventLogClient; - savedObjectsRepository: ISavedObjectsRepository; - logger: Logger; -}) => { - const { backfill, logger, eventLogClient, eventLogger } = params; - - const ruleId = backfill.rule.id; - const start = new Date(backfill.start); - const end = backfill?.end ? new Date(backfill.end) : new Date(); - - try { - if (!eventLogger) { - throw new Error('Add in-progress intervals to gaps: Event logger is not defined'); - } - - const alertingEventLogger = new AlertingEventLogger(eventLogger); - const allGaps = await findAllGaps({ - eventLogClient, - logger, - params: { - ruleId, - start, - end, - statuses: [gapStatus.PARTIALLY_FILLED, gapStatus.UNFILLED], - }, - }); - - for (const gap of allGaps) { - // TODO: we shouldn't write gap into alertevent log if not real update - backfill.schedule.forEach((scheduleItem) => { - const runAt = new Date(scheduleItem.runAt).getTime(); - const intervalDuration = parseDuration(scheduleItem.interval); - const from = runAt - intervalDuration; - const to = runAt; - if ( - scheduleItem.status === adHocRunStatus.PENDING || - scheduleItem.status === adHocRunStatus.RUNNING - ) { - gap.addInProgress({ - gte: new Date(from), - lte: new Date(to), - }); - } - }); - - const esGap = gap.getEsObject(); - const internalFields = gap.internalFields; - - if (internalFields) { - try { - await alertingEventLogger.updateGap({ - internalFields, - gap: esGap, - }); - } catch (e) { - // TODO version mismatch -> - // refetch gap - // check status - // retry - - logger.error( - 'Failed to add in-progress intervals into gap, because of conflicting versions' - ); - } - } - } - } catch (err) { - logger.error( - `Failed to add in-progress for rule ${ruleId} from: ${start.toISOString()} to: ${end.toISOString()}: ${ - err.message - }` - ); - throw err; - } -}; diff --git a/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/caclculate_in_progress_intervals_for_gaps.ts b/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/caclculate_in_progress_intervals_for_gaps.ts deleted file mode 100644 index 622c93762ed3f..0000000000000 --- a/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/caclculate_in_progress_intervals_for_gaps.ts +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ISavedObjectsRepository, Logger } from '@kbn/core/server'; -import { IEventLogClient, IEventLogger } from '@kbn/event-log-plugin/server'; -import { AD_HOC_RUN_SAVED_OBJECT_TYPE, RULE_SAVED_OBJECT_TYPE } from '../../../saved_objects'; -import { AlertingEventLogger } from '../../alerting_event_logger/alerting_event_logger'; -import { findAllGaps } from '../find_gaps'; -import { AdHocRunSO } from '../../../data/ad_hoc_run/types'; -import { adHocRunStatus, gapStatus } from '../../../../common/constants'; -import { parseDuration } from '../../../../common'; -import { transformAdHocRunToBackfillResult } from '../../../application/backfill/transforms'; -import { Gap } from '../gap'; - -/** - * Find all overlapping backfill tasks and update the gap status accordingly - */ -const updateGapStatus = async ({ - gap, - savedObjectsRepository, - ruleId, -}: { - gap: Gap; - savedObjectsRepository: ISavedObjectsRepository; - ruleId: string; -}): Promise => { - // TODO: get all backfill, not first page - const { saved_objects: backfillSOs } = await savedObjectsRepository.find({ - type: AD_HOC_RUN_SAVED_OBJECT_TYPE, - hasReference: { - type: RULE_SAVED_OBJECT_TYPE, - id: ruleId, - }, - // Filter for backfills that overlap with our interval - filter: ` - ad_hoc_run_params.attributes.start <= "${gap.range.lte.toISOString()}" and - ad_hoc_run_params.attributes.end >= "${gap.range.gte.toISOString()}" - `, - page: 1, - perPage: 10000, - }); - - // TODO: Extract backfill transform to another function, to reuse in API - const transformedBackfills = backfillSOs.map((data) => transformAdHocRunToBackfillResult(data)); - - gap.resetInProgressIntervals(); - for (const backfill of transformedBackfills) { - if ('error' in backfill) { - break; - } - const backfillScheduleIntervals = backfill?.schedule ?? []; - for (const scheduleItem of backfillScheduleIntervals) { - const runAt = new Date(scheduleItem.runAt).getTime(); - const intervalDuration = parseDuration(scheduleItem.interval); - const from = runAt - intervalDuration; - const to = runAt; - const scheduleInterval = { - gte: new Date(from), - lte: new Date(to), - }; - if ( - scheduleItem.status === adHocRunStatus.PENDING || - scheduleItem.status === adHocRunStatus.RUNNING - ) { - gap.addInProgress(scheduleInterval); - } - } - } - return gap; -}; - -/** - * Find all gaps for this date range and ruleId - * Then find all backfill tasks that overlap with these gaps - * Update the gap status accordingly, be reset and then calculate the new inProgressIntervals - */ -export async function calculateInProgressIntervalsForGaps(params: { - ruleId: string; - start: Date; - end: Date; - eventLogger: IEventLogger | undefined; - eventLogClient: IEventLogClient; - savedObjectsRepository: ISavedObjectsRepository; - logger: Logger; -}): Promise { - const { ruleId, start, end, logger, savedObjectsRepository, eventLogClient, eventLogger } = - params; - - try { - if (!eventLogger) { - throw new Error('Event logger is required'); - } - - const alertingEventLogger = new AlertingEventLogger(eventLogger); - - const allGaps = await findAllGaps({ - eventLogClient, - logger, - params: { - ruleId, - start, - end, - statuses: [gapStatus.PARTIALLY_FILLED, gapStatus.UNFILLED], - }, - }); - - for (const gap of allGaps) { - await updateGapStatus({ - gap, - savedObjectsRepository, - ruleId, - }); - - const esGap = gap.getEsObject(); - const internalFields = gap.internalFields; - - if (internalFields) { - try { - await alertingEventLogger.updateGap({ - internalFields, - gap: esGap, - }); - } catch (e) { - // TODO version mismatch -> - // refetch gap - // check status - // retry - - logger.error('failed update'); - } - } - } - } catch (err) { - logger.error(`Failed to process gaps for rule ${ruleId}: ${err.message}`); - throw err; - } -} diff --git a/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/calculate_gaps_state.test.ts b/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/calculate_gaps_state.test.ts new file mode 100644 index 0000000000000..91e99a836d438 --- /dev/null +++ b/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/calculate_gaps_state.test.ts @@ -0,0 +1,146 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { savedObjectsRepositoryMock } from '@kbn/core/server/mocks'; +import { backfillClientMock } from '../../../backfill_client/backfill_client.mock'; +import { calculateGapStateFromAllBackfills } from './calculate_gaps_state'; +import { Gap } from '../gap'; +import { adHocRunStatus } from '../../../../common/constants'; + +describe('calculateGapStateFromAllBackfills', () => { + const mockSavedObjectsRepository = savedObjectsRepositoryMock.create(); + const mockBackfillClient = backfillClientMock.create(); + + beforeEach(() => { + jest.resetAllMocks(); + mockBackfillClient.findOverlappingBackfills.mockResolvedValue([]); + }); + + it('should calculate gap state', async () => { + const testGap = new Gap({ + range: { + gte: '2024-01-01T00:00:00.000Z', + lte: '2024-01-01T01:00:00.000Z', + }, + }); + + await calculateGapStateFromAllBackfills({ + gap: testGap, + savedObjectsRepository: mockSavedObjectsRepository, + ruleId: 'test-rule-id', + backfillClient: mockBackfillClient, + }); + + expect(mockBackfillClient.findOverlappingBackfills).toHaveBeenCalledWith({ + ruleId: 'test-rule-id', + start: testGap.range.gte, + end: testGap.range.lte, + savedObjectsRepository: mockSavedObjectsRepository, + }); + }); + + it('should reset in-progress intervals before processing backfills', async () => { + const testGap = new Gap({ + range: { + gte: '2024-01-01T00:00:00.000Z', + lte: '2024-01-01T01:00:00.000Z', + }, + inProgressIntervals: [ + { + gte: '2024-01-01T00:00:00.000Z', + lte: '2024-01-01T00:30:00.000Z', + }, + ], + }); + + const spy = jest.spyOn(testGap, 'resetInProgressIntervals'); + + await calculateGapStateFromAllBackfills({ + gap: testGap, + savedObjectsRepository: mockSavedObjectsRepository, + ruleId: 'test-rule-id', + backfillClient: mockBackfillClient, + }); + + expect(spy).toHaveBeenCalled(); + expect(testGap.inProgressIntervals).toHaveLength(0); + }); + + it('should update gap with backfill schedules from overlapping backfills', async () => { + const testGap = new Gap({ + range: { + gte: '2024-01-01T00:00:00.000Z', + lte: '2024-01-01T01:00:00.000Z', + }, + }); + + mockBackfillClient.findOverlappingBackfills.mockResolvedValueOnce([ + { + schedule: [ + { + runAt: '2024-01-01T00:15:00.000Z', + interval: '15m', + status: adHocRunStatus.RUNNING, + }, + { + runAt: '2024-01-01T00:20:00.000Z', + interval: '15m', + status: adHocRunStatus.PENDING, + }, + ], + }, + ]); + + const updatedGap = await calculateGapStateFromAllBackfills({ + gap: testGap, + savedObjectsRepository: mockSavedObjectsRepository, + ruleId: 'test-rule-id', + backfillClient: mockBackfillClient, + }); + + expect(updatedGap.inProgressIntervals).toHaveLength(1); + expect(updatedGap.inProgressIntervals[0]).toEqual({ + gte: new Date('2024-01-01T00:00:00.000Z'), + lte: new Date('2024-01-01T00:20:00.000Z'), + }); + }); + + it('should filter out backfills with an error', async () => { + const testGap = new Gap({ + range: { + gte: '2024-01-01T00:00:00.000Z', + lte: '2024-01-01T01:00:00.000Z', + }, + }); + + mockBackfillClient.findOverlappingBackfills.mockResolvedValueOnce([ + { error: 'Some error' }, + { + schedule: [ + { + runAt: '2024-01-01T00:15:00.000Z', + interval: '15m', + status: adHocRunStatus.RUNNING, + }, + ], + }, + ]); + + const updatedGap = await calculateGapStateFromAllBackfills({ + gap: testGap, + savedObjectsRepository: mockSavedObjectsRepository, + ruleId: 'test-rule-id', + backfillClient: mockBackfillClient, + }); + + expect(updatedGap.inProgressIntervals).toHaveLength(1); + expect(updatedGap.inProgressIntervals[0]).toEqual({ + gte: new Date('2024-01-01T00:00:00.000Z'), + lte: new Date('2024-01-01T00:15:00.000Z'), + }); + }); +}); diff --git a/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/calculate_gaps_state.ts b/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/calculate_gaps_state.ts new file mode 100644 index 0000000000000..af175c8b46f0a --- /dev/null +++ b/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/calculate_gaps_state.ts @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ISavedObjectsRepository } from '@kbn/core/server'; +import { Gap } from '../gap'; +import { updateGapFromSchedule } from './update_gap_from_schedule'; +import { BackfillClient } from '../../../backfill_client/backfill_client'; + +/** + * Find all overlapping backfill tasks and update the gap status accordingly + */ +export const calculateGapStateFromAllBackfills = async ({ + gap, + savedObjectsRepository, + ruleId, + backfillClient, +}: { + gap: Gap; + savedObjectsRepository: ISavedObjectsRepository; + ruleId: string; + backfillClient: BackfillClient; +}): Promise => { + const transformedBackfills = await backfillClient.findOverlappingBackfills({ + ruleId, + start: gap.range.gte, + end: gap.range.lte, + savedObjectsRepository, + }); + + gap.resetInProgressIntervals(); + for (const backfill of transformedBackfills) { + if ('error' in backfill) { + continue; + } + gap = updateGapFromSchedule({ + gap, + backfillSchedule: backfill?.schedule ?? [], + }); + } + + return gap; +}; diff --git a/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/update_gap_from_schedule.test.ts b/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/update_gap_from_schedule.test.ts new file mode 100644 index 0000000000000..068541c54a0fe --- /dev/null +++ b/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/update_gap_from_schedule.test.ts @@ -0,0 +1,154 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Gap } from '../gap'; +import { updateGapFromSchedule } from './update_gap_from_schedule'; +import { adHocRunStatus } from '../../../../common/constants'; + +describe('updateGapFromSchedule', () => { + const createTestGap = () => + new Gap({ + range: { + gte: '2024-01-01T00:00:00.000Z', + lte: '2024-01-01T01:00:00.000Z', + }, + }); + + describe('schedule processing', () => { + it('should handle empty schedule', () => { + const gap = createTestGap(); + const updatedGap = updateGapFromSchedule({ + gap, + backfillSchedule: [], + }); + + expect(updatedGap.filledIntervals).toHaveLength(0); + expect(updatedGap.inProgressIntervals).toHaveLength(0); + }); + + it('should update filled intervals for completed backfills', () => { + const gap = createTestGap(); + const updatedGap = updateGapFromSchedule({ + gap, + backfillSchedule: [ + { + runAt: '2024-01-01T00:15:00.000Z', + interval: '15m', + status: adHocRunStatus.COMPLETE, + }, + ], + }); + + expect(updatedGap.filledIntervals).toHaveLength(1); + expect(updatedGap.filledIntervals[0]).toEqual({ + gte: new Date('2024-01-01T00:00:00.000Z'), + lte: new Date('2024-01-01T00:15:00.000Z'), + }); + }); + + it('should update in-progress intervals for running backfills', () => { + const gap = createTestGap(); + const updatedGap = updateGapFromSchedule({ + gap, + backfillSchedule: [ + { + runAt: '2024-01-01T00:15:00.000Z', + interval: '15m', + status: adHocRunStatus.RUNNING, + }, + ], + }); + + expect(updatedGap.inProgressIntervals).toHaveLength(1); + expect(updatedGap.inProgressIntervals[0]).toEqual({ + gte: new Date('2024-01-01T00:00:00.000Z'), + lte: new Date('2024-01-01T00:15:00.000Z'), + }); + }); + }); + + describe('multiple intervals handling', () => { + it('should handle overlapping intervals', () => { + const gap = createTestGap(); + const updatedGap = updateGapFromSchedule({ + gap, + backfillSchedule: [ + { + runAt: '2024-01-01T00:15:00.000Z', + interval: '15m', + status: adHocRunStatus.COMPLETE, + }, + { + runAt: '2024-01-01T00:20:00.000Z', + interval: '15m', + status: adHocRunStatus.COMPLETE, + }, + ], + }); + + expect(updatedGap.filledIntervals).toHaveLength(1); + expect(updatedGap.filledIntervals[0]).toEqual({ + gte: new Date('2024-01-01T00:00:00.000Z'), + lte: new Date('2024-01-01T00:20:00.000Z'), + }); + }); + + it('should handle mixed status intervals', () => { + const gap = createTestGap(); + const updatedGap = updateGapFromSchedule({ + gap, + backfillSchedule: [ + { + runAt: '2024-01-01T00:15:00.000Z', + interval: '15m', + status: adHocRunStatus.COMPLETE, + }, + { + runAt: '2024-01-01T00:30:00.000Z', + interval: '15m', + status: adHocRunStatus.RUNNING, + }, + { + runAt: '2024-01-01T00:45:00.000Z', + interval: '15m', + status: adHocRunStatus.PENDING, + }, + ], + }); + + expect(updatedGap.filledIntervals).toHaveLength(1); + expect(updatedGap.inProgressIntervals).toHaveLength(1); + expect(updatedGap.filledIntervals[0]).toEqual({ + gte: new Date('2024-01-01T00:00:00.000Z'), + lte: new Date('2024-01-01T00:15:00.000Z'), + }); + expect(updatedGap.inProgressIntervals[0]).toEqual({ + gte: new Date('2024-01-01T00:15:00.000Z'), + lte: new Date('2024-01-01T00:45:00.000Z'), + }); + }); + }); + + describe('edge cases', () => { + it('should handle intervals outside gap range', () => { + const gap = createTestGap(); + const updatedGap = updateGapFromSchedule({ + gap, + backfillSchedule: [ + { + runAt: '2024-01-01T02:00:00.000Z', // Outside gap range + interval: '15m', + status: adHocRunStatus.COMPLETE, + }, + ], + }); + + expect(updatedGap.filledIntervals).toHaveLength(0); + expect(updatedGap.inProgressIntervals).toHaveLength(0); + }); + }); +}); diff --git a/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/update_gap_from_schedule.ts b/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/update_gap_from_schedule.ts new file mode 100644 index 0000000000000..5b15a754a8e11 --- /dev/null +++ b/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/update_gap_from_schedule.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Gap } from '../gap'; +import { adHocRunStatus } from '../../../../common/constants'; +import { parseDuration } from '../../../../common'; +import { BackfillSchedule } from '../../../application/backfill/result/types'; + +export const updateGapFromSchedule = ({ + gap, + backfillSchedule, +}: { + gap: Gap; + backfillSchedule: BackfillSchedule[]; +}) => { + for (const scheduleItem of backfillSchedule) { + const runAt = new Date(scheduleItem.runAt).getTime(); + const intervalDuration = parseDuration(scheduleItem.interval); + const from = runAt - intervalDuration; + const to = runAt; + const scheduleInterval = { + gte: new Date(from), + lte: new Date(to), + }; + if ( + scheduleItem.status === adHocRunStatus.PENDING || + scheduleItem.status === adHocRunStatus.RUNNING + ) { + gap.addInProgress(scheduleInterval); + } else if (scheduleItem.status === adHocRunStatus.COMPLETE) { + gap.fillGap(scheduleInterval); + } + } + + return gap; +}; diff --git a/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/update_gaps.test.ts b/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/update_gaps.test.ts new file mode 100644 index 0000000000000..1bae4aa0d2e1d --- /dev/null +++ b/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/update_gaps.test.ts @@ -0,0 +1,223 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { updateGaps } from './update_gaps'; +import { findAllGaps } from '../find_gaps'; +import { updateGapFromSchedule } from './update_gap_from_schedule'; +import { calculateGapStateFromAllBackfills } from './calculate_gaps_state'; +import { backfillClientMock } from '../../../backfill_client/backfill_client.mock'; + +import { loggerMock } from '@kbn/logging-mocks'; +import { eventLoggerMock } from '@kbn/event-log-plugin/server/event_logger.mock'; +import { eventLogClientMock } from '@kbn/event-log-plugin/server/event_log_client.mock'; +import { savedObjectsRepositoryMock } from '@kbn/core/server/mocks'; +import { Gap } from '../gap'; +import { adHocRunStatus } from '../../../../common/constants'; +import { errors as EsErrors } from '@elastic/elasticsearch'; + +jest.mock('../find_gaps'); +jest.mock('./update_gap_from_schedule'); +jest.mock('./calculate_gaps_state'); + +describe('updateGaps', () => { + const mockLogger = loggerMock.create(); + const mockEventLogger = eventLoggerMock.create(); + const mockEventLogClient = eventLogClientMock.create(); + const mockSavedObjectsRepository = savedObjectsRepositoryMock.create(); + const mockBackfillClient = backfillClientMock.create(); + + const createTestGap = () => + new Gap({ + range: { + gte: '2024-01-01T00:00:00.000Z', + lte: '2024-01-01T01:00:00.000Z', + }, + internalFields: { + _id: 'test-id', + _index: 'test-index', + _seq_no: 1, + _primary_term: 1, + }, + }); + + beforeEach(() => { + jest.resetAllMocks(); + (findAllGaps as jest.Mock).mockResolvedValue([]); + }); + + describe('updateGaps', () => { + it('should orchestrate the gap update process', async () => { + const testGap = createTestGap(); + (findAllGaps as jest.Mock).mockResolvedValue([testGap]); + + await updateGaps({ + ruleId: 'test-rule-id', + start: new Date('2024-01-01T00:00:00.000Z'), + end: new Date('2024-01-01T01:00:00.000Z'), + eventLogger: mockEventLogger, + eventLogClient: mockEventLogClient, + logger: mockLogger, + savedObjectsRepository: mockSavedObjectsRepository, + backfillClient: mockBackfillClient, + }); + + expect(findAllGaps).toHaveBeenCalledWith({ + eventLogClient: mockEventLogClient, + logger: mockLogger, + params: { + ruleId: 'test-rule-id', + start: expect.any(Date), + end: expect.any(Date), + statuses: ['partially_filled', 'unfilled'], + }, + }); + expect(mockEventLogger.updateEvent).toHaveBeenCalled(); + }); + + it('should handle multiple gaps in the time range', async () => { + const gaps = [ + createTestGap(), + new Gap({ + range: { + gte: '2024-01-01T01:00:00.000Z', + lte: '2024-01-01T02:00:00.000Z', + }, + internalFields: { + _id: 'test-id-2', + _index: 'test-index', + _seq_no: 2, + _primary_term: 1, + }, + }), + ]; + (findAllGaps as jest.Mock).mockResolvedValue(gaps); + + await updateGaps({ + ruleId: 'test-rule-id', + start: new Date('2024-01-01T00:00:00.000Z'), + end: new Date('2024-01-01T02:00:00.000Z'), + eventLogger: mockEventLogger, + eventLogClient: mockEventLogClient, + logger: mockLogger, + savedObjectsRepository: mockSavedObjectsRepository, + backfillClient: mockBackfillClient, + }); + + expect(mockEventLogger.updateEvent).toHaveBeenCalledTimes(2); + }); + }); + + describe('error handling', () => { + it('should handle findAllGaps errors', async () => { + (findAllGaps as jest.Mock).mockRejectedValue(new Error('Find gaps failed')); + + await updateGaps({ + ruleId: 'test-rule-id', + start: new Date('2024-01-01T00:00:00.000Z'), + end: new Date('2024-01-01T01:00:00.000Z'), + eventLogger: mockEventLogger, + eventLogClient: mockEventLogClient, + logger: mockLogger, + savedObjectsRepository: mockSavedObjectsRepository, + backfillClient: mockBackfillClient, + }); + + expect(mockLogger.error).toHaveBeenCalledWith( + expect.stringContaining('Failed to update gaps for rule test-rule-id') + ); + }); + + it('should retry on conflict errors', async () => { + const testGap = createTestGap(); + (findAllGaps as jest.Mock).mockResolvedValue([testGap]); + + const conflictError = new EsErrors.ResponseError({ + statusCode: 409, + body: { error: { type: 'version_conflict_engine_exception' } }, + headers: {}, + meta: {} as any, + warnings: [], + }); + + mockEventLogger.updateEvent + .mockRejectedValueOnce(conflictError) + .mockRejectedValueOnce(conflictError) + .mockResolvedValue(); + + await updateGaps({ + ruleId: 'test-rule-id', + start: new Date('2024-01-01T00:00:00.000Z'), + end: new Date('2024-01-01T01:00:00.000Z'), + eventLogger: mockEventLogger, + eventLogClient: mockEventLogClient, + logger: mockLogger, + savedObjectsRepository: mockSavedObjectsRepository, + backfillClient: mockBackfillClient, + }); + + expect(mockEventLogger.updateEvent).toHaveBeenCalledTimes(3); + }); + }); + + describe('backfill handling', () => { + beforeEach(() => { + (updateGapFromSchedule as jest.Mock).mockImplementation((params) => params.gap); + (calculateGapStateFromAllBackfills as jest.Mock).mockImplementation((params) => params.gap); + }); + + it('should handle direct schedule updates', async () => { + const testGap = createTestGap(); + (findAllGaps as jest.Mock).mockResolvedValue([testGap]); + + const backfillSchedule = [ + { + runAt: '2024-01-01T00:30:00.000Z', + interval: '30m', + status: adHocRunStatus.COMPLETE, + }, + ]; + + await updateGaps({ + ruleId: 'test-rule-id', + start: new Date('2024-01-01T00:00:00.000Z'), + end: new Date('2024-01-01T01:00:00.000Z'), + eventLogger: mockEventLogger, + eventLogClient: mockEventLogClient, + logger: mockLogger, + savedObjectsRepository: mockSavedObjectsRepository, + backfillSchedule, + backfillClient: mockBackfillClient, + }); + + expect(updateGapFromSchedule).toHaveBeenCalledWith({ + gap: testGap, + backfillSchedule, + }); + expect(calculateGapStateFromAllBackfills).not.toHaveBeenCalled(); + }); + + it('should trigger refetch when shouldRefetchAllBackfills is true', async () => { + const testGap = createTestGap(); + (findAllGaps as jest.Mock).mockResolvedValue([testGap]); + + await updateGaps({ + ruleId: 'test-rule-id', + start: new Date('2024-01-01T00:00:00.000Z'), + end: new Date('2024-01-01T01:00:00.000Z'), + eventLogger: mockEventLogger, + eventLogClient: mockEventLogClient, + logger: mockLogger, + savedObjectsRepository: mockSavedObjectsRepository, + shouldRefetchAllBackfills: true, + backfillClient: mockBackfillClient, + }); + + expect(updateGapFromSchedule).not.toHaveBeenCalled(); + expect(calculateGapStateFromAllBackfills).toHaveBeenCalled(); + }); + }); +}); diff --git a/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/update_gaps.ts b/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/update_gaps.ts new file mode 100644 index 0000000000000..458fcc2607a90 --- /dev/null +++ b/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/update/update_gaps.ts @@ -0,0 +1,131 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Logger, ISavedObjectsRepository } from '@kbn/core/server'; +import { IEventLogClient, IEventLogger } from '@kbn/event-log-plugin/server'; +import { BackfillClient } from '../../../backfill_client/backfill_client'; +import { AlertingEventLogger } from '../../alerting_event_logger/alerting_event_logger'; +import { findAllGaps } from '../find_gaps'; +import { retryTransientEsErrors } from '../../../alerts_service/lib/retry_transient_es_errors'; +import { gapStatus } from '../../../../common/constants'; +import { BackfillSchedule } from '../../../application/backfill/result/types'; +import { adHocRunStatus } from '../../../../common/constants'; +import { calculateGapStateFromAllBackfills } from './calculate_gaps_state'; +import { updateGapFromSchedule } from './update_gap_from_schedule'; + +interface UpdateGapsParams { + ruleId: string; + backfillSchedule?: BackfillSchedule[]; + start: Date; + end: Date; + eventLogger?: IEventLogger; + eventLogClient: IEventLogClient; + logger: Logger; + savedObjectsRepository: ISavedObjectsRepository; + shouldRefetchAllBackfills?: boolean; + backfillClient: BackfillClient; +} + +const CONFLICT_STATUS_CODE = 409; + +export const updateGaps = async (params: UpdateGapsParams) => { + const { ruleId, start, end, logger } = params; + try { + await retryTransientEsErrors(() => _updateGaps(params), { + logger, + additionalRetryableStatusCodes: [CONFLICT_STATUS_CODE], + }); + } catch (e) { + logger.error( + `Failed to update gaps for rule ${ruleId} from: ${start.toISOString()} to: ${end.toISOString()}: ${ + e.message + }` + ); + } +}; + +/** + * Find all gaps for this date range and ruleId + * Then add the filled interval to the gaps + * Then update the gaps in the event log + */ +const _updateGaps = async (params: UpdateGapsParams) => { + const { + ruleId, + start, + end, + logger, + eventLogClient, + eventLogger, + backfillSchedule, + savedObjectsRepository, + shouldRefetchAllBackfills, + backfillClient, + } = params; + + if (!eventLogger) { + throw new Error('Event logger is required'); + } + + const alertingEventLogger = new AlertingEventLogger(eventLogger); + + const allGaps = await findAllGaps({ + eventLogClient, + logger, + params: { + ruleId, + start, + end, + statuses: [gapStatus.PARTIALLY_FILLED, gapStatus.UNFILLED], + }, + }); + + for (const gap of allGaps) { + const hasFailedBackfillTask = backfillSchedule?.some( + (scheduleItem) => + scheduleItem.status === adHocRunStatus.ERROR || + scheduleItem.status === adHocRunStatus.TIMEOUT + ); + + // add filled and in progress intervals from schedule + if (backfillSchedule && !hasFailedBackfillTask) { + updateGapFromSchedule({ + gap, + backfillSchedule, + }); + } + + // reset in progress intervals and refetch all backfills, to calculate the correct state + // if there failed backfill task, or we remove backfill, we can't just remove in progress intervals from the gap + // we need to refetch all backfills to calculate in-progress intervals + if (hasFailedBackfillTask || !backfillSchedule || shouldRefetchAllBackfills) { + await calculateGapStateFromAllBackfills({ + gap, + savedObjectsRepository, + ruleId, + backfillClient, + }); + } + + const esGap = gap.getEsObject(); + const internalFields = gap.internalFields; + + if (internalFields) { + try { + await alertingEventLogger.updateGap({ + internalFields, + gap: esGap, + }); + } catch (e) { + if (e.statusCode === CONFLICT_STATUS_CODE) { + logger.error('Failed to udpate gap, because of conflicting versions'); + } + throw e; + } + } + } +}; diff --git a/x-pack/platform/plugins/shared/alerting/server/routes/gaps/apis/find/transforms/transform_response/v1.ts b/x-pack/platform/plugins/shared/alerting/server/routes/gaps/apis/find/transforms/transform_response/v1.ts index a3e9b3b5852d1..0ec42b426584b 100644 --- a/x-pack/platform/plugins/shared/alerting/server/routes/gaps/apis/find/transforms/transform_response/v1.ts +++ b/x-pack/platform/plugins/shared/alerting/server/routes/gaps/apis/find/transforms/transform_response/v1.ts @@ -24,7 +24,7 @@ export const transformResponse = ({ per_page: perPage, total, data: gapsData.map((gap) => ({ - _id: gap?.meta?._id, + _id: gap?.internalFields?._id, ...gap.getEsObject(), '@timestamp': gap.timestamp, })), diff --git a/x-pack/platform/plugins/shared/alerting/server/task_runner/ad_hoc_task_runner.ts b/x-pack/platform/plugins/shared/alerting/server/task_runner/ad_hoc_task_runner.ts index b7970394f95b7..81848adc29784 100644 --- a/x-pack/platform/plugins/shared/alerting/server/task_runner/ad_hoc_task_runner.ts +++ b/x-pack/platform/plugins/shared/alerting/server/task_runner/ad_hoc_task_runner.ts @@ -24,7 +24,7 @@ import { CancellableTask, RunResult } from '@kbn/task-manager-plugin/server/task import { AdHocRunStatus, adHocRunStatus } from '../../common/constants'; import { RuleRunnerErrorStackTraceLog, RuleTaskStateAndMetrics, TaskRunnerContext } from './types'; import { getExecutorServices } from './get_executor_services'; -import { ErrorWithReason, parseDuration, validateRuleTypeParams } from '../lib'; +import { ErrorWithReason, validateRuleTypeParams } from '../lib'; import { AlertInstanceContext, AlertInstanceState, @@ -52,8 +52,7 @@ import { import { RuleRunMetrics, RuleRunMetricsStore } from '../lib/rule_run_metrics_store'; import { getEsErrorMessage } from '../lib/errors'; import { Result, isOk, asOk, asErr } from '../lib/result_type'; -import { addFilledIntervalToGaps } from '../lib/rule_gaps/update/add_filled_interval_to_gaps'; -import { calculateInProgressIntervalsForGaps } from '../lib/rule_gaps/update/caclculate_in_progress_intervals_for_gaps'; +import { updateGaps } from '../lib/rule_gaps/update/update_gaps'; interface ConstructorParams { context: TaskRunnerContext; @@ -76,6 +75,7 @@ export class AdHocTaskRunner implements CancellableTask { private readonly taskInstance: ConcreteTaskInstance; private adHocRunSchedule: AdHocRunSchedule[] = []; + private adHocRange: { start: string; end: string | undefined } | null = null; private alertingEventLogger: AlertingEventLogger; private cancelled: boolean = false; private logger: Logger; @@ -321,7 +321,7 @@ export class AdHocTaskRunner implements CancellableTask { ); } - const { rule, apiKeyToUse, schedule } = adHocRunData; + const { rule, apiKeyToUse, schedule, start, end } = adHocRunData; this.apiKeyToUse = apiKeyToUse; let ruleType: UntypedNormalizedRuleType; @@ -387,6 +387,7 @@ export class AdHocTaskRunner implements CancellableTask { // Determine which schedule entry we're going to run // Find the first index where the status is pending this.adHocRunSchedule = schedule; + this.adHocRange = { start, end }; this.scheduleToRunIndex = (this.adHocRunSchedule ?? []).findIndex( (s: AdHocRunSchedule) => s.status === adHocRunStatus.PENDING ); @@ -487,12 +488,6 @@ export class AdHocTaskRunner implements CancellableTask { executionStatus?.error?.reason === RuleExecutionStatusErrorReasons.Validate); if (startedAt) { - if (executionStatus.status === 'error') { - await this.calculateInProgressIntervalsForGaps(); - } else { - await this.addFilledIntervalToGaps(); - } - // Capture how long it took for the rule to run after being claimed this.timer.setDuration(TaskRunnerTimerSpan.TotalRunDuration, startedAt); } @@ -546,6 +541,9 @@ export class AdHocTaskRunner implements CancellableTask { await this.processAdHocRunResults(runMetrics); this.shouldDeleteTask = this.shouldDeleteTask || !this.hasAnyPendingRuns(); + + // await this.updateGapsAfterBackfillComplete(); + return { state: {}, ...(this.shouldDeleteTask ? {} : { runAt: new Date() }), @@ -587,8 +585,6 @@ export class AdHocTaskRunner implements CancellableTask { }); this.shouldDeleteTask = !this.hasAnyPendingRuns(); - await this.calculateInProgressIntervalsForGaps(); - // cleanup function is not called for timed out tasks await this.cleanup(); } @@ -596,6 +592,8 @@ export class AdHocTaskRunner implements CancellableTask { async cleanup() { if (!this.shouldDeleteTask) return; + await this.updateGapsAfterBackfillComplete(); + try { await this.internalSavedObjectsRepository.delete( AD_HOC_RUN_SAVED_OBJECT_TYPE, @@ -613,46 +611,29 @@ export class AdHocTaskRunner implements CancellableTask { } } - private async getParamsForGapsUpdate() { - if (this.scheduleToRunIndex < 0) return null; + private async updateGapsAfterBackfillComplete() { + if (!this.shouldDeleteTask) return; - const intervalInMs = parseDuration(this.adHocRunSchedule[this.scheduleToRunIndex].interval); + if (this.scheduleToRunIndex < 0 || !this.adHocRange) return null; - const endGapsRange = new Date(this.adHocRunSchedule[this.scheduleToRunIndex].runAt); - const startGapsRange = new Date(endGapsRange.getTime() - intervalInMs); const fakeRequest = getFakeKibanaRequest( this.context, this.taskInstance.params.spaceId, this.apiKeyToUse ); + const eventLogClient = await this.context.getEventLogClient(fakeRequest); - return { + return updateGaps({ ruleId: this.ruleId, - start: startGapsRange, - end: endGapsRange, + start: new Date(this.adHocRange.start), + end: this.adHocRange.end ? new Date(this.adHocRange.end) : new Date(), eventLogger: this.context.eventLogger, eventLogClient, logger: this.logger, - }; - } - - private async calculateInProgressIntervalsForGaps() { - const params = await this.getParamsForGapsUpdate(); - if (params === null) return; - - return calculateInProgressIntervalsForGaps({ - ...params, + backfillSchedule: this.adHocRunSchedule, savedObjectsRepository: this.internalSavedObjectsRepository, - }); - } - - private async addFilledIntervalToGaps() { - const params = await this.getParamsForGapsUpdate(); - if (params === null) return; - - return addFilledIntervalToGaps({ - ...params, + backfillClient: this.context.backfillClient, }); } } diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/gap/index.ts b/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/gap/index.ts index 9784827bbe3bc..fb61386cf5452 100644 --- a/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/gap/index.ts +++ b/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/gap/index.ts @@ -11,5 +11,6 @@ import { FtrProviderContext } from '../../../../../common/ftr_provider_context'; export default function gapsTests({ loadTestFile }: FtrProviderContext) { describe('rule gaps', () => { loadTestFile(require.resolve('./find')); + loadTestFile(require.resolve('./update_gaps')); }); } diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/gap/update_gaps.ts b/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/gap/update_gaps.ts new file mode 100644 index 0000000000000..52a8042405bcc --- /dev/null +++ b/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/gap/update_gaps.ts @@ -0,0 +1,519 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import moment from 'moment'; +import { AD_HOC_RUN_SAVED_OBJECT_TYPE } from '@kbn/alerting-plugin/server/saved_objects'; +import { SuperuserAtSpace1 } from '../../../../scenarios'; +import { getUrlPrefix, ObjectRemover, getTestRuleData } from '../../../../../common/lib'; +import { FtrProviderContext } from '../../../../../common/ftr_provider_context'; +import { getEventLog } from '../../../../../common/lib/get_event_log'; + +// eslint-disable-next-line import/no-default-export +export default function updateGapsTests({ getService }: FtrProviderContext) { + const supertest = getService('supertest'); + const retry = getService('retry'); + + describe.only('update gaps', () => { + const objectRemover = new ObjectRemover(supertest); + const gapStart = moment().subtract(14, 'days').startOf('day').toISOString(); + const gapEnd = moment().subtract(13, 'days').startOf('day').toISOString(); + + afterEach(async () => { + await objectRemover.removeAll(); + }); + + function getRule(overwrites = {}) { + return getTestRuleData({ + rule_type_id: 'test.patternFiringAutoRecoverFalse', + params: { + pattern: { + instance: [true, false, true], + }, + }, + schedule: { interval: '12h' }, + ...overwrites, + }); + } + + async function waitForBackfillComplete(backfillId: string, spaceId: string) { + await retry.try(async () => { + const response = await supertest + .get(`${getUrlPrefix(spaceId)}/internal/alerting/rules/backfill/${backfillId}`) + .set('kbn-xsrf', 'foo'); + + expect(response.statusCode).to.eql(404); + }); + } + + it('should update gap status after backfill execution', async () => { + const { space } = SuperuserAtSpace1; + + // Create a rule + const ruleResponse = await supertest + .post(`${getUrlPrefix(space.id)}/api/alerting/rule`) + .set('kbn-xsrf', 'foo') + .send(getRule()) + .expect(200); + const ruleId = ruleResponse.body.id; + objectRemover.add(space.id, ruleId, 'rule', 'alerting'); + + // Report a gap + await supertest + .post(`${getUrlPrefix(space.id)}/_test/report_gap`) + .set('kbn-xsrf', 'foo') + .send({ + ruleId, + start: gapStart, + end: gapEnd, + spaceId: space.id, + }); + + // Verify gap exists and is unfilled + const initialGapResponse = await supertest + .post(`${getUrlPrefix(space.id)}/internal/alerting/rules/gaps/_find`) + .set('kbn-xsrf', 'foo') + .send({ + rule_id: ruleId, + start: gapStart, + end: gapEnd, + }); + + expect(initialGapResponse.statusCode).to.eql(200); + expect(initialGapResponse.body.total).to.eql(1); + const initialGap = initialGapResponse.body.data[0]; + expect(initialGap.status).to.eql('unfilled'); + expect(initialGap.unfilled_duration_ms).to.eql(86400000); + expect(initialGap.unfilled_intervals).to.have.length(1); + expect(initialGap.unfilled_intervals[0].gte).to.eql(gapStart); + expect(initialGap.unfilled_intervals[0].lte).to.eql(gapEnd); + expect(initialGap.filled_intervals).to.have.length(0); + expect(initialGap.in_progress_intervals).to.have.length(0); + + // Schedule backfill for the gap period + const scheduleResponse = await supertest + .post(`${getUrlPrefix(space.id)}/internal/alerting/rules/backfill/_schedule`) + .set('kbn-xsrf', 'foo') + .send([{ rule_id: ruleId, start: gapStart, end: gapEnd }]); + + expect(scheduleResponse.statusCode).to.eql(200); + const backfillId = scheduleResponse.body[0].id; + + // Wait for backfill to complete and verify all executions + await waitForBackfillComplete(backfillId, space.id); + + // Verify gap is now filled + const finalGapResponse = await supertest + .post(`${getUrlPrefix(space.id)}/internal/alerting/rules/gaps/_find`) + .set('kbn-xsrf', 'foo') + .send({ + rule_id: ruleId, + start: gapStart, + end: gapEnd, + }); + + expect(finalGapResponse.statusCode).to.eql(200); + expect(finalGapResponse.body.total).to.eql(1); + const finalGap = finalGapResponse.body.data[0]; + expect(finalGap.status).to.eql('filled'); + expect(finalGap.filled_duration_ms).to.eql(86400000); + expect(finalGap.unfilled_duration_ms).to.eql(0); + expect(finalGap.filled_intervals).to.have.length(1); + expect(finalGap.filled_intervals[0].gte).to.eql(gapStart); + expect(finalGap.filled_intervals[0].lte).to.eql(gapEnd); + expect(finalGap.unfilled_intervals).to.have.length(0); + expect(finalGap.in_progress_intervals).to.have.length(0); + }); + + it('should mark intervals as in_progress immediately after scheduling backfill', async () => { + const { space } = SuperuserAtSpace1; + + // Create a rule with timeout pattern to ensure it stays in progress + const ruleResponse = await supertest + .post(`${getUrlPrefix(space.id)}/api/alerting/rule`) + .set('kbn-xsrf', 'foo') + .send( + getRule({ + params: { + pattern: { + instance: ['timeout'], + }, + }, + }) + ) + .expect(200); + const ruleId = ruleResponse.body.id; + objectRemover.add(space.id, ruleId, 'rule', 'alerting'); + + // Report a gap + await supertest + .post(`${getUrlPrefix(space.id)}/_test/report_gap`) + .set('kbn-xsrf', 'foo') + .send({ + ruleId, + start: gapStart, + end: gapEnd, + spaceId: space.id, + }); + + // Schedule backfill + const scheduleResponse = await supertest + .post(`${getUrlPrefix(space.id)}/internal/alerting/rules/backfill/_schedule`) + .set('kbn-xsrf', 'foo') + .send([{ rule_id: ruleId, start: gapStart, end: gapEnd }]); + + expect(scheduleResponse.statusCode).to.eql(200); + + // Verify intervals are marked as in_progress immediately + await retry.try(async () => { + const inProgressResponse = await supertest + .post(`${getUrlPrefix(space.id)}/internal/alerting/rules/gaps/_find`) + .set('kbn-xsrf', 'foo') + .send({ + rule_id: ruleId, + start: gapStart, + end: gapEnd, + }); + + expect(inProgressResponse.statusCode).to.eql(200); + expect(inProgressResponse.body.total).to.eql(1); + const gap = inProgressResponse.body.data[0]; + expect(gap.status).to.eql('partially_filled'); + expect(gap.in_progress_intervals).to.have.length(1); + expect(gap.in_progress_intervals[0].gte).to.eql(gapStart); + expect(gap.in_progress_intervals[0].lte).to.eql(gapEnd); + }); + + // Wait for timeout event + await retry.try(async () => { + const events = await getEventLog({ + getService, + spaceId: space.id, + type: AD_HOC_RUN_SAVED_OBJECT_TYPE, + id: scheduleResponse.body[0].id, + provider: 'alerting', + actions: new Map([['execute-timeout', { equal: 1 }]]), + }); + expect(events.length).to.eql(1); + }); + + await waitForBackfillComplete(scheduleResponse.body[0].id, space.id); + + await retry.try(async () => { + // Verify in-progress intervals are removed after timeout + const finalGapResponse = await supertest + .post(`${getUrlPrefix(space.id)}/internal/alerting/rules/gaps/_find`) + .set('kbn-xsrf', 'foo') + .send({ + rule_id: ruleId, + start: gapStart, + end: gapEnd, + }); + + expect(finalGapResponse.statusCode).to.eql(200); + expect(finalGapResponse.body.total).to.eql(1); + const finalGap = finalGapResponse.body.data[0]; + expect(finalGap.in_progress_intervals).to.have.length(0); + expect(finalGap.status).to.eql('unfilled'); + expect(finalGap.unfilled_intervals).to.have.length(1); + expect(finalGap.unfilled_intervals[0].gte).to.eql(gapStart); + expect(finalGap.unfilled_intervals[0].lte).to.eql(gapEnd); + expect(finalGap.filled_intervals).to.have.length(0); + }); + }); + + it('should handle partial gap filling when backfill overlaps', async () => { + const { space } = SuperuserAtSpace1; + + // Create a rule + const ruleResponse = await supertest + .post(`${getUrlPrefix(space.id)}/api/alerting/rule`) + .set('kbn-xsrf', 'foo') + .send(getRule()) + .expect(200); + const ruleId = ruleResponse.body.id; + objectRemover.add(space.id, ruleId, 'rule', 'alerting'); + + // Report a gap + await supertest + .post(`${getUrlPrefix(space.id)}/_test/report_gap`) + .set('kbn-xsrf', 'foo') + .send({ + ruleId, + start: gapStart, + end: gapEnd, + spaceId: space.id, + }); + + // Schedule backfill for only part of the gap + const partialStart = moment(gapStart).add(12, 'hours').toISOString(); + const partialEnd = moment(gapEnd).add(6, 'hours').toISOString(); + + const scheduleResponse = await supertest + .post(`${getUrlPrefix(space.id)}/internal/alerting/rules/backfill/_schedule`) + .set('kbn-xsrf', 'foo') + .send([{ rule_id: ruleId, start: partialStart, end: partialEnd }]); + + expect(scheduleResponse.statusCode).to.eql(200); + const backfillId = scheduleResponse.body[0].id; + + // Wait for backfill to complete + await waitForBackfillComplete(backfillId, space.id); + + // Verify gap is partially filled + const finalGapResponse = await supertest + .post(`${getUrlPrefix(space.id)}/internal/alerting/rules/gaps/_find`) + .set('kbn-xsrf', 'foo') + .send({ + rule_id: ruleId, + start: gapStart, + end: gapEnd, + }); + + expect(finalGapResponse.statusCode).to.eql(200); + expect(finalGapResponse.body.total).to.eql(1); + const gap = finalGapResponse.body.data[0]; + expect(gap.status).to.eql('partially_filled'); + expect(gap.filled_duration_ms).to.eql(12 * 60 * 60 * 1000); + expect(gap.filled_intervals[0].gte).to.eql(partialStart); + expect(gap.filled_intervals[0].lte).to.eql(gapEnd); + expect(gap.unfilled_intervals[0].gte).to.eql(gapStart); + expect(gap.unfilled_intervals[0].lte).to.eql(partialStart); + expect(gap.unfilled_duration_ms).to.be.eql(12 * 60 * 60 * 1000); + expect(gap.unfilled_intervals).to.have.length(1); + expect(gap.filled_intervals).to.have.length(1); + expect(gap.in_progress_intervals).to.have.length(0); + }); + + it('should fill gap with multiple backfills', async () => { + const { space } = SuperuserAtSpace1; + + // Create a rule + const ruleResponse = await supertest + .post(`${getUrlPrefix(space.id)}/api/alerting/rule`) + .set('kbn-xsrf', 'foo') + .send(getRule()) + .expect(200); + const ruleId = ruleResponse.body.id; + objectRemover.add(space.id, ruleId, 'rule', 'alerting'); + + // Report a gap + await supertest + .post(`${getUrlPrefix(space.id)}/_test/report_gap`) + .set('kbn-xsrf', 'foo') + .send({ + ruleId, + start: gapStart, + end: gapEnd, + spaceId: space.id, + }); + + // Schedule two backfills that together cover the entire gap + const firstHalfEnd = moment(gapStart).add(12, 'hours').toISOString(); + const secondHalfStart = firstHalfEnd; + + const scheduleResponse = await supertest + .post(`${getUrlPrefix(space.id)}/internal/alerting/rules/backfill/_schedule`) + .set('kbn-xsrf', 'foo') + .send([ + { rule_id: ruleId, start: gapStart, end: firstHalfEnd }, + { rule_id: ruleId, start: secondHalfStart, end: gapEnd }, + ]); + + expect(scheduleResponse.statusCode).to.eql(200); + expect(scheduleResponse.body).to.have.length(2); + + // Wait for both backfills to complete + await Promise.all( + scheduleResponse.body.map((result: { id: string }) => + waitForBackfillComplete(result.id, space.id) + ) + ); + + // Verify gap is completely filled + const finalGapResponse = await supertest + .post(`${getUrlPrefix(space.id)}/internal/alerting/rules/gaps/_find`) + .set('kbn-xsrf', 'foo') + .send({ + rule_id: ruleId, + start: gapStart, + end: gapEnd, + }); + + expect(finalGapResponse.statusCode).to.eql(200); + expect(finalGapResponse.body.total).to.eql(1); + const finalGap = finalGapResponse.body.data[0]; + expect(finalGap.status).to.eql('filled'); + expect(finalGap.filled_duration_ms).to.eql(86400000); + expect(finalGap.unfilled_duration_ms).to.eql(0); + expect(finalGap.filled_intervals).to.have.length(1); + expect(finalGap.filled_intervals[0].gte).to.eql(gapStart); + expect(finalGap.filled_intervals[0].lte).to.eql(gapEnd); + expect(finalGap.unfilled_intervals).to.have.length(0); + expect(finalGap.in_progress_intervals).to.have.length(0); + }); + + it('should update gap status after backfill is deleted', async () => { + const { space } = SuperuserAtSpace1; + + // Create a rule with timeout pattern to ensure we can catch it in progress + const ruleResponse = await supertest + .post(`${getUrlPrefix(space.id)}/api/alerting/rule`) + .set('kbn-xsrf', 'foo') + .send( + getRule({ + params: { + pattern: { + instance: ['timeout'], + }, + }, + }) + ) + .expect(200); + const ruleId = ruleResponse.body.id; + objectRemover.add(space.id, ruleId, 'rule', 'alerting'); + + // Report a gap + await supertest + .post(`${getUrlPrefix(space.id)}/_test/report_gap`) + .set('kbn-xsrf', 'foo') + .send({ + ruleId, + start: gapStart, + end: gapEnd, + spaceId: space.id, + }); + + // Schedule backfill + const scheduleResponse = await supertest + .post(`${getUrlPrefix(space.id)}/internal/alerting/rules/backfill/_schedule`) + .set('kbn-xsrf', 'foo') + .send([{ rule_id: ruleId, start: gapStart, end: gapEnd }]); + + expect(scheduleResponse.statusCode).to.eql(200); + const backfillId = scheduleResponse.body[0].id; + + // Wait for in-progress state + await retry.try(async () => { + const inProgressResponse = await supertest + .post(`${getUrlPrefix(space.id)}/internal/alerting/rules/gaps/_find`) + .set('kbn-xsrf', 'foo') + .send({ + rule_id: ruleId, + start: gapStart, + end: gapEnd, + }); + + expect(inProgressResponse.statusCode).to.eql(200); + expect(inProgressResponse.body.data[0].status).to.eql('partially_filled'); + expect(inProgressResponse.body.data[0].in_progress_intervals).to.have.length(1); + }); + + // Delete backfill while in progress + await supertest + .delete(`${getUrlPrefix(space.id)}/internal/alerting/rules/backfill/${backfillId}`) + .set('kbn-xsrf', 'foo') + .expect(204); + + // Verify gap status is updated + const finalGapResponse = await supertest + .post(`${getUrlPrefix(space.id)}/internal/alerting/rules/gaps/_find`) + .set('kbn-xsrf', 'foo') + .send({ + rule_id: ruleId, + start: gapStart, + end: gapEnd, + }); + + expect(finalGapResponse.statusCode).to.eql(200); + expect(finalGapResponse.body.total).to.eql(1); + const gap = finalGapResponse.body.data[0]; + expect(gap.status).to.eql('unfilled'); + expect(gap.in_progress_intervals).to.have.length(0); + expect(gap.unfilled_intervals).to.have.length(1); + expect(gap.unfilled_intervals[0].gte).to.eql(gapStart); + expect(gap.unfilled_intervals[0].lte).to.eql(gapEnd); + expect(gap.filled_intervals).to.have.length(0); + }); + + it('should handle task failures', async () => { + const { space } = SuperuserAtSpace1; + + // Create a rule that always errors + const ruleResponse = await supertest + .post(`${getUrlPrefix(space.id)}/api/alerting/rule`) + .set('kbn-xsrf', 'foo') + .send( + getRule({ + params: { + pattern: { + instance: ['error'], + }, + }, + }) + ) + .expect(200); + const ruleId = ruleResponse.body.id; + objectRemover.add(space.id, ruleId, 'rule', 'alerting'); + + // Report a gap + await supertest + .post(`${getUrlPrefix(space.id)}/_test/report_gap`) + .set('kbn-xsrf', 'foo') + .send({ + ruleId, + start: gapStart, + end: gapEnd, + spaceId: space.id, + }); + + // Schedule backfill + const scheduleResponse = await supertest + .post(`${getUrlPrefix(space.id)}/internal/alerting/rules/backfill/_schedule`) + .set('kbn-xsrf', 'foo') + .send([{ rule_id: ruleId, start: gapStart, end: gapEnd }]); + + expect(scheduleResponse.statusCode).to.eql(200); + const backfillId = scheduleResponse.body[0].id; + + // Wait for task failure event + await retry.try(async () => { + const events = await getEventLog({ + getService, + spaceId: space.id, + type: AD_HOC_RUN_SAVED_OBJECT_TYPE, + id: backfillId, + provider: 'alerting', + actions: new Map([['execute-backfill', { equal: 1 }]]), + }); + expect(events.length).to.eql(1); + expect(events[0]?.event?.outcome).to.eql('failure'); + expect(events[0]?.error?.message).to.eql('rule executor error'); + }); + + // Verify gap status is updated + const finalGapResponse = await supertest + .post(`${getUrlPrefix(space.id)}/internal/alerting/rules/gaps/_find`) + .set('kbn-xsrf', 'foo') + .send({ + rule_id: ruleId, + start: gapStart, + end: gapEnd, + }); + + expect(finalGapResponse.statusCode).to.eql(200); + expect(finalGapResponse.body.total).to.eql(1); + const gap = finalGapResponse.body.data[0]; + expect(gap.status).to.eql('unfilled'); + expect(gap.in_progress_intervals).to.have.length(0); + expect(gap.unfilled_intervals).to.have.length(1); + expect(gap.unfilled_intervals[0].gte).to.eql(gapStart); + expect(gap.unfilled_intervals[0].lte).to.eql(gapEnd); + expect(gap.filled_intervals).to.have.length(0); + }); + }); +} From 034cb7436e12f518706bc7e57e2fc42267181e1c Mon Sep 17 00:00:00 2001 From: Nikita Khristinin Date: Fri, 10 Jan 2025 09:22:12 +0100 Subject: [PATCH 11/17] remoe some code --- .../common/routes/gaps/apis/fill/index.ts | 13 - .../routes/gaps/apis/fill/schemas/latest.ts | 8 - .../routes/gaps/apis/fill/schemas/v1.ts | 12 - .../routes/gaps/apis/fill/types/latest.ts | 8 - .../common/routes/gaps/apis/fill/types/v1.ts | 11 - .../gaps/apis/get_rules_with_gaps/index.ts | 24 - .../get_rules_with_gaps/schemas/latest.ts | 8 - .../apis/get_rules_with_gaps/schemas/v1.ts | 36 -- .../apis/get_rules_with_gaps/types/latest.ts | 8 - .../gaps/apis/get_rules_with_gaps/types/v1.ts | 16 - .../methods/fill_gap_by_id/fill_gap_by_id.ts | 44 -- .../rule/methods/fill_gap_by_id/index.ts | 8 - .../server/lib/rule_gaps/find_gap_by_id.ts | 43 -- .../server/lib/rule_gaps/schemas/index.ts | 5 - .../server/lib/rule_gaps/types/index.ts | 3 +- .../gaps/apis/fill/fill_gap_by_id_route.ts | 51 -- .../routes/gaps/apis/fill/transforms/index.ts | 10 - .../transforms/transform_request/latest.ts | 8 - .../fill/transforms/transform_request/v1.ts | 18 - .../get_rules_with_gaps_route.ts | 47 -- .../server/rules_client/rules_client.ts | 6 +- .../pages/rule_details/index.tsx | 2 - .../rule_gaps/components/rule_gaps/index.tsx | 495 ------------------ .../components/rule_gaps/status_filter.tsx | 54 -- .../components/rule_gaps/translations.tsx | 126 ----- .../rule_gaps/components/rule_gaps/utils.ts | 21 - .../rules_with_gaps_overview_panel/index.tsx | 237 --------- .../translations.tsx | 50 -- .../components/rules_table/rules_tables.tsx | 8 - 29 files changed, 3 insertions(+), 1377 deletions(-) delete mode 100644 x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/fill/index.ts delete mode 100644 x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/fill/schemas/latest.ts delete mode 100644 x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/fill/schemas/v1.ts delete mode 100644 x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/fill/types/latest.ts delete mode 100644 x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/fill/types/v1.ts delete mode 100644 x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/get_rules_with_gaps/index.ts delete mode 100644 x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/get_rules_with_gaps/schemas/latest.ts delete mode 100644 x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/get_rules_with_gaps/schemas/v1.ts delete mode 100644 x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/get_rules_with_gaps/types/latest.ts delete mode 100644 x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/get_rules_with_gaps/types/v1.ts delete mode 100644 x-pack/platform/plugins/shared/alerting/server/application/rule/methods/fill_gap_by_id/fill_gap_by_id.ts delete mode 100644 x-pack/platform/plugins/shared/alerting/server/application/rule/methods/fill_gap_by_id/index.ts delete mode 100644 x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/find_gap_by_id.ts delete mode 100644 x-pack/platform/plugins/shared/alerting/server/routes/gaps/apis/fill/fill_gap_by_id_route.ts delete mode 100644 x-pack/platform/plugins/shared/alerting/server/routes/gaps/apis/fill/transforms/index.ts delete mode 100644 x-pack/platform/plugins/shared/alerting/server/routes/gaps/apis/fill/transforms/transform_request/latest.ts delete mode 100644 x-pack/platform/plugins/shared/alerting/server/routes/gaps/apis/fill/transforms/transform_request/v1.ts delete mode 100644 x-pack/platform/plugins/shared/alerting/server/routes/gaps/apis/get_rules_with_gaps/get_rules_with_gaps_route.ts delete mode 100644 x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_gaps/components/rule_gaps/index.tsx delete mode 100644 x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_gaps/components/rule_gaps/status_filter.tsx delete mode 100644 x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_gaps/components/rule_gaps/translations.tsx delete mode 100644 x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_gaps/components/rule_gaps/utils.ts delete mode 100644 x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_gaps/components/rules_with_gaps_overview_panel/index.tsx delete mode 100644 x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_gaps/components/rules_with_gaps_overview_panel/translations.tsx diff --git a/x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/fill/index.ts b/x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/fill/index.ts deleted file mode 100644 index 3fb3dde4ed2b8..0000000000000 --- a/x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/fill/index.ts +++ /dev/null @@ -1,13 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export { fillGapByIdQuerySchema } from './schemas/latest'; - -export { fillGapByIdQuerySchema as fillGapByIdQuerySchemaV1 } from './schemas/v1'; - -export type { FillGapByIdQuery as FillGapByIdQueryV1 } from './types/v1'; -export type { ScheduleBackfillRequestBodyV1 as FillGapByIdResponseV1 } from '../../../backfill/apis/schedule'; diff --git a/x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/fill/schemas/latest.ts b/x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/fill/schemas/latest.ts deleted file mode 100644 index 25300c97a6d2e..0000000000000 --- a/x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/fill/schemas/latest.ts +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export * from './v1'; diff --git a/x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/fill/schemas/v1.ts b/x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/fill/schemas/v1.ts deleted file mode 100644 index 122cd35a498db..0000000000000 --- a/x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/fill/schemas/v1.ts +++ /dev/null @@ -1,12 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import { schema } from '@kbn/config-schema'; - -export const fillGapByIdQuerySchema = schema.object({ - rule_id: schema.string(), - gap_id: schema.string(), -}); diff --git a/x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/fill/types/latest.ts b/x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/fill/types/latest.ts deleted file mode 100644 index 25300c97a6d2e..0000000000000 --- a/x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/fill/types/latest.ts +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export * from './v1'; diff --git a/x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/fill/types/v1.ts b/x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/fill/types/v1.ts deleted file mode 100644 index 64bee71b53643..0000000000000 --- a/x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/fill/types/v1.ts +++ /dev/null @@ -1,11 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { TypeOf } from '@kbn/config-schema'; -import { fillGapByIdQuerySchemaV1 } from '..'; - -export type FillGapByIdQuery = TypeOf; diff --git a/x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/get_rules_with_gaps/index.ts b/x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/get_rules_with_gaps/index.ts deleted file mode 100644 index 9f3b190999fb7..0000000000000 --- a/x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/get_rules_with_gaps/index.ts +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export { getRulesWithGapQuerySchema, getRulesWithGapResponseSchema } from './schemas/latest'; -export type { - GetRulesWithGapQuery, - GetRulesWithGapResponse, - GetRulesWithGapResponseBody, -} from './types/latest'; - -export { - getRulesWithGapQuerySchema as getRulesWithGapQuerySchemaV1, - getRulesWithGapResponseSchema as getRulesWithGapResponseSchemaV1, -} from './schemas/v1'; - -export type { - GetRulesWithGapQuery as GetRulesWithGapQueryV1, - GetRulesWithGapResponse as GetRulesWithGapResponseV1, - GetRulesWithGapResponseBody as GetRulesWithGapResponseBodyV1, -} from './types/v1'; diff --git a/x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/get_rules_with_gaps/schemas/latest.ts b/x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/get_rules_with_gaps/schemas/latest.ts deleted file mode 100644 index 25300c97a6d2e..0000000000000 --- a/x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/get_rules_with_gaps/schemas/latest.ts +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export * from './v1'; diff --git a/x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/get_rules_with_gaps/schemas/v1.ts b/x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/get_rules_with_gaps/schemas/v1.ts deleted file mode 100644 index f07e9ef93cd12..0000000000000 --- a/x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/get_rules_with_gaps/schemas/v1.ts +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import { schema } from '@kbn/config-schema'; - -export const getRulesWithGapQuerySchema = schema.object( - { - end: schema.string(), - start: schema.string(), - statuses: schema.maybe(schema.arrayOf(schema.string())), - }, - { - validate({ start, end }) { - if (start) { - const parsedStart = Date.parse(start); - if (isNaN(parsedStart)) { - return `[start]: query start must be valid date`; - } - } - if (end) { - const parsedEnd = Date.parse(end); - if (isNaN(parsedEnd)) { - return `[end]: query end must be valid date`; - } - } - }, - } -); - -export const getRulesWithGapResponseSchema = schema.object({ - total: schema.number(), - ruleIds: schema.arrayOf(schema.string()), -}); diff --git a/x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/get_rules_with_gaps/types/latest.ts b/x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/get_rules_with_gaps/types/latest.ts deleted file mode 100644 index 25300c97a6d2e..0000000000000 --- a/x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/get_rules_with_gaps/types/latest.ts +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export * from './v1'; diff --git a/x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/get_rules_with_gaps/types/v1.ts b/x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/get_rules_with_gaps/types/v1.ts deleted file mode 100644 index 55cf952ba0cd4..0000000000000 --- a/x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/get_rules_with_gaps/types/v1.ts +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { TypeOf } from '@kbn/config-schema'; -import { getRulesWithGapQuerySchemaV1, getRulesWithGapResponseSchemaV1 } from '..'; - -export type GetRulesWithGapQuery = TypeOf; -export type GetRulesWithGapResponseBody = TypeOf; - -export interface GetRulesWithGapResponse { - body: GetRulesWithGapResponseBody; -} diff --git a/x-pack/platform/plugins/shared/alerting/server/application/rule/methods/fill_gap_by_id/fill_gap_by_id.ts b/x-pack/platform/plugins/shared/alerting/server/application/rule/methods/fill_gap_by_id/fill_gap_by_id.ts deleted file mode 100644 index f506a5c45e273..0000000000000 --- a/x-pack/platform/plugins/shared/alerting/server/application/rule/methods/fill_gap_by_id/fill_gap_by_id.ts +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import Boom from '@hapi/boom'; - -import { RulesClientContext } from '../../../../rules_client'; - -import { findGapById as _findGapById } from '../../../../lib/rule_gaps/find_gap_by_id'; -import { FindGapByIdParams } from '../../../../lib/rule_gaps/types'; -import { scheduleBackfill } from '../../../backfill/methods/schedule'; - -export async function fillGapById(context: RulesClientContext, params: FindGapByIdParams) { - try { - const eventLogClient = await context.getEventLogClient(); - const gap = await _findGapById({ - params, - eventLogClient, - logger: context.logger, - }); - - if (!gap) { - throw Boom.notFound(`Gap not found for ruleId ${params.ruleId} and gapId ${params.gapId}`); - } - - const gapState = gap.getState(); - - const allGapsScheduled = - gapState.unfilledIntervals.map((interval) => ({ - ruleId: params.ruleId, - start: interval.gte, - end: interval.lte, - })) ?? []; - - return scheduleBackfill(context, allGapsScheduled); - } catch (err) { - const errorMessage = `Failed to find gap and schedule manual rule run`; - context.logger.error(`${errorMessage} - ${err}`); - throw Boom.boomify(err, { message: errorMessage }); - } -} diff --git a/x-pack/platform/plugins/shared/alerting/server/application/rule/methods/fill_gap_by_id/index.ts b/x-pack/platform/plugins/shared/alerting/server/application/rule/methods/fill_gap_by_id/index.ts deleted file mode 100644 index 51e0f74dd65c2..0000000000000 --- a/x-pack/platform/plugins/shared/alerting/server/application/rule/methods/fill_gap_by_id/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export { fillGapById } from './fill_gap_by_id'; diff --git a/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/find_gap_by_id.ts b/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/find_gap_by_id.ts deleted file mode 100644 index 977d7cb9a6a9c..0000000000000 --- a/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/find_gap_by_id.ts +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { IEventLogClient } from '@kbn/event-log-plugin/server'; -import { Logger } from '@kbn/core/server'; -import { RULE_SAVED_OBJECT_TYPE } from '../../saved_objects'; -import { FindGapByIdParams } from './types'; -import { Gap } from './gap'; -import { transformToGap } from './transforms/transform_to_gap'; - -export const findGapById = async ({ - eventLogClient, - logger, - params, -}: { - eventLogClient: IEventLogClient; - logger: Logger; - params: FindGapByIdParams; -}): Promise => { - const { gapId, ruleId } = params; - try { - const gapsResponse = await eventLogClient.findEventsBySavedObjectIds( - RULE_SAVED_OBJECT_TYPE, - [ruleId], - { - filter: `_id: ${gapId}`, - } - ); - - if (gapsResponse.total === 0) return null; - - const gap = transformToGap(gapsResponse)[0]; - - return gap; - } catch (err) { - logger.error(`Failed to find gap by id ${gapId} for rule ${ruleId.toString()}: ${err.message}`); - throw err; - } -}; diff --git a/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/schemas/index.ts b/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/schemas/index.ts index ab85499d32dca..646d4b14bbc62 100644 --- a/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/schemas/index.ts +++ b/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/schemas/index.ts @@ -67,8 +67,3 @@ export const findGapsParamsSchema = schema.object( }, } ); - -export const findGapByIdParamsSchema = schema.object({ - gapId: schema.string(), - ruleId: schema.string(), -}); diff --git a/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/types/index.ts b/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/types/index.ts index 53c837a5f37f0..aaf2ff68a376d 100644 --- a/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/types/index.ts +++ b/x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/types/index.ts @@ -6,11 +6,10 @@ */ import { TypeOf } from '@kbn/config-schema'; -import { gapBaseSchema, findGapsParamsSchema, findGapByIdParamsSchema } from '../schemas'; +import { gapBaseSchema, findGapsParamsSchema } from '../schemas'; export type GapBase = TypeOf; export type FindGapsParams = TypeOf; -export type FindGapByIdParams = TypeOf; export interface Interval { gte: Date; diff --git a/x-pack/platform/plugins/shared/alerting/server/routes/gaps/apis/fill/fill_gap_by_id_route.ts b/x-pack/platform/plugins/shared/alerting/server/routes/gaps/apis/fill/fill_gap_by_id_route.ts deleted file mode 100644 index 678cf5ee7df4a..0000000000000 --- a/x-pack/platform/plugins/shared/alerting/server/routes/gaps/apis/fill/fill_gap_by_id_route.ts +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import { IRouter } from '@kbn/core/server'; -import { - fillGapByIdQuerySchemaV1, - FillGapByIdQueryV1, -} from '../../../../../common/routes/gaps/apis/fill'; -import { ScheduleBackfillResponseV1 } from '../../../../../common/routes/backfill/apis/schedule'; -import { ILicenseState } from '../../../../lib'; -import { verifyAccessAndContext } from '../../../lib'; -import { - AlertingRequestHandlerContext, - INTERNAL_ALERTING_GAPS_FILL_BY_ID_API_PATH, -} from '../../../../types'; -import { transformRequestV1 } from './transforms'; -import { transformResponseV1 } from '../../../backfill/apis/schedule/transforms'; - -export const fillGapByIdRoute = ( - router: IRouter, - licenseState: ILicenseState -) => { - router.post( - { - path: `${INTERNAL_ALERTING_GAPS_FILL_BY_ID_API_PATH}`, - validate: { - query: fillGapByIdQuerySchemaV1, - }, - options: { - access: 'internal', - }, - }, - router.handleLegacyErrors( - verifyAccessAndContext(licenseState, async function (context, req, res) { - const alertingContext = await context.alerting; - const rulesClient = await alertingContext.getRulesClient(); - const query: FillGapByIdQueryV1 = req.query; - - const result = await rulesClient.fillGapById(transformRequestV1(query)); - - const response: ScheduleBackfillResponseV1 = { - body: transformResponseV1(result), - }; - return res.ok(response); - }) - ) - ); -}; diff --git a/x-pack/platform/plugins/shared/alerting/server/routes/gaps/apis/fill/transforms/index.ts b/x-pack/platform/plugins/shared/alerting/server/routes/gaps/apis/fill/transforms/index.ts deleted file mode 100644 index 8f53ad06bea41..0000000000000 --- a/x-pack/platform/plugins/shared/alerting/server/routes/gaps/apis/fill/transforms/index.ts +++ /dev/null @@ -1,10 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export { transformRequest } from './transform_request/latest'; - -export { transformRequest as transformRequestV1 } from './transform_request/v1'; diff --git a/x-pack/platform/plugins/shared/alerting/server/routes/gaps/apis/fill/transforms/transform_request/latest.ts b/x-pack/platform/plugins/shared/alerting/server/routes/gaps/apis/fill/transforms/transform_request/latest.ts deleted file mode 100644 index 25300c97a6d2e..0000000000000 --- a/x-pack/platform/plugins/shared/alerting/server/routes/gaps/apis/fill/transforms/transform_request/latest.ts +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export * from './v1'; diff --git a/x-pack/platform/plugins/shared/alerting/server/routes/gaps/apis/fill/transforms/transform_request/v1.ts b/x-pack/platform/plugins/shared/alerting/server/routes/gaps/apis/fill/transforms/transform_request/v1.ts deleted file mode 100644 index 8e57187c21d5c..0000000000000 --- a/x-pack/platform/plugins/shared/alerting/server/routes/gaps/apis/fill/transforms/transform_request/v1.ts +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -/* eslint-disable @typescript-eslint/naming-convention */ - -import { FillGapByIdQueryV1 } from '../../../../../../../common/routes/gaps/apis/fill'; -import { FindGapByIdParams } from '../../../../../../lib/rule_gaps/types'; - -export const transformRequest = ({ - rule_id, - gap_id, -}: FillGapByIdQueryV1): FindGapByIdParams => ({ - gapId: gap_id, - ruleId: rule_id, -}); diff --git a/x-pack/platform/plugins/shared/alerting/server/routes/gaps/apis/get_rules_with_gaps/get_rules_with_gaps_route.ts b/x-pack/platform/plugins/shared/alerting/server/routes/gaps/apis/get_rules_with_gaps/get_rules_with_gaps_route.ts deleted file mode 100644 index 4c05ded9cc88f..0000000000000 --- a/x-pack/platform/plugins/shared/alerting/server/routes/gaps/apis/get_rules_with_gaps/get_rules_with_gaps_route.ts +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import { IRouter } from '@kbn/core/server'; -import { - getRulesWithGapQuerySchemaV1, - GetRulesWithGapQueryV1, - GetRulesWithGapResponseV1, -} from '../../../../../common/routes/gaps/apis/get_rules_with_gaps'; -import { ILicenseState } from '../../../../lib'; -import { verifyAccessAndContext } from '../../../lib'; -import { - AlertingRequestHandlerContext, - INTERNAL_ALERTING_GAPS_GET_RULES_API_PATH, -} from '../../../../types'; - -export const getRulesWithGapsRoute = ( - router: IRouter, - licenseState: ILicenseState -) => { - router.get( - { - path: `${INTERNAL_ALERTING_GAPS_GET_RULES_API_PATH}`, - validate: { - query: getRulesWithGapQuerySchemaV1, - }, - options: { - access: 'internal', - }, - }, - router.handleLegacyErrors( - verifyAccessAndContext(licenseState, async function (context, req, res) { - const rulesClient = (await context.alerting).getRulesClient(); - const query: GetRulesWithGapQueryV1 = req.query; - - const result = await rulesClient.getRulesWithGaps(query); - const response: GetRulesWithGapResponseV1 = { - body: result, - }; - return res.ok(response); - }) - ) - ); -}; diff --git a/x-pack/platform/plugins/shared/alerting/server/rules_client/rules_client.ts b/x-pack/platform/plugins/shared/alerting/server/rules_client/rules_client.ts index 85a9db84b7052..799e686e95125 100644 --- a/x-pack/platform/plugins/shared/alerting/server/rules_client/rules_client.ts +++ b/x-pack/platform/plugins/shared/alerting/server/rules_client/rules_client.ts @@ -77,10 +77,10 @@ import { FindBackfillParams } from '../application/backfill/methods/find/types'; import { DisableRuleParams } from '../application/rule/methods/disable'; import { EnableRuleParams } from '../application/rule/methods/enable_rule'; import { findGaps } from '../application/rule/methods/find_gaps'; -import { fillGapById } from '../application/rule/methods/fill_gap_by_id'; + import { GetRulesWithGapsParams } from '../application/rule/methods/get_rules_with_gaps/types'; import { getRulesWithGaps } from '../application/rule/methods/get_rules_with_gaps'; -import { FindGapsParams, FindGapByIdParams } from '../lib/rule_gaps/types'; +import { FindGapsParams } from '../lib/rule_gaps/types'; export type ConstructorOptions = Omit< RulesClientContext, @@ -215,8 +215,6 @@ export class RulesClient { public findGaps = (params: FindGapsParams) => findGaps(this.context, params); - public fillGapById = (params: FindGapByIdParams) => fillGapById(this.context, params); - public getRulesWithGaps = (params: GetRulesWithGapsParams) => getRulesWithGaps(this.context, params); } diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_details_ui/pages/rule_details/index.tsx b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_details_ui/pages/rule_details/index.tsx index c5c9285581409..e25e56692e40a 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_details_ui/pages/rule_details/index.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_details_ui/pages/rule_details/index.tsx @@ -797,8 +797,6 @@ const RuleDetailsPageComponent: React.FC = ({ theme={theme} /> - - diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_gaps/components/rule_gaps/index.tsx b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_gaps/components/rule_gaps/index.tsx deleted file mode 100644 index 419b29687a825..0000000000000 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_gaps/components/rule_gaps/index.tsx +++ /dev/null @@ -1,495 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { UseMutationOptions, UseQueryOptions } from '@tanstack/react-query'; -import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query'; -import { - INTERNAL_ALERTING_GAPS_FIND_API_PATH, - INTERNAL_ALERTING_GAPS_FILL_BY_ID_API_PATH, -} from '@kbn/alerting-plugin/common'; -import styled from 'styled-components'; -import React, { useCallback, useState } from 'react'; -import dateMath from '@kbn/datemath'; -import type { - CriteriaWithPagination, - EuiBasicTableColumn, - OnRefreshChangeProps, - OnTimeChangeProps, -} from '@elastic/eui'; -import { - EuiBasicTable, - EuiFlexGroup, - EuiFlexItem, - EuiPanel, - EuiBetaBadge, - EuiProgress, - EuiText, - EuiHealth, - EuiButtonEmpty, - EuiSuperDatePicker, -} from '@elastic/eui'; - -const DatePickerEuiFlexItem = styled(EuiFlexItem)` - max-width: 582px; -`; - -import type { FindGapsResponseBody } from '@kbn/alerting-plugin/common/routes/gaps/apis/find'; - -import type { FillGapByIdResponseV1 } from '@kbn/alerting-plugin/common/routes/gaps/apis/fill'; -import type { IHttpFetchError } from '@kbn/core/public'; -import { useAppToasts } from '../../../../common/hooks/use_app_toasts'; -import { useUserData } from '../../../../detections/components/user_info'; -import { hasUserCRUDPermission } from '../../../../common/utils/privileges'; - -import { BETA, BETA_TOOLTIP } from '../../../../common/translations'; - -import { HeaderSection } from '../../../../common/components/header_section'; -import { TableHeaderTooltipCell } from '../../../rule_management_ui/components/rules_table/table_header_tooltip_cell'; - -import { FormattedDate } from '../../../../common/components/formatted_date'; -import { KibanaServices, useKibana } from '../../../../common/lib/kibana'; -import { getFormattedDuration } from '../../../rule_details_ui/pages/rule_details/execution_log_table/rule_duration_format'; -import * as i18n from './translations'; -import type { Gap, GapStatus } from '../../types'; -import { getStatusLabel } from './utils'; -import { GapStatusFilter } from './status_filter'; - -const FIND_GAPS_FOR_RULE = 'FIND_GAP_FOR_RULE'; - -/** - * Find gaps for the given rule ID - * @param ruleIds string[] - * @param signal? AbortSignal - * @returns - */ -export const findGapsForRule = async ({ - ruleId, - page, - perPage, - signal, - sortField = '@timestamp', - sortOrder = 'desc', - start, - end, - statuses, -}: { - ruleId: string; - page: number; - perPage: number; - start: string; - end: string; - statuses: GapStatus[]; - signal?: AbortSignal; - sortField?: string; - sortOrder?: string; -}): Promise => { - const startDate = dateMath.parse(start); - const endDate = dateMath.parse(end, { roundUp: true }); - - return KibanaServices.get().http.fetch( - INTERNAL_ALERTING_GAPS_FIND_API_PATH, - { - method: 'POST', - body: JSON.stringify({ - rule_id: ruleId, - page, - per_page: perPage, - sort_field: sortField, - sort_order: sortOrder, - start: startDate?.utc().toISOString(), - end: endDate?.utc().toISOString(), - statuses, - }), - signal, - } - ); -}; - -/** - * Fill gap by Id for the given rule ID - * @param ruleIds string[] - * @param signal? AbortSignal - * @returns - */ -export const fillGapByIdForRule = async ({ - ruleId, - gapId, - signal, -}: FillGapQuery & { - signal?: AbortSignal; -}): Promise => - KibanaServices.get().http.fetch( - INTERNAL_ALERTING_GAPS_FILL_BY_ID_API_PATH, - { - method: 'POST', - query: { - rule_id: ruleId, - gap_id: gapId, - }, - signal, - } - ); - -export const useInvalidateFindGapsQuery = () => { - const queryClient = useQueryClient(); - - return useCallback(() => { - queryClient.invalidateQueries([FIND_GAPS_FOR_RULE], { - refetchType: 'active', - }); - }, [queryClient]); -}; - -export const useFindGapsForRule = ( - { - ruleId, - page, - perPage, - start, - end, - statuses, - sortField, - sortOrder, - }: { - ruleId: string; - page: number; - perPage: number; - start: string; - end: string; - statuses: GapStatus[]; - sortField: keyof Gap; - sortOrder: string; - }, - options?: UseQueryOptions -) => { - return useQuery( - [FIND_GAPS_FOR_RULE, ruleId, page, perPage, statuses?.join(','), sortField, sortOrder], - async ({ signal }) => { - const response = await findGapsForRule({ - signal, - ruleId, - page, - perPage, - start, - end, - statuses, - sortField, - sortOrder, - }); - - return response; - }, - { - retry: 0, - keepPreviousData: true, - ...options, - } - ); -}; - -export const FILL_GAP_BY_ID_MUTATION_KEY = ['POST', INTERNAL_ALERTING_GAPS_FILL_BY_ID_API_PATH]; - -interface FillGapQuery { - ruleId: string; - gapId: string; -} -export const useFillGapMutation = ( - options?: UseMutationOptions, FillGapQuery> -) => { - const invalidateFindGapsQuery = useInvalidateFindGapsQuery(); - return useMutation((fillGapsOptions: FillGapQuery) => fillGapByIdForRule(fillGapsOptions), { - ...options, - onSettled: (...args) => { - invalidateFindGapsQuery(); - if (options?.onSettled) { - options.onSettled(...args); - } - }, - mutationKey: FILL_GAP_BY_ID_MUTATION_KEY, - }); -}; - -const FillGap = ({ ruleId, gap }: { ruleId: string; gap: Gap }) => { - const { addSuccess, addError } = useAppToasts(); - const fillGapMutation = useFillGapMutation({ - onSuccess: () => { - addSuccess(i18n.GAP_FILL_REQUEST_SUCCESS_MESSAGE, { - toastMessage: i18n.GAP_FILL_REQUEST_SUCCESS_MESSAGE_TOOLTIP, - }); - }, - onError: (error) => { - addError(error, { - title: i18n.GAP_FILL_REQUEST_ERROR_MESSAGE, - toastMessage: error?.body?.message ?? error.message, - }); - }, - }); - - if (gap.status === 'filled' || gap.unfilled_intervals.length === 0) { - return null; - } - - const title = - gap.in_progress_intervals.length > 0 || gap.filled_intervals.length > 0 - ? i18n.GAPS_TABLE_FILL_REMAINING_GAP_BUTTON_LABEL - : i18n.GAPS_TABLE_FILL_GAP_BUTTON_LABEL; - - return ( - <> - - fillGapMutation.mutate({ - ruleId, - gapId: gap._id, - }) - } - > - {title} - - - ); -}; - -const getGapsTableColumns = (hasCRUDPermissions: boolean, ruleId: string) => { - const fillActions = { - name: i18n.GAPS_TABLE_ACTIONS_LABEL, - align: 'right' as const, - render: (gap: Gap) => , - width: '15%', - }; - - const columns: Array> = [ - { - field: 'status', - sortable: true, - name: , - render: (value: GapStatus) => getStatusLabel(value), - width: '10%', - }, - { - field: '@timestamp', - sortable: true, - name: , - render: (value: Gap['@timestamp']) => ( - - ), - width: '15%', - }, - { - field: 'in_progress_intervals', - name: ( - - ), - render: (value: Gap['in_progress_intervals']) => { - if (!value || !value.length) return null; - return {i18n.GAPS_TABLE_IN_PROGRESS_LABEL}; - }, - width: '10%', - }, - { - width: '10%', - align: 'right', - name: ( - - ), - render: (item: Gap) => { - if (!item) return null; - const value = Math.ceil((item.filled_duration_ms * 100) / item.total_gap_duration_ms); - return ( - - - -

- {value} - {'%'} -

- - - - - - - ); - }, - }, - { - field: 'range', - name: , - render: (value: Gap['range']) => ( - <> - - {' - '} - - - ), - width: '40%', - }, - { - field: 'total_gap_duration_ms', - sortable: true, - name: ( - - ), - render: (value: Gap['total_gap_duration_ms']) => <>{getFormattedDuration(value)}, - width: '10%', - }, - ]; - - if (hasCRUDPermissions) { - columns.push(fillActions); - } - - return columns; -}; - -const DEFAULT_PAGE_SIZE = 10; - -export const RuleGaps = ({ ruleId }: { ruleId: string }) => { - const [pageIndex, setPageIndex] = useState(0); - const [pageSize, setPageSize] = useState(DEFAULT_PAGE_SIZE); - const [dateRange, setDateRange] = useState<{ start: string; end: string }>({ - start: 'now-24h', - end: 'now', - }); - const [{ canUserCRUD }] = useUserData(); - const { timelines } = useKibana().services; - const hasCRUDPermissions = hasUserCRUDPermission(canUserCRUD); - const [refreshInterval, setRefreshInterval] = useState(1000); - const [isPaused, setIsPaused] = useState(true); - const [selectedStatuses, setSelectedStatuses] = useState([]); - const [sort, setSort] = useState<{ field: keyof Gap; direction: 'desc' | 'asc' }>({ - field: '@timestamp', - direction: 'desc', - }); - - const { data, isLoading, isError, isFetching, refetch, dataUpdatedAt } = useFindGapsForRule({ - ruleId, - page: pageIndex + 1, - perPage: pageSize, - start: dateRange.start, - end: dateRange.end, - statuses: selectedStatuses, - sortField: sort.field, - sortOrder: sort.direction, - }); - - const pagination = { - pageIndex, - pageSize, - totalItemCount: data?.total ?? 0, - }; - - const columns = getGapsTableColumns(hasCRUDPermissions, ruleId); - - const onRefreshCallback = () => { - refetch(); - }; - - const handleTableChange: (params: CriteriaWithPagination) => void = ({ - page, - sort: newSort, - }) => { - if (page) { - setPageIndex(page.index); - setPageSize(page.size); - } - if (newSort) { - setSort(newSort); - } - }; - - const onTimeChangeCallback = useCallback( - (props: OnTimeChangeProps) => { - setDateRange({ start: props.start, end: props.end }); - }, - [setDateRange] - ); - - const onRefreshChangeCallback = useCallback( - (props: OnRefreshChangeProps) => { - setIsPaused(props.isPaused); - // Only support auto-refresh >= 5s -- no current ability to limit within component - setRefreshInterval(props.refreshInterval > 5000 ? props.refreshInterval : 5000); - }, - [setIsPaused, setRefreshInterval] - ); - - const handleStatusChange = useCallback( - (statuses: GapStatus[]) => { - setSelectedStatuses(statuses); - }, - [setSelectedStatuses] - ); - - return ( - - - - - - - - - - - - - - - - - - - - - - - - - {timelines.getLastUpdated({ - showUpdating: isLoading, - updatedAt: dataUpdatedAt, - })} - - - - - ); -}; diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_gaps/components/rule_gaps/status_filter.tsx b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_gaps/components/rule_gaps/status_filter.tsx deleted file mode 100644 index ebe90a9a5ace0..0000000000000 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_gaps/components/rule_gaps/status_filter.tsx +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import React, { useCallback } from 'react'; -import { EuiFilterGroup } from '@elastic/eui'; -import { MultiselectFilter } from '../../../../common/components/multiselect_filter'; -import * as i18n from './translations'; -import type { GapStatus } from '../../types'; -import { getStatusLabel } from './utils'; - -interface GapStatusFilterComponent { - selectedItems: GapStatus[]; - onChange: (selectedItems: GapStatusItem[]) => void; -} - -const items: GapStatus[] = ['partially_filled', 'unfilled', 'filled']; - -const GapStatusFilterComponent: React.FC = ({ - selectedItems, - onChange, -}) => { - const renderItem = useCallback((status: GapStatus) => { - return getStatusLabel(status); - }, []); - - const handleSelectionChange = useCallback( - (statuses: GapStatus[]) => { - console.log('statuses', statuses); - onChange(statuses); - }, - [onChange] - ); - - return ( - - - data-test-subj="GapStatusTypeFilter" - title={i18n.GAP_STATUS_FILTER_TITLE} - items={items} - selectedItems={selectedItems} - onSelectionChange={handleSelectionChange} - renderItem={renderItem} - width={200} - /> - - ); -}; - -export const GapStatusFilter = React.memo(GapStatusFilterComponent); -GapStatusFilter.displayName = 'GapStatusFilter'; diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_gaps/components/rule_gaps/translations.tsx b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_gaps/components/rule_gaps/translations.tsx deleted file mode 100644 index 5e14d78ab8874..0000000000000 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_gaps/components/rule_gaps/translations.tsx +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import { i18n } from '@kbn/i18n'; - -export const GAPS_TABLE_STATUS_LABEL = i18n.translate( - 'xpack.securitySolution.gapsTable.statusLabel', - { - defaultMessage: 'Status', - } -); - -export const GAPS_TABLE_ACTIONS_LABEL = i18n.translate( - 'xpack.securitySolution.gapsTable.actionsLabel', - { - defaultMessage: 'Actions', - } -); - -export const GAP_STATUS_UNFILLED = i18n.translate( - 'xpack.securitySolution.gapsTable.gapStatus.unfilled', - { - defaultMessage: 'Unfilled', - } -); - -export const GAP_STATUS_PARTIALLY_FILLED = i18n.translate( - 'xpack.securitySolution.gapsTable.gapStatus.partiallyFilled', - { - defaultMessage: 'Partially filled', - } -); - -export const GAP_STATUS_FILLED = i18n.translate( - 'xpack.securitySolution.gapsTable.gapStatus.filled', - { - defaultMessage: 'Filled', - } -); - -export const GAPS_TABLE_MANUAL_FILL_TASKS_LABEL = i18n.translate( - 'xpack.securitySolution.gapsTable.manualFillTasksLabel', - { - defaultMessage: 'Manual fill tasks', - } -); - -export const GAPS_TABLE_IN_PROGRESS_LABEL = i18n.translate( - 'xpack.securitySolution.gapsTable.inProgressIntervalsLabel', - { - defaultMessage: 'In progress', - } -); - -export const GAPS_TABLE_EVENT_TIME_COVERED_LABEL = i18n.translate( - 'xpack.securitySolution.gapsTable.eventTimeCoveredLabel', - { - defaultMessage: 'Event time covered', - } -); - -export const GAPS_TABLE_GAP_RANGE_LABEL = i18n.translate( - 'xpack.securitySolution.gapsTable.gapRangeLabel', - { - defaultMessage: 'Range', - } -); - -export const GAPS_TABLE_GAP_DURATION_TOOLTIP = i18n.translate( - 'xpack.securitySolution.gapsTable.gapDurationTooltip', - { - defaultMessage: 'Total gap duration', - } -); - -export const GAPS_TABLE_FILL_GAP_BUTTON_LABEL = i18n.translate( - 'xpack.securitySolution.gapsTable.fillGapButtonLabel', - { - defaultMessage: 'Fill gap', - } -); - -export const GAPS_TABLE_FILL_REMAINING_GAP_BUTTON_LABEL = i18n.translate( - 'xpack.securitySolution.gapsTable.fillRemainingGapButtonLabel', - { - defaultMessage: 'Fill remaining gap', - } -); - -export const GAP_FILL_REQUEST_SUCCESS_MESSAGE = i18n.translate( - 'xpack.securitySolution.gapsTable.gapFillRequestSuccessMessage', - { - defaultMessage: 'Manual run requested', - } -); - -export const GAP_FILL_REQUEST_SUCCESS_MESSAGE_TOOLTIP = i18n.translate( - 'xpack.securitySolution.gapsTable.gapFillRequestSuccessMessageTooltip', - { - defaultMessage: 'Check status in rule execution logs. Actions for this execution will be run.', - } -); - -export const GAP_FILL_REQUEST_ERROR_MESSAGE = i18n.translate( - 'xpack.securitySolution.gapsTable.gapFillRequestErrorMessage', - { - defaultMessage: 'Failed to request manual run', - } -); - -export const GAP_STATUS_FILTER_TITLE = i18n.translate( - 'xpack.securitySolution.gapsTable.gapStatusFilterTitle', - { - defaultMessage: 'Status', - } -); - -export const GAPS_TABLE_EVENT_TIME_LABEL = i18n.translate( - 'xpack.securitySolution.gapsTable.eventTimeLabel', - { - defaultMessage: 'Detected at', - } -); diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_gaps/components/rule_gaps/utils.ts b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_gaps/components/rule_gaps/utils.ts deleted file mode 100644 index 71ada001dfa33..0000000000000 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_gaps/components/rule_gaps/utils.ts +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { GapStatus } from '../../types'; -import * as i18n from './translations'; - -export const getStatusLabel = (status: GapStatus) => { - switch (status) { - case 'partially_filled': - return i18n.GAP_STATUS_PARTIALLY_FILLED; - case 'unfilled': - return i18n.GAP_STATUS_UNFILLED; - case 'filled': - return i18n.GAP_STATUS_FILLED; - } - return ''; -}; diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_gaps/components/rules_with_gaps_overview_panel/index.tsx b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_gaps/components/rules_with_gaps_overview_panel/index.tsx deleted file mode 100644 index 7540d1451e645..0000000000000 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_gaps/components/rules_with_gaps_overview_panel/index.tsx +++ /dev/null @@ -1,237 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { GetRulesWithGapResponseBody } from '@kbn/alerting-plugin/common/routes/gaps/apis/get_rules_with_gaps'; -import type { UseQueryOptions } from '@tanstack/react-query'; -import { useQuery } from '@tanstack/react-query'; -import { INTERNAL_ALERTING_GAPS_GET_RULES_API_PATH } from '@kbn/alerting-plugin/common'; -import React, { useEffect, useState } from 'react'; - -import { - EuiButton, - EuiFlexGroup, - EuiFlexItem, - EuiPanel, - EuiText, - EuiContextMenuPanel, - EuiPopover, - EuiContextMenuItem, - EuiBadge, - EuiFilterButton, - EuiFilterGroup, -} from '@elastic/eui'; - -import { KibanaServices } from '../../../../common/lib/kibana'; -import { useRulesTableContext } from '../../../rule_management_ui/components/rules_table/rules_table/rules_table_context'; -import * as i18n from './translations'; - -/** - * Find gaps for the given rule ID - * @param ruleIds string[] - * @param signal? AbortSignal - * @returns - */ -export const getRulesWithGaps = async ({ - signal, - start, - end, -}: { - start: string; - end: string; - signal?: AbortSignal; -}): Promise => - KibanaServices.get().http.fetch( - INTERNAL_ALERTING_GAPS_GET_RULES_API_PATH, - { - method: 'GET', - query: { - start, - end, - }, - signal, - } - ); - -const GET_RULES_WITH_GAPS = ['GET_RULES_WITH_GAPS']; -export const useGetRulesWithGaps = ( - { - start, - end, - }: { - start: string; - end: string; - }, - options?: UseQueryOptions -) => { - return useQuery( - [GET_RULES_WITH_GAPS, start, end], - async ({ signal }) => { - const response = await getRulesWithGaps({ signal, start, end }); - - return response; - }, - { - retry: 0, - keepPreviousData: true, - ...options, - } - ); -}; - -enum RangeValue { - LAST_24_H = 'last_24_h', - LAST_3_D = 'last_3_d', - LAST_7_D = 'last_7_d', -} - -const defaultRangeValue = RangeValue.LAST_24_H; -export const RulesWithGapsOverviewPanel = () => { - const [rangeValue, setRangeValue] = useState(defaultRangeValue); - const [showRulesWithGaps, setShowRulesWithGaps] = useState(false); - const [gapsInterval, setGapsInterval] = useState<{ start: string; end: string } | null>(null); - const { data } = useGetRulesWithGaps( - { - start: gapsInterval?.start ?? '', - end: gapsInterval?.end ?? '', - }, - { - enabled: Boolean(gapsInterval), - } - ); - const { - actions: { setFilterOptions }, - } = useRulesTableContext(); - const [isPopoverOpen, setPopover] = useState(false); - - const rangeValueToLabel = { - [RangeValue.LAST_24_H]: i18n.RULE_GAPS_OVERVIEW_PANEL_LAST_24_HOURS_LABEL, - [RangeValue.LAST_3_D]: i18n.RULE_GAPS_OVERVIEW_PANEL_LAST_3_DAYS_LABEL, - [RangeValue.LAST_7_D]: i18n.RULE_GAPS_OVERVIEW_PANEL_LAST_7_DAYS_LABEL, - }; - - useEffect(() => { - if (rangeValue) { - const now = new Date(); - const dayMs = 24 * 60 * 60 * 1000; - let amountOfDays = 1; - switch (rangeValue) { - case RangeValue.LAST_24_H: - amountOfDays = 1; - break; - case RangeValue.LAST_3_D: - amountOfDays = 3; - break; - case RangeValue.LAST_7_D: - amountOfDays = 7; - break; - } - - const start = new Date(now.getTime() - amountOfDays * dayMs).toISOString(); - const end = now.toISOString(); - setGapsInterval({ start, end }); - } - }, [rangeValue]); - - const onButtonClick = () => { - setPopover(!isPopoverOpen); - }; - - const closePopover = () => { - setPopover(false); - }; - - const items = Object.values(RangeValue).map((value) => ({ - value, - label: rangeValueToLabel[value], - })); - - const button = ( - - {rangeValueToLabel[rangeValue]} - - ); - - const handleShowRulesWithGapsFilterButtonClick = (value: boolean) => { - setShowRulesWithGaps(value); - if (!data) return; - if (value) { - setFilterOptions({ - ruleIds: data.ruleIds, - }); - } else { - setFilterOptions({ - ruleIds: [], - }); - } - }; - - return ( - - - - - ( - { - setRangeValue(item.value); - closePopover(); - }} - > - {item.label} - - ))} - /> - - - - - - - {i18n.RULE_GAPS_OVERVIEW_PANEL_LABEL} - - - - {data?.total} - - - - - - handleShowRulesWithGapsFilterButtonClick(false)} - > - {i18n.RULE_GAPS_OVERVIEW_PANEL_SHOW_ALL_RULES_LABEL} - - handleShowRulesWithGapsFilterButtonClick(true)} - > - {i18n.RULE_GAPS_OVERVIEW_PANEL_SHOW_RULES_WITH_GAPS_LABEL} - - - - - - ); -}; diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_gaps/components/rules_with_gaps_overview_panel/translations.tsx b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_gaps/components/rules_with_gaps_overview_panel/translations.tsx deleted file mode 100644 index fe9963eace3d1..0000000000000 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_gaps/components/rules_with_gaps_overview_panel/translations.tsx +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { i18n } from '@kbn/i18n'; - -export const RULE_GAPS_OVERVIEW_PANEL_LABEL = i18n.translate( - 'xpack.securitySolution.ruleGapsOverviewPanel.label', - { - defaultMessage: 'Total rules with gaps:', - } -); - -export const RULE_GAPS_OVERVIEW_PANEL_SHOW_ALL_RULES_LABEL = i18n.translate( - 'xpack.securitySolution.ruleGapsOverviewPanel.showAllRulesLabel', - { - defaultMessage: 'Show all rules', - } -); - -export const RULE_GAPS_OVERVIEW_PANEL_SHOW_RULES_WITH_GAPS_LABEL = i18n.translate( - 'xpack.securitySolution.ruleGapsOverviewPanel.showRulesWithGapsLabel', - { - defaultMessage: 'Show rules with gaps', - } -); - -export const RULE_GAPS_OVERVIEW_PANEL_LAST_24_HOURS_LABEL = i18n.translate( - 'xpack.securitySolution.ruleGapsOverviewPanel.last24HoursLabel', - { - defaultMessage: 'Last 24 hours', - } -); - -export const RULE_GAPS_OVERVIEW_PANEL_LAST_3_DAYS_LABEL = i18n.translate( - 'xpack.securitySolution.ruleGapsOverviewPanel.last3DaysLabel', - { - defaultMessage: 'Last 3 days', - } -); - -export const RULE_GAPS_OVERVIEW_PANEL_LAST_7_DAYS_LABEL = i18n.translate( - 'xpack.securitySolution.ruleGapsOverviewPanel.last7DaysLabel', - { - defaultMessage: 'Last 7 days', - } -); diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_tables.tsx b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_tables.tsx index 3fbaf1a6d4709..fda4ab21b119c 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_tables.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_tables.tsx @@ -44,7 +44,6 @@ import { useIsUpgradingSecurityPackages } from '../../../rule_management/logic/u import { useManualRuleRunConfirmation } from '../../../rule_gaps/components/manual_rule_run/use_manual_rule_run_confirmation'; import { ManualRuleRunModal } from '../../../rule_gaps/components/manual_rule_run'; import { BulkManualRuleRunLimitErrorModal } from './bulk_actions/bulk_manual_rule_run_limit_error_modal'; -import { RulesWithGapsOverviewPanel } from '../../../rule_gaps/components/rules_with_gaps_overview_panel'; const INITIAL_SORT_FIELD = 'enabled'; @@ -320,13 +319,6 @@ export const RulesTables = React.memo(({ selectedTab }) => { )} {shouldShowRulesTable && ( <> - {selectedTab === AllRulesTabs.monitoring && ( - <> - - - - - )} Date: Fri, 10 Jan 2025 09:34:04 +0100 Subject: [PATCH 12/17] remove more --- .../feature_privilege_builder/alerting.ts | 1 - .../plugins/shared/alerting/common/index.ts | 6 -- .../get_rules_with_gaps.ts | 93 ------------------- .../rule/methods/get_rules_with_gaps/index.ts | 8 -- .../schemas/get_rules_with_gaps.ts | 19 ---- .../get_rules_with_gaps/schemas/index.ts | 8 -- .../types/get_rule_params.ts | 12 --- .../get_rules_with_gaps/types/index.ts | 8 -- ...ransform_rule_attributes_to_rule_domain.ts | 4 +- .../alerting/server/authorization/types.ts | 1 - .../shared/alerting/server/lib/monitoring.ts | 3 +- .../pages/rule_details/index.tsx | 1 - .../detection_engine/rule_gaps/types.ts | 5 - .../rule_management/logic/types.ts | 1 - .../rules_table/rules_table_context.tsx | 2 - .../components/rules_table/rules_tables.tsx | 8 +- .../rule_types/utils/utils.ts | 2 +- 17 files changed, 4 insertions(+), 178 deletions(-) delete mode 100644 x-pack/platform/plugins/shared/alerting/server/application/rule/methods/get_rules_with_gaps/get_rules_with_gaps.ts delete mode 100644 x-pack/platform/plugins/shared/alerting/server/application/rule/methods/get_rules_with_gaps/index.ts delete mode 100644 x-pack/platform/plugins/shared/alerting/server/application/rule/methods/get_rules_with_gaps/schemas/get_rules_with_gaps.ts delete mode 100644 x-pack/platform/plugins/shared/alerting/server/application/rule/methods/get_rules_with_gaps/schemas/index.ts delete mode 100644 x-pack/platform/plugins/shared/alerting/server/application/rule/methods/get_rules_with_gaps/types/get_rule_params.ts delete mode 100644 x-pack/platform/plugins/shared/alerting/server/application/rule/methods/get_rules_with_gaps/types/index.ts diff --git a/x-pack/platform/packages/private/security/authorization_core/src/privileges/feature_privilege_builder/alerting.ts b/x-pack/platform/packages/private/security/authorization_core/src/privileges/feature_privilege_builder/alerting.ts index cc94f400ddab1..ffb3be1686bb2 100644 --- a/x-pack/platform/packages/private/security/authorization_core/src/privileges/feature_privilege_builder/alerting.ts +++ b/x-pack/platform/packages/private/security/authorization_core/src/privileges/feature_privilege_builder/alerting.ts @@ -54,7 +54,6 @@ const writeOperations: Record = { 'runSoon', 'scheduleBackfill', 'deleteBackfill', - 'fillGaps', ], alert: ['update'], }; diff --git a/x-pack/platform/plugins/shared/alerting/common/index.ts b/x-pack/platform/plugins/shared/alerting/common/index.ts index 069f59fd988a5..82c6eca266337 100644 --- a/x-pack/platform/plugins/shared/alerting/common/index.ts +++ b/x-pack/platform/plugins/shared/alerting/common/index.ts @@ -98,12 +98,6 @@ export const INTERNAL_ALERTING_GAPS_API_PATH = export const INTERNAL_ALERTING_GAPS_FIND_API_PATH = `${INTERNAL_ALERTING_GAPS_API_PATH}/_find` as const; -export const INTERNAL_ALERTING_GAPS_GET_RULES_API_PATH = - `${INTERNAL_ALERTING_GAPS_API_PATH}/_get_rules` as const; - -export const INTERNAL_ALERTING_GAPS_FILL_BY_ID_API_PATH = - `${INTERNAL_ALERTING_GAPS_API_PATH}/_fill_by_id` as const; - export const ALERTING_FEATURE_ID = 'alerts'; export const MONITORING_HISTORY_LIMIT = 200; export const ENABLE_MAINTENANCE_WINDOWS = true; diff --git a/x-pack/platform/plugins/shared/alerting/server/application/rule/methods/get_rules_with_gaps/get_rules_with_gaps.ts b/x-pack/platform/plugins/shared/alerting/server/application/rule/methods/get_rules_with_gaps/get_rules_with_gaps.ts deleted file mode 100644 index f6516470b6513..0000000000000 --- a/x-pack/platform/plugins/shared/alerting/server/application/rule/methods/get_rules_with_gaps/get_rules_with_gaps.ts +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import Boom from '@hapi/boom'; -import { KueryNode } from '@kbn/es-query'; -import { - AlertingAuthorizationEntity, - AlertingAuthorizationFilterType, -} from '../../../../authorization'; -import { RulesClientContext } from '../../../../rules_client'; -import { GetRulesWithGapsParams, GetRulesWithGapsResponse } from './types'; - -export const RULE_SAVED_OBJECT_TYPE = 'alert'; - -export async function getRulesWithGaps( - context: RulesClientContext, - params: GetRulesWithGapsParams -) { - try { - let authorizationTuple; - try { - authorizationTuple = await context.authorization.getFindAuthorizationFilter( - AlertingAuthorizationEntity.Alert, - { - type: AlertingAuthorizationFilterType.KQL, - fieldNames: { - ruleTypeId: 'kibana.alert.rule.rule_type_id', - consumer: 'kibana.alert.rule.consumer', - }, - } - ); - } catch (error) { - // context.auditLogger?.log( - // ruleAuditEvent({ - // action: RuleAuditAction.GET_GLOBAL_EXECUTION_KPI, - // error, - // }) - // ); - throw error; - } - - const { start, end, statuses } = params; - const eventLogClient = await context.getEventLogClient(); - - const filter = 'kibana.alert.rule.gap: *'; - const statusFilter = statuses - ?.map((status) => `kibana.alert.rule.gap.status:${status}`) - .join(' OR '); - - const aggs = await eventLogClient.aggregateEventsWithAuthFilter( - RULE_SAVED_OBJECT_TYPE, - authorizationTuple.filter as KueryNode, - { - start, - end, - filter: `${filter} ${statusFilter ? `AND (${statusFilter})` : ''}`, - aggs: { - unique_rule_ids: { - terms: { - field: 'rule.id', - size: 10000, - }, - }, - }, - } - ); - - interface UniqueRuleIdsAgg { - buckets: Array<{ key: string }>; - } - - const uniqueRuleIdsAgg = aggs.aggregations?.unique_rule_ids as UniqueRuleIdsAgg; - - const resultBuckets = uniqueRuleIdsAgg?.buckets ?? []; - - const ruleIds = resultBuckets.map((bucket) => bucket.key) ?? []; - - const result: GetRulesWithGapsResponse = { - total: ruleIds?.length, - ruleIds, - }; - - return result; - } catch (err) { - const errorMessage = `Failed to find rules with gaps`; - context.logger.error(`${errorMessage} - ${err}`); - throw Boom.boomify(err, { message: errorMessage }); - } -} diff --git a/x-pack/platform/plugins/shared/alerting/server/application/rule/methods/get_rules_with_gaps/index.ts b/x-pack/platform/plugins/shared/alerting/server/application/rule/methods/get_rules_with_gaps/index.ts deleted file mode 100644 index 6130adad1b7a8..0000000000000 --- a/x-pack/platform/plugins/shared/alerting/server/application/rule/methods/get_rules_with_gaps/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export { getRulesWithGaps } from './get_rules_with_gaps'; diff --git a/x-pack/platform/plugins/shared/alerting/server/application/rule/methods/get_rules_with_gaps/schemas/get_rules_with_gaps.ts b/x-pack/platform/plugins/shared/alerting/server/application/rule/methods/get_rules_with_gaps/schemas/get_rules_with_gaps.ts deleted file mode 100644 index 995826f6b11e4..0000000000000 --- a/x-pack/platform/plugins/shared/alerting/server/application/rule/methods/get_rules_with_gaps/schemas/get_rules_with_gaps.ts +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { schema } from '@kbn/config-schema'; - -export const getRulesWithGapsParamsSchema = schema.object({ - start: schema.string(), - end: schema.string(), - statuses: schema.maybe(schema.arrayOf(schema.string())), -}); - -export const getRulesWithGapsResponseSchema = schema.object({ - total: schema.number(), - ruleIds: schema.arrayOf(schema.string()), -}); diff --git a/x-pack/platform/plugins/shared/alerting/server/application/rule/methods/get_rules_with_gaps/schemas/index.ts b/x-pack/platform/plugins/shared/alerting/server/application/rule/methods/get_rules_with_gaps/schemas/index.ts deleted file mode 100644 index 2b3b33f812c16..0000000000000 --- a/x-pack/platform/plugins/shared/alerting/server/application/rule/methods/get_rules_with_gaps/schemas/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export * from './get_rules_with_gaps'; diff --git a/x-pack/platform/plugins/shared/alerting/server/application/rule/methods/get_rules_with_gaps/types/get_rule_params.ts b/x-pack/platform/plugins/shared/alerting/server/application/rule/methods/get_rules_with_gaps/types/get_rule_params.ts deleted file mode 100644 index e801681e19764..0000000000000 --- a/x-pack/platform/plugins/shared/alerting/server/application/rule/methods/get_rules_with_gaps/types/get_rule_params.ts +++ /dev/null @@ -1,12 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { TypeOf } from '@kbn/config-schema'; -import { getRulesWithGapsParamsSchema, getRulesWithGapsResponseSchema } from '../schemas'; - -export type GetRulesWithGapsParams = TypeOf; -export type GetRulesWithGapsResponse = TypeOf; diff --git a/x-pack/platform/plugins/shared/alerting/server/application/rule/methods/get_rules_with_gaps/types/index.ts b/x-pack/platform/plugins/shared/alerting/server/application/rule/methods/get_rules_with_gaps/types/index.ts deleted file mode 100644 index 553499a994ff6..0000000000000 --- a/x-pack/platform/plugins/shared/alerting/server/application/rule/methods/get_rules_with_gaps/types/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export * from './get_rule_params'; diff --git a/x-pack/platform/plugins/shared/alerting/server/application/rule/transforms/transform_rule_attributes_to_rule_domain.ts b/x-pack/platform/plugins/shared/alerting/server/application/rule/transforms/transform_rule_attributes_to_rule_domain.ts index 3b4bc7d07900f..3d2f25691b086 100644 --- a/x-pack/platform/plugins/shared/alerting/server/application/rule/transforms/transform_rule_attributes_to_rule_domain.ts +++ b/x-pack/platform/plugins/shared/alerting/server/application/rule/transforms/transform_rule_attributes_to_rule_domain.ts @@ -26,9 +26,7 @@ const INITIAL_LAST_RUN_METRICS = { total_alerts_detected: null, total_alerts_created: null, gap_duration_s: null, - // TODO: should initialised field, after inermidiate release - // gap_range: null, - // unfilled_gaps_ms: null, + gap_range: null, }; const transformEsExecutionStatus = ( diff --git a/x-pack/platform/plugins/shared/alerting/server/authorization/types.ts b/x-pack/platform/plugins/shared/alerting/server/authorization/types.ts index 8668884463667..5950ed3f259ba 100644 --- a/x-pack/platform/plugins/shared/alerting/server/authorization/types.ts +++ b/x-pack/platform/plugins/shared/alerting/server/authorization/types.ts @@ -44,5 +44,4 @@ export enum WriteOperations { RunSoon = 'runSoon', ScheduleBackfill = 'scheduleBackfill', DeleteBackfill = 'deleteBackfill', - FillGaps = 'fillGaps', } diff --git a/x-pack/platform/plugins/shared/alerting/server/lib/monitoring.ts b/x-pack/platform/plugins/shared/alerting/server/lib/monitoring.ts index b25592b88bdb0..cd0b300f8f8eb 100644 --- a/x-pack/platform/plugins/shared/alerting/server/lib/monitoring.ts +++ b/x-pack/platform/plugins/shared/alerting/server/lib/monitoring.ts @@ -21,8 +21,7 @@ const INITIAL_LAST_RUN_METRICS: RuleMonitoringLastRunMetrics = { total_alerts_detected: null, total_alerts_created: null, gap_duration_s: null, - // TODO: should initialised field, after inermidiate release - // gap_range: null, + gap_range: null, }; export const getDefaultMonitoring = (timestamp: string): RawRuleMonitoring => { diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_details_ui/pages/rule_details/index.tsx b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_details_ui/pages/rule_details/index.tsx index e25e56692e40a..851d219ad43d3 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_details_ui/pages/rule_details/index.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_details_ui/pages/rule_details/index.tsx @@ -114,7 +114,6 @@ import { import { ExecutionEventsTable } from '../../../rule_monitoring'; import { ExecutionLogTable } from './execution_log_table/execution_log_table'; import { RuleBackfillsInfo } from '../../../rule_gaps/components/rule_backfills_info'; -import { RuleGaps } from '../../../rule_gaps/components/rule_gaps'; import * as ruleI18n from '../../../../detections/pages/detection_engine/rules/translations'; diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_gaps/types.ts b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_gaps/types.ts index 223c6ad97aac3..03b9ed4a4bea0 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_gaps/types.ts +++ b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_gaps/types.ts @@ -5,17 +5,12 @@ * 2.0. */ -import type { FindGapsResponseBody } from '@kbn/alerting-plugin/common/routes/gaps/apis/find'; - import type { FindBackfillResponseBody } from '@kbn/alerting-plugin/common/routes/backfill/apis/find'; export type Backfill = FindBackfillResponseBody['data']['0']; export type BackfillStatus = Backfill['status']; -export type Gap = FindGapsResponseBody['data']['0']; -export type GapStatus = Gap['status']; - export interface BackfillStats { total: number; complete: number; diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management/logic/types.ts b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management/logic/types.ts index 620ea5c30d5df..59ac52d592bcd 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management/logic/types.ts +++ b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management/logic/types.ts @@ -100,7 +100,6 @@ export interface FilterOptions { enabled?: boolean; // undefined is to display all the rules ruleExecutionStatus?: RuleExecutionStatus; // undefined means "all" ruleSource?: RuleCustomizationEnum[]; // undefined is to display all the rules - ruleIds?: string[]; } export interface FetchRulesResponse { diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_table/rules_table_context.tsx b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_table/rules_table_context.tsx index 143991d33bbad..8f0cac0130083 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_table/rules_table_context.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_table/rules_table_context.tsx @@ -209,7 +209,6 @@ export const RulesTableContextProvider = ({ children }: RulesTableContextProvide enabled: savedFilter?.enabled, ruleExecutionStatus: savedFilter?.ruleExecutionStatus ?? DEFAULT_FILTER_OPTIONS.ruleExecutionStatus, - ruleIds: [], }); const [sortingOptions, setSortingOptions] = useState({ @@ -281,7 +280,6 @@ export const RulesTableContextProvider = ({ children }: RulesTableContextProvide } }, [selectedRuleIds, isRefreshOn]); - console.log('filterOptions', filterOptions); // Fetch rules const { data: { rules, total } = { rules: [], total: 0 }, diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_tables.tsx b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_tables.tsx index fda4ab21b119c..3a9c5dfb8ee00 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_tables.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/rules_tables.tsx @@ -5,13 +5,7 @@ * 2.0. */ -import { - EuiBasicTable, - EuiConfirmModal, - EuiEmptyPrompt, - EuiProgress, - EuiSpacer, -} from '@elastic/eui'; +import { EuiBasicTable, EuiConfirmModal, EuiEmptyPrompt, EuiProgress } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n-react'; import React, { useCallback, useMemo, useRef } from 'react'; import { Loader } from '../../../../common/components/loader'; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts index efcf16d7b4913..d18020c7dbee9 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts @@ -93,7 +93,7 @@ import type { BuildReasonMessage } from './reason_formatters'; import { getSuppressionTerms } from './suppression_utils'; import { robustGet } from './source_fields_merging/utils/robust_field_access'; -export const MAX_RULE_GAP_RATIO = 1; +export const MAX_RULE_GAP_RATIO = 4; export const hasReadIndexPrivileges = async (args: { privileges: Privilege; From 45e3118b26112ea6b57406350a2dffd962c19003 Mon Sep 17 00:00:00 2001 From: Nikita Khristinin Date: Fri, 10 Jan 2025 09:36:19 +0100 Subject: [PATCH 13/17] remove rule filtering --- .../detection_engine/rule_management/rule_filtering.ts | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/common/detection_engine/rule_management/rule_filtering.ts b/x-pack/solutions/security/plugins/security_solution/common/detection_engine/rule_management/rule_filtering.ts index eed3bc13818b6..bfc31df88ec97 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/detection_engine/rule_management/rule_filtering.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/detection_engine/rule_management/rule_filtering.ts @@ -32,7 +32,6 @@ interface RulesFilterOptions { tags: string[]; excludeRuleTypes: Type[]; ruleExecutionStatus: RuleExecutionStatus; - ruleIds: string[]; } /** @@ -50,7 +49,6 @@ export function convertRulesFilterToKQL({ tags, excludeRuleTypes = [], ruleExecutionStatus, - ruleIds, }: Partial): string { const kql: string[] = []; @@ -86,10 +84,6 @@ export function convertRulesFilterToKQL({ kql.push(`${LAST_RUN_OUTCOME_FIELD}: "failed"`); } - if (ruleIds?.length) { - kql.push(convertRuleIdsToKQL(ruleIds)); - } - return kql.join(' AND '); } @@ -118,7 +112,3 @@ export function convertRuleTagsToKQL(tags: string[]): string { export function convertRuleTypesToKQL(ruleTypes: Type[]): string { return `${PARAMS_TYPE_FIELD}: (${ruleTypes.map(prepareKQLStringParam).join(' OR ')})`; } - -export function convertRuleIdsToKQL(ruleIds: string[]): string { - return `(${ruleIds.map((ruleId) => `alert.id: ("alert:${ruleId}")`).join(' OR ')})`; -} From d2350f5409944b925b9d540b40e2eaa39414263a Mon Sep 17 00:00:00 2001 From: Nikita Khristinin Date: Fri, 10 Jan 2025 12:10:33 +0100 Subject: [PATCH 14/17] add some tests --- .../lib/retry_transient_es_errors.test.ts | 23 ++ .../methods/delete/delete_backfill.test.ts | 39 ++- .../methods/delete/delete_backfill.ts | 7 +- .../schedule/schedule_backfill.test.ts | 150 +++++----- .../rule/methods/find_gaps/find_gaps.test.ts | 150 ++++++++++ .../rule/methods/find_gaps/find_gaps.ts | 7 + .../backfill_client/backfill_client.test.ts | 262 +++++++++++++++++- .../shared/alerting/server/routes/index.ts | 6 +- .../alerting/server/rules_client.mock.ts | 2 - .../server/rules_client/rules_client.ts | 6 - 10 files changed, 559 insertions(+), 93 deletions(-) create mode 100644 x-pack/platform/plugins/shared/alerting/server/application/rule/methods/find_gaps/find_gaps.test.ts diff --git a/x-pack/platform/plugins/shared/alerting/server/alerts_service/lib/retry_transient_es_errors.test.ts b/x-pack/platform/plugins/shared/alerting/server/alerts_service/lib/retry_transient_es_errors.test.ts index 3a59b07556f9b..8ee707bc96b37 100644 --- a/x-pack/platform/plugins/shared/alerting/server/alerts_service/lib/retry_transient_es_errors.test.ts +++ b/x-pack/platform/plugins/shared/alerting/server/alerts_service/lib/retry_transient_es_errors.test.ts @@ -92,4 +92,27 @@ describe('retryTransientErrors', () => { await expect(retryTransientEsErrors(esCallMock, { logger })).rejects.toThrow(error); expect(esCallMock).toHaveBeenCalledTimes(1); }); + + it('retries with additional status codes when provided', async () => { + const customStatusCode = 409; // Conflict version + const error = new EsErrors.ResponseError({ + statusCode: customStatusCode, + meta: {} as any, + warnings: [], + body: 'Conflict Version', + }); + const esCallMock = jest.fn().mockRejectedValueOnce(error).mockResolvedValue('success'); + + expect( + await retryTransientEsErrors(esCallMock, { + logger, + additionalRetryableStatusCodes: [customStatusCode], + }) + ).toEqual('success'); + expect(esCallMock).toHaveBeenCalledTimes(2); + expect(logger.warn).toHaveBeenCalledTimes(1); + expect(logger.warn.mock.calls[0][0]).toMatch( + `Retrying Elasticsearch operation after [2s] due to error: ResponseError: Conflict Version` + ); + }); }); diff --git a/x-pack/platform/plugins/shared/alerting/server/application/backfill/methods/delete/delete_backfill.test.ts b/x-pack/platform/plugins/shared/alerting/server/application/backfill/methods/delete/delete_backfill.test.ts index b0879613d069f..e102d0eb0d6a5 100644 --- a/x-pack/platform/plugins/shared/alerting/server/application/backfill/methods/delete/delete_backfill.test.ts +++ b/x-pack/platform/plugins/shared/alerting/server/application/backfill/methods/delete/delete_backfill.test.ts @@ -28,6 +28,11 @@ import { AD_HOC_RUN_SAVED_OBJECT_TYPE } from '../../../../saved_objects'; import { SavedObject } from '@kbn/core-saved-objects-api-server'; import { AdHocRunSO } from '../../../../data/ad_hoc_run/types'; import { SavedObjectsErrorHelpers } from '@kbn/core/server'; +import { updateGaps } from '../../../../lib/rule_gaps/update/update_gaps'; + +jest.mock('../../../../lib/rule_gaps/update/update_gaps', () => ({ + updateGaps: jest.fn(), +})); const kibanaVersion = 'v8.0.0'; const taskManager = taskManagerMock.createStart(); @@ -159,7 +164,10 @@ describe('deleteBackfill()', () => { }); expect(unsecuredSavedObjectsClient.delete).toHaveBeenLastCalledWith( AD_HOC_RUN_SAVED_OBJECT_TYPE, - '1' + '1', + { + refresh: 'wait_for', + } ); expect(taskManager.removeIfExists).toHaveBeenCalledWith('1'); expect(logger.error).not.toHaveBeenCalled(); @@ -167,6 +175,35 @@ describe('deleteBackfill()', () => { expect(result).toEqual({}); }); + test('should call updateGaps with correct parameters when deleting backfill', async () => { + const mockEventLogClient = { mockEventLogClient: true } as any; + rulesClientParams.getEventLogClient.mockResolvedValue(mockEventLogClient); + + await rulesClient.deleteBackfill('1'); + + const updateGapsCall = (updateGaps as jest.Mock).mock.calls[0][0]; + expect(updateGapsCall.ruleId).toBe('abc'); + expect(updateGapsCall.start).toEqual(new Date('2023-10-19T15:07:40.011Z')); + expect(updateGapsCall.end).toBeInstanceOf(Date); + expect(updateGapsCall.backfillSchedule).toEqual([ + { + interval: '12h', + status: adHocRunStatus.PENDING, + runAt: '2023-10-20T03:07:40.011Z', + }, + { + interval: '12h', + status: adHocRunStatus.PENDING, + runAt: '2023-10-20T15:07:40.011Z', + }, + ]); + expect(updateGapsCall.savedObjectsRepository).toBe(internalSavedObjectsRepository); + expect(updateGapsCall.logger).toBe(logger); + expect(updateGapsCall.eventLogClient).toBe(mockEventLogClient); + expect(updateGapsCall.shouldRefetchAllBackfills).toBe(true); + expect(updateGapsCall.backfillClient).toBe(backfillClient); + }); + describe('error handling', () => { test('should retry if conflict error', async () => { unsecuredSavedObjectsClient.delete.mockImplementationOnce(() => { diff --git a/x-pack/platform/plugins/shared/alerting/server/application/backfill/methods/delete/delete_backfill.ts b/x-pack/platform/plugins/shared/alerting/server/application/backfill/methods/delete/delete_backfill.ts index 51b6b1de4c1f2..9a13609dcdd72 100644 --- a/x-pack/platform/plugins/shared/alerting/server/application/backfill/methods/delete/delete_backfill.ts +++ b/x-pack/platform/plugins/shared/alerting/server/application/backfill/methods/delete/delete_backfill.ts @@ -84,12 +84,7 @@ async function deleteWithOCC(context: RulesClientContext, { id }: { id: string } }) ); - const backfill = await context.unsecuredSavedObjectsClient.get( - AD_HOC_RUN_SAVED_OBJECT_TYPE, - id - ); - - const backfillResult = transformAdHocRunToBackfillResult(backfill); + const backfillResult = transformAdHocRunToBackfillResult(result); // delete the saved object const removeResult = await context.unsecuredSavedObjectsClient.delete( diff --git a/x-pack/platform/plugins/shared/alerting/server/application/backfill/methods/schedule/schedule_backfill.test.ts b/x-pack/platform/plugins/shared/alerting/server/application/backfill/methods/schedule/schedule_backfill.test.ts index a0a79421c7358..54f36fed3c27a 100644 --- a/x-pack/platform/plugins/shared/alerting/server/application/backfill/methods/schedule/schedule_backfill.test.ts +++ b/x-pack/platform/plugins/shared/alerting/server/application/backfill/methods/schedule/schedule_backfill.test.ts @@ -24,6 +24,8 @@ import { fromKueryExpression } from '@kbn/es-query'; import { auditLoggerMock } from '@kbn/security-plugin/server/audit/mocks'; import { asyncForEach } from '@kbn/std'; import { taskManagerMock } from '@kbn/task-manager-plugin/server/mocks'; +import { eventLoggerMock } from '@kbn/event-log-plugin/server/event_logger.mock'; +import { eventLogClientMock } from '@kbn/event-log-plugin/server/event_log_client.mock'; import { ConstructorOptions, RulesClient } from '../../../../rules_client'; import { ScheduleBackfillParam } from './types'; import { adHocRunStatus } from '../../../../../common/constants'; @@ -39,6 +41,8 @@ const actionsAuthorization = actionsAuthorizationMock.create(); const auditLogger = auditLoggerMock.create(); const internalSavedObjectsRepository = savedObjectsRepositoryMock.create(); const backfillClient = backfillClientMock.create(); +const eventLogger = eventLoggerMock.create(); +const eventLogClient = eventLogClientMock.create(); const filter = fromKueryExpression( '((alert.attributes.alertTypeId:myType and alert.attributes.consumer:myApp) or (alert.attributes.alertTypeId:myOtherType and alert.attributes.consumer:myApp) or (alert.attributes.alertTypeId:myOtherType and alert.attributes.consumer:myOtherApp))' @@ -71,6 +75,7 @@ const rulesClientParams: jest.Mocked = { isSystemAction: jest.fn(), connectorAdapterRegistry: new ConnectorAdapterRegistry(), uiSettings: uiSettingsServiceMock.createStartContract(), + eventLogger, }; const fakeRuleName = 'fakeRuleName'; @@ -249,6 +254,8 @@ describe('scheduleBackfill()', () => { test('should successfully schedule backfill', async () => { const mockData = [getMockData(), getMockData({ ruleId: '2', end: '2023-11-17T08:00:00.000Z' })]; + rulesClientParams.getEventLogClient.mockResolvedValue(eventLogClient); + const result = await rulesClient.scheduleBackfill(mockData); expect(authorization.getFindAuthorizationFilter).toHaveBeenCalledWith({ @@ -401,77 +408,82 @@ describe('scheduleBackfill()', () => { message: 'User has scheduled backfill for rule [id=2] [name=fakeRuleName]', }); - expect(backfillClient.bulkQueue).toHaveBeenCalledWith({ - auditLogger, - params: mockData, - ruleTypeRegistry, - unsecuredSavedObjectsClient, - spaceId: 'default', - rules: [ - { - id: existingDecryptedRule1.id, - legacyId: null, - actions: existingDecryptedRule1.attributes.actions, - alertTypeId: existingDecryptedRule1.attributes.alertTypeId, - apiKey: existingDecryptedRule1.attributes.apiKey, - apiKeyCreatedByUser: existingDecryptedRule1.attributes.apiKeyCreatedByUser, - consumer: existingDecryptedRule1.attributes.consumer, - createdAt: new Date(existingDecryptedRule1.attributes.createdAt), - createdBy: existingDecryptedRule1.attributes.createdBy, - enabled: true, - executionStatus: { - ...existingDecryptedRule1.attributes.executionStatus, - lastExecutionDate: new Date( - existingDecryptedRule1.attributes.executionStatus.lastExecutionDate - ), + expect(backfillClient.bulkQueue).toHaveBeenCalledWith( + expect.objectContaining({ + auditLogger, + params: mockData, + ruleTypeRegistry, + unsecuredSavedObjectsClient, + spaceId: 'default', + rules: [ + { + id: existingDecryptedRule1.id, + legacyId: null, + actions: existingDecryptedRule1.attributes.actions, + alertTypeId: existingDecryptedRule1.attributes.alertTypeId, + apiKey: existingDecryptedRule1.attributes.apiKey, + apiKeyCreatedByUser: existingDecryptedRule1.attributes.apiKeyCreatedByUser, + consumer: existingDecryptedRule1.attributes.consumer, + createdAt: new Date(existingDecryptedRule1.attributes.createdAt), + createdBy: existingDecryptedRule1.attributes.createdBy, + enabled: true, + executionStatus: { + ...existingDecryptedRule1.attributes.executionStatus, + lastExecutionDate: new Date( + existingDecryptedRule1.attributes.executionStatus.lastExecutionDate + ), + }, + muteAll: existingDecryptedRule1.attributes.muteAll, + mutedInstanceIds: existingDecryptedRule1.attributes.mutedInstanceIds, + name: existingDecryptedRule1.attributes.name, + notifyWhen: existingDecryptedRule1.attributes.notifyWhen, + params: existingDecryptedRule1.attributes.params, + revision: existingDecryptedRule1.attributes.revision, + schedule: existingDecryptedRule1.attributes.schedule, + scheduledTaskId: existingDecryptedRule1.attributes.scheduledTaskId, + snoozeSchedule: existingDecryptedRule1.attributes.snoozeSchedule, + systemActions: existingDecryptedRule1.attributes.systemActions, + tags: existingDecryptedRule1.attributes.tags, + throttle: existingDecryptedRule1.attributes.throttle, + updatedAt: new Date(existingDecryptedRule1.attributes.updatedAt), }, - muteAll: existingDecryptedRule1.attributes.muteAll, - mutedInstanceIds: existingDecryptedRule1.attributes.mutedInstanceIds, - name: existingDecryptedRule1.attributes.name, - notifyWhen: existingDecryptedRule1.attributes.notifyWhen, - params: existingDecryptedRule1.attributes.params, - revision: existingDecryptedRule1.attributes.revision, - schedule: existingDecryptedRule1.attributes.schedule, - scheduledTaskId: existingDecryptedRule1.attributes.scheduledTaskId, - snoozeSchedule: existingDecryptedRule1.attributes.snoozeSchedule, - systemActions: existingDecryptedRule1.attributes.systemActions, - tags: existingDecryptedRule1.attributes.tags, - throttle: existingDecryptedRule1.attributes.throttle, - updatedAt: new Date(existingDecryptedRule1.attributes.updatedAt), - }, - { - id: existingDecryptedRule2.id, - legacyId: null, - actions: existingDecryptedRule2.attributes.actions, - alertTypeId: existingDecryptedRule2.attributes.alertTypeId, - apiKey: existingDecryptedRule2.attributes.apiKey, - apiKeyCreatedByUser: existingDecryptedRule2.attributes.apiKeyCreatedByUser, - consumer: existingDecryptedRule2.attributes.consumer, - createdAt: new Date(existingDecryptedRule2.attributes.createdAt), - createdBy: existingDecryptedRule2.attributes.createdBy, - enabled: true, - executionStatus: { - ...existingDecryptedRule2.attributes.executionStatus, - lastExecutionDate: new Date( - existingDecryptedRule2.attributes.executionStatus.lastExecutionDate - ), + { + id: existingDecryptedRule2.id, + legacyId: null, + actions: existingDecryptedRule2.attributes.actions, + alertTypeId: existingDecryptedRule2.attributes.alertTypeId, + apiKey: existingDecryptedRule2.attributes.apiKey, + apiKeyCreatedByUser: existingDecryptedRule2.attributes.apiKeyCreatedByUser, + consumer: existingDecryptedRule2.attributes.consumer, + createdAt: new Date(existingDecryptedRule2.attributes.createdAt), + createdBy: existingDecryptedRule2.attributes.createdBy, + enabled: true, + executionStatus: { + ...existingDecryptedRule2.attributes.executionStatus, + lastExecutionDate: new Date( + existingDecryptedRule2.attributes.executionStatus.lastExecutionDate + ), + }, + muteAll: existingDecryptedRule2.attributes.muteAll, + mutedInstanceIds: existingDecryptedRule2.attributes.mutedInstanceIds, + name: existingDecryptedRule2.attributes.name, + notifyWhen: existingDecryptedRule2.attributes.notifyWhen, + params: existingDecryptedRule2.attributes.params, + revision: existingDecryptedRule2.attributes.revision, + schedule: existingDecryptedRule2.attributes.schedule, + scheduledTaskId: existingDecryptedRule2.attributes.scheduledTaskId, + snoozeSchedule: existingDecryptedRule2.attributes.snoozeSchedule, + systemActions: existingDecryptedRule2.attributes.systemActions, + tags: existingDecryptedRule2.attributes.tags, + throttle: existingDecryptedRule2.attributes.throttle, + updatedAt: new Date(existingDecryptedRule2.attributes.updatedAt), }, - muteAll: existingDecryptedRule2.attributes.muteAll, - mutedInstanceIds: existingDecryptedRule2.attributes.mutedInstanceIds, - name: existingDecryptedRule2.attributes.name, - notifyWhen: existingDecryptedRule2.attributes.notifyWhen, - params: existingDecryptedRule2.attributes.params, - revision: existingDecryptedRule2.attributes.revision, - schedule: existingDecryptedRule2.attributes.schedule, - scheduledTaskId: existingDecryptedRule2.attributes.scheduledTaskId, - snoozeSchedule: existingDecryptedRule2.attributes.snoozeSchedule, - systemActions: existingDecryptedRule2.attributes.systemActions, - tags: existingDecryptedRule2.attributes.tags, - throttle: existingDecryptedRule2.attributes.throttle, - updatedAt: new Date(existingDecryptedRule2.attributes.updatedAt), - }, - ], - }); + ], + eventLogClient, + internalSavedObjectsRepository, + eventLogger, + }) + ); expect(result).toEqual(mockBulkQueueResult); }); diff --git a/x-pack/platform/plugins/shared/alerting/server/application/rule/methods/find_gaps/find_gaps.test.ts b/x-pack/platform/plugins/shared/alerting/server/application/rule/methods/find_gaps/find_gaps.test.ts new file mode 100644 index 0000000000000..7ea2a14872225 --- /dev/null +++ b/x-pack/platform/plugins/shared/alerting/server/application/rule/methods/find_gaps/find_gaps.test.ts @@ -0,0 +1,150 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { findGaps } from './find_gaps'; +import { getRule } from '../get/get_rule'; +import { loggerMock } from '@kbn/logging-mocks'; +import { eventLogClientMock } from '@kbn/event-log-plugin/server/event_log_client.mock'; +import { auditLoggerMock } from '@kbn/security-plugin/server/audit/mocks'; +import { ReadOperations, AlertingAuthorizationEntity } from '../../../../authorization'; +import { RULE_SAVED_OBJECT_TYPE } from '../../../../saved_objects'; + +jest.mock('../get/get_rule'); +jest.mock('../../../../lib/rule_gaps/find_gaps'); + +const mockRule = { + id: '1', + name: 'Test Rule', + alertTypeId: 'test-type', + consumer: 'test-consumer', + enabled: true, + tags: [], + actions: [], + schedule: { interval: '1m' }, + createdAt: new Date(), + updatedAt: new Date(), + params: {}, + executionStatus: { + status: 'ok' as const, + lastExecutionDate: new Date(), + }, + notifyWhen: 'onActiveAlert' as const, + muteAll: false, + mutedInstanceIds: [], + updatedBy: null, + createdBy: null, + apiKeyOwner: null, + throttle: null, + legacyId: null, + revision: 1, +}; + +describe('findGaps', () => { + const mockedGetRule = getRule as jest.MockedFunction; + const auditLogger = auditLoggerMock.create(); + let context: any; + + const params = { + ruleId: '1', + page: 1, + perPage: 10, + start: '2024-01-01T00:00:00.000Z', + end: '2024-01-02T00:00:00.000Z', + }; + + beforeEach(() => { + context = { + authorization: { + ensureAuthorized: jest.fn(), + }, + logger: loggerMock.create(), + auditLogger, + getEventLogClient: jest.fn().mockResolvedValue(eventLogClientMock.create()), + }; + (auditLogger.log as jest.Mock).mockClear(); + mockedGetRule.mockResolvedValue(mockRule); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + describe('authorization', () => { + it('should authorize and find gaps successfully', async () => { + await findGaps(context, params); + + expect(context.authorization.ensureAuthorized).toHaveBeenCalledWith({ + ruleTypeId: mockRule.alertTypeId, + consumer: mockRule.consumer, + operation: ReadOperations.FindGaps, + entity: AlertingAuthorizationEntity.Rule, + }); + }); + + it('should throw error when not authorized', async () => { + const authError = new Error('Unauthorized'); + context.authorization.ensureAuthorized.mockRejectedValue(authError); + + await expect(findGaps(context, params)).rejects.toThrow(authError); + }); + }); + + describe('auditLogger', () => { + it('logs audit event when finding gaps successfully', async () => { + await findGaps(context, params); + + expect(auditLogger.log).toHaveBeenCalledWith( + expect.objectContaining({ + event: expect.objectContaining({ + action: 'rule_find_gaps', + outcome: 'success', + }), + kibana: { saved_object: { id: '1', type: RULE_SAVED_OBJECT_TYPE, name: 'Test Rule' } }, + }) + ); + }); + + it('logs audit event when not authorized to find gaps', async () => { + const authError = new Error('Unauthorized'); + context.authorization.ensureAuthorized.mockRejectedValue(authError); + + await expect(findGaps(context, params)).rejects.toThrow(authError); + + expect(auditLogger.log).toHaveBeenCalledWith( + expect.objectContaining({ + event: expect.objectContaining({ + action: 'rule_find_gaps', + outcome: 'failure', + }), + kibana: { saved_object: { id: '1', type: RULE_SAVED_OBJECT_TYPE, name: 'Test Rule' } }, + error: { + code: 'Error', + message: 'Unauthorized', + }, + }) + ); + }); + }); + + describe('error handling', () => { + it('should handle and wrap errors from getRule', async () => { + const error = new Error('Rule not found'); + mockedGetRule.mockRejectedValue(error); + + await expect(findGaps(context, params)).rejects.toThrow('Failed to find gaps'); + expect(context.logger.error).toHaveBeenCalled(); + }); + + it('should handle errors from findGaps implementation', async () => { + const error = new Error('Failed to find gaps'); + context.getEventLogClient.mockRejectedValue(error); + + await expect(findGaps(context, params)).rejects.toThrow('Failed to find gaps'); + expect(context.logger.error).toHaveBeenCalled(); + }); + }); +}); diff --git a/x-pack/platform/plugins/shared/alerting/server/application/rule/methods/find_gaps/find_gaps.ts b/x-pack/platform/plugins/shared/alerting/server/application/rule/methods/find_gaps/find_gaps.ts index 612c2452b9f88..801b054577404 100644 --- a/x-pack/platform/plugins/shared/alerting/server/application/rule/methods/find_gaps/find_gaps.ts +++ b/x-pack/platform/plugins/shared/alerting/server/application/rule/methods/find_gaps/find_gaps.ts @@ -49,6 +49,13 @@ export async function findGaps(context: RulesClientContext, params: FindGapsPara logger: context.logger, }); + context.auditLogger?.log( + ruleAuditEvent({ + action: RuleAuditAction.FIND_GAPS, + savedObject: { type: RULE_SAVED_OBJECT_TYPE, id: rule.id, name: rule.name }, + }) + ); + return gaps; } catch (err) { const errorMessage = `Failed to find gaps`; diff --git a/x-pack/platform/plugins/shared/alerting/server/backfill_client/backfill_client.test.ts b/x-pack/platform/plugins/shared/alerting/server/backfill_client/backfill_client.test.ts index 771f5a4db34b9..83e069ad05ecc 100644 --- a/x-pack/platform/plugins/shared/alerting/server/backfill_client/backfill_client.test.ts +++ b/x-pack/platform/plugins/shared/alerting/server/backfill_client/backfill_client.test.ts @@ -8,7 +8,7 @@ import { adHocRunStatus } from '../../common/constants'; import { loggingSystemMock } from '@kbn/core-logging-server-mocks'; import { SavedObject, SavedObjectsBulkResponse } from '@kbn/core/server'; -import { savedObjectsClientMock } from '@kbn/core/server/mocks'; +import { savedObjectsClientMock, savedObjectsRepositoryMock } from '@kbn/core/server/mocks'; import { ScheduleBackfillParam } from '../application/backfill/methods/schedule/types'; import { RuleDomain } from '../application/rule/types'; import { ruleTypeRegistryMock } from '../rule_type_registry.mock'; @@ -22,6 +22,12 @@ import { taskManagerMock } from '@kbn/task-manager-plugin/server/mocks'; import { TaskRunnerFactory } from '../task_runner'; import { TaskPriority } from '@kbn/task-manager-plugin/server'; import { UntypedNormalizedRuleType } from '../rule_type_registry'; +import { eventLogClientMock, eventLoggerMock } from '@kbn/event-log-plugin/server/mocks'; +import { updateGaps } from '../lib/rule_gaps/update/update_gaps'; + +jest.mock('../lib/rule_gaps/update/update_gaps', () => ({ + updateGaps: jest.fn(), +})); const logger = loggingSystemMock.create().get(); const taskManagerSetup = taskManagerMock.createSetup(); @@ -29,6 +35,9 @@ const taskManagerStart = taskManagerMock.createStart(); const ruleTypeRegistry = ruleTypeRegistryMock.create(); const unsecuredSavedObjectsClient = savedObjectsClientMock.create(); const auditLogger = auditLoggerMock.create(); +const eventLogClient = eventLogClientMock.create(); +const eventLogger = eventLoggerMock.create(); +const internalSavedObjectsRepository = savedObjectsRepositoryMock.create(); function getMockData(overwrites: Record = {}): ScheduleBackfillParam { return { @@ -173,9 +182,23 @@ function getBulkCreateParam( } const mockCreatePointInTimeFinderAsInternalUser = ( - response = { + response: { + saved_objects: Array<{ + id: string; + type: string; + attributes: AdHocRunSO; + references?: Array<{ id: string; name: string; type: string }>; + version?: string; + }>; + } = { saved_objects: [ - { id: 'abc', type: AD_HOC_RUN_SAVED_OBJECT_TYPE, attributes: getMockAdHocRunAttributes() }, + { + id: 'abc', + type: AD_HOC_RUN_SAVED_OBJECT_TYPE, + attributes: getMockAdHocRunAttributes(), + references: [{ id: '1', name: 'rule', type: RULE_SAVED_OBJECT_TYPE }], + version: '1', + }, ], } ) => { @@ -189,7 +212,6 @@ const mockCreatePointInTimeFinderAsInternalUser = ( describe('BackfillClient', () => { let backfillClient: BackfillClient; - beforeAll(() => { jest.useFakeTimers().setSystemTime(new Date('2024-01-30T00:00:00.000Z')); }); @@ -277,6 +299,9 @@ describe('BackfillClient', () => { ruleTypeRegistry, spaceId: 'default', unsecuredSavedObjectsClient, + eventLogClient, + internalSavedObjectsRepository, + eventLogger, }); const bulkCreateParams = [ @@ -389,6 +414,9 @@ describe('BackfillClient', () => { ruleTypeRegistry, spaceId: 'default', unsecuredSavedObjectsClient, + eventLogClient, + internalSavedObjectsRepository, + eventLogger, }); const bulkCreateParams = [ @@ -482,6 +510,9 @@ describe('BackfillClient', () => { ruleTypeRegistry, spaceId: 'default', unsecuredSavedObjectsClient, + eventLogClient, + internalSavedObjectsRepository, + eventLogger, }); const bulkCreateParams = [ @@ -597,6 +628,9 @@ describe('BackfillClient', () => { ruleTypeRegistry, spaceId: 'default', unsecuredSavedObjectsClient, + eventLogClient, + internalSavedObjectsRepository, + eventLogger, }); expect(auditLogger.log).toHaveBeenCalledTimes(5); expect(auditLogger.log).toHaveBeenNthCalledWith(1, { @@ -745,6 +779,9 @@ describe('BackfillClient', () => { ruleTypeRegistry, spaceId: 'default', unsecuredSavedObjectsClient, + eventLogClient, + internalSavedObjectsRepository, + eventLogger, }); expect(unsecuredSavedObjectsClient.bulkCreate).not.toHaveBeenCalled(); @@ -847,6 +884,9 @@ describe('BackfillClient', () => { ruleTypeRegistry, spaceId: 'default', unsecuredSavedObjectsClient, + eventLogClient, + internalSavedObjectsRepository, + eventLogger, }); expect(taskManagerStart.bulkSchedule).not.toHaveBeenCalled(); @@ -884,6 +924,127 @@ describe('BackfillClient', () => { }, ]); }); + + test('should call updateGaps with correct params for each backfill', async () => { + const mockData = [getMockData(), getMockData({ ruleId: '2' })]; + const rule1 = getMockRule(); + const rule2 = getMockRule({ id: '2' }); + const mockRules = [rule1, rule2]; + + const mockAttributes = getMockAdHocRunAttributes(); + const bulkCreateResult = { + saved_objects: [ + getBulkCreateParam('abc', '1', mockAttributes), + getBulkCreateParam('def', '2', mockAttributes), + ], + }; + + unsecuredSavedObjectsClient.bulkCreate.mockResolvedValueOnce(bulkCreateResult); + + await backfillClient.bulkQueue({ + auditLogger, + params: mockData, + rules: mockRules, + ruleTypeRegistry, + spaceId: 'default', + unsecuredSavedObjectsClient, + eventLogClient, + internalSavedObjectsRepository, + eventLogger, + }); + + expect(updateGaps).toHaveBeenCalledTimes(2); + expect(updateGaps).toHaveBeenNthCalledWith(1, { + backfillSchedule: mockAttributes.schedule, + ruleId: '1', + start: new Date(mockAttributes.start), + end: new Date(), + eventLogger, + eventLogClient, + savedObjectsRepository: internalSavedObjectsRepository, + logger, + backfillClient, + }); + expect(updateGaps).toHaveBeenNthCalledWith(2, { + backfillSchedule: mockAttributes.schedule, + ruleId: '2', + start: new Date(mockAttributes.start), + end: new Date(), + eventLogger, + eventLogClient, + savedObjectsRepository: internalSavedObjectsRepository, + logger, + backfillClient, + }); + }); + + test('should handle updateGaps errors gracefully', async () => { + const mockData = [getMockData()]; + const rule1 = getMockRule(); + const mockRules = [rule1]; + + const mockAttributes = getMockAdHocRunAttributes(); + const bulkCreateResult = { + saved_objects: [getBulkCreateParam('abc', '1', mockAttributes)], + }; + + unsecuredSavedObjectsClient.bulkCreate.mockResolvedValueOnce(bulkCreateResult); + (updateGaps as jest.Mock).mockRejectedValueOnce(new Error('Failed to update gaps')); + + await backfillClient.bulkQueue({ + auditLogger, + params: mockData, + rules: mockRules, + ruleTypeRegistry, + spaceId: 'default', + unsecuredSavedObjectsClient, + eventLogClient, + internalSavedObjectsRepository, + eventLogger, + }); + + expect(logger.warn).toHaveBeenCalledWith('Error updating gaps for backfill jobs: abc'); + }); + + test('should call updateGaps with end date from backfill param if provided', async () => { + const endDate = '2024-02-01T00:00:00.000Z'; + const mockData = [getMockData({ end: endDate })]; + const rule1 = getMockRule(); + const mockRules = [rule1]; + + const mockAttributes = getMockAdHocRunAttributes({ + overwrites: { end: endDate }, + }); + const bulkCreateResult = { + saved_objects: [getBulkCreateParam('abc', '1', mockAttributes)], + }; + + unsecuredSavedObjectsClient.bulkCreate.mockResolvedValueOnce(bulkCreateResult); + + await backfillClient.bulkQueue({ + auditLogger, + params: mockData, + rules: mockRules, + ruleTypeRegistry, + spaceId: 'default', + unsecuredSavedObjectsClient, + eventLogClient, + internalSavedObjectsRepository, + eventLogger, + }); + + expect(updateGaps).toHaveBeenCalledWith({ + backfillSchedule: mockAttributes.schedule, + ruleId: '1', + start: new Date(mockAttributes.start), + end: new Date(endDate), + eventLogger, + eventLogClient, + savedObjectsRepository: internalSavedObjectsRepository, + logger, + backfillClient, + }); + }); }); describe('deleteBackfillForRules()', () => { @@ -1190,4 +1351,97 @@ describe('BackfillClient', () => { ); }); }); + + describe('findOverlappingBackfills()', () => { + test('should find overlapping backfills', async () => { + const mockSavedObjectsRepository = savedObjectsRepositoryMock.create(); + const mockStart = new Date('2024-01-01T00:00:00.000Z'); + const mockEnd = new Date('2024-01-02T00:00:00.000Z'); + + mockCreatePointInTimeFinderAsInternalUser({ + saved_objects: [ + { + id: 'abc', + type: AD_HOC_RUN_SAVED_OBJECT_TYPE, + attributes: getMockAdHocRunAttributes(), + references: [{ id: '1', name: 'rule', type: RULE_SAVED_OBJECT_TYPE }], + version: '1', + }, + { + id: 'def', + type: AD_HOC_RUN_SAVED_OBJECT_TYPE, + attributes: getMockAdHocRunAttributes(), + references: [{ id: '1', name: 'rule', type: RULE_SAVED_OBJECT_TYPE }], + version: '1', + }, + ], + }); + + mockSavedObjectsRepository.createPointInTimeFinder = + unsecuredSavedObjectsClient.createPointInTimeFinder; + + const result = await backfillClient.findOverlappingBackfills({ + ruleId: '1', + start: mockStart, + end: mockEnd, + savedObjectsRepository: mockSavedObjectsRepository, + }); + + expect(mockSavedObjectsRepository.createPointInTimeFinder).toHaveBeenCalledWith({ + type: AD_HOC_RUN_SAVED_OBJECT_TYPE, + perPage: 100, + hasReference: [{ id: '1', type: RULE_SAVED_OBJECT_TYPE }], + filter: ` + ad_hoc_run_params.attributes.start <= "${mockEnd.toISOString()}" and + ad_hoc_run_params.attributes.end >= "${mockStart.toISOString()}" + `, + }); + + expect(result).toHaveLength(2); + expect('id' in result[0] && result[0].id).toBe('abc'); + expect('id' in result[1] && result[1].id).toBe('def'); + }); + + test('should handle errors and close finder', async () => { + const mockSavedObjectsRepository = savedObjectsRepositoryMock.create(); + const mockStart = new Date('2024-01-01T00:00:00.000Z'); + const mockEnd = new Date('2024-01-02T00:00:00.000Z'); + + mockSavedObjectsRepository.createPointInTimeFinder.mockResolvedValue({ + close: jest.fn(), + *find() { + throw new Error('Failed to find'); + }, + }); + + await expect( + backfillClient.findOverlappingBackfills({ + ruleId: '1', + start: mockStart, + end: mockEnd, + savedObjectsRepository: mockSavedObjectsRepository, + }) + ).rejects.toThrow('Failed to find'); + }); + + test('should return empty array when no overlapping backfills found', async () => { + const mockSavedObjectsRepository = savedObjectsRepositoryMock.create(); + const mockStart = new Date('2024-01-01T00:00:00.000Z'); + const mockEnd = new Date('2024-01-02T00:00:00.000Z'); + + mockCreatePointInTimeFinderAsInternalUser({ saved_objects: [] }); + + mockSavedObjectsRepository.createPointInTimeFinder = + unsecuredSavedObjectsClient.createPointInTimeFinder; + + const result = await backfillClient.findOverlappingBackfills({ + ruleId: '1', + start: mockStart, + end: mockEnd, + savedObjectsRepository: mockSavedObjectsRepository, + }); + + expect(result).toHaveLength(0); + }); + }); }); diff --git a/x-pack/platform/plugins/shared/alerting/server/routes/index.ts b/x-pack/platform/plugins/shared/alerting/server/routes/index.ts index dca12e0b160e3..fa895c764d4a6 100644 --- a/x-pack/platform/plugins/shared/alerting/server/routes/index.ts +++ b/x-pack/platform/plugins/shared/alerting/server/routes/index.ts @@ -72,10 +72,8 @@ import { getBackfillRoute } from './backfill/apis/get/get_backfill_route'; import { findBackfillRoute } from './backfill/apis/find/find_backfill_route'; import { deleteBackfillRoute } from './backfill/apis/delete/delete_backfill_route'; -// Gaps ApI +// Gaps API import { findGapsRoute } from './gaps/apis/find/find_gaps_route'; -import { fillGapByIdRoute } from './gaps/apis/fill/fill_gap_by_id_route'; -import { getRulesWithGapsRoute } from './gaps/apis/get_rules_with_gaps/get_rules_with_gaps_route'; export interface RouteOptions { router: IRouter; @@ -153,8 +151,6 @@ export function defineRoutes(opts: RouteOptions) { // Gaps APIs findGapsRoute(router, licenseState); - fillGapByIdRoute(router, licenseState); - getRulesWithGapsRoute(router, licenseState); // Other APIs registerFieldsRoute(router, licenseState); diff --git a/x-pack/platform/plugins/shared/alerting/server/rules_client.mock.ts b/x-pack/platform/plugins/shared/alerting/server/rules_client.mock.ts index b1e0b0d225fc4..13847ac1bea9e 100644 --- a/x-pack/platform/plugins/shared/alerting/server/rules_client.mock.ts +++ b/x-pack/platform/plugins/shared/alerting/server/rules_client.mock.ts @@ -57,8 +57,6 @@ const createRulesClientMock = () => { getScheduleFrequency: jest.fn(), bulkUntrackAlerts: jest.fn(), findGaps: jest.fn(), - fillGapById: jest.fn(), - getRulesWithGaps: jest.fn(), }; return mocked; }; diff --git a/x-pack/platform/plugins/shared/alerting/server/rules_client/rules_client.ts b/x-pack/platform/plugins/shared/alerting/server/rules_client/rules_client.ts index 799e686e95125..9a1b6a9e8de5e 100644 --- a/x-pack/platform/plugins/shared/alerting/server/rules_client/rules_client.ts +++ b/x-pack/platform/plugins/shared/alerting/server/rules_client/rules_client.ts @@ -77,9 +77,6 @@ import { FindBackfillParams } from '../application/backfill/methods/find/types'; import { DisableRuleParams } from '../application/rule/methods/disable'; import { EnableRuleParams } from '../application/rule/methods/enable_rule'; import { findGaps } from '../application/rule/methods/find_gaps'; - -import { GetRulesWithGapsParams } from '../application/rule/methods/get_rules_with_gaps/types'; -import { getRulesWithGaps } from '../application/rule/methods/get_rules_with_gaps'; import { FindGapsParams } from '../lib/rule_gaps/types'; export type ConstructorOptions = Omit< @@ -214,7 +211,4 @@ export class RulesClient { public getScheduleFrequency = () => getScheduleFrequency(this.context); public findGaps = (params: FindGapsParams) => findGaps(this.context, params); - - public getRulesWithGaps = (params: GetRulesWithGapsParams) => - getRulesWithGaps(this.context, params); } From cff950b260cd86d0da790bd93895112297c016b4 Mon Sep 17 00:00:00 2001 From: Nikita Khristinin Date: Fri, 10 Jan 2025 13:08:15 +0100 Subject: [PATCH 15/17] add more tests --- .../alerting_event_logger.test.ts | 61 +++++++++++++++++-- .../rules_client/common/audit_events.ts | 1 - .../task_runner/ad_hoc_task_runner.test.ts | 28 ++++++++- .../server/task_runner/ad_hoc_task_runner.ts | 12 ++-- 4 files changed, 89 insertions(+), 13 deletions(-) diff --git a/x-pack/platform/plugins/shared/alerting/server/lib/alerting_event_logger/alerting_event_logger.test.ts b/x-pack/platform/plugins/shared/alerting/server/lib/alerting_event_logger/alerting_event_logger.test.ts index da9f54567d886..1f118229c02f5 100644 --- a/x-pack/platform/plugins/shared/alerting/server/lib/alerting_event_logger/alerting_event_logger.test.ts +++ b/x-pack/platform/plugins/shared/alerting/server/lib/alerting_event_logger/alerting_event_logger.test.ts @@ -6,7 +6,7 @@ */ import { eventLoggerMock } from '@kbn/event-log-plugin/server/event_logger.mock'; -import { IEvent, SAVED_OBJECT_REL_PRIMARY } from '@kbn/event-log-plugin/server'; +import { IEvent, SAVED_OBJECT_REL_PRIMARY, InternalFields } from '@kbn/event-log-plugin/server'; import { ActionsCompletion } from '@kbn/alerting-state-types'; import { AlertingEventLogger, @@ -36,6 +36,7 @@ import { TaskRunnerTimerSpan } from '../../task_runner/task_runner_timer'; import { schema } from '@kbn/config-schema'; import { RULE_SAVED_OBJECT_TYPE } from '../..'; import { AD_HOC_RUN_SAVED_OBJECT_TYPE } from '../../saved_objects'; +import { GapBase } from '../rule_gaps/types'; const mockNow = '2020-01-01T02:00:00.000Z'; const eventLogger = eventLoggerMock.create(); @@ -1476,14 +1477,66 @@ describe('AlertingEventLogger', () => { }; alertingEventLogger.reportGap({ gap: range }); - const event = createGapRecord(ruleContext, ruleData, [alertSO], { - status: 'unfilled', + const gap: GapBase = { + status: 'unfilled' as const, range, - }); + filled_intervals: [], + unfilled_intervals: [range], + in_progress_intervals: [], + total_gap_duration_ms: 3600000, + filled_duration_ms: 0, + unfilled_duration_ms: 3600000, + in_progress_duration_ms: 0, + }; + const event = createGapRecord(ruleContext, ruleData, [alertSO], gap); expect(eventLogger.logEvent).toHaveBeenCalledWith(event); }); }); + + describe('updateGap()', () => { + const mockInternalFields: InternalFields = { + _id: 'test-id', + _index: 'test-index', + _seq_no: 1, + _primary_term: 1, + }; + + const mockGap: GapBase = { + status: 'filled' as const, + range: { + gte: '2022-05-05T15:59:54.480Z', + lte: '2022-05-05T16:59:54.480Z', + }, + filled_intervals: [ + { + gte: '2022-05-05T15:59:54.480Z', + lte: '2022-05-05T16:59:54.480Z', + }, + ], + unfilled_intervals: [], + in_progress_intervals: [], + total_gap_duration_ms: 3600000, + filled_duration_ms: 3600000, + unfilled_duration_ms: 0, + in_progress_duration_ms: 0, + }; + + test('should call eventLogger.updateEvent with correct parameters', async () => { + alertingEventLogger.initialize({ context: ruleContext, runDate, ruleData }); + await alertingEventLogger.updateGap({ internalFields: mockInternalFields, gap: mockGap }); + + expect(eventLogger.updateEvent).toHaveBeenCalledWith(mockInternalFields, { + kibana: { + alert: { + rule: { + gap: mockGap, + }, + }, + }, + }); + }); + }); }); describe('helper functions', () => { diff --git a/x-pack/platform/plugins/shared/alerting/server/rules_client/common/audit_events.ts b/x-pack/platform/plugins/shared/alerting/server/rules_client/common/audit_events.ts index ba74527a9f6b3..7d61fad2f47f7 100644 --- a/x-pack/platform/plugins/shared/alerting/server/rules_client/common/audit_events.ts +++ b/x-pack/platform/plugins/shared/alerting/server/rules_client/common/audit_events.ts @@ -47,7 +47,6 @@ export enum AdHocRunAuditAction { DELETE = 'ad_hoc_run_delete', } - type VerbsTuple = [string, string, string]; const ruleEventVerbs: Record = { diff --git a/x-pack/platform/plugins/shared/alerting/server/task_runner/ad_hoc_task_runner.test.ts b/x-pack/platform/plugins/shared/alerting/server/task_runner/ad_hoc_task_runner.test.ts index ae6467d0dcbf8..7af3bdee961ef 100644 --- a/x-pack/platform/plugins/shared/alerting/server/task_runner/ad_hoc_task_runner.test.ts +++ b/x-pack/platform/plugins/shared/alerting/server/task_runner/ad_hoc_task_runner.test.ts @@ -23,7 +23,7 @@ import { dataViewPluginMocks } from '@kbn/data-views-plugin/public/mocks'; import { DataViewsServerPluginStart } from '@kbn/data-views-plugin/server'; import { encryptedSavedObjectsMock } from '@kbn/encrypted-saved-objects-plugin/server/mocks'; import { IEventLogger } from '@kbn/event-log-plugin/server'; -import { eventLoggerMock } from '@kbn/event-log-plugin/server/mocks'; +import { eventLogClientMock, eventLoggerMock } from '@kbn/event-log-plugin/server/mocks'; import { SharePluginStart } from '@kbn/share-plugin/server'; import { ConcreteTaskInstance, TaskStatus } from '@kbn/task-manager-plugin/server'; import { usageCountersServiceMock } from '@kbn/usage-collection-plugin/server/usage_counters/usage_counters_service.mock'; @@ -93,7 +93,9 @@ import { RuleRunMetricsStore } from '../lib/rule_run_metrics_store'; import { ConnectorAdapterRegistry } from '../connector_adapters/connector_adapter_registry'; import { rulesSettingsServiceMock } from '../rules_settings/rules_settings_service.mock'; import { maintenanceWindowsServiceMock } from './maintenance_windows/maintenance_windows_service.mock'; +import { updateGaps } from '../lib/rule_gaps/update/update_gaps'; +jest.mock('../lib/rule_gaps/update/update_gaps'); const UUID = '5f6aa57d-3e22-484e-bae8-cbed868f4d28'; jest.mock('uuid', () => ({ @@ -149,6 +151,7 @@ const ruleTypeRegistry = ruleTypeRegistryMock.create(); const savedObjectsService = savedObjectsServiceMock.createInternalStartContract(); const services = alertsMock.createRuleExecutorServices(); const uiSettingsService = uiSettingsServiceMock.createStartContract(); +const eventLogClient = eventLogClientMock.create(); const taskRunnerFactoryInitializerParams: TaskRunnerFactoryInitializerParamsType = { actionsConfigMap: { default: { max: 1000 } }, @@ -176,6 +179,7 @@ const taskRunnerFactoryInitializerParams: TaskRunnerFactoryInitializerParamsType uiSettings: uiSettingsService, usageCounter: mockUsageCounter, isServerless: false, + getEventLogClient: jest.fn(), }; const mockedTaskInstance: ConcreteTaskInstance = { @@ -226,6 +230,7 @@ describe('Ad Hoc Task Runner', () => { let schedule4: AdHocRunSchedule; let schedule5: AdHocRunSchedule; let alertingEventLoggerInitializer: ContextOpts; + const mockUpdateGaps = updateGaps as jest.MockedFunction; beforeAll(() => { fakeTimer = sinon.useFakeTimers(); @@ -652,6 +657,9 @@ describe('Ad Hoc Task Runner', () => { }); test('should delete ad hoc run SO and not return a new runAt date when all schedules have been processed ', async () => { + taskRunnerFactoryInitializerParams.getEventLogClient = jest + .fn() + .mockResolvedValue(eventLogClient); ruleTypeWithAlerts.executor.mockImplementation( async ({ services: executorServices, @@ -744,6 +752,24 @@ describe('Ad Hoc Task Runner', () => { { refresh: false, namespace: undefined } ); + expect(mockUpdateGaps).toHaveBeenCalledWith({ + ruleId: RULE_ID, + start: new Date(mockedAdHocRunSO.attributes.start), + end: mockedAdHocRunSO.attributes.end ? new Date(mockedAdHocRunSO.attributes.end) : new Date(), + eventLogger: taskRunnerFactoryInitializerParams.eventLogger, + eventLogClient, + logger: taskRunnerFactoryInitializerParams.logger, + backfillSchedule: [ + { ...schedule1, status: adHocRunStatus.COMPLETE }, + { ...schedule2, status: adHocRunStatus.TIMEOUT }, + { ...schedule3, status: adHocRunStatus.ERROR }, + { ...schedule4, status: adHocRunStatus.COMPLETE }, + { ...schedule5, status: adHocRunStatus.COMPLETE }, + ], + savedObjectsRepository: internalSavedObjectsRepository, + backfillClient: taskRunnerFactoryInitializerParams.backfillClient, + }); + testAlertingEventLogCalls({ status: 'ok', backfillRunAt: schedule5.runAt, diff --git a/x-pack/platform/plugins/shared/alerting/server/task_runner/ad_hoc_task_runner.ts b/x-pack/platform/plugins/shared/alerting/server/task_runner/ad_hoc_task_runner.ts index 81848adc29784..010b48d276339 100644 --- a/x-pack/platform/plugins/shared/alerting/server/task_runner/ad_hoc_task_runner.ts +++ b/x-pack/platform/plugins/shared/alerting/server/task_runner/ad_hoc_task_runner.ts @@ -487,16 +487,16 @@ export class AdHocTaskRunner implements CancellableTask { executionStatus?.error?.reason === RuleExecutionStatusErrorReasons.License || executionStatus?.error?.reason === RuleExecutionStatusErrorReasons.Validate); - if (startedAt) { - // Capture how long it took for the rule to run after being claimed - this.timer.setDuration(TaskRunnerTimerSpan.TotalRunDuration, startedAt); - } - await this.updateAdHocRunSavedObjectPostRun(adHocRunParamsId, namespace, { ...(this.shouldDeleteTask ? { status: adHocRunStatus.ERROR } : {}), ...(this.scheduleToRunIndex > -1 ? { schedule: this.adHocRunSchedule } : {}), }); + if (startedAt) { + // Capture how long it took for the rule to run after being claimed + this.timer.setDuration(TaskRunnerTimerSpan.TotalRunDuration, startedAt); + } + return { executionStatus, executionMetrics }; }); this.alertingEventLogger.done({ @@ -542,8 +542,6 @@ export class AdHocTaskRunner implements CancellableTask { this.shouldDeleteTask = this.shouldDeleteTask || !this.hasAnyPendingRuns(); - // await this.updateGapsAfterBackfillComplete(); - return { state: {}, ...(this.shouldDeleteTask ? {} : { runAt: new Date() }), From f556e106815fa4b5a0eb9341b5f7ba8e2fa64ee6 Mon Sep 17 00:00:00 2001 From: Nikita Khristinin Date: Fri, 10 Jan 2025 14:29:07 +0100 Subject: [PATCH 16/17] remove only test --- .../group1/tests/alerting/gap/update_gaps.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/gap/update_gaps.ts b/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/gap/update_gaps.ts index 52a8042405bcc..b94d5ce4fdbc9 100644 --- a/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/gap/update_gaps.ts +++ b/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/gap/update_gaps.ts @@ -18,7 +18,7 @@ export default function updateGapsTests({ getService }: FtrProviderContext) { const supertest = getService('supertest'); const retry = getService('retry'); - describe.only('update gaps', () => { + describe('update gaps', () => { const objectRemover = new ObjectRemover(supertest); const gapStart = moment().subtract(14, 'days').startOf('day').toISOString(); const gapEnd = moment().subtract(13, 'days').startOf('day').toISOString(); From c6bcaf01813f6cea50cd9070a7d9a7a6c218d9f5 Mon Sep 17 00:00:00 2001 From: Nikita Khristinin Date: Fri, 10 Jan 2025 15:24:34 +0100 Subject: [PATCH 17/17] fix tests a bit --- .../event_log/server/event_log_client.test.ts | 4 ++++ .../common/plugins/alerts/server/routes.ts | 5 ++--- .../group1/tests/alerting/gap/update_gaps.ts | 22 +++++++++++++++++++ 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/x-pack/platform/plugins/shared/event_log/server/event_log_client.test.ts b/x-pack/platform/plugins/shared/event_log/server/event_log_client.test.ts index 5abd44061eb4d..190046777bc65 100644 --- a/x-pack/platform/plugins/shared/event_log/server/event_log_client.test.ts +++ b/x-pack/platform/plugins/shared/event_log/server/event_log_client.test.ts @@ -285,6 +285,10 @@ describe('EventLogStart', () => { function fakeEvent(overrides = {}) { return merge( { + _id: '1', + _index: '1', + _seq_no: 1, + _primary_term: 1, event: { provider: 'actions', action: 'execute', diff --git a/x-pack/test/alerting_api_integration/common/plugins/alerts/server/routes.ts b/x-pack/test/alerting_api_integration/common/plugins/alerts/server/routes.ts index e857f7f10287b..2a708e93a0c56 100644 --- a/x-pack/test/alerting_api_integration/common/plugins/alerts/server/routes.ts +++ b/x-pack/test/alerting_api_integration/common/plugins/alerts/server/routes.ts @@ -628,6 +628,7 @@ export function defineRoutes( const alertingEventLogger = new AlertingEventLogger(eventLogger); + // mocked ruleData to initialize alert event logger alertingEventLogger.initialize({ context: { savedObjectId: req.body.ruleId, @@ -648,9 +649,7 @@ export function defineRoutes( defaultActionGroupId: 'default', minimumLicenseRequired: 'basic', isExportable: true, - executor: { - execute: async () => ({ state: {} }), - }, + executor: async () => ({ state: {} }), category: 'test', producer: 'alerts', cancelAlertsOnRuleTimeout: true, diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/gap/update_gaps.ts b/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/gap/update_gaps.ts index b94d5ce4fdbc9..9576dbd495f72 100644 --- a/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/gap/update_gaps.ts +++ b/x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/gap/update_gaps.ts @@ -478,6 +478,26 @@ export default function updateGapsTests({ getService }: FtrProviderContext) { .send([{ rule_id: ruleId, start: gapStart, end: gapEnd }]); expect(scheduleResponse.statusCode).to.eql(200); + + await retry.try(async () => { + const firstGapResponse = await supertest + .post(`${getUrlPrefix(space.id)}/internal/alerting/rules/gaps/_find`) + .set('kbn-xsrf', 'foo') + .send({ + rule_id: ruleId, + start: gapStart, + end: gapEnd, + }); + + const firstGap = firstGapResponse.body.data[0]; + expect(firstGap.status).to.eql('partially_filled'); + expect(firstGap.unfilled_intervals).to.have.length(0); + expect(firstGap.in_progress_intervals).to.have.length(1); + expect(firstGap.in_progress_intervals[0].gte).to.eql(gapStart); + expect(firstGap.in_progress_intervals[0].lte).to.eql(gapEnd); + expect(firstGap.filled_intervals).to.have.length(0); + }); + const backfillId = scheduleResponse.body[0].id; // Wait for task failure event @@ -495,6 +515,8 @@ export default function updateGapsTests({ getService }: FtrProviderContext) { expect(events[0]?.error?.message).to.eql('rule executor error'); }); + await waitForBackfillComplete(backfillId, space.id); + // Verify gap status is updated const finalGapResponse = await supertest .post(`${getUrlPrefix(space.id)}/internal/alerting/rules/gaps/_find`)