Gaps find udpate

x-pack/platform/packages/private/security/authorization_core/src/privileges/feature_privilege_builder/alerting.ts

@@ -28,6 +28,7 @@ const readOperations: Record<AlertingEntity, string[]> = {
     'getRuleExecutionKPI',
     'getBackfill',
     'findBackfill',
+    'findGaps',
   ],
   alert: ['get', 'find', 'getAuthorizedAlertsIndices', 'getAlertSummary'],
 };

x-pack/platform/plugins/shared/alerting/common/constants/gap_status.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const gapStatus = {
+  UNFILLED: 'unfilled',
+  FILLED: 'filled',
+  PARTIALLY_FILLED: 'partially_filled',
+} as const;
+
+export type GapStatus = (typeof gapStatus)[keyof typeof gapStatus];

x-pack/platform/plugins/shared/alerting/common/constants/index.ts

@@ -13,3 +13,5 @@ export {
   MAX_SCHEDULE_BACKFILL_LOOKBACK_WINDOW_MS,
 } from './backfill';
 export { PLUGIN } from './plugin';
+export { gapStatus } from './gap_status';
+export type { GapStatus } from './gap_status';

x-pack/platform/plugins/shared/alerting/common/index.ts

@@ -92,6 +92,12 @@ export const INTERNAL_ALERTING_BACKFILL_FIND_API_PATH =
 export const INTERNAL_ALERTING_BACKFILL_SCHEDULE_API_PATH =
   `${INTERNAL_ALERTING_BACKFILL_API_PATH}/_schedule` as const;
 
+export const INTERNAL_ALERTING_GAPS_API_PATH =
+  `${INTERNAL_BASE_ALERTING_API_PATH}/rules/gaps` as const;
+
+export const INTERNAL_ALERTING_GAPS_FIND_API_PATH =
+  `${INTERNAL_ALERTING_GAPS_API_PATH}/_find` as const;
+
 export const ALERTING_FEATURE_ID = 'alerts';
 export const MONITORING_HISTORY_LIMIT = 200;
 export const ENABLE_MAINTENANCE_WINDOWS = true;

x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/find/index.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export { findGapsBodySchema, findGapsResponseSchema } from './schemas/latest';
+export type { FindGapsRequestQuery, FindGapsResponseBody, FindGapsResponse } from './types/latest';
+
+export {
+  findGapsBodySchema as findGapsBodySchemaV1,
+  findGapsResponseSchema as findGapsResponseSchemaV1,
+} from './schemas/v1';
+export type {
+  FindGapsRequestQuery as FindGapsRequestQueryV1,
+  FindGapsResponseBody as FindGapsResponseBodyV1,
+  FindGapsResponse as FindGapsResponseV1,
+} from './types/v1';

x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/find/schemas/latest.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export * from './v1';

x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/find/schemas/v1.ts

@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { schema } from '@kbn/config-schema';
+import { gapsResponseSchemaV1 } from '../../../response';
+
+export const findGapsBodySchema = schema.object(
+  {
+    end: schema.maybe(schema.string()),
+    page: schema.number({ defaultValue: 1, min: 1 }),
+    per_page: schema.number({ defaultValue: 10, min: 0 }),
+    rule_id: schema.maybe(schema.string()),
+    start: schema.maybe(schema.string()),
+    sort_field: schema.maybe(
+      schema.oneOf([
+        schema.literal('@timestamp'),
+        schema.literal('status'),
+        schema.literal('total_gap_duration_ms'),
+      ])
+    ),
+    sort_order: schema.maybe(schema.oneOf([schema.literal('asc'), schema.literal('desc')])),
+    statuses: schema.maybe(
+      schema.arrayOf(
+        schema.oneOf([
+          schema.literal('partially_filled'),
+          schema.literal('unfilled'),
+          schema.literal('filled'),
+        ])
+      )
+    ),
+  },
+  {
+    validate({ start, end }) {
+      if (start) {
+        const parsedStart = Date.parse(start);
+        if (isNaN(parsedStart)) {
+          return `[start]: query start must be valid date`;
+        }
+      }
+      if (end) {
+        const parsedEnd = Date.parse(end);
+        if (isNaN(parsedEnd)) {
+          return `[end]: query end must be valid date`;
+        }
+      }
+    },
+  }
+);
+
+export const findGapsResponseSchema = schema.object({
+  page: schema.number(),
+  per_page: schema.number(),
+  total: schema.number(),
+  data: schema.arrayOf(gapsResponseSchemaV1),
+});

x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/find/types/latest.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export * from './v1';

x-pack/platform/plugins/shared/alerting/common/routes/gaps/apis/find/types/v1.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { TypeOf } from '@kbn/config-schema';
+import { findQuerySchemaV1, findResponseSchemaV1 } from '..';
+
+export type FindGapsRequestQuery = TypeOf<typeof findQuerySchemaV1>;
+export type FindGapsResponseBody = TypeOf<typeof findResponseSchemaV1>;
+
+export interface FindGapsResponse {
+  body: FindGapsResponseBody;
+}

x-pack/platform/plugins/shared/alerting/common/routes/gaps/response/index.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export { gapsResponseSchema, errorResponseSchema } from './schemas/latest';
+export type { GapsResponse } from './types/latest';
+
+export {
+  gapsResponseSchema as gapsResponseSchemaV1,
+  errorResponseSchema as errorResponseSchemaV1,
+} from './schemas/v1';
+
+export type { GapsResponse as GapsResponseV1, ErrorResponse as ErrorResponseV1 } from './types/v1';

x-pack/platform/plugins/shared/alerting/common/routes/gaps/response/schemas/latest.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export * from './v1';

x-pack/platform/plugins/shared/alerting/common/routes/gaps/response/schemas/v1.ts

@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { schema } from '@kbn/config-schema';
+
+import { gapStatus } from '../../../../constants';
+
+export const gapStatusSchema = schema.oneOf([
+  schema.literal(gapStatus.UNFILLED),
+  schema.literal(gapStatus.FILLED),
+  schema.literal(gapStatus.PARTIALLY_FILLED),
+]);
+
+export const rangeSchema = schema.object({
+  lte: schema.string(),
+  gte: schema.string(),
+});
+
+export const rangeListSchema = schema.arrayOf(rangeSchema);
+
+export const gapsResponseSchema = schema.object({
+  '@timestamp': schema.string(),
+  _id: schema.string(),
+  status: gapStatusSchema,
+  range: rangeSchema,
+  in_progress_intervals: rangeListSchema,
+  filled_intervals: rangeListSchema,
+  unfilled_intervals: rangeListSchema,
+  total_gap_duration_ms: schema.number(),
+  filled_duration_ms: schema.number(),
+  unfilled_duration_ms: schema.number(),
+  in_progress_duration_ms: schema.number(),
+});
+
+export const errorResponseSchema = schema.object({
+  error: schema.object({
+    message: schema.string(),
+    status: schema.maybe(schema.number()),
+  }),
+});

x-pack/platform/plugins/shared/alerting/common/routes/gaps/response/types/latest.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export * from './v1';

x-pack/platform/plugins/shared/alerting/common/routes/gaps/response/types/v1.ts

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { TypeOf } from '@kbn/config-schema';
+import { gapsResponseSchemaV1, errorResponseSchemaV1 } from '..';
+
+export type GapsResponse = TypeOf<typeof gapsResponseSchemaV1>;
+export type ErrorResponse = TypeOf<typeof errorResponseSchemaV1>;

x-pack/platform/plugins/shared/alerting/server/alerts_service/lib/retry_transient_es_errors.test.ts

@@ -92,4 +92,27 @@ describe('retryTransientErrors', () => {
     await expect(retryTransientEsErrors(esCallMock, { logger })).rejects.toThrow(error);
     expect(esCallMock).toHaveBeenCalledTimes(1);
   });
+
+  it('retries with additional status codes when provided', async () => {
+    const customStatusCode = 409; // Conflict version
+    const error = new EsErrors.ResponseError({
+      statusCode: customStatusCode,
+      meta: {} as any,
+      warnings: [],
+      body: 'Conflict Version',
+    });
+    const esCallMock = jest.fn().mockRejectedValueOnce(error).mockResolvedValue('success');
+
+    expect(
+      await retryTransientEsErrors(esCallMock, {
+        logger,
+        additionalRetryableStatusCodes: [customStatusCode],
+      })
+    ).toEqual('success');
+    expect(esCallMock).toHaveBeenCalledTimes(2);
+    expect(logger.warn).toHaveBeenCalledTimes(1);
+    expect(logger.warn.mock.calls[0][0]).toMatch(
+      `Retrying Elasticsearch operation after [2s] due to error: ResponseError: Conflict Version`
+    );
+  });
 });

x-pack/platform/plugins/shared/alerting/server/alerts_service/lib/retry_transient_es_errors.ts

@@ -16,11 +16,12 @@ const retryResponseStatuses = [
   410, // Gone
 ];
 
-const isRetryableError = (e: Error) =>
+const isRetryableError = (e: Error, additionalRetryableStatusCodes: number[]) =>
   e instanceof EsErrors.NoLivingConnectionsError ||
   e instanceof EsErrors.ConnectionError ||
   e instanceof EsErrors.TimeoutError ||
-  (e instanceof EsErrors.ResponseError && retryResponseStatuses.includes(e?.statusCode!));
+  (e instanceof EsErrors.ResponseError &&
+    [...retryResponseStatuses, ...additionalRetryableStatusCodes].includes(e?.statusCode!));
 
 const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
 
@@ -29,15 +30,17 @@ export const retryTransientEsErrors = async <T>(
   {
     logger,
     attempt = 0,
+    additionalRetryableStatusCodes = [],
   }: {
     logger: Logger;
     attempt?: number;
+    additionalRetryableStatusCodes?: number[];
   }
 ): Promise<T> => {
   try {
     return await esCall();
   } catch (e) {
-    if (attempt < MAX_ATTEMPTS && isRetryableError(e)) {
+    if (attempt < MAX_ATTEMPTS && isRetryableError(e, additionalRetryableStatusCodes)) {
       const retryCount = attempt + 1;
       const retryDelaySec: number = Math.min(Math.pow(2, retryCount), 30); // 2s, 4s, 8s, 16s, 30s, 30s, 30s...
 
@@ -49,7 +52,11 @@ export const retryTransientEsErrors = async <T>(
 
       // delay with some randomness
       await delay(retryDelaySec * 1000 * Math.random());
-      return retryTransientEsErrors(esCall, { logger, attempt: retryCount });
+      return retryTransientEsErrors(esCall, {
+        logger,
+        attempt: retryCount,
+        additionalRetryableStatusCodes,
+      });
     }
 
     throw e;

x-pack/platform/plugins/shared/alerting/server/application/backfill/methods/delete/delete_backfill.test.ts

@@ -28,6 +28,11 @@ import { AD_HOC_RUN_SAVED_OBJECT_TYPE } from '../../../../saved_objects';
 import { SavedObject } from '@kbn/core-saved-objects-api-server';
 import { AdHocRunSO } from '../../../../data/ad_hoc_run/types';
 import { SavedObjectsErrorHelpers } from '@kbn/core/server';
+import { updateGaps } from '../../../../lib/rule_gaps/update/update_gaps';
+
+jest.mock('../../../../lib/rule_gaps/update/update_gaps', () => ({
+  updateGaps: jest.fn(),
+}));
 
 const kibanaVersion = 'v8.0.0';
 const taskManager = taskManagerMock.createStart();
@@ -159,14 +164,46 @@ describe('deleteBackfill()', () => {
     });
     expect(unsecuredSavedObjectsClient.delete).toHaveBeenLastCalledWith(
       AD_HOC_RUN_SAVED_OBJECT_TYPE,
-      '1'
+      '1',
+      {
+        refresh: 'wait_for',
+      }
     );
     expect(taskManager.removeIfExists).toHaveBeenCalledWith('1');
     expect(logger.error).not.toHaveBeenCalled();
 
     expect(result).toEqual({});
   });
 
+  test('should call updateGaps with correct parameters when deleting backfill', async () => {
+    const mockEventLogClient = { mockEventLogClient: true } as any;
+    rulesClientParams.getEventLogClient.mockResolvedValue(mockEventLogClient);
+
+    await rulesClient.deleteBackfill('1');
+
+    const updateGapsCall = (updateGaps as jest.Mock).mock.calls[0][0];
+    expect(updateGapsCall.ruleId).toBe('abc');
+    expect(updateGapsCall.start).toEqual(new Date('2023-10-19T15:07:40.011Z'));
+    expect(updateGapsCall.end).toBeInstanceOf(Date);
+    expect(updateGapsCall.backfillSchedule).toEqual([
+      {
+        interval: '12h',
+        status: adHocRunStatus.PENDING,
+        runAt: '2023-10-20T03:07:40.011Z',
+      },
+      {
+        interval: '12h',
+        status: adHocRunStatus.PENDING,
+        runAt: '2023-10-20T15:07:40.011Z',
+      },
+    ]);
+    expect(updateGapsCall.savedObjectsRepository).toBe(internalSavedObjectsRepository);
+    expect(updateGapsCall.logger).toBe(logger);
+    expect(updateGapsCall.eventLogClient).toBe(mockEventLogClient);
+    expect(updateGapsCall.shouldRefetchAllBackfills).toBe(true);
+    expect(updateGapsCall.backfillClient).toBe(backfillClient);
+  });
+
   describe('error handling', () => {
     test('should retry if conflict error', async () => {
       unsecuredSavedObjectsClient.delete.mockImplementationOnce(() => {