[ResponseOps] Clear flapping when rules are enabled or bulk enabled

x-pack/platform/plugins/shared/alerting/server/alerts_service/alerts_service.mock.ts

@@ -13,6 +13,7 @@ const creatAlertsServiceMock = () => {
       getContextInitializationPromise: jest.fn(),
       createAlertsClient: jest.fn(),
       setAlertsToUntracked: jest.fn(),
+      clearAlertFlappingHistory: jest.fn(),
     };
   });
 };

x-pack/platform/plugins/shared/alerting/server/alerts_service/alerts_service.ts

@@ -50,6 +50,8 @@ import { AlertsClient } from '../alerts_client';
 import type { IAlertsClient } from '../alerts_client/types';
 import type { SetAlertsToUntrackedParams } from './lib/set_alerts_to_untracked';
 import { setAlertsToUntracked } from './lib/set_alerts_to_untracked';
+import type { ClearAlertFlappingHistoryParams } from './lib/clear_alert_flapping_history';
+import { clearAlertFlappingHistory } from './lib/clear_alert_flapping_history';
 
 export const TOTAL_FIELDS_LIMIT = 2500;
 const LEGACY_ALERT_CONTEXT = 'legacy-alert';
@@ -501,4 +503,12 @@ export class AlertsService implements IAlertsService {
       ...opts,
     });
   }
+
+  public async clearAlertFlappingHistory(opts: ClearAlertFlappingHistoryParams) {
+    return clearAlertFlappingHistory({
+      logger: this.options.logger,
+      esClient: await this.options.elasticsearchClientPromise,
+      ...opts,
+    });
+  }
 }

x-pack/platform/plugins/shared/alerting/server/alerts_service/lib/clear_alert_flapping_history.test.ts

@@ -0,0 +1,183 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { ElasticsearchClientMock } from '@kbn/core/server/mocks';
+import { elasticsearchServiceMock, loggingSystemMock } from '@kbn/core/server/mocks';
+import { clearAlertFlappingHistory } from './clear_alert_flapping_history';
+import { ALERT_UUID } from '@kbn/rule-data-utils';
+
+let clusterClient: ElasticsearchClientMock;
+let logger: ReturnType<(typeof loggingSystemMock)['createLogger']>;
+
+describe('clearAlertFlappingHistory()', () => {
+  beforeEach(() => {
+    logger = loggingSystemMock.createLogger();
+    clusterClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
+    clusterClient.updateByQuery.mockResponse({
+      total: 1,
+      updated: 1,
+    });
+
+    clusterClient.search.mockResponse({
+      took: 1,
+      timed_out: false,
+      _shards: {
+        total: 1,
+        successful: 1,
+        skipped: 0,
+        failed: 0,
+      },
+      hits: {
+        hits: [
+          {
+            _index: 'test-index',
+            _source: {
+              [ALERT_UUID]: 'test-alert-id',
+            },
+          },
+        ],
+      },
+    });
+  });
+
+  afterEach(() => {
+    jest.resetAllMocks();
+  });
+
+  test('should call updateByQuery with provided rule id and index', async () => {
+    await clearAlertFlappingHistory({
+      logger,
+      esClient: clusterClient,
+      indices: ['test-index'],
+      ruleIds: ['test-rule'],
+    });
+
+    expect(clusterClient.updateByQuery).toHaveBeenCalledTimes(1);
+    expect(clusterClient.updateByQuery.mock.lastCall).toMatchInlineSnapshot(`
+      Array [
+        Object {
+          "allow_no_indices": true,
+          "conflicts": "proceed",
+          "index": Array [
+            "test-index",
+          ],
+          "query": Object {
+            "bool": Object {
+              "must": Array [
+                Object {
+                  "bool": Object {
+                    "should": Array [
+                      Object {
+                        "term": Object {
+                          "kibana.alert.status": Object {
+                            "value": "active",
+                          },
+                        },
+                      },
+                      Object {
+                        "term": Object {
+                          "kibana.alert.status": Object {
+                            "value": "recovered",
+                          },
+                        },
+                      },
+                    ],
+                  },
+                },
+                Object {
+                  "bool": Object {
+                    "should": Array [
+                      Object {
+                        "term": Object {
+                          "kibana.alert.rule.uuid": Object {
+                            "value": "test-rule",
+                          },
+                        },
+                      },
+                    ],
+                  },
+                },
+              ],
+            },
+          },
+          "refresh": true,
+          "script": Object {
+            "lang": "painless",
+            "source": "
+                  ctx._source['kibana.alert.flapping'] = false;
+                  ctx._source['kibana.alert.flapping_history'] = new ArrayList();
+                ",
+          },
+        },
+      ]
+    `);
+  });
+
+  test('should throw if either indices or ruleIds is empty', async () => {
+    await expect(
+      clearAlertFlappingHistory({
+        logger,
+        esClient: clusterClient,
+        indices: [],
+        ruleIds: ['test-rule'],
+      })
+    ).rejects.toThrow('Rule Ids and indices must be provided');
+
+    await expect(
+      clearAlertFlappingHistory({
+        logger,
+        esClient: clusterClient,
+        indices: ['test-index'],
+        ruleIds: [],
+      })
+    ).rejects.toThrow('Rule Ids and indices must be provided');
+
+    expect(clusterClient.updateByQuery).toHaveBeenCalledTimes(0);
+  });
+
+  test('should throw if could not clear flapping history', async () => {
+    clusterClient.updateByQuery.mockRejectedValue(Error('something went wrong!'));
+    await expect(
+      clearAlertFlappingHistory({
+        logger,
+        esClient: clusterClient,
+        indices: ['test-index'],
+        ruleIds: ['test-rule'],
+      })
+    ).rejects.toThrow('something went wrong!');
+
+    expect(clusterClient.updateByQuery).toHaveBeenCalledTimes(1);
+
+    expect(logger.error).toHaveBeenCalledWith(
+      'Error clearing alert flapping for indices: test-index, ruleIds: test-rule - something went wrong!'
+    );
+  });
+
+  test('should retry updateByQuery if it does not finish in 1 attempt', async () => {
+    clusterClient.updateByQuery.mockResolvedValueOnce({
+      total: 3,
+      updated: 1,
+    });
+    clusterClient.updateByQuery.mockResolvedValueOnce({
+      total: 3,
+      updated: 2,
+    });
+    clusterClient.updateByQuery.mockResolvedValueOnce({
+      total: 3,
+      updated: 3,
+    });
+
+    await clearAlertFlappingHistory({
+      logger,
+      esClient: clusterClient,
+      indices: ['test-index'],
+      ruleIds: ['test-rule'],
+    });
+
+    expect(clusterClient.updateByQuery).toHaveBeenCalledTimes(3);
+  });
+});

x-pack/platform/plugins/shared/alerting/server/alerts_service/lib/clear_alert_flapping_history.ts

@@ -0,0 +1,123 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
+import type { Logger } from '@kbn/logging';
+import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types';
+import {
+  ALERT_RULE_UUID,
+  ALERT_STATUS,
+  ALERT_STATUS_ACTIVE,
+  ALERT_STATUS_RECOVERED,
+  ALERT_FLAPPING,
+  ALERT_FLAPPING_HISTORY,
+} from '@kbn/rule-data-utils';
+
+export interface ClearAlertFlappingHistoryParams {
+  indices: string[];
+  ruleIds: string[];
+}
+
+interface ClearAlertFlappingHistoryParamsWithDeps extends ClearAlertFlappingHistoryParams {
+  logger: Logger;
+  esClient: ElasticsearchClient;
+}
+
+const clearAlertFlappingHistoryQuery = (ruleIds: string[]): QueryDslQueryContainer => {
+  const shouldStatusTerms = [ALERT_STATUS_ACTIVE, ALERT_STATUS_RECOVERED].map((status) => {
+    return {
+      term: {
+        [ALERT_STATUS]: { value: status },
+      },
+    };
+  });
+
+  const shouldRuleIdsTerms = ruleIds.map((ruleId) => {
+    return {
+      term: {
+        [ALERT_RULE_UUID]: { value: ruleId },
+      },
+    };
+  });
+
+  return {
+    bool: {
+      must: [
+        {
+          bool: {
+            should: shouldStatusTerms,
+          },
+        },
+        {
+          bool: {
+            should: shouldRuleIdsTerms,
+          },
+        },
+      ],
+    },
+  };
+};
+
+export const clearAlertFlappingHistory = async (
+  params: ClearAlertFlappingHistoryParamsWithDeps
+) => {
+  const { indices, ruleIds, esClient, logger } = params;
+
+  if (!ruleIds.length || !indices.length) {
+    throw new Error('Rule Ids and indices must be provided');
+  }
+
+  try {
+    let total = 0;
+
+    for (let retryCount = 0; retryCount < 3; retryCount++) {
+      const response = await esClient.updateByQuery({
+        index: indices,
+        allow_no_indices: true,
+        conflicts: 'proceed',
+        script: {
+          source: `
+            ctx._source['${ALERT_FLAPPING}'] = false;
+            ctx._source['${ALERT_FLAPPING_HISTORY}'] = new ArrayList();
+          `,
+          lang: 'painless',
+        },
+        query: clearAlertFlappingHistoryQuery(ruleIds),
+        refresh: true,
+      });
+
+      if (total === 0 && response.total === 0) {
+        logger.debug('No active or recovered alerts matched the query');
+        break;
+      }
+
+      if (response.total) {
+        total = response.total;
+      }
+
+      if (response.total === response.updated) {
+        break;
+      }
+
+      logger.warn(
+        `Attempt ${retryCount + 1}: Failed to clear flapping ${
+          (response.total ?? 0) - (response.updated ?? 0)
+        } of ${response.total}; indices: ${indices}, ruleIds: ${ruleIds}
+        }`
+      );
+    }
+
+    if (total === 0) {
+      return [];
+    }
+  } catch (err) {
+    logger.error(
+      `Error clearing alert flapping for indices: ${indices}, ruleIds: ${ruleIds} - ${err.message}`
+    );
+    throw err;
+  }
+};