feat(balances): Update write alert scripts with helpul prompts (#5628)

### Description

- Updates the write alert scripts with helpful prompts and general QoL
improvements
- Checks if the proposed change introduces a regression i.e remove a
chain from the existing alert, logs an error and aborts
- Read the actual alert threshold via grafana api, compares with
proposed thresholds, prompts the user to confirm that they want to
introduce these changes

![image](https://github.com/user-attachments/assets/dc61b247-d131-4b07-b6c5-634094865e5f)
- Check if proposed update will lead to alert firing and prompts the
user to confirm with these changes

![image](https://github.com/user-attachments/assets/eb9972b1-3a7f-401e-9ae5-33d234da20bc)

### Testing

Manual

Author: Mo-Hussain

typescript/infra/scripts/funding/calculate-relayer-daily-burn.ts

@@ -20,6 +20,8 @@ import {
   sortThresholds,
 } from '../../src/funding/balances.js';
 import {
+  LOCAL_PROM_URL,
+  PROMETHEUS_LOCAL_PORT,
   PrometheusInstantResult,
   fetchPrometheusInstantExpression,
   portForwardPrometheusServer,
@@ -38,7 +40,6 @@ const SCRAPER_READ_ONLY_DB_SECRET_NAME =
 
 const LOOK_BACK_DAYS = 10; // the number of days to look back for average destination tx costs
 const MIN_NUMBER_OF_TXS = 100; // the minimum number of txs to consider for daily burn
-const PROMETHEUS_LOCAL_PORT = 9090;
 const MIN_BURN_INCREASE_FACTOR = 0.05; // burn should be at least 5% higher than current to be updated
 const LOW_PROPOSED_BURN_FACTOR = 0.5; // proposed burn should be at least 50% lower than current to initiate user review
 
@@ -146,8 +147,6 @@ async function getSealevelBurnProm(
     PROMETHEUS_LOCAL_PORT,
   );
 
-  const promUrl = `http://localhost:${PROMETHEUS_LOCAL_PORT}`;
-
   const burn: ChainMap<number> = {};
 
   const rangeHours = LOOK_BACK_DAYS * 24;
@@ -186,7 +185,10 @@ async function getSealevelBurnProm(
   let results: PrometheusInstantResult[];
 
   try {
-    results = await fetchPrometheusInstantExpression(promUrl, promQlQuery);
+    results = await fetchPrometheusInstantExpression(
+      LOCAL_PROM_URL,
+      promQlQuery,
+    );
   } finally {
     portForwardProcess.kill();
     rootLogger.info('Prometheus server port-forward process killed');

typescript/infra/scripts/funding/write-alert.ts

@@ -1,3 +1,6 @@
+import { confirm } from '@inquirer/prompts';
+import { ChildProcess } from 'child_process';
+
 import { ChainMap } from '@hyperlane-xyz/sdk';
 import { rootLogger } from '@hyperlane-xyz/utils';
 
@@ -9,66 +12,295 @@ import {
 } from '../../src/config/funding/balances.js';
 import {
   AlertType,
+  ProvisionedAlertRule,
   alertConfigMapping,
 } from '../../src/config/funding/grafanaAlerts.js';
+import { parseBalancesPromQLQuery } from '../../src/funding/alerts.js';
 import { validateThresholds } from '../../src/funding/balances.js';
 import {
   fetchGrafanaAlert,
   fetchGrafanaServiceAccountToken,
   generateQuery,
   updateGrafanaAlert,
 } from '../../src/infrastructure/monitoring/grafana.js';
+import {
+  LOCAL_PROM_URL,
+  PROMETHEUS_LOCAL_PORT,
+  fetchPrometheusInstantExpression,
+  portForwardPrometheusServer,
+} from '../../src/infrastructure/monitoring/prometheus.js';
 import { readJSONAtPath } from '../../src/utils/utils.js';
 
+interface AlertUpdateInfo {
+  alertType: AlertType;
+  grafanaAlertId: string;
+  provisionedAlertRule: ProvisionedAlertRule;
+  query: string;
+}
+
+interface RegressionError {
+  alertType: AlertType;
+  missingChains: string[];
+}
+
 async function main() {
+  // runs a validation check to ensure the threshold configs are valid relative to each other
+  await validateBalanceThresholdConfigs();
+
   const saToken = await fetchGrafanaServiceAccountToken();
+  const portForwardProcess = await portForwardPrometheusServer(
+    PROMETHEUS_LOCAL_PORT,
+  );
+
+  const alertsToUpdate = Object.values(AlertType);
+  const alertUpdateInfo: AlertUpdateInfo[] = [];
+  const missingChainErrors: RegressionError[] = [];
+
+  try {
+    for (const alert of alertsToUpdate) {
+      // fetch alertRule config from Grafana via the Grafana API
+      const alertRule = await fetchGrafanaAlert(alert, saToken);
+
+      // read the proposed thresholds from the config file
+      let proposedThresholds: ChainMap<number> = {};
+      try {
+        proposedThresholds = readJSONAtPath(
+          `${THRESHOLD_CONFIG_PATH}/${alertConfigMapping[alert].configFileName}`,
+        );
+      } catch (e) {
+        rootLogger.error(`Error reading ${alert} config: ${e}`);
+        process.exit(1);
+      }
+
+      // parse the current thresholds from the existing query
+      const existingQuery = alertRule.queries[0];
+      const currentThresholds = parseBalancesPromQLQuery(
+        existingQuery,
+        alertConfigMapping[alert].walletName,
+      );
+
+      // log an error if a chain is defined in current thresholds but not in the proposed thresholds
+      // this is to ensure that we don't introduce a regression where a chain is no longer being monitored
+      const missingChains = Object.keys(currentThresholds).filter(
+        (chain) => !proposedThresholds[chain],
+      );
+      if (missingChains.length > 0) {
+        missingChainErrors.push({
+          alertType: alert,
+          missingChains,
+        });
+        rootLogger.error(
+          `Missing thresholds for chains: ${missingChains.join(
+            ', ',
+          )} for ${alert} config, skipping updating this alert`,
+        );
+        continue;
+      }
+
+      // generate a table of the differences in the thresholds, prompt the user to confirm the changes
+      const diffTable = generateDiffTable(
+        currentThresholds,
+        proposedThresholds,
+      );
+      if (diffTable.length > 0) {
+        rootLogger.info(`Differences in ${alert} thresholds:`);
+        console.table(diffTable);
+
+        const confirmed = await confirm({
+          message: `Do you want to update thresholds for ${alert}?`,
+        });
+
+        if (!confirmed) {
+          rootLogger.info(
+            `Exiting without updating any alerts, this is to avoid thresholds from being out of sync`,
+          );
+          process.exit(0);
+        }
+      } else {
+        rootLogger.info(
+          `Proposed thresholds for ${alert} are the same as existing thresholds, skipping`,
+        );
+        continue;
+      }
+
+      // prompt the user to confirm that they are ok with the alert firing for chains after the update
+      const query = generateQuery(alert, proposedThresholds);
+      await confirmFiringAlerts(
+        alert,
+        query,
+        currentThresholds,
+        proposedThresholds,
+      );
+
+      alertUpdateInfo.push({
+        alertType: alert,
+        grafanaAlertId: alertConfigMapping[alert].grafanaAlertId,
+        provisionedAlertRule: alertRule.rawData,
+        query,
+      });
+    }
+
+    // abort if there are any missing thresholds in the config to avoid introducing a regression
+    handleMissingChainErrors(missingChainErrors);
+
+    // update the alerts with the new thresholds via the Grafana API
+    await updateAlerts(alertUpdateInfo, saToken, portForwardProcess);
+  } finally {
+    portForwardProcess.kill();
+  }
+}
 
+async function validateBalanceThresholdConfigs() {
   const balanceThresholdTypes = Object.values(BalanceThresholdType);
-  const balanceThresholdConfigs: ThresholdsData = balanceThresholdTypes.reduce(
+  const balanceThresholdConfigs = balanceThresholdTypes.reduce(
     (acc, balanceThresholdType) => {
       const thresholds = readJSONAtPath(
         `${THRESHOLD_CONFIG_PATH}/${balanceThresholdConfigMapping[balanceThresholdType].configFileName}`,
       ) as ChainMap<string>;
 
       return {
         ...acc,
-        [balanceThresholdType]: {
-          thresholds,
-        },
+        [balanceThresholdType]: thresholds,
       };
     },
     {} as ThresholdsData,
   );
 
   validateThresholds(balanceThresholdConfigs);
+}
 
-  const alertsToUpdate = Object.values(AlertType);
+async function fetchFiringThresholdAlert(query: string): Promise<string[]> {
+  const results = await fetchPrometheusInstantExpression(LOCAL_PROM_URL, query);
+
+  const alertingChains: string[] = [];
+
+  for (const series of results) {
+    const chain = series.metric.chain;
+
+    if (series.value && parseFloat(series.value[1]) < 0) {
+      alertingChains.push(chain);
+    } else if (series.histogram) {
+      rootLogger.warn(
+        `Unexpected histogram data found for "${chain} in Prometheus, skipping.`,
+      );
+    }
+  }
+
+  return alertingChains;
+}
 
-  for (const alert of alertsToUpdate) {
-    // fetch alertRule config from Grafana
-    const alertRule = await fetchGrafanaAlert(alert, saToken);
+async function updateAlerts(
+  alertUpdateInfo: AlertUpdateInfo[],
+  saToken: string,
+  portForwardProcess: ChildProcess,
+) {
+  // sort alertUpdateInfo by alertConfigMapping writePriority in descending order
+  // the intention is to update alerts with higher writePriority first
+  // if there are any errors, we don't want to continue updating alert thresholds with lower writePriority
+  // to avoid the thresholds being out of sync, this is only effective when we are increasing thresholds which is the most common case
+  alertUpdateInfo.sort(
+    (a, b) =>
+      alertConfigMapping[b.alertType].writePriority -
+      alertConfigMapping[a.alertType].writePriority,
+  );
 
-    let thresholds: ChainMap<string> = {};
+  for (const alertInfo of alertUpdateInfo) {
     try {
-      thresholds = readJSONAtPath(
-        `${THRESHOLD_CONFIG_PATH}/${alertConfigMapping[alert].configFileName}`,
+      await updateGrafanaAlert(
+        alertInfo.grafanaAlertId,
+        alertInfo.provisionedAlertRule,
+        alertInfo.query,
+        saToken,
       );
+      rootLogger.info(`Updated ${alertInfo.alertType} alert`);
     } catch (e) {
-      rootLogger.error(`Error reading ${alert} config: ${e}`);
+      rootLogger.error(
+        `Error updating ${alertInfo.alertType} alert, aborting updating the rest of the alerts: ${e}`,
+      );
+      // exiting here so we don't continue updating alerts with lower writePriority
+      portForwardProcess.kill();
       process.exit(1);
     }
+  }
+}
 
-    const query = generateQuery(alert, thresholds);
+function generateDiffTable(
+  currentThresholds: ChainMap<number>,
+  proposedThresholds: ChainMap<number>,
+) {
+  const diffTable = Object.entries(proposedThresholds).reduce(
+    (acc, [chain, newThreshold]) => {
+      const currentThreshold = currentThresholds[chain];
+      if (currentThreshold !== proposedThresholds[chain]) {
+        acc.push({
+          chain,
+          current: currentThreshold,
+          new: newThreshold,
+          change:
+            currentThreshold === undefined
+              ? 'new'
+              : currentThreshold < newThreshold
+              ? 'increase'
+              : 'decrease',
+        });
+      }
+      return acc;
+    },
+    [] as {
+      chain: string;
+      current: number;
+      new: number;
+      change: 'increase' | 'decrease' | 'new';
+    }[],
+  );
+
+  return diffTable;
+}
 
-    // only change the query
-    await updateGrafanaAlert(
-      alertConfigMapping[alert].grafanaAlertId,
-      alertRule.rawData,
-      query,
-      saToken,
+function handleMissingChainErrors(missingChainErrors: RegressionError[]) {
+  if (missingChainErrors.length === 0) return;
+
+  for (const error of missingChainErrors) {
+    rootLogger.error(
+      `Missing thresholds for chains: ${error.missingChains.join(', ')} for ${
+        error.alertType
+      } config`,
     );
+  }
+  rootLogger.error(
+    `Aborting updating alerts due to missing thresholds in config`,
+  );
+  process.exit(1);
+}
 
-    rootLogger.info(`Updated ${alert} alert`);
+async function confirmFiringAlerts(
+  alert: AlertType,
+  query: string,
+  currentThresholds: ChainMap<number>,
+  proposedThresholds: ChainMap<number>,
+) {
+  const alertingChains = await fetchFiringThresholdAlert(query);
+  if (alertingChains.length === 0) return;
+
+  rootLogger.warn(
+    `updating ${alert} alert will result in alerting for the following chains`,
+  );
+  console.table(
+    alertingChains.map((chain) => ({
+      chain,
+      current: currentThresholds[chain],
+      proposed: proposedThresholds[chain],
+    })),
+  );
+
+  const confirmed = await confirm({
+    message: `Do you want to proceed with updating the alert thresholds for ${alert}?`,
+  });
+  if (!confirmed) {
+    rootLogger.info(
+      `Exiting without updating any alerts, this is to avoid thresholds from being out of sync as we do not want to update the ${alert} alert`,
+    );
+    process.exit(0);
   }
 }