fix: service healthchecker checks all configured DAL and verifier endpoints for issues

Author: kuco23

packages/fasset-bots-core/src/actors/service-health-checker.ts

@@ -24,30 +24,41 @@ async function checkNativeRpcHealth(): Promise<string | undefined> {
 async function checkIndexerHealth(
     indexer: BlockchainIndexerHelper
 ): Promise<string | undefined> {
-    try {
-        const ok = await indexer.getHealth()
-        if (!ok) {
-            return 'health check failed'
+    const msgs: string[] = []
+    for (const [i, client] of indexer.verifier.clients.entries()) {
+        const url = indexer.indexerWebServerUrls[i]
+        try {
+            const healthy = await indexer.getHealth(client)
+            if (!healthy) {
+                msgs.push(`client ${url} is unhealthy`)
+            }
+        } catch (e: any) {
+            msgs.push(`client ${url} connection errored with ${String(e?.message)}`)
         }
-    } catch (e: any) {
-        return String(e?.message)
+    }
+    if (msgs.length > 0) {
+        return msgs.join(', ')
     }
 }
 
+
 async function checkDalHealth(
     fdc: FlareDataConnectorClientHelper
 ): Promise<string | undefined> {
-    try {
-        const healthy = await fdc.getHealth()
-        const n = healthy.filter(h => h).length
-        if (n < fdc.dataAccessLayerClients.length) {
-            const unhealthy = healthy.map((h, i) => h ? '' : fdc.dataAccessLayerUrls[i]).join(',')
-            return `clients ${unhealthy} are healthy`
-        } else if (n == 0) {
-            return 'no clients available'
+    const msgs: string[] = []
+    for (const [i, client] of fdc.dataAccessLayerClients.entries()) {
+        const url = fdc.dataAccessLayerUrls[i]
+        try {
+            const healthy = await fdc.getHealth(client)
+            if (!healthy) {
+                msgs.push(`client ${url} is unhealthy`)
+            }
+        } catch (e: any) {
+            msgs.push(`client ${url} connection errored with ${String(e?.message)}`)
         }
-    } catch (e: any) {
-        return String(e?.message)
+    }
+    if (msgs.length > 0) {
+        return msgs.join(', ')
     }
 }
 
@@ -71,17 +82,20 @@ async function serviceHealthcheck(
     const errors: [string, string][] = []
     for (const c of config.fAssets.values()) {
         const rpcHealth = await checkNativeRpcHealth()
-        const indexerHealth = await checkIndexerHealth(c.blockchainIndexerClient)
-        // @ts-ignore flareDataConnector has the right implementation
-        const dalHealth = await checkDalHealth(c.flareDataConnector)
-        const walletHealth = await checkUnderlyingWalletHealth(c.wallet)
         if (rpcHealth != null) {
             errors.push(['Native RPC', rpcHealth])
-        } else if (indexerHealth != null) {
+        }
+        const indexerHealth = await checkIndexerHealth(c.blockchainIndexerClient)
+        if (indexerHealth != null) {
             errors.push(['FDC verifier', indexerHealth])
-        } else if (dalHealth != null) {
+        }
+        // @ts-ignore flareDataConnector implements FlareDataConnectorClientHelper
+        const dalHealth = await checkDalHealth(c.flareDataConnector)
+        if (dalHealth != null) {
             errors.push(['DAL', dalHealth])
-        } else if (walletHealth != null) {
+        }
+        const walletHealth = await checkUnderlyingWalletHealth(c.wallet)
+        if (walletHealth != null) {
             errors.push(['Underlying RPC', walletHealth])
         }
     }

packages/fasset-bots-core/src/underlying-chain/BlockchainIndexerHelper.ts

@@ -2,6 +2,7 @@ import BN from "bn.js";
 import { formatArgs } from "../utils/formatting";
 import { DEFAULT_RETRIES, prefix0x, requireNotNull, retry, sleep, toBN, ZERO_BYTES32 } from "../utils/helpers";
 import { logger } from "../utils/logger";
+import { HttpApiClient } from "../utils/HttpApiClient";
 import { MultiApiClient } from "../utils/MultiApiClient";
 import { ChainId } from "./ChainId";
 import { IBlock, IBlockChain, IBlockId, ITransaction, TX_BLOCKED, TX_FAILED, TX_SUCCESS, TxInputOutput } from "./interfaces/IBlockChain";
@@ -202,9 +203,14 @@ export class BlockchainIndexerHelper implements IBlockChain {
         return blockHeight;
     }
 
-    async getHealth(): Promise<boolean> {
-        const respdata = await this.verifier.get<ApiWrapper<boolean>>(`/api/health`, "getHealth");
-        return respdata.status == "OK";
+    async getHealth(client?: HttpApiClient): Promise<boolean> {
+        if (client == null) {
+            const respdata = await this.verifier.get<ApiWrapper<boolean>>(`/api/health`, "getHealth");
+            return respdata.status == "OK";
+        } else {
+            const respdata = await client.get<ApiWrapper<boolean>>(`/api/health`, "getHealth", 0);
+            return respdata.status == "OK";
+        }
     }
 
     async getCurrentBlockHeightFromIndexer(): Promise<number> {

packages/fasset-bots-core/src/underlying-chain/FlareDataConnectorClientHelper.ts

@@ -257,11 +257,9 @@ export class FlareDataConnectorClientHelper implements IFlareDataConnectorClient
         return proof;
     }
 
-    async getHealth(): Promise<boolean[]> {
-        const responses = await Promise.all(this.dataAccessLayerClients.map(
-            client => client.get<DalHealthResult>('/api/health', 'getHealth', 0))
-        );
-        return responses.map(r => r.healthy)
+    async getHealth(client: HttpApiClient): Promise<boolean> {
+        const response = await client.get<DalHealthResult>('/api/health', 'getHealth', 0);
+        return response.healthy;
     }
 
     /* istanbul ignore next */