Handle healthcheck errors gracefully

lib/plugins/healthcheck/commonHealthcheckPlugin.spec.ts

@@ -14,6 +14,9 @@ const positiveHealthcheckChecker: HealthChecker = () => {
 const negativeHealthcheckChecker: HealthChecker = () => {
   return Promise.resolve({ error: new Error('Something exploded') })
 }
+const throwingHealthcheckChecker: HealthChecker = () => {
+  throw new Error('Connection refused')
+}
 
 async function initApp(opts: CommonHealthcheckPluginOptions) {
   const app = fastify()
@@ -317,5 +320,34 @@ describe('commonHealthcheckPlugin', () => {
         ],
       })
     })
+
+    it('handles checker that throws an error as a failed healthcheck', async () => {
+      app = await initApp({
+        responsePayload: { version: 1 },
+        healthChecks: [
+          {
+            name: 'check1',
+            isMandatory: false,
+            checker: throwingHealthcheckChecker,
+          },
+          {
+            name: 'check2',
+            isMandatory: true,
+            checker: positiveHealthcheckChecker,
+          },
+        ],
+      })
+
+      const response = await app.inject().get(PRIVATE_ENDPOINT).end()
+      expect(response.statusCode).toBe(200)
+      expect(response.json()).toEqual({
+        heartbeat: 'PARTIALLY_HEALTHY',
+        version: 1,
+        checks: {
+          check1: 'FAIL',
+          check2: 'HEALTHY',
+        },
+      })
+    })
   })
 })

lib/plugins/healthcheck/commonHealthcheckPlugin.ts

@@ -1,4 +1,4 @@
-import type { Either } from '@lokalise/node-core'
+import { type Either, isError } from '@lokalise/node-core'
 import type { FastifyPluginCallback } from 'fastify'
 import fp from 'fastify-plugin'
 import type { AnyFastifyInstance } from '../pluginsCommon.js'
@@ -97,7 +97,12 @@ function addRoute(
       if (opts.healthChecks.length) {
         const results = await Promise.all(
           opts.healthChecks.map(async (healthcheck) => {
-            const result = await healthcheck.checker(app)
+            let result: Either<Error, true>
+            try {
+              result = await healthcheck.checker(app)
+            } catch (err) {
+              result = { error: isError(err) ? err : new Error(String(err)) }
+            }
             if (result.error) {
               app.log.error(result.error, `${healthcheck.name} healthcheck has failed`)
             }

lib/plugins/healthcheck/commonSyncHealthcheckPlugin.spec.ts

@@ -14,6 +14,9 @@ const positiveHealthcheckChecker: HealthCheckerSync = () => {
 const negativeHealthcheckChecker: HealthCheckerSync = () => {
   return new Error('Something exploded')
 }
+const throwingHealthcheckChecker: HealthCheckerSync = () => {
+  throw new Error('Connection refused')
+}
 
 async function initApp(opts: CommonSyncHealthcheckPluginOptions) {
   const app = fastify()
@@ -317,5 +320,34 @@ describe('commonSyncHealthcheckPlugin', () => {
         ],
       })
     })
+
+    it('handles checker that throws an error as a failed healthcheck', async () => {
+      app = await initApp({
+        responsePayload: { version: 1 },
+        healthChecks: [
+          {
+            name: 'check1',
+            isMandatory: false,
+            checker: throwingHealthcheckChecker,
+          },
+          {
+            name: 'check2',
+            isMandatory: true,
+            checker: positiveHealthcheckChecker,
+          },
+        ],
+      })
+
+      const response = await app.inject().get(PRIVATE_ENDPOINT).end()
+      expect(response.statusCode).toBe(200)
+      expect(response.json()).toEqual({
+        heartbeat: 'PARTIALLY_HEALTHY',
+        version: 1,
+        checks: {
+          check1: 'FAIL',
+          check2: 'HEALTHY',
+        },
+      })
+    })
   })
 })

lib/plugins/healthcheck/commonSyncHealthcheckPlugin.ts

@@ -1,3 +1,4 @@
+import { isError } from '@lokalise/node-core'
 import type { FastifyPluginCallback } from 'fastify'
 import fp from 'fastify-plugin'
 import type { AnyFastifyInstance } from '../pluginsCommon.js'
@@ -95,7 +96,12 @@ function addRoute(
 
       if (opts.healthChecks.length) {
         const results = opts.healthChecks.map((healthcheck) => {
-          const healthcheckError = healthcheck.checker(app)
+          let healthcheckError: Error | null
+          try {
+            healthcheckError = healthcheck.checker(app)
+          } catch (err) {
+            healthcheckError = isError(err) ? err : new Error(String(err))
+          }
           if (healthcheckError) {
             app.log.error(healthcheckError, `${healthcheck.name} healthcheck has failed`)
           }

lib/plugins/healthcheck/publicHealthcheckPlugin.spec.ts

@@ -11,6 +11,9 @@ const positiveHealthcheckChecker: HealthChecker = () => {
 const negativeHealthcheckChecker: HealthChecker = () => {
   return Promise.resolve({ error: new Error('Something exploded') })
 }
+const throwingHealthcheckChecker: HealthChecker = () => {
+  throw new Error('Connection refused')
+}
 
 async function initApp(opts: PublicHealthcheckPluginOptions) {
   const app = fastify()
@@ -180,4 +183,33 @@ describe('publicHealthcheckPlugin', () => {
       ],
     })
   })
+
+  it('handles checker that throws an error as a failed healthcheck', async () => {
+    app = await initApp({
+      responsePayload: { version: 1 },
+      healthChecks: [
+        {
+          name: 'check1',
+          isMandatory: false,
+          checker: throwingHealthcheckChecker,
+        },
+        {
+          name: 'check2',
+          isMandatory: true,
+          checker: positiveHealthcheckChecker,
+        },
+      ],
+    })
+
+    const response = await app.inject().get('/health').end()
+    expect(response.statusCode).toBe(200)
+    expect(response.json()).toEqual({
+      heartbeat: 'PARTIALLY_HEALTHY',
+      version: 1,
+      checks: {
+        check1: 'FAIL',
+        check2: 'HEALTHY',
+      },
+    })
+  })
 })

lib/plugins/healthcheck/publicHealthcheckPlugin.ts

@@ -1,3 +1,4 @@
+import { type Either, isError } from '@lokalise/node-core'
 import type { FastifyPluginCallback } from 'fastify'
 import fp from 'fastify-plugin'
 import type { AnyFastifyInstance } from '../pluginsCommon.js'
@@ -45,17 +46,21 @@ function plugin(
 
       if (opts.healthChecks.length) {
         const results = await Promise.all(
-          opts.healthChecks.map((healthcheck) => {
-            return healthcheck.checker(app).then((result) => {
-              if (result.error) {
-                app.log.error(result.error, `${healthcheck.name} healthcheck has failed`)
-              }
-              return {
-                name: healthcheck.name,
-                result,
-                isMandatory: healthcheck.isMandatory,
-              }
-            })
+          opts.healthChecks.map(async (healthcheck) => {
+            let result: Either<Error, true>
+            try {
+              result = await healthcheck.checker(app)
+            } catch (err) {
+              result = { error: isError(err) ? err : new Error(String(err)) }
+            }
+            if (result.error) {
+              app.log.error(result.error, `${healthcheck.name} healthcheck has failed`)
+            }
+            return {
+              name: healthcheck.name,
+              result,
+              isMandatory: healthcheck.isMandatory,
+            }
           }),
         )
 

lib/plugins/healthcheck/startupHealthcheckPlugin.spec.ts

@@ -14,6 +14,9 @@ const positiveHealthcheckChecker: HealthChecker = () => {
 const negativeHealthcheckChecker: HealthChecker = () => {
   return Promise.resolve({ error: new Error('Something exploded') })
 }
+const throwingHealthcheckChecker: HealthChecker = () => {
+  throw new Error('Connection refused')
+}
 
 async function initApp(opts: StartupHealthcheckPluginOptions) {
   const app = fastify()
@@ -85,5 +88,36 @@ describe('startupHealthcheckPlugin', () => {
         ],
       })
     })
+
+    it('handles checker that throws as a failed healthcheck', async () => {
+      await expect(
+        initApp({
+          healthChecks: [
+            {
+              name: 'check1',
+              isMandatory: true,
+              checker: throwingHealthcheckChecker,
+            },
+          ],
+        }),
+      ).rejects.toThrowErrorMatchingInlineSnapshot(`[Error: Healthchecks failed: ["check1"]]`)
+    })
+
+    it('app starts if throwing checker is optional', async () => {
+      app = await initApp({
+        healthChecks: [
+          {
+            name: 'check1',
+            isMandatory: false,
+            checker: throwingHealthcheckChecker,
+          },
+          {
+            name: 'check2',
+            isMandatory: true,
+            checker: positiveHealthcheckChecker,
+          },
+        ],
+      })
+    })
   })
 })

lib/plugins/healthcheck/startupHealthcheckPlugin.ts

@@ -1,7 +1,20 @@
+import { type Either, isError } from '@lokalise/node-core'
 import type { FastifyPluginCallback } from 'fastify'
 import fp from 'fastify-plugin'
 import { stdSerializers } from 'pino'
 import { type HealthCheck, resolveHealthcheckResults } from './commonHealthcheckPlugin.js'
+import type { HealthChecker } from './healthcheckCommons.js'
+
+async function executeHealthCheck(
+  checker: HealthChecker,
+  app: Parameters<HealthChecker>[0],
+): Promise<Either<Error, true>> {
+  try {
+    return await checker(app)
+  } catch (err) {
+    return { error: isError(err) ? err : new Error(String(err)) }
+  }
+}
 
 export interface StartupHealthcheckPluginOptions {
   resultsLogLevel?: 'fatal' | 'error' | 'warn' | 'info' | 'debug' | 'trace' | 'silent'
@@ -18,7 +31,7 @@ const plugin: FastifyPluginCallback<StartupHealthcheckPluginOptions> = (app, opt
     if (opts.healthChecks.length) {
       const results = await Promise.all(
         opts.healthChecks.map(async (healthcheck) => {
-          const result = await healthcheck.checker(app)
+          const result = await executeHealthCheck(healthcheck.checker, app)
           if (result.error) {
             app.log.error(
               {