Retry sending emails on transient errors (#1309)

* Retry sending emails on transient errors

* Code style

* Fixes

Author: nygrenh

backend/bin/kafkaConsumer/common/EmailTemplater/sendEmailTemplate.ts

@@ -37,20 +37,24 @@ export async function sendEmailTemplateToUser({
     : undefined
 
   if (context.test) {
-    ;(context.logger?.info || console.log)(
+    const logger = context.logger ?? console
+    logger.info(
       `To: ${
         email ?? user.email
       }\nSubject: ${title}\nText: ${text}\nHtml: ${html_body}`,
     )
 
     return
   }
-  await sendMail({
-    to: email ?? user.email,
-    subject: emptyOrNullToUndefined(title),
-    text,
-    html: emptyOrNullToUndefined(html_body),
-  })
+  await sendMail(
+    {
+      to: email ?? user.email,
+      subject: emptyOrNullToUndefined(title),
+      text,
+      html: emptyOrNullToUndefined(html_body),
+    },
+    { logger: context.logger },
+  )
 }
 
 interface ApplyTemplateArgs {

backend/util/sendMail.ts

@@ -31,7 +31,8 @@ export async function sendMail(
   { to, text, subject, html }: SendMailOptions,
   context?: SendMailContext,
 ) {
-  const { logger } = context ?? {}
+  const logger = context?.logger
+  const logInfo = logger?.info.bind(logger) ?? console.log
 
   const options: SMTPTransport.Options = {
     host: SMTP_HOST,
@@ -41,17 +42,103 @@ export async function sendMail(
       user: SMTP_USER, // generated ethereal user
       pass: SMTP_PASS, // generated ethereal password
     },
+    connectionTimeout: 15000,
+    greetingTimeout: 15000,
+    socketTimeout: 30000,
   }
 
-  const transporter = createTransport(options)
-
-  const info = await transporter.sendMail({
-    from: SMTP_FROM, // sender address
-    to, // list of receivers
-    subject, // Subject line
-    text, // plain text body
-    html, // html body
+  await withRetries({
+    maxRetries: 3,
+    logInfo,
+    operationName: "SMTP send",
+    isTransientError: isTransientSmtpError,
+    operation: async (attempt) => {
+      const transporter = createTransport(options)
+      try {
+        const info = await transporter.sendMail({
+          from: SMTP_FROM, // sender address
+          to, // list of receivers
+          subject, // Subject line
+          text, // plain text body
+          html, // html body
+        })
+        logInfo(
+          `Message sent: ${info.messageId}${
+            attempt > 1 ? ` (succeeded on attempt ${attempt})` : ""
+          }`,
+        )
+      } finally {
+        transporter.close()
+      }
+    },
   })
-  ;(logger?.info ?? console.log)(`Message sent: ${info.messageId}`)
-  // Message sent: <[email protected]>
+}
+
+interface RetryOptions {
+  maxRetries: number
+  logInfo?: (message: string) => void
+  operationName: string
+  operation: (attempt: number) => Promise<void>
+  isTransientError: (error: unknown) => boolean
+}
+
+async function withRetries({
+  maxRetries,
+  logInfo,
+  operationName,
+  operation,
+  isTransientError,
+}: RetryOptions) {
+  const BASE_DELAY_MS = 1000
+  const jitter = () => Math.floor(Math.random() * 250)
+  const log = logInfo ?? ((message: string) => console.log(message))
+
+  for (let attempt = 1; attempt <= maxRetries; attempt++) {
+    try {
+      await operation(attempt)
+      return
+    } catch (error: any) {
+      if (attempt >= maxRetries || !isTransientError(error)) {
+        throw error
+      }
+      log(
+        `${operationName} failed (attempt ${attempt}/${maxRetries}, retrying): ${
+          error?.message ?? error
+        }`,
+      )
+      const delay =
+        Math.min(BASE_DELAY_MS * 2 ** (attempt - 1), 8000) + jitter()
+      await new Promise((resolve) => setTimeout(resolve, delay))
+    }
+  }
+}
+
+function isTransientSmtpError(error: unknown) {
+  interface SmtpError {
+    responseCode?: number
+    response?: string
+    message?: string
+    code?: string
+  }
+  const smtpError = error as SmtpError
+  const responseCode = Number(smtpError?.responseCode)
+  if (responseCode >= 400 && responseCode < 500) {
+    return true
+  }
+  const response = String(smtpError?.response ?? "")
+  if (/^4\d\d/.test(response)) {
+    return true
+  }
+  const message = String(smtpError?.message ?? "")
+  if (/\b4\d\d/.test(message)) {
+    return true
+  }
+  const code = String(smtpError?.code ?? "")
+  return [
+    "ECONNRESET",
+    "ETIMEDOUT",
+    "ESOCKET",
+    "EPIPE",
+    "ECONNREFUSED",
+  ].includes(code)
 }