chore: temporal slack alerting [IN-904] (#1574)

Signed-off-by: Joana Maia <jmaia@contractor.linuxfoundation.org>

Author: joanagmaia

submodules/crowd.dev

@@ -16,6 +16,7 @@
     "@crowd/temporal": "workspace:*",
     "@crowd/telemetry": "workspace:*",
     "@crowd/logging": "workspace:*",
+    "@crowd/slack": "workspace:*",
     "@temporalio/worker": "~1.11.1",
     "@temporalio/activity": "~1.11.1",
     "@temporalio/workflow": "~1.11.1"

workers/temporal/base/package.json

@@ -1,7 +1,11 @@
 // Copyright (c) 2025 The Linux Foundation and each contributor.
 // SPDX-License-Identifier: MIT
+import { getServiceChildLogger } from '@crowd/logging'
+import { SlackChannel, SlackPersona, sendSlackNotificationAsync } from '@crowd/slack'
 import telemetry from '@crowd/telemetry'
 
+const log = getServiceChildLogger('activity-interceptor')
+
 async function telemetryDistribution(
   name: string,
   value: number,
@@ -10,4 +14,15 @@ async function telemetryDistribution(
   telemetry.distribution(name, value, tags)
 }
 
-export { telemetryDistribution }
+async function slackNotify(message: string, persona: SlackPersona) {
+  // Accept string to allow workflow code to pass string literals without importing enum
+  await sendSlackNotificationAsync(
+    SlackChannel.ALERTS,
+    persona as SlackPersona,
+    'Temporal Alert',
+    message,
+  )
+  log.info('Slack notification sent from Temporal activity')
+}
+
+export { telemetryDistribution, slackNotify }

workers/temporal/base/src/activities/index.ts

@@ -1,6 +1,8 @@
 // Copyright (c) 2025 The Linux Foundation and each contributor.
 // SPDX-License-Identifier: MIT
 import {
+  ActivityFailure,
+  ApplicationFailure,
   Next,
   WorkflowExecuteInput,
   WorkflowInboundCallsInterceptor,
@@ -10,10 +12,41 @@ import {
 
 import * as activities from './activities'
 
+// Use string literal instead of enum to avoid bundling @slack/webhook in workflow code
+// The activity will handle the type conversion
+const SLACK_PERSONA_ERROR_REPORTER = 'ERROR_REPORTER' as const
+
 const activity = proxyActivities<typeof activities>({
   startToCloseTimeout: '10 seconds',
 })
 
+/**
+ * Extract detailed error information when an activity reaches retry limit
+ */
+function getActivityRetryLimitDetails(err: ActivityFailure): string {
+  let details = `*Activity:* \`${err.activityType}\`\n`
+  details += `*Activity ID:* \`${err.activityId || 'N/A'}\`\n`
+  details += `*Retry State:* ${err.retryState}\n\n`
+
+  // Get the root cause error message and type
+  if (err.cause) {
+    details += `*Error:* ${err.cause.message}\n`
+
+    // If it's an ApplicationFailure, get the type (e.g., AxiosError)
+    if (err.cause instanceof ApplicationFailure && err.cause.type) {
+      details += `*Error Type:* ${err.cause.type}\n`
+    }
+
+    // Add stack trace (first 10 lines for context)
+    if (err.cause.stack) {
+      const stackLines = err.cause.stack.split('\n').slice(0, 10)
+      details += `\n*Stack Trace (first 10 lines):*\n\`\`\`\n${stackLines.join('\n')}\n\`\`\``
+    }
+  }
+
+  return details
+}
+
 export class WorkflowMonitoringInterceptor implements WorkflowInboundCallsInterceptor {
   async execute(
     input: WorkflowExecuteInput,
@@ -33,6 +66,26 @@ export class WorkflowMonitoringInterceptor implements WorkflowInboundCallsInterc
     try {
       const result = await next(input)
       return result
+    } catch (err) {
+      // Type guard to ensure err is an Error object
+      const error = err instanceof Error ? err : new Error(String(err))
+
+      if (error.message !== 'Workflow continued as new') {
+        // Only send detailed notification if it's an activity that reached retry limit
+        if (err instanceof ActivityFailure && err.retryState === 'MAXIMUM_ATTEMPTS_REACHED') {
+          const errorDetails = getActivityRetryLimitDetails(err)
+          const message = `*Workflow Failed: Activity Retry Limit Reached*\n\n*Workflow:* \`${info.workflowType}\`\n*Workflow ID:* \`${info.workflowId}\`\n*Run ID:* \`${info.runId}\`\n\n${errorDetails}`
+
+          await activity.slackNotify(message, SLACK_PERSONA_ERROR_REPORTER)
+        } else {
+          // For other errors, send a simpler notification
+          const message = `*Workflow Failed*\n\n*Workflow:* \`${info.workflowType}\`\n*Workflow ID:* \`${info.workflowId}\`\n*Run ID:* \`${info.runId}\`\n*Error:* ${error.message}`
+
+          await activity.slackNotify(message, SLACK_PERSONA_ERROR_REPORTER)
+        }
+      }
+
+      throw err
     } finally {
       const end = new Date()
       const duration = end.getTime() - start.getTime()