[ENG-3665] Fix AWS source dying when a single region times out (#4670)

## Summary

- **Hotfix**: A single unreachable AWS region (e.g. `me-south-1` being
decommissioned) was causing the entire AWS source to stay permanently
unready, because the STS timeout error wasn't handled gracefully.
- **Assistant awareness**: The explore assistant now knows when sources
are unhealthy and tells users results may be incomplete, instead of
claiming resources don't exist.

## Linear Ticket

- **Ticket**:
[ENG-3665](https://linear.app/overmind/issue/ENG-3665/aws-source-is-dead-when-a-single-region-times-out)
— AWS source is dead when a single region times out
- **Purpose**: Prevent a single dead region from taking down the entire
AWS source, and surface health degradation to users via the assistant.

## Changes

### 1. `aws-source/proc/proc.go` + `proc_test.go`

Expanded `isOptInRegionError` to also match `context.DeadlineExceeded`
and `context.Canceled`. When a region times out on the STS
`GetCallerIdentity` call, it is now skipped (with a warning log) instead
of failing the entire source initialization. Four new test cases cover
bare and wrapped timeout/cancellation errors.

### 2. `services/gateway/service/assistant.go` + related files

`setupTools` now returns a `setupToolsResult` struct containing both the
tool list and a health summary string. `buildSourceHealthSummary`
iterates all sources and formats a markdown section listing any that
aren't healthy (with name, type, status, and error message). This is
appended to the system prompt when creating the LLM conversation, so the
assistant can proactively inform users about degraded sources.

**Reviewers should focus on**: the error-matching logic in
`isOptInRegionError` (is `DeadlineExceeded`/`Canceled` the right
scope?), and the system prompt injection wording in
`buildSourceHealthSummary`.

Made with [Cursor](https://cursor.com)

<!-- CURSOR_SUMMARY -->
---

> [!NOTE]
> **Medium Risk**
> Changes AWS source initialization to skip regions on STS timeouts (and
opt-in/OIDC errors) instead of failing, which could mask genuine
regional issues if misclassified. Also injects source health into the
assistant system prompt, so prompt-sanitization and wording correctness
matter.
>
> **Overview**
> Prevents the AWS source from getting stuck unready when a single
region can’t respond: STS `GetCallerIdentity` failures that are
*timeouts* (and existing opt-in/OIDC failures) are now treated as
**skippable**, recorded, and initialization continues for the remaining
regions (with improved log messaging and an OTel
`ovm.adapter.regionSkipped` event).
>
> Makes the Explore assistant **aware of degraded sources** by changing
`setupTools` to return both the tool list and a generated “Source Health
Warnings” block, sanitising source-provided strings to reduce
prompt-injection risk, and appending that health summary to the
assistant system prompt; tests were added/updated to cover the new
timeout/skip logic and prompt sanitisation/summary generation.
>
> <sup>Reviewed by [Cursor Bugbot](https://cursor.com/bugbot) for commit
e6f9e3241f554ea9e69b5b0907dd83d83323a454. Bugbot is set up for automated
code reviews on this repo. Configure
[here](https://www.cursor.com/dashboard/bugbot).</sup>
<!-- /CURSOR_SUMMARY -->

---------

Co-authored-by: Cursor Agent <cursoragent@cursor.com>
Co-authored-by: David Schmitt <DavidS-ovm@users.noreply.github.com>
GitOrigin-RevId: c195aaa468142d0a75206c7ef5d8948066c8e34a

Author: 2 people

aws-source/proc/proc.go

@@ -46,6 +46,8 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/viper"
 	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
 )
 
 // This package contains a few functions needed by the CLI to load this in-proc.
@@ -82,20 +84,23 @@ func ConfigFromViper() ([]aws.Config, error) {
 	return CreateAWSConfigs(authConfig)
 }
 
-// isOptInRegionError checks if an error indicates an opt-in region that is not enabled.
-// This typically occurs when trying to authenticate with IRSA in a region that hasn't
-// been enabled in the AWS account. These errors should not cause source initialization
-// to fail - the region should simply be skipped.
+// isTimeoutError checks if an error is a context deadline exceeded.
+// A single unresponsive region (e.g. me-south-1 being decommissioned) must
+// not take down the whole source — see ENG-3665.
+func isTimeoutError(err error) bool {
+	return err != nil && errors.Is(err, context.DeadlineExceeded)
+}
+
+// isOptInRegionError checks if an error indicates an opt-in region that is not
+// enabled in the AWS account (InvalidIdentityToken + OIDC).
 func isOptInRegionError(err error) bool {
 	if err == nil {
 		return false
 	}
 
-	// Check for the InvalidIdentityToken error code from STS AssumeRoleWithWebIdentity
 	var apiErr smithy.APIError
 	if errors.As(err, &apiErr) {
 		if apiErr.ErrorCode() == "InvalidIdentityToken" {
-			// Additional validation: check if it's specifically about OIDC provider
 			errMsg := err.Error()
 			if strings.Contains(errMsg, "No OpenIDConnect provider found") {
 				return true
@@ -106,13 +111,22 @@ func isOptInRegionError(err error) bool {
 	return false
 }
 
+// isSkippableRegionError checks if an error indicates a region that cannot be
+// reached and should be skipped rather than failing the entire source.
+func isSkippableRegionError(err error) bool {
+	return isTimeoutError(err) || isOptInRegionError(err)
+}
+
 // wrapRegionError wraps misleading AWS errors with more helpful context
 func wrapRegionError(err error, region string) error {
 	if err == nil {
 		return nil
 	}
 
-	// Check for opt-in region errors and provide helpful context
+	if isTimeoutError(err) {
+		return fmt.Errorf("%w. Region '%s' is unreachable (timeout); it may be decommissioned or experiencing an outage", err, region)
+	}
+
 	if isOptInRegionError(err) {
 		return fmt.Errorf("%w. This error often occurs when region '%s' is not enabled in the target AWS account", err, region)
 	}
@@ -322,18 +336,32 @@ func InitializeAwsSourceAdapters(ctx context.Context, e *discovery.Engine, confi
 					"region": cfg.Region,
 				}
 
-				// Check if this is an opt-in region error
-				if isOptInRegionError(err) {
-					// This region is not enabled in the account - skip it but don't fail
+				// Check if this is a skippable region error (timeout or opt-in)
+				if isSkippableRegionError(err) {
 					wrappedErr := wrapRegionError(err, cfg.Region)
 					skippedRegionsMu.Lock()
 					skippedRegions = append(skippedRegions, skippedRegion{
 						region: cfg.Region,
 						err:    wrappedErr,
 					})
 					skippedRegionsMu.Unlock()
-					log.WithError(wrappedErr).WithFields(lf).Warn("Skipping region - not enabled in account")
-					return nil // Don't fail the pool for opt-in regions
+
+					reason := "opt-in region not enabled"
+					if isTimeoutError(err) {
+						reason = "timeout"
+						log.WithError(wrappedErr).WithFields(lf).Warn("Skipping region - unreachable (timeout)")
+					} else {
+						log.WithError(wrappedErr).WithFields(lf).Warn("Skipping region - not enabled in account")
+					}
+
+					span := trace.SpanFromContext(ctx)
+					span.AddEvent("ovm.adapter.regionSkipped", trace.WithAttributes(
+						attribute.String("ovm.adapter.region", cfg.Region),
+						attribute.String("ovm.adapter.skipReason", reason),
+						attribute.String("ovm.adapter.error", wrappedErr.Error()),
+					))
+
+					return nil // Don't fail the pool for skippable regions
 				}
 
 				// Wrap misleading OIDC errors with helpful region enablement context
@@ -645,7 +673,7 @@ func InitializeAwsSourceAdapters(ctx context.Context, e *discovery.Engine, confi
 		log.WithFields(log.Fields{
 			"skipped_regions": skippedRegionNames,
 			"count":           len(skippedRegions),
-		}).Warn("Some regions were skipped because they are not enabled in the AWS account. The source will operate normally with the remaining regions.")
+		}).Warn("Some regions were skipped because they are unreachable or not enabled in the AWS account. The source will operate normally with the remaining regions.")
 	}
 
 	log.Debug("Sources initialized")

aws-source/proc/proc_test.go

@@ -176,6 +176,16 @@ func TestIsOptInRegionError(t *testing.T) {
 			err:            errors.New("No OpenIDConnect provider found"),
 			expectedResult: false,
 		},
+		{
+			name:           "context.DeadlineExceeded returns false",
+			err:            context.DeadlineExceeded,
+			expectedResult: false,
+		},
+		{
+			name:           "context.Canceled returns false",
+			err:            context.Canceled,
+			expectedResult: false,
+		},
 	}
 
 	for _, tt := range tests {
@@ -188,6 +198,105 @@ func TestIsOptInRegionError(t *testing.T) {
 	}
 }
 
+func TestIsTimeoutError(t *testing.T) {
+	tests := []struct {
+		name           string
+		err            error
+		expectedResult bool
+	}{
+		{
+			name:           "nil error returns false",
+			err:            nil,
+			expectedResult: false,
+		},
+		{
+			name:           "context.DeadlineExceeded returns true",
+			err:            context.DeadlineExceeded,
+			expectedResult: true,
+		},
+		{
+			name:           "wrapped context.DeadlineExceeded returns true",
+			err:            fmt.Errorf("operation error STS: GetCallerIdentity: %w", context.DeadlineExceeded),
+			expectedResult: true,
+		},
+		{
+			name:           "context.Canceled returns false",
+			err:            context.Canceled,
+			expectedResult: false,
+		},
+		{
+			name:           "wrapped context.Canceled returns false",
+			err:            fmt.Errorf("operation error STS: GetCallerIdentity: %w", context.Canceled),
+			expectedResult: false,
+		},
+		{
+			name:           "non-timeout error returns false",
+			err:            errors.New("some random error"),
+			expectedResult: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := isTimeoutError(tt.err)
+			if result != tt.expectedResult {
+				t.Errorf("isTimeoutError() = %v, want %v for error: %v", result, tt.expectedResult, tt.err)
+			}
+		})
+	}
+}
+
+func TestIsSkippableRegionError(t *testing.T) {
+	tests := []struct {
+		name           string
+		err            error
+		expectedResult bool
+	}{
+		{
+			name:           "nil error returns false",
+			err:            nil,
+			expectedResult: false,
+		},
+		{
+			name:           "context.DeadlineExceeded returns true (ENG-3665)",
+			err:            context.DeadlineExceeded,
+			expectedResult: true,
+		},
+		{
+			name:           "wrapped context.DeadlineExceeded returns true (ENG-3665)",
+			err:            fmt.Errorf("operation error STS: GetCallerIdentity: %w", context.DeadlineExceeded),
+			expectedResult: true,
+		},
+		{
+			name:           "context.Canceled returns false (parent cancellation, not region timeout)",
+			err:            context.Canceled,
+			expectedResult: false,
+		},
+		{
+			name: "opt-in region error returns true",
+			err: &mockAPIError{
+				code:    "InvalidIdentityToken",
+				message: "No OpenIDConnect provider found in your account",
+			},
+			expectedResult: true,
+		},
+		{
+			name:           "non-skippable error returns false",
+			err:            errors.New("some random error"),
+			expectedResult: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := isSkippableRegionError(tt.err)
+			if result != tt.expectedResult {
+				t.Errorf("isSkippableRegionError() = %v, want %v for error: %v", result, tt.expectedResult, tt.err)
+			}
+		})
+	}
+}
+
 func TestWrapRegionError(t *testing.T) {
 	tests := []struct {
 		name         string
@@ -240,6 +349,27 @@ func TestWrapRegionError(t *testing.T) {
 			shouldWrap:   false,
 			expectedText: "",
 		},
+		{
+			name:         "timeout error gets timeout-specific message",
+			err:          context.DeadlineExceeded,
+			region:       "me-south-1",
+			shouldWrap:   true,
+			expectedText: "unreachable (timeout)",
+		},
+		{
+			name:         "wrapped timeout error gets timeout-specific message",
+			err:          fmt.Errorf("operation error STS: GetCallerIdentity: %w", context.DeadlineExceeded),
+			region:       "me-south-1",
+			shouldWrap:   true,
+			expectedText: "unreachable (timeout)",
+		},
+		{
+			name:         "canceled error is not wrapped (parent cancellation, not region timeout)",
+			err:          context.Canceled,
+			region:       "me-south-1",
+			shouldWrap:   false,
+			expectedText: "",
+		},
 	}
 
 	for _, tt := range tests {