x-pack/filebeat/input/awss3: fix priority of region name configurations (#36034) (#36096)

The code currently prioritises the region_name configuration, even when
it is not provided, against the claims of the documentation. Make a
check whether it is empty before claiming a conflict and using it.

(cherry picked from commit 57d649d)

Co-authored-by: Dan Kortschak <[email protected]>
Co-authored-by: Bharat Pasupula <[email protected]>

Author: 3 people

CHANGELOG.next.asciidoc

@@ -147,6 +147,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Improve error reporting and fix IPv6 handling of TCP and UDP metric collection. {pull}35996[35996]
 - Fix handling of NUL-terminated log lines in Fortinet Firewall module. {issue}36026[36026] {pull}36027[36027]
 - Make redact field configuration recommended in CEL input and log warning if missing. {pull}36008[36008]
+- Fix handling of region name configuration in awss3 input {pull}36034[36034]
 
 *Heartbeat*
 

x-pack/filebeat/input/awss3/input.go

@@ -119,10 +119,10 @@ func (in *s3Input) Run(inputContext v2.Context, pipeline beat.Pipeline) error {
 		if err != nil && in.config.RegionName == "" {
 			return fmt.Errorf("failed to get AWS region from queue_url: %w", err)
 		}
-		if regionName != in.config.RegionName {
-			inputContext.Logger.Warnf("configured region disagrees with queue_url region: %q != %q: using %[1]q",
-				in.config.RegionName, regionName)
-			regionName = in.config.RegionName
+		var warn regionMismatchError
+		if errors.As(err, &warn) {
+			// Warn of mismatch, but go ahead with configured region name.
+			inputContext.Logger.Warnf("%v: using %q", err, regionName)
 		}
 		in.awsConfig.Region = regionName
 
@@ -306,7 +306,7 @@ func (in *s3Input) createS3Lister(ctx v2.Context, cancelCtx context.Context, cli
 
 var errBadQueueURL = errors.New("QueueURL is not in format: https://sqs.{REGION_ENDPOINT}.{ENDPOINT}/{ACCOUNT_NUMBER}/{QUEUE_NAME}")
 
-func getRegionFromQueueURL(queueURL string, endpoint, defaultRegion string) (string, error) {
+func getRegionFromQueueURL(queueURL string, endpoint, defaultRegion string) (region string, err error) {
 	// get region from queueURL
 	// Example: https://sqs.us-east-1.amazonaws.com/627959692251/test-s3-logs
 	u, err := url.Parse(queueURL)
@@ -317,7 +317,11 @@ func getRegionFromQueueURL(queueURL string, endpoint, defaultRegion string) (str
 		queueHostSplit := strings.SplitN(u.Host, ".", 3)
 		if len(queueHostSplit) == 3 {
 			if queueHostSplit[2] == endpoint || (endpoint == "" && strings.HasPrefix(queueHostSplit[2], "amazonaws.")) {
-				return queueHostSplit[1], nil
+				region = queueHostSplit[1]
+				if defaultRegion != "" && region != defaultRegion {
+					return defaultRegion, regionMismatchError{queueURLRegion: region, defaultRegion: defaultRegion}
+				}
+				return region, nil
 			}
 		} else if defaultRegion != "" {
 			return defaultRegion, nil
@@ -326,6 +330,15 @@ func getRegionFromQueueURL(queueURL string, endpoint, defaultRegion string) (str
 	return "", errBadQueueURL
 }
 
+type regionMismatchError struct {
+	queueURLRegion string
+	defaultRegion  string
+}
+
+func (e regionMismatchError) Error() string {
+	return fmt.Sprintf("configured region disagrees with queue_url region: %q != %q", e.queueURLRegion, e.defaultRegion)
+}
+
 func getRegionForBucket(ctx context.Context, s3Client *s3.Client, bucketName string) (string, error) {
 	getBucketLocationOutput, err := s3Client.GetBucketLocation(ctx, &s3.GetBucketLocationInput{
 		Bucket: awssdk.String(bucketName),

x-pack/filebeat/input/awss3/input_test.go

@@ -81,6 +81,19 @@ func TestGetRegionFromQueueURL(t *testing.T) {
 			endpoint: "googlecloud.com",
 			wantErr:  errBadQueueURL,
 		},
+		{
+			name:     "mismatch_regions_no_default",
+			queueURL: "https://sqs.us-east-1.amazonaws.com/627959692251/test-s3-logs",
+			deflt:    "",
+			want:     "us-east-1",
+		},
+		{
+			name:     "mismatch_regions",
+			queueURL: "https://sqs.us-east-1.amazonaws.com/627959692251/test-s3-logs",
+			deflt:    "ap-west-1",
+			want:     "ap-west-1",
+			wantErr:  regionMismatchError{queueURLRegion: "us-east-1", defaultRegion: "ap-west-1"},
+		},
 		{
 			name:     "localstack",
 			queueURL: "http://localhost:4566/000000000000/filebeat-s3-integtest-d9clk9",