Migrate AWS API calls from SDK Go v1 to v2

Squashed application of the aws-sdk-v2 migration (feature/aws-sdk-v2) onto
latest main. Replaces the SDKv1 credential chain, service clients, retryers,
and request handlers with SDKv2 equivalents, and removes the hand-vendored
SDKv1 cloudwatch/cloudwatchlogs services (~44k lines).

Source PRs (Jeffrey Chien):
- #1981 Support credential chain for AWS SDK Go v2
- #1985 Migrate CloudWatch exporter to SDKv2
- #1989 Migrate CloudWatch logs output plugin to SDKv2
- #1992 Migrate EC2 metadata to SDKv2
- #1994 Migrate ECS service discovery to SDKv2
- #1995 Migrate wizard, downloader, and translator to SDKv2
- #2001 Remove SDKv1 direct dependencies
- Use a shared HTTP client to reduce file descriptor usage

aws-sdk-go (v1) is no longer a direct dependency. cfg/aws now exposes the v2
credential chain (CredentialsConfig.LoadConfig), STS regional->partitional
fallback, smithy custom-header middleware, and a shared BuildableClient.

Author: jefchien

.github/workflows/PR-build.yml

@@ -72,6 +72,16 @@ jobs:
         if: needs.changes.outputs.lint == 'true'
         run: make lint
 
+      - name: Check AWS SDK partition data
+        if: needs.changes.outputs.lint == 'true'
+        run: |
+          make copy-aws-sdk-partition
+          if [ ! -z "`git status --porcelain`" ]; then
+            echo "make copy-aws-sdk-partition changed files"
+            git status
+            exit 1
+          fi
+
   build:
     needs: [lint, changes]
     name: Build ${{ matrix.os }}

.golangci.yml

@@ -33,6 +33,7 @@ linters:
       - third_party$
       - builtin$
       - examples$
+      - sdk/
     rules:
       - linters:
         - revive

Makefile

@@ -391,3 +391,18 @@ dockerized-windows-build:
 	$(DOCKER_WINDOWS_BUILD_FROM_SOURCE) .
 	@echo Built image:
 	@echo $(IMAGE)
+
+copy-aws-sdk-partition:
+	# aws-sdk-go-v2 does not export partition metadata, so we need to copy the files from vendor to make it accessible
+	# for the credential chain.
+	go mod vendor
+	cp vendor/github.com/aws/aws-sdk-go-v2/internal/endpoints/awsrulesfn/partition.go ./sdk/endpoints/awsrulesfn/partition.go
+	cp vendor/github.com/aws/aws-sdk-go-v2/internal/endpoints/awsrulesfn/partitions.go ./sdk/endpoints/awsrulesfn/partitions.go
+	echo '// Code generated by aws-sdk-go-v2. DO NOT EDIT.' > temp_partition.go
+	cat ./sdk/endpoints/awsrulesfn/partition.go >> temp_partition.go
+	mv temp_partition.go ./sdk/endpoints/awsrulesfn/partition.go
+	echo '// Code generated by aws-sdk-go-v2. DO NOT EDIT.' > temp_partitions.go
+	cat ./sdk/endpoints/awsrulesfn/partitions.go >> temp_partitions.go
+	mv temp_partitions.go ./sdk/endpoints/awsrulesfn/partitions.go
+	rm -rf vendor/
+	$(MAKE) fmt

cfg/aws/aws_sdk_logging.go

@@ -4,36 +4,50 @@
 package aws
 
 import (
+	"fmt"
 	"log"
 	"strings"
 
-	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/smithy-go/logging"
 )
 
 // Hard coded strings that match actual variable names in the AWS SDK.
-// I don't expect these names to change, or their meaning to change.
-// Update this map if/when AWS SDK adds more levels (unlikely).
-var stringToLevelMap map[string]aws.LogLevelType = map[string]aws.LogLevelType{
-	"LogDebug":                    aws.LogDebug,
-	"LogDebugWithSigning":         aws.LogDebugWithSigning,
-	"LogDebugWithHTTPBody":        aws.LogDebugWithHTTPBody,
-	"LogDebugWithRequestRetries":  aws.LogDebugWithRequestRetries,
-	"LogDebugWithRequestErrors":   aws.LogDebugWithRequestErrors,
-	"LogDebugWithEventStreamBody": aws.LogDebugWithEventStreamBody,
+var stringToLevelMap = map[string]aws.ClientLogMode{
+	// AWS SDK v2 Levels
+	"LogRequest":              aws.LogRequest,
+	"LogResponse":             aws.LogResponse,
+	"LogSigning":              aws.LogSigning,
+	"LogRequestWithBody":      aws.LogRequestWithBody,
+	"LogResponseWithBody":     aws.LogResponseWithBody,
+	"LogRetries":              aws.LogRetries,
+	"LogRequestEventMessage":  aws.LogRequestEventMessage,
+	"LogResponseEventMessage": aws.LogResponseEventMessage,
+	"LogDeprecatedUsage":      aws.LogDeprecatedUsage,
+	// AWS SDK v1 Levels
+	"LogDebug":                    aws.LogRequest | aws.LogResponse,
+	"LogDebugWithSigning":         aws.LogRequest | aws.LogResponse | aws.LogSigning,
+	"LogDebugWithHTTPBody":        aws.LogRequestWithBody | aws.LogResponseWithBody,
+	"LogDebugWithRequestRetries":  aws.LogRequest | aws.LogResponse | aws.LogRetries,
+	"LogDebugWithRequestErrors":   aws.LogRequest | aws.LogResponse, // no equivalent in AWS SDK v2
+	"LogDebugWithEventStreamBody": aws.LogRequestEventMessage | aws.LogResponseEventMessage,
 }
-var sdkLogLevel aws.LogLevelType = aws.LogOff
+var sdkLogLevel aws.ClientLogMode
 
 // SetSDKLogLevel sets the global log level which will be used in all AWS SDK calls.
 // The levels are a bit field that is OR'd together.
 // So the user can specify multiple levels and we OR them together.
 // Example: "aws_sdk_log_level": "LogDebugWithSigning | LogDebugWithRequestErrors".
-// JSON string value must contain the levels seperated by "|" and optionally whitespace.
+// JSON string value must contain the levels separated by "|" and optionally whitespace.
 func SetSDKLogLevel(sdkLogLevelString string) {
-	var temp aws.LogLevelType = aws.LogOff
+	var temp aws.ClientLogMode
 
 	levels := strings.Split(sdkLogLevelString, "|")
 	for _, v := range levels {
 		trimmed := strings.TrimSpace(v)
+		if trimmed == "LogDebugWithRequestErrors" {
+			log.Println(`W! AWS SDK log level "LogDebugWithRequestErrors" is no longer supported. This will be evaluated as the equivalent of "LogDebug".`)
+		}
 		// If v not in map, then OR with 0 is harmless.
 		temp |= stringToLevelMap[trimmed]
 	}
@@ -43,17 +57,23 @@ func SetSDKLogLevel(sdkLogLevelString string) {
 
 // SDKLogLevel returns the single global value so it can be used in all
 // AWS SDK calls scattered throughout the Agent.
-func SDKLogLevel() *aws.LogLevelType {
-	return &sdkLogLevel
+func SDKLogLevel() aws.ClientLogMode {
+	return sdkLogLevel
 }
 
 // SDKLogger implements the aws.Logger interface.
 type SDKLogger struct {
 }
 
-// Log is the only method in the aws.Logger interface.
-func (SDKLogger) Log(args ...interface{}) {
-	// Always use info logging level.
-	tempSlice := append([]interface{}{"I!"}, args...)
-	log.Println(tempSlice...)
+var _ logging.Logger = (*SDKLogger)(nil)
+
+func (SDKLogger) Logf(classification logging.Classification, format string, args ...interface{}) {
+	logLevelPrefix := "I!"
+	switch classification {
+	case logging.Debug:
+		logLevelPrefix = "D!"
+	case logging.Warn:
+		logLevelPrefix = "W!"
+	}
+	log.Printf(fmt.Sprintf("%s %s", logLevelPrefix, format), args...)
 }

cfg/aws/aws_sdk_logging_test.go

@@ -4,49 +4,83 @@
 package aws
 
 import (
+	"bytes"
+	"log"
+	"os"
 	"testing"
 
-	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/smithy-go/logging"
+	"github.com/stretchr/testify/assert"
 )
 
 func TestSetSDKLogLevel(t *testing.T) {
-	cases := []struct {
-		sdkLogLevelString string
-		expectedVal       aws.LogLevelType
+	testCases := []struct {
+		input string
+		want  aws.ClientLogMode
 	}{
-		// sdkLogLevelString does not match
-		{"FOO", aws.LogOff},
+		// Invalid input
+		{input: "FOO", want: aws.ClientLogMode(0)},
 		// Wrong case.
-		{"logDEBUG", aws.LogOff},
+		{input: "logrequest", want: aws.ClientLogMode(0)},
 		// Extra char.
-		{"LogDebug1", aws.LogOff},
+		{input: "LogRequest1", want: aws.ClientLogMode(0)},
 		// Single match.
-		{"LogDebug", aws.LogDebug},
-		{"LogDebugWithEventStreamBody", aws.LogDebugWithEventStreamBody},
-		{"LogDebugWithHTTPBody", aws.LogDebugWithHTTPBody},
-		{"LogDebugWithRequestRetries", aws.LogDebugWithRequestRetries},
-		{"LogDebugWithRequestErrors", aws.LogDebugWithRequestErrors},
-		{"LogDebugWithEventStreamBody", aws.LogDebugWithEventStreamBody},
+		{input: "LogRequest", want: aws.LogRequest},
+		{input: "LogResponse", want: aws.LogResponse},
+		{input: "LogSigning", want: aws.LogSigning},
+		{input: "LogRequestWithBody", want: aws.LogRequestWithBody},
+		{input: "LogResponseWithBody", want: aws.LogResponseWithBody},
+		{input: "LogRetries", want: aws.LogRetries},
+		{input: "LogRequestEventMessage", want: aws.LogRequestEventMessage},
+		{input: "LogResponseEventMessage", want: aws.LogResponseEventMessage},
+		{input: "LogDeprecatedUsage", want: aws.LogDeprecatedUsage},
+		// v1 compatibility
+		{input: "LogDebug", want: aws.LogRequest | aws.LogResponse},
+		{input: "LogDebugWithSigning", want: aws.LogRequest | aws.LogResponse | aws.LogSigning},
+		{input: "LogDebugWithHTTPBody", want: aws.LogRequestWithBody | aws.LogResponseWithBody},
+		{input: "LogDebugWithRequestRetries", want: aws.LogRequest | aws.LogResponse | aws.LogRetries},
+		{input: "LogDebugWithRequestErrors", want: aws.LogRequest | aws.LogResponse},
+		{input: "LogDebugWithEventStreamBody", want: aws.LogRequestEventMessage | aws.LogResponseEventMessage},
 		// Extra space around is allowed.
-		{"   LogDebug  ", aws.LogDebug},
+		{input: "   LogRequest  ", want: aws.LogRequest},
 		// Multiple matches.
-		{"LogDebugWithEventStreamBody|LogDebugWithHTTPBody",
-			aws.LogDebugWithEventStreamBody | aws.LogDebugWithHTTPBody},
-		{"  LogDebugWithHTTPBody  |  LogDebugWithEventStreamBody  ",
-			aws.LogDebugWithEventStreamBody | aws.LogDebugWithHTTPBody},
-		{"LogDebugWithRequestRetries|LogDebugWithEventStreamBody",
-			aws.LogDebugWithEventStreamBody | aws.LogDebugWithRequestRetries},
-		{"LogDebugWithRequestRetries|LogDebugWithRequestErrors",
-			aws.LogDebugWithRequestRetries | aws.LogDebugWithRequestErrors},
-		{"LogDebugWithRequestRetries|LogDebugWithRequestErrors|LogDebugWithEventStreamBody",
-			aws.LogDebugWithRequestRetries | aws.LogDebugWithRequestErrors | aws.LogDebugWithEventStreamBody},
+		{input: "LogRequest|LogResponse", want: aws.LogRequest | aws.LogResponse},
+		{input: "  LogRequestWithBody  |  LogResponseWithBody  ", want: aws.LogRequestWithBody | aws.LogResponseWithBody},
+		{input: "LogRetries|LogSigning", want: aws.LogRetries | aws.LogSigning},
+		{input: "LogRequest|LogResponse|LogSigning", want: aws.LogRequest | aws.LogResponse | aws.LogSigning},
 	}
 
-	for _, tc := range cases {
-		SetSDKLogLevel(tc.sdkLogLevelString)
-		// check the internal var
-		if *SDKLogLevel() != tc.expectedVal {
-			t.Errorf("input: %v, actual: %v", tc, sdkLogLevel)
-		}
+	for _, testCase := range testCases {
+		SetSDKLogLevel(testCase.input)
+		assert.Equal(t, testCase.want, SDKLogLevel())
+	}
+}
+
+func TestSDKLogger(t *testing.T) {
+	var buf bytes.Buffer
+	log.SetOutput(&buf)
+	defer log.SetOutput(os.Stderr)
+
+	logger := SDKLogger{}
+
+	tests := []struct {
+		classification logging.Classification
+		expectedPrefix string
+	}{
+		{classification: logging.Debug, expectedPrefix: "D!"},
+		{classification: logging.Warn, expectedPrefix: "W!"},
+		{classification: logging.Classification("TEST"), expectedPrefix: "I!"},
+	}
+
+	for _, tt := range tests {
+		t.Run(string(tt.classification), func(t *testing.T) {
+			buf.Reset()
+			logger.Logf(tt.classification, "test message: %s", "arg")
+
+			output := buf.String()
+			assert.Contains(t, output, tt.expectedPrefix)
+			assert.Contains(t, output, "test message: arg")
+		})
 	}
 }

cfg/aws/client.go

@@ -0,0 +1,28 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: MIT
+
+package aws
+
+import (
+	"sync"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
+)
+
+var (
+	sharedHTTPClient     aws.HTTPClient
+	sharedHTTPClientOnce sync.Once
+)
+
+// getSharedHTTPClient returns a singleton HTTP client for all AWS SDK operations. Sharing the client enables connection
+// pooling and reuse across all AWS API calls, which reduces memory and file descriptor usage. It has to be an
+// BuildableClient because the SDK will only append custom CA bundles if the client is of that type.
+// https://github.com/aws/aws-sdk-go-v2/blob/v1.41.1/config/resolve.go#L57
+func getSharedHTTPClient() aws.HTTPClient {
+	sharedHTTPClientOnce.Do(func() {
+		sharedHTTPClient = awshttp.NewBuildableClient().WithTimeout(1 * time.Minute)
+	})
+	return sharedHTTPClient
+}

cfg/aws/client_test.go

@@ -0,0 +1,38 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: MIT
+
+package aws
+
+import (
+	"sync"
+	"testing"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetSharedHTTPClient_Concurrent(t *testing.T) {
+	sharedHTTPClientOnce = sync.Once{}
+	sharedHTTPClient = nil
+
+	const count = 100
+	clients := make([]aws.HTTPClient, count)
+	var wg sync.WaitGroup
+
+	for i := 0; i < count; i++ {
+		wg.Add(1)
+		go func(index int) {
+			defer wg.Done()
+			clients[index] = getSharedHTTPClient()
+		}(i)
+	}
+
+	wg.Wait()
+
+	// All goroutines should get the same instance
+	first := clients[0]
+	assert.NotNil(t, first)
+	for i := 1; i < count; i++ {
+		assert.Same(t, first, clients[i])
+	}
+}