Refactor test infrastructure and consolidate test utilities

Modernize the test infrastructure by consolidating test helpers into a
shared testutil package and enhancing connection reliability:

- Move test utilities from internal/test to opensearchutil/testutil for
  broader reusability across the project and external packages
- Remove obsolete internal/test/config.go in favor of improved helper
  functions with better error handling and connection management
- Add dynamic field filtering for JSON comparison tests to handle
  version-specific and environment-dependent OpenSearch responses
- Enhance connection robustness with improved readiness checks and
  health monitoring in opensearchtransport layer
- Update all integration tests across opensearchapi, plugins, and
  transport packages to use the new unified test infrastructure
- Add comprehensive documentation and examples for the new test utilities

This refactor provides a more maintainable foundation for testing across
different OpenSearch versions and environments while reducing code
duplication and improving test reliability.

Signed-off-by: Sean Chittenden <sean.chittenden@crowdstrike.com>

Author: sean-

.ci/opensearch/Dockerfile.opensearch

@@ -44,7 +44,6 @@ RUN if [ "$SECURE_INTEGRATION" != "true" ] ; then \
 # Ensure the repository mount point has correct permissions for snapshots
 RUN mkdir -p /usr/share/opensearch/mnt && chown opensearch:opensearch /usr/share/opensearch/mnt
 
-HEALTHCHECK --start-period=20s --interval=30s \
-  CMD curl -sf --retry 5 --max-time 5 --retry-delay 5 --retry-max-time 30 --retry-all-errors \
-  $(if $SECURE_INTEGRATION; then echo "--cert config/kirk.pem --key config/kirk-key.pem -k https://"; fi)"localhost:9200" \
-  || bash -c 'kill -s 15 -1 && (sleep 10; kill -s 9 -1)'
+HEALTHCHECK --start-period=60s --interval=30s --timeout=10s --retries=3 \
+  CMD curl -sf --max-time 5 \
+  $(if [ "$SECURE_INTEGRATION" = "true" ]; then echo "--cert config/kirk.pem --key config/kirk-key.pem -k https://"; else echo "http://"; fi)localhost:9200/_cluster/health

.ci/opensearch/docker-compose.yml

@@ -28,7 +28,7 @@ services:
       - http.publish_host=localhost
       - http.publish_port=9200
       # Memory settings
-      - OPENSEARCH_JAVA_OPTS=-Xms${OPENSEARCH_HEAP_SIZE:-1g} -Xmx${OPENSEARCH_HEAP_SIZE:-1g}
+      - OPENSEARCH_JAVA_OPTS=-Xms${OPENSEARCH_HEAP_SIZE:-1g} -Xmx${OPENSEARCH_HEAP_SIZE:-1g} ${OPENSEARCH_JAVA_OPTS_EXTRA}
     ports:
       - "9200:9200"
       - "9300:9300"
@@ -72,7 +72,7 @@ services:
       - http.publish_host=localhost
       - http.publish_port=9201
       # Memory settings
-      - OPENSEARCH_JAVA_OPTS=-Xms${OPENSEARCH_HEAP_SIZE:-1g} -Xmx${OPENSEARCH_HEAP_SIZE:-1g}
+      - OPENSEARCH_JAVA_OPTS=-Xms${OPENSEARCH_HEAP_SIZE:-1g} -Xmx${OPENSEARCH_HEAP_SIZE:-1g} ${OPENSEARCH_JAVA_OPTS_EXTRA}
     ports:
       - "9201:9200"
       - "9301:9300"
@@ -116,7 +116,7 @@ services:
       - http.publish_host=localhost
       - http.publish_port=9202
       # Memory settings
-      - OPENSEARCH_JAVA_OPTS=-Xms${OPENSEARCH_HEAP_SIZE:-1g} -Xmx${OPENSEARCH_HEAP_SIZE:-1g}
+      - OPENSEARCH_JAVA_OPTS=-Xms${OPENSEARCH_HEAP_SIZE:-1g} -Xmx${OPENSEARCH_HEAP_SIZE:-1g} ${OPENSEARCH_JAVA_OPTS_EXTRA}
     ports:
       - "9202:9200"
       - "9302:9300"

.github/workflows/lint.yml

@@ -18,7 +18,7 @@ jobs:
         with:
           version: v2.8.0
           only-new-issues: true
-          args: --fix --build-tags "integration core"
+          args: --fix --build-tags "integration core plugins plugin_security plugin_index_management multinode"
 
   prettify:
     name: Prettify

.github/workflows/test-integration.yml

@@ -57,7 +57,12 @@ jobs:
             fi
             echo "=====> Attempt $attempt/25 - waiting..."
           done
-      - run: make test-integ-core test-integ-plugins race=true
+      - run: make test-integ-core test-integ-plugins race=true coverage=true
+      - uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: tmp/integ.cov
+          flags: integration
 
   secured:
     name: Tests against secure cluster

CHANGELOG.md

@@ -6,7 +6,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 
 ### Added
 
-- Enhanced cluster readiness checking for improved test reliability: `ostest.NewClient()` now includes readiness validation (health + cluster state + nodes info)
+- Enhanced cluster readiness checking for improved test reliability: `testutil.NewClient()` now includes readiness validation (health + cluster state + nodes info)
 - Test parallelization support via TEST_PARALLEL environment variable (default: CPU cores - 1, minimum 1)
 - opensearchutil/testutil package with PollUntil helper for eventual consistency testing (ISM policies, index readiness, cluster state changes)
 - Configuration option `IncludeDedicatedClusterManagers` for controlling cluster manager node routing ([#765](https://github.com/opensearch-project/opensearch-go/issues/765))
@@ -44,6 +44,12 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
   - `opensearch.Config` and `opensearchtransport.Config` now accept optional `Context` and `CancelFunc` fields
   - `opensearchutil.BulkIndexerConfig` now accepts optional `Context` and `CancelFunc` fields
   - Enables proper context propagation for timeouts, cancellation, and graceful shutdown
+  - Role compatibility validation prevents conflicting role assignments (master+cluster_manager, warm+search)
+  - OpenSearch 3.0+ searchable snapshots now use `warm` role instead of deprecated `search` role
+- **BREAKING**: Migrate `signer/aws` package from AWS SDK v1 to AWS SDK v2 due to AWS SDK v1 reaching end-of-support on July 31, 2025
+  - Constructor now takes `aws.Config` instead of `session.Options`
+  - See USER_GUIDE.md for details required to migrate
+  - Users who need access to the existing `signer/awsv2` API can still use it, however they are encouraged to migrate to `signer/aws`
 
 ### Deprecated
 
@@ -323,7 +329,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 
 - Updates workflow action versions ([#488](https://github.com/opensearch-project/opensearch-go/pull/488))
 - Changes integration tests to work with secure and unsecure OpenSearch ([#488](https://github.com/opensearch-project/opensearch-go/pull/488))
-- Moves functions from `opensearch/internal/test` to `internal/test` for more general test uses ([#488](https://github.com/opensearch-project/opensearch-go/pull/488))
+- Moves functions from `opensearch/internal/test` to `opensearchutil/testutil` for shared test utilities ([#488](https://github.com/opensearch-project/opensearch-go/pull/488))
 - Changes `custom_foldername` field to pointer as it can be `null` ([#488](https://github.com/opensearch-project/opensearch-go/pull/488))
 - Changs cat indices Primary and Replica field to pointer as it can be `null` ([#488](https://github.com/opensearch-project/opensearch-go/pull/488))
 - Replaces `ioutil` with `io` in examples and integration tests [#495](https://github.com/opensearch-project/opensearch-go/pull/495)

Makefile

@@ -16,6 +16,7 @@ ifdef race
 	$(eval testunitargs += "-race")
 endif
 	$(eval testunitargs += "-cover" "./..." "-args" "-test.gocoverdir=$(PWD)/tmp/unit")
+	@rm -rf $(PWD)/tmp/unit
 	@mkdir -p $(PWD)/tmp/unit
 	@echo "go test -v" $(testunitargs); \
 	go test -v $(testunitargs);
@@ -35,6 +36,7 @@ ifdef race
 endif
 	$(eval TEST_PARALLEL ?= $(shell ncpu=$$(go env GOMAXPROCS 2>/dev/null); [ -z "$$ncpu" ] && ncpu=$$(sysctl -n hw.ncpu 2>/dev/null || nproc 2>/dev/null || echo 4); parallel=$$((ncpu - 1)); [ $$parallel -lt 1 ] && parallel=1; echo $$parallel))
 	$(eval testintegargs += "-cover" "-tags=$(testintegtags)" "-timeout=10m" "-parallel=$(TEST_PARALLEL)" "./..." "-args" "-test.gocoverdir=$(PWD)/tmp/integration")
+	@rm -rf $(PWD)/tmp/integration
 	@mkdir -p $(PWD)/tmp/integration
 	@echo "go test -v" $(testintegargs); \
 	go test -v $(testintegargs);
@@ -230,6 +232,7 @@ cluster.build:
 
 cluster.start:
 	@$(MAKE) cluster.docker-up
+	@$(MAKE) cluster.wait-ready
 	@$(MAKE) cluster.get-cert
 
 cluster.stop:
@@ -238,6 +241,7 @@ cluster.stop:
 cluster.docker-build:
 	@# Determine version-specific settings
 	$(eval OPENSEARCH_VERSION ?= latest)
+	$(eval SECURE_INTEGRATION ?= false)
 	$(eval version_major := $(shell \
 		if [ "$(OPENSEARCH_VERSION)" = "latest" ]; then \
 			echo "2"; \
@@ -252,13 +256,14 @@ cluster.docker-build:
 			echo "cluster_manager"; \
 		fi \
 	))
-	@echo "Building OpenSearch $(OPENSEARCH_VERSION) with role: $(manager_role)"
+	@echo "Building OpenSearch $(OPENSEARCH_VERSION) with role: $(manager_role), secure: $(SECURE_INTEGRATION)"
 	OPENSEARCH_MANAGER_ROLE=$(manager_role) OPENSEARCH_MANAGER_SETTING=$(manager_role) \
 		docker compose --project-directory .ci/opensearch build --pull
 
 cluster.docker-up:
 	@# Determine version-specific settings
 	$(eval OPENSEARCH_VERSION ?= latest)
+	$(eval SECURE_INTEGRATION ?= false)
 	$(eval version_major := $(shell \
 		if [ "$(OPENSEARCH_VERSION)" = "latest" ]; then \
 			echo "2"; \
@@ -273,9 +278,25 @@ cluster.docker-up:
 			echo "cluster_manager"; \
 		fi \
 	))
-	@echo "Starting OpenSearch $(OPENSEARCH_VERSION) with role: $(manager_role)"
-	OPENSEARCH_MANAGER_ROLE=$(manager_role) OPENSEARCH_MANAGER_SETTING=$(manager_role) \
-		docker compose --project-directory .ci/opensearch up -d
+	@# Apply cgroup workaround for OpenSearch 2.0.1-2.3.0
+	$(eval java_opts_extra := $(shell \
+		if [ "$(OPENSEARCH_VERSION)" != "latest" ]; then \
+			version() { echo "$$@" | awk -F. '{ printf("%d%03d%03d%03d\n", $$1,$$2,$$3,$$4); }'; }; \
+			v=$$(version $(OPENSEARCH_VERSION)); \
+			v_min=$$(version 2.0.1); \
+			v_max=$$(version 2.3.0); \
+			if [ $$v -ge $$v_min ] && [ $$v -le $$v_max ]; then \
+				echo " -XX:-UseContainerSupport"; \
+			fi; \
+		fi \
+	))
+	@echo "Starting OpenSearch $(OPENSEARCH_VERSION) with role: $(manager_role), secure: $(SECURE_INTEGRATION)"
+	export SECURE_INTEGRATION=$(SECURE_INTEGRATION); \
+	export OPENSEARCH_VERSION=$(OPENSEARCH_VERSION); \
+	export OPENSEARCH_MANAGER_ROLE=$(manager_role); \
+	export OPENSEARCH_MANAGER_SETTING=$(manager_role); \
+	export OPENSEARCH_JAVA_OPTS_EXTRA="$(java_opts_extra)"; \
+	docker compose --project-directory .ci/opensearch up -d
 
 cluster.scale.1: ## Start single-node cluster
 	docker compose --project-directory .ci/opensearch up -d --scale opensearch-node2=0 --scale opensearch-node3=0;
@@ -297,6 +318,84 @@ cluster.get-cert:
 		docker cp $$CONTAINER:/usr/share/opensearch/config/kirk-key.pem admin.key; \
 	fi
 
+cluster.wait-ready: ## Poll cluster until health status is green or yellow
+	@printf "\033[2m-> Waiting for cluster to be ready...\033[0m\n"
+	@{ \
+		set -e; \
+		MAX_ATTEMPTS=60; \
+		ATTEMPT=1; \
+		HTTP_URL="http://localhost:9200/_cluster/health"; \
+		HTTPS_URL="https://localhost:9200/_cluster/health"; \
+		HEALTH_URL=""; \
+		CURL_OPTS=""; \
+		VERSION="$${OPENSEARCH_VERSION:-latest}"; \
+		if [ "$$VERSION" = "latest" ]; then \
+			PASSWORD="myStrongPassword123!"; \
+		else \
+			MAJOR=$$(echo "$$VERSION" | cut -d. -f1); \
+			MINOR=$$(echo "$$VERSION" | cut -d. -f2); \
+			if [ $$MAJOR -gt 2 ] || ([ $$MAJOR -eq 2 ] && [ $$MINOR -ge 12 ]); then \
+				PASSWORD="myStrongPassword123!"; \
+			else \
+				PASSWORD="admin"; \
+			fi; \
+		fi; \
+		while [ $$ATTEMPT -le $$MAX_ATTEMPTS ]; do \
+			if [ -z "$$HEALTH_URL" ]; then \
+				if curl -sf "$$HTTP_URL" > /dev/null 2>&1; then \
+					printf "\033[36m→ Detected insecure cluster (HTTP)\033[0m\n"; \
+					HEALTH_URL="$$HTTP_URL"; \
+					CURL_OPTS=""; \
+				elif curl -sf -k -u "admin:$$PASSWORD" "$$HTTPS_URL" > /dev/null 2>&1; then \
+					printf "\033[36m→ Detected secure cluster (HTTPS)\033[0m\n"; \
+					HEALTH_URL="$$HTTPS_URL"; \
+					CURL_OPTS="-k -u admin:$$PASSWORD"; \
+				else \
+					printf "\033[33m⋯ Waiting for cluster to respond (attempt $$ATTEMPT/$$MAX_ATTEMPTS)\033[0m\n"; \
+					ATTEMPT=$$((ATTEMPT + 1)); \
+					sleep 2; \
+					continue; \
+				fi; \
+			fi; \
+			if curl -sf $$CURL_OPTS "$$HEALTH_URL" > /dev/null 2>&1; then \
+				STATUS=$$(curl -sf $$CURL_OPTS "$$HEALTH_URL" | grep -o '"status":"[^"]*"' | cut -d'"' -f4); \
+				if [ "$$STATUS" = "green" ] || [ "$$STATUS" = "yellow" ]; then \
+					printf "\033[32m✓ Cluster is ready (status: $$STATUS) after $$ATTEMPT attempts\033[0m\n"; \
+					INFO_URL="$${HEALTH_URL%%/_cluster/health}"; \
+					CLUSTER_INFO=$$(curl -sf $$CURL_OPTS "$$INFO_URL" 2>/dev/null); \
+					if [ -n "$$CLUSTER_INFO" ]; then \
+						CLUSTER_NAME=$$(echo "$$CLUSTER_INFO" | grep -o '"cluster_name":"[^"]*"' | cut -d'"' -f4); \
+						CLUSTER_VERSION=$$(echo "$$CLUSTER_INFO" | grep -o '"number":"[^"]*"' | head -1 | cut -d'"' -f4); \
+						printf "\033[2m  Cluster: $$CLUSTER_NAME\033[0m\n"; \
+						printf "\033[2m  Version: $$CLUSTER_VERSION\033[0m\n"; \
+						printf "\033[2m  URL:     $$INFO_URL\033[0m\n"; \
+						if [ -n "$$CURL_OPTS" ]; then \
+							printf "\033[2m  Auth:    admin:****\033[0m\n"; \
+						fi; \
+					fi; \
+					exit 0; \
+				fi; \
+				printf "\033[33m⋯ Cluster status: $$STATUS (attempt $$ATTEMPT/$$MAX_ATTEMPTS)\033[0m\n"; \
+			else \
+				printf "\033[33m⋯ Waiting for cluster to respond (attempt $$ATTEMPT/$$MAX_ATTEMPTS)\033[0m\n"; \
+			fi; \
+			ATTEMPT=$$((ATTEMPT + 1)); \
+			sleep 2; \
+		done; \
+		printf "\033[31m✗ Cluster failed to become ready after $$MAX_ATTEMPTS attempts\033[0m\n"; \
+		printf "\033[2m\n--- Diagnostic Information ---\033[0m\n"; \
+		printf "\033[2mDocker containers:\033[0m\n"; \
+		docker compose --project-directory .ci/opensearch ps || true; \
+		printf "\033[2m\nFull logs from all containers:\033[0m\n"; \
+		docker compose --project-directory .ci/opensearch logs || true; \
+		printf "\033[2m\nAttempted URLs:\033[0m\n"; \
+		printf "  HTTP:  $$HTTP_URL\n"; \
+		printf "  HTTPS: $$HTTPS_URL\n"; \
+		printf "\033[2m\nCurl test results:\033[0m\n"; \
+		printf "  HTTP: "; curl -sf "$$HTTP_URL" && echo "✓ OK" || echo "✗ Failed"; \
+		printf "  HTTPS: "; curl -sf -k -u "admin:$$PASSWORD" "$$HTTPS_URL" && echo "✓ OK" || echo "✗ Failed"; \
+		exit 1; \
+	}
 
 cluster.clean: ## Remove unused Docker volumes and networks
 	@printf "\033[2m-> Cleaning up Docker assets...\033[0m\n"

USER_GUIDE.md

@@ -221,13 +221,19 @@ Before starting, we strongly recommend reading the full AWS documentation regard
 >
 > See [Managed Domains signing-service requests.](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/ac.html#managedomains-signing-service-requests)
 
-Depending on the version of AWS SDK used, import the v1 or v2 request signer from `signer/aws` or `signer/awsv2` respectively. Both signers are equivalent in their functionality, they provide AWS Signature Version 4 (SigV4).
+Depending on the version of AWS SDK used, import the request signer from `signer/aws` (recommended) or `signer/awsv2`. Both signers use AWS SDK v2 and provide AWS Signature Version 4 (SigV4).
+
+**BREAKING CHANGE**: As of this version, the main `signer/aws` package has been migrated from AWS SDK v1 to AWS SDK v2 due to AWS SDK v1 reaching end-of-support on July 31, 2025.
 
 To read more about SigV4 see [Signature Version 4 signing process](https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html)
 
 Here are some Go samples that show how to sign each OpenSearch request and automatically search for AWS credentials from the ~/.aws folder or environment variables:
 
-### AWS SDK v1
+### AWS SDK v2 (Recommended)
+
+**Migration Note**: If you were previously using `signer/aws` with AWS SDK v1, you need to update your imports and configuration as shown below.
+
+**Credential Caching**: The signer automatically enables credential caching for improved performance, especially when using STS credentials (assume role, web identity, etc.). This reduces API calls to AWS STS and improves signing performance.
 
 ```go
 package main
@@ -237,7 +243,8 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
 	requestsigner "github.com/opensearch-project/opensearch-go/v4/signer/aws"
 
 	"github.com/opensearch-project/opensearch-go/v4"
@@ -256,16 +263,22 @@ func main() {
 const endpoint = "" // e.g. https://opensearch-domain.region.com
 
 func example() error {
-	// Create an AWS request Signer and load AWS configuration using default config folder or env vars.
-	// See https://docs.aws.amazon.com/opensearch-service/latest/developerguide/request-signing.html#request-signing-go
-	signer, err := requestsigner.NewSignerWithService(
-		session.Options{SharedConfigState: session.SharedConfigEnable},
-		requestsigner.OpenSearchService, // Use requestsigner.OpenSearchServerless for Amazon OpenSearch Serverless.
-	)
+	ctx := context.Background()
+
+	// Load AWS configuration
+	awsCfg, err := config.LoadDefaultConfig(ctx)
 	if err != nil {
 		return err
 	}
-	// Create an opensearch client and use the request-signer.
+
+	// Create an AWS request Signer
+	signer, err := requestsigner.NewSignerWithService(awsCfg, requestsigner.OpenSearchService)
+	// Use requestsigner.OpenSearchServerless for Amazon OpenSearch Serverless
+	if err != nil {
+		return err
+	}
+
+	// Create an opensearch client and use the request-signer
 	client, err := opensearchapi.NewClient(
 		opensearchapi.Config{
 			Client: opensearch.Config{
@@ -278,20 +291,18 @@ func example() error {
 		return err
 	}
 
-	ctx := context.Background()
-
     ping, err := client.Ping(ctx, nil)
     if err != nil {
         return err
     }
 
-    fmt.Println(ping)
+	fmt.Println(ping)
 
 	return nil
 }
 ```
 
-### AWS SDK v2
+### Alternative: Using signer/awsv2
 
 Use the AWS SDK v2 for Go to authenticate with Amazon OpenSearch service.