Refactor test infrastructure and consolidate test utilities

Modernize the test infrastructure by consolidating test helpers into a
shared testutil package and enhancing connection reliability:

- Move test utilities from internal/test to opensearchutil/testutil for
  broader reusability across the project and external packages
- Remove obsolete internal/test/config.go in favor of improved helper
  functions with better error handling and connection management
- Add dynamic field filtering for JSON comparison tests to handle
  version-specific and environment-dependent OpenSearch responses
- Enhance connection robustness with improved readiness checks and
  health monitoring in opensearchtransport layer
- Update all integration tests across opensearchapi, plugins, and
  transport packages to use the new unified test infrastructure
- Add comprehensive documentation and examples for the new test utilities

This refactor provides a more maintainable foundation for testing across
different OpenSearch versions and environments while reducing code
duplication and improving test reliability.

Signed-off-by: Sean Chittenden <sean.chittenden@crowdstrike.com>

Author: sean-

Makefile

@@ -227,8 +227,8 @@ cluster.scale.3: ## Start full 3-node cluster
 
 cluster.get-cert:
 	@if [[ -v SECURE_INTEGRATION ]] && [[ $$SECURE_INTEGRATION == "true" ]]; then \
-		docker cp $$(docker compose --project-directory .ci/opensearch ps --format '{{.Name}}'):/usr/share/opensearch/config/kirk.pem admin.pem && \
-		docker cp $$(docker compose --project-directory .ci/opensearch ps --format '{{.Name}}'):/usr/share/opensearch/config/kirk-key.pem admin.key; \
+		docker cp $$(docker compose --project-directory .ci/opensearch ps --format '{{.Name}}' | head -1):/usr/share/opensearch/config/kirk.pem admin.pem && \
+		docker cp $$(docker compose --project-directory .ci/opensearch ps --format '{{.Name}}' | head -1):/usr/share/opensearch/config/kirk-key.pem admin.key; \
 	fi
 
 

error_test.go

@@ -50,7 +50,7 @@ func TestError(t *testing.T) {
 			assert.True(t, resp.IsError())
 			err := opensearch.ParseError(resp)
 			var testError *opensearch.StructError
-			require.True(t, errors.As(err, &testError))
+			require.ErrorAs(t, err, &testError)
 			assert.Equal(t, http.StatusBadRequest, testError.Status)
 			assert.Equal(t, "resource_already_exists_exception", testError.Err.Type)
 			assert.Equal(t, "index [test/HU2mN_RMRXGcS38j3yV-VQ] already exists", testError.Err.Reason)
@@ -92,7 +92,7 @@ func TestError(t *testing.T) {
 			assert.True(t, resp.IsError())
 			err := opensearch.ParseError(resp)
 			var testError *opensearch.StructError
-			require.True(t, errors.As(err, &testError))
+			require.ErrorAs(t, err, &testError)
 			assert.Equal(t, http.StatusBadRequest, testError.Status)
 			assert.Equal(t, "illegal_argument_exception", testError.Err.Type)
 			assert.Equal(t, "composable template [posts] template after composition is invalid", testError.Err.Reason)
@@ -117,14 +117,14 @@ func TestError(t *testing.T) {
 					}`),
 				)
 				body, err := io.ReadAll(reader)
-				require.Nil(t, err)
+				require.NoError(t, err)
 
 				var errStruct *opensearch.StructError
 				err = json.Unmarshal(body, &errStruct)
-				assert.NotNil(t, err)
+				assert.Error(t, err)
 
 				var jsonError *json.UnmarshalTypeError
-				assert.True(t, errors.As(err, &jsonError))
+				assert.ErrorAs(t, err, &jsonError)
 			})
 			t.Run("string", func(t *testing.T) {
 				reader := io.NopCloser(
@@ -134,14 +134,14 @@ func TestError(t *testing.T) {
 					}`),
 				)
 				body, err := io.ReadAll(reader)
-				require.Nil(t, err)
+				require.NoError(t, err)
 
 				var errStruct *opensearch.StructError
 				err = json.Unmarshal(body, &errStruct)
-				assert.NotNil(t, err)
+				assert.Error(t, err)
 
 				var errStr *opensearch.StringError
-				require.True(t, errors.As(err, &errStr))
+				require.ErrorAs(t, err, &errStr)
 			})
 		})
 	})
@@ -159,7 +159,7 @@ func TestError(t *testing.T) {
 		assert.True(t, resp.IsError())
 		err := opensearch.ParseError(resp)
 		var testError *opensearch.StringError
-		require.True(t, errors.As(err, &testError))
+		require.ErrorAs(t, err, &testError)
 		assert.Equal(t, http.StatusMethodNotAllowed, testError.Status)
 		assert.Contains(t, testError.Err, "Incorrect HTTP method for uri")
 		_ = fmt.Sprintf("%s", testError)
@@ -175,7 +175,7 @@ func TestError(t *testing.T) {
 		assert.True(t, resp.IsError())
 		err := opensearch.ParseError(resp)
 		var testError *opensearch.StringError
-		require.True(t, errors.As(err, &testError))
+		require.ErrorAs(t, err, &testError)
 		assert.Equal(t, http.StatusNotFound, testError.Status)
 		assert.Contains(t, testError.Err, "{\"_index\":\"index\",\"_id\":\"2\",\"matched\":false}")
 		_ = fmt.Sprintf("%s", testError)
@@ -193,7 +193,7 @@ func TestError(t *testing.T) {
 		assert.True(t, resp.IsError())
 		err := opensearch.ParseError(resp)
 		var testError *opensearch.Error
-		require.True(t, errors.As(err, &testError))
+		require.ErrorAs(t, err, &testError)
 		assert.Contains(t, testError.Err, "no handler found for uri [/_plugins/_security/xxx] and method [GET]")
 		_ = fmt.Sprintf("%s", testError)
 	})
@@ -214,7 +214,7 @@ func TestError(t *testing.T) {
 		assert.True(t, resp.IsError())
 		err := opensearch.ParseError(resp)
 		var testError *opensearch.ReasonError
-		require.True(t, errors.As(err, &testError))
+		require.ErrorAs(t, err, &testError)
 		assert.Equal(t, "error", testError.Status)
 		assert.Contains(t, testError.Reason, "Invalid configuration")
 		_ = fmt.Sprintf("%s", testError)
@@ -230,7 +230,7 @@ func TestError(t *testing.T) {
 		assert.True(t, resp.IsError())
 		err := opensearch.ParseError(resp)
 		var testError *opensearch.MessageError
-		require.True(t, errors.As(err, &testError))
+		require.ErrorAs(t, err, &testError)
 		assert.Equal(t, "BAD_REQUEST", testError.Status)
 		assert.Contains(t, testError.Message, "Wrong request body")
 		_ = fmt.Sprintf("%s", testError)
@@ -277,7 +277,7 @@ func TestError(t *testing.T) {
 			t.Run(tt.Name, func(t *testing.T) {
 				err := opensearch.ParseError(tt.Resp)
 				for _, wantedError := range tt.WantedErrors {
-					assert.True(t, errors.Is(err, wantedError))
+					assert.ErrorIs(t, err, wantedError)
 				}
 			})
 		}
@@ -289,7 +289,7 @@ func TestError(t *testing.T) {
 			}
 			assert.True(t, resp.IsError())
 			err := opensearch.ParseError(resp)
-			assert.True(t, errors.Is(err, opensearch.ErrJSONUnmarshalBody))
+			assert.ErrorIs(t, err, opensearch.ErrJSONUnmarshalBody)
 		})
 		t.Run("too many requests", func(t *testing.T) {
 			resp := &opensearch.Response{
@@ -298,7 +298,7 @@ func TestError(t *testing.T) {
 			}
 			assert.True(t, resp.IsError())
 			err := opensearch.ParseError(resp)
-			assert.True(t, errors.Is(err, opensearch.ErrJSONUnmarshalBody))
+			assert.ErrorIs(t, err, opensearch.ErrJSONUnmarshalBody)
 		})
 	})
 }

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/config v1.32.7
 	github.com/stretchr/testify v1.11.1
 	github.com/wI2L/jsondiff v0.7.0
+	golang.org/x/mod v0.32.0
 )
 
 require (

go.sum

@@ -52,6 +52,8 @@ github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
 github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
 github.com/wI2L/jsondiff v0.7.0 h1:1lH1G37GhBPqCfp/lrs91rf/2j3DktX6qYAKZkLuCQQ=
 github.com/wI2L/jsondiff v0.7.0/go.mod h1:KAEIojdQq66oJiHhDyQez2x+sRit0vIzC9KeK0yizxM=
+golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c=
+golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=

internal/test/config.go

@@ -4,6 +4,65 @@
 // this file be licensed under the Apache-2.0 license or a
 // compatible open source license.
 
-// Package ostest provides OpenSearch testing utilities and infrastructure.
-// Integration-specific functionality is available only when building with integration tags.
+// Package ostest provides internal integration test utilities for the OpenSearch Go client.
+//
+// This package contains complex test orchestration, OpenSearch-specific utilities,
+// and internal integration test helpers. It builds upon the basic utilities
+// provided by opensearchutil/testutil but adds OpenSearch-aware functionality.
+//
+// # Design Principles
+//
+// Functions in this package follow the same design principles as testutil:
+//   - MUST take `t *testing.T` as the first parameter when applicable
+//   - MUST call `t.Helper()` as the first statement for test helpers
+//   - SHOULD delegate to testutil for basic utilities rather than duplicating them
+//
+// # When to Add Functions Here
+//
+// Add functions to this package when:
+//   - The utility requires complex OpenSearch cluster orchestration
+//   - The functionality is specific to internal integration testing
+//   - The utility needs to perform readiness checks or cluster health validation
+//   - The functionality involves OpenSearch version-specific logic
+//   - The utility is only needed by internal test packages
+//
+// # Key Functionality
+//
+// This package provides:
+//   - NewClient() - Creates OpenSearch clients with automatic readiness checking
+//   - Cluster health and readiness validation
+//   - JSON comparison utilities for response validation
+//   - OpenSearchTestSuite for structured test organization
+//   - Version-specific test utilities and skip logic
+//   - Complex client configuration scenarios
+//
+// # Relationship to opensearchutil/testutil
+//
+// This package imports and uses opensearchutil/testutil for basic utilities:
+//   - Uses testutil.ClientConfig(t) for basic client configuration
+//   - Uses testutil.IsSecure(t) for security checks
+//   - Uses testutil.GetPassword(t) for authentication
+//   - Provides enhanced functionality on top of these basics
+//
+// Basic utilities should be added to testutil, while OpenSearch-specific
+// orchestration should be added here.
+//
+// # Examples
+//
+//	// Create client with automatic readiness checking
+//	client, err := ostest.NewClient(t)
+//	require.NoError(t, err)
+//
+//	// Use test suite with version checking
+//	type MyTestSuite struct {
+//		ostest.OpenSearchTestSuite
+//	}
+//
+//	func (s *MyTestSuite) TestFeature() {
+//		s.SkipIfBelowVersion(2, 4, "FeatureName")
+//		// test implementation
+//	}
+//
+//	// Validate JSON response completeness
+//	ostest.CompareRawJSONwithParsedJSON(t, resp, resp.Inspect().Response)
 package ostest