fix tests

Signed-off-by: Sean Chittenden <sean.chittenden@crowdstrike.com>

Author: sean-

.ci/opensearch/Dockerfile.opensearch

@@ -1,4 +1,4 @@
-ARG OPENSEARCH_VERSION
+ARG OPENSEARCH_VERSION=latest
 FROM opensearchproject/opensearch:${OPENSEARCH_VERSION}
 
 ARG opensearch_path=/usr/share/opensearch

.ci/opensearch/docker-compose.yml

@@ -13,9 +13,9 @@ services:
     environment:
       - cluster.name=opensearch-cluster
       - node.name=opensearch-node1
-      - node.roles=cluster_manager,data,ingest
+      - node.roles=${OPENSEARCH_MANAGER_ROLE:-cluster_manager},data,ingest
       - discovery.seed_hosts=opensearch-node1,opensearch-node2,opensearch-node3
-      - cluster.initial_cluster_manager_nodes=opensearch-node1,opensearch-node2,opensearch-node3
+      - cluster.initial_${OPENSEARCH_MANAGER_SETTING:-cluster_manager}_nodes=opensearch-node1,opensearch-node2,opensearch-node3
       - bootstrap.memory_lock=false  # Disable memory locking for development
       - path.repo=/usr/share/opensearch/mnt
       - OPENSEARCH_INITIAL_ADMIN_PASSWORD=myStrongPassword123!
@@ -28,7 +28,7 @@ services:
       - http.publish_host=localhost
       - http.publish_port=9200
       # Memory settings
-      - OPENSEARCH_JAVA_OPTS=-Xms1g -Xmx1g
+      - OPENSEARCH_JAVA_OPTS=-Xms${OPENSEARCH_HEAP_SIZE:-1g} -Xmx${OPENSEARCH_HEAP_SIZE:-1g}
     ports:
       - "9200:9200"
       - "9300:9300"
@@ -57,9 +57,9 @@ services:
     environment:
       - cluster.name=opensearch-cluster
       - node.name=opensearch-node2
-      - node.roles=cluster_manager,data,ingest
+      - node.roles=${OPENSEARCH_MANAGER_ROLE:-cluster_manager},data,ingest
       - discovery.seed_hosts=opensearch-node1,opensearch-node2,opensearch-node3
-      - cluster.initial_cluster_manager_nodes=opensearch-node1,opensearch-node2,opensearch-node3
+      - cluster.initial_${OPENSEARCH_MANAGER_SETTING:-cluster_manager}_nodes=opensearch-node1,opensearch-node2,opensearch-node3
       - bootstrap.memory_lock=false  # Disable memory locking for development
       - path.repo=/usr/share/opensearch/mnt
       - OPENSEARCH_INITIAL_ADMIN_PASSWORD=myStrongPassword123!
@@ -72,7 +72,7 @@ services:
       - http.publish_host=localhost
       - http.publish_port=9201
       # Memory settings
-      - OPENSEARCH_JAVA_OPTS=-Xms1g -Xmx1g
+      - OPENSEARCH_JAVA_OPTS=-Xms${OPENSEARCH_HEAP_SIZE:-1g} -Xmx${OPENSEARCH_HEAP_SIZE:-1g}
     ports:
       - "9201:9200"
       - "9301:9300"
@@ -101,9 +101,9 @@ services:
     environment:
       - cluster.name=opensearch-cluster
       - node.name=opensearch-node3
-      - node.roles=cluster_manager,data,ingest
+      - node.roles=${OPENSEARCH_MANAGER_ROLE:-cluster_manager},data,ingest
       - discovery.seed_hosts=opensearch-node1,opensearch-node2,opensearch-node3
-      - cluster.initial_cluster_manager_nodes=opensearch-node1,opensearch-node2,opensearch-node3
+      - cluster.initial_${OPENSEARCH_MANAGER_SETTING:-cluster_manager}_nodes=opensearch-node1,opensearch-node2,opensearch-node3
       - bootstrap.memory_lock=false  # Disable memory locking for development
       - path.repo=/usr/share/opensearch/mnt
       - OPENSEARCH_INITIAL_ADMIN_PASSWORD=myStrongPassword123!
@@ -116,7 +116,7 @@ services:
       - http.publish_host=localhost
       - http.publish_port=9202
       # Memory settings
-      - OPENSEARCH_JAVA_OPTS=-Xms1g -Xmx1g
+      - OPENSEARCH_JAVA_OPTS=-Xms${OPENSEARCH_HEAP_SIZE:-1g} -Xmx${OPENSEARCH_HEAP_SIZE:-1g}
     ports:
       - "9202:9200"
       - "9302:9300"

.gitignore

@@ -6,4 +6,7 @@ tmp/
 .vscode
 bin/
 
-.DS_Store
+.DS_Store
+# Admin certificates for testing
+admin.pem
+admin.key

CHANGELOG.md

@@ -43,6 +43,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 ### Fixed
 
 - Fix flaky connection integration test by replacing arbitrary sleep times with proper server readiness polling
+- Fix OpenSearch 2.8.0+ Tasks API compatibility by adding cancellation_time_millis field to TasksListTask struct
 - Fix OpenSearch 3.1.0+ API compatibility by adding phase_results_processors field to nodes API and time_in_execution fields to cluster pending tasks API
 - Fix OpenSearch 3.2.0+ API compatibility by adding max_last_index_request_timestamp and startree query fields across nodes stats, indices stats, and cat APIs, plus settings field to security plugin health API
 - Fix OpenSearch 3.3.0+ API compatibility by adding neural_search breaker, query_failed and startree_query_failed search fields, search pipeline system_generated fields across multiple APIs, plus ingestion_status field to cluster state API and jwks_uri field to security config API

Makefile

@@ -33,7 +33,8 @@ endif
 ifdef race
 	$(eval testintegargs += "-race")
 endif
-	$(eval testintegargs += "-cover" "-tags=$(testintegtags)" "-timeout=1h" "./..." "-args" "-test.gocoverdir=$(PWD)/tmp/integration")
+	$(eval TEST_PARALLEL ?= $(shell ncpu=$$(go env GOMAXPROCS 2>/dev/null); [ -z "$$ncpu" ] && ncpu=$$(sysctl -n hw.ncpu 2>/dev/null || nproc 2>/dev/null || echo 4); parallel=$$((ncpu - 1)); [ $$parallel -lt 1 ] && parallel=1; echo $$parallel))
+	$(eval testintegargs += "-cover" "-tags=$(testintegtags)" "-timeout=10m" "-parallel=$(TEST_PARALLEL)" "./..." "-args" "-test.gocoverdir=$(PWD)/tmp/integration")
 	@mkdir -p $(PWD)/tmp/integration
 	@echo "go test -v" $(testintegargs); \
 	go test -v $(testintegargs);
@@ -126,10 +127,10 @@ lint.markdown:
 		npm install -g $(package) --no-shrinkwrap; \
 	fi
 	@printf "\033[2m-> Running markdown lint...\033[0m\n"
-	if npx $(package) --prose-wrap never --check **/*.md; [[ $$? -ne 0 ]]; then \
+	if npx $(package) --prose-wrap never --check **/*.md; [ $$? -ne 0 ]; then \
 		echo -e "\033[32m-> Found invalid files. Want to auto-format invalid files? (y/n) \033[0m"; \
 		read RESP; \
-		if [[ $$RESP = "y" || $$RESP = "Y" ]]; then \
+		if [ "$$RESP" = "y" ] || [ "$$RESP" = "Y" ]; then \
 		  echo -e "\033[33m Formatting...\033[0m"; \
 		  npx $(package) --prose-wrap never --write **/*.md; \
 		  echo -e "\033[34m \nAll invalid files are formatted\033[0m"; \
@@ -197,7 +198,7 @@ endif
 		set -e -o pipefail; \
 		printf "\033[2m-> Commit and create Git tag? (y/n): \033[0m\c"; \
 		read continue; \
-		if [[ $$continue == "y" ]]; then \
+		if [ "$$continue" = "y" ]; then \
 			git add internal/version/version.go && \
 			git commit --no-status --quiet --message "Release $(version)" && \
 			git tag --annotate v$(version) --message 'Release $(version)'; \
@@ -225,14 +226,57 @@ godoc: ## Display documentation for the package
 	godoc --http=localhost:6060 --play
 
 cluster.build:
-	docker compose --project-directory .ci/opensearch build --pull;
+	@$(MAKE) cluster.docker-build
 
 cluster.start:
-	docker compose --project-directory .ci/opensearch up -d;
+	@$(MAKE) cluster.docker-up
+	@$(MAKE) cluster.get-cert
 
 cluster.stop:
 	docker compose --project-directory .ci/opensearch down;
 
+cluster.docker-build:
+	@# Determine version-specific settings
+	$(eval OPENSEARCH_VERSION ?= latest)
+	$(eval version_major := $(shell \
+		if [ "$(OPENSEARCH_VERSION)" = "latest" ]; then \
+			echo "2"; \
+		else \
+			echo "$(OPENSEARCH_VERSION)" | awk -F. '{print $$1}'; \
+		fi \
+	))
+	$(eval manager_role := $(shell \
+		if [ "$(version_major)" = "1" ]; then \
+			echo "master"; \
+		else \
+			echo "cluster_manager"; \
+		fi \
+	))
+	@echo "Building OpenSearch $(OPENSEARCH_VERSION) with role: $(manager_role)"
+	OPENSEARCH_MANAGER_ROLE=$(manager_role) OPENSEARCH_MANAGER_SETTING=$(manager_role) \
+		docker compose --project-directory .ci/opensearch build --pull
+
+cluster.docker-up:
+	@# Determine version-specific settings
+	$(eval OPENSEARCH_VERSION ?= latest)
+	$(eval version_major := $(shell \
+		if [ "$(OPENSEARCH_VERSION)" = "latest" ]; then \
+			echo "2"; \
+		else \
+			echo "$(OPENSEARCH_VERSION)" | awk -F. '{print $$1}'; \
+		fi \
+	))
+	$(eval manager_role := $(shell \
+		if [ "$(version_major)" = "1" ]; then \
+			echo "master"; \
+		else \
+			echo "cluster_manager"; \
+		fi \
+	))
+	@echo "Starting OpenSearch $(OPENSEARCH_VERSION) with role: $(manager_role)"
+	OPENSEARCH_MANAGER_ROLE=$(manager_role) OPENSEARCH_MANAGER_SETTING=$(manager_role) \
+		docker compose --project-directory .ci/opensearch up -d
+
 cluster.scale.1: ## Start single-node cluster
 	docker compose --project-directory .ci/opensearch up -d --scale opensearch-node2=0 --scale opensearch-node3=0;
 
@@ -243,14 +287,24 @@ cluster.scale.3: ## Start full 3-node cluster
 	docker compose --project-directory .ci/opensearch up -d --scale opensearch-node1=1 --scale opensearch-node2=1 --scale opensearch-node3=1;
 
 cluster.get-cert:
-	@if [[ -v SECURE_INTEGRATION ]] && [[ $$SECURE_INTEGRATION == "true" ]]; then \
-		docker cp $$(docker compose --project-directory .ci/opensearch ps --format '{{.Name}}' | head -1):/usr/share/opensearch/config/kirk.pem admin.pem && \
-		docker cp $$(docker compose --project-directory .ci/opensearch ps --format '{{.Name}}' | head -1):/usr/share/opensearch/config/kirk-key.pem admin.key; \
+	@if [ -n "$${SECURE_INTEGRATION}" ] && [ "$${SECURE_INTEGRATION}" = "true" ]; then \
+		CONTAINER=$$(docker compose --project-directory .ci/opensearch ps --format '{{.Name}}' | head -1); \
+		if [ -z "$$CONTAINER" ]; then \
+			echo "Error: No OpenSearch containers running. Start cluster first with 'make cluster.start'"; \
+			exit 1; \
+		fi; \
+		docker cp $$CONTAINER:/usr/share/opensearch/config/kirk.pem admin.pem && \
+		docker cp $$CONTAINER:/usr/share/opensearch/config/kirk-key.pem admin.key; \
 	fi
 
 
 cluster.clean: ## Remove unused Docker volumes and networks
 	@printf "\033[2m-> Cleaning up Docker assets...\033[0m\n"
+	@# Remove OpenSearch built images to ensure clean rebuilds when switching versions
+	@docker images -q opensearch-opensearch-node* | xargs -r docker rmi -f || true
+	@# Remove OpenSearch volumes to clear stale data
+	@docker volume ls -q --filter "name=opensearch" | xargs -r docker volume rm || true
+	@# Clean up unused Docker resources
 	docker volume prune --force
 	docker network prune --force
 	docker system prune --volumes --force
@@ -281,5 +335,5 @@ help:  ## Display help
 #------------- <https://suva.sh/posts/well-documented-makefiles> --------------
 
 .DEFAULT_GOAL := help
-.PHONY: help backport cluster cluster.clean coverage godoc lint lint.local release test test-all test-race test-bench test-integ test-unit linters linters.install
+.PHONY: help backport cluster.build cluster.start cluster.stop cluster.docker-build cluster.docker-up cluster.clean coverage godoc lint lint.local release test test-all test-race test-bench test-integ test-unit linters linters.install
 .SILENT: lint.markdown

go.mod

@@ -30,5 +30,6 @@ require (
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.1 // indirect
 	github.com/tidwall/sjson v1.2.5 // indirect
+	golang.org/x/mod v0.33.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -52,6 +52,8 @@ github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
 github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
 github.com/wI2L/jsondiff v0.7.0 h1:1lH1G37GhBPqCfp/lrs91rf/2j3DktX6qYAKZkLuCQQ=
 github.com/wI2L/jsondiff v0.7.0/go.mod h1:KAEIojdQq66oJiHhDyQez2x+sRit0vIzC9KeK0yizxM=
+golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8=
+golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=

internal/test/config.go

@@ -8,13 +8,25 @@ package ostest
 
 import (
 	"crypto/tls"
+	"fmt"
 	"net/http"
 	"os"
 
+	"golang.org/x/mod/semver"
+
 	"github.com/opensearch-project/opensearch-go/v4"
 	"github.com/opensearch-project/opensearch-go/v4/opensearchapi"
 )
 
+const (
+	// OpenSearch default admin passwords
+	defaultPasswordPre212  = "admin"                // Default admin password in OpenSearch < 2.12.0
+	defaultPasswordPost212 = "myStrongPassword123!" // Default admin password in OpenSearch >= 2.12.0
+
+	// Version where default admin password changed
+	defaultPasswordChangeVersion = "v2.12.0"
+)
+
 // IsSecure returns true when SECURE_INTEGRATION env is set to true
 func IsSecure() bool {
 	return os.Getenv("SECURE_INTEGRATION") == "true"
@@ -48,25 +60,61 @@ func ClientConfig() (*opensearchapi.Config, error) {
 	}, nil
 }
 
-// GetPassword returns the password suited for the opensearch version
+// GetPassword returns the admin password for the opensearch version.
+// OpenSearch 2.12.0+ changed the default admin password from "admin" to "myStrongPassword123!".
+//
+// Note: This function tries to determine the correct password based on OPENSEARCH_VERSION env var.
+// If the env var doesn't match the actual running cluster, authentication may fail.
 func GetPassword() (string, error) {
-	var (
-		major, minor int64
-		err          error
-	)
-	password := "admin"
 	version := os.Getenv("OPENSEARCH_VERSION")
 
-	if version != "latest" && version != "" {
-		major, minor, _, err = opensearch.ParseVersion(version)
-		if err != nil {
-			return "", err
+	// Default to pre-2.12 password for empty version or versions < 2.12.0
+	password := defaultPasswordPre212
+
+	if version == "latest" {
+		// Latest uses the post-2.12 default password
+		password = defaultPasswordPost212
+	} else if version != "" {
+		// Normalize version to semver format (v2.12.0)
+		if version[0] != 'v' {
+			version = "v" + version
+		}
+
+		// Validate semver format
+		if !semver.IsValid(version) {
+			return "", fmt.Errorf("invalid version format: %s", version)
 		}
-		if version == "latest" || major > 2 || (major == 2 && minor >= 12) {
-			password = "myStrongPassword123!"
+
+		// OpenSearch 2.12.0+ uses the new default password
+		if semver.Compare(version, defaultPasswordChangeVersion) >= 0 {
+			password = defaultPasswordPost212
 		}
-	} else {
-		password = "myStrongPassword123!"
 	}
+
 	return password, nil
 }
+
+// GetPasswordForCluster returns the admin password by trying to detect the actual cluster version.
+// This is more reliable than GetPassword() when OPENSEARCH_VERSION env var might not match reality.
+// It returns both possible passwords to try in order.
+func GetPasswordForCluster() []string {
+	version := os.Getenv("OPENSEARCH_VERSION")
+
+	// If version is set and valid, trust it
+	if version != "" && version != "latest" {
+		if version[0] != 'v' {
+			version = "v" + version
+		}
+		if semver.IsValid(version) {
+			if semver.Compare(version, defaultPasswordChangeVersion) >= 0 {
+				// For 2.12+, try post-2.12 password first, then fallback to pre-2.12
+				return []string{defaultPasswordPost212, defaultPasswordPre212}
+			}
+			// For < 2.12, try pre-2.12 password first, then fallback to post-2.12
+			return []string{defaultPasswordPre212, defaultPasswordPost212}
+		}
+	}
+
+	// Unknown or latest version: try pre-2.12 password first (most common), then post-2.12
+	return []string{defaultPasswordPre212, defaultPasswordPost212}
+}