Makefile

SHELL := /bin/bash

# Tool versions
GOLANGCI_LINT_VERSION := v2.12.2

# Build tags for linting.
#
# golangci-lint can only compile ONE point in the build-tag space per run, so a
# single invocation can never lint every file. Two mutually-exclusive boolean
# axes partition the tree:
#   - integration vs !integration: every unit *_test.go is `!integration`; every
#     integration test is `integration`. Setting `integration` drops all unit
#     test files from the type-check, and omitting it drops all integration ones.
#   - multinode vs !multinode (within integration only): the single-node
#     integration files are `!multinode`; the multinode ones require `multinode`.
# The feature tags below are pure-OR alternatives -- `core` satisfies every
# `(core || X)` constraint and `plugins` every `(plugins || X)` one -- so they
# union harmlessly into every run. Complete coverage therefore requires running
# golangci-lint once per (integration, multinode) combination with the feature
# tags unioned in. GOLANGCI_LINT_TAG_SETS enumerates those runs; each element is
# one quoted --build-tags argument. GOLANGCI_LINT_BUILD_TAGS is retained as the
# maximal set for the osgen module, which has no build-constrained files.
GOLANGCI_LINT_FEATURE_TAGS := core plugins plugin_security plugin_index_management
GOLANGCI_LINT_BUILD_TAGS := "integration $(GOLANGCI_LINT_FEATURE_TAGS) multinode"
GOLANGCI_LINT_TAG_SETS := \
	"$(GOLANGCI_LINT_FEATURE_TAGS)" \
	"integration $(GOLANGCI_LINT_FEATURE_TAGS)" \
	"integration $(GOLANGCI_LINT_FEATURE_TAGS) multinode"

# Container provider detection: prefer Colima, then Rancher Desktop, then Docker.
# A provider is the machine/daemon backing the containers; it drives the docker
# context commands talk to, the CLI runtime $(CTR) resolves to, whether the VM
# is started (cluster.provider.ensure), and how vm.max_map_count is set
# (cluster.sysctl). Override with CONTAINER_PROVIDER=colima|rancher|docker.
# Detection is by CLI presence in the order colima -> rancher (rdctl) -> docker.
ifndef CONTAINER_PROVIDER
  ifneq (,$(shell command -v colima 2>/dev/null))
    CONTAINER_PROVIDER := colima
  else ifneq (,$(shell command -v rdctl 2>/dev/null))
    CONTAINER_PROVIDER := rancher
  else ifneq (,$(shell command -v docker 2>/dev/null))
    CONTAINER_PROVIDER := docker
  endif
endif

# Docker context to pin per provider. Colima and Rancher Desktop each register a
# named docker context; the plain docker provider leaves the active context
# alone. Exported below so every recipe shell targets the right daemon.
PROVIDER_CONTEXT := $(if $(filter colima,$(CONTAINER_PROVIDER)),colima,$(if $(filter rancher,$(CONTAINER_PROVIDER)),rancher-desktop,))
ifneq ($(PROVIDER_CONTEXT),)
  ifneq ($(origin DOCKER_CONTEXT),environment)
    export DOCKER_CONTEXT := $(PROVIDER_CONTEXT)
  endif
endif

# Container runtime (CLI): defaults to docker for every provider, since Colima
# and Rancher Desktop both ship a docker shim and register a docker context.
# Override with CONTAINER_RUNTIME=nerdctl for advanced containerd use (docker
# context pinning does not apply to nerdctl).
# Detection is lazy: only targets that use $(CTR) or $(CTR_COMPOSE) will fail
# if no runtime is available, allowing test-unit to work without a provider.
ifndef CONTAINER_RUNTIME
  ifneq (,$(shell command -v docker 2>/dev/null))
    CONTAINER_RUNTIME := docker
  else ifneq (,$(shell command -v nerdctl 2>/dev/null))
    CONTAINER_RUNTIME := nerdctl
  endif
endif
CTR = $(if $(CONTAINER_RUNTIME),$(CONTAINER_RUNTIME),$(error No container runtime found. Install a provider (colima, rancher, or docker) or set CONTAINER_RUNTIME.))
REPO_ROOT := $(shell git rev-parse --show-toplevel 2>/dev/null || pwd)

# Sub-modules are auto-discovered: every go.mod except the root one and testdata
# fixtures. New nested modules (cmd/*, osprom, osotel, ...) are picked up by
# test-unit and lint with no Makefile change. testdata/ holds corpus fixtures
# (stub + golden modules for the rewrite tests), not real modules to build or
# lint. Paths are sorted for deterministic ordering.
SUBMODULES := $(shell find . -name go.mod -not -path './go.mod' -not -path '*/.*' -not -path '*/testdata/*' -exec dirname {} \; | sed 's|^\./||' | sort)

# Compose with optional override files for heterogeneous clusters.
# Override files are generated by cluster.heterogeneous.* targets and removed
# by cluster.homogeneous. When no overrides exist, CTR_COMPOSE falls back
# to the default: $(CTR) compose --project-directory .ci/opensearch.
COMPOSE_DIR := .ci/opensearch
COMPOSE_OVERRIDES = $(wildcard $(COMPOSE_DIR)/docker-compose.*-override.yml)
COMPOSE_FILES = $(if $(COMPOSE_OVERRIDES),-f $(COMPOSE_DIR)/docker-compose.yml $(addprefix -f ,$(COMPOSE_OVERRIDES)))
CTR_COMPOSE = $(CTR) compose --project-directory $(COMPOSE_DIR) $(COMPOSE_FILES)

##@ Formatting
format:  ## Format all Go files with goimports
	goimports -w .;

##@ Testing
test-unit:  ## Run unit tests across all modules (root + every nested go.mod)
	@printf "\033[2m-> Running unit tests...\033[0m\n"
ifdef race
	$(eval testunitargs += "-race")
	$(eval submoduleargs += -race)
endif
	$(eval testunitargs += "-cover" "./..." "-args" "-test.gocoverdir=$(PWD)/tmp/unit")
	@rm -rf $(PWD)/tmp/unit
	@mkdir -p $(PWD)/tmp/unit
	@echo "go test -v" $(testunitargs); \
	go test -v $(testunitargs) 2>&1 | tee test-unit.log; \
	exit $${PIPESTATUS[0]};
	@for mod in $(SUBMODULES); do \
		tag=$$(echo "$$mod" | tr '/' '-'); \
		printf "\033[2m-> Running %s unit tests (separate module)...\033[0m\n" "$$mod"; \
		modargs="$(submoduleargs)"; \
		if [ -n "$(coverage)" ]; then \
			rm -rf "$(PWD)/tmp/$$tag"; mkdir -p "$(PWD)/tmp/$$tag"; \
			modargs="$$modargs -cover ./... -args -test.gocoverdir=$(PWD)/tmp/$$tag"; \
		else \
			modargs="$$modargs ./..."; \
		fi; \
		echo "(cd $$mod && go test -v $$modargs)"; \
		(cd "$$mod" && go test -v $$modargs) 2>&1 | tee "$(PWD)/test-$$tag.log"; \
		rc=$${PIPESTATUS[0]}; \
		if [ "$$rc" -ne 0 ]; then exit "$$rc"; fi; \
	done
ifdef coverage
	@go tool covdata textfmt -i=$(PWD)/tmp/unit -o $(PWD)/tmp/unit.cov
	@for mod in $(SUBMODULES); do \
		tag=$$(echo "$$mod" | tr '/' '-'); \
		go tool covdata textfmt -i="$(PWD)/tmp/$$tag" -o "$(PWD)/tmp/$$tag.cov"; \
	done
endif
test: test-unit

test-integ:  ## Run integration tests
	@printf "\033[2m-> Running integration tests...\033[0m\n"
	$(eval testintegtags += "integration,core,plugins")
	$(eval testintegdir ?= integration)
	$(eval OPENSEARCH_NODE_COUNT ?= 3)
ifdef multinode
	$(eval testintegtags += "multinode")
endif
ifdef race
	$(eval testintegargs += "-race")
endif
	$(eval TEST_PARALLEL ?= $(shell ncpu=$$(sysctl -n hw.ncpu 2>/dev/null || nproc 2>/dev/null || echo 4); parallel=$$((ncpu / 2)); [ $$parallel -lt 1 ] && parallel=1; echo $$parallel))
	$(eval TEST_TIMEOUT ?= 10m)
	$(eval testintegargs += "-cover" "-tags=$(testintegtags)" "-timeout=$(TEST_TIMEOUT)" "-parallel=$(TEST_PARALLEL)" "./..." "-args" "-test.gocoverdir=$(PWD)/tmp/$(testintegdir)")
	@rm -rf $(PWD)/tmp/$(testintegdir)
	@mkdir -p $(PWD)/tmp/$(testintegdir)
	@echo "go test -v" $(testintegargs); \
	OPENSEARCH_NODE_COUNT=$(OPENSEARCH_NODE_COUNT) go test -v $(testintegargs) 2>&1 | tee test-integ.log; \
	exit $${PIPESTATUS[0]};
ifdef coverage
	@go tool covdata textfmt -i=$(PWD)/tmp/$(testintegdir) -o $(PWD)/tmp/$(testintegdir).cov
endif

test-integ-core:  ## Run base integration tests
	@$(MAKE) test-integ testintegtags=integration,core testintegdir=integration-core

test-integ-plugins:  ## Run plugin integration tests
	@$(MAKE) test-integ testintegtags=integration,plugins testintegdir=integration-plugins

test-integ-secure:  ## Run secure integration tests
	@SECURE_INTEGRATION=true $(MAKE) test-integ

test-integ-core-secure:  ## Run secure base integration tests
	@SECURE_INTEGRATION=true $(MAKE) test-integ testintegtags=integration,core

test-integ-plugins-secure:  ## Run secure plugin integration tests
	@SECURE_INTEGRATION=true $(MAKE) test-integ testintegtags=integration,plugins

test-all:  ## Run all tests with all build tags (unit + integration)
	@printf "\033[2m-> Running all unit tests...\033[0m\n"
	@$(MAKE) test-unit
	@printf "\033[2m-> Running all integration tests with all tags...\033[0m\n"
	@$(MAKE) test-integ testintegtags=integration,core,plugins,plugin_security,plugin_index_management,multinode

test-race:  ## Run all tests with race detection enabled
	@printf "\033[2m-> Running all unit tests with race detection...\033[0m\n"
	@$(MAKE) test-unit race=true
	@printf "\033[2m-> Running all integration tests with race detection and all tags...\033[0m\n"
	@$(MAKE) test-integ race=true testintegtags=integration,core,plugins,plugin_security,plugin_index_management,multinode

test-bench:  ## Run benchmarks
	@printf "\033[2m-> Running benchmarks...\033[0m\n"
	go test -run=none -bench=. -benchmem -benchtime=200ms ./...

build-samples:  ## Compile and vet each _samples/*.go program
	@printf "\033[2m-> Building _samples...\033[0m\n"
	@failed=0; \
	for f in _samples/*.go; do \
		printf "   %s\n" "$$f"; \
		go build -o /dev/null "$$f" || failed=1; \
		go vet "$$f" || failed=1; \
	done; \
	if [ $$failed -ne 0 ]; then \
		printf "\033[31m-> _samples build/vet failed\033[0m\n"; \
		exit 1; \
	fi

coverage:  ## Print test coverage report
	@$(MAKE) gen-coverage
	@go tool cover -func=$(PWD)/tmp/total.cov
	@printf "\033[0m--------------------------------------------------------------------------------\n\033[0m"

coverage-html: ## Open test coverage report in browser
	@$(MAKE) gen-coverage
	@go tool cover -html $(PWD)/tmp/total.cov

gen-coverage:  ## Generate test coverage report
	@printf "\033[2m-> Generating test coverage report...\033[0m\n"
	@rm -rf tmp
	@mkdir tmp
	@mkdir tmp/unit
	@mkdir tmp/integration
	@$(MAKE) test-unit coverage=true
	@$(MAKE) test-integ coverage=true
	@$(MAKE) build-coverage

build-coverage:
	@go tool covdata textfmt -i=$(PWD)/tmp/unit,$(PWD)/tmp/integration -o $(PWD)/tmp/total.cov

OPENAPI_SPEC := $(REPO_ROOT)/opensearch-openapi.yaml
OPENAPI_SPEC_URL := https://github.com/opensearch-project/opensearch-api-specification/releases/latest/download/opensearch-openapi.yaml

# Generated code output directories.
GEN_PATH_DIR    := $(REPO_ROOT)/internal/path
GEN_OSAPI_DIR   := $(REPO_ROOT)/opensearchapi
GEN_PLUGINS_DIR := $(REPO_ROOT)/plugins

# Version filtering defaults for code generation.
# Override on the command line to scope generated code to a version window:
#   make gen GEN_MIN_VERSION=2.0 GEN_REMOVE_DEPRECATED=2.0
GEN_MIN_VERSION        ?= epoch
GEN_MAX_VERSION        ?= latest
GEN_REMOVE_DEPRECATED  ?= epoch

##@ Development
fetch-opensearch-spec: ## Download the OpenSearch OpenAPI spec (skips if already present)
	@if [ ! -f "$(OPENAPI_SPEC)" ]; then \
		printf "\033[2m-> Downloading %s...\033[0m\n" "$(OPENAPI_SPEC)"; \
		curl -sSfL "$(OPENAPI_SPEC_URL)" -o "$(OPENAPI_SPEC)"; \
	else \
		printf "\033[2m-> %s already present\033[0m\n" "$(OPENAPI_SPEC)"; \
	fi

fetch-opensearch-spec-force: ## Re-download the OpenSearch OpenAPI spec from upstream
	@printf "\033[2m-> Downloading %s...\033[0m\n" "$(OPENAPI_SPEC)"
	@curl -sSfL "$(OPENAPI_SPEC_URL)" -o "$(OPENAPI_SPEC)"

clean-gen:  ## Remove all generated Go files (opensearchapi, plugins, internal/path)
	@printf "\033[2m-> Removing generated files...\033[0m\n"
	@rm -f $(GEN_PATH_DIR)/builders_gen.go $(GEN_PATH_DIR)/builders_gen_test.go
	@rm -f $(GEN_OSAPI_DIR)/*_gen.go $(GEN_OSAPI_DIR)/*_gen_test.go
	@find $(GEN_PLUGINS_DIR) -name '*_gen.go' -o -name '*_gen_test.go' | xargs rm -f 2>/dev/null || true

gen-paths: fetch-opensearch-spec  ## Regenerate path builders only
	@printf "\033[2m-> Regenerating path builders...\033[0m\n"
	cd $(REPO_ROOT)/cmd/osgen && go run . paths \
		-spec $(OPENAPI_SPEC) \
		-pkg path \
		-o $(GEN_PATH_DIR)/builders_gen.go \
		-test-out $(GEN_PATH_DIR)/builders_gen_test.go \
		-min-version=$(GEN_MIN_VERSION) \
		-max-version=$(GEN_MAX_VERSION) \
		-remove-deprecated=$(GEN_REMOVE_DEPRECATED)

gen-api: fetch-opensearch-spec  ## Regenerate API consumer files only
	@printf "\033[2m-> Regenerating API consumer files...\033[0m\n"
	cd $(REPO_ROOT)/cmd/osgen && go run . api \
		-spec $(OPENAPI_SPEC) \
		-out $(GEN_OSAPI_DIR) \
		-pkg opensearchapi \
		-plugins-out $(GEN_PLUGINS_DIR) \
		-min-version=$(GEN_MIN_VERSION) \
		-max-version=$(GEN_MAX_VERSION) \
		-remove-deprecated=$(GEN_REMOVE_DEPRECATED)

gen-api-update-rawlist: fetch-opensearch-spec  ## Regenerate API files and refresh the json.RawMessage allowlist
	@printf "\033[2m-> Regenerating API consumer files and json.RawMessage allowlist...\033[0m\n"
	cd $(REPO_ROOT)/cmd/osgen && go run . api \
		-spec $(OPENAPI_SPEC) \
		-out $(GEN_OSAPI_DIR) \
		-pkg opensearchapi \
		-plugins-out $(GEN_PLUGINS_DIR) \
		-min-version=$(GEN_MIN_VERSION) \
		-max-version=$(GEN_MAX_VERSION) \
		-remove-deprecated=$(GEN_REMOVE_DEPRECATED) \
		-update-raw-message-allowlist

gen: gen-paths gen-api  ## Regenerate all code from OpenAPI spec (run gen-paths and gen-api in parallel with `make -j gen`)

regen: clean-gen gen  ## Clean generated files then regenerate from spec

test-gen: regen  ## Regen then run unit + integration tests (ensures tests use fresh output)
	@$(MAKE) test-unit
	@printf "\033[2m-> Running integration tests...\033[0m\n"
	$(eval SECURE_INTEGRATION ?= true)
	@SECURE_INTEGRATION=$(SECURE_INTEGRATION) go test -v -tags=integration -count=1 -timeout=5m ./opensearchapi/...

lint:  ## Run lint on the package
	@$(MAKE) linters

lint.headers:  ## Check license headers on all Go files (same check as CI)
	@.github/check-license-headers.sh

lint.local:  ## Run lint locally (not in Docker) across all build-tag combinations
	@printf "\033[2m-> Running golangci-lint locally across all build-tag sets...\033[0m\n"
	@for tags in $(GOLANGCI_LINT_TAG_SETS); do \
		printf "\033[2m   --build-tags %s\033[0m\n" "$$tags"; \
		golangci-lint run --fix --build-tags "$$tags" --timeout=5m -v ./... || exit $$?; \
	done
	@for mod in $(SUBMODULES); do \
		printf "\033[2m-> Running golangci-lint in %s (separate Go module)...\033[0m\n" "$$mod"; \
		(cd "$$mod" && golangci-lint run --fix --build-tags $(GOLANGCI_LINT_BUILD_TAGS) --timeout=5m -v ./...) || exit $$?; \
	done

package := "prettier"
PRETTIER_ARGS := --prose-wrap never --print-width 300 "**/*.md"

lint.markdown:  ## Reformat markdown files with prettier
	npx $(package) $(PRETTIER_ARGS) --write


backport: ## Backport one or more commits from main into version branches
ifeq ($(origin commits), undefined)
	@echo "Missing commit(s), exiting..."
	@exit 2
endif
ifndef branches
	$(eval branches_list = '1.x')
else
	$(eval branches_list = $(shell echo $(branches) | tr ',' ' ') )
endif
	$(eval commits_list = $(shell echo $(commits) | tr ',' ' '))
	@printf "\033[2m-> Backporting commits [$(commits)]\033[0m\n"
	@{ \
		set -e -o pipefail; \
		for commit in $(commits_list); do \
			git show --pretty='%h | %s' --no-patch $$commit; \
		done; \
		echo ""; \
		for branch in $(branches_list); do \
			printf "\033[2m-> $$branch\033[0m\n"; \
			git checkout $$branch; \
			for commit in $(commits_list); do \
				git cherry-pick -x $$commit; \
			done; \
			git status --short --branch; \
			echo ""; \
		done; \
		printf "\033[2m-> Push updates to Github:\033[0m\n"; \
		for branch in $(branches_list); do \
			echo "git push --verbose origin $$branch"; \
		done; \
	}

release: ## Release a new version to Github
	$(eval branch = $(shell git rev-parse --abbrev-ref HEAD))
	$(eval current_version = $(shell cat internal/version/version.go | sed -Ee 's/const Client = "(.*)"/\1/' | tail -1))
	@printf "\033[2m-> [$(branch)] Current version: $(current_version)...\033[0m\n"
ifndef version
	@printf "\033[31m[!] Missing version argument, exiting...\033[0m\n"
	@exit 2
endif
ifeq ($(version), "")
	@printf "\033[31m[!] Empty version argument, exiting...\033[0m\n"
	@exit 2
endif
	@printf "\033[2m-> [$(branch)] Creating version $(version)...\033[0m\n"
	@{ \
		set -e -o pipefail; \
		cp internal/version/version.go internal/version/version.go.OLD && \
		cat internal/version/version.go.OLD | sed -e 's/Client = ".*"/Client = "$(version)"/' > internal/version/version.go && \
		go vet internal/version/version.go && \
		go fmt internal/version/version.go && \
		git diff --color-words internal/version/version.go | tail -n 1; \
	}
	@{ \
		set -e -o pipefail; \
		printf "\033[2m-> Commit and create Git tag? (y/n): \033[0m\c"; \
		read continue; \
		if [ "$$continue" = "y" ]; then \
			git add internal/version/version.go && \
			git commit --no-status --quiet --message "Release $(version)" && \
			git tag --annotate v$(version) --message 'Release $(version)'; \
			printf "\033[2m-> Push `git show --pretty='%h (%s)' --no-patch HEAD` to Github:\033[0m\n\n"; \
			printf "\033[1m  git push origin HEAD && git push origin v$(version)\033[0m\n\n"; \
			mv internal/version/version.go.OLD internal/version/version.go && \
			git add internal/version/version.go && \
			original_version=`cat internal/version/version.go | sed -ne 's;^const Client = "\(.*\)"$$;\1;p'` && \
			git commit --no-status --quiet --message "Update version to $$original_version"; \
			printf "\033[2m-> Version updated to [$$original_version].\033[0m\n\n"; \
		else \
			echo "Aborting..."; \
			rm internal/version/version.go.OLD; \
			exit 1; \
		fi; \
	}

godoc: ## Display documentation for the package
	@printf "\033[2m-> Generating documentation...\033[0m\n"
	@echo "* http://localhost:6060/pkg/github.com/opensearch-project/opensearch-go"
	@echo "* http://localhost:6060/pkg/github.com/opensearch-project/opensearch-go/opensearchapi"
	@echo "* http://localhost:6060/pkg/github.com/opensearch-project/opensearch-go/opensearchtransport"
	@echo "* http://localhost:6060/pkg/github.com/opensearch-project/opensearch-go/opensearchutil"
	@printf "\n"
	godoc --http=localhost:6060 --play

workflow:  ## Run full CI workflow locally (lint, test, integration)
# Lint
	$(MAKE) lint
# License Checker
	.github/check-license-headers.sh
# Unit Test
	$(MAKE) test-unit race=true
# Benchmarks Test
	$(MAKE) test-bench
# Integration Test
### OpenSearch
	$(MAKE) cluster.clean cluster.build cluster.start
	$(MAKE) test-integ race=true
	$(MAKE) cluster.stop

##@ Cluster Lifecycle
cluster.runtime:  ## Show detected container provider, docker context, and runtime
	@echo "Container provider: $(if $(CONTAINER_PROVIDER),$(CONTAINER_PROVIDER),(none detected))"
	@echo "Docker context:     $(if $(DOCKER_CONTEXT),$(DOCKER_CONTEXT),(active default))"
	@echo "Container runtime:  $(CTR)"
	@$(CTR) --version
	@$(CTR) compose version

cluster.provider.ensure:  ## Ensure the selected provider's VM/daemon is running (starts it if needed)
	@case "$(CONTAINER_PROVIDER)" in \
	colima) \
		if colima status >/dev/null 2>&1; then \
			printf "\033[2m-> Colima already running\033[0m\n"; \
		else \
			printf "\033[2m-> Starting Colima...\033[0m\n"; colima start; \
		fi ;; \
	rancher) \
		if rdctl shell true >/dev/null 2>&1; then \
			printf "\033[2m-> Rancher Desktop already running\033[0m\n"; \
		else \
			printf "\033[2m-> Starting Rancher Desktop...\033[0m\n"; rdctl start; \
		fi ;; \
	docker) \
		if $(CTR) info >/dev/null 2>&1; then \
			printf "\033[2m-> Docker daemon already running\033[0m\n"; \
		elif [ "$$(uname)" = "Darwin" ]; then \
			printf "\033[2m-> Starting Docker Desktop...\033[0m\n"; open -a Docker; \
			for i in $$(seq 30); do $(CTR) info >/dev/null 2>&1 && break; sleep 2; done; \
			$(CTR) info >/dev/null 2>&1 || { echo "Docker daemon did not become ready"; exit 1; }; \
		else \
			echo "Docker daemon is not running; start it and retry."; exit 1; \
		fi ;; \
	*) echo "No container provider detected. Install colima, rancher, or docker, or set CONTAINER_PROVIDER."; exit 1 ;; \
	esac

cluster.sysctl:  ## Ensure vm.max_map_count is set for OpenSearch (Linux, or macOS via Colima/Rancher/Docker)
	@if [ "$$(uname)" != "Darwin" ]; then \
		vmexec=""; setter="sudo sysctl -w vm.max_map_count=262144"; \
	elif [ "$(CONTAINER_PROVIDER)" = "colima" ]; then \
		vmexec="colima ssh --"; setter="colima ssh -- sudo sysctl -w vm.max_map_count=262144"; \
	elif [ "$(CONTAINER_PROVIDER)" = "rancher" ]; then \
		vmexec="rdctl shell"; setter="rdctl shell sudo sysctl -w vm.max_map_count=262144"; \
	else \
		vmexec="$(CTR) run --rm --privileged --net=host busybox"; \
		setter="$(CTR) run --rm --privileged --net=host busybox sysctl -w vm.max_map_count=262144"; \
	fi; \
	current=$$($$vmexec cat /proc/sys/vm/max_map_count 2>/dev/null || echo 0); \
	if [ "$$current" -ge 262144 ]; then \
		printf "\033[2m-> vm.max_map_count already $$current (>= 262144)\033[0m\n"; \
	else \
		printf "\033[2m-> Setting vm.max_map_count=262144 (was $$current)...\033[0m\n"; \
		$$setter; \
	fi

cluster.build:  ## Build OpenSearch Docker images (version-aware)
	@$(MAKE) cluster.docker-build

cluster.start:  ## Build, start cluster, wait for ready, and fetch certs
	@$(MAKE) cluster.provider.ensure
	@$(MAKE) cluster.sysctl
	@$(MAKE) cluster.docker-up
	@$(MAKE) cluster.wait-ready
	@$(MAKE) cluster.get-cert

cluster.stop:  ## Stop the OpenSearch cluster
	$(CTR_COMPOSE) down;

cluster.status: ## Show detailed cluster status and health information
	@printf "\033[1m=== OpenSearch Cluster Status ===\033[0m\n\n"
	@# Determine auth settings
	@{ \
		set -e; \
		HTTP_URL="http://localhost:9200"; \
		HTTPS_URL="https://localhost:9200"; \
		BASE_URL=""; \
		CURL_OPTS=""; \
		VERSION="$${OPENSEARCH_VERSION:-latest}"; \
		if [ "$$VERSION" = "latest" ]; then \
			PASSWORD="myStrongPassword123!"; \
		else \
			MAJOR=$$(echo "$$VERSION" | cut -d. -f1); \
			MINOR=$$(echo "$$VERSION" | cut -d. -f2); \
			if [ $$MAJOR -gt 2 ] || ([ $$MAJOR -eq 2 ] && [ $$MINOR -ge 12 ]); then \
				PASSWORD="myStrongPassword123!"; \
			else \
				PASSWORD="admin"; \
			fi; \
		fi; \
		if curl -sf "$$HTTP_URL" > /dev/null 2>&1; then \
			BASE_URL="$$HTTP_URL"; \
			CURL_OPTS=""; \
			printf "\033[36m→ Using HTTP (insecure cluster)\033[0m\n\n"; \
		elif curl -sf -k -u "admin:$$PASSWORD" "$$HTTPS_URL" > /dev/null 2>&1; then \
			BASE_URL="$$HTTPS_URL"; \
			CURL_OPTS="-k -u admin:$$PASSWORD"; \
			printf "\033[36m→ Using HTTPS (secure cluster)\033[0m\n\n"; \
		else \
			printf "\033[31m✗ Cannot connect to cluster at $$HTTP_URL or $$HTTPS_URL\033[0m\n"; \
			printf "\033[2m\nDocker containers:\033[0m\n"; \
			$(CTR_COMPOSE) ps || echo "No containers running"; \
			exit 1; \
		fi; \
		printf "\033[1m--- Cluster Info ---\033[0m\n"; \
		curl -sf $$CURL_OPTS "$$BASE_URL" | jq -C . || echo "Failed to get cluster info"; \
		printf "\n\033[1m--- Cluster Health ---\033[0m\n"; \
		curl -sf $$CURL_OPTS "$$BASE_URL/_cluster/health?pretty" | jq -C . || echo "Failed to get cluster health"; \
		printf "\n\033[1m--- Nodes ---\033[0m\n"; \
		curl -sf $$CURL_OPTS "$$BASE_URL/_cat/nodes?v&h=name,ip,heap.percent,ram.percent,cpu,load_1m,load_5m,load_15m,node.role,master,version" || echo "Failed to get nodes"; \
		printf "\n\n\033[1m--- Node Details (roles, CPUs) ---\033[0m\n"; \
		curl -sf $$CURL_OPTS "$$BASE_URL/_nodes/http,os" | jq -C '.nodes | to_entries[] | {name: .value.name, roles: .value.roles, allocated_processors: .value.os.allocated_processors, http_address: .value.http.publish_address}' || echo "Failed to get node details"; \
		printf "\n\033[1m--- Containers ---\033[0m\n"; \
		$(CTR_COMPOSE) ps; \
		printf "\n\033[1m--- Indices ---\033[0m\n"; \
		curl -sf $$CURL_OPTS "$$BASE_URL/_cat/indices?v&h=health,status,index,docs.count,store.size,pri,rep" || echo "Failed to get indices"; \
		printf "\n\n\033[1m--- Shards ---\033[0m\n"; \
		curl -sf $$CURL_OPTS "$$BASE_URL/_cat/shards?v&h=index,shard,prirep,state,docs,store,node" || echo "Failed to get shards"; \
		printf "\n"; \
	}

cluster.docker-build:
	@# Determine version-specific settings
	$(eval OPENSEARCH_VERSION ?= latest)
	$(eval SECURE_INTEGRATION ?= true)
	$(eval version_major := $(shell \
		if [ "$(OPENSEARCH_VERSION)" = "latest" ]; then \
			echo "2"; \
		else \
			echo "$(OPENSEARCH_VERSION)" | awk -F. '{print $$1}'; \
		fi \
	))
	$(eval manager_role := $(shell \
		if [ "$(version_major)" = "1" ]; then \
			echo "master"; \
		else \
			echo "cluster_manager"; \
		fi \
	))
	@echo "Building OpenSearch $(OPENSEARCH_VERSION) with role: $(manager_role), secure: $(SECURE_INTEGRATION)"
	@echo "Pre-pulling base image opensearchproject/opensearch:$(OPENSEARCH_VERSION)..."
	@for attempt in $$(seq 30); do \
		$(CTR) pull opensearchproject/opensearch:$(OPENSEARCH_VERSION) && break; \
		echo "Pull attempt $$attempt/30 failed, retrying in 10s..."; \
		[ "$$attempt" -eq 30 ] && { echo "All 30 pull attempts failed, aborting."; exit 1; }; \
		sleep 10; \
	done
	OPENSEARCH_MANAGER_ROLE=$(manager_role) OPENSEARCH_MANAGER_SETTING=$(manager_role) \
		$(CTR_COMPOSE) build --pull

cluster.docker-up:
	@# Determine version-specific settings
	$(eval OPENSEARCH_VERSION ?= latest)
	$(eval SECURE_INTEGRATION ?= true)
	$(eval version_major := $(shell \
		if [ "$(OPENSEARCH_VERSION)" = "latest" ]; then \
			echo "2"; \
		else \
			echo "$(OPENSEARCH_VERSION)" | awk -F. '{print $$1}'; \
		fi \
	))
	$(eval manager_role := $(shell \
		if [ "$(version_major)" = "1" ]; then \
			echo "master"; \
		else \
			echo "cluster_manager"; \
		fi \
	))
	@# Apply cgroup workaround for OpenSearch 2.0.1-2.3.0
	$(eval java_opts_extra := $(shell \
		if [ "$(OPENSEARCH_VERSION)" != "latest" ]; then \
			version() { echo "$$@" | awk -F. '{ printf("%d%03d%03d%03d\n", $$1,$$2,$$3,$$4); }'; }; \
			v=$$(version $(OPENSEARCH_VERSION)); \
			v_min=$$(version 2.0.1); \
			v_max=$$(version 2.3.0); \
			if [ $$v -ge $$v_min ] && [ $$v -le $$v_max ]; then \
				echo " -XX:-UseContainerSupport"; \
			fi; \
		fi \
	))
	$(eval OPENSEARCH_HEAP_SIZE ?= 1g)
	$(eval OPENSEARCH_NODE_COUNT ?= 3)
	@# OpenSearch <=2.17.x carries a node-join/node-left race condition
	@# (fixed in 2.18 via opensearch-project/OpenSearch#15521, backported
	@# via opensearch-project/OpenSearch#16118) that leaves a node in
	@# cluster state but disconnected at the transport layer, breaking
	@# NodesStats RPC fan-out indefinitely. Single-node clusters cannot
	@# hit the race; the workflow sets OPENSEARCH_NODE_COUNT=1 for those
	@# versions. Scale flags are derived inline below; an earlier form
	@# that nested a case statement inside an eval/shell broke CI.
	@echo "Starting OpenSearch $(OPENSEARCH_VERSION) with role: $(manager_role), secure: $(SECURE_INTEGRATION), heap: $(OPENSEARCH_HEAP_SIZE), nodes: $(OPENSEARCH_NODE_COUNT)"
	@OVERRIDES="$$(ls $(COMPOSE_DIR)/docker-compose.*-override.yml 2>/dev/null | xargs -n1 basename 2>/dev/null)"; \
	if [ -n "$$OVERRIDES" ]; then echo "Active overrides: $$OVERRIDES"; fi
	@export SECURE_INTEGRATION=$(SECURE_INTEGRATION); \
	export OPENSEARCH_VERSION=$(OPENSEARCH_VERSION); \
	export OPENSEARCH_MANAGER_ROLE=$(manager_role); \
	export OPENSEARCH_MANAGER_SETTING=$(manager_role); \
	export OPENSEARCH_HEAP_SIZE=$(OPENSEARCH_HEAP_SIZE); \
	export OPENSEARCH_JAVA_OPTS_EXTRA="$(java_opts_extra)"; \
	SCALE_ARGS=""; \
	if [ "$(OPENSEARCH_NODE_COUNT)" = "1" ]; then \
		SCALE_ARGS="--scale opensearch-node2=0 --scale opensearch-node3=0"; \
		export OPENSEARCH_SEED_HOSTS="opensearch-node1"; \
		export OPENSEARCH_INITIAL_MANAGER_NODES="opensearch-node1"; \
	elif [ "$(OPENSEARCH_NODE_COUNT)" = "2" ]; then \
		SCALE_ARGS="--scale opensearch-node3=0"; \
		export OPENSEARCH_SEED_HOSTS="opensearch-node1,opensearch-node2"; \
		export OPENSEARCH_INITIAL_MANAGER_NODES="opensearch-node1,opensearch-node2"; \
	fi; \
	$(CTR_COMPOSE) up -d $$SCALE_ARGS

##@ Cluster Scaling & Configuration
cluster.scale.1: ## Start single-node cluster
	$(CTR_COMPOSE) up -d --scale opensearch-node2=0 --scale opensearch-node3=0;

cluster.scale.2: ## Start 2-node cluster
	$(CTR_COMPOSE) up -d --scale opensearch-node1=1 --scale opensearch-node2=1 --scale opensearch-node3=0;

cluster.scale.3: ## Start full 3-node cluster
	$(CTR_COMPOSE) up -d --scale opensearch-node1=1 --scale opensearch-node2=1 --scale opensearch-node3=1;

cluster.get-cert:
	@if curl -sf http://localhost:9200 >/dev/null 2>&1; then \
		printf "\033[2m-> Cluster is HTTP (insecure), skipping cert extraction\033[0m\n"; \
	elif curl -sk -o /dev/null -w '' https://localhost:9200 2>/dev/null; then \
		CONTAINER=$$($(CTR_COMPOSE) ps --format '{{.Name}}' | head -1); \
		if [ -z "$$CONTAINER" ]; then \
			echo "Error: No OpenSearch containers running. Start cluster first with 'make cluster.start'"; \
			exit 1; \
		fi; \
		printf "\033[2m-> Extracting admin certs from $$CONTAINER...\033[0m\n"; \
		$(CTR) cp $$CONTAINER:/usr/share/opensearch/config/kirk.pem admin.pem && \
		$(CTR) cp $$CONTAINER:/usr/share/opensearch/config/kirk-key.pem admin.key; \
	else \
		printf "\033[33m-> Cluster not responding on HTTP or HTTPS, skipping cert extraction\033[0m\n"; \
	fi

cluster.wait-ready: ## Poll cluster until health status is green or yellow
	@printf "\033[2m-> Waiting for cluster to be ready...\033[0m\n"
	@{ \
		set -e; \
		MAX_ATTEMPTS=60; \
		ATTEMPT=1; \
		HTTP_URL="http://localhost:9200/_cluster/health"; \
		HTTPS_URL="https://localhost:9200/_cluster/health"; \
		HEALTH_URL=""; \
		CURL_OPTS=""; \
		VERSION="$${OPENSEARCH_VERSION:-latest}"; \
		if [ "$$VERSION" = "latest" ]; then \
			PASSWORD="myStrongPassword123!"; \
		else \
			MAJOR=$$(echo "$$VERSION" | cut -d. -f1); \
			MINOR=$$(echo "$$VERSION" | cut -d. -f2); \
			if [ $$MAJOR -gt 2 ] || ([ $$MAJOR -eq 2 ] && [ $$MINOR -ge 12 ]); then \
				PASSWORD="myStrongPassword123!"; \
			else \
				PASSWORD="admin"; \
			fi; \
		fi; \
		while [ $$ATTEMPT -le $$MAX_ATTEMPTS ]; do \
			if [ -z "$$HEALTH_URL" ]; then \
				if curl -sf "$$HTTP_URL" > /dev/null 2>&1; then \
					printf "\033[36m→ Detected insecure cluster (HTTP)\033[0m\n"; \
					HEALTH_URL="$$HTTP_URL"; \
					CURL_OPTS=""; \
				elif curl -sf -k -u "admin:$$PASSWORD" "$$HTTPS_URL" > /dev/null 2>&1; then \
					printf "\033[36m→ Detected secure cluster (HTTPS)\033[0m\n"; \
					HEALTH_URL="$$HTTPS_URL"; \
					CURL_OPTS="-k -u admin:$$PASSWORD"; \
				else \
					printf "\033[33m⋯ Waiting for cluster to respond (attempt $$ATTEMPT/$$MAX_ATTEMPTS)\033[0m\n"; \
					ATTEMPT=$$((ATTEMPT + 1)); \
					sleep 3; \
					continue; \
				fi; \
			fi; \
			if curl -sf $$CURL_OPTS "$$HEALTH_URL" > /dev/null 2>&1; then \
				STATUS=$$(curl -sf $$CURL_OPTS "$$HEALTH_URL" | grep -o '"status":"[^"]*"' | cut -d'"' -f4); \
				if [ "$$STATUS" = "green" ] || [ "$$STATUS" = "yellow" ]; then \
					printf "\033[32m✓ Cluster is ready (status: $$STATUS) after $$ATTEMPT attempts\033[0m\n"; \
					INFO_URL="$${HEALTH_URL%%/_cluster/health}"; \
					CLUSTER_INFO=$$(curl -sf $$CURL_OPTS "$$INFO_URL" 2>/dev/null); \
					if [ -n "$$CLUSTER_INFO" ]; then \
						CLUSTER_NAME=$$(echo "$$CLUSTER_INFO" | grep -o '"cluster_name":"[^"]*"' | cut -d'"' -f4); \
						CLUSTER_VERSION=$$(echo "$$CLUSTER_INFO" | grep -o '"number":"[^"]*"' | head -1 | cut -d'"' -f4); \
						printf "\033[2m  Cluster: $$CLUSTER_NAME\033[0m\n"; \
						printf "\033[2m  Version: $$CLUSTER_VERSION\033[0m\n"; \
						printf "\033[2m  URL:     $$INFO_URL\033[0m\n"; \
						if [ -n "$$CURL_OPTS" ]; then \
							printf "\033[2m  Auth:    admin:****\033[0m\n"; \
						fi; \
					fi; \
					exit 0; \
				fi; \
				printf "\033[33m⋯ Cluster status: $$STATUS (attempt $$ATTEMPT/$$MAX_ATTEMPTS)\033[0m\n"; \
			else \
				printf "\033[33m⋯ Waiting for cluster to respond (attempt $$ATTEMPT/$$MAX_ATTEMPTS)\033[0m\n"; \
			fi; \
			ATTEMPT=$$((ATTEMPT + 1)); \
			sleep 3; \
		done; \
		printf "\033[31m✗ Cluster failed to become ready after $$MAX_ATTEMPTS attempts\033[0m\n"; \
		printf "\033[2m\n--- Diagnostic Information ---\033[0m\n"; \
		printf "\033[2mDocker containers:\033[0m\n"; \
		$(CTR_COMPOSE) ps || true; \
		printf "\033[2m\nFull logs from all containers:\033[0m\n"; \
		$(CTR_COMPOSE) logs || true; \
		printf "\033[2m\nAttempted URLs:\033[0m\n"; \
		printf "  HTTP:  $$HTTP_URL\n"; \
		printf "  HTTPS: $$HTTPS_URL\n"; \
		printf "\033[2m\nCurl test results:\033[0m\n"; \
		printf "  HTTP: "; curl -sf "$$HTTP_URL" && echo "✓ OK" || echo "✗ Failed"; \
		printf "  HTTPS: "; curl -sf -k -u "admin:$$PASSWORD" "$$HTTPS_URL" && echo "✓ OK" || echo "✗ Failed"; \
		exit 1; \
	}

cluster.clean: ## Remove unused container volumes and networks
	@printf "\033[2m-> Cleaning up container assets...\033[0m\n"
	@# Stop and remove containers first to release volumes
	@$(CTR_COMPOSE) down --volumes 2>/dev/null || true
	@# Remove OpenSearch built images to ensure clean rebuilds when switching versions
	@imgs=$$($(CTR) images -q --filter 'reference=opensearch-opensearch-node*' 2>/dev/null); \
		[ -n "$$imgs" ] && echo "$$imgs" | xargs $(CTR) rmi -f || true
	@# Remove OpenSearch volumes to clear stale data
	@vols=$$($(CTR) volume ls -q --filter "name=opensearch" 2>/dev/null); \
		[ -n "$$vols" ] && echo "$$vols" | xargs $(CTR) volume rm || true
	@# Clean up unused container resources
	$(CTR) volume prune --force
	$(CTR) network prune --force
	$(CTR) system prune --volumes --force

cluster.heterogeneous.cpu.1:  ## Set CPU limits: node1=2, node2=2, node3=4
	@printf '%s\n' \
		'services:' \
		'  opensearch-node1:' \
		'    deploy:' \
		'      resources:' \
		'        limits:' \
		"          cpus: '2'" \
		'  opensearch-node2:' \
		'    deploy:' \
		'      resources:' \
		'        limits:' \
		"          cpus: '2'" \
		'  opensearch-node3:' \
		'    deploy:' \
		'      resources:' \
		'        limits:' \
		"          cpus: '4'" \
		> $(COMPOSE_DIR)/docker-compose.cpu-override.yml
	@echo "CPU override: node1=2, node2=2, node3=4 (weights [1,1,2])"

cluster.heterogeneous.cpu.2:  ## Set CPU limits: node1=1, node2=2, node3=4
	@printf '%s\n' \
		'services:' \
		'  opensearch-node1:' \
		'    deploy:' \
		'      resources:' \
		'        limits:' \
		"          cpus: '1'" \
		'  opensearch-node2:' \
		'    deploy:' \
		'      resources:' \
		'        limits:' \
		"          cpus: '2'" \
		'  opensearch-node3:' \
		'    deploy:' \
		'      resources:' \
		'        limits:' \
		"          cpus: '4'" \
		> $(COMPOSE_DIR)/docker-compose.cpu-override.yml
	@echo "CPU override: node1=1, node2=2, node3=4 (weights [1,2,4])"

cluster.heterogeneous.roles:  ## Set roles: node1=cm+ingest, node2=data+ingest, node3=data
	@printf '%s\n' \
		'services:' \
		'  opensearch-node1:' \
		'    environment:' \
		'      - node.roles=$${OPENSEARCH_MANAGER_ROLE:-cluster_manager},ingest' \
		'  opensearch-node2:' \
		'    environment:' \
		'      - node.roles=data,ingest' \
		'  opensearch-node3:' \
		'    environment:' \
		'      - node.roles=data' \
		> $(COMPOSE_DIR)/docker-compose.roles-override.yml
	@echo "Roles override: node1=cm+ingest, node2=data+ingest, node3=data"

cluster.homogeneous:  ## Remove all override files (reset to defaults)
	@rm -f $(COMPOSE_DIR)/docker-compose.*-override.yml
	@echo "Removed all overrides — cluster will use default (homogeneous) config"

##@ Network Latency Simulation
cluster.latency.asymmetric:  ## Add asymmetric latency: node1=0ms, node2=50ms, node3=150ms
	@printf "\033[2m-> Applying asymmetric latency...\033[0m\n"
	@NODES="$$($(CTR_COMPOSE) ps --format '{{.Name}}')"; \
	for NODE in $$NODES; do \
		$(CTR) exec --user root $$NODE tc qdisc del dev eth0 root 2>/dev/null || true; \
	done; \
	NODE1=$$($(CTR_COMPOSE) ps --format '{{.Name}}' opensearch-node1 2>/dev/null | head -1); \
	NODE2=$$($(CTR_COMPOSE) ps --format '{{.Name}}' opensearch-node2 2>/dev/null | head -1); \
	NODE3=$$($(CTR_COMPOSE) ps --format '{{.Name}}' opensearch-node3 2>/dev/null | head -1); \
	if [ -n "$$NODE1" ]; then echo "  node1 ($$NODE1): 0ms (no delay)"; fi; \
	if [ -n "$$NODE2" ]; then $(CTR) exec --user root $$NODE2 tc qdisc add dev eth0 root netem delay 50ms 5ms && echo "  node2 ($$NODE2): 50ms ±5ms"; fi; \
	if [ -n "$$NODE3" ]; then $(CTR) exec --user root $$NODE3 tc qdisc add dev eth0 root netem delay 150ms 15ms && echo "  node3 ($$NODE3): 150ms ±15ms"; fi; \
	echo "Latency applied. Use 'make cluster.latency.show' to verify."

cluster.latency.symmetric:  ## Add symmetric latency: all nodes 1ms (single tier, bucket ~10)
	@printf "\033[2m-> Applying symmetric latency...\033[0m\n"
	@NODES="$$($(CTR_COMPOSE) ps --format '{{.Name}}')"; \
	for NODE in $$NODES; do \
		$(CTR) exec --user root $$NODE tc qdisc del dev eth0 root 2>/dev/null || true; \
		$(CTR) exec --user root $$NODE tc qdisc add dev eth0 root netem delay 1ms 100us && \
			echo "  $$NODE: 1ms ±100us (bucket ~10)"; \
	done; \
	echo "Symmetric latency applied (all nodes same tier)."

cluster.latency.bimodal:  ## Add bimodal latency: node1=1ms, node2=1ms, node3=20ms (2 local + 1 remote)
	@printf "\033[2m-> Applying bimodal latency...\033[0m\n"
	@NODES="$$($(CTR_COMPOSE) ps --format '{{.Name}}')"; \
	for NODE in $$NODES; do \
		$(CTR) exec --user root $$NODE tc qdisc del dev eth0 root 2>/dev/null || true; \
	done; \
	NODE1=$$($(CTR_COMPOSE) ps --format '{{.Name}}' opensearch-node1 2>/dev/null | head -1); \
	NODE2=$$($(CTR_COMPOSE) ps --format '{{.Name}}' opensearch-node2 2>/dev/null | head -1); \
	NODE3=$$($(CTR_COMPOSE) ps --format '{{.Name}}' opensearch-node3 2>/dev/null | head -1); \
	if [ -n "$$NODE1" ]; then $(CTR) exec --user root $$NODE1 tc qdisc add dev eth0 root netem delay 1ms 100us && echo "  node1 ($$NODE1): 1ms ±100us (bucket ~10)"; fi; \
	if [ -n "$$NODE2" ]; then $(CTR) exec --user root $$NODE2 tc qdisc add dev eth0 root netem delay 1ms 100us && echo "  node2 ($$NODE2): 1ms ±100us (bucket ~10)"; fi; \
	if [ -n "$$NODE3" ]; then $(CTR) exec --user root $$NODE3 tc qdisc add dev eth0 root netem delay 20ms 2ms && echo "  node3 ($$NODE3): 20ms ±2ms (bucket ~14)"; fi; \
	echo "Bimodal latency applied (2 local + 1 remote)."

cluster.latency.graduated:  ## Add graduated latency: node1=1ms, node2=10ms, node3=100ms (buckets 10,14,17)
	@printf "\033[2m-> Applying graduated latency...\033[0m\n"
	@NODES="$$($(CTR_COMPOSE) ps --format '{{.Name}}')"; \
	for NODE in $$NODES; do \
		$(CTR) exec --user root $$NODE tc qdisc del dev eth0 root 2>/dev/null || true; \
	done; \
	NODE1=$$($(CTR_COMPOSE) ps --format '{{.Name}}' opensearch-node1 2>/dev/null | head -1); \
	NODE2=$$($(CTR_COMPOSE) ps --format '{{.Name}}' opensearch-node2 2>/dev/null | head -1); \
	NODE3=$$($(CTR_COMPOSE) ps --format '{{.Name}}' opensearch-node3 2>/dev/null | head -1); \
	if [ -n "$$NODE1" ]; then $(CTR) exec --user root $$NODE1 tc qdisc add dev eth0 root netem delay 1ms 100us && echo "  node1 ($$NODE1): 1ms ±100us (bucket ~10)"; fi; \
	if [ -n "$$NODE2" ]; then $(CTR) exec --user root $$NODE2 tc qdisc add dev eth0 root netem delay 10ms 1ms && echo "  node2 ($$NODE2): 10ms ±1ms (bucket ~14)"; fi; \
	if [ -n "$$NODE3" ]; then $(CTR) exec --user root $$NODE3 tc qdisc add dev eth0 root netem delay 100ms 10ms && echo "  node3 ($$NODE3): 100ms ±10ms (bucket ~17)"; fi; \
	echo "Graduated latency applied (3 distinct tiers)."

cluster.latency.clear:  ## Remove all artificial latency from nodes
	@printf "\033[2m-> Clearing network latency...\033[0m\n"
	@NODES="$$($(CTR_COMPOSE) ps --format '{{.Name}}')"; \
	for NODE in $$NODES; do \
		$(CTR) exec --user root $$NODE tc qdisc del dev eth0 root 2>/dev/null && echo "  Cleared $$NODE" || echo "  $$NODE: no qdisc to clear"; \
	done; \
	echo "All latency rules removed."

cluster.latency.show:  ## Show current tc qdisc rules on each node
	@NODES="$$($(CTR_COMPOSE) ps --format '{{.Name}}')"; \
	for NODE in $$NODES; do \
		echo "--- $$NODE ---"; \
		$(CTR) exec --user root $$NODE tc qdisc show dev eth0 2>/dev/null || echo "  (tc not available)"; \
	done

linters:
	@for tags in $(GOLANGCI_LINT_TAG_SETS); do \
		printf "\033[2m-> golangci-lint --build-tags %s\033[0m\n" "$$tags"; \
		$(CTR) run -t --rm -v $$(pwd):/app -v ~/.cache/golangci-lint/$(GOLANGCI_LINT_VERSION):/root/.cache -w /app golangci/golangci-lint:$(GOLANGCI_LINT_VERSION) golangci-lint run --fix --build-tags "$$tags" --timeout=5m -v ./... || exit $$?; \
	done

##@ GitHub CI
#------------------------------------------------------------------------------
# Fetch and filter CI check results from GitHub Actions using the gh CLI.
# These targets extract only the actual failures from CI logs, avoiding false
# positives from test names that contain words like "fail", "error", or "panic".

# GH_RUN_ID: override to inspect a specific run. When unset, targets auto-detect
# the most recent failed run on the current branch.
GH_RUN_ID ?=

# Internal: resolve GH_RUN_ID lazily. If the caller didn't set it, pick the
# latest failed run for the current branch.
_gh_run_id = $(or $(GH_RUN_ID),$(shell gh run list --branch "$$(git branch --show-current)" --status failure --limit 1 --json databaseId --jq '.[0].databaseId'))

gh.checks:  ## Show all CI check statuses for the current branch
	@gh run list --branch "$$(git branch --show-current)" --limit 10

gh.checks.failed:  ## List only failed CI runs for the current branch
	@gh run list --branch "$$(git branch --show-current)" --status failure --limit 10

gh.fail:  ## Show failed test names and error messages from the latest (or GH_RUN_ID) failed run
	@RUN_ID=$(_gh_run_id); \
	if [ -z "$$RUN_ID" ]; then \
		echo "No failed runs found on branch $$(git branch --show-current)"; \
		exit 0; \
	fi; \
	printf "\033[1m=== Failed run: $$RUN_ID ===\033[0m\n"; \
	gh run view "$$RUN_ID" --json jobs --jq '.jobs[] | select(.conclusion == "failure") | "  \(.name)"' 2>/dev/null; \
	printf "\n"; \
	gh run view "$$RUN_ID" --log-failed 2>&1 \
	| grep -E -e '--- FAIL:' -e 'FAIL	' -e 'panic:' \
	| cut -f3- \
	| sed 's/^[^ ]* //'

gh.fail.full:  ## Show full failed-step logs from the latest (or GH_RUN_ID) failed run
	@RUN_ID=$(_gh_run_id); \
	if [ -z "$$RUN_ID" ]; then \
		echo "No failed runs found on branch $$(git branch --show-current)"; \
		exit 0; \
	fi; \
	printf "\033[1m=== Failed run: $$RUN_ID ===\033[0m\n\n"; \
	gh run view "$$RUN_ID" --log-failed

gh.fail.context:  ## Show failed tests with 5 lines of context before each failure
	@RUN_ID=$(_gh_run_id); \
	if [ -z "$$RUN_ID" ]; then \
		echo "No failed runs found on branch $$(git branch --show-current)"; \
		exit 0; \
	fi; \
	printf "\033[1m=== Failed run: $$RUN_ID ===\033[0m\n\n"; \
	gh run view "$$RUN_ID" --log-failed 2>&1 \
	| cut -f3- \
	| sed 's/^[^ ]* //' \
	| grep -B5 -E -e '--- FAIL:' -e 'panic:'

gh.fail.summary:  ## One-line-per-failure summary from the latest (or GH_RUN_ID) failed run
	@RUN_ID=$(_gh_run_id); \
	if [ -z "$$RUN_ID" ]; then \
		echo "No failed runs found on branch $$(git branch --show-current)"; \
		exit 0; \
	fi; \
	printf "\033[1mRun $$RUN_ID — failed jobs:\033[0m\n"; \
	gh run view "$$RUN_ID" --json jobs --jq '.jobs[] | select(.conclusion == "failure") | "  \u001b[31m✗\u001b[0m \(.name)"' 2>/dev/null; \
	printf "\n\033[1mFailed tests:\033[0m\n"; \
	gh run view "$$RUN_ID" --log-failed 2>&1 \
	| grep -E -- '--- FAIL:' \
	| cut -f3- \
	| sed 's/^[^ ]* //' \
	| sort -u \
	| sed 's/^/  /'

##@ Other
#------------------------------------------------------------------------------
help:  ## Display help
	@awk 'BEGIN {FS = ":.*##"; printf "Usage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_][a-zA-Z0-9._-]+:.*?##/ { printf "  \033[36m%-35s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
#------------- <https://suva.sh/posts/well-documented-makefiles> --------------

.DEFAULT_GOAL := help
.PHONY: help backport cluster.runtime cluster.provider.ensure cluster.sysctl cluster.build cluster.start cluster.stop cluster.docker-build cluster.docker-up cluster.clean cluster.heterogeneous.cpu.1 cluster.heterogeneous.cpu.2 cluster.heterogeneous.roles cluster.homogeneous cluster.latency.asymmetric cluster.latency.symmetric cluster.latency.bimodal cluster.latency.graduated cluster.latency.clear cluster.latency.show gh.checks gh.checks.failed gh.fail gh.fail.full gh.fail.context gh.fail.summary coverage godoc lint lint.local release test test-all test-race test-bench test-integ test-unit linters linters.install build-samples
.SILENT: lint.markdown