PMM-7 API tests rework

.github/workflows/api-tests.yml

@@ -81,8 +81,7 @@ jobs:
       - name: Build the test image
         shell: bash
         run: |
-          echo "Make sure to set the context to the root of the repository."
-          docker buildx build -f api-tests/Dockerfile -t percona/pmm-api-tests .
+          make docker-build-image
 
       - name: Run compose up for test DBs
         shell: bash
@@ -98,14 +97,7 @@ jobs:
       - name: Run API tests
         shell: bash
         run: |
-          docker run \
-            -e PMM_SERVER_URL=${{env.PMM_URL}} \
-            -e PMM_RUN_UPDATE_TEST=0 \
-            -e PMM_RUN_ADVISOR_TESTS=0 \
-            -e PMM_SERVER_INSECURE_TLS=1 \
-            --name pmm-api-tests \
-            --network host \
-            percona/pmm-api-tests
+          PMM_SERVER_URL=${{env.PMM_URL}} make docker-run-tests
 
       - name: Get PMM logs
         if: ${{ failure() }}

api-tests/.dockerignore

@@ -0,0 +1 @@
+/bin/

api-tests/Dockerfile

@@ -1,11 +1,25 @@
 # This Dockerfile is used only for API tests.
+ARG VERSION=dev
+ARG GIT_COMMIT=unknown
+ARG BUILD_TIME=unknown
+ARG GO_VERSION="1.25"
+ARG GO_BASE_IMAGE="golang:${GO_VERSION}"
 
-FROM golang:1.25
+FROM $GO_BASE_IMAGE AS builder
+ENV GONOPROXY='github.com/percona,github.com/Percona-Lab'
+ENV GONOSUMDB='github.com/percona,github.com/Percona-Lab'
+ENV GOPRIVATE='github.com/percona,github.com/Percona-Lab'
 
-RUN export GOPATH=$(go env GOPATH) && \
-    mkdir -p $GOPATH/pmm
+WORKDIR $GOPATH/pmm
 
-COPY . $GOPATH/pmm/
+COPY go.mod go.sum ./
+RUN --mount=type=cache,target=/go/pkg/mod \
+    go mod download
+
+COPY . $GOPATH/pmm
 WORKDIR $GOPATH/pmm/api-tests/
+RUN --mount=type=cache,target=/go/pkg/mod \
+    --mount=type=cache,target=/root/.cache/go-build \
+    make -j4 init
 
-CMD ["make", "init", "run-race"]
+CMD ["make", "test-report"]

api-tests/Makefile

@@ -1,28 +1,93 @@
-all: build
+.DEFAULT_GOAL := help
+BIN_DIR := $(CURDIR)/bin
 
-init:           ## Installs development tools
+# --- Go-related variables ----------------------------------------------------------------
+GO_VERSION := $(shell grep '^go ' $(CURDIR)/../go.mod | awk '{print $$2}')
+
+# --- Git variables ------------------------------------------------------------------------
+GIT_VERSION := $(shell git describe --tags)
+GIT_COMMIT := $(shell git rev-parse --short HEAD)
+BUILD_TIME := $(shell date -u +%Y-%m-%dT%H:%M:%SZ)
+
+# --- Test-related variables --------------------------------------------------------------
+TEST_DIRS := $(shell find . -type f -name '*_test.go' -not -path '*/.*' -exec dirname {} + | sort -u)
+TEST_BINARY_TARGETS := $(subst /,-,$(subst ./,,$(TEST_DIRS)))
+RUN_TEST_TARGETS := $(addprefix test-,$(TEST_BINARY_TARGETS))
+# Exclude server-settings tests from this list, as they require special handling (Grafana restart).
+RUN_TEST_TARGETS := $(filter-out test-server-settings%,$(RUN_TEST_TARGETS))
+REPORT_OUT := pmm-api-tests-junit-report.xml
+TEST_OUTPUT_PREFIX := pmm-api-tests-output
+TEST_FLAGS := -test.count=1 -test.v
+
+# --- Docker related variables -----------------------------------------------------------
+DOCKER_IMAGE := local/pmm-api-tests
+PMM_SERVER_INSECURE_TLS ?= 1
+PMM_RUN_UPDATE_TEST ?= 0
+PMM_RUN_ADVISOR_TESTS ?= 0
+
+.PHONY: help guard-% init $(TEST_BINARY_TARGETS) $(RUN_TEST_TARGETS)
+.PHONY: test-server-settings test test-report clean docker-build-image docker-run-tests
+
+help: ## Show available targets
+	@echo "Available targets: $(TARGETS)"
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "  \033[36m%-20s\033[0m %s\n", $$1, $$2}'
+
+guard-%:
+	@[ -n "${$*}" ] || (echo "ERROR: $* is not set"; exit 1)
+
+$(TEST_BINARY_TARGETS): ## Build test binaries
+	@dir="$(subst -,/,$@)"; \
+	echo "-> Building test binary for dir: $$dir"; \
+	go test -c -v -race -o $(BIN_DIR)/$@ $(CURDIR)/$$dir
+
+init: $(TEST_BINARY_TARGETS) ## Installs development tools
+	@echo "-> Installing tools"
 	cd tools && go generate -x -tags=tools
 
-build:
-	go install -v ./...
-	go test -c -v ./alerting
-	go test -c -v ./backup
-	go test -c -v ./inventory
-	go test -c -v ./management
-	go test -c -v ./server
-	go test -c -v ./user
-
-run:
-	go test -count=1 -p 1 -v ./... 2>&1 | tee pmm-api-tests-output.txt
-	cat pmm-api-tests-output.txt | bin/go-junit-report > pmm-api-tests-junit-report.xml
-
-run-dev:
-	go test -count=1 -p 1 -v ./...
-
-run-race:
-	go test -count=1 -p 1 -v -race ./... 2>&1 | tee pmm-api-tests-output.txt
-	cat pmm-api-tests-output.txt | bin/go-junit-report > pmm-api-tests-junit-report.xml
-
-clean:
-	rm -f ./pmm-api-tests-output.txt
-	rm -f ./pmm-api-tests-junit-report.xml
+$(RUN_TEST_TARGETS):
+	@binary="$(subst test-,,$@)"; \
+	echo "🚀 Run $$binary tests"; \
+	$(BIN_DIR)/$$binary $(TEST_FLAGS) 2>&1 | tee $(TEST_OUTPUT_PREFIX)-$@.txt
+
+# This target shall be running separately from the rest of tests,
+# as it triggers Grafana restart that may cause other tests to fail.
+test-server-settings: guard-PMM_SERVER_URL ## Run server-settings tests (requires Grafana restart)
+	@binary="$(subst test-,,$@)"; \
+	echo "🚀 Run $$binary tests"; \
+	$(BIN_DIR)/$$binary $(TEST_FLAGS) 2>&1 | tee $(TEST_OUTPUT_PREFIX)-$@.txt
+
+test: guard-PMM_SERVER_URL $(RUN_TEST_TARGETS) ## Run tests
+	$(MAKE) test-server-settings
+	@echo "✅ All tests completed."
+
+test-report: test ## Run tests and generate JUnit report
+	cat $(CURDIR)/$(TEST_OUTPUT_PREFIX)-*.txt | $(BIN_DIR)/go-junit-report > $(REPORT_OUT)
+	@echo "✅ Report is generated: $(REPORT_OUT)."
+
+clean:	## Cleanup reports
+	rm -f $(CURDIR)/pmm-api-tests-*.txt
+	rm -f $(CURDIR)/$(REPORT_OUT)
+	@echo "✅ Cleanup completed."
+
+## Docker-related targets
+
+docker-build-image: ## Build Docker image with tests
+	docker build \
+	-f Dockerfile \
+	--build-arg GO_VERSION=$(GO_VERSION) \
+	--build-arg VERSION=$(GIT_VERSION) \
+	--build-arg GIT_COMMIT=$(GIT_COMMIT) \
+	--build-arg BUILD_TIME=$(BUILD_TIME) \
+	-t $(DOCKER_IMAGE) ../
+	@echo "✅ Docker image '$(DOCKER_IMAGE)' built successfully."
+
+docker-run-tests: guard-PMM_SERVER_URL ## Run tests inside Docker container
+	docker run \
+	-e PMM_SERVER_URL=$(PMM_SERVER_URL) \
+	-e PMM_SERVER_INSECURE_TLS=$(PMM_SERVER_INSECURE_TLS) \
+	-e PMM_RUN_UPDATE_TEST=$(PMM_RUN_UPDATE_TEST) \
+	-e PMM_RUN_ADVISOR_TESTS=$(PMM_RUN_ADVISOR_TESTS) \
+	--name api-tests \
+	--network host \
+	$(DOCKER_IMAGE)
+	@echo "✅ Tests executed successfully inside Docker container."

api-tests/README.md

@@ -8,28 +8,33 @@ Make sure you have the latest Go version installed on your systems, execute the
 to set up API-tests in your local systems.
 
 1. Run PMM Server. This can be done by running `make env-up` in the root (`pmm`) directory.
-2. Replace `$PMM_SERVER_URL` with a URL in format `http://USERNAME:PASSWORD@HOST`. For local development it's usually `http://admin:admin@127.0.0.1`.
+2. Replace `$PMM_SERVER_URL` with a URL in format `https://USERNAME:PASSWORD@HOST`. For local development it's usually `http://admin:admin@127.0.0.1`.
 
 # Usage
 
+Precompile tests using the following command:
+```
+make -j4 init
+```
+
 Run the tests using the following command:
 
 ```
-go test ./... -pmm.server-url $PMM_SERVER_URL -v
+PMM_SERVER_URL=$PMM_SERVER_URL make test
 ```
 
 # Docker
 
 Build Docker image using the following command:
 
 ```
-docker build -t IMAGENAME .
+make docker-build-image
 ```
 
 Run Docker container using the following command:
 
 ```
-docker run -e PMM_SERVER_URL=**pmm-server-url** IMAGENAME
+PMM_SERVER_URL=$PMM_SERVER_URL make docker-run-tests
 ```
 
 where `PMM_SERVER_URL` should be pointing to a running PMM Server.

api-tests/alerting/alerting_test.go

@@ -19,11 +19,14 @@ import (
 	"bytes"
 	"encoding/base64"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
 	"net/url"
 	"os"
+	"path"
+	"runtime"
 	"strings"
 	"testing"
 	"time"
@@ -170,13 +173,13 @@ func TestModifyTemplatesAPI(t *testing.T) {
 	t.Parallel()
 	client := alertingClient.Default.AlertingService
 
-	templateData, err := os.ReadFile("../testdata/alerting/template.yaml")
+	templateData, err := readTemplateContent(t, "../testdata/alerting/template.yaml")
 	require.NoError(t, err)
 
-	multipleTemplatesData, err := os.ReadFile("../testdata/alerting/multiple-templates.yaml")
+	multipleTemplatesData, err := readTemplateContent(t, "../testdata/alerting/multiple-templates.yaml")
 	require.NoError(t, err)
 
-	invalidTemplateData, err := os.ReadFile("../testdata/alerting/invalid-template.yaml")
+	invalidTemplateData, err := readTemplateContent(t, "../testdata/alerting/invalid-template.yaml")
 	require.NoError(t, err)
 
 	t.Run("add", func(t *testing.T) {
@@ -429,11 +432,15 @@ func TestModifyTemplatesAPI(t *testing.T) {
 // We keep it separate from the tests in TestModifyTemplatesAPI to avoid
 // race conditions when other tests add or remove templates while we are listing them.
 func TestListTemplatesAPI(t *testing.T) {
+	t.Parallel()
+
 	client := alertingClient.Default.AlertingService
 
-	templateData, err := os.ReadFile("../testdata/alerting/template.yaml")
+	templateData, err := readTemplateContent(t, "../testdata/alerting/template.yaml")
 	require.NoError(t, err)
 	t.Run("list", func(t *testing.T) {
+		t.Parallel()
+
 		t.Run("without pagination", func(t *testing.T) {
 			name := uuid.New().String()
 			expr := uuid.New().String()
@@ -680,7 +687,7 @@ func createAlertRuleParams(templateName, folderUID string, filter *alerting.Crea
 func createTemplate(t *testing.T) string {
 	t.Helper()
 
-	b, err := os.ReadFile("../testdata/alerting/template.yaml")
+	b, err := readTemplateContent(t, "../testdata/alerting/template.yaml")
 	require.NoError(t, err)
 
 	templateName := uuid.New().String()
@@ -695,3 +702,13 @@ func createTemplate(t *testing.T) string {
 
 	return templateName
 }
+
+func readTemplateContent(t *testing.T, filePath string) ([]byte, error) {
+	t.Helper()
+
+	_, file, _, ok := runtime.Caller(0)
+	if !ok {
+		return nil, errors.New("failed to get current file path")
+	}
+	return os.ReadFile(path.Join(path.Dir(file), filePath)) //nolint:gosec
+}

api-tests/helpers.go

@@ -24,7 +24,6 @@ import (
 	"reflect"
 	"strings"
 	"testing"
-	"time"
 
 	"github.com/AlekSi/pointer"
 	"github.com/stretchr/testify/assert"
@@ -294,32 +293,6 @@ func AddPMMAgent(t TestingT, nodeID string) *agents.AddAgentOKBody {
 	return res.Payload
 }
 
-// retryWithBackoff retries fn with capped exponential backoff until it succeeds,
-// attempts are exhausted, or ctx is done.
-func retryWithBackoff(ctx context.Context, attempts int, fn func() error) error {
-	var lastErr error
-	for i := range attempts {
-		lastErr = fn()
-		if lastErr == nil {
-			return nil
-		}
-		if i == attempts-1 {
-			break
-		}
-		select {
-		case <-time.After(backoff(i)):
-		case <-ctx.Done():
-			return ctx.Err()
-		}
-	}
-	return fmt.Errorf("retries exhausted: %w", lastErr)
-}
-
-func backoff(attempt int) time.Duration {
-	d := time.Duration(1<<attempt) * time.Second
-	return min(d, 5*time.Second)
-}
-
 // check interfaces.
 var (
 	_ assert.TestingT  = (*expectedFailureTestingT)(nil)

api-tests/init.go

@@ -36,16 +36,15 @@ import (
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
 
+	"github.com/percona/pmm/api-tests/utils"
 	actionsClient "github.com/percona/pmm/api/actions/v1/json/client"
 	advisorClient "github.com/percona/pmm/api/advisors/v1/json/client"
 	alertingClient "github.com/percona/pmm/api/alerting/v1/json/client"
 	backupsClient "github.com/percona/pmm/api/backup/v1/json/client"
 	inventoryClient "github.com/percona/pmm/api/inventory/v1/json/client"
 	managementClient "github.com/percona/pmm/api/management/v1/json/client"
 	serverClient "github.com/percona/pmm/api/server/v1/json/client"
-	"github.com/percona/pmm/api/server/v1/json/client/server_service"
 	userClient "github.com/percona/pmm/api/user/v1/json/client"
-	"github.com/percona/pmm/api/user/v1/json/client/user_service"
 	"github.com/percona/pmm/utils/tlsconfig"
 )
 
@@ -207,22 +206,7 @@ func init() {
 	userClient.Default = userClient.New(transport, nil)
 
 	// do not run tests if server is not available
-	serverReadyErr := retryWithBackoff(Context, 10, func() error {
-		_, err = serverClient.Default.ServerService.Readiness(&server_service.ReadinessParams{
-			Context: Context,
-		})
-		if err != nil {
-			return fmt.Errorf("failed to pass the server readiness probe: %w", err)
-		}
-
-		_, err = userClient.Default.UserService.GetUser(&user_service.GetUserParams{
-			Context: Context,
-		})
-		if err != nil {
-			return fmt.Errorf("failed to get user details: %w", err)
-		}
-		return nil
-	})
+	serverReadyErr := utils.WaitServerReady(Context)
 
 	if serverReadyErr != nil {
 		logrus.Fatalf("PMM Server is not ready: %s", serverReadyErr)

api-tests/inventory/agents_azure_database_exporter_test.go

@@ -29,8 +29,7 @@ import (
 )
 
 func TestAzureDatabaseExporter(t *testing.T) { //nolint:tparallel
-	// TODO Fix this test to run in parallel.
-	// t.Parallel()
+	t.Parallel()
 	t.Run("Basic", func(t *testing.T) {
 		t.Parallel()
 		genericNodeID := pmmapitests.AddGenericNode(t, pmmapitests.TestString(t, "")).NodeID
@@ -219,6 +218,8 @@ func TestAzureDatabaseExporter(t *testing.T) { //nolint:tparallel
 	})
 
 	t.Run("With PushMetrics", func(t *testing.T) {
+		t.Parallel()
+
 		genericNodeID := pmmapitests.AddGenericNode(t, pmmapitests.TestString(t, "")).NodeID
 		require.NotEmpty(t, genericNodeID)
 		defer pmmapitests.RemoveNodes(t, genericNodeID)

api-tests/inventory/agents_proxysql_exporter_test.go

@@ -283,6 +283,8 @@ func TestProxySQLExporter(t *testing.T) {
 		}
 	})
 	t.Run("With PushMetrics", func(t *testing.T) {
+		t.Parallel()
+
 		genericNodeID := pmmapitests.AddGenericNode(t, pmmapitests.TestString(t, "")).NodeID
 		require.NotEmpty(t, genericNodeID)
 		defer pmmapitests.RemoveNodes(t, genericNodeID)