update integration tests

Author: sean-breen

.github/workflows/ci.yml

@@ -64,26 +64,21 @@ jobs:
             OSARCH=amd64
             GO_VERSION=${{ env.GO_VERSION }}
 
-      - name: Run Load Test with CPU profiling
+      - name: Run CPU Profiling Tests
         run: |
-          make run-load-test-with-cpu-profiling
-          make generate-pgo-profile
+          cp profile.pgo old_profile.pgo || echo "No previous pgo profile found"
+          make generate-pgo-profile || echo "No changes to profile"
+          ls -l profile.pgo
+          diff old_profile.pgo profile.pgo || echo "No differences found"
 
-      - name: Archive Load Test CPU profile
-        id: artifact-upload-step
-        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
-        with:
-          name: cpu-profile
-          path: build/test/load/profile.pgo
-          if-no-files-found: error
-
-      - name: Push PGO profile
+      - name: Upload CPU Profile
         run: |
           git config --global user.name 'github-actions'
           git config --global user.email '41898282+github-actions[bot]@users.noreply.github.com'
           git commit -m "CI Autogenerated" -- ${{ steps.artifact-upload-step.outputs.artifact-id }} || echo "No changes to commit"
           git log -1
 
+
   proxy-sanity-check:
     name: Proxy Sanity Check
     runs-on: ubuntu-22.04

Makefile

@@ -283,15 +283,14 @@ local-rpm-package: ## Create local rpm package
 
 generate-pgo-profile: build-mock-management-plane-grpc run-load-test-with-cpu-profiling ## Generate PGO profile
 	@echo "Generating PGO profile"
-	cp default.pgo old.pprof
 	TEST_ENV="Container" CONTAINER_OS_TYPE=$(CONTAINER_OS_TYPE) BUILD_TARGET="install-agent-local" \
 	PACKAGES_REPO=$(OSS_PACKAGES_REPO) PACKAGE_NAME=$(PACKAGE_NAME) BASE_IMAGE=$(BASE_IMAGE) \
 	OS_VERSION=$(OS_VERSION) OS_RELEASE=$(OS_RELEASE) DOCKERFILE_PATH=$(DOCKERFILE_PATH) \
 	IMAGE_PATH=$(IMAGE_PATH) TAG=${IMAGE_TAG} CONTAINER_NGINX_IMAGE_REGISTRY=${CONTAINER_NGINX_IMAGE_REGISTRY} \
 	scripts/performance/profiling.sh
 
 	@(GOTOOL) pprof -proto -output=default.pgo merged.pprof build/test/load-cpu-profiling/load/metrics_load_cpu.pprof
-
+	rm $(find . -type f -name "*.test" | xargs)
 
 # run under sudo locally
 load-test-image: ## Build performance load testing image

scripts/performance/profiling.sh

@@ -1,38 +1,17 @@
 #!/usr/bin/env bash
 
-# This script runs Go tests with CPU profiling enabled for all test packages in the test/integration directory.
-# It saves the CPU profiles in the profiles directory with the format <package-name>_<test_type>.pprof
-# Usage: ./scripts/performance/profiling.sh
-#
-# Variables:
-#   - TEST_ENV: The test environment (default: "local")
-#   - CONTAINER_OS_TYPE: The container OS type (default: "linux")
-#   - BUILD_TARGET: The build target (default: "agent")
-#   - PACKAGES_REPO: The packages repository (default: "packages.nginx.org")
-#   - PACKAGE_NAME: The package name (default: "nginx-agent
-#   - BASE_IMAGE: The base image (default: "ubuntu")
-#   - OS_VERSION: The OS version (default: "20.04")
-#   - OS_RELEASE: The OS release (default: "focal")
-#   - DOCKERFILE_PATH: The Dockerfile path (default: "Dockerfile")
-#   - IMAGE_PATH: The image path (default: "nginxinc/nginx-agent")
-#   - TAG: The image tag (default: "latest")
-#   - CONTAINER_NGINX_IMAGE_REGISTRY: The container registry (default: "docker.io")
-# Example:
-#   TEST_ENV=ci CONTAINER_OS_TYPE=linux BUILD_TARGET=agent PACKAGES_REPO=nginxinc PACKAGE_NAME=nginx-agent BASE_IMAGE=ubuntu OS_VERSION=20.04 OS_RELEASE=focal DOCKERFILE_PATH=Dockerfile IMAGE_PATH=nginxinc/nginx-agent TAG=latest CONTAINER_NGINX_IMAGE_REGISTRY=docker.io ./scripts/performance/profiling.sh
-
-## Print all variables in the environment
-#echo "TEST_ENV=$TEST_ENV"
-#echo "CONTAINER_OS_TYPE=$CONTAINER_OS_TYPE"
-#echo "BUILD_TARGET=$BUILD_TARGET"
-#echo "PACKAGES_REPO=$PACKAGES_REPO"
-#echo "PACKAGE_NAME=$PACKAGE_NAME"
-#echo "BASE_IMAGE=$BASE_IMAGE"
-#echo "OS_VERSION=$OS_VERSION"
-#echo "OS_RELEASE=$OS_RELEASE"
-#echo "DOCKERFILE_PATH=$DOCKERFILE_PATH"
-#echo "IMAGE_PATH=$IMAGE_PATH"
-#echo "TAG=$TAG"
-#echo "CONTAINER_NGINX_IMAGE_REGISTRY=$CONTAINER_NGINX_IMAGE_REGISTRY"
+# This script runs Go tests with CPU profiling enabled for all test packages found under the directories:
+#   - internal/watcher
+#   - test/integration
+# It saves the CPU profiles in the $PROFILES_DIR directory with the format <package-name>_<test_type>.pprof
+#   e.g. <package-name>_watcher_cpu.pprof or <package-name>_integration_cpu.pprof
+
+# The variables below can be set to customize the environment for the integration tests:
+# Example using variables defined in our Makefile:
+#   TEST_ENV=ci CONTAINER_OS_TYPE=linux BUILD_TARGET=agent PACKAGES_REPO=nginxinc \
+#     PACKAGE_NAME=nginx-agent BASE_IMAGE=ubuntu OS_VERSION=20.04 OS_RELEASE=focal \
+#     DOCKERFILE_PATH=Dockerfile IMAGE_PATH=nginxinc/nginx-agent TAG=latest CONTAINER_NGINX_IMAGE_REGISTRY=docker.io \
+#     ./scripts/performance/profiling.sh
 
 set -e
 set -o pipefail
@@ -48,7 +27,7 @@ echo "$packages"
 for pkg in $packages; do
     echo "Running tests in package: ${pkg}"
     go test \
-      -count 10 -timeout 1m \
+      -count 10 -timeout 3m \
       -cpuprofile "${PROFILES_DIR}/$(basename $pkg)_watcher_cpu.pprof" \
       "./${pkg}" || { echo "Tests failed in package: ${pkg}, but continuing..."; continue; }
     echo "Profile saved to: ${PROFILES_DIR}/$(basename $pkg)_watcher_cpu.pprof"
@@ -62,18 +41,14 @@ echo "$packages"
 for pkg in $packages; do
     echo "Running tests in package: ${pkg}"
     go test -v \
-      -count 3 -timeout 1m \
-      -cpuprofile "profiles/$(basename $pkg)_integration_cpu.pprof" \
+      -count 3 -timeout 3m \
+      -cpuprofile "${PROFILES_DIR}/$(basename $pkg)_integration_cpu.pprof" \
       "./${pkg}" || { echo "Tests failed in package: ${pkg}, but continuing..."; continue; }
-    mv $(basename $pkg).test profiles/$(basename $pkg)_integration_cpu.test
-    echo "Profile saved to: profiles/${profile_name}"
+    echo "Profile saved to: ${PROFILES_DIR}/$(basename $pkg)_integration_cpu.pprof"
 done
 
 ## Merge all CPU profiles
 files=$(ls ${PROFILES_DIR}/*.pprof)
 echo "Merging CPU profiles: $files"
 go tool pprof -proto -output=${PROFILES_DIR}/merged.pgo $files
-echo "Merged CPU profile saved to: default.pgo"
-
-## Cleanup
-rm *.pprof
+echo "Merged CPU profile saved to: ${PROFILES_DIR}/merged.pgo"