fix load test, benchmark json file

Author: sean-breen

.github/workflows/ci.yml

@@ -67,6 +67,7 @@ jobs:
       - name: Run Load Test with CPU profiling
         run: |
           make run-load-test-with-cpu-profiling
+          make generate-pgo-profile
 
       - name: Archive Load Test CPU profile
         uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
@@ -75,10 +76,6 @@ jobs:
           path: build/test/load/profile.pgo
           if-no-files-found: error
 
-      - name: Run Profiling script
-        run: |
-          make generate-pgo-profile
-
       - name: Push PGO profile
         run: |
           git config --global user.name 'github-actions'

Makefile

@@ -46,6 +46,7 @@ PROJECT_FILE	= main.go
 COLLECTOR_PATH  ?= /etc/nginx-agent/opentelemetry-collector-agent.yaml
 MANIFEST_DIR	?= /var/lib/nginx-agent
 DIRS            = $(BUILD_DIR) $(TEST_BUILD_DIR) $(BUILD_DIR)/$(DOCS_DIR) $(BUILD_DIR)/$(DOCS_DIR)/$(PROTO_DIR)
+TIME_NOW		:= $(shell date +%s)
 $(shell mkdir -p $(DIRS))
 
 VERSION ?= $(shell git describe --match "v[0-9]*" --abbrev=0 --tags)
@@ -289,49 +290,78 @@ generate-pgo-profile: build-mock-management-plane-grpc ## Generate PGO profile
 
 # run under sudo locally
 load-test-image: ## Build performance load testing image
-	@echo "🚚 Building load test image"
+	@echo "🚚  Building load test image"
 	$(CONTAINER_BUILDENV) $(CONTAINER_CLITOOL) build \
 		-t $(IMAGE_TAG)_load_test:1.0.0 . \
+		--no-cache \
 		-f ./test/docker/load/Dockerfile \
 		--secret id=nginx-crt,src=$(CERTS_DIR)/nginx-repo.crt \
 		--secret id=nginx-key,src=$(CERTS_DIR)/nginx-repo.key \
 		--build-arg OSARCH=$(OSARCH) \
 		--build-arg GO_VERSION=$(GO_VERSION)
 
 run-load-test-image: ## Run performance load testing image
-	@echo "🚚 Running load tests"
+	@mkdir -p $(TEST_BUILD_DIR)/load
+	@echo "🚚  Running load tests"
 	$(CONTAINER_CLITOOL) rm -f agent-load-test || true
 
-	## Run the load test container
+# Run the load test container
 	@$(CONTAINER_BUILDENV) $(CONTAINER_CLITOOL) run \
 		--name agent-load-test \
 		$(IMAGE_TAG)_load_test:1.0.0
 
-	# Copy the files generated by the load tests
+# Copy the files generated by the load tests
 	@$(CONTAINER_CLITOOL) cp \
-		agent-load-test:/agent/performance/load-tests \
+		agent-load-test:/agent/performance/load \
 		$(TEST_BUILD_DIR)/load
-	@echo "Benchmarks saved to $(TEST_BUILD_DIR)/load"
+	@echo "Results saved to $(TEST_BUILD_DIR)/load"
 	@find $(TEST_BUILD_DIR)/load -type f -exec ls -lh {} \;
 
-	## Stop and remove the container
-	$(CONTAINER_CLITOOL) stop agent-load-test && $(CONTAINER_CLITOOL) rm -f agent-load-test
+# Copy the benchmark json to the $(TEST_BUILD_DIR)/load directory for easier access
+	@$(CONTAINER_CLITOOL) cp \
+		agent-load-test:/agent/test/load \
+		$(TEST_BUILD_DIR)/load
+	@echo "Benchmark results saved to $(TEST_BUILD_DIR)/load/benchmarks.json"
+
+# Copy the results to the $(TEST_BUILD_DIR)/load directory for easier access
+	@$(CONTAINER_CLITOOL) cp \
+		agent-load-test:/agent/test/load/results \
+		$(TEST_BUILD_DIR)/load/
+	@echo "Benchmark results saved to $(TEST_BUILD_DIR)/load/results"
+
+# Stop and remove the container
+	@$(CONTAINER_CLITOOL) stop agent-load-test && $(CONTAINER_CLITOOL) rm -f agent-load-test
 
 run-load-test-with-cpu-profiling: ## Run performance load testing with cpu profiling
-	@echo "🚚 Running load tests with cpu profiling"
+	@echo "🚚  Running load tests with cpu profiling"
+	@mkdir -p $(TEST_BUILD_DIR)
 	@$(CONTAINER_CLITOOL) rm -f agent-load-test-with-cpu-profiling || true
+	@echo "Current epoch time is $(TIME_NOW)"
 
-	## Run the load test container
+# Run the load test container
 	@$(CONTAINER_BUILDENV) $(CONTAINER_CLITOOL) run \
 		--name agent-load-test-with-cpu-profiling \
 		--env PROFILE=true \
 		$(IMAGE_TAG)_load_test:1.0.0
 
-	# Copy the files generated by the load tests
+# Copy the files generated by the load tests
+	DIR=profiles/$(TIME_NOW)
 	@$(CONTAINER_CLITOOL) cp \
-		agent-load-test-with-cpu-profiling:/agent/performance/load-tests \
-		profiles
-	@echo "Benchmarks and profile saved to profiles directory"
+		agent-load-test-with-cpu-profiling:/agent/performance/load \
+		$(DIR)
+	@echo "Benchmarks and profile saved to $(DIR) directory"
+
+# Copy the benchmark results to the root of build directory for easier access
+	@$(CONTAINER_CLITOOL) cp \
+		agent-load-test-with-cpu-profiling:/agent/test/load/benchmarks.json \
+		$(TEST_BUILD_DIR)/load/benchmarks.json
+	@echo "Benchmark results saved to $(TEST_BUILD_DIR)/load/benchmarks.json"
+
+# Copy the results to the $(TEST_BUILD_DIR)/load directory for easier access
+	@$(CONTAINER_CLITOOL) cp \
+		agent-load-test:/agent/test/load/results \
+		$(TEST_BUILD_DIR)/load
+	@echo "Benchmark results saved to $(TEST_BUILD_DIR)/load/results"
 
-	# Stop and remove the container
+# Stop and remove the container
 	@$(CONTAINER_CLITOOL) stop agent-load-test-with-cpu-profiling && $(CONTAINER_CLITOOL) rm -f agent-load-test-with-cpu-profiling

test/docker/load/Dockerfile

@@ -57,7 +57,7 @@ ARG OSARCH
 
 # Install Go
 RUN wget -q https://go.dev/dl/go${GO_VERSION}.linux-${OSARCH}.tar.gz \
-    && tar -xvf go${GO_VERSION}.linux-${OSARCH}.tar.gz \
+    && tar -xf go${GO_VERSION}.linux-${OSARCH}.tar.gz \
     && mv go /usr/local > /dev/null \
     && rm go${GO_VERSION}.linux-${OSARCH}.tar.gz
 

test/docker/load/entrypoint.sh

@@ -1,49 +1,44 @@
 #!/bin/bash
 
 PROFILE=${PROFILE:-"false"}
-DESTINATION="/agent/performance"
-BENCHMARKS_DIR="${DESTINATION}/load-tests"
-mkdir -p ${BENCHMARKS_DIR}
+BENCHMARKS_DIR="/agent/performance/load"
 
 load_test() {
   echo "Running load tests..."
-  echo "Results will be saved to ${BENCHMARKS_DIR}/benchmarks.json"
-  go test -v -timeout 1m ./test/load/... \
-      | tee ${BENCHMARKS_DIR}/benchmarks.json || { echo "Load tests failed, exiting..."; exit 1; }
-  find ${BENCHMARKS_DIR}
+  pushd test/load
+  go test -v -timeout 1m ./...
+
+  cp benchmarks.json ${BENCHMARKS_DIR}
+  cp -r results ${BENCHMARKS_DIR}/results
+  popd
 }
 
 load_test_with_profile() {
   echo "Running load tests with CPU Profiling enabled..."
-  mkdir -p ${BENCHMARKS_DIR}
-  go test -v -timeout 1m ./test/load/... \
-      -cpuprofile ${BENCHMARKS_DIR}/metrics_cpu.pprof \
-      | tee ${BENCHMARKS_DIR}/benchmarks.json || { echo "Load tests failed, exiting..."; exit 1; }
-  cp ${BENCHMARKS_DIR}/metrics_load_cpu.pprof ${BENCHMARKS_DIR}/profile.pgo
-  find ${BENCHMARKS_DIR}
-}
+  pushd test/load
+  go test -v -timeout 1m ./... \
+    -cpuprofile ${BENCHMARKS_DIR}/metrics_load_cpu.pprof
+
+  cp benchmarks.json ${BENCHMARKS_DIR}/benchmarks.json
+  cp -r results ${BENCHMARKS_DIR}/results
 
-copy_agent_files() {
-  echo "Copying agent files..."
-  cp /agent/test/load/results/TestMetric10kDPS/OTLP-linux-build/nginx-agent/agent.log ${BENCHMARKS_DIR}/agent.log \
-    || echo "No agent.log file found"
-  cp /agent/test/load/results/TestMetric10kDPS/OTLP-linux-build/nginx-agent/opentelemetry-collector-agent.log ${BENCHMARKS_DIR}/otel.log \
-    || echo "No opentelemetry-collector-agent.log file found"
+  popd
 }
 
 ## Main script execution starts here
-
-mkdir -p $DESTINATION
+mkdir -p ${BENCHMARKS_DIR}
 
 echo "Running in $(pwd)"
 # Run load tests
 #   $PROFILE - if true, run with CPU profiling enabled
 if [[ "$PROFILE" == "true" ]]; then
   echo "CPU Profiling is enabled."
-  load_test_with_profile
+  load_test_with_profile || { echo "Load tests with profiling failed"; exit 1; }
 else
-  load_test
+  load_test || { echo "Load tests failed"; exit 1; }
 fi
 
-copy_agent_files
+echo "Listing contents of ${BENCHMARKS_DIR}:"
+ls -la ${BENCHMARKS_DIR}
+
 echo "Done."

test/load/otel_collector_plugin_load_test.go

@@ -42,10 +42,6 @@ func TestMetric10kDPS(t *testing.T) {
 			name:        name + "-none",
 			compression: "none",
 		},
-		{
-			name:        name + "-gzip",
-			compression: "gzip",
-		},
 	}
 
 	for _, test := range tests {
@@ -98,6 +94,7 @@ func TestMetric10kDPS(t *testing.T) {
 			tc.ValidateData()
 		})
 
+		// Save intermediate results after each test case
 		testbed.SaveResults(performanceResultsSummary)
 	}
 }