Integration tests for N1C metrics

go.mod

@@ -36,6 +36,8 @@ require (
 	github.com/open-telemetry/opentelemetry-collector-contrib/receiver/hostmetricsreceiver v0.124.1
 	github.com/open-telemetry/opentelemetry-collector-contrib/receiver/tcplogreceiver v0.124.1
 	github.com/open-telemetry/opentelemetry-collector-contrib/testbed v0.124.1
+	github.com/prometheus/client_model v0.6.1
+	github.com/prometheus/common v0.62.0
 	github.com/shirou/gopsutil/v4 v4.25.3
 	github.com/spf13/pflag v1.0.6
 	github.com/stretchr/testify v1.10.0
@@ -202,8 +204,6 @@ require (
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
-	github.com/prometheus/client_model v0.6.1 // indirect
-	github.com/prometheus/common v0.62.0 // indirect
 	github.com/prometheus/procfs v0.16.0 // indirect
 	github.com/rs/cors v1.11.1 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect

test/helpers/test_containers_utils.go

@@ -25,6 +25,13 @@ type Parameters struct {
 	LogMessage           string
 }
 
+type MockCollectorContainers struct {
+	// AgentPlus  testcontainers.Container
+	AgentOSS   testcontainers.Container
+	Otel       testcontainers.Container
+	Prometheus testcontainers.Container
+}
+
 func StartContainer(
 	ctx context.Context,
 	tb testing.TB,
@@ -296,6 +303,161 @@ func StartAuxiliaryMockManagementPlaneGrpcContainer(ctx context.Context, tb test
 	return container
 }
 
+func StartMockCollectorStack(ctx context.Context, tb testing.TB,
+	containerNetwork *testcontainers.DockerNetwork, agentConfig string,
+) *MockCollectorContainers {
+	tb.Helper()
+
+	packageName := Env(tb, "PACKAGE_NAME")
+	packageRepo := Env(tb, "PACKAGES_REPO")
+	baseImage := Env(tb, "BASE_IMAGE")
+	// osRelease := Env(tb, "OS_RELEASE")
+	// osVersion := Env(tb, "OS_VERSION")
+	buildTarget := Env(tb, "BUILD_TARGET")
+	dockerfilePath := Env(tb, "DOCKERFILE_PATH")
+	// containerRegistry := Env(tb, "CONTAINER_NGINX_IMAGE_REGISTRY")
+	// tag := Env(tb, "TAG")
+	// imagePath := Env(tb, "IMAGE_PATH")
+
+	// agentPlus, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
+	//	ContainerRequest: testcontainers.ContainerRequest{
+	//		FromDockerfile: testcontainers.FromDockerfile{
+	//			Context:       "../../../",
+	//			Dockerfile:    "./test/docker/nginx-plus/deb/Dockerfile",
+	//			KeepImage:     false,
+	//			PrintBuildLog: true,
+	//			BuildArgs: map[string]*string{
+	//				"PACKAGE_NAME":                   ToPtr(packageName),
+	//				"PACKAGES_REPO":                  ToPtr(packageRepo),
+	//				"BASE_IMAGE":                     ToPtr(baseImage),
+	//				"OS_RELEASE":                     ToPtr(osRelease),
+	//				"OS_VERSION":                     ToPtr(osVersion),
+	//				"ENTRY_POINT":                    ToPtr("./test/docker/entrypoint.sh"),
+	//				"CONTAINER_NGINX_IMAGE_REGISTRY": ToPtr(containerRegistry),
+	//				"IMAGE_PATH":                     ToPtr(imagePath),
+	//				"TAG":                            ToPtr(tag),
+	//			},
+	//			BuildOptionsModifier: func(buildOptions *types.ImageBuildOptions) {
+	//				buildOptions.Target = "install-nginx"
+	//			},
+	//		},
+	//		Name:     "agent-with-nginx-plus",
+	//		Networks: []string{containerNetwork.Name},
+	//		Files: []testcontainers.ContainerFile{
+	//			{
+	//				HostFilePath:      agentConfig,
+	//				ContainerFilePath: "/etc/nginx-agent/nginx-agent.conf",
+	//				FileMode:          configFilePermissions,
+	//			},
+	//			{
+	//				HostFilePath:      "../../mock/collector/nginx-plus/nginx.conf",
+	//				ContainerFilePath: "/etc/nginx/nginx.conf",
+	//				FileMode:          configFilePermissions,
+	//			},
+	//			{
+	//				HostFilePath:      "../../mock/collector/nginx-plus/conf.d/default.conf",
+	//				ContainerFilePath: "/etc/nginx/conf.d/default.conf",
+	//				FileMode:          configFilePermissions,
+	//			},
+	//		},
+	//	},
+	//	Started: true,
+	// })
+	// require.NoError(tb, err)
+
+	agentOSS, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
+		ContainerRequest: testcontainers.ContainerRequest{
+			FromDockerfile: testcontainers.FromDockerfile{
+				Context:       "../../../",
+				Dockerfile:    dockerfilePath,
+				KeepImage:     false,
+				PrintBuildLog: true,
+				BuildArgs: map[string]*string{
+					"PACKAGE_NAME":  ToPtr(packageName),
+					"PACKAGES_REPO": ToPtr(packageRepo),
+					"BASE_IMAGE":    ToPtr(baseImage),
+					"ENTRY_POINT":   ToPtr("./test/docker/entrypoint.sh"),
+				},
+				BuildOptionsModifier: func(buildOptions *types.ImageBuildOptions) {
+					buildOptions.Target = buildTarget
+				},
+			},
+			Name:     "agent-with-nginx-oss",
+			Networks: []string{containerNetwork.Name},
+			Files: []testcontainers.ContainerFile{
+				{
+					HostFilePath:      agentConfig,
+					ContainerFilePath: "/etc/nginx-agent/nginx-agent.conf",
+					FileMode:          configFilePermissions,
+				},
+				{
+					HostFilePath:      "../../mock/collector/nginx-oss/nginx.conf",
+					ContainerFilePath: "/etc/nginx/nginx.conf",
+					FileMode:          configFilePermissions,
+				},
+				{
+					HostFilePath:      "../../mock/collector/nginx-oss/conf.d/default.conf",
+					ContainerFilePath: "/etc/nginx/conf.d/default.conf",
+					FileMode:          configFilePermissions,
+				},
+			},
+		},
+		Started: true,
+	})
+	require.NoError(tb, err)
+
+	otel, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
+		ContainerRequest: testcontainers.ContainerRequest{
+			FromDockerfile: testcontainers.FromDockerfile{
+				Context:       "../../../",
+				Dockerfile:    "./test/mock/collector/mock-collector/Dockerfile",
+				KeepImage:     false,
+				PrintBuildLog: true,
+			},
+			Name:         "otel-collector",
+			ExposedPorts: []string{"4317/tcp", "9090/tcp", "9775/tcp"},
+			Networks:     []string{containerNetwork.Name},
+			Files: []testcontainers.ContainerFile{
+				{
+					HostFilePath:      "../../mock/collector/otel-collector.yaml",
+					ContainerFilePath: "/etc/otel-collector.yaml",
+					FileMode:          configFilePermissions,
+				},
+			},
+			WaitingFor: wait.ForLog("Everything is ready. Begin running and processing data."),
+		},
+		Started: true,
+	})
+	require.NoError(tb, err)
+
+	prometheus, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
+		ContainerRequest: testcontainers.ContainerRequest{
+			Image:        "prom/prometheus:latest",
+			Name:         "prometheus",
+			ExposedPorts: []string{"9090/tcp"},
+			Networks:     []string{containerNetwork.Name},
+			Files: []testcontainers.ContainerFile{
+				{
+					HostFilePath:      "../../mock/collector/prometheus.yaml",
+					ContainerFilePath: "/etc/prometheus/prometheus.yaml",
+					FileMode:          configFilePermissions,
+				},
+			},
+			Cmd:        []string{"--config.file=/etc/prometheus/prometheus.yml"},
+			WaitingFor: wait.ForLog("Server is ready to receive web requests."),
+		},
+		Started: true,
+	})
+	require.NoError(tb, err)
+
+	return &MockCollectorContainers{
+		// AgentPlus:  agentPlus,
+		AgentOSS:   agentOSS,
+		Otel:       otel,
+		Prometheus: prometheus,
+	}
+}
+
 func ToPtr[T any](value T) *T {
 	return &value
 }
@@ -359,3 +521,34 @@ func LogAndTerminateContainers(
 		require.NoError(tb, err)
 	}
 }
+
+func LogAndTerminateStack(ctx context.Context, tb testing.TB,
+	containers *MockCollectorContainers,
+) {
+	tb.Helper()
+
+	logAndTerminate := func(name string, container testcontainers.Container) {
+		if container == nil {
+			tb.Logf("Skipping log collection for %s: container is nil", name)
+			return
+		}
+
+		tb.Logf("======================== Logging %s Container Logs ========================", name)
+		logReader, err := container.Logs(ctx)
+		require.NoError(tb, err)
+
+		buf, err := io.ReadAll(logReader)
+		require.NoError(tb, err)
+		logs := string(buf)
+
+		tb.Log(logs)
+
+		err = container.Terminate(ctx)
+		require.NoError(tb, err)
+	}
+
+	// logAndTerminate("agent-plus", containers.AgentPlus)
+	logAndTerminate("agent-oss", containers.AgentOSS)
+	logAndTerminate("otel", containers.Otel)
+	logAndTerminate("prometheus", containers.Prometheus)
+}

test/integration/metrics/metrics_test.go

@@ -0,0 +1,112 @@
+// Copyright (c) F5, Inc.
+//
+// This source code is licensed under the Apache License, Version 2.0 license found in the
+// LICENSE file in the root directory of this source tree.
+
+package metrics
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/nginx/agent/v3/test/integration/utils"
+
+	dto "github.com/prometheus/client_model/go"
+	"github.com/stretchr/testify/suite"
+)
+
+type MetricsTestSuite struct {
+	suite.Suite
+	ctx            context.Context
+	teardownTest   func(testing.TB)
+	metricFamilies map[string]*dto.MetricFamily
+}
+
+func (s *MetricsTestSuite) SetupSuite() {
+	s.ctx = context.Background()
+	s.teardownTest = utils.SetupMetricsTest(s.T())
+	time.Sleep(30 * time.Second)
+	s.metricFamilies = utils.ScrapeCollectorMetricFamilies(s.T(), s.ctx, utils.MockCollectorStack.Otel)
+}
+
+func (s *MetricsTestSuite) TearDownSuite() {
+	s.teardownTest(s.T())
+}
+
+func (s *MetricsTestSuite) TestNginxOSS_Test1_TestRequestCount() {
+	family := s.metricFamilies["nginx_http_request_count"]
+	s.T().Logf("nginx_http_request_count metric family: %v", family)
+	s.Require().NotNil(family)
+
+	baselineMetric := utils.SumMetricFamily(family)
+	s.T().Logf("NGINX HTTP request count total: %v", baselineMetric)
+
+	requestCount := 5
+	for range requestCount {
+		url := "http://127.0.0.1/"
+		_, _, err := utils.MockCollectorStack.AgentOSS.Exec(
+			s.ctx,
+			[]string{"curl", "-s", url},
+		)
+		s.Require().NoError(err)
+	}
+
+	time.Sleep(65 * time.Second)
+
+	s.metricFamilies = utils.ScrapeCollectorMetricFamilies(s.T(), s.ctx, utils.MockCollectorStack.Otel)
+	family = s.metricFamilies["nginx_http_request_count"]
+	s.T().Logf("nginx_http_request_count metric family: %v", family)
+	s.Require().NotNil(family)
+
+	got := utils.SumMetricFamily(family)
+
+	s.T().Logf("NGINX HTTP request count total: %v", got)
+	s.Require().GreaterOrEqual(got, baselineMetric+float64(requestCount))
+}
+
+func (s *MetricsTestSuite) TestNginxOSS_Test2_TestResponseCode() {
+	family := s.metricFamilies["nginx_http_response_count"]
+	s.T().Logf("nginx_http_response_count family: %v", family)
+	s.Require().NotNil(family)
+
+	responseCodes := []string{"1xx", "2xx", "3xx", "4xx"}
+	codeRes := make([]float64, 0, len(responseCodes))
+	for code := range responseCodes {
+		codeRes = append(codeRes, utils.SumMetricFamilyLabel(family, "nginx_status_range", responseCodes[code]))
+		s.T().Logf("NGINX HTTP response code %s total: %v", responseCodes[code], codeRes[code])
+		s.Require().NotNil(codeRes[code])
+	}
+}
+
+func (s *MetricsTestSuite) TestHostMetrics_Test1_TestSystemCPUUtilization() {
+	family := s.metricFamilies["system_cpu_utilization"]
+	s.T().Logf("system_cpu_utilization metric family: %v", family)
+	s.Require().NotNil(family)
+
+	cpuUtilizationSystem := utils.SumMetricFamilyLabel(family, "state", "system")
+	cpuUtilizationUser := utils.SumMetricFamilyLabel(family, "state", "user")
+
+	s.T().Logf("System cpu utilization: %v", cpuUtilizationSystem)
+	s.T().Logf("System cpu utilization: %v", cpuUtilizationUser)
+	s.Require().NotNil(cpuUtilizationSystem)
+	s.Require().NotNil(cpuUtilizationUser)
+}
+
+func (s *MetricsTestSuite) TestHostMetrics_Test2_TestSystemMemoryUsage() {
+	family := s.metricFamilies["system_memory_usage"]
+	s.T().Logf("system_memory_usage metric family: %v", family)
+	s.Require().NotNil(family)
+
+	memoryUsageFree := utils.SumMetricFamilyLabel(family, "state", "free")
+	memoryUsageUsed := utils.SumMetricFamilyLabel(family, "state", "used")
+
+	s.T().Logf("System memory usage: %v", memoryUsageFree)
+	s.T().Logf("System memory usage: %v", memoryUsageUsed)
+	s.Require().NotNil(memoryUsageFree)
+	s.Require().NotNil(memoryUsageUsed)
+}
+
+func TestMetricsTestSuite(t *testing.T) {
+	suite.Run(t, new(MetricsTestSuite))
+}