Visualize NAP logs in mock collector grafana (#939)

Author: dhurley

Makefile

@@ -69,12 +69,12 @@ OLD_BENCHMARK_RESULTS_FILE ?= $(TEST_BUILD_DIR)/benchmark.txt
 uname_m    := $(shell uname -m)
 
 ifeq ($(uname_m),aarch64)
-	OSARCH = arm64
+	OSARCH ?= arm64
 else
 	ifeq ($(uname_m),x86_64)
-		OSARCH = amd64
+		OSARCH ?= amd64
 	else
-		OSARCH = $(uname_m)
+		OSARCH ?= $(uname_m)
 	endif
 endif
 
@@ -150,9 +150,9 @@ build-mock-management-plane-grpc:
 	mkdir -p $(BUILD_DIR)/mock-management-plane-grpc
 	@CGO_ENABLED=0 GOARCH=$(OSARCH) GOOS=linux $(GOBUILD) -o $(BUILD_DIR)/mock-management-plane-grpc/server test/mock/grpc/cmd/main.go
 
-build-mock-management-plane-collector:
-	mkdir -p $(BUILD_DIR)/mock-management-plane-collector
-	@CGO_ENABLED=0 GOARCH=$(OSARCH) GOOS=linux $(GOBUILD) -o $(BUILD_DIR)/mock-management-plane-collector/collector test/mock/collector/mock-collector/main.go
+build-mock-management-otel-collector:
+	mkdir -p $(BUILD_DIR)/mock-management-otel-collector
+	@CGO_ENABLED=0 GOARCH=$(OSARCH) GOOS=linux $(GOBUILD) -o $(BUILD_DIR)/mock-management-otel-collector/collector test/mock/collector/mock-collector/main.go
 
 integration-test: $(SELECTED_PACKAGE) build-mock-management-plane-grpc
 	TEST_ENV="Container" CONTAINER_OS_TYPE=$(CONTAINER_OS_TYPE) BUILD_TARGET="install-agent-local" CONTAINER_NGINX_IMAGE_REGISTRY=${CONTAINER_NGINX_IMAGE_REGISTRY} \
@@ -190,6 +190,18 @@ run-mock-management-grpc-server: ## Run mock management plane gRPC server
 	@echo "🖲️ Running mock management plane gRPC server"
 	$(GORUN) test/mock/grpc/cmd/main.go -configDirectory=$(MOCK_MANAGEMENT_PLANE_CONFIG_DIRECTORY) -logLevel=$(MOCK_MANAGEMENT_PLANE_LOG_LEVEL) -grpcAddress=$(MOCK_MANAGEMENT_PLANE_GRPC_ADDRESS) -apiAddress=$(MOCK_MANAGEMENT_PLANE_API_ADDRESS)
 
+
+.PHONY: build-test-nginx-plus-and-nap-image
+build-test-nginx-plus-and-nap-image:
+	$(CONTAINER_BUILDENV) $(CONTAINER_CLITOOL) build -t nginx_plus_and_nap_$(IMAGE_TAG) . \
+		--no-cache -f ./test/docker/nginx-plus-and-nap/deb/Dockerfile \
+		--secret id=nginx-crt,src=$(CERTS_DIR)/nginx-repo.crt \
+		--secret id=nginx-key,src=$(CERTS_DIR)/nginx-repo.key \
+		--build-arg PACKAGE_NAME=$(PACKAGE_NAME) \
+		--build-arg PACKAGES_REPO=$(OSS_PACKAGES_REPO) \
+		--build-arg BASE_IMAGE=$(BASE_IMAGE) \
+		--build-arg ENTRY_POINT=./test/docker/entrypoint.sh
+
 .PHONY: build-test-plus-image
 build-test-plus-image:
 	$(CONTAINER_BUILDENV) $(CONTAINER_CLITOOL) build -t nginx_plus_$(IMAGE_TAG) . \
@@ -211,20 +223,20 @@ build-test-oss-image:
 		--build-arg BASE_IMAGE=$(BASE_IMAGE) \
 		--build-arg ENTRY_POINT=./test/docker/entrypoint.sh
 
-.PHONY: build-mock-collector-image
-build-mock-collector-image: build-mock-management-plane-collector
+.PHONY: build-mock-management-otel-collector-image
+build-mock-management-otel-collector-image: build-mock-management-otel-collector
 	$(CONTAINER_BUILDENV) $(CONTAINER_CLITOOL) build -t mock-collector . \
 		--no-cache -f ./test/mock/collector/mock-collector/Dockerfile
 
 .PHONY: run-mock-management-otel-collector
 run-mock-management-otel-collector: ## Run mock management plane OTel collector
 	@echo "🚀 Running mock management plane OTel collector"
-	AGENT_IMAGE_WITH_NGINX_PLUS=nginx_plus_$(IMAGE_TAG):latest AGENT_IMAGE_WITH_NGINX_OSS=nginx_oss_$(IMAGE_TAG):latest $(CONTAINER_COMPOSE) -f ./test/mock/collector/docker-compose.yaml up -d
+	AGENT_IMAGE_WITH_NGINX_PLUS=nginx_plus_$(IMAGE_TAG):latest AGENT_IMAGE_WITH_NGINX_OSS=nginx_oss_$(IMAGE_TAG):latest AGENT_IMAGE_WITH_NGINX_PLUS_AND_NAP=nginx_plus_and_nap_$(IMAGE_TAG):latest $(CONTAINER_COMPOSE) -f ./test/mock/collector/docker-compose.yaml up -d
 
 .PHONY: stop-mock-management-otel-collector
 stop-mock-management-otel-collector: ## Stop running mock management plane OTel collector
 	@echo "Stopping mock management plane OTel collector"
-	AGENT_IMAGE_WITH_NGINX_PLUS=nginx_plus_$(IMAGE_TAG):latest AGENT_IMAGE_WITH_NGINX_OSS=nginx_oss_$(IMAGE_TAG):latest $(CONTAINER_COMPOSE) -f ./test/mock/collector/docker-compose.yaml down
+	AGENT_IMAGE_WITH_NGINX_PLUS=nginx_plus_$(IMAGE_TAG):latest AGENT_IMAGE_WITH_NGINX_OSS=nginx_oss_$(IMAGE_TAG):latest AGENT_IMAGE_WITH_NGINX_PLUS_AND_NAP=nginx_plus_and_nap_$(IMAGE_TAG):latest $(CONTAINER_COMPOSE) -f ./test/mock/collector/docker-compose.yaml down
 
 generate: ## Generate golang code
 	@echo "🗄️ Generating proto files"

test/docker/entrypoint.sh

@@ -15,14 +15,15 @@ trap 'handle_term' TERM
 
 # Launch nginx
 echo "starting nginx ..."
+/bin/su -s /bin/sh -c "/usr/share/ts/bin/bd-socket-plugin tmm_count 4 proc_cpuinfo_cpu_mhz 2000000 total_xml_memory 307200000 total_umu_max_size 3129344 sys_max_account_id 1024 no_static_config 2>&1 >> /var/log/app_protect/bd-socket-plugin.log &" nginx
 /usr/sbin/nginx -g "daemon off;" &
 
 nginx_pid=$!
 
 SECONDS=0
 
 while ! ps -ef | grep "nginx: master process" | grep -v grep; do 
-    if (( SECONDS > 5 )); then 
+    if (( SECONDS > 30 )); then
         echo "couldn't find nginx master process"
         exit 1
     fi

test/docker/nginx-plus-and-nap/deb/Dockerfile

@@ -0,0 +1,84 @@
+ARG BASE_IMAGE
+FROM --platform=linux/amd64 ${BASE_IMAGE} as install-nginx
+LABEL maintainer="NGINX Docker Maintainers <docker-maint@nginx.com>"
+
+ARG DEBIAN_FRONTEND=noninteractive
+ARG ENTRY_POINT
+ARG PACKAGE_NAME
+ARG PACKAGES_REPO
+
+WORKDIR /agent
+COPY ./build /agent/build
+COPY $ENTRY_POINT /agent/entrypoint.sh
+
+ENV PLUS_VERSION=R32
+
+RUN --mount=type=secret,id=nginx-crt,dst=nginx-repo.crt \
+    --mount=type=secret,id=nginx-key,dst=nginx-repo.key \
+    set -x \
+# Create nginx user/group first, to be consistent throughout Docker variants
+    && groupadd --system --gid 101 nginx \
+    && useradd --system --gid nginx --no-create-home --home-dir /nonexistent --uid 101 nginx \
+    && apt-get update --allow-releaseinfo-change \
+    && apt-get install --no-install-recommends --no-install-suggests -y \
+                        ca-certificates \
+                        gnupg1 \
+                        lsb-release \
+                        apt-transport-https \
+                        git \
+                        wget \
+                        make \
+                        gnupg2 \
+                        ubuntu-keyring \
+    && wget -qO - https://cs.nginx.com/static/keys/app-protect-security-updates.key | gpg --dearmor | tee /usr/share/keyrings/app-protect-security-updates.gpg >/dev/null \
+    && wget -qO - https://cs.nginx.com/static/keys/nginx_signing.key | gpg --dearmor | tee /usr/share/keyrings/nginx-archive-keyring.gpg >/dev/null \
+    && \
+    NGINX_GPGKEY=573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62; \
+    found=''; \
+    for server in \
+        hkp://keyserver.ubuntu.com:80 \
+        pgp.mit.edu \
+    ; do \
+        echo "Fetching GPG key $NGINX_GPGKEY from $server"; \
+        apt-key adv --keyserver "$server" --keyserver-options timeout=10 --recv-keys "$NGINX_GPGKEY" && found=yes && break; \
+    done; \
+    test -z "$found" && echo >&2 "error: failed to fetch GPG key $NGINX_GPGKEY" && exit 1; \
+    apt-get remove --purge --auto-remove -y gnupg1 && rm -rf /var/lib/apt/lists/* \
+# Install the latest release of NGINX Plus and/or NGINX Plus modules
+# Uncomment individual modules if necessary
+# Use versioned packages over defaults to specify a release
+    && nginxPackages=" \
+        app-protect \
+    " \
+    && echo "Acquire::https::pkgs.nginx.com::Verify-Peer \"true\";" > /etc/apt/apt.conf.d/90nginx \
+    && echo "Acquire::https::pkgs.nginx.com::Verify-Host \"true\";" >> /etc/apt/apt.conf.d/90nginx \
+    && echo "Acquire::https::pkgs.nginx.com::SslCert     \"/etc/ssl/nginx/nginx-repo.crt\";" >> /etc/apt/apt.conf.d/90nginx \
+    && echo "Acquire::https::pkgs.nginx.com::SslKey      \"/etc/ssl/nginx/nginx-repo.key\";" >> /etc/apt/apt.conf.d/90nginx \
+    && printf "deb https://pkgs.nginx.com/plus/${PLUS_VERSION}/ubuntu/ `lsb_release -cs` nginx-plus\n" > /etc/apt/sources.list.d/nginx-plus.list \
+    && printf "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] https://pkgs.nginx.com/app-protect/${PLUS_VERSION}/ubuntu `lsb_release -cs` nginx-plus\n" | tee /etc/apt/sources.list.d/nginx-app-protect.list \
+    && printf "deb [signed-by=/usr/share/keyrings/app-protect-security-updates.gpg] https://pkgs.nginx.com/app-protect-security-updates/ubuntu `lsb_release -cs` nginx-plus\n" | tee -a /etc/apt/sources.list.d/nginx-app-protect.list \
+    && mkdir -p /etc/ssl/nginx \
+    && cat nginx-repo.crt > /etc/ssl/nginx/nginx-repo.crt \
+    && cat nginx-repo.key > /etc/ssl/nginx/nginx-repo.key \
+    && apt-get update \
+    && apt-get install --no-install-recommends --no-install-suggests -y \
+                        $nginxPackages \
+                        curl \
+                        gettext-base \
+                        jq \
+    && apt-get remove --purge -y lsb-release \
+    && apt-get remove --purge --auto-remove -y && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/nginx-plus.list /etc/apt/sources.list.d/nginx-app-protect.list \
+    && rm -rf /etc/apt/apt.conf.d/90nginx /etc/ssl/nginx
+
+EXPOSE 80
+
+STOPSIGNAL SIGQUIT
+
+RUN chmod +x /agent/entrypoint.sh
+RUN apt install -y /agent/build/${PACKAGE_NAME}.deb
+
+STOPSIGNAL SIGTERM
+
+EXPOSE 80 443
+
+ENTRYPOINT ["/agent/entrypoint.sh"]

test/mock/collector/README.md

@@ -1,30 +1,38 @@
 # Mock Management OTel Collector
 
-There are 3 images that need to be built in order to use the mock management OTel collector
+There are 4 images that need to be built in order to use the mock management OTel collector
 * Agent with NGINX Plus image
+* Agent with NGINX Plus & NGINX App Protect image
 * Agent with NGINX OSS image
 * Custom OTel collector image
 
 To build these images run the following
 ```
-make local-deb-package build-test-plus-image build-test-oss-image build-mock-collector-image
+OSARCH=amd64 make local-deb-package build-test-nginx-plus-and-nap-image
+make local-deb-package build-test-oss-image build-test-plus-image build-mock-management-otel-collector-image
 ```
 
+[**Note:** We need to build the test NGINX Plus with NAP image with the environment variable `OSARCH=amd64` since NGINX App Protect doesn't support ARM yet.]
+
 To start run everything run the following
 ```
 make run-mock-management-otel-collector
 ```
 
-Once everything is started there should be 5 containers running
-```
-8e6df6d0bc73  localhost/nginx_plus_agent_ubuntu_22.04:latest                        4 minutes ago  Up 4 minutes              80/tcp, 443/tcp                                 mock-collector-agent-with-nginx-plus
-a65a7efaf2b3  localhost/nginx_oss_agent_ubuntu_22.04:latest                         4 minutes ago  Up 4 minutes              80/tcp, 443/tcp                                 mock-collector-agent-with-nginx-oss
-bf0f247991c0  localhost/mock-collector:latest                 go run main.go        4 minutes ago  Up 4 minutes              0.0.0.0:4320->4317/tcp, 0.0.0.0:9775->9090/tcp  mock-collector-otel-collector
-67bb7bde6392  docker.io/prom/prometheus:latest                --config.file=/et...  4 minutes ago  Up 4 minutes              0.0.0.0:9090->9090/tcp, 9090/tcp                mock-collector-prometheus
-a83a997eb652  docker.io/grafana/grafana:latest                                      4 minutes ago  Up 4 minutes              0.0.0.0:3002->3000/tcp, 3000/tcp                mock-collector-grafana
+Once everything is started there should be 7 containers running
+```
+CONTAINER ID  IMAGE                                                   COMMAND               CREATED         STATUS         PORTS                                           NAMES
+e0e57897223d  localhost/nginx_plus_and_nap_agent_ubuntu_22.04:latest                        17 seconds ago  Up 17 seconds  80/tcp, 443/tcp                                 mock-collector-agent-with-nginx-plus-and-nap
+f4e7264ceb0b  localhost/nginx_plus_agent_ubuntu_22.04:latest                                15 seconds ago  Up 16 seconds  80/tcp, 443/tcp                                 mock-collector-agent-with-nginx-plus
+97e9e84679a2  localhost/nginx_oss_agent_ubuntu_22.04:latest                                 14 seconds ago  Up 14 seconds  80/tcp, 443/tcp                                 mock-collector-agent-with-nginx-oss
+04a3bce14c7a  localhost/mock-collector:latest                         /mock-management-...  12 seconds ago  Up 13 seconds  0.0.0.0:4320->4317/tcp, 0.0.0.0:9775->9090/tcp  mock-collector-otel-collector
+8efe150cbd96  docker.io/prom/prometheus:latest                        --config.file=/et...  11 seconds ago  Up 11 seconds  0.0.0.0:9090->9090/tcp, 9090/tcp                mock-collector-prometheus
+e55572131d2a  docker.io/grafana/grafana:latest                                              9 seconds ago   Up 10 seconds  0.0.0.0:3002->3000/tcp, 3000/tcp                mock-collector-grafana
+69c8efab591b  docker.io/grafana/loki:latest                           -config.file=/etc...  8 seconds ago   Up 8 seconds   0.0.0.0:3100->3100/tcp, 3100/tcp                mock-collector-loki
+
 ```
 
-To view the metrics, the grafana UI can be used by accessing this URL http://localhost:3002/login (Note: username/password is admin/admin)
+To view the metrics & logs, the grafana UI can be used by accessing this URL http://localhost:3002/login (Note: username/password is admin/admin)
 
 To stop everything run the following
 ```

test/mock/collector/docker-compose.yaml

@@ -7,12 +7,23 @@ volumes:
   grafana-storage:
 
 services:
+  agent-with-nginx-plus-and-nap:
+    image: ${AGENT_IMAGE_WITH_NGINX_PLUS_AND_NAP}
+    container_name: mock-collector-agent-with-nginx-plus-and-nap
+    volumes:
+      - ./nginx-agent.conf:/etc/nginx-agent/nginx-agent.conf
+      - ./nginx-plus-and-nap/nginx.conf:/etc/nginx/nginx.conf
+      - ./nginx-plus-and-nap/conf.d/default.conf:/etc/nginx/conf.d/default.conf
+    networks:
+      - metrics
+
   agent-with-nginx-plus:
     image: ${AGENT_IMAGE_WITH_NGINX_PLUS}
     container_name: mock-collector-agent-with-nginx-plus
     volumes:
       - ./nginx-agent.conf:/etc/nginx-agent/nginx-agent.conf
-      - ./nginx-plus:/etc/nginx/
+      - ./nginx-plus/nginx.conf:/etc/nginx/nginx.conf
+      - ./nginx-plus/conf.d/default.conf:/etc/nginx/conf.d/default.conf
     networks:
       - metrics
 
@@ -53,12 +64,27 @@ services:
     image: grafana/grafana:latest
     container_name: mock-collector-grafana
     restart: unless-stopped
+    environment:
+      GF_INSTALL_PLUGINS: "grafana-lokiexplore-app"
     ports:
       - "3002:3000"
     volumes:
       - grafana-storage:/var/lib/grafana
       - ./grafana/provisioning/datasources:/etc/grafana/provisioning/datasources
       - ./grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards
+      - ./grafana/provisioning/plugins:/etc/grafana/provisioning/plugins
       - ./grafana/provisioning/dashboards:/var/lib/grafana/dashboards
     networks:
       - metrics
+
+  loki:
+    image: grafana/loki:latest
+    container_name: mock-collector-loki
+    restart: unless-stopped
+    ports:
+      - "3100:3100"
+    volumes:
+      - ./loki-config.yaml:/etc/loki/local-config.yaml
+    command: -config.file=/etc/loki/local-config.yaml
+    networks:
+      - metrics

test/mock/collector/grafana/provisioning/datasources/datasource.yml

@@ -7,3 +7,8 @@ datasources:
     access: proxy
     url: http://prometheus:9090
     isDefault: true
+  - name: Loki
+    type: loki
+    uid: otel-loki-scraper
+    access: proxy
+    url: http://loki:3100

test/mock/collector/grafana/provisioning/plugins/plugin.yaml

@@ -0,0 +1,5 @@
+apiVersion: 1
+
+apps:
+  - type: grafana-lokiexplore-app
+    org_id: 1

test/mock/collector/loki-config.yaml

@@ -0,0 +1,36 @@
+auth_enabled: false
+
+limits_config:
+  allow_structured_metadata: true
+  volume_enabled: true
+
+server:
+  http_listen_port: 3100
+
+common:
+  ring:
+    instance_addr: 0.0.0.0
+    kvstore:
+      store: inmemory
+  replication_factor: 1
+  path_prefix: /tmp/loki
+
+schema_config:
+  configs:
+    - from: 2020-05-15
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+storage_config:
+  tsdb_shipper:
+    active_index_directory: /tmp/loki/index
+    cache_location: /tmp/loki/index_cache
+  filesystem:
+    directory: /tmp/loki/chunks
+
+pattern_ingester:
+  enabled: true

test/mock/collector/mock-collector/Dockerfile

@@ -1,6 +1,6 @@
 FROM golang:bookworm
 
-WORKDIR /mock-management-plane-collector
-COPY ./build/mock-management-plane-collector ./
+WORKDIR /mock-management-otel-collector
+COPY ./build/mock-management-otel-collector ./
 
-CMD ["/mock-management-plane-collector/collector"]
+CMD ["/mock-management-otel-collector/collector"]

test/mock/collector/mock-collector/main.go

@@ -16,10 +16,12 @@ import (
 
 	"github.com/nginx/agent/v3/test/mock/collector/mock-collector/auth"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusexporter"
+	"github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourceprocessor"
 	"go.opentelemetry.io/collector/connector"
 	"go.opentelemetry.io/collector/exporter"
 	"go.opentelemetry.io/collector/exporter/debugexporter"
 	"go.opentelemetry.io/collector/exporter/otlpexporter"
+	"go.opentelemetry.io/collector/exporter/otlphttpexporter"
 	"go.opentelemetry.io/collector/extension"
 	"go.opentelemetry.io/collector/processor"
 	"go.opentelemetry.io/collector/processor/batchprocessor"
@@ -85,13 +87,15 @@ func components() (otelcol.Factories, error) {
 		debugexporter.NewFactory(),
 		otlpexporter.NewFactory(),
 		prometheusexporter.NewFactory(),
+		otlphttpexporter.NewFactory(),
 	)
 	if err != nil {
 		return otelcol.Factories{}, err
 	}
 
 	factories.Processors, err = processor.MakeFactoryMap(
 		batchprocessor.NewFactory(),
+		resourceprocessor.NewFactory(),
 	)
 	if err != nil {
 		return otelcol.Factories{}, err