Add telemetry catalogs for NGINX OSS and Plus metrics (#1444)

Author: dhurley

.github/workflows/ci.yml

@@ -81,6 +81,7 @@ jobs:
         uses: golangci/golangci-lint-action@1e7e51e771db61008b38414a730f564565cf7c20 # v9.2.0
         with:
           version: v2.4.0
+          skip-cache: true
 
   vulnerability-scan:
     name: Vulnerability Scan

.gitignore

@@ -7,6 +7,7 @@ whitesource/
 .vscode/
 .idea/
 *.log
+!**/testdata/*.log
 *.test
 *.orig
 sdk/certs/**

Makefile

@@ -110,6 +110,7 @@ $(RPM_PACKAGE):
 include Makefile.tools
 include Makefile.containers
 include Makefile.packaging
+include Makefile.weaver
 
 .PHONY: help clean no-local-changes build lint format unit-test integration-test run dev run-mock-management-grpc-server generate generate-mocks local-apk-package local-deb-package local-rpm-package
 help: ## Show help message
@@ -282,7 +283,7 @@ stop-mock-otel-collector-without-nap: ## Stop running mock management plane OTel
 	@echo "Stopping mock management plane OTel collector without NAP"
 	AGENT_IMAGE_WITH_NGINX_PLUS=nginx_plus_$(IMAGE_TAG):latest AGENT_IMAGE_WITH_NGINX_OSS=nginx_oss_$(IMAGE_TAG):latest $(CONTAINER_COMPOSE) -f ./test/mock/collector/docker-compose.yaml down
 
-generate: ## Generate golang code
+generate: nginx-metadata-gen nginxplus-metadata-gen ## Generate golang code
 	@echo "🗄️ Generating proto files"
 	@cd api/grpc && $(GORUN) $(BUF) generate
 	@echo "🗃️ Generating go files"

Makefile.weaver

@@ -0,0 +1,82 @@
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+# https://github.com/open-telemetry/weaver                                                    #
+# These images are for invoking weaver                                                        #
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+# From where to resolve the containers (e.g. "otel/weaver").
+CONTAINER_REPOSITORY=docker.io
+
+# Per container overrides for the repository resolution.
+WEAVER_CONTAINER_REPOSITORY=$(CONTAINER_REPOSITORY)
+SEMCONVGEN_CONTAINER_REPOSITORY=$(CONTAINER_REPOSITORY)
+OPA_CONTAINER_REPOSITORY=$(CONTAINER_REPOSITORY)
+
+CHECK_TARGETS=install-tools markdownlint
+
+# Versioned, non-qualified references to containers used in this Makefile.
+# These are parsed from dependencies.Dockerfile so dependabot will autoupdate
+# the versions of docker files we use.
+VERSIONED_WEAVER_CONTAINER_NO_REPO=$(shell cat dependencies.Dockerfile | awk '$$4=="weaver" {print $$2}')
+VERSIONED_SEMCONVGEN_CONTAINER_NO_REPO=$(shell cat dependencies.Dockerfile | awk '$$4=="semconvgen" {print $$2}')
+VERSIONED_OPA_CONTAINER_NO_REPO=$(shell cat dependencies.Dockerfile | awk '$$4=="opa" {print $$2}')
+
+# Fully qualified references to containers used in this Makefile. These
+# include the container repository, so that the build will work with tools
+# like "podman" with a default "/etc/containers/registries.conf", where
+# a default respository of "docker.io" is not assumed. This is intended to
+# eliminate errors from podman such as:
+#
+#    Error: short-name "otel/weaver:v1.2.3" did not resolve to an alias
+#    and no unqualified-search registries are defined in "/etc/containers/registries.conf"
+WEAVER_CONTAINER=$(WEAVER_CONTAINER_REPOSITORY)/$(VERSIONED_WEAVER_CONTAINER_NO_REPO)
+SEMCONVGEN_CONTAINER=$(SEMCONVGEN_CONTAINER_REPOSITORY)/$(VERSIONED_SEMCONVGEN_CONTAINER_NO_REPO)
+OPA_CONTAINER=$(OPA_CONTAINER_REPOSITORY)/$(VERSIONED_OPA_CONTAINER_NO_REPO)
+
+# Determine if "docker" is actually podman
+DOCKER_VERSION_OUTPUT := $(shell docker --version 2>&1)
+DOCKER_IS_PODMAN := $(shell echo $(DOCKER_VERSION_OUTPUT) | grep -c podman)
+
+ifeq ($(DOCKER_IS_PODMAN),0)
+    DOCKER_COMMAND := docker
+else
+    DOCKER_COMMAND := podman
+endif
+
+# Debug printing
+ifdef DEBUG
+$(info Docker version output: $(DOCKER_VERSION_OUTPUT))
+$(info Is Docker actually Podman? $(DOCKER_IS_PODMAN))
+$(info Using command: $(DOCKER_COMMAND))
+endif
+
+DOCKER_RUN=$(DOCKER_COMMAND) run
+DOCKER_USER=$(shell id -u):$(shell id -g)
+DOCKER_USER_IS_HOST_USER_ARG=-u $(DOCKER_USER)
+ifeq ($(DOCKER_COMMAND),podman)
+ # On podman, additional arguments are needed to make "-u" work
+ # correctly with the host user ID and host group ID.
+ #
+ #      Error: OCI runtime error: crun: setgroups: Invalid argument
+ DOCKER_USER_IS_HOST_USER_ARG=--userns=keep-id -u $(DOCKER_USER)
+endif
+
+# Generate attribute registry metadata for mdatagen.
+# Example uses: 
+#   make nginxplus-metadata-gen
+#   make nginx-metadata-gen
+.PHONY: %-metadata-gen
+%-metadata-gen:
+	$(DOCKER_RUN) --rm \
+		$(DOCKER_USER_IS_HOST_USER_ARG) \
+		--mount 'type=bind,source=$(PWD)/templates,target=/home/weaver/templates,readonly' \
+		--mount 'type=bind,source=$(PWD)/catalog/$*,target=/home/weaver/source/,readonly' \
+		--mount 'type=bind,source=$(PWD)/internal/collector,target=/home/weaver/target' \
+		$(WEAVER_CONTAINER) registry generate \
+		  --registry=/home/weaver/source \
+		  --templates=/home/weaver/templates \
+		  metadata \
+		  /home/weaver/target/$*receiver
+	cat $(PWD)/internal/collector/$*receiver/*_metadata.yaml > $(PWD)/internal/collector/$*receiver/metadata.yaml
+	rm $(PWD)/internal/collector/$*receiver/*_metadata.yaml
+	@echo "🗃️ Generating go files"
+	@$(GOGEN) $(PWD)/internal/collector/$*receiver/...

catalog/nginx/metrics.yaml

@@ -0,0 +1,63 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/open-telemetry/weaver/v0.9.2/schemas/semconv.schema.json
+groups:
+  # nginx.http.* metrics
+  - id: metric.nginx.http.connections
+    type: metric
+    unit: "connections"
+    metric_name: nginx.http.connections
+    stability: experimental
+    brief: "The total number of connections, since NGINX was last started or reloaded."
+    instrument: counter
+    note: |
+      Source: HTTP GET `/status`
+    attributes:
+      - ref: nginx.connections.outcome
+        requirement_level: required
+
+  - id: metric.nginx.http.connection.count
+    type: metric
+    unit: "connections"
+    metric_name: nginx.http.connection.count
+    stability: experimental
+    brief: "The current number of connections."
+    instrument: gauge
+    note: |
+      Source: HTTP GET `/status`
+    attributes:
+      - ref: nginx.connections.outcome
+        requirement_level: required
+
+  - id: metric.nginx.http.requests
+    type: metric
+    unit: "requests"
+    metric_name: nginx.http.requests
+    stability: experimental
+    brief: "The total number of client requests received, since NGINX was last started or reloaded."
+    instrument: counter
+    note: |
+      Source: HTTP GET `/status`
+    attributes: []
+
+  - id: metric.nginx.http.request.count
+    type: metric
+    unit: "requests"
+    metric_name: nginx.http.request.count
+    stability: experimental
+    brief: "The total number of client requests received, since the last collection interval."
+    instrument: gauge
+    note: |
+      Source: HTTP API `/status`
+    attributes: []
+
+  - id: metric.nginx.http.response.count
+    type: metric
+    unit: "responses"
+    metric_name: nginx.http.response.count
+    stability: experimental
+    brief: "The total number of HTTP responses since the last collection interval, grouped by status code range."
+    instrument: gauge
+    note: |
+      Source: Logs `access.log`
+    attributes:
+      - ref: nginx.status_range
+        requirement_level: required

catalog/nginx/registry.yaml

@@ -0,0 +1,19 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/open-telemetry/weaver/v0.9.2/schemas/semconv.schema.json
+groups:
+  # General NGINX OSS attributes
+  - id: registry.nginx
+    type: attribute_group
+    stability: experimental
+    display_name: General NGINX OSS Attributes
+    brief: "Describes NGINX attributes"
+    attributes:
+      - id: nginx.status_range
+        type: string
+        stability: development
+        brief: "A status code range or bucket for a HTTP response's status code."
+        examples: ["1xx","2xx","3xx","4xx","5xx"]
+      - id: nginx.connections.outcome
+        type: string
+        stability: development
+        brief: "The outcome of a connection"
+        examples: ["ACCEPTED","ACTIVE","HANDLED","READING","WRITING","WAITING"]