diff --git a/labs/lab7/docker-compose.yml b/labs/lab7/docker-compose.yml index e9c563b..9b1b73d 100644 --- a/labs/lab7/docker-compose.yml +++ b/labs/lab7/docker-compose.yml @@ -1,45 +1,52 @@ -# NGINX Plus Proxy with tools build +# NGINX Plus Proxy with NGINX Agent # NGINX webservers with ingress-demo pages -# NGINX Basics, Nov 2024 +# NGINX One Console, Mar 2025 # Chris Akker, Shouvik Dutta, Adam Currier # services: - nginx-plus: # NGINX Plus Web / Load Balancer - hostname: nginx-plus - container_name: nginx-plus - image: nginx-plus:workshop # From Lab1 - volumes: # Sync these folders to container + nginx-plus: # NGINX Plus Web / Load Balancer + environment: + NGINX_AGENT_SERVER_HOST: 'agent.connect.nginx.com' + NGINX_AGENT_SERVER_GRPCPORT: '443' + NGINX_AGENT_TLS_ENABLE: 'true' + NGINX_AGENT_SERVER_TOKEN: $TOKEN # Datakey From One Console + hostname: $NAME-nginx-plus + container_name: $NAME-nginx-plus + image: private-registry.nginx.com/nginx-plus/agent:nginx-plus-r32-debian # From Nginx Private Registry + volumes: # Sync these folders to container - ./nginx-plus/etc/nginx/nginx.conf:/etc/nginx/nginx.conf - ./nginx-plus/etc/nginx/conf.d:/etc/nginx/conf.d - ./nginx-plus/etc/nginx/includes:/etc/nginx/includes - ./nginx-plus/usr/share/nginx/html:/usr/share/nginx/html + - ./nginx-plus/usr/share/nginx-plus-module-prometheus:/usr/share/nginx-plus-module-prometheus ports: - 80:80 # Open for HTTP - 443:443 # Open for HTTPS - - 9000:9000 # Open for stub status page - - 9113:9113 # Open for Prometheus Scraper page - restart: always + - 9000:9000 # Open for API / Dashboard page + - 9113:9113 # Open for Prometheus scraper page + restart: always + # web1: - hostname: web1 - container_name: web1 + hostname: $NAME-web1 + container_name: $NAME-web1 image: nginxinc/ingress-demo # Image from Docker Hub ports: - "80" # Open for HTTP - "443" # Open for HTTPS web2: - hostname: web2 - container_name: web2 + hostname: $NAME-web2 + container_name: $NAME-web2 image: nginxinc/ingress-demo ports: - "80" - "433" web3: - hostname: web3 - container_name: web3 + hostname: $NAME-web3 + container_name: $NAME-web3 image: nginxinc/ingress-demo ports: - "80" - - "443" + - "443" prometheus: hostname: prometheus container_name: prometheus @@ -47,7 +54,7 @@ services: volumes: - ./nginx-plus/etc/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml ports: - - "9090:9090" + - "9090:9090" # Prometheus Server Web Console restart: always depends_on: - nginx-plus @@ -58,7 +65,7 @@ services: - grafana-storage:/var/lib/grafana image: grafana/grafana ports: - - "3000:3000" + - "3000:3000" # Grafana Server Web Console restart: always depends_on: - nginx-plus diff --git a/labs/lab7/final/nginx.conf b/labs/lab7/final/nginx.conf new file mode 100644 index 0000000..e0f0aad --- /dev/null +++ b/labs/lab7/final/nginx.conf @@ -0,0 +1,43 @@ +# NGINX One Console, Mar 2025 +# Chris Akker, Shouvik Dutta, Adam Currier +# nginx.conf +# +user nginx; +worker_processes 1; + +error_log /var/log/nginx/error.log info; +pid /var/run/nginx.pid; + +# Uncomment to enable NGINX Java Script Module +load_module modules/ngx_http_js_module.so; # Added for Prometheus + +events { + worker_connections 1024; +} + + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + include /etc/nginx/includes/log_formats/*.conf; # Custom Access logs formats found here + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + + include /etc/nginx/conf.d/*.conf; + + # Uncomment for Prometheus scraper page output + subrequest_output_buffer_size 32k; # Added for Prometheus + +} diff --git a/labs/lab7/final/prometheus.conf b/labs/lab7/final/prometheus.conf new file mode 100644 index 0000000..da60b30 --- /dev/null +++ b/labs/lab7/final/prometheus.conf @@ -0,0 +1,22 @@ +# Nginx One Console +# NGINX Plus Prometheus configuration, for HTTP scraper page +# Chris Akker, Shouvik Dutta, Adam Currier, Mar 2025 +# Based on: https://www.f5.com/company/blog/nginx/how-to-visualize-nginx-plus-with-prometheus-and-grafana +# prometheus.conf +# +# Uncomment all lines below +js_import /usr/share/nginx-plus-module-prometheus/prometheus.js; + +server { + + listen 9113; # This is the default port for Prometheus scraper page + + location = /metrics { + js_content prometheus.metrics; + } + + location /api { + api; + } + +} diff --git a/labs/lab7/media/lab7_edit-nginx-conf.png b/labs/lab7/media/lab7_edit-nginx-conf.png new file mode 100644 index 0000000..7a22317 Binary files /dev/null and b/labs/lab7/media/lab7_edit-nginx-conf.png differ diff --git a/labs/lab7/media/lab7_edit-prometheus-conf.png b/labs/lab7/media/lab7_edit-prometheus-conf.png new file mode 100644 index 0000000..63ce93f Binary files /dev/null and b/labs/lab7/media/lab7_edit-prometheus-conf.png differ diff --git a/labs/lab7/media/lab7_prometheus-targets.png b/labs/lab7/media/lab7_prometheus-targets.png new file mode 100644 index 0000000..a5fa7fb Binary files /dev/null and b/labs/lab7/media/lab7_prometheus-targets.png differ diff --git a/labs/lab7/media/lab7_prometheus-upstream-response-time.png b/labs/lab7/media/lab7_prometheus-upstream-response-time.png new file mode 100644 index 0000000..6837324 Binary files /dev/null and b/labs/lab7/media/lab7_prometheus-upstream-response-time.png differ diff --git a/labs/lab7/nginx-plus/Dockerfile b/labs/lab7/nginx-plus/Dockerfile.old similarity index 100% rename from labs/lab7/nginx-plus/Dockerfile rename to labs/lab7/nginx-plus/Dockerfile.old diff --git a/labs/lab7/nginx-plus/Dockerfile.test b/labs/lab7/nginx-plus/Dockerfile.test new file mode 100644 index 0000000..ab17479 --- /dev/null +++ b/labs/lab7/nginx-plus/Dockerfile.test @@ -0,0 +1,38 @@ +# NginxPlus with Agent and NJS and Prometheus modules +# Nginx One Workshop - Internal F5 +# Chris Akker, Shouvik Dutta, Adam Currier - Mar 2025 +# +FROM private-registry.nginx.com/nginx-plus/agent:r32 +COPY --from=private-registry.nginx.com/nginx-plus/modules:r32-prometheus-debian usr/ /usr/ +# +# +# Install prerequisite packages: +RUN apt-get update && apt-get install -y apt-transport-https lsb-release ca-certificates curl wget gnupg2 net-tools vim tree openssl jq +# +# Download the apt configuration to `/etc/apt/apt.conf.d`: +RUN wget -P /etc/apt/apt.conf.d https://cs.nginx.com/static/files/90pkgs-nginx +# +# Copy certificate files and dhparams +# COPY etc/ssl /etc/ssl +# +# COPY /etc/nginx (Nginx configuration) directory +COPY etc/nginx /etc/nginx +RUN chown -R nginx:nginx /etc/nginx +# # Forward request logs to Docker log collector: +# && ln -sf /dev/stdout /var/log/nginx/access.log \ +# && ln -sf /dev/stderr /var/log/nginx/error.log \ +# Remove the cert/keys from the image +# && rm /etc/ssl/nginx/nginx-repo.crt /etc/ssl/nginx/nginx-repo.key +# +# COPY /usr/share/nginx/html (Nginx files) directory +COPY usr/share/nginx/html /usr/share/nginx/html +RUN chown -R nginx:nginx /usr/share/nginx/html +# +# COPY /etc/prometheus (Prometheus files) directory +COPY /etc/prometheus /etc/prometheus +RUN chown -R nginx:nginx /etc/prometheus +# +# EXPOSE ports, HTTP 80, HTTP 8080, HTTPS 443, Nginx API / status page 9000, Prometheus Exporter page 9113 +EXPOSE 80 443 8080 9000 9113 +STOPSIGNAL SIGTERM +CMD ["nginx", "-g", "daemon off;"] diff --git a/labs/lab7/nginx-plus/etc/nginx/conf.d/dashboard.conf b/labs/lab7/nginx-plus/etc/nginx/conf.d/dashboard.conf index 3b1ace3..2f4c14b 100644 --- a/labs/lab7/nginx-plus/etc/nginx/conf.d/dashboard.conf +++ b/labs/lab7/nginx-plus/etc/nginx/conf.d/dashboard.conf @@ -1,4 +1,4 @@ -# NGINX Plus Basics, Nov 2024 +# NGINX One Console, Nov 2024 # Chris Akker, Shouvik Dutta, Adam Currier # dashboard.conf # diff --git a/labs/lab7/nginx-plus/etc/nginx/conf.d/prometheus.conf b/labs/lab7/nginx-plus/etc/nginx/conf.d/prometheus.conf index 56b2ecf..6a5480b 100644 --- a/labs/lab7/nginx-plus/etc/nginx/conf.d/prometheus.conf +++ b/labs/lab7/nginx-plus/etc/nginx/conf.d/prometheus.conf @@ -1,7 +1,7 @@ # NGINX Plus Prometheus configuration, for HTTP scraper page # Chris Akker, Nov 2024 # Based on: https://www.f5.com/company/blog/nginx/how-to-visualize-nginx-plus-with-prometheus-and-grafana -# Nginx Basics +# Nginx One Console # # Uncomment all lines below # js_import /usr/share/nginx-plus-module-prometheus/prometheus.js; diff --git a/labs/lab7/nginx-plus/etc/nginx/conf.d/status_ok.conf b/labs/lab7/nginx-plus/etc/nginx/conf.d/status_ok.conf index adc6bbe..4a86c95 100644 --- a/labs/lab7/nginx-plus/etc/nginx/conf.d/status_ok.conf +++ b/labs/lab7/nginx-plus/etc/nginx/conf.d/status_ok.conf @@ -1,5 +1,9 @@ +# NGINX One Console, Nov 2024 +# Chris Akker, Shouvik Dutta, Adam Currier +# status_ok.conf # Simple health check expecting http 200 and correct Content-Type +# match status_ok { status 200; - header Content-Type = "text/html; charset=utf-8"; # For the nginx-cafe html -} \ No newline at end of file + header Content-Type = "text/html; charset=utf-8"; # For the nginx-cafe html +} diff --git a/labs/lab7/nginx-plus/etc/nginx/conf.d/upstreams.conf b/labs/lab7/nginx-plus/etc/nginx/conf.d/upstreams.conf index 9879cf6..fb96cb1 100644 --- a/labs/lab7/nginx-plus/etc/nginx/conf.d/upstreams.conf +++ b/labs/lab7/nginx-plus/etc/nginx/conf.d/upstreams.conf @@ -1,5 +1,5 @@ # NGINX Basics, Plus Proxy to three upstream NGINX containers -# Nov 2024 - Chris Akker, Shouvik Dutta, Adam Currier +# Chris Akker, Shouvik Dutta, Adam Currier - Mar 2025 # nginx_cafe servers # upstream nginx_cafe { # Upstream block, the name is "nginx_cafe" diff --git a/labs/lab7/nginx-plus/etc/nginx/includes/keepalive.conf b/labs/lab7/nginx-plus/etc/nginx/includes/keepalive.conf index a9fbcf4..d920a64 100644 --- a/labs/lab7/nginx-plus/etc/nginx/includes/keepalive.conf +++ b/labs/lab7/nginx-plus/etc/nginx/includes/keepalive.conf @@ -1,4 +1,4 @@ -# Default is HTTP/1, keepalive is only enabled in HTTP/1.1 +# Nginx default to upstreams is HTTP/1.0, keepalive is only enabled in HTTP/1.1 proxy_http_version 1.1; # Remove the Connection header if the client sends it, diff --git a/labs/lab7/nginx-plus/etc/nginx/includes/proxy_headers.conf b/labs/lab7/nginx-plus/etc/nginx/includes/proxy_headers.conf index 23a83d1..f328b28 100644 --- a/labs/lab7/nginx-plus/etc/nginx/includes/proxy_headers.conf +++ b/labs/lab7/nginx-plus/etc/nginx/includes/proxy_headers.conf @@ -1,6 +1,6 @@ ## Set Headers to the proxied servers ## -# client address in a binary form, value’s length is always 4 bytes for IPv4 addresses or 16 bytes for IPv6 addresses +# client's IP address proxy_set_header X-Real-IP $remote_addr; # X-Forwarded-For client request header field with the $remote_addr variable appended to it, diff --git a/labs/lab7/nginx-plus/etc/prometheus/prometheus.yml b/labs/lab7/nginx-plus/etc/prometheus/prometheus.yml index 3bc7f48..f0a0043 100644 --- a/labs/lab7/nginx-plus/etc/prometheus/prometheus.yml +++ b/labs/lab7/nginx-plus/etc/prometheus/prometheus.yml @@ -1,3 +1,7 @@ +# Prometheus config file +# Chris Akker, Shouvik Dutta, Adam Currier - Mar 2025 +# prometheus.yml +# global: scrape_interval: 15s @@ -10,4 +14,4 @@ scrape_configs: scrape_interval: 5s static_configs: - - targets: ['nginx-plus:9113'] # NGINX Plus container + - targets: ['nginx-plus:9113'] # NGINX Plus container diff --git a/labs/lab7/nginx-plus/usr/share/nginx-plus-module-prometheus/prometheus.js b/labs/lab7/nginx-plus/usr/share/nginx-plus-module-prometheus/prometheus.js new file mode 100644 index 0000000..305110a --- /dev/null +++ b/labs/lab7/nginx-plus/usr/share/nginx-plus-module-prometheus/prometheus.js @@ -0,0 +1,1730 @@ +var responseBuffer = []; +var contentLength = 0; +var NGINX_PLUS_CONTEXT = "nginxplus"; +var numSubrequests; +var rss = 0 +var private_mem = 0; + +var ProcessMetrics = { + "respawned": { + "help": "Total number of abnormally terminated and respawned child processes", + "type": "counter" + } +}; + +var WorkerConnectionsMetrics = [ + "accepted", + "dropped", + "active", + "idle" +]; + +var WorkerHttp = []; + +var WorkerHttpRequestsMetrics = [ + "total", + "current" +]; + +var SslServerZoneEventsMetrics = [ + "handshakes", + "session_reuses", + "handshakes_failed", + "no_common_protocol", + "no_common_cipher", + "handshake_timeout", + "peer_rejected_cert" +]; + +var SslPeerEventsMetrics = [ + "handshakes", + "session_reuses", + "handshakes_failed", + "no_common_protocol", + "handshake_timeout", + "peer_rejected_cert" +]; + +var SslServerZoneVerifyFailuresMetrics = [ + "expired_cert", + "revoked_cert", + "no_cert", + "other" +]; + +var SslPeerVerifyFailuresMetrics = [ + "expired_cert", + "revoked_cert", + "hostname_mismatch", + "other" +]; + +var ResponseMetrics = [ + "1xx", + "2xx", + "3xx", + "4xx", + "5xx" +]; + +var ResponseCodesMetrics = [ + "100", + "101", + "102", + "103", + "200", + "201", + "202", + "203", + "204", + "205", + "206", + "207", + "208", + "218", + "226", + "300", + "301", + "302", + "303", + "304", + "305", + "307", + "308", + "400", + "401", + "402", + "403", + "404", + "405", + "406", + "407", + "408", + "409", + "410", + "411", + "412", + "413", + "414", + "415", + "416", + "417", + "419", + "420", + "421", + "422", + "423", + "424", + "425", + "426", + "428", + "429", + "430", + "431", + "440", + "444", + "449", + "450", + "451", + "460", + "463", + "494", + "495", + "496", + "497", + "498", + "499", + "500", + "501", + "502", + "503", + "504", + "505", + "506", + "507", + "508", + "509", + "510", + "511", + "520", + "521", + "522", + "523", + "524", + "525", + "526", + "527", + "529", + "530", + "561", + "598", +]; + +var SessionMetrics = [ + "2xx", + "4xx", + "5xx" +]; + +var NGINXStatusMetrics = { + "config_generation": { + "help": "The total number of configuration reloads", + "type": "counter" + }, + "config_uptime": { + "help": "Time since the last reload of configuration in seconds", + "type": "counter" + } +}; + +var HealthCheckMetrics = { + "checks": { + "help": "Total health check requests", + "type": "counter" + }, + "fails": { + "help": "Failed health checks", + "type": "counter" + }, + "unhealthy": { + "help": "How many times the server became unhealthy (state 'unhealthy')", + "type": "counter" + } +}; + +var LimitConnsMetrics = { + "passed": { + "help": "The total number of connections that were neither limited nor accounted as limited", + "type": "counter" + }, + "rejected": { + "help": "The total number of connections that were rejected", + "type": "counter" + }, + "rejected_dry_run": { + "help": "The total number of connections accounted as rejected in the 'dry run' mode", + "type": "counter" + } +}; + +var LimitReqsMetrics = { + "passed": { + "help": "The total number of requests that were neither limited nor accounted as limited", + "type": "counter" + }, + "delayed": { + "help": "The total number of requests that were delayed", + "type": "counter" + }, + "rejected": { + "help": "The total number of requests that were rejected", + "type": "counter" + }, + "delayed_dry_run": { + "help": "The total number of requests accounted as delayed in the 'dry run' mode", + "type": "counter" + }, + "rejected_dry_run": { + "help": "The total number of requests accounted as rejected in the 'dry run' mode", + "type": "counter" + } +}; + +var UpstreamServerMetrics = { + + "active": { + "help": "Active client connections", + "type": "gauge" + }, + + "fails": { + "help": "Total number of unsuccessful attempts to communicate with the server", + "type": "counter" + }, + + "header_time": { + "help": "Average time to get the response header from the server", + "type": "gauge" + }, + + "health_checks": { + "handler": prepend_metric_handler, + "metrics": HealthCheckMetrics + }, + "received": { + "help": "Bytes received from clients", + "type": "counter" + }, + "requests": { + "help": "Total client requests", + "type": "counter" + }, + "response_time": { + "help": "Average time to get the full response from the server", + "type": "gauge" + }, + "responses": { + "help": "The total number of responses obtained from this server", + "type": "counter", + "handler": labeled_metric_handler, + "nested_handler": nested_metric_handler, + "metrics": ResponseMetrics, + "label": "code", + "codes": { + "help": "The number of responses per each status code", + "type": "counter", + "handler": labeled_metric_handler, + "metrics": ResponseCodesMetrics, + "label": "code" + } + }, + "sent": { + "help": "Bytes sent to clients", + "type": "counter" + }, + "state": { + "help": "Current state", + "type": "gauge", + "printFilter": convert_state + }, + "unavail": { + "help": "How many times the server became unavailable for client requests (state 'unavail') due to the number of unsuccessful attempts reaching the max_fails threshold", + "type": "counter" + }, + "ssl": { + "help": "SSL events", + "type": "counter", + "handler": labeled_metric_handler, + "nested_handler": nested_metric_handler, + "metrics": SslPeerEventsMetrics, + "label": "event", + "verify_failures": { + "help": "SSL certificate verification errors", + "type": "counter", + "handler": labeled_metric_handler, + "metrics": SslPeerVerifyFailuresMetrics, + "label": "verify_failure" + } + } +}; + +var UpstreamMetrics = { + + "keepalive": { + "help": "Idle keepalive connections", + "type": "gauge" + }, + "peers": { + "handler": upstream_server_handler, + "metrics": UpstreamServerMetrics, + "context": "upstream_server", + "specifierContext": "server" + }, + "zombies": { + "help": "Servers removed from the group but still processing active client requests", + "type": "gauge" + } +}; + +var ServerZoneMetrics = { + "discarded": { + "help": "Requests completed without sending a response", + "type": "counter" + }, + "processing": { + "help": "Client requests that are currently being processed", + "type": "gauge" + }, + "received": { + "help": "Bytes received from clients", + "type": "counter" + }, + "requests": { + "help": "Total client requests", + "type": "counter" + }, + "responses": { + "help": "Total responses sent to clients", + "type": "counter", + "handler": labeled_metric_handler, + "nested_handler": nested_metric_handler, + "metrics": ResponseMetrics, + "label": "code", + "codes": { + "help": "The number of responses per each status code", + "type": "counter", + "handler": labeled_metric_handler, + "metrics": ResponseCodesMetrics, + "label": "code" + } + }, + "sent": { + "help": "Bytes sent to clients", + "type": "counter" + }, + "ssl": { + "help": "SSL events", + "type": "counter", + "handler": labeled_metric_handler, + "nested_handler": nested_metric_handler, + "metrics": SslServerZoneEventsMetrics, + "label": "event", + "verify_failures": { + "help": "SSL certificate verification errors", + "type": "counter", + "handler": labeled_metric_handler, + "metrics": SslServerZoneVerifyFailuresMetrics, + "label": "verify_failure" + } + } +}; + +var LocationZoneMetrics = { + "discarded": { + "help": "Requests completed without sending a response", + "type": "counter" + }, + "received": { + "help": "Bytes received from clients", + "type": "counter" + }, + "requests": { + "help": "Total client requests", + "type": "counter" + }, + "responses": { + "help": "Total responses sent to clients", + "type": "counter", + "handler": labeled_metric_handler, + "nested_handler": nested_metric_handler, + "metrics": ResponseMetrics, + "label": "code", + "codes": { + "help": "The number of responses per each status code", + "type": "counter", + "handler": labeled_metric_handler, + "metrics": ResponseCodesMetrics, + "label": "code" + } + }, + "sent": { + "help": "Bytes sent to clients", + "type": "counter" + } +}; + +var ResolverRequestMetrics = [ + "name", + "srv", + "addr" +]; + +var ResolverResponseMetrics = [ + "noerror", + "formerr", + "servfail", + "nxdomain", + "notimp", + "refused", + "timedout", + "unknown" +]; + +var ResolverMetrics = { + "requests": { + "help": "Request types sent to resolver", + "type": "counter", + "handler": labeled_metric_handler, + "metrics": ResolverRequestMetrics, + "label": "type" + }, + "responses": { + "help": "Response types sent to clients", + "type": "counter", + "handler": labeled_metric_handler, + "metrics": ResolverResponseMetrics, + "label": "type" + } +}; + +var StreamUpstreamServerMetrics = { + "active": { + "help": "Active client connections", + "type": "gauge" + }, + "connect_time": { + "help": "Average time to connect to the upstream server", + "type": "gauge" + }, + "connections": { + "help": "Total number of client connections forwarded to this server", + "type": "counter" + }, + "fails": { + "help": "Total number of unsuccessful attempts to communicate with the server", + "type": "counter" + }, + "first_byte_time": { + "help": "Average time to receive the first byte of data", + "type": "gauge" + }, + "health_checks": { + "handler": prepend_metric_handler, + "metrics": HealthCheckMetrics + }, + "received": { + "help": "Bytes received from clients", + "type": "counter" + }, + "response_time": { + "help": "Average time to receive the last byte of data", + "type": "gauge" + }, + "sent": { + "help": "Bytes sent to clients", + "type": "counter" + }, + "state": { + "help": "Current state", + "type": "gauge", + "printFilter": convert_state + }, + "unavail": { + "help": "How many times the server became unavailable for client requests (state 'unavail') due to the number of unsuccessful attempts reaching the max_fails threshold", + "type": "counter" + }, + "ssl": { + "help": "SSL events", + "type": "counter", + "handler": labeled_metric_handler, + "nested_handler": nested_metric_handler, + "metrics": SslPeerEventsMetrics, + "label": "event", + "verify_failures": { + "help": "SSL certificate verification errors", + "type": "counter", + "handler": labeled_metric_handler, + "metrics": SslPeerVerifyFailuresMetrics, + "label": "verify_failure" + } + } +}; + +var StreamUpstreamMetrics = { + "zombies": { + "help": "Servers removed from the group but still processing active client requests", + "type": "gauge" + }, + "peers": { + "handler": upstream_server_handler, + "metrics": StreamUpstreamServerMetrics, + "context": "stream_upstream_server", + "specifierContext": "server" + } +}; + +var StreamServerZoneMetrics = { + "connections": { + "help": "Total connections", + "type": "counter" + }, + "discarded": { + "help": "Connections completed without creating a session", + "type": "counter" + }, + "processing": { + "help": "Client connections that are currently being processed", + "type": "gauge" + }, + "received": { + "help": "Bytes received from clients", + "type": "counter" + }, + "sent": { + "help": "Bytes sent to clients", + "type": "counter" + }, + "sessions": { + "help": "Total sessions completed", + "type": "counter", + "handler": labeled_metric_handler, + "metrics": SessionMetrics, + "label": "code" + }, + "ssl": { + "help": "SSL events", + "type": "counter", + "handler": labeled_metric_handler, + "nested_handler": nested_metric_handler, + "metrics": SslServerZoneEventsMetrics, + "label": "event", + "verify_failures": { + "help": "SSL certificate verification errors", + "type": "counter", + "handler": labeled_metric_handler, + "metrics": SslServerZoneVerifyFailuresMetrics, + "label": "verify_failure" + } + } +}; + +var WorkerMetrics = { + "pid" : { + "help": "Worker PID", + "type": "gauge" + }, + "connections": { + "help": "Client connections handled by worker", + "type": "gauge", + "handler": labeled_metric_handler, + "metrics": WorkerConnectionsMetrics, + "label": "connections" + }, + "http": { + "nested_handler": nested_metric_handler, + "handler": labeled_metric_handler, + "metrics" : WorkerHttp, + "label": "http", + "requests": { + "help": "HTTP requests handled by worker", + "type": "gauge", + "handler": labeled_metric_handler, + "metrics": WorkerHttpRequestsMetrics, + "label": "requests" + } + } +}; + +var ConnectionMetrics = { + "accepted": { + "help": "Accepted client connections", + "type": "counter" + }, + "active": { + "help": "Active client connections", + "type": "gauge" + }, + "dropped": { + "help": "Dropped client connections", + "type": "counter" + }, + "idle": { + "help": "Idle client connections", + "type": "gauge" + } +}; + +var HttpRequestMetrics = { + "total": { + "help": "Total http requests", + "type": "counter" + }, + "current": { + "help": "Current http requests", + "type": "gauge" + } +}; + +var HttpCacheHitMetrics = { + "responses": { + "help": "Total number of valid responses read from the cache", + "type": "counter" + }, + "bytes": { + "help": "Total number of bytes read from the cache", + "type": "counter" + } +}; + +var HttpCacheStaleMetrics = { + "responses": { + "help": "Total number of expired responses read from the cache", + "type": "counter" + }, + "bytes": { + "help": "Total number of bytes read from the cache", + "type": "counter" + } +}; + +var HttpCacheUpdatingMetrics = { + "responses": { + "help": "Total number of expired responses read from the cache while responses were being updated", + "type": "counter" + }, + "bytes": { + "help": "Total number of bytes read from the cache", + "type": "counter" + } +}; + +var HttpCacheRevalidatedMetrics = { + "responses": { + "help": "Total number of expired and revalidated responses read from the cache", + "type": "counter" + }, + "bytes": { + "help": "Total number of bytes read from the cache", + "type": "counter" + } +}; + +var HttpCacheMissMetrics = { + "responses": { + "help": "Total number of responses not found in the cache", + "type": "counter" + }, + "bytes": { + "help": "Total number of bytes read from the proxied server", + "type": "counter" + }, + "responses_written": { + "help": "Total number of responses written to the cache", + "type": "counter" + }, + "bytes_written": { + "help": "Total number of bytes written to the cache", + "type": "counter" + } +}; + +var HttpCacheExpiredMetrics = { + "responses": { + "help": "Total number of expired responses not taken from the cache", + "type": "counter" + }, + "bytes": { + "help": "Total number of bytes read from the proxied server", + "type": "counter" + }, + "responses_written": { + "help": "Total number of responses written to the cache", + "type": "counter" + }, + "bytes_written": { + "help": "Total number of bytes written to the cache", + "type": "counter" + } +}; + +var HttpCacheBypassMetrics = { + "responses": { + "help": "Total number of responses not looked up in the cache", + "type": "counter" + }, + "bytes": { + "help": "Total number of bytes read from the proxied server", + "type": "counter" + }, + "responses_written": { + "help": "Total number of responses written to the cache", + "type": "counter" + }, + "bytes_written": { + "help": "Total number of bytes written to the cache", + "type": "counter" + } +}; + +var HttpCacheMetrics = { + "size": { + "help": "Current size of the cache", + "type": "gauge" + }, + "max_size": { + "help": "Limit on the maximum size of the cache", + "type": "gauge" + }, + "cold": { + "help": "Indicates whether the “cache loader” process is still loading data from disk into the cache", + "type": "gauge", + "printFilter": convert_state + }, + "hit": { + "handler": prepend_metric_handler, + "metrics": HttpCacheHitMetrics + }, + "stale": { + "handler": prepend_metric_handler, + "metrics": HttpCacheStaleMetrics + }, + "updating": { + "handler": prepend_metric_handler, + "metrics": HttpCacheUpdatingMetrics + }, + "revalidated": { + "handler": prepend_metric_handler, + "metrics": HttpCacheRevalidatedMetrics + }, + "miss": { + "handler": prepend_metric_handler, + "metrics": HttpCacheMissMetrics + }, + "expired": { + "handler": prepend_metric_handler, + "metrics": HttpCacheExpiredMetrics + }, + "bypass": { + "handler": prepend_metric_handler, + "metrics": HttpCacheBypassMetrics + } +}; + +var SslMetrics = { + "handshakes": { + "help": "Successful SSL handshakes", + "type": "counter" + }, + "handshakes_failed": { + "help": "Failed SSL handshakes", + "type": "counter" + }, + "session_reuses": { + "help": "Session reuses during SSL handshake", + "type": "counter" + }, + "no_common_protocol": { + "help": "SSL handshakes failed because of no common protocol", + "type": "counter" + }, + "no_common_cipher": { + "help": "SSL handshakes failed because of no shared cipher", + "type": "counter" + }, + "handshake_timeout": { + "help": "SSL handshakes failed because of a timeout", + "type": "counter" + }, + "peer_rejected_cert": { + "help": "Failed SSL handshakes due to rejected certificate by peer", + "type": "counter" + }, + "verify_failures": { + "expired_cert": { + "help": "An expired or not yet valid certificate was presented by a client", + "type": "counter" + }, + "revoked_cert": { + "help": "A revoked certificate was presented by a client", + "type": "counter" + }, + "no_cert": { + "help": "A client did not provide the required certificate", + "type": "counter" + }, + "hostname_mismatch": { + "help": "Server's certificate doesn't match the hostname", + "type": "counter" + }, + "other": { + "help": "Other SSL certificate verification errors", + "type": "counter" + } + } +}; + +var PageMetrics = { + "used": { + "help": "Current number of used memory pages", + "type": "gauge" + }, + "free": { + "help": "Current number of free memory pages", + "type": "gauge" + }, +}; + +var SlotSpecifiers = [ + "8", + "16", + "32", + "64", + "128", + "256", + "512", + "1024", + "2048" +]; + +var SlotMetrics = { + "used": { + "help": "Current number of used memory slots", + "type": "gauge", + "metrics": SlotSpecifiers, + "label": "slot" + }, + "free": { + "help": "Current number of free memory slots", + "type": "gauge", + "metrics": SlotSpecifiers, + "label": "slot" + }, + "reqs": { + "help": "Total number of attempts to allocate memory of specified size", + "type": "gauge", + "metrics": SlotSpecifiers, + "label": "slot" + }, + "fails": { + "help": "Total number of unsuccessful attempts to allocate memory of specified size", + "type": "gauge", + "metrics": SlotSpecifiers, + "label": "slot" + } +}; + +var SlabMetrics = { + "pages": { + "metrics": PageMetrics + }, + "slots": { + "handler": slot_handler, + "specifiers": SlotSpecifiers, + "metrics": SlotMetrics + } +}; + +var ZoneSyncStatusMetrics = { + "nodes_online": { + "help": "Total number of peers this node is conected to", + "type": "counter" + }, + "msgs_in": { + "help": "Total messages received by this node", + "type": "counter" + }, + "msgs_out": { + "help": "Total messages sent by this node", + "type": "counter" + }, + "bytes_in": { + "help": "Bytes received by this node", + "type": "counter" + }, + "bytes_out": { + "help": "Bytes sent by this node", + "type": "counter" + } +}; + +var ZoneSyncZoneMetrics = { + "records_total": { + "help": "The total number of records stored in the shared memory zone", + "type": "counter", + }, + "records_pending": { + "help": "The number of records that need to be sent to the cluster", + "type": "counter", + } +}; + +var ZoneSyncMetrics = { + "status": { + "handler": change_base, + "next_handler": default_metric_callback, + "metrics": ZoneSyncStatusMetrics, + }, + "zones": { + "handler": change_base, + "next_handler": named_metric_callback, + "metrics": ZoneSyncZoneMetrics, + "specifierContext": "zone" + } +}; + +var WorkersMemMetrics ={ + "rss": { + "handler": system_rss_callback, + "help": "Resident set size (pss), physical memory NGINX workers are using, including shared libraries", + "type": "counter" + }, + "private": { + "handler": system_private_callback, + "help": "Private memory used by NGINX workers, does not include shared libraries", + "type": "counter" + }, +} + +var NGINXStatusContext = { + "handler": build_nginx_status, + "metrics": NGINXStatusMetrics, + "context": "nginx" +}; + +var ProcessesContext = { + "handler": default_metric_callback, + "metrics": ProcessMetrics, + "context": "processes" +}; + +var WorkersContext = { + "handler": named_metric_callback, + "metrics": WorkerMetrics, + "context": "workers", + "specifierContext": "id" +} + +var ConnectionContext = { + "handler": default_metric_callback, + "metrics": ConnectionMetrics, + "context": "connections" +}; + +var SslContext = { + "handler": default_metric_callback, + "metrics": SslMetrics, + "context": "ssl" +}; + +var SlabContext = { + "handler": prepend_metric_handler, + "metrics": SlabMetrics, + "context": "slab", + "specifierContext": "zone" +}; + +var HttpRequestContext = { + "handler": default_metric_callback, + "metrics": HttpRequestMetrics, + "context": "http_requests" +}; + +var HttpCacheContext = { + "handler": named_metric_callback, + "metrics": HttpCacheMetrics, + "context": "cache", + "specifierContext": "zone" +} + +var UpstreamContext = { + "handler": named_metric_callback, + "metrics": UpstreamMetrics, + "context": "upstream" +}; + +var ServerZoneContext = { + "handler": named_metric_callback, + "metrics": ServerZoneMetrics, + "context": "server_zone" +}; + +var LocationZoneContext = { + "handler": named_metric_callback, + "metrics": LocationZoneMetrics, + "context": "location_zone" +}; + +var ResolverContext = { + "handler": named_metric_callback, + "metrics": ResolverMetrics, + "context": "resolver" +}; + +var StreamUpstreamContext = { + "handler": named_metric_callback, + "metrics": StreamUpstreamMetrics, + "context": "stream_upstream", + "specifierContext": "upstream" +}; + +var StreamServerZoneContext = { + "handler": named_metric_callback, + "metrics": StreamServerZoneMetrics, + "context": "stream_server_zone", + "specifierContext": "server_zone" +}; + +var ZoneSyncContext = { + "metrics": ZoneSyncMetrics, + "context": "stream_zone_sync" +}; + +var WorkersMemContext = { + "metrics": WorkersMemMetrics, + "context": "workers_mem" +}; + +var LimitConnsContext = { + "handler": named_metric_callback, + "metrics": LimitConnsMetrics, + "context": "limit_conns", + "specifierContext": "zone" +}; + +var LimitReqsContext = { + "handler": named_metric_callback, + "metrics": LimitReqsMetrics, + "context": "limit_reqs", + "specifierContext": "zone" +}; + +var Metrics = [ + [ + "/api/9/nginx", + NGINXStatusContext + ], + [ + "/api/9/processes", + ProcessesContext + ], + [ + "/api/9/connections", + ConnectionContext + ], + [ + "/api/9/ssl", + SslContext + ], + [ + "/api/9/workers", + WorkersContext + ], + [ + "/api/9/slabs", + SlabContext + ], + [ + "/api/9/resolvers", + ResolverContext + ], + [ + "/api/9/http/requests", + HttpRequestContext + ], + [ + "/api/9/http/caches", + HttpCacheContext + ], + [ + "/api/9/http/limit_conns", + LimitConnsContext + ], + [ + "/api/9/http/upstreams", + UpstreamContext, + replace_with_keyval + ], + [ + "/api/9/http/server_zones", + ServerZoneContext + ], + [ + "/api/9/http/location_zones", + LocationZoneContext + ], + [ + "/api/9/stream/limit_conns", + LimitConnsContext + ], + [ + "/api/9/http/limit_reqs", + LimitReqsContext + ], + [ + "/api/9/stream/upstreams", + StreamUpstreamContext, + replace_with_keyval + ], + [ + "/api/9/stream/server_zones", + StreamServerZoneContext + ], + [ + "/api/9/stream/zone_sync", + ZoneSyncContext + ], + [ + "workers/mem", + WorkersMemContext, + workers_mem_collector + ] +]; + +function metrics(r) { + numSubrequests = Metrics.length; + + disable_metrics(r); + Metrics.forEach(function(m) { + ngx_collect_metrics.apply(null, [r].concat(m)); + }); +} + +function disable_metrics(r) { + if (r.variables.prom_metrics_disabled == undefined || + r.variables.prom_metrics_disabled == "") { + return; + } + var disabledMetrics = r.variables.prom_metrics_disabled.split(/, */g); + + for (var i = 0; i < disabledMetrics.length; i++) { + var metric = disabledMetrics[i]; + for (var j = 0; j < Metrics.length; j++) { + var urlSplit; + if(Metrics[j][0].startsWith("/api")) { + urlSplit = Metrics[j][0].slice(7); + } else { + urlSplit = Metrics[j][0] + } + if (urlSplit == metric) { + Metrics[j][1].disabled = true; + numSubrequests--; + break; + } + if (j == Metrics.length - 1) { + r.error("specified metric to disable does not exist: " + metric); + } + } + } +} + +// ngx_collect_metrics makes a subrequest to a designated url and then passes the +// resulting object to a handler to build the response for the metrics passed in +function ngx_collect_metrics(r, url, objContext, filterHandler) { + if (!objContext.disabled) { + if (url.startsWith("/api")) { + r.subrequest(url, { method: 'GET' }, function (res) { + if (res.status >= 300 || res.responseText == "{}") { + if (--numSubrequests == 0) { + ngx_send_response(r); + } + return; + } + contentLength += res.responseText.length; + + var obj = JSON.parse(res.responseText); + var resp = null; + if (filterHandler) { + resp = filterHandler(r, obj, objContext, res.uri); + } + if (resp == null) { + var buf = process_obj(obj, objContext); + responseBuffer.push(buf); + if (--numSubrequests == 0) { + ngx_send_response(r); + return; + } + } + }); + } else { + var buf; + if (filterHandler) { + filterHandler(r, "", objContext, "") ; + } + buf = process_obj("", objContext); + responseBuffer.push(buf); + if (--numSubrequests == 0) { + ngx_send_response(r); + return; + } + } + } + if (numSubrequests == 0) { + ngx_send_response(r); + } +} + +// replace_with_keyval makes a subrequest to the keyval specified by the variable +// "prom_keyval" and replaces all object names if they are in the keyval map +// keyval map should take the form replacement : object-to-replace +function replace_with_keyval(r, obj, objContext, uri) { + var zone = uri.split("/")[3]; + var keyValUpstream = NaN; + if (zone == "http") + keyValUpstream = r.variables.prom_keyval; + else if (zone == "stream") + keyValUpstream = r.variables.prom_keyval_stream; + if (keyValUpstream) { + r.subrequest("/api/9/" + zone + "/keyvals/" + keyValUpstream, { method: 'GET' }, + function (res) { + if (res.status < 300 && res.responseText != "{}") { + var keyVal = JSON.parse(res.responseText); + var keyValEnum = Object.entries(keyVal); + for (var i = 0; i < keyValEnum.length; i++) { + var upstreamMap = keyValEnum[i]; + if (obj[upstreamMap[1]] != null) { + obj[upstreamMap[0]] = obj[upstreamMap[1]]; + delete obj[upstreamMap[1]]; + } + } + } + var buf = process_obj(obj, objContext); + responseBuffer.push(buf); + if (--numSubrequests == 0) { + ngx_send_response(r); + return; + } + }); + return NaN; + } else { + return null; + } +} + +function convert_to_number(r, s) { + var number = parseInt(s.split(" ")[0]); + var unit = s.split(" ")[1]; + var multiplier = 1; + + switch (unit.toLowerCase()) { + case "kb": + multiplier = 1024; + break; + case "mb": + multiplier = 1024*1024; + break; + case "gb": + multiplier = 1024*1024*1024; + break; + default: + multiplier = 1 + + } + + return number * multiplier; +} + +function workers_mem_collector(r, obj, objContext, uri) { + var fs = require('fs'); + var fileName; + var procList; + var procName; + var pid; + var match; + var vmRSS; + var rssAnon; + + try { + fileName = "/proc/" + process.ppid + "/task/" + process.ppid + "/children"; + procList = fs.readFileSync(fileName, 'utf8').trim().split(" "); + + for (pid in procList) { + // Skip the cache manager process + fileName = "/proc/" + procList[pid] + "/cmdline" + procName = fs.readFileSync(fileName, "utf8").toString(); + if (procName.match("nginx: worker process") == null) { + continue; + } + + fileName = "/proc/" + procList[pid] + "/status" + var procStatus = fs.readFileSync(fileName, 'utf8'); + match = procStatus.match(/VmRSS:\s*(.*)\n/) + if (match != null) { + vmRSS = convert_to_number(r, match[1]); + } + match = procStatus.match(/RssAnon:\s*(.*)\n/) + if (match != null) { + rssAnon = convert_to_number(r, match[1]); + } + + private_mem += rssAnon; + if (rss == 0) { + // Only count the shared libraries once + rss += vmRSS + } else { + rss += rssAnon + } + } + } + catch(err) { + rss = 0; + private_mem = 0; + r.log("Unable to read " + fileName + ": " + err.message); + } +} + +// process_obj is the callback from the ngx_collect_metrics function and is +// responsible for printing the metric help text as well as calling the handler +// associated with a certain metric +function process_obj(obj, objContext) { + var buffer = []; + var metricsEnum = Object.entries(objContext.metrics); + var metricContext = objContext.context; + var handler = objContext.handler; + + for (var i = 0; i < metricsEnum.length; i++) { + var metric = metricsEnum[i]; + // Print metric information + if (metric[1].help != null) { + buffer.push(print_metric_help_text(metricContext, metric)); + } + + var specifierContext = objContext.context; + if (objContext.specifierContext != null) { + specifierContext = objContext.specifierContext; + } + // Call if metric has associated handler + if (metric[1].handler != null) { + buffer.push(metric[1].handler(obj, metric, metricContext, + specifierContext)); + // Call if metric has any nested metric handler + if (metric[1].nested_handler != null) { + buffer.push(metric[1].nested_handler(obj, objContext, metric)); + } + } else { + // Else call default handler + buffer.push(handler(obj, metric, metricContext, specifierContext)); + } + } + + return buffer.join(""); +} + +// build_nginx_status builds the status metrics for NGINX, using version, +// build, address, ppid metrics in a specifier. +function build_nginx_status(obj, metric, context) { + var objString = JSON.stringify(obj).replace(/":([^",}]+)/g, "\":\"$1\""). + replace(/"generation":"\d+",/, ""). + replace(/"load_timestamp":".*",/, ""). + replace(/"timestamp":".*",/, ""). + replace(/"pid":"\d+",/, ""); + + switch (metric[0]) { + case "config_generation": + return build_metric(context, + metric[0], + obj["generation"], + JSON.parse(objString)); + case "config_uptime": + return build_metric(context, + metric[0], + parseInt(((new Date(obj["timestamp"])).getTime() - + (new Date(obj["load_timestamp"])).getTime())/1000), + JSON.parse(objString)); + default: + return build_metric(context, + metric[0], + obj[metric[0]], + JSON.parse(objString)); + } +} + +// default_metric_callback assumes that there is only a single object of metrics +// to be printed. It returns metrics without a specifier attached to them +function default_metric_callback(obj, metric, context) { + if (typeof(obj[metric[0]]) === 'object' && !Array.isArray(obj[metric[0]])) { + return process_obj(obj[metric[0]], { + "metrics": metric[1], + "context": context + "_" + metric[0], + "handler": default_metric_callback + }); + } else { + return build_metric(context, metric[0], obj[metric[0]], null); + } +} + +// named_metric_callback assumes that there are multiple elements to be returned, +// each formatted as a name : [object] pair in which the name will be used +// in conjunction with the specifierContext to build a specifier +function named_metric_callback(obj, metric, context, specifierContext) { + var buffer = []; + var arrayEnum = Object.entries(obj); + + for (var j = 0; j < arrayEnum.length; j++) { + var arrayKey = arrayEnum[j][0]; + var arrayObj = arrayEnum[j][1]; + var metric_val = arrayObj[metric[0]]; + var specifierObj = {}; + if (metric[1].printFilter != null) { + metric_val = metric[1].printFilter(arrayObj[metric[0]]); + } + // On iterations after initial, specifier object may be set, representing + // past base contexts to be printed + if (arrayObj.specifierObj != null) { + specifierObj = arrayObj.specifierObj; + } + + var set = false; + if (specifierObj[specifierContext] == null) { + specifierObj[specifierContext] = arrayKey; + set = true; + } + buffer.push(build_metric(context, metric[0], metric_val, specifierObj)); + if (set) { + delete specifierObj[specifierContext]; + } + } + + return buffer.join(""); +} + +// upstream_server_handler is responsible for building the name : object pair +// that named_metric_callback expects, except with peer servers instead of +// upstreams +function upstream_server_handler(obj, metric, context, specifierContext) { + var nextObj = {}; + var upstreamsEnum = Object.entries(obj); + var metric_name = metric[0]; + var metric_obj = metric[1]; + + for (var upstreamIdx = 0; upstreamIdx < upstreamsEnum.length; upstreamIdx++) { + var upstream = upstreamsEnum[upstreamIdx][1]; + + for (var peerIdx = 0; peerIdx < upstream[metric_name].length; peerIdx++) { + var peer = upstream[metric_name][peerIdx]; + var specifierObj = {}; + var placeholderKey = peer.server + "_" + upstreamIdx; + + // Add specifier for upstream to object to attach to peer + specifierObj[specifierContext] = upstreamsEnum[upstreamIdx][0]; + specifierObj[metric_obj.specifierContext] = peer.server; + + nextObj[placeholderKey] = peer; + nextObj[placeholderKey].specifierObj = specifierObj; + } + } + + metric_obj.handler = named_metric_callback; + + return process_obj(nextObj, metric_obj); +} + +// labeled_metric_handler allows for object metrics to be printed, in which each +// member in the object is added to the metric as a specifier represented by +// specifierContext +function labeled_metric_handler(obj, metric, context, specifierContext) { + var buffer = []; + var objEnum = Object.entries(obj); + + for (var i = 0; i < objEnum.length; i++) { + var iterKey = objEnum[i][0]; + var iterObj = objEnum[i][1]; + var labelObj = iterObj[metric[0]]; + var specifierObj = {} + if (iterObj.specifierObj != null) { + specifierObj = iterObj.specifierObj; + } + var set = false; + if (specifierObj[specifierContext] == null) { + specifierObj[specifierContext] = iterKey; + set = true; + } + if (labelObj != null) { + buffer.push(build_labeled_metrics(labelObj, metric, context, specifierObj)); + } + if (set) { + delete specifierObj[specifierContext]; + } + } + + return buffer.join(""); +} + +// nested_metric_handler allows to exclusively process nested metric +// objects by calling process_obj, while the object and its context are changed +// in such a way that the nested metric is moved one level up. +function nested_metric_handler(obj, objContext, metric) { + var parentMetricKey = metric[0]; + var nestedContext = Object.assign({}, objContext); + + var metricEnum = Object.entries(metric[1]); + for (var i=0; i < metricEnum.length; i++) { + var iterKey = metricEnum[i][0]; + var iterObj = metricEnum[i][1]; + + if (typeof (iterObj) === 'object' && !Array.isArray(iterObj)) { + var nestedMetricKey = parentMetricKey + "_" + iterKey; + nestedContext.metrics = {}; + nestedContext.metrics[nestedMetricKey] = objContext.metrics + [parentMetricKey][iterKey]; + + var objEnum = Object.entries(obj); + for (var j = 0; j < objEnum.length; j++) { + var iterObjKey = objEnum[j][0]; + if (obj[iterObjKey][parentMetricKey] != null) { + obj[iterObjKey][nestedMetricKey] = obj[iterObjKey] + [parentMetricKey][iterKey]; + } + } + } + } + + return process_obj(obj, nestedContext); +} + +// build_labeled_metrics builds the metrics and adds the "code" to the specifier +function build_labeled_metrics(labelObj, metric, context, specifierObj) { + var buffer = []; + var labelMetrics = metric[1].metrics; + + for (var i = 0; i < labelMetrics.length; i++) { + var labelMetric = labelMetrics[i]; + specifierObj[metric[1].label] = labelMetric; + buffer.push(build_metric(context, metric[0], labelObj[labelMetric], + specifierObj)); + } + + delete specifierObj[metric[1].label]; + + return buffer.join(""); +} + +// prepend_metric_handler allows for object metrics to be printed, in which each +// member is appended to the base object name and which corresponds to an object +// in the subrequest response +function prepend_metric_handler(obj, metric, context, specifierContext) { + var buffer = []; + var objEnum = Object.entries(obj); + var prependMetrics = metric[1].metrics; + var prependMetricsEnum = Object.entries(prependMetrics); + + for (var i = 0; i < prependMetricsEnum.length; i++) { + var prependMetric = prependMetricsEnum[i]; + var printableMetric = {}; + printableMetric[metric[0] + "_" + prependMetric[0]] = prependMetric[1]; + if (prependMetric[1].help != null) { + buffer.push(print_metric_help_text(context, + Object.entries(printableMetric)[0])); + } + + for (var j = 0; j < objEnum.length; j++) { + var iterKey = objEnum[j][0]; + var iterObj = objEnum[j][1]; + if (specifierContext != null) { + var specifierObj = {}; + if (iterObj.specifierObj != null) { + specifierObj = iterObj.specifierObj; + } + var set = false; + if (specifierContext != null && specifierObj[specifierContext] == null) { + specifierObj[specifierContext] = iterKey; + set = true; + } + } + var prependMetricValue = iterObj[metric[0]][prependMetric[0]]; + buffer.push(build_metric(context, metric[0] + "_" + prependMetric[0], + prependMetricValue, specifierObj)); + if (set) { + delete specifierObj[specifierContext]; + } + } + } + + return buffer.join(""); +} + +// change_base modifies the passed obj to go down one node based on the metric +// name. +// zones: { becomes { +// "records_total", ----> "records_total", +// "records_pending" "records_pending" +// } } +// then calls process_obj again with handler replaced with next_handler +function change_base(obj, metric, context, specifierContext) { + var newObj = obj[metric[0]]; + var metric_obj = metric[1]; + + if (metric_obj.next_handler == null) { + return null; + } + + metric_obj.handler = metric_obj.next_handler; + metric_obj.context = context + "_" + metric[0]; + + return process_obj(newObj, metric_obj); +} + +// convert_state converts the metric "state" value to a prometheus-compliant value +function convert_state(metric) { + switch (metric) { + case "up": + return 1; + case "draining": + return 2; + case "down": + return 3; + case "unavail": + return 4; + case "checking": + return 5; + case "unhealthy": + return 6; + case true: + return 1; + default: + return 0; + } +} + +// slot_handler swaps the slot and metric information so that it can call +// labeled_metric_handler +function slot_handler(obj, metric, context, specifierContext) { + var objEnum = Object.entries(obj); + var metricsEnum = Object.entries(metric[1].metrics); + var specifiers = metric[1].specifiers; + + for (var i = 0; i < objEnum.length; i++) { + var iterKey = objEnum[i][0]; + var iterValue = objEnum[i][1]; + obj[iterKey] = iterValue["slots"]; + for (var j = 0; j < metricsEnum.length; j++) { + var iterMetric = metricsEnum[j][0]; + obj[iterKey][iterMetric] = {} + for (var k = 0; k < specifiers.length; k++) { + obj[iterKey][iterMetric][specifiers[k]] = + obj[iterKey][specifiers[k]][iterMetric]; + } + } + for (var j = 0; j < specifiers.length; j++) { + delete obj[iterKey][specifiers[j]]; + } + } + + metric[1].handler = labeled_metric_handler; + metric[1].context = context; + metric[1].specifierContext = specifierContext; + + return process_obj(obj, metric[1]); +} + +// system_rss_callback returns prometheus formatted NGINX resident +// set size (rss) +function system_rss_callback(obj, metric, context, specifierContext) { + return build_metric(context, metric[0], rss, null); +} + +// system_private_callback returns prometheus formatted NGINX private +// clean + dirty pages +function system_private_callback(obj, metric, context, specifierContext) { + return build_metric(context, metric[0], private_mem, null); +} + +// build_metric returns the prometheus-formatted metric for a particular NGINX +// metric +function build_metric(context, metric, metric_val, specifierObj) { + var specifier = ""; + + if (specifierObj != null) { + specifier = build_specifier(specifierObj); + } + if (metric_val == undefined) { + metric_val = 0; + } + + return NGINX_PLUS_CONTEXT + "_" + context + "_" + metric + " " + + specifier + metric_val + "\n"; +} + +// build_specifier returns the JSON-like prometheus specifier associated with a +// metric +function build_specifier(specifierObj) { + return JSON.stringify(specifierObj).replace(/\"([^(\")"]+)\":/g, "$1=") + " "; +} + +// print_metric_help_text returns the "HELP" and "TYPE" text associated with a +// metric +function print_metric_help_text(context, metric) { + var metric_context = NGINX_PLUS_CONTEXT + "_" + context + "_" + metric[0]; + var metric_type = metric[1].type == undefined ? "untyped" : metric[1].type; + + return "# HELP " + metric_context + " " + metric[1].help + "\n" + + "# TYPE " + metric_context + " " + metric_type + "\n"; +} + +// ngx_send_response returns the response through the NGINX response object with +// the built response buffer +function ngx_send_response(r) { + r.headersOut['Content-Type'] = "text/plain"; + r.return(200, responseBuffer.join("")); +} + +export default { metrics }; \ No newline at end of file diff --git a/labs/lab7/readme.md b/labs/lab7/readme.md index c4c2c44..d097a91 100644 --- a/labs/lab7/readme.md +++ b/labs/lab7/readme.md @@ -4,7 +4,7 @@ In this lab, you will be exploring the integration between NGINX Plus, Prometheus and Grafana. -This Solution requires the use of the NGINX provided Javascript and Prometheus modules to collect metrics from the NGINX Plus API, and export those metrics as an HTTP html/text page, commonly called the `scaper page` because it scrapes statistics for publication. The metrics on this export page are then read and imported into Prometheus and Grafana's time-series database. Once these metrics are in the database, you can create many different Dashboards, Thresholds, Alerts, and other types of graphs for Visualization and Reporting. As you can imagine, there are literally hundreds of Grafana dashboards written by users of NGINX that you can try out for free. Grafana also allows you to create and edit your own Dashboards. +This Solution requires the use of the NGINX provided Javascript and Prometheus modules to collect metrics from the NGINX Plus API, and export those metrics as an HTTP html/text page, commonly called the `scaper page` because it scrapes statistics for publication. The metrics on this export page are then read and imported into a Prometheus time-series database. Once these metrics are in the database, you can use Grafana to create many different Dashboards, set Thresholds, Alerts, and other types of graphs for Visualization and Reporting. As you can imagine, there are literally hundreds of Grafana dashboards written by users of NGINX that you can try out for free. Grafana also allows you to create and edit your own Dashboards. You will enable Prometheus on Nginx, and use Docker containers to collect and display Nginx metrics. NGINX Plus | Prometheus | Grafana :-------------------------:|:-------------------------:|:-----: @@ -14,15 +14,17 @@ NGINX Plus | Prometheus | Grafana By the end of the lab you will be able to: -- Enable and configure NGINX Java Script +- Enable and configure NGINX Javascript (NJS) - Create Prometheus Exporter configuration - Test the Prometheus Server - Test the Grafana Server -- View Grafana Dashboard +- Import and View a simple NGINX Grafana Dashboard -## Pre-Requisites +## Prerequisites -- Nginx-Plus container from Lab1 +- Nginx-Plus container from Lab5 +- Nginx ONE Dataplane Key +- NginxPlus License JWT file - You must have Docker installed and running - You must have Docker-compose installed - See `Lab0` for instructions on setting up your system for this Workshop @@ -32,17 +34,62 @@ By the end of the lab you will be able to: - Familiarity with Prometheus - Familiartiy with Grafana -As part of your Dockerfile, your NGINX Plus container already has the added `NGINX Java Script and NGINX Prometheus dynamic module` installed during the build process. Refer to the /lab1/nginx-plus/Dockerfile if you want to check it out. +
+ +As part of your Docker image, your NGINX Plus container already has the required `NGINX Java Script and NGINX Prometheus dynamic modules` needed for Prometheus installed for you. If you are curious, Docker Exec into your Nginx Plus container, and look in the `/etc/nginx/modules` folder for the modules available. + +*NOTE: Make sure you `docker compose down` any running containers from previous labs before starting this lab.* + +1. Make sure your three environment variables are still set in your Visual Studio terminal. If they are not set, then refer to Lab2 to set them up before proceeding to next step. + + ```bash + # Check if below environment variables are present in your terminal + + echo "----------------------------------------------" + echo $TOKEN + echo "----------------------------------------------" + echo $JWT + echo "----------------------------------------------" + echo $NAME + echo "----------------------------------------------" + ``` + +1. Confirm you are still logged in to the NGINX Private Registry, using the `$JWT` environment variable for the username, as follows. (Your system may require sudo) + + ```bash + docker login private-registry.nginx.com --username=$JWT --password=none -1. Ensure you are in the `lab7` folder. Using a Terminal, run Docker Compose to build and run all the containers. + ``` + It should respond with `Login Succeeded`. + +1. Ensure you are in the `lab7` folder. Using the VSCode Terminal, run Docker Compose to download/run all the containers. ```bash - cd lab6 + cd lab7 docker compose up --force-recreate -d ``` -1. Edit your `nginx.conf` file, you will make 2 changes. +1. Verify all SIX of your Lab7 containers have started (ignore the Guacamole container). + + ```bash + docker ps + + ``` + + ```bash + ## Sample output ## + CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES + 8664bad14c05 private-registry.nginx.com/nginx-plus/agent:nginx-plus-r32-debian "/docker-entrypoint.…" 4 hours ago Up 4 hours 0.0.0.0:80->80/tcp, :::80->80/tcp, 0.0.0.0:443->443/tcp, :::443->443/tcp, 0.0.0.0:9000->9000/tcp, :::9000->9000/tcp, 0.0.0.0:9113->9113/tcp, :::9113->9113/tcp $NAME-nginx-plus + 302cdf1a17dc nginxinc/ingress-demo "/docker-entrypoint.…" 4 hours ago Up 4 hours 443/tcp, 0.0.0.0:32779->80/tcp, :::32779->80/tcp, 0.0.0.0:32777->433/tcp, :::32777->433/tcp $NAME-web2 + a39caa99adf0 nginxinc/ingress-demo "/docker-entrypoint.…" 4 hours ago Up 4 hours 0.0.0.0:32775->80/tcp, :::32775->80/tcp, 0.0.0.0:32774->443/tcp, :::32774->443/tcp $NAME-web3 + 173bab90ecfa nginxinc/ingress-demo "/docker-entrypoint.…" 4 hours ago Up 4 hours 0.0.0.0:32778->80/tcp, :::32778->80/tcp, 0.0.0.0:32776->443/tcp, :::32776->443/tcp $NAME-web1 + cc9ba360acff prom/prometheus "/bin/prometheus --c…" 4 hours ago Up 4 hours 0.0.0.0:9090->9090/tcp, :::9090->9090/tcp prometheus + 7bb2da3684cd grafana/grafana "/run.sh" 4 hours ago Up 4 hours 0.0.0.0:3000->3000/tcp, :::3000->3000/tcp grafana + + ``` + +1. Using the One Console, find your `<$NAME>-nginx-plus` under Manage > Instance. Click on the name, Configuration, then Edit your `nginx.conf` file, you will make 2 changes. - Uncomment Line #8 to enable the `ngx_http_js_module` module. - Uncomment Line #37 to set a parameter for an NGINX buffer called `subrequest_output_buffer_size`. @@ -51,7 +98,7 @@ As part of your Dockerfile, your NGINX Plus container already has the added `NGI ...snip user nginx; - worker_processes auto; + worker_processes 1; error_log /var/log/nginx/error.log info; pid /var/run/nginx.pid; @@ -61,20 +108,22 @@ As part of your Dockerfile, your NGINX Plus container already has the added `NGI ...snip - # Uncomment for Prometheus scraper page output + # Uncomment for Prometheus scraper page output buffer subrequest_output_buffer_size 32k; # Added for Prometheus ...snip ``` -1. Inspect the `prometheus.conf` file in the `labs/lab6/nginx-plus/etc/nginx/conf.d` folder. This is the NGINX config file which opens up port 9113, and provides access to the scraper page. Uncomment all the lines to enable this. + ![Edit Nginx.conf](media/lab7_edit-nginx-conf.png) + +1. Also Inspect, then Edit the `/etc/nginx/conf.d/prometheus.conf` file. This is the NGINX config file which opens up port 9113, and provides access to the scraper page. Uncomment all the lines to enable this. ```nginx # NGINX Plus Prometheus configuration, for HTTP scraper page - # Chris Akker, Shouvik Dutta - Feb 2024 + # Chris Akker, Shouvik Dutta, Adam Currier - Mar 2025 # https://www.nginx.com/blog/how-to-visualize-nginx-plus-with-prometheus-and-grafana/ - # Nginx Basics + # Nginx One # # Uncomment all lines below js_import /usr/share/nginx-plus-module-prometheus/prometheus.js; @@ -95,28 +144,25 @@ As part of your Dockerfile, your NGINX Plus container already has the added `NGI ``` -1. Once the contents of both files has been updated and saved, Docker Exec into the nginx-plus container. - - ```bash - docker exec -it nginx-plus bin/bash - - ``` + ![Edit prometheus.conf](media/lab7_edit-prometheus-conf.png) -1. Test and reload your NGINX config by running `nginx -t` and `nginx -s reload` commands respectively from within the container. +1. After your edits are completed, click Next and Publish your changes to enable Prometheus NJS and the scraper page on port 9113. -1. Start the WRK load generation tool. This will provide some traffic to the nginx-plus container, so the statistics will be increasing. +1. Start the WRK load generation tool. This will provide some traffic to the $NAME-nginx-plus container, so the statistics will be increasing. You may have to change $NAME to your name, like `s.jobs-nginx-plus'. ```bash - docker run --name wrk --network=lab6_default --rm williamyeh/wrk -t4 -c200 -d20m -H 'Host: cafe.example.com' --timeout 2s http://nginx-plus/coffee + docker run --name wrk --network=lab7_default --rm elswork/wrk -t4 -c200 -d20m -H 'Host: cafe.example.com' --timeout 2s http://s.jobs-nginx-plus/coffee ``` -1. Test the Prometheus scraper page. Open your browser to . You should see an html/text page like this one. You will notice there are MANY statistcs available, this page is like a text version of the NGINX Plus dashboard. This page can be easily imported into your existing Performance Management and Monitoring tools. You will see how to do this in the next section with Prometheus and Grafana. +1. Test the Prometheus scraper page. Open your browser to . You should see an html/text page like this one. You will notice there are MANY statistcs available, this page is like a text version of the NGINX Plus dashboard. This page can be easily imported into many different Performance Management and Monitoring tools. You will see how to do this in the next section with Prometheus and Grafana. Click refresh a couple times, and some of the metrics should increment. ![Scraper page](media/lab7_scraper_page1.png) +1. Check your Nginx Plus Dashboard at , you should also see HTTP Zone and Upstream counters incrementing during the loadtest. +
## Prometheus and Grafana Server Docker containers @@ -126,6 +172,8 @@ As part of your Dockerfile, your NGINX Plus container already has the added `NGI ![prometheus](media/prometheus-icon.png) |![grafana](media/grafana-icon.png) --- | --- +In this section, you will use and explore the two Docker containers for the Prometheus and Grafana servers. These servers will collect and display various metrics from the Nginx Plus container. You will also run a load test to create some traffic and watch the metrics Dashboards in real-time. + 1. Inspect your `docker-compose.yml` file, you will see it includes 2 additional Docker containers for this lab, one for a Prometheus server, and one for a Grafana server. These have been configured to run for you, but the images will be pulled from public repos. ```bash @@ -138,7 +186,7 @@ As part of your Dockerfile, your NGINX Plus container already has the added `NGI volumes: - ./nginx-plus/etc/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml ports: - - "9090:9090" + - "9090:9090" # Prometheus Server Web Console restart: always depends_on: - nginx-plus @@ -149,7 +197,8 @@ As part of your Dockerfile, your NGINX Plus container already has the added `NGI - grafana-storage:/var/lib/grafana image: grafana/grafana ports: - - "3000:3000" + - "3000:3000" + - "3000:3000" # Grafana Server Web Console restart: always depends_on: - nginx-plus @@ -177,21 +226,37 @@ As part of your Dockerfile, your NGINX Plus container already has the added `NGI ``` +If both are running, you can continue with the next section. +
### Prometheus
-Prometheus is a software package that can watch and collect statistics from many different NGINX instances. The Prometheus server will collect the statistics from the scraper page that you enabled in the previous section. +![Prom](media/prometheus-icon.png) + +
+ +**Prometheus** is a very popular software package that can collect statistics from many different NGINX instances. The Prometheus server will collect the NGINX Plus statistics from the scraper page that you enabled in the previous section.
-1. Using Chrome, navigate to . You should see a Prometheus webpage like this one. Search for `nginxplus_` in the query box to see a list of all the Nginx Plus statistics that Prometheus is collecting for you. Select `nginxplus_http_requests_total` from the list, click on Graph, and then click the "Execute" Button. Change the Time window if needed. This will provide a graph similar to this one: +1. Using Chrome, navigate to . You should see a Prometheus webpage like this one. In the dark grey banner, click on `Status`, then `Target Health`. You should see that Prometheus is configured to scape statistics from your `nginx-plus` Endpoint on port 9113 (as you configured Nginx earlier). The State should be `Green`. Last Scrape shows how long since the last pull of data. + + ![Prom Target](media/lab7_prometheus-targets.png) + +1. On the Grey Banner, click on `Query`. Type `nginxplus_` in the query box to see a list of all the Nginx Plus statistics that Prometheus is collecting for you. Select `nginxplus_http_requests_total` from the list, click on Graph, and then click the "Execute" Button. Change the Time window if needed. This will provide a graph similar to this one: ![Prom Graph](media/lab7_prometheus-graph.png) - Take a few minutes to explore other metrics available from NGINX Plus. What are the Upstream Response Times of your 3 backend web servers??? + *Take a few minutes to explore other metrics available from NGINX Plus. Can you find the `HTTP Upstream Response Times` of your 3 backend web servers???* + + **HINT** + + ![Prom Graph](media/lab7_prometheus-upstream-response-time.png) + + >*Pop Quiz: How much memory did Nginx use during your WRK loadtest?*
@@ -199,33 +264,39 @@ Prometheus is a software package that can watch and collect statistics from many
-Grafana is a data visualization tool, which contains a time series database and graphical web presentation tools. Grafana imports the Prometheus scraper page statistics into it's database, and allows you to create Dashboards of the statistics that are important to you. +![grafana](media/grafana-icon.png) + +
-1. Log into the Web console access for Grafana at . The default Login should be user/pass of `admin/admin`. This will present the main Grafana page. +Grafana is a popular data visualization tool, which can link to a time-series database and graphical web presentation tools. Grafana uses the Prometheus database as a `DataSource`, and allows you to create Dashboards of the Nginx statistics that are important to you. -1. Create a Prometheus Data Source. In the middle of the Grafana Welcome page, click on `Add your first Data Source`, and Select the Prometheus icon. +1. Log into the Web console for Grafana at . The default Login should be user/pass of `admin/admin`. This will present the main Grafana page. -1. Set the Connection URL to `http://prometheus:9090` as shown: +1. Create a Prometheus Data Source. In the middle of the Grafana Welcome page, click on `Add your first Data Source`, and select Prometheus. + +1. Set the Connection URL to `http://prometheus:9090` as shown. (this is your Prometheus container): ![Prom Datasource](media/lab7_prometheus-datasource.png) -1. Scroll to the bottom and click `Save and Test`. You should see a green `Successfully queried the Prometheus API` message. Click the Home page link to return the main menu. +1. Scroll to the bottom and click the `Save and Test` button. You should see a green `Successfully queried the Prometheus API` message. Click the Home page link to return to the Home page. -1. Import the provided `labs/lab7/NGINX-Basics.json` file to see statistics like the NGINX Plus HTTP Requests Per Second and Upstream Response Times. +1. Grafana Dashboard definitions can be saved as JSON files. Import the provided `labs/lab7/NGINX-Basics.json` file to see statistics like the NGINX Plus HTTP Requests Per Second, Connections, Upstream Response Times, and HTTP Response Codes. - - Click on Create New Dashboard from Home page and then `Import dashboard`. - - Copy and Paste the `labs/lab7/NGINX-Basics.json` file provided. + - Click on `Create New Dashboard` from Home page and then `Import dashboard`. + - Copy and Paste the contents of the `labs/lab7/NGINX-Basics.json` file provided. - Click on the `Load` button. - - Set the data source to `prometheus` and then click on the `Import` button. - - Sometimes you have change the datasource, Dashboard ID, Name, or other settings for it to Import properly. + - Set the Datasource to `Prometheus` and then click on the `Import` button. + - Sometimes you have change the Datasource, Dashboard ID, Name, or other settings for it to Import properly. - You should see a Grafana Dashboard like this one: + You should see a Grafana Dashboard similar to this one. Set the Time Window to `Last 5 minutes`, and the Refresh to `5s`: ![Grafana Dashboard](media/lab7_grafana-dashboard.png) - There are many different Grafana Dashboards available, and you have the option to create and build dashboards to suite your needs. NGINX Plus provides over 240 metrics for TCP, HTTP, SSL, Virtual Servers, Locations, Rate Limits, and Upstreams. + If needed, restart your `WRK loadtest` to create some traffic and metrics to view. + + There are many different Grafana Dashboards available, and you have the option to create and build Dashboards to suite your needs. NGINX Plus provides over 240 metrics for Nginx, TCP, HTTP, TLS, Virtual Servers, Location Blocks, Rate Limits, Caching, Resolvers, and Upstreams. - Take a few minutes to explore Grafana, and you can also import Dashboards that other people have created, by exploring the Grafana website and searching for "nginx". + Take a few minutes to explore Grafana, and you can also import shared Dashboards that other people have created, by exploring the Grafana website and searching for "nginx".
@@ -244,13 +315,13 @@ Grafana is a data visualization tool, which contains a time series database and ```bash ##Sample output## Running 5/5 - Container nginx-plus Removed - Container web2 Removed - Container prometheus Removed - Container web3 Removed - Container web1 Removed - Container grafana Removed - Network lab7_default Removed + Container s.jobs-nginx-plus Removed + Container s.jobs-web2 Removed + Container prometheus Removed + Container s.jobs-web3 Removed + Container s.jobs-web1 Removed + Container grafana Removed + Network lab7_default Removed ```