feat: working metrics

Author: gjuchault

.cspell.json

@@ -9,6 +9,7 @@
 		"eqalpha",
 		"esbuild",
 		"formbody",
+		"fsreqcallback",
 		"gjuchault",
 		"healthcheck",
 		"isready",
@@ -23,8 +24,10 @@
 		"ryansonshine",
 		"slonik",
 		"socio",
+		"tcpsocketwrap",
 		"timestamptz",
 		"tini",
+		"tlswrap",
 		"traceparent",
 		"tsdoc",
 		"typescript-service-starter",

docker-compose.yml

@@ -29,6 +29,7 @@ services:
       HTTP_ADDRESS: 0.0.0.0
       HTTP_PORT: 3000
       OTLP_TRACE_ENDPOINT: http://jaeger:4318
+      OTLP_METRICS_ENDPOINT: http://prometheus:9090
 
   postgres:
     container_name: postgres
@@ -62,6 +63,10 @@ services:
 
   prometheus:
     image: prom/prometheus
+    container_name: prometheus
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+      - --web.enable-otlp-receiver
     volumes:
       - ./prometheus.yml:/etc/prometheus/prometheus.yml
     ports:
@@ -76,6 +81,7 @@ services:
 
   jaeger:
     image: jaegertracing/jaeger
+    container_name: jaeger
     ports:
       - 16686:16686 # web ui
       - 4317:4317 # accept OpenTelemetry Protocol (OTLP) over gRPC

otel-collector-config.yml

@@ -11,7 +11,7 @@ import rateLimit from "@fastify/rate-limit";
 import session from "@fastify/session";
 import swagger from "@fastify/swagger";
 import underPressure from "@fastify/under-pressure";
-import { context, propagation, SpanKind, trace } from "@opentelemetry/api";
+import { SpanKind, context, propagation } from "@opentelemetry/api";
 import { RedisStore } from "connect-redis";
 import {
 	type FastifyBaseLogger,

src/infrastructure/http-server/http-server.ts

@@ -0,0 +1,90 @@
+import { monitorEventLoopDelay } from "node:perf_hooks";
+import type { Meter } from "@opentelemetry/api";
+import type { MonitorOptions } from "./index.ts";
+
+export function monitorEventLoopLag(
+	nodeMetrics: Meter,
+	{ prefix, labels, eventLoopMonitoringPrecision }: MonitorOptions,
+): void {
+	const nodejsEventLoopLag = "nodejs_eventloop_lag_seconds";
+	const nodejsEventLoopLagMin = "nodejs_eventloop_lag_min_seconds";
+	const nodejsEventLoopLagMax = "nodejs_eventloop_lag_max_seconds";
+	const nodejsEventLoopLagMean = "nodejs_eventloop_lag_mean_seconds";
+	const nodejsEventLoopLagStddev = "nodejs_eventloop_lag_stddev_seconds";
+	const nodejsEventLoopLagP50 = "nodejs_eventloop_lag_p50_seconds";
+	const nodejsEventLoopLagP90 = "nodejs_eventloop_lag_p90_seconds";
+	const nodejsEventLoopLagP99 = "nodejs_eventloop_lag_p99_seconds";
+
+	const eventLoopDelayMonitor = monitorEventLoopDelay({
+		resolution: eventLoopMonitoringPrecision,
+	});
+	eventLoopDelayMonitor.enable();
+
+	nodeMetrics
+		.createObservableGauge(prefix + nodejsEventLoopLag, {
+			description: "Lag of event loop in seconds.",
+		})
+		.addCallback(async (observable) => {
+			const startTime = process.hrtime();
+			await new Promise((resolve) => setImmediate(() => resolve(undefined)));
+			const delta = process.hrtime(startTime);
+			const seconds = delta[0] + delta[1] / 1e9;
+			observable.observe(seconds, labels);
+		});
+
+	nodeMetrics
+		.createObservableGauge(prefix + nodejsEventLoopLagMin, {
+			description: "The minimum recorded event loop delay.",
+		})
+		.addCallback((observable) => {
+			observable.observe(eventLoopDelayMonitor.min / 1e9, labels);
+		});
+
+	nodeMetrics
+		.createObservableGauge(prefix + nodejsEventLoopLagMax, {
+			description: "The maximum recorded event loop delay.",
+		})
+		.addCallback((observable) => {
+			observable.observe(eventLoopDelayMonitor.max / 1e9, labels);
+		});
+
+	nodeMetrics
+		.createObservableGauge(prefix + nodejsEventLoopLagMean, {
+			description: "The mean of the recorded event loop delays.",
+		})
+		.addCallback((observable) => {
+			observable.observe(eventLoopDelayMonitor.mean / 1e9, labels);
+		});
+
+	nodeMetrics
+		.createObservableGauge(prefix + nodejsEventLoopLagStddev, {
+			description: "The standard deviation of the recorded event loop delays.",
+		})
+		.addCallback((observable) => {
+			observable.observe(eventLoopDelayMonitor.stddev / 1e9, labels);
+		});
+
+	nodeMetrics
+		.createObservableGauge(prefix + nodejsEventLoopLagP50, {
+			description: "The 50th percentile of the recorded event loop delays.",
+		})
+		.addCallback((observable) => {
+			observable.observe(eventLoopDelayMonitor.percentile(50) / 1e9, labels);
+		});
+
+	nodeMetrics
+		.createObservableGauge(prefix + nodejsEventLoopLagP90, {
+			description: "The 90th percentile of the recorded event loop delays.",
+		})
+		.addCallback((observable) => {
+			observable.observe(eventLoopDelayMonitor.percentile(90) / 1e9, labels);
+		});
+
+	nodeMetrics
+		.createObservableGauge(prefix + nodejsEventLoopLagP99, {
+			description: "The 99th percentile of the recorded event loop delays.",
+		})
+		.addCallback((observable) => {
+			observable.observe(eventLoopDelayMonitor.percentile(99) / 1e9, labels);
+		});
+}

src/infrastructure/telemetry/node-metrics/event-loop-lag.ts

@@ -0,0 +1,46 @@
+import { constants, PerformanceObserver } from "node:perf_hooks";
+import type { Meter } from "@opentelemetry/api";
+import type { MonitorOptions } from "./index.ts";
+
+export function monitorGc(
+	meter: Meter,
+	{ prefix, labels }: MonitorOptions,
+): void {
+	const nodejsGcDurationSeconds = "nodejs_gc_duration_seconds";
+
+	const histogram = meter.createHistogram(prefix + nodejsGcDurationSeconds, {
+		description:
+			"Garbage collection duration by kind, one of major, minor, incremental or weakcb.",
+	});
+
+	const kinds: Record<number, Record<string, string>> = {};
+	kinds[constants.NODE_PERFORMANCE_GC_MAJOR] = { ...labels, kind: "major" };
+	kinds[constants.NODE_PERFORMANCE_GC_MINOR] = { ...labels, kind: "minor" };
+	kinds[constants.NODE_PERFORMANCE_GC_INCREMENTAL] = {
+		...labels,
+		kind: "incremental",
+	};
+	kinds[constants.NODE_PERFORMANCE_GC_WEAKCB] = { ...labels, kind: "weakcb" };
+
+	const obs = new PerformanceObserver((list) => {
+		const entry = list.getEntries().at(0);
+
+		if (
+			entry === undefined ||
+			!(
+				typeof entry.detail === "object" &&
+				entry.detail !== null &&
+				"kind" in entry.detail &&
+				typeof entry.detail.kind === "number"
+			)
+		) {
+			return;
+		}
+
+		// Convert duration from milliseconds to seconds
+		histogram.record(entry.duration / 1000, kinds[entry.detail.kind]);
+	});
+
+	// We do not expect too many gc events per second, so we do not use buffering
+	obs.observe({ entryTypes: ["gc"], buffered: false });
+}

src/infrastructure/telemetry/node-metrics/gc.ts

@@ -0,0 +1,63 @@
+import type { Meter } from "@opentelemetry/api";
+import { safeMemoryUsage } from "./helpers/safe-memory-usage.ts";
+import type { MonitorOptions } from "./index.ts";
+
+export function monitorHeapSizeAndUsed(
+	meter: Meter,
+	{ prefix, labels }: MonitorOptions,
+): void {
+	const nodejsHeapSizeTotal = "nodejs_heap_size_total_bytes";
+	const nodejsHeapSizeUsed = "nodejs_heap_size_used_bytes";
+	const nodejsExternalMemory = "nodejs_external_memory_bytes";
+
+	let stats: NodeJS.MemoryUsage | undefined;
+	function getStats() {
+		if (stats !== undefined) {
+			return stats;
+		}
+
+		stats = safeMemoryUsage() ?? undefined;
+
+		setTimeout(() => {
+			stats = undefined;
+		}, 1000).unref();
+
+		return stats;
+	}
+
+	meter
+		.createObservableGauge(prefix + nodejsHeapSizeTotal, {
+			description: "Process heap size from Node.js in bytes.",
+		})
+		.addCallback((observable) => {
+			getStats();
+
+			if (stats?.heapTotal !== undefined) {
+				observable.observe(stats.heapTotal, labels);
+			}
+		});
+
+	meter
+		.createObservableGauge(prefix + nodejsHeapSizeUsed, {
+			description: "Process heap size used from Node.js in bytes.",
+		})
+		.addCallback((observable) => {
+			getStats();
+
+			if (stats?.heapTotal !== undefined) {
+				observable.observe(stats.heapUsed, labels);
+			}
+		});
+
+	meter
+		.createObservableGauge(prefix + nodejsExternalMemory, {
+			description: "Node.js external memory size in bytes.",
+		})
+		.addCallback((observable) => {
+			getStats();
+
+			if (stats?.external !== undefined) {
+				observable.observe(stats.external, labels);
+			}
+		});
+}

src/infrastructure/telemetry/node-metrics/heap-size-and-used.ts

@@ -0,0 +1,103 @@
+import v8 from "node:v8";
+import type { Meter } from "@opentelemetry/api";
+import type { MonitorOptions } from "./index.ts";
+
+const nodeJsHeapSize = {
+	total: "nodejs_heap_space_size_total_bytes",
+	used: "nodejs_heap_space_size_used_bytes",
+	available: "nodejs_heap_space_size_available_bytes",
+};
+
+const spaceRegex = /_space$/;
+
+export function monitorHeapSpacesSizeAndUsed(
+	meter: Meter,
+	{ prefix, labels }: MonitorOptions,
+): void {
+	const labelsBySpace: Record<
+		string,
+		Record<"total" | "used" | "available", Record<string, string>>
+	> = {};
+
+	let stats:
+		| {
+				total: { value: number; labels: Record<string, string> };
+				used: { value: number; labels: Record<string, string> };
+				available: { value: number; labels: Record<string, string> };
+		  }[]
+		| undefined;
+	function getStats() {
+		if (stats !== undefined) {
+			return stats;
+		}
+
+		stats = v8.getHeapSpaceStatistics().map((space) => {
+			const spaceLabels =
+				labelsBySpace[space.space_name] ?? defaultLabelsBySpace(space, labels);
+			labelsBySpace[space.space_name] = spaceLabels;
+
+			return {
+				total: { value: space.space_size, labels: spaceLabels.total },
+				used: { value: space.space_used_size, labels: spaceLabels.used },
+				available: {
+					value: space.space_available_size,
+					labels: spaceLabels.available,
+				},
+			};
+		});
+
+		setTimeout(() => {
+			stats = undefined;
+		}, 1000).unref();
+		return stats;
+	}
+
+	meter
+		.createObservableGauge(prefix + nodeJsHeapSize.total, {
+			description: "Process heap space size total from Node.js in bytes.",
+		})
+		.addCallback((observable) => {
+			getStats();
+
+			for (const stat of stats ?? []) {
+				observable.observe(stat.total.value, stat.total.labels);
+			}
+		});
+
+	meter
+		.createObservableGauge(prefix + nodeJsHeapSize.used, {
+			description: "Process heap space size used from Node.js in bytes.",
+		})
+		.addCallback((observable) => {
+			getStats();
+
+			for (const stat of stats ?? []) {
+				observable.observe(stat.used.value, stat.used.labels);
+			}
+		});
+
+	meter
+		.createObservableGauge(prefix + nodeJsHeapSize.available, {
+			description: "Process heap space size available from Node.js in bytes.",
+		})
+		.addCallback((observable) => {
+			getStats();
+
+			for (const stat of stats ?? []) {
+				observable.observe(stat.available.value, stat.available.labels);
+			}
+		});
+}
+
+function defaultLabelsBySpace(
+	space: v8.HeapSpaceInfo,
+	labels: Record<string, string>,
+): Record<"total" | "used" | "available", Record<string, string>> {
+	const spaceName = space.space_name.replace(spaceRegex, "");
+
+	return {
+		total: { ...labels, space: spaceName },
+		used: { ...labels, space: spaceName },
+		available: { ...labels, space: spaceName },
+	};
+}