forked from Talenttrust/Talenttrust-Backend
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathmetrics-service.ts
More file actions
341 lines (279 loc) · 10.2 KB
/
Copy pathmetrics-service.ts
File metadata and controls
341 lines (279 loc) · 10.2 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
import { NextFunction, Request, Response } from 'express';
import {
collectDefaultMetrics,
Counter,
Gauge,
Histogram,
Registry,
} from 'prom-client';
import { ServiceStatus } from './types';
import {
assertDlqDepth,
assertServiceStatus,
assertWebhookOutcome,
WebhookOutcome as ValidatedWebhookOutcome,
} from './metrics-validation';
import { DEFAULT_HISTOGRAM_BUCKETS, validateHistogramBuckets } from './observability-config';
/**
* Re-exported from metrics-validation to preserve existing import paths.
* The canonical definition lives in metrics-validation.ts where all metric
* input types are colocated.
*/
export type WebhookOutcome = ValidatedWebhookOutcome;
/**
* Canonical list of metric family names documented in docs/observability.md.
* This constant enables round-trip verification: tests assert that the set of
* metrics registered by MetricsService matches this list exactly.
*/
export const CATALOG_METRIC_NAMES: readonly string[] = [
'http_requests_total',
'http_request_duration_seconds',
'service_health_status',
'webhook_deliveries_total',
'webhook_dlq_depth',
'webhook_rate_limit_tokens',
'webhook_rate_limit_queue_depth',
] as const;
export interface MetricsServiceLike {
contentType: string;
trackHttpRequest: (req: Request, res: Response, next: NextFunction) => void;
getMetrics: () => Promise<string>;
recordHealthStatus: (status: ServiceStatus) => void;
recordWebhookDelivery: (outcome: WebhookOutcome) => void;
setWebhookDlqDepth: (depth: number) => void;
startRateLimitMetricsSampling?: (limiter: any, intervalMs?: number) => void;
stopRateLimitMetricsSampling?: () => void;
}
const HEALTH_STATUS_VALUE: Record<ServiceStatus, number> = {
up: 2,
degraded: 1,
down: 0,
};
const DEFAULT_HTTP_ROUTE_LABEL_LIMIT = 100;
const OTHER_ROUTE_LABEL = 'other';
const UNMATCHED_ROUTE_LABEL = 'unmatched';
export interface MetricsServiceOptions {
httpRouteLabelLimit?: number;
/**
* Custom histogram bucket boundaries (in seconds) for
* `http_request_duration_seconds`. Must be a non-empty array of strictly
* increasing positive numbers. Falls back to {@link DEFAULT_HISTOGRAM_BUCKETS}
* when absent or invalid.
*/
histogramBuckets?: number[];
}
/**
* Manages Prometheus metrics registration and request instrumentation.
*/
export class MetricsService implements MetricsServiceLike {
readonly contentType: string;
private readonly register: Registry;
private readonly httpRequestsTotal: Counter;
private readonly httpRequestDurationSeconds: Histogram;
private readonly serviceHealthStatus: Gauge;
private readonly webhookDeliveriesTotal: Counter;
private readonly webhookDlqDepth: Gauge;
private readonly webhookRateLimitTokens: Gauge;
private readonly webhookRateLimitQueueDepth: Gauge;
private readonly httpRouteLabelLimit: number;
private readonly observedHttpRouteLabels = new Set<string>();
private rateLimitStopSampling: (() => void) | null = null;
constructor(
private readonly serviceName: string,
register?: Registry,
options: MetricsServiceOptions = {},
) {
this.register = register ?? new Registry();
this.httpRouteLabelLimit = options.httpRouteLabelLimit ?? DEFAULT_HTTP_ROUTE_LABEL_LIMIT;
// Resolve histogram buckets: validate caller-supplied values and fall back
// to defaults when absent or invalid, so misconfiguration is non-fatal.
const resolvedBuckets = resolveHistogramBuckets(options.histogramBuckets);
collectDefaultMetrics({
register: this.register,
prefix: `${sanitizeMetricPrefix(serviceName)}_`,
});
this.httpRequestsTotal = new Counter({
name: 'http_requests_total',
help: 'Total number of HTTP requests.',
labelNames: ['method', 'route', 'status_code'],
registers: [this.register],
});
this.httpRequestDurationSeconds = new Histogram({
name: 'http_request_duration_seconds',
help: 'Duration of HTTP requests in seconds.',
labelNames: ['method', 'route', 'status_code'],
buckets: resolvedBuckets,
registers: [this.register],
});
this.serviceHealthStatus = new Gauge({
name: 'service_health_status',
help: 'Current service health status. up=2, degraded=1, down=0.',
labelNames: ['service'],
registers: [this.register],
});
this.serviceHealthStatus.set({ service: this.serviceName }, HEALTH_STATUS_VALUE.up);
this.contentType = this.register.contentType;
this.webhookDeliveriesTotal = new Counter({
name: 'webhook_deliveries_total',
help: 'Total webhook delivery attempts by outcome.',
labelNames: ['outcome'],
registers: [this.register],
});
this.webhookDlqDepth = new Gauge({
name: 'webhook_dlq_depth',
help: 'Current number of entries in the webhook dead-letter queue.',
registers: [this.register],
});
this.webhookRateLimitTokens = new Gauge({
name: 'webhook_rate_limit_tokens',
help: 'Current token count per provider in the rate-limiter bucket.',
labelNames: ['provider_id'],
registers: [this.register],
});
this.webhookRateLimitQueueDepth = new Gauge({
name: 'webhook_rate_limit_queue_depth',
help: 'Current queue depth (number of waiting deliveries) per provider in the rate-limiter.',
labelNames: ['provider_id'],
registers: [this.register],
});
}
trackHttpRequest(req: Request, res: Response, next: NextFunction): void {
const start = process.hrtime.bigint();
res.on('finish', () => {
const duration = Number(process.hrtime.bigint() - start) / 1_000_000_000;
const route = this.boundRouteLabel(extractRoute(req));
const labels = {
method: req.method,
route,
status_code: String(res.statusCode),
};
this.httpRequestsTotal.inc(labels);
this.httpRequestDurationSeconds.observe(labels, duration);
});
next();
}
recordHealthStatus(status: ServiceStatus): void {
// Runtime guard: reject unknown status strings that bypass TypeScript types
// (e.g. from JSON-deserialized or cross-process call sites).
const validated = assertServiceStatus(status);
this.serviceHealthStatus.set(
{ service: this.serviceName },
HEALTH_STATUS_VALUE[validated],
);
}
recordWebhookDelivery(outcome: WebhookOutcome): void {
// Runtime guard: reject unknown outcome strings.
const validated = assertWebhookOutcome(outcome);
this.webhookDeliveriesTotal.inc({ outcome: validated });
}
setWebhookDlqDepth(depth: number): void {
// Runtime guard: reject NaN, ±Infinity, negative values, and unreasonably
// large values that would indicate a bug or injection attempt.
const validated = assertDlqDepth(depth);
this.webhookDlqDepth.set(validated);
}
startRateLimitMetricsSampling(limiter: any, intervalMs: number = 10000): void {
if (this.rateLimitStopSampling !== null) {
console.warn('[MetricsService] Rate limit metrics sampling already active.');
return;
}
this.rateLimitStopSampling = limiter.startMetricsSampling(
this.webhookRateLimitTokens,
this.webhookRateLimitQueueDepth,
intervalMs,
);
}
stopRateLimitMetricsSampling(): void {
if (this.rateLimitStopSampling !== null) {
this.rateLimitStopSampling();
this.rateLimitStopSampling = null;
}
}
getMetrics(): Promise<string> {
return this.register.metrics();
}
private boundRouteLabel(route: string): string {
// Never collapse unmatched routes — they are not user-controlled and must
// always be tracked separately so operators can monitor 404 rates.
if (route === UNMATCHED_ROUTE_LABEL) {
return route;
}
if (this.observedHttpRouteLabels.has(route)) {
return route;
}
if (this.observedHttpRouteLabels.size < this.httpRouteLabelLimit) {
this.observedHttpRouteLabels.add(route);
return route;
}
return OTHER_ROUTE_LABEL;
}
}
/**
* Validate the caller-supplied bucket array and return it if valid.
* Falls back to {@link DEFAULT_HISTOGRAM_BUCKETS} when the input is absent or
* fails validation, ensuring that misconfiguration is non-fatal and existing
* dashboards keep working.
*/
function resolveHistogramBuckets(buckets: number[] | undefined): number[] {
if (buckets === undefined) {
return [...DEFAULT_HISTOGRAM_BUCKETS];
}
const result = validateHistogramBuckets(buckets);
if (!result.valid) {
console.warn(
`[MetricsService] Invalid histogramBuckets option (${result.reason}); falling back to defaults.`,
);
return [...DEFAULT_HISTOGRAM_BUCKETS];
}
return result.buckets;
}
function sanitizeMetricPrefix(input: string): string {
const sanitized = input.replace(/[^a-zA-Z0-9_:]/g, '_');
return sanitized.length > 0 ? sanitized : 'service';
}
/**
* Returns a bounded, non-user-controlled route label for HTTP metrics.
*
* Express exposes the matched route template at `req.route.path`; joining it
* with the static mount point in `req.baseUrl` preserves useful labels such as
* `/api/v1/contracts/:id` without using concrete request paths that may contain
* attacker-controlled identifiers. Requests that never match a route collapse
* into one shared bucket.
*/
function extractRoute(req: Request): string {
const routePath = formatExpressPath(req.route?.path);
if (routePath === null) {
return UNMATCHED_ROUTE_LABEL;
}
const baseUrl = normalizeRoutePart(req.baseUrl);
const route = joinRouteParts(baseUrl, routePath);
return route.length > 0 ? route : '/';
}
function formatExpressPath(path: unknown): string | null {
if (typeof path === 'string') {
return normalizeRoutePart(path);
}
if (path instanceof RegExp) {
return path.toString();
}
if (Array.isArray(path)) {
const parts = path.map(formatExpressPath).filter((part): part is string => part !== null);
return parts.length > 0 ? parts.join('|') : null;
}
return null;
}
function normalizeRoutePart(part: string | undefined): string {
if (!part || part === '/') {
return '';
}
return part.startsWith('/') ? part : `/${part}`;
}
function joinRouteParts(baseUrl: string, routePath: string): string {
if (!baseUrl) {
return routePath;
}
if (!routePath) {
return baseUrl;
}
return `${baseUrl}${routePath}`;
}