Skip to content

Commit 369fd8e

Browse files
tdurieuxtdurieux
committed
redis cache
1 parent b37a814 commit 369fd8e

3 files changed

Lines changed: 98 additions & 51 deletions

File tree

‎public/script/admin.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -995,7 +995,7 @@ angular
995995
getQueues();
996996
getMetrics();
997997
}
998-
}, 5000);
998+
}, 15000);
999999
$scope.$on("$destroy", () => $interval.cancel(stop));
10001000

10011001
$scope.refreshNow = function () { getQueues(); getMetrics(); };
@@ -1732,7 +1732,7 @@ angular
17321732
load();
17331733
const stop = $interval(() => {
17341734
if ($scope.query.autoRefresh) load();
1735-
}, 5000);
1735+
}, 15000);
17361736
$scope.$on("$destroy", () => $interval.cancel(stop));
17371737

17381738
$scope.$watch("query.search", recompute);

‎src/queue/queueMetrics.ts‎

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -69,10 +69,18 @@ export interface MetricPoint {
6969
avgMs: number;
7070
}
7171

72+
const METRIC_FIELDS = ["c", "f", "ms"];
73+
const metricsCache = new Map<string, { data: MetricPoint[]; ts: number }>();
74+
const METRICS_CACHE_TTL = 30_000;
75+
7276
export async function queryMetrics(
7377
queue: string,
7478
rangeMinutes: number
7579
): Promise<MetricPoint[]> {
80+
const cacheKey = `${queue}:${rangeMinutes}`;
81+
const cached = metricsCache.get(cacheKey);
82+
if (cached && Date.now() - cached.ts < METRICS_CACHE_TTL) return cached.data;
83+
7684
const c = getClient();
7785
if (!c || !c.isOpen) return [];
7886

@@ -88,14 +96,14 @@ export async function queryMetrics(
8896

8997
try {
9098
const pipe = c.multi();
91-
for (const k of keys) pipe.hGetAll(k);
99+
for (const k of keys) pipe.hmGet(k, METRIC_FIELDS);
92100
const results = await pipe.exec();
93101

94-
return timestamps.map((ts, i) => {
95-
const h = (results[i] as unknown as Record<string, string>) || {};
96-
const completed = parseInt(h.c || "0", 10) || 0;
97-
const failed = parseInt(h.f || "0", 10) || 0;
98-
const totalMs = parseInt(h.ms || "0", 10) || 0;
102+
const points = timestamps.map((ts, i) => {
103+
const vals = (results[i] as unknown as (string | null)[]) || [];
104+
const completed = parseInt(vals[0] || "0", 10) || 0;
105+
const failed = parseInt(vals[1] || "0", 10) || 0;
106+
const totalMs = parseInt(vals[2] || "0", 10) || 0;
99107
const total = completed + failed;
100108
return {
101109
ts,
@@ -104,6 +112,8 @@ export async function queryMetrics(
104112
avgMs: total > 0 ? Math.round(totalMs / total) : 0,
105113
};
106114
});
115+
metricsCache.set(cacheKey, { data: points, ts: Date.now() });
116+
return points;
107117
} catch {
108118
return [];
109119
}

‎src/server/routes/admin.ts‎

Lines changed: 80 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,9 @@ import config from "../../config";
2828
const logger = createLogger("admin");
2929

3030
let errorLogClient: RedisClientType | null = null;
31+
32+
let errorStatsCache: { data: unknown; ts: number } | null = null;
33+
const ERROR_STATS_CACHE_TTL = 30_000;
3134
async function getErrorLogClient(): Promise<RedisClientType | null> {
3235
if (errorLogClient && errorLogClient.isOpen) return errorLogClient;
3336
try {
@@ -69,6 +72,15 @@ router.use(
6972

7073
router.use("/tokens", adminTokensRouter);
7174

75+
function dashboardCache(
76+
_req: express.Request,
77+
res: express.Response,
78+
next: express.NextFunction
79+
) {
80+
res.set("Cache-Control", "private, max-age=10, must-revalidate");
81+
next();
82+
}
83+
7284
const QUEUE_STATES = [
7385
"waiting",
7486
"active",
@@ -256,7 +268,16 @@ router.post("/queues/pause-all", async (_req, res) => {
256268
}
257269
});
258270

271+
const queueStatsCache = new Map<
272+
string,
273+
{ data: Record<string, unknown>; ts: number }
274+
>();
275+
const QUEUE_STATS_CACHE_TTL = 15_000;
276+
259277
async function queueStats(queueKey: string, queue: Queue) {
278+
const cached = queueStatsCache.get(queueKey);
279+
if (cached && Date.now() - cached.ts < QUEUE_STATS_CACHE_TTL) return cached.data;
280+
260281
const [counts, workers, paused, metrics24h] =
261282
await Promise.all([
262283
queue.getJobCounts(...QUEUE_STATES),
@@ -275,14 +296,16 @@ async function queueStats(queueKey: string, queue: Queue) {
275296
failed24h += p.failed;
276297
}
277298

278-
return {
299+
const data = {
279300
counts,
280301
paused,
281302
workers: workerCount,
282303
concurrency,
283304
completed24h,
284305
failed24h,
285306
};
307+
queueStatsCache.set(queueKey, { data, ts: Date.now() });
308+
return data;
286309
}
287310

288311
const RANGE_MINUTES: Record<string, number> = {
@@ -292,7 +315,7 @@ const RANGE_MINUTES: Record<string, number> = {
292315
"7d": 10080,
293316
};
294317

295-
router.get("/queues/metrics", async (req, res) => {
318+
router.get("/queues/metrics", dashboardCache, async (req, res) => {
296319
const queueName = String(req.query.queue || "download");
297320
if (!pickQueue(queueName)) return res.status(404).json({ error: "queue_not_found" });
298321
const range = String(req.query.range || "1h");
@@ -305,63 +328,70 @@ router.get("/queues/metrics", async (req, res) => {
305328
}
306329
});
307330

308-
router.get("/queues", async (req, res) => {
331+
const queuesCache = new Map<string, { data: unknown; ts: number }>();
332+
const QUEUES_CACHE_TTL = 10_000;
333+
334+
router.get("/queues", dashboardCache, async (req, res) => {
309335
const search = req.query.search ? String(req.query.search).toLowerCase() : "";
310336
const queueName = req.query.queue ? String(req.query.queue) : "";
337+
const cacheKey = `${queueName}|${search}`;
338+
const cached = queuesCache.get(cacheKey);
339+
if (cached && Date.now() - cached.ts < QUEUES_CACHE_TTL) {
340+
return res.json(cached.data);
341+
}
311342

312343
const allQueues: { key: string; label: string; queue: Queue }[] = [
313344
{ key: "download", label: "Download", queue: downloadQueue },
314345
{ key: "remove", label: "Remove", queue: removeQueue },
315346
{ key: "cache", label: "Cache cleanup", queue: cacheQueue },
316347
];
317348

318-
const statsResults = await Promise.all(
319-
allQueues.map(async (q) => ({
320-
key: q.key,
321-
label: q.label,
322-
...(await queueStats(q.key, q.queue)),
323-
}))
324-
);
325-
326349
const target = queueName
327350
? allQueues.find((q) => q.key === queueName)
328351
: allQueues[0];
329352
const targetQueue = target ? target.queue : downloadQueue;
330353

331-
const matches = (job: { id?: string | undefined; name?: string }) => {
332-
if (!search) return true;
333-
return (
334-
(job.id || "").toLowerCase().includes(search) ||
335-
(job.name || "").toLowerCase().includes(search)
336-
);
337-
};
338-
339-
// Fetch all states in parallel, tag each job with its state
340-
const jobsByState = await Promise.all(
341-
QUEUE_STATES.map(async (state) => {
342-
const jobs = await targetQueue.getJobs([state]);
354+
const [statsResults, ...jobsByState] = await Promise.all([
355+
Promise.all(
356+
allQueues.map(async (q) => ({
357+
key: q.key,
358+
label: q.label,
359+
...(await queueStats(q.key, q.queue)),
360+
}))
361+
),
362+
...QUEUE_STATES.map(async (state) => {
363+
const jobs = await targetQueue.getJobs([state], 0, 199);
343364
return jobs.map((j) => {
344365
const json: Record<string, unknown> = { ...j.asJSON(), _state: state };
345366
if (state === "delayed" && j.delay > 0) {
346367
json.delayUntil = j.timestamp + j.delay;
347368
}
348369
return json;
349370
});
350-
})
351-
);
352-
const allJobs = jobsByState.flat().filter(matches);
371+
}),
372+
]);
373+
374+
const matches = (job: { id?: string | undefined; name?: string }) => {
375+
if (!search) return true;
376+
return (
377+
(job.id || "").toLowerCase().includes(search) ||
378+
(job.name || "").toLowerCase().includes(search)
379+
);
380+
};
381+
const allJobs = (jobsByState as Record<string, unknown>[][]).flat().filter(matches);
353382

354-
// Sort: active first, then waiting, delayed, failed, completed
355383
const stateOrder: Record<string, number> = {
356384
active: 0, waiting: 1, delayed: 2, failed: 3, completed: 4,
357385
};
358386
allJobs.sort((a, b) => (stateOrder[a._state as string] ?? 9) - (stateOrder[b._state as string] ?? 9));
359387

360-
res.json({
388+
const data = {
361389
queues: statsResults,
362390
selectedQueue: target?.key || "download",
363391
jobs: allJobs,
364-
});
392+
};
393+
queuesCache.set(cacheKey, { data, ts: Date.now() });
394+
res.json(data);
365395
});
366396

367397
// Errors captured by the logger sink. Server-paginated to avoid pulling
@@ -412,22 +442,29 @@ router.get("/errors", async (req, res) => {
412442

413443
// Aggregated stats from the precomputed hourly counters (HINCRBY on each
414444
// persistError). No JSON parsing of stored entries — O(48 small HGETALLs).
415-
router.get("/errors/stats", async (req, res) => {
445+
router.get("/errors/stats", dashboardCache, async (req, res) => {
416446
try {
447+
if (
448+
errorStatsCache &&
449+
Date.now() - errorStatsCache.ts < ERROR_STATS_CACHE_TTL
450+
) {
451+
return res.json(errorStatsCache.data);
452+
}
453+
417454
const client = await getErrorLogClient();
418455
if (!client) {
419-
return res.json({
456+
const data = {
420457
available: false,
421458
last24h: 0,
422459
prev24h: 0,
423460
severity: { error: 0, warn: 0, info: 0 },
424461
unique: { error: 0, warn: 0, info: 0 },
425462
buckets: [],
426463
dropped: getInProcessDropped(),
427-
});
464+
};
465+
return res.json(data);
428466
}
429467
const now = new Date();
430-
// Build the 48 hour keys to fetch (24 for current window + 24 for prev).
431468
function hourKey(d: Date) {
432469
const y = d.getUTCFullYear();
433470
const m = String(d.getUTCMonth() + 1).padStart(2, "0");
@@ -440,8 +477,6 @@ router.get("/errors/stats", async (req, res) => {
440477
const bucketHourTs: number[] = [];
441478
for (let i = 23; i >= 0; i--) {
442479
const d = new Date(now.getTime() - i * 3600 * 1000);
443-
// Anchor each bar at the end of its hour so a "9s ago" event lands in
444-
// the rightmost bar.
445480
const anchor = new Date(
446481
Date.UTC(
447482
d.getUTCFullYear(),
@@ -467,17 +502,17 @@ router.get("/errors/stats", async (req, res) => {
467502
}
468503
const pipe = client.multi();
469504
for (const k of currentKeys) pipe.hGetAll(k);
470-
for (const k of prevKeys) pipe.hGetAll(k);
505+
for (const k of prevKeys) pipe.hmGet(k, ["total"]);
471506
pipe.get(ERROR_LOG_DROPPED_KEY);
472507
const results = (await pipe.exec()) as unknown[];
473508
const currentHashes = results.slice(0, currentKeys.length) as Record<
474509
string,
475510
string
476511
>[];
477-
const prevHashes = results.slice(
512+
const prevTotals = results.slice(
478513
currentKeys.length,
479514
currentKeys.length + prevKeys.length
480-
) as Record<string, string>[];
515+
) as (string | null)[][];
481516
const droppedRedis =
482517
parseInt(String(results[results.length - 1] || "0"), 10) || 0;
483518

@@ -504,7 +539,6 @@ router.get("/errors/stats", async (req, res) => {
504539
sev.warn += w;
505540
sev.info += inf;
506541
last24h += parseInt(flat.total || "0", 10) || 0;
507-
// cb:<bucket>:<code> fields → unique code sets.
508542
for (const k of Object.keys(flat)) {
509543
if (!k.startsWith("cb:")) continue;
510544
const sep = k.indexOf(":", 3);
@@ -515,11 +549,11 @@ router.get("/errors/stats", async (req, res) => {
515549
}
516550
});
517551
let prev24h = 0;
518-
for (const h of prevHashes) {
519-
prev24h += parseInt((h || {}).total || "0", 10) || 0;
552+
for (const row of prevTotals) {
553+
prev24h += parseInt((row && row[0]) || "0", 10) || 0;
520554
}
521555

522-
res.json({
556+
const data = {
523557
available: true,
524558
last24h,
525559
prev24h,
@@ -531,7 +565,9 @@ router.get("/errors/stats", async (req, res) => {
531565
},
532566
buckets,
533567
dropped: droppedRedis + getInProcessDropped(),
534-
});
568+
};
569+
errorStatsCache = { data, ts: Date.now() };
570+
res.json(data);
535571
} catch (error) {
536572
handleError(error, res, req);
537573
}
@@ -559,6 +595,7 @@ router.delete("/errors", async (req, res) => {
559595
pipe.del(ERROR_LOG_DROPPED_KEY);
560596
if (hourlyKeys.length) pipe.del(hourlyKeys);
561597
await pipe.exec();
598+
errorStatsCache = null;
562599
res.json({ ok: true, cleared: len, hourlyCleared: hourlyKeys.length });
563600
} catch (error) {
564601
handleError(error, res, req);

0 commit comments

Comments
 (0)