diff --git a/packages/backend/src/services/__tests__/staleCache.test.ts b/packages/backend/src/services/__tests__/staleCache.test.ts new file mode 100644 index 00000000..31b02e93 --- /dev/null +++ b/packages/backend/src/services/__tests__/staleCache.test.ts @@ -0,0 +1,445 @@ +/** + * @file staleCache.test.ts + * @description Load and concurrency tests for stale cache with probabilistic eviction. + * Tests verify that the stale cache mechanism handles high-throughput Web3 webhook + * traffic without blocking the Node.js event loop and maintains performance under load. + */ + +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { StaleCacheService } from "../staleCache.js"; +import { safeGet, safeSet } from "../cache.js"; +import { evictionEngine } from "../probabilisticEviction.js"; +import { logger } from "../../utils/logger.js"; + +// Mock dependencies +vi.mock("../cache.js"); +vi.mock("../probabilisticEviction.js"); +vi.mock("../../utils/logger.js"); + +describe("Stale Cache Load Tests", () => { + let staleCache: StaleCacheService; + + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(logger).debug = vi.fn(); + vi.mocked(logger).warn = vi.fn(); + vi.mocked(logger).error = vi.fn(); + vi.mocked(evictionEngine).recordAccess = vi.fn(); + vi.mocked(evictionEngine).getFrequency = vi.fn(() => 0); + vi.mocked(evictionEngine).totalAccesses = 0; + vi.mocked(evictionEngine).memoryBytes = 1600000; // 1.6 MB default + + staleCache = new StaleCacheService({ + staleThresholdMs: 1000, + expireThresholdMs: 5000, + baseRefreshProbability: 0.5, + defaultTTL: 10, + }); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + describe("Event Loop Non-Blocking Tests", () => { + it("should not block event loop during cache miss", async () => { + vi.mocked(safeGet).mockResolvedValue(null); + vi.mocked(safeSet).mockResolvedValue(undefined); + + let fetcherResolved = false; + const fetcher = vi.fn(async () => { + await new Promise(resolve => setTimeout(resolve, 50)); + fetcherResolved = true; + return { data: "fresh" }; + }); + + const getPromise = staleCache.get("test-key", fetcher); + + // Check if event loop is still responsive + let eventLoopResponsive = true; + try { + await new Promise(resolve => setTimeout(resolve, 10)); + } catch (error) { + eventLoopResponsive = false; + } + + expect(eventLoopResponsive).toBe(true); + expect(fetcherResolved).toBe(false); // Fetcher should still be processing + + await getPromise; + expect(fetcherResolved).toBe(true); + expect(fetcher).toHaveBeenCalledTimes(1); + }); + + it("should not block event loop during stale cache hit", async () => { + const now = Date.now(); + const meta = JSON.stringify({ timestamp: now - 2000, version: 1 }); + + vi.mocked(safeGet).mockImplementation(async (key) => { + if (key.includes("meta")) return meta; + return JSON.stringify({ data: "stale" }); + }); + vi.mocked(safeSet).mockResolvedValue(undefined); + vi.mocked(evictionEngine).getFrequency = vi.fn(() => 10); + vi.mocked(evictionEngine).totalAccesses = 100; + + const fetcher = vi.fn(async () => { + await new Promise(resolve => setTimeout(resolve, 50)); + return { data: "fresh" }; + }); + + const result = await staleCache.get("test-key", fetcher); + + // Should return stale data immediately + expect(result).toEqual({ data: "stale" }); + + // Event loop should remain responsive during background refresh + let eventLoopResponsive = true; + try { + await new Promise(resolve => setTimeout(resolve, 20)); + } catch (error) { + eventLoopResponsive = false; + } + expect(eventLoopResponsive).toBe(true); + }); + + it("should handle 100 concurrent cache operations without blocking", async () => { + vi.mocked(safeGet).mockResolvedValue(null); + vi.mocked(safeSet).mockResolvedValue(undefined); + + const fetcher = vi.fn(async (key) => { + await new Promise(resolve => setTimeout(resolve, 10)); + return { key, data: "fresh" }; + }); + + const concurrentRequests = 100; + const promises = []; + + for (let i = 0; i < concurrentRequests; i++) { + promises.push(staleCache.get(`key-${i}`, fetcher)); + } + + const startTime = Date.now(); + const results = await Promise.all(promises); + const duration = Date.now() - startTime; + + // Should complete quickly (not 100 * 10ms = 1000ms if blocking) + expect(duration).toBeLessThan(500); + expect(results.length).toBe(concurrentRequests); + expect(fetcher).toHaveBeenCalledTimes(concurrentRequests); + }); + }); + + describe("High Throughput Tests", () => { + it("should handle rapid cache hits without performance degradation", async () => { + const now = Date.now(); + const meta = JSON.stringify({ timestamp: now, version: 1 }); + + vi.mocked(safeGet).mockResolvedValue(JSON.stringify({ data: "cached" })); + vi.mocked(evictionEngine).getFrequency = vi.fn(() => 100); + + const fetcher = vi.fn(); + + const iterations = 1000; + const startTime = Date.now(); + + for (let i = 0; i < iterations; i++) { + await staleCache.get(`key-${i}`, fetcher); + } + + const duration = Date.now() - startTime; + const avgTimePerOp = duration / iterations; + + // Should average less than 1ms per operation for cache hits + expect(avgTimePerOp).toBeLessThan(1); + expect(fetcher).not.toHaveBeenCalled(); + }); + + it("should handle mixed cache hits and misses efficiently", async () => { + let callCount = 0; + vi.mocked(safeGet).mockImplementation(async () => { + callCount++; + if (callCount % 3 === 0) return null; // Every 3rd call is a miss + return JSON.stringify({ data: "cached" }); + }); + vi.mocked(safeSet).mockResolvedValue(undefined); + + const fetcher = vi.fn(async () => { + await new Promise(resolve => setTimeout(resolve, 5)); + return { data: "fresh" }; + }); + + const operations = 100; + const promises = []; + + for (let i = 0; i < operations; i++) { + promises.push(staleCache.get(`key-${i}`, fetcher)); + } + + const startTime = Date.now(); + await Promise.all(promises); + const duration = Date.now() - startTime; + + // Should complete quickly despite mixed hits/misses + expect(duration).toBeLessThan(500); + expect(fetcher).toHaveBeenCalledTimes(Math.ceil(operations / 3)); + }); + + it("should handle burst traffic during simulated block finalization", async () => { + vi.mocked(safeGet).mockResolvedValue(null); + vi.mocked(safeSet).mockResolvedValue(undefined); + + const fetcher = vi.fn(async () => { + await new Promise(resolve => setTimeout(resolve, 1)); + return { blockData: "simulated" }; + }); + + // Simulate burst of 50 webhooks in 100ms (block finalization spike) + const burstSize = 50; + const promises = []; + + const burstStart = Date.now(); + for (let i = 0; i < burstSize; i++) { + promises.push(staleCache.get(`block-${i}`, fetcher)); + // Small delay to simulate realistic webhook timing + await new Promise(resolve => setTimeout(resolve, 2)); + } + + await Promise.all(promises); + const burstDuration = Date.now() - burstStart; + + // Burst should complete in reasonable time + expect(burstDuration).toBeLessThan(500); + expect(fetcher).toHaveBeenCalledTimes(burstSize); + }); + }); + + describe("Probabilistic Refresh Behavior", () => { + it("should probabilistically refresh stale entries based on frequency", async () => { + const now = Date.now(); + const staleMeta = JSON.stringify({ timestamp: now - 2000, version: 1 }); + + vi.mocked(safeGet).mockResolvedValue(JSON.stringify({ data: "stale" })); + vi.mocked(safeSet).mockResolvedValue(undefined); + + // High frequency should increase refresh probability + vi.mocked(evictionEngine).getFrequency = vi.fn(() => 100); + vi.mocked(evictionEngine).totalAccesses = 1000; + + const fetcher = vi.fn(async () => { + await new Promise(resolve => setTimeout(resolve, 10)); + return { data: "fresh" }; + }); + + // Run multiple times to observe probabilistic behavior + const results = []; + for (let i = 0; i < 20; i++) { + const result = await staleCache.get(`hot-key-${i}`, fetcher); + results.push(result); + // Wait for potential background refresh + await new Promise(resolve => setTimeout(resolve, 15)); + } + + // High frequency keys should trigger background refreshes + // (not deterministic due to probability, but should happen) + expect(fetcher).toHaveBeenCalled(); + }); + + it("should not refresh cold keys as frequently", async () => { + const now = Date.now(); + const staleMeta = JSON.stringify({ timestamp: now - 2000, version: 1 }); + + vi.mocked(safeGet).mockResolvedValue(JSON.stringify({ data: "stale" })); + vi.mocked(safeSet).mockResolvedValue(undefined); + + // Low frequency should decrease refresh probability + vi.mocked(evictionEngine).getFrequency = vi.fn(() => 1); + vi.mocked(evictionEngine).totalAccesses = 1000; + + const fetcher = vi.fn(async () => { + await new Promise(resolve => setTimeout(resolve, 10)); + return { data: "fresh" }; + }); + + const results = []; + for (let i = 0; i < 20; i++) { + const result = await staleCache.get(`cold-key-${i}`, fetcher); + results.push(result); + await new Promise(resolve => setTimeout(resolve, 15)); + } + + // Cold keys should trigger fewer refreshes + expect(results.every(r => r.data === "stale")).toBe(true); + }); + }); + + describe("Error Handling Under Load", () => { + it("should handle Redis failures gracefully without blocking", async () => { + vi.mocked(safeGet).mockRejectedValue(new Error("Redis connection failed")); + + const fetcher = vi.fn(async () => { + await new Promise(resolve => setTimeout(resolve, 10)); + return { data: "fresh" }; + }); + + const result = await staleCache.get("test-key", fetcher); + + // Should fall back to fetcher + expect(result).toEqual({ data: "fresh" }); + expect(fetcher).toHaveBeenCalledTimes(1); + expect(vi.mocked(logger).error).toHaveBeenCalled(); + }); + + it("should handle malformed cache data gracefully", async () => { + vi.mocked(safeGet).mockResolvedValue("invalid-json"); + + const fetcher = vi.fn(async () => { + await new Promise(resolve => setTimeout(resolve, 10)); + return { data: "fresh" }; + }); + + const result = await staleCache.get("test-key", fetcher); + + // Should fall back to fetcher on parse error + expect(result).toEqual({ data: "fresh" }); + expect(fetcher).toHaveBeenCalledTimes(1); + expect(vi.mocked(logger).warn).toHaveBeenCalled(); + }); + + it("should handle fetcher failures without crashing", async () => { + vi.mocked(safeGet).mockResolvedValue(null); + vi.mocked(safeSet).mockResolvedValue(undefined); + + const fetcher = vi.fn(async () => { + throw new Error("Fetcher failed"); + }); + + await expect(staleCache.get("test-key", fetcher)).rejects.toThrow("Fetcher failed"); + expect(vi.mocked(logger).error).toHaveBeenCalled(); + }); + }); + + describe("Memory and Resource Management", () => { + it("should not cause memory leaks with repeated operations", async () => { + vi.mocked(safeGet).mockResolvedValue(null); + vi.mocked(safeSet).mockResolvedValue(undefined); + + const fetcher = vi.fn(async () => ({ data: "fresh" })); + + const initialMemory = process.memoryUsage().heapUsed; + + // Perform many operations + for (let i = 0; i < 1000; i++) { + await staleCache.get(`key-${i}`, fetcher); + } + + // Force garbage collection if available + if (global.gc) { + global.gc(); + } + + const finalMemory = process.memoryUsage().heapUsed; + const memoryIncrease = finalMemory - initialMemory; + + // Memory increase should be reasonable + expect(memoryIncrease).toBeLessThan(5 * 1024 * 1024); // < 5MB + }); + + it("should deduplicate concurrent refreshes for the same key", async () => { + const now = Date.now(); + const staleMeta = JSON.stringify({ timestamp: now - 2000, version: 1 }); + + vi.mocked(safeGet).mockResolvedValue(JSON.stringify({ data: "stale" })); + vi.mocked(safeSet).mockResolvedValue(undefined); + vi.mocked(evictionEngine).getFrequency = vi.fn(() => 100); + vi.mocked(evictionEngine).totalAccesses = 1000; + + let fetcherCallCount = 0; + const fetcher = vi.fn(async () => { + fetcherCallCount++; + await new Promise(resolve => setTimeout(resolve, 50)); + return { data: "fresh" }; + }); + + // Trigger multiple concurrent gets for the same key + const promises = []; + for (let i = 0; i < 10; i++) { + promises.push(staleCache.get("same-key", fetcher)); + } + + await Promise.all(promises); + + // Should deduplicate background refreshes + expect(fetcherCallCount).toBeLessThanOrEqual(2); + }); + }); + + describe("Input Validation Edge Cases", () => { + it("should reject invalid cache keys", async () => { + await expect(staleCache.get("", vi.fn())).rejects.toThrow("Cache key must be a non-empty string"); + await expect(staleCache.get(null as any, vi.fn())).rejects.toThrow("Cache key must be a non-empty string"); + await expect(staleCache.get(undefined as any, vi.fn())).rejects.toThrow("Cache key must be a non-empty string"); + }); + + it("should reject invalid fetcher", async () => { + await expect(staleCache.get("key", null as any)).rejects.toThrow("Fetcher must be a function"); + await expect(staleCache.get("key", "not a function" as any)).rejects.toThrow("Fetcher must be a function"); + }); + + it("should reject invalid config options", async () => { + const invalidCache = new StaleCacheService({ + staleThresholdMs: -1, + }); + + vi.mocked(safeGet).mockResolvedValue(null); + vi.mocked(safeSet).mockResolvedValue(undefined); + + await expect(invalidCache.get("key", vi.fn())).rejects.toThrow("staleThresholdMs must be non-negative"); + }); + + it("should reject undefined values in set", async () => { + await expect(staleCache.set("key", undefined as any)).rejects.toThrow("Cannot cache undefined value"); + }); + + it("should reject invalid TTL in set", async () => { + await expect(staleCache.set("key", { data: "test" }, -1)).rejects.toThrow("TTL must be a positive number"); + await expect(staleCache.set("key", { data: "test" }, 0)).rejects.toThrow("TTL must be a positive number"); + }); + }); + + describe("Cache Statistics and Monitoring", () => { + it("should provide accurate cache statistics", async () => { + const stats = staleCache.getStats(); + + expect(stats).toHaveProperty("pendingRefreshes"); + expect(stats).toHaveProperty("config"); + expect(stats).toHaveProperty("evictionEngine"); + expect(stats.evictionEngine).toHaveProperty("memoryBytes"); + expect(stats.evictionEngine).toHaveProperty("totalAccesses"); + }); + + it("should track pending refreshes accurately", async () => { + const now = Date.now(); + const staleMeta = JSON.stringify({ timestamp: now - 2000, version: 1 }); + + vi.mocked(safeGet).mockResolvedValue(JSON.stringify({ data: "stale" })); + vi.mocked(safeSet).mockResolvedValue(undefined); + vi.mocked(evictionEngine).getFrequency = vi.fn(() => 100); + vi.mocked(evictionEngine).totalAccesses = 1000; + + const fetcher = vi.fn(async () => { + await new Promise(resolve => setTimeout(resolve, 100)); + return { data: "fresh" }; + }); + + // Trigger background refresh + staleCache.get("key", fetcher); + + // Wait a bit for refresh to start + await new Promise(resolve => setTimeout(resolve, 10)); + + const stats = staleCache.getStats(); + expect(stats.pendingRefreshes).toBeGreaterThan(0); + }); + }); +}); diff --git a/packages/backend/src/services/staleCache.ts b/packages/backend/src/services/staleCache.ts new file mode 100644 index 00000000..e7aefe1c --- /dev/null +++ b/packages/backend/src/services/staleCache.ts @@ -0,0 +1,434 @@ +/** + * @file staleCache.ts + * @description Stale cache service with probabilistic eviction for high-throughput webhook ingestion. + * + * This service implements a stale-while-revalidate pattern with probabilistic eviction to: + * - Serve stale data immediately to avoid blocking the event loop + * - Refresh cache entries asynchronously in the background + * - Use probabilistic logic to determine when to refresh vs serve stale + * - Maintain state consistency during heavy Web3 block finalization spikes + * + * ## Design Rationale + * + * During heavy webhook ingestion (e.g., block finalization spikes), synchronous cache + * refreshes can block the Node.js event loop. This service: + * 1. Returns stale data if available (immediate response) + * 2. Probabilistically decides whether to refresh in background + * 3. Uses the existing Count-Min Sketch for access frequency tracking + * 4. Ensures non-blocking operations for all cache interactions + * + * ## Probabilistic Refresh Strategy + * + * - Hot keys (high frequency): Higher probability of background refresh + * - Cold keys (low frequency): Lower probability, serve stale longer + * - Stale threshold: Configurable max age before forced refresh + * - Refresh probability: Computed based on access frequency and staleness + */ + +import { safeGet, safeSet } from "./cache.js"; +import { evictionEngine } from "./probabilisticEviction.js"; +import { logger } from "../utils/logger.js"; + +// ─── Configuration ─────────────────────────────────────────────────────────── + +export interface StaleCacheConfig { + /** + * Maximum age (ms) before data is considered stale. + * Default: 30000 (30 seconds) + */ + staleThresholdMs?: number; + /** + * Maximum age (ms) before data is considered expired and must be refreshed. + * Default: 300000 (5 minutes) + */ + expireThresholdMs?: number; + /** + * Base probability (0-1) of refreshing a stale entry. + * Adjusted by access frequency. Default: 0.3 + */ + baseRefreshProbability?: number; + /** + * TTL for fresh cache entries (seconds). + * Default: 60 + */ + defaultTTL?: number; + /** + * Whether to enable probabilistic refresh. + * Default: true + */ + enableProbabilisticRefresh?: boolean; +} + +const DEFAULT_CONFIG: Required = { + staleThresholdMs: 30000, + expireThresholdMs: 300000, + baseRefreshProbability: 0.3, + defaultTTL: 60, + enableProbabilisticRefresh: true, +}; + +// ─── Cache Entry Metadata ───────────────────────────────────────────────────── + +interface CacheEntry { + data: T; + timestamp: number; + version: number; +} + +interface CacheMetadata { + timestamp: number; + version: number; +} + +// ─── Stale Cache Service ────────────────────────────────────────────────────── + +export class StaleCacheService { + private readonly config: Required; + private readonly pendingRefreshes = new Map>(); + + constructor(config: StaleCacheConfig = {}) { + this.config = { ...DEFAULT_CONFIG, ...config }; + } + + /** + * Get cached data with stale-while-revalidate semantics. + * + * Returns stale data immediately if available, then probabilistically + * refreshes in the background. Never blocks the event loop. + * + * @param key - Cache key + * @param fetcher - Async function to fetch fresh data + * @param options - Override default config for this operation + * @returns Cached data (stale or fresh) or fresh data if cache miss + */ + async get( + key: string, + fetcher: () => Promise, + options?: StaleCacheConfig + ): Promise { + // Input validation + if (!key || typeof key !== "string") { + throw new Error("Cache key must be a non-empty string"); + } + if (typeof fetcher !== "function") { + throw new Error("Fetcher must be a function"); + } + + const effectiveConfig = { ...this.config, ...options }; + + // Validate config + if (effectiveConfig.staleThresholdMs < 0) { + throw new Error("staleThresholdMs must be non-negative"); + } + if (effectiveConfig.expireThresholdMs < effectiveConfig.staleThresholdMs) { + throw new Error("expireThresholdMs must be >= staleThresholdMs"); + } + if (effectiveConfig.baseRefreshProbability < 0 || effectiveConfig.baseRefreshProbability > 1) { + throw new Error("baseRefreshProbability must be in [0, 1]"); + } + if (effectiveConfig.defaultTTL <= 0) { + throw new Error("defaultTTL must be positive"); + } + + const cacheKey = this._buildCacheKey(key); + const metaKey = this._buildMetaKey(key); + + try { + // Try to get cached entry + const cached = await safeGet(cacheKey); + const metaStr = await safeGet(metaKey); + + if (cached !== null && metaStr !== null) { + let meta: CacheMetadata; + try { + meta = JSON.parse(metaStr) as CacheMetadata; + } catch (parseError) { + logger.warn({ err: parseError, key }, "Cache metadata corrupted, forcing refresh"); + return this._forceRefresh(key, cacheKey, metaKey, fetcher, effectiveConfig); + } + + // Validate metadata structure + if (!meta || typeof meta.timestamp !== "number" || typeof meta.version !== "number") { + logger.warn({ key, meta }, "Invalid cache metadata structure, forcing refresh"); + return this._forceRefresh(key, cacheKey, metaKey, fetcher, effectiveConfig); + } + + const now = Date.now(); + const age = now - meta.timestamp; + + // Record access for probabilistic eviction engine + evictionEngine.recordAccess(key); + + // Check if data is expired (must refresh) + if (age > effectiveConfig.expireThresholdMs) { + logger.debug({ key, age }, "Cache entry expired, forcing refresh"); + return this._forceRefresh(key, cacheKey, metaKey, fetcher, effectiveConfig); + } + + // Data is stale but usable + if (age > effectiveConfig.staleThresholdMs) { + const shouldRefresh = this._shouldRefresh(key, age, effectiveConfig); + + if (shouldRefresh) { + // Trigger background refresh without blocking + this._backgroundRefresh(key, cacheKey, metaKey, fetcher, effectiveConfig); + } + + // Return stale data immediately + logger.debug({ key, age }, "Returning stale data"); + try { + return JSON.parse(cached) as T; + } catch (parseError) { + logger.warn({ err: parseError, key }, "Cache data corrupted, forcing refresh"); + return this._forceRefresh(key, cacheKey, metaKey, fetcher, effectiveConfig); + } + } + + // Data is fresh + logger.debug({ key, age }, "Cache hit (fresh)"); + try { + return JSON.parse(cached) as T; + } catch (parseError) { + logger.warn({ err: parseError, key }, "Cache data corrupted, forcing refresh"); + return this._forceRefresh(key, cacheKey, metaKey, fetcher, effectiveConfig); + } + } + + // Cache miss - fetch fresh data + logger.debug({ key }, "Cache miss, fetching fresh data"); + return this._forceRefresh(key, cacheKey, metaKey, fetcher, effectiveConfig); + } catch (error) { + logger.error({ err: error, key }, "Stale cache get failed, falling back to fetcher"); + // On any error, fall back to fetcher + try { + return await fetcher(); + } catch (fetcherError) { + logger.error({ err: fetcherError, key }, "Fetcher also failed, propagating error"); + throw fetcherError; + } + } + } + + /** + * Set data in cache with metadata. + * + * @param key - Cache key + * @param data - Data to cache + * @param ttl - TTL in seconds (overrides default) + */ + async set(key: string, data: T, ttl?: number): Promise { + // Input validation + if (!key || typeof key !== "string") { + throw new Error("Cache key must be a non-empty string"); + } + if (data === undefined) { + throw new Error("Cannot cache undefined value"); + } + if (ttl !== undefined && (ttl <= 0 || !Number.isFinite(ttl))) { + throw new Error("TTL must be a positive number"); + } + + const cacheKey = this._buildCacheKey(key); + const metaKey = this._buildMetaKey(key); + const effectiveTTL = ttl ?? this.config.defaultTTL; + + try { + const entry: CacheEntry = { + data, + timestamp: Date.now(), + version: 1, + }; + + const meta: CacheMetadata = { + timestamp: entry.timestamp, + version: entry.version, + }; + + await Promise.all([ + safeSet(cacheKey, JSON.stringify(entry.data), effectiveTTL), + safeSet(metaKey, JSON.stringify(meta), effectiveTTL), + ]); + + // Record write for probabilistic eviction + evictionEngine.recordAccess(key); + } catch (error) { + logger.error({ err: error, key }, "Stale cache set failed"); + } + } + + /** + * Invalidate a cache entry. + */ + async invalidate(key: string): Promise { + // Input validation + if (!key || typeof key !== "string") { + throw new Error("Cache key must be a non-empty string"); + } + + const cacheKey = this._buildCacheKey(key); + const metaKey = this._buildMetaKey(key); + + try { + const { safeDel } = await import("./cache.js"); + await Promise.all([safeDel(cacheKey), safeDel(metaKey)]); + + // Remove from pending refreshes if present + this.pendingRefreshes.delete(key); + } catch (error) { + logger.error({ err: error, key }, "Stale cache invalidate failed"); + } + } + + /** + * Get cache statistics for monitoring. + */ + getStats() { + return { + pendingRefreshes: this.pendingRefreshes.size, + config: this.config, + evictionEngine: { + memoryBytes: evictionEngine.memoryBytes, + totalAccesses: evictionEngine.totalAccesses, + }, + }; + } + + // ── Private Helpers ────────────────────────────────────────────────────── + + private _buildCacheKey(key: string): string { + return `stale:${key}`; + } + + private _buildMetaKey(key: string): string { + return `stale:meta:${key}`; + } + + /** + * Force a synchronous refresh (used for cache misses or expired entries). + */ + private async _forceRefresh( + key: string, + cacheKey: string, + metaKey: string, + fetcher: () => Promise, + config: Required + ): Promise { + const data = await fetcher(); + await this._storeEntry(cacheKey, metaKey, data, config.defaultTTL); + return data; + } + + /** + * Trigger an asynchronous background refresh. + * Multiple concurrent refreshes for the same key are deduplicated. + */ + private _backgroundRefresh( + key: string, + cacheKey: string, + metaKey: string, + fetcher: () => Promise, + config: Required + ): void { + // Deduplicate concurrent refreshes + if (this.pendingRefreshes.has(key)) { + return; + } + + const refreshPromise = (async () => { + try { + const data = await fetcher(); + await this._storeEntry(cacheKey, metaKey, data, config.defaultTTL); + logger.debug({ key }, "Background refresh completed"); + } catch (error) { + logger.error({ err: error, key }, "Background refresh failed"); + } finally { + this.pendingRefreshes.delete(key); + } + })(); + + this.pendingRefreshes.set(key, refreshPromise); + } + + /** + * Store an entry in cache with metadata. + */ + private async _storeEntry( + cacheKey: string, + metaKey: string, + data: T, + ttl: number + ): Promise { + const meta: CacheMetadata = { + timestamp: Date.now(), + version: 1, + }; + + await Promise.all([ + safeSet(cacheKey, JSON.stringify(data), ttl), + safeSet(metaKey, JSON.stringify(meta), ttl), + ]); + } + + /** + * Determine whether to refresh a stale entry based on: + * - Access frequency (from Count-Min Sketch) + * - Staleness (age vs thresholds) + * - Base refresh probability + */ + private _shouldRefresh( + key: string, + age: number, + config: Required + ): boolean { + if (!config.enableProbabilisticRefresh) { + return true; + } + + // Get frequency score from probabilistic eviction engine + const frequency = evictionEngine.getFrequency(key); + const totalAccesses = evictionEngine.totalAccesses; + + // Normalize frequency to [0, 1] + const frequencyScore = totalAccesses > 0 + ? Math.min(frequency / totalAccesses, 1) + : 0; + + // Calculate staleness score (0 = fresh, 1 = at expire threshold) + const stalenessScore = Math.min( + (age - config.staleThresholdMs) / + (config.expireThresholdMs - config.staleThresholdMs), + 1 + ); + + // Combine frequency and staleness for refresh probability + // Hot keys and stale data have higher refresh probability + const refreshProbability = config.baseRefreshProbability + + (frequencyScore * 0.4) + + (stalenessScore * 0.3); + + // Cap at 1.0 + const finalProbability = Math.min(refreshProbability, 1.0); + + // Probabilistic decision + return Math.random() < finalProbability; + } +} + +// ─── Singleton ─────────────────────────────────────────────────────────────── + +/** + * Global singleton for the stale cache service. + * Configuration can be overridden via environment variables: + * + * STALE_CACHE_THRESHOLD_MS — Stale threshold (default 30000) + * STALE_CACHE_EXPIRE_MS — Expire threshold (default 300000) + * STALE_CACHE_REFRESH_PROB — Base refresh probability (default 0.3) + * STALE_CACHE_DEFAULT_TTL — Default TTL in seconds (default 60) + */ +export const staleCacheService = new StaleCacheService({ + staleThresholdMs: parseInt(process.env["STALE_CACHE_THRESHOLD_MS"] ?? "30000", 10), + expireThresholdMs: parseInt(process.env["STALE_CACHE_EXPIRE_MS"] ?? "300000", 10), + baseRefreshProbability: parseFloat(process.env["STALE_CACHE_REFRESH_PROB"] ?? "0.3"), + defaultTTL: parseInt(process.env["STALE_CACHE_DEFAULT_TTL"] ?? "60", 10), + enableProbabilisticRefresh: process.env["STALE_CACHE_ENABLE_REFRESH"] !== "false", +}); diff --git a/packages/backend/src/services/webhookService.ts b/packages/backend/src/services/webhookService.ts index ac083920..74406610 100644 --- a/packages/backend/src/services/webhookService.ts +++ b/packages/backend/src/services/webhookService.ts @@ -14,6 +14,7 @@ import { type WebhookJobData, } from "../schemas/webhookJobSchemas.js"; import { bullRedisConnection } from "./cache.js"; +import { staleCacheService } from "./staleCache.js"; import { logger } from "../utils/logger.js"; import { calculateSignature as hmacCalculateSignature } from "../utils/signatureVerify.js"; @@ -89,24 +90,44 @@ export class WebhookService { /** * Retrieves the current webhook configuration for an organization. + * Uses stale cache for non-blocking reads during high load. * @param organizationId The ID of the organization. */ async getConfig(organizationId: string) { - return webhookRepository.getConfig(organizationId); + const cacheKey = `webhook:config:${organizationId}`; + + return staleCacheService.get( + cacheKey, + () => webhookRepository.getConfig(organizationId), + { + staleThresholdMs: 15000, // 15 seconds stale threshold for webhook configs + expireThresholdMs: 120000, // 2 minutes expire threshold + baseRefreshProbability: 0.2, + defaultTTL: 30, + } + ); } /** * Updates or creates a webhook URL configuration for an organization. + * Invalidates stale cache after update to ensure consistency. * @param organizationId The ID of the organization. * @param url The external HTTP POST endpoint. */ async updateConfig(organizationId: string, url: string) { const secret = await this.generateSecretForOrganization(organizationId); - return webhookRepository.upsertConfig(organizationId, url, secret); + const result = await webhookRepository.upsertConfig(organizationId, url, secret); + + // Invalidate cache to ensure consistency + const cacheKey = `webhook:config:${organizationId}`; + await staleCacheService.invalidate(cacheKey); + + return result; } /** * Dispatches a webhook asynchronously using BullMQ or SQS. + * Uses stale cache for config lookup to avoid blocking during high load. * @param organizationId The organization to notify. * @param event The event name. * @param data The payload data. @@ -116,7 +137,7 @@ export class WebhookService { event: string, data: WebhookEventData, ) { - const config = await webhookRepository.getConfig(organizationId); + const config = await this.getConfig(organizationId); if (!config || !config.url) { logger.debug( { organizationId, event }, @@ -212,10 +233,11 @@ export class WebhookService { /** * Dispatches a test webhook event. + * Uses stale cache for config lookup. * @param organizationId The ID of the organization. */ async sendTestWebhook(organizationId: string) { - const config = await webhookRepository.getConfig(organizationId); + const config = await this.getConfig(organizationId); if (!config || !config.url) { throw new Error("No webhook configuration found for this organization"); } diff --git a/packages/backend/src/trpc/staleCacheMiddleware.ts b/packages/backend/src/trpc/staleCacheMiddleware.ts new file mode 100644 index 00000000..5ea4d199 --- /dev/null +++ b/packages/backend/src/trpc/staleCacheMiddleware.ts @@ -0,0 +1,120 @@ +/** + * @file staleCacheMiddleware.ts + * @description Type-safe tRPC middleware with stale cache probabilistic eviction. + * + * This middleware provides stale-while-revalidate caching for tRPC procedures: + * - Returns stale data immediately to avoid blocking + * - Probabilistically refreshes cache in background + * - Maintains type safety across the tRPC boundary + * - Integrates with the existing probabilistic eviction engine + * + * IMPORTANT: Chain `.input()` before `.use(withStaleCache(...))` so parsed input + * is available when building the cache key. + */ + +import { staleCacheService, type StaleCacheConfig } from "../services/staleCache.js"; +import { logger } from "../utils/logger.js"; +import { t } from "./trpc.js"; + +/** + * Wraps a tRPC query procedure with stale-while-revalidate caching. + * Mutations bypass the cache entirely. + * + * @param buildKey - Function to build cache key from input + * @param config - Stale cache configuration (optional) + * @returns tRPC middleware + */ +export function withStaleCache( + buildKey: (input: TInput) => string, + config?: StaleCacheConfig +) { + return t.middleware(async ({ next, input, type }: { next: any; input: any; type: string }) => { + if (type !== "query") { + return next(); + } + + const key = buildKey(input as TInput); + + try { + // Use stale cache service for non-blocking reads + const result = await staleCacheService.get( + key, + async () => { + const procedureResult = await next(); + if (!procedureResult.ok) { + throw new Error("Procedure failed, cannot cache error"); + } + return procedureResult.data; + }, + config + ); + + return { + ok: true as const, + data: result as unknown, + marker: undefined as never, + }; + } catch (error) { + logger.error({ err: error, key }, "Stale cache middleware failed, falling through to procedure"); + + // Fall back to direct procedure call on cache failure + return next(); + } + }); +} + +/** + * Wraps a tRPC mutation procedure with cache invalidation. + * Automatically invalidates cache entries after successful mutations. + * + * @param buildKey - Function to build cache key from input + * @returns tRPC middleware + */ +export function withCacheInvalidation( + buildKey: (input: TInput) => string | string[] +) { + return t.middleware(async ({ next, input, type }: { next: any; input: any; type: string }) => { + const result = await next(); + + // Only invalidate on successful mutations + if (type === "mutation" && result.ok) { + try { + const keys = buildKey(input as TInput); + const keyArray = Array.isArray(keys) ? keys : [keys]; + + await Promise.all( + keyArray.map(key => staleCacheService.invalidate(key)) + ); + } catch (error) { + logger.error({ err: error }, "Cache invalidation failed"); + } + } + + return result; + }); +} + +/** + * Combines stale cache for queries with automatic invalidation for mutations. + * Useful for CRUD operations where mutations should invalidate related query caches. + * + * @param buildKey - Function to build cache key from input + * @param config - Stale cache configuration (optional) + * @returns Object with query and mutation middleware + */ +export function withStaleCacheAndInvalidation( + buildKey: (input: TInput) => string | string[], + config?: StaleCacheConfig +) { + const keyBuilder = typeof buildKey === "function" + ? (input: TInput) => { + const keys = buildKey(input); + return Array.isArray(keys) ? keys[0]! : keys; + } + : buildKey; + + return { + query: withStaleCache(keyBuilder, config), + mutation: withCacheInvalidation(buildKey), + }; +}