Merge pull request #21 from HarperFast/spread-render-schedule

harper-joseph · web-flow · commit 992155e67c54 · 2026-07-08T15:31:41.000-04:00
feat: spread render schedule + claim-per-capacity (plugin v0.4.0, browser v1.7.0)
diff --git a/package-lock.json b/package-lock.json
diff --git a/packages/browser/package.json b/packages/browser/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "@harperfast/prerender-browser",
-	"version": "1.6.0",
+	"version": "1.7.0",
 	"type": "module",
 	"description": "Headless-browser render library for Harper Prerender: claims render jobs from the @harperfast/prerender queue, renders pages in headless Chrome (Puppeteer), and posts the HTML back. Embedded by a render service and configured entirely via startWorker() options.",
 	"keywords": [
diff --git a/packages/browser/src/settings.ts b/packages/browser/src/settings.ts
@@ -229,7 +229,10 @@ export const applySettings = (options: BrowserOptions): Settings => {
 	fresh.config = typeof options.config === 'string' ? loadConfig(options.config) : mergeConfig(options.config ?? {});
 	fresh.concurrency = options.concurrency ?? fresh.concurrency;
 	fresh.rps = options.rps ?? fresh.rps;
-	fresh.jobClaimLimit = options.jobClaimLimit ?? fresh.concurrency * 2;
+	// Claim at most what this worker can actually render at once. Over-claiming (the old
+	// concurrency*2) made one worker grab jobs it couldn't start and hold them leased-idle,
+	// starving other renderers of a burst and doubling the per-claim lease-write transaction.
+	fresh.jobClaimLimit = options.jobClaimLimit ?? fresh.concurrency;
 	fresh.browserExpirationThreshold = options.browserExpirationThreshold ?? fresh.browserExpirationThreshold;
 	fresh.incognitoPages = options.incognitoPages ?? fresh.incognitoPages;
 	fresh.contentEncoding = options.contentEncoding ?? fresh.contentEncoding;
diff --git a/packages/browser/test/settings.test.ts b/packages/browser/test/settings.test.ts
@@ -17,7 +17,7 @@ test('applies defaults and derives jobClaimLimit from concurrency', () => {
 	applySettings({ harper: HARPER, concurrency: 4 });
 	assert.equal(settings.harper.workerId, 'w1');
 	assert.equal(settings.concurrency, 4);
-	assert.equal(settings.jobClaimLimit, 8); // concurrency * 2
+	assert.equal(settings.jobClaimLimit, 4); // defaults to concurrency
 	assert.equal(settings.queuePort, 9926);
 	assert.equal(settings.bypass.header, 'x-harper-renderer-bypass');
 	assert.equal(settings.contentEncoding, 'gzip');
diff --git a/packages/plugin/README.md b/packages/plugin/README.md
@@ -79,9 +79,7 @@ rest: true # required for the @export-ed table REST endpoints
     swrTtl: 10800000 # 3h  — stale-while-revalidate window
 
   render:
-    defaultInterval: 86400000 # 24h — how often a target is re-rendered
-    time: '07:00' # local time-of-day for the daily render run
-    timezone: America/New_York
+    defaultInterval: 86400000 # 24h — how often a target is re-rendered (relative to completion)
 
   sitemap:
     refreshTime: '12:00' # local time-of-day for the daily sitemap refresh
diff --git a/packages/plugin/package.json b/packages/plugin/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "@harperfast/prerender",
-	"version": "0.3.6",
+	"version": "0.4.0",
 	"type": "module",
 	"description": "Configurable Harper plugin for prerendering pages for bots and crawlers",
 	"license": "Apache-2.0",
diff --git a/packages/plugin/src/config.js b/packages/plugin/src/config.js
@@ -163,9 +163,11 @@ const defaultConfig = () => ({
 	},
 
 	render: {
-		defaultInterval: DAY, // how often a target is re-rendered
-		time: '07:00', // local time-of-day for the daily render run
-		timezone: 'America/New_York',
+		// How often a target is re-rendered. Cadence is relative to each render's
+		// completion (not a fixed time-of-day), and a target's first render is jittered
+		// across this interval — so the fleet renders as a smooth stream rather than a
+		// daily herd. Sitemap-derived targets override this per-URL from `changefreq`.
+		defaultInterval: DAY,
 	},
 
 	sitemap: {
@@ -180,6 +182,11 @@ const defaultConfig = () => ({
 	queue: {
 		jobLeaseTime: 10 * MINUTE, // how long a claimed job is leased before re-claim
 		statusSyncInterval: MINUTE, // how often queue status is recomputed/broadcast
+		// Hard ceiling on jobs granted per claim, regardless of what a consumer asks for.
+		// Each claimed job costs a lease write held under the claim mutex, so this bounds
+		// the per-claim transaction (keeps one greedy/misconfigured worker from grabbing a
+		// huge batch — long lock hold + starving other renderers of the burst).
+		maxClaimLimit: 25,
 	},
 
 	// Per-device-type User-Agent strings sent to the origin.
diff --git a/packages/plugin/src/http_handlers/bot_request.js b/packages/plugin/src/http_handlers/bot_request.js
@@ -176,9 +176,10 @@ async function handlePageScheduling(resource) {
 					const cacheKey = CacheKey.toCacheKey({ url: cacheKeyUrl(resource.url), deviceType });
 					const existingTarget = await RenderTarget.get({ id: cacheKey, select: 'cacheKey' });
 					if (!existingTarget) {
+						// No explicit time → RenderTarget.put jitters the first render across the
+						// interval, so a crawl that discovers many URLs at once doesn't stampede.
 						await RenderTarget.put(cacheKey, {
 							renderInterval: config.render.defaultInterval,
-							nextRenderTime: currentMinuteMs(),
 						});
 					}
 				}
diff --git a/packages/plugin/src/resources/RenderQueue.js b/packages/plugin/src/resources/RenderQueue.js
@@ -1,6 +1,6 @@
 import { getMutex } from '../util/coordination.js';
 import { config } from '../config.js';
-import { currentMinuteMs, getNextRenderTime } from '../util/time.js';
+import { currentMinuteMs } from '../util/time.js';
 import { QueueState } from './QueueState.js';
 import { CacheKey } from '../util/cacheKey.js';
 import { cacheKeyUrl, normalizeUrl } from '../util/url.js';
@@ -112,7 +112,18 @@ export class RenderQueue extends Resource {
 			const renderTarget = await RenderTarget.get({ id: cacheKey, select: ['renderInterval', 'sitemapUrl'] });
 			const renderInterval = renderTarget?.renderInterval;
 
-			const nextRenderTime = getNextRenderTime();
+			// Schedule the next render relative to when THIS one completed (now), not a
+			// fixed wall-clock time — so renders stay spread across the interval instead of
+			// realigning into a daily herd, and the cadence self-paces to fleet throughput.
+			// The per-target renderInterval drives the recurring cadence; fall back to the
+			// default when a target exists without a valid interval (a bare number check also
+			// rejects NaN from an arbitrary API PUT).
+			const interval =
+				Number.isFinite(renderInterval) && renderInterval > 0 ? renderInterval : config.render.defaultInterval;
+			// The cached page expires when the next render is due; the swrTtl window then keeps
+			// it served while the re-render lands, so render latency up to swrTtl never causes
+			// a cache miss.
+			const nextRenderTime = currentMinuteMs() + interval;
 
 			if (result.content) {
 				result.headers['x-harper-rendered'] = '1';
@@ -126,11 +137,14 @@ export class RenderQueue extends Resource {
 				});
 			}
 
-			if (typeof renderInterval === 'number' && renderInterval > 0) {
-				// Refresh fromSitemap from the live target so it self-corrects if the URL
-				// has since left its sitemap.
+			if (renderTarget) {
+				// A target owns this schedule → recurring. Reschedule relative to completion
+				// using the resolved interval (so a target lacking an explicit renderInterval
+				// falls back to the default instead of getting stuck re-claiming every lease
+				// period). Refresh fromSitemap from the live target so it self-corrects if the
+				// URL has since left its sitemap.
 				await RenderSchedule.put(cacheKey, { nextRenderTime, fromSitemap: !!renderTarget.sitemapUrl });
-			} else if (!renderTarget) {
+			} else {
 				// No target owns this schedule: it's a one-off (render-now) or an orphaned
 				// row. Nothing sets a recurring cadence, so drop the schedule instead of
 				// leaving it to be re-claimed when the lease expires.
@@ -169,6 +183,11 @@ export class RenderQueue extends Resource {
 			return [];
 		}
 
+		// Bound the batch server-side so no consumer can over-claim: a large grant means a
+		// large lease-write burst held under this mutex (long lock hold) and lets one worker
+		// hoard a burst other renderers should share.
+		limit = Math.min(Math.max(1, limit | 0), config.queue.maxClaimLimit);
+
 		const currentMinute = currentMinuteMs();
 		// Fully drain the search (read) transaction into memory BEFORE issuing any
 		// RenderSchedule.put leases. Interleaving the puts inside the `for await` keeps the
diff --git a/packages/plugin/src/resources/RenderTarget.js b/packages/plugin/src/resources/RenderTarget.js
@@ -1,7 +1,7 @@
 import { config } from '../config.js';
 import { CacheKey } from '../util/cacheKey.js';
 import { getResidencyByUrl } from '../util/residency.js';
-import { currentMinuteMs, getNextRenderTime } from '../util/time.js';
+import { currentMinuteMs, getInitialRenderTime } from '../util/time.js';
 import { setImmediate } from 'node:timers/promises';
 
 const {
@@ -28,8 +28,20 @@ export class RenderTarget extends databases.render_service.RenderTarget {
 		// schedule — is benign and self-heals on the next sitemap refresh / revalidate.
 		const result = await super.put({ ...CacheKey.parse(cacheKey), ...data }, target);
 
+		// Absent a valid explicit time, jitter the first render across the interval (keyed
+		// off the cacheKey) so bulk-created targets don't all come due at once. RenderTarget
+		// is API-exposed, so validate the numbers (reject negatives / NaN / non-numbers)
+		// rather than trust the payload.
+		const interval =
+			Number.isFinite(data.renderInterval) && data.renderInterval > 0
+				? data.renderInterval
+				: config.render.defaultInterval;
+
 		await RenderSchedule.put(cacheKey, {
-			nextRenderTime: typeof nextRenderTime === 'number' ? nextRenderTime : getNextRenderTime(),
+			nextRenderTime:
+				Number.isFinite(nextRenderTime) && nextRenderTime > 0
+					? nextRenderTime
+					: getInitialRenderTime(cacheKey, interval),
 			fromSitemap: !!data.sitemapUrl,
 		});
 
diff --git a/packages/plugin/src/resources/Sitemap.js b/packages/plugin/src/resources/Sitemap.js
@@ -70,11 +70,10 @@ class Sitemap extends databases.sitemaps.Sitemap {
 
 						for (const deviceType of deviceTypes) {
 							let updateTarget = false;
-							let revalidateNow = revalidate;
 
 							const cacheKey = CacheKey.toCacheKey({ url, deviceType });
 
-							if (revalidateNow) {
+							if (revalidate) {
 								updateTarget = true;
 								created++;
 							} else {
@@ -90,15 +89,17 @@ class Sitemap extends databases.sitemaps.Sitemap {
 								} else {
 									created++;
 									updateTarget = true;
-									revalidateNow = true;
 								}
 							}
 
 							if (updateTarget) {
+								// Explicit revalidate renders now; a newly-discovered target omits the
+								// time so RenderTarget.put jitters its first render across the interval,
+								// keeping bulk sitemap population from stampeding the queue.
 								lastPromise = RenderTarget.put(cacheKey, {
 									renderInterval,
 									sitemapUrl,
-									nextRenderTime: revalidateNow ? currentMinuteMs() : undefined,
+									nextRenderTime: revalidate ? currentMinuteMs() : undefined,
 								});
 								inflightCount++;
 							} else {
diff --git a/packages/plugin/src/util/hash.js b/packages/plugin/src/util/hash.js
@@ -0,0 +1,11 @@
+// FNV-1a 32-bit (fast, deterministic). Used for node-residency rendezvous hashing
+// and for deterministic per-key render-schedule jitter.
+export function fnv1a32(str) {
+	const s = String(str ?? '');
+	let h = 0x811c9dc5;
+	for (let i = 0; i < s.length; i++) {
+		h ^= s.charCodeAt(i);
+		h = (h + (h << 1) + (h << 4) + (h << 7) + (h << 8) + (h << 24)) >>> 0;
+	}
+	return h >>> 0;
+}
diff --git a/packages/plugin/src/util/residency.js b/packages/plugin/src/util/residency.js
@@ -1,12 +1,4 @@
-// FNV-1a 32-bit (fast, deterministic)
-function fnv1a32(str) {
-	let h = 0x811c9dc5;
-	for (let i = 0; i < str.length; i++) {
-		h ^= str.charCodeAt(i);
-		h = (h + (h << 1) + (h << 4) + (h << 7) + (h << 8) + (h << 24)) >>> 0;
-	}
-	return h >>> 0;
-}
+import { fnv1a32 } from './hash.js';
 
 export const nodes = [server.hostname, ...(server.nodes?.map(({ name }) => name) ?? [])].sort();
 
diff --git a/packages/plugin/src/util/time.js b/packages/plugin/src/util/time.js
@@ -1,4 +1,5 @@
 import { config } from '../config.js';
+import { fnv1a32 } from './hash.js';
 
 export const SECOND = 1000;
 export const MINUTE = 60 * SECOND;
@@ -44,6 +45,20 @@ export const getNextTimeOfDay = (timeStr, timezone) => {
 	return currentMinuteMs(tzNow.getTime() + targetOffset);
 };
 
-export const getNextRenderTime = () => getNextTimeOfDay(config.render.time, config.render.timezone);
+/**
+ * Deterministic first-render time: `now` plus a per-key offset in `[0, interval)`,
+ * floored to the minute. Spreads the initial render of freshly-scheduled targets
+ * across the render interval instead of firing them all at once (the thundering
+ * herd on bulk sitemap population / crawl spikes). The offset is keyed off the
+ * cacheKey so it's stable and reproducible. Recurring re-renders are scheduled
+ * relative to render completion (see RenderQueue.processJobResult), so this initial
+ * spread is preserved cycle over cycle rather than realigning to a fixed instant.
+ */
+export const getInitialRenderTime = (key, interval) => {
+	// Guard against a zero/negative/NaN interval (which would make the modulo NaN and
+	// schedule an invalid time) so the helper is safe to call with unvalidated inputs.
+	const safeInterval = Number.isFinite(interval) && interval > 0 ? interval : config.render.defaultInterval;
+	return currentMinuteMs(Date.now() + (fnv1a32(key) % safeInterval));
+};
 
 export const getNextSitemapRefreshTime = () => getNextTimeOfDay(config.sitemap.refreshTime, config.sitemap.timezone);
diff --git a/packages/plugin/test/time.test.js b/packages/plugin/test/time.test.js
@@ -1,7 +1,7 @@
 import { test, beforeEach } from 'node:test';
 import assert from 'node:assert/strict';
 import { applyOptions } from '../src/config.js';
-import { currentMinuteMs, getNextTimeOfDay, getNextRenderTime, MINUTE } from '../src/util/time.js';
+import { currentMinuteMs, getNextTimeOfDay, getInitialRenderTime, MINUTE } from '../src/util/time.js';
 
 beforeEach(() => applyOptions({}));
 
@@ -24,14 +24,22 @@ test('getNextTimeOfDay tolerates a missing minute component', () => {
 	assert.ok(next > Date.now());
 });
 
-test('getNextRenderTime reflects configured render time/timezone', () => {
-	applyOptions({ render: { time: '03:30', timezone: 'UTC' } });
-	const next = getNextRenderTime();
-	const asUtc = new Date(next);
-	// minute-aligned and in the future
-	assert.equal(next % MINUTE, 0);
-	assert.ok(next > Date.now());
-	// the UTC time-of-day should be 03:30
-	assert.equal(asUtc.getUTCHours(), 3);
-	assert.equal(asUtc.getUTCMinutes(), 30);
+test('getInitialRenderTime is minute-aligned within [now, now+interval)', () => {
+	const interval = 24 * 60 * MINUTE; // a day
+	const base = currentMinuteMs();
+	const t = getInitialRenderTime('https://x.test/a|desktop', interval);
+	assert.equal(t % MINUTE, 0);
+	assert.ok(t >= base, 'not scheduled before now');
+	assert.ok(t < base + interval + MINUTE, 'within the render interval');
+});
+
+test('getInitialRenderTime is stable per key and spreads across keys', () => {
+	const interval = 24 * 60 * MINUTE;
+	const a = getInitialRenderTime('key-a', interval);
+	// Same key resolves to the same minute (deterministic offset; allow a 1-minute
+	// window in case the wall clock ticks over a minute between the two calls).
+	assert.ok(Math.abs(getInitialRenderTime('key-a', interval) - a) <= MINUTE, 'stable for a given key');
+	// Distinct keys spread across the interval rather than collapsing to one time.
+	const values = new Set(Array.from({ length: 100 }, (_, i) => getInitialRenderTime(`key-${i}`, interval)));
+	assert.ok(values.size > 1, 'distinct keys spread across times');
 });

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "@harperfast/prerender-browser",`
`3`		`- "version": "1.6.0",`
	`3`	`+ "version": "1.7.0",`
`4`	`4`	`"type": "module",`
`5`	`5`	`"description": "Headless-browser render library for Harper Prerender: claims render jobs from the @harperfast/prerender queue, renders pages in headless Chrome (Puppeteer), and posts the HTML back. Embedded by a render service and configured entirely via startWorker() options.",`
`6`	`6`	`"keywords": [`
Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "@harperfast/prerender",`
`3`		`- "version": "0.3.6",`
	`3`	`+ "version": "0.4.0",`
`4`	`4`	`"type": "module",`
`5`	`5`	`"description": "Configurable Harper plugin for prerendering pages for bots and crawlers",`
`6`	`6`	`"license": "Apache-2.0",`
Original file line number	Diff line number	Diff line change
`@@ -176,9 +176,10 @@ async function handlePageScheduling(resource) {`
`176`	`176`	`const cacheKey = CacheKey.toCacheKey({ url: cacheKeyUrl(resource.url), deviceType });`
`177`	`177`	`const existingTarget = await RenderTarget.get({ id: cacheKey, select: 'cacheKey' });`
`178`	`178`	`if (!existingTarget) {`
	`179`	`+ // No explicit time → RenderTarget.put jitters the first render across the`
	`180`	`+ // interval, so a crawl that discovers many URLs at once doesn't stampede.`
`179`	`181`	`await RenderTarget.put(cacheKey, {`
`180`	`182`	`renderInterval: config.render.defaultInterval,`
`181`		`- nextRenderTime: currentMinuteMs(),`
`182`	`183`	`});`
`183`	`184`	`}`
`184`	`185`	`}`