Skip to content

Commit da83693

Browse files
Petrus Pennanenclaude
authored andcommitted
poller: per-room watermark so evicted old ids never resurface as new (complements #95)
After #92 widened the fetch window, a restart replayed months-old messages in quiet rooms: their ids had fallen off the capped seen file, so they read as new (claudemm, 2 Sep). Fix that does not depend on the cap and does not drop a legitimate backlog: RoomHistory keeps a per-room watermark (newest created_at ever processed or seeded, persisted under "_marks" in the history file). A fetched message older than the watermark by more than 120 s and not in the seen set is marked seen and skipped, never notified. Anything newer than the watermark is delivered no matter how old, so a poller that was down for hours still delivers what it missed (#90 item 1's promise). seedRoom() sets the watermark; classifyFetched() is the exported decision; the loop logs how many stale messages it swallowed per room. Tests: unit (marks persist, stale/new/seen classification, tolerance, no watermark = nothing stale) and a live poller run with the curl stub: seed sets the mark, a wide window with three evicted-old messages and one new one notifies only the new one and remembers all ids. Suite 314/314. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
1 parent 93293f5 commit da83693

4 files changed

Lines changed: 152 additions & 4 deletions

File tree

‎src/common/room-history.mjs‎

Lines changed: 40 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,13 @@ import { replyIdOf } from './reply-context.mjs';
2424

2525
const BODY_KEEP = 600;
2626
export const STATE_PREFIX = /^\s*(?:state|settled)\s*:\s*(.+)$/i;
27+
const MARKS_KEY = '_marks';
28+
// A message this much older than the room's watermark, and not in the seen
29+
// set, is treated as already handled: the seen-id cap (2000 ids across all
30+
// rooms) evicts old ids, and a fetch window wider than 10 then re-surfaces
31+
// months-old messages in quiet rooms as "new" (claudemm, 2 Sep 2026, after
32+
// #92 raised the window to 25). The tolerance covers out-of-order arrival.
33+
export const STALE_TOLERANCE_S = 120;
2734

2835
function oneLine(s, n) {
2936
return String(s || '').replace(/\s+/g, ' ').trim().slice(0, n);
@@ -39,30 +46,61 @@ export class RoomHistory {
3946
this.path = path;
4047
this.maxPerRoom = maxPerRoom;
4148
this.rooms = {};
49+
this.marks = {}; // room -> newest created_at ever processed or seeded
4250
this.load();
4351
}
4452

4553
load() {
4654
try {
4755
const data = JSON.parse(readFileSync(this.path, 'utf8'));
48-
if (data && typeof data === 'object') this.rooms = data;
56+
if (data && typeof data === 'object') {
57+
const { [MARKS_KEY]: marks, ...rooms } = data;
58+
this.rooms = rooms;
59+
this.marks = (marks && typeof marks === 'object') ? marks : {};
60+
}
4961
} catch {
5062
this.rooms = {};
63+
this.marks = {};
5164
}
5265
}
5366

5467
save() {
5568
try {
5669
mkdirSync(dirname(this.path), { recursive: true });
5770
const tmp = this.path + '.tmp';
58-
writeFileSync(tmp, JSON.stringify(this.rooms));
71+
writeFileSync(tmp, JSON.stringify({ ...this.rooms, [MARKS_KEY]: this.marks }));
5972
renameSync(tmp, this.path);
6073
return true;
6174
} catch {
6275
return false;
6376
}
6477
}
6578

79+
/** Advance the room's watermark to `createdAt` if it is newer. */
80+
markProcessed(room, createdAt) {
81+
if (!room || !createdAt) return;
82+
const t = Date.parse(createdAt);
83+
if (!Number.isFinite(t)) return;
84+
const cur = Date.parse(this.marks[room] || '');
85+
if (!Number.isFinite(cur) || t > cur) this.marks[room] = new Date(t).toISOString();
86+
}
87+
88+
watermark(room) {
89+
return this.marks[room] || '';
90+
}
91+
92+
/**
93+
* True when a message is older than the room's watermark by more than the
94+
* tolerance: it predates everything already handled here, so it can only
95+
* be an evicted-from-seen resurfacing, never a genuinely new message.
96+
*/
97+
isStale(room, createdAt, toleranceS = STALE_TOLERANCE_S) {
98+
const mark = Date.parse(this.marks[room] || '');
99+
const t = Date.parse(createdAt || '');
100+
if (!Number.isFinite(mark) || !Number.isFinite(t)) return false;
101+
return t < mark - toleranceS * 1000;
102+
}
103+
66104
/** Add or refresh a fetched batch. Idempotent; keeps newest-last order. */
67105
remember(room, msgs) {
68106
if (!room || !Array.isArray(msgs) || msgs.length === 0) return;

‎src/team-relay/room-poller.mjs‎

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -176,11 +176,31 @@ export function seedRoom({ seen, history, room, msgs }) {
176176
let added = 0;
177177
for (const m of msgs || []) {
178178
if (m && m.id && !seen.has(m.id)) { seen.add(m.id); added++; }
179+
if (history && typeof history.markProcessed === 'function' && m && m.created_at) {
180+
history.markProcessed(room, m.created_at);
181+
}
179182
}
180183
if (history && typeof history.remember === 'function') history.remember(room, msgs || []);
181184
return added;
182185
}
183186

187+
/**
188+
* Is this fetched message one to handle, or an old one resurfacing? The
189+
* seen set is capped, so a wide fetch window can show months-old messages
190+
* whose ids were evicted; the room watermark catches those (they are older
191+
* than everything already handled) and they are marked seen, never
192+
* notified. Exported for the regression test.
193+
*/
194+
export function classifyFetched({ seen, history, room, m }) {
195+
if (!m || !m.id) return 'skip';
196+
if (seen.has(m.id)) return 'seen';
197+
if (history && typeof history.isStale === 'function' && history.isStale(room, m.created_at)) {
198+
seen.add(m.id);
199+
return 'stale';
200+
}
201+
return 'new';
202+
}
203+
184204
export async function startRoomPoller({ rooms, apiKey, handle, interval, config, sessionOpt }) {
185205
const seenFile = config?.poller?.seen_file || SEEN_FILE_DEFAULT;
186206
const heartbeatFile = config?.poller?.heartbeat_file || HEARTBEAT_FILE_DEFAULT;
@@ -312,10 +332,13 @@ export async function startRoomPoller({ rooms, apiKey, handle, interval, config,
312332
const roomLines = [];
313333
let roomPriority = false;
314334
let roomHeaderWritten = false;
335+
let staleCount = 0;
315336
for (const m of msgs) {
316337
const mid = m.id;
317-
if (!mid || seen.has(mid)) continue;
338+
const kind = classifyFetched({ seen, history, room, m });
339+
if (kind !== 'new') { if (kind === 'stale') staleCount++; continue; }
318340
seen.add(mid);
341+
history.markProcessed(room, m.created_at);
319342

320343
const sender = m.from || m.sender || '?';
321344
const normalizedSender = normalizeHandle(sender);
@@ -386,6 +409,10 @@ export async function startRoomPoller({ rooms, apiKey, handle, interval, config,
386409
// Lines and the context header were written per message above;
387410
// newMessages only feeds the count/log below.
388411
newMessages.push(...roomLines);
412+
if (staleCount) {
413+
console.log(` ${room}: ${staleCount} old message(s) below the watermark ${history.watermark(room)} marked seen, not notified`);
414+
saveSeenIds(seenFile, seen);
415+
}
389416
}
390417

391418
saveSeenIds(seenFile, seen);

‎test/room-history.test.mjs‎

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import { tmpdir } from 'node:os';
77
import { join } from 'node:path';
88
import { RoomHistory, threadSuffix, previousSuffix, stateOfPlayLine, ownLastPostLine } from '../src/common/room-history.mjs';
99
import { resolveReplyTargets } from '../src/common/reply-context.mjs';
10-
import { seedRoom } from '../src/team-relay/room-poller.mjs';
10+
import { seedRoom, classifyFetched } from '../src/team-relay/room-poller.mjs';
1111

1212
const R = 'thinkoff-development';
1313
function msg(id, from, body, created_at, reply_to) {
@@ -131,3 +131,35 @@ describe('first-run seed keeps the thread (codexmb, PR #92)', () => {
131131
assert.ok(threadSuffix(h, R, batch[0]).includes('Card v2 for your review'));
132132
});
133133
});
134+
135+
136+
describe('room watermark: evicted old ids must not resurface as new (claudemm, 2 Sep)', () => {
137+
it('marks advance on seed and processing, persist, and classify stale vs new', () => {
138+
const h = fresh();
139+
const seen = new Set();
140+
seedRoom({ seen, history: h, room: R, msgs: [
141+
msg('s1', 'petrus', 'old one', '2026-06-01T10:00:00Z'),
142+
msg('s2', 'petrus', 'old two', '2026-08-31T12:00:00Z')
143+
] });
144+
assert.equal(h.watermark(R), '2026-08-31T12:00:00.000Z');
145+
assert.ok(h.save());
146+
const again = new RoomHistory(h.path);
147+
assert.equal(again.watermark(R), '2026-08-31T12:00:00.000Z', 'watermark survives restart');
148+
assert.equal(again.get(R, 's2').body, 'old two', 'rooms still load beside the marks');
149+
// the seen cap evicted every old id: a wide window now shows a February message
150+
const evicted = new Set();
151+
const feb = msg('feb', 'petrus', 'from february', '2026-02-10T09:00:00Z');
152+
assert.equal(classifyFetched({ seen: evicted, history: again, room: R, m: feb }), 'stale');
153+
assert.ok(evicted.has('feb'), 'stale message is marked seen so it never comes back');
154+
// a genuinely new message is new; one 60 s before the mark is within tolerance
155+
assert.equal(classifyFetched({ seen: evicted, history: again, room: R, m: msg('n1', 'petrus', 'new', '2026-09-02T14:50:00Z') }), 'new');
156+
assert.equal(classifyFetched({ seen: evicted, history: again, room: R, m: msg('n2', 'petrus', 'late arrival', '2026-08-31T11:59:00Z') }), 'new');
157+
evicted.add('n2'); // the loop adds a NEW id after classifying it
158+
assert.equal(classifyFetched({ seen: evicted, history: again, room: R, m: msg('n2', 'petrus', 'dup', '2026-08-31T11:59:00Z') }), 'seen');
159+
again.markProcessed(R, '2026-09-02T14:50:00Z');
160+
again.markProcessed(R, '2026-09-01T00:00:00Z'); // older: never moves the mark back
161+
assert.equal(again.watermark(R), '2026-09-02T14:50:00.000Z');
162+
// no watermark yet (fresh poller, first run): nothing is stale
163+
assert.equal(classifyFetched({ seen: new Set(), history: fresh(), room: R, m: feb }), 'new');
164+
});
165+
});

‎test/watermark-live.test.mjs‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
// SPDX-License-Identifier: AGPL-3.0-only
2+
// Regression (claudemm, 2 Sep 2026): after #92 widened the fetch window, a
3+
// restart replayed months-old messages whose ids the seen cap had evicted.
4+
import assert from 'node:assert/strict';
5+
import { mkdtempSync, rmSync, writeFileSync, readFileSync, chmodSync, mkdirSync } from 'node:fs';
6+
import { tmpdir } from 'node:os';
7+
import path from 'node:path';
8+
import test from 'node:test';
9+
import { loadSeenIds } from '../src/common/seen-ids.mjs';
10+
import { startRoomPoller } from '../src/team-relay/room-poller.mjs';
11+
12+
test('old messages below the room watermark are marked seen and never notified, new ones still are', async () => {
13+
const dir = mkdtempSync(path.join(tmpdir(), 'iak-wm-'));
14+
const savedPath = process.env.PATH;
15+
let timers;
16+
try {
17+
const stubDir = path.join(dir, 'bin'); mkdirSync(stubDir);
18+
// seed (limit=50): two messages from 31 Aug set the watermark.
19+
// first poll (limit=25): the wide window now shows three February/June
20+
// messages with ids nobody has seen (evicted from the seen cap on a real
21+
// box) plus one genuinely new message.
22+
writeFileSync(path.join(stubDir, 'curl'), `#!/bin/sh
23+
case "$*" in
24+
*limit=50*) echo '[{"id":"a1","from":"petrus","body":"old aug one","created_at":"2026-08-31T12:00:00Z"},{"id":"a2","from":"petrus","body":"old aug two","created_at":"2026-08-31T12:05:00Z"}]';;
25+
*) echo '[{"id":"n1","from":"petrus","body":"genuinely new","created_at":"2026-09-02T14:50:00Z"},{"id":"a2","from":"petrus","body":"old aug two","created_at":"2026-08-31T12:05:00Z"},{"id":"f1","from":"petrus","body":"from february","created_at":"2026-02-10T09:00:00Z"},{"id":"f2","from":"@x","body":"from june","created_at":"2026-06-03T09:00:00Z"},{"id":"f3","from":"petrus","body":"from may","created_at":"2026-05-01T09:00:00Z"}]';;
26+
esac
27+
`);
28+
chmodSync(path.join(stubDir, 'curl'), 0o755);
29+
process.env.PATH = `${stubDir}:${savedPath}`;
30+
const seenFile = path.join(dir, 'seen'); const notifyFile = path.join(dir, 'notify');
31+
const origLog = console.log; console.log = () => {};
32+
try {
33+
timers = await startRoomPoller({
34+
rooms: ['r'], apiKey: 'k', handle: '@t', interval: 3600,
35+
config: { poller: { seen_file: seenFile, notification_file: notifyFile, heartbeat_file: path.join(dir, 'hb'), nudge_mode: 'none', owner_handle: 'petrus', history_file: path.join(dir, 'hist.json') }, queue: { path: path.join(dir, 'q.jsonl') } }
36+
});
37+
} finally { console.log = origLog; }
38+
const lines = readFileSync(notifyFile, 'utf8').split('\n').filter(Boolean);
39+
assert.equal(lines.filter((l) => /petrus: genuinely new/.test(l)).length, 1, `the new message is notified once: ${lines.join(' | ')}`);
40+
assert.ok(!lines.some((l) => /from february|from june|from may/.test(l)), `old messages never reach the notification file: ${lines.join(' | ')}`);
41+
const seen = loadSeenIds(seenFile);
42+
for (const id of ['a1', 'a2', 'n1', 'f1', 'f2', 'f3']) assert.ok(seen.has(id), `${id} is in the seen file`);
43+
const hist = JSON.parse(readFileSync(path.join(dir, 'hist.json'), 'utf8'));
44+
assert.equal(hist._marks.r, '2026-09-02T14:50:00.000Z', 'watermark advanced to the new message');
45+
} finally {
46+
if (timers?.roomTimer) clearInterval(timers.roomTimer);
47+
if (timers?.dmTimer) clearInterval(timers.dmTimer);
48+
process.env.PATH = savedPath;
49+
rmSync(dir, { recursive: true, force: true });
50+
}
51+
});

0 commit comments

Comments
 (0)