Skip to content

Commit 52a5d33

Browse files
ryan-williamsclaude
andcommitted
gbfs/cascade: consolidate-the-dust same-tier sources (fix midnight wedge)
`writeSameTierCascade` sourced a rigid prev-rung × k key set, but the min-cover dust never materializes the LAST constituent of a closing rung — it closes and is superseded in the same tick (`5min@00:35` for `10min@00:30`, `30min@00:30` for `1h@00:00`, ...), so it's never in any tick's missing list and never written. Result: every consolidation from the first 10min rung up wedged with `no_inputs` — prod stuck from 2026-07-09T00:00Z (the first boundary after the pure-DAG deploy) with only the `/1m@5min` tail advancing. New source rule = "produce the shard from the dust": - `planDustTiling`: greedy largest-first tiling of the period from EXISTING finer same-tier shards (R2 HEAD probes, largest aligned rung first). Steady state leaves exactly ONE hole of finest-rung width at the period tail. - Holes fill from the tier's upstream source: raw WAL for `/1m` (buffered+sorted, capped at `MAX_RAW_FILL_MIN=30`), or `sourceTierFor`'s cover shards for `/Nm` (clipped per-hole via `clipDtRange` so they can't double-count rows the tiles already cover; `kwayMerge`'s tier-bin floor rebins them exactly). - Fragmented dust (outage/wedge scar, > `MAX_TILES=32` tiles+holes): abandon the tiling wholesale and fill the WHOLE period as one hole — pure upstream fill with fewer, larger sources. For `/1m` this defers to the raw cap (offline fsck beyond 30 min); for `/Nm` the source tier's cover is exactly the right-sized read, bounded by the existing `MAX_SOURCE_BYTES` / `MAX_OUTPUT_ROWS` guards. Same tier → same bin, so tiled sources merge with no rebin (tier-bin floor is an identity on aligned rows). Verified in prod: post-deploy converge healed the recent window (`1m/1h/22:00` wrote 36/36 = 6 surviving first-half 5min tiles + 30 raw minutes; `/2m@1h` × 2 via cross-tier fill), and after a local `ctbk pyramid-cascade --fsck --fill` closed the wedge-day scars the CFW converged the rest — including its first midnight boundary — to `totalMissing: 0` / `/api/health` `allComplete: true`. Also: extract shared `readRawRows` (raw-WAL read + LUC expand) used by `writeRawIngest` + hole-fill, move the ranged-R2 test mock to `mock-r2.ts` (shared by `aggregate.test.ts` + new `cascade-write.test.ts`), export `writeShard`/`planDustTiling` for tests. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
1 parent 2fbab77 commit 52a5d33

4 files changed

Lines changed: 586 additions & 130 deletions

File tree

gbfs/cascade/src/avail3/aggregate.test.ts

Lines changed: 2 additions & 89 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
1-
import { describe, test, expect, vi } from 'vitest';
1+
import { describe, test, expect } from 'vitest';
22
import { parquetReadObjects } from 'hyparquet';
33
import { kwayMerge, aggregateStream, streamShardRows, writeShardStreaming, MultipartR2Writer } from './aggregate';
44
import { sortRows, type AvailV3Row } from './transform';
5+
import { makeR2 } from './mock-r2';
56

67
// Convenience: build a wide row with per-metric histograms.
78
function row(s2_cell: string, dt: bigint, hists: Partial<Record<'bikes' | 'ebikes' | 'docks' | 'disabled' | 'pending', Record<number, number>>> = {}): AvailV3Row {
@@ -157,94 +158,6 @@ describe('aggregateStream', () => {
157158
});
158159
});
159160

160-
// ─── Ranged R2 mock ────────────────────────────────────────────────
161-
// Supports { range: { offset, length } } — required by streamShardRows'
162-
// r2SlicedBuffer wrapper. Real R2 always returns an ArrayBuffer from
163-
// R2Object.arrayBuffer(); normalize on put+get so callers can put
164-
// either ArrayBuffer or a Uint8Array view (e.g., ByteWriter.getBytes()).
165-
// Also implements minimal `createMultipartUpload` API used by
166-
// `MultipartR2Writer` — uploaded parts accumulate in-memory and are
167-
// concatenated in `complete()`.
168-
function makeR2() {
169-
const store = new Map<string, ArrayBuffer>();
170-
const multiparts = new Map<string, { key: string; parts: Map<number, Uint8Array>; aborted: boolean }>();
171-
let uploadCounter = 0;
172-
return {
173-
head: vi.fn(async (key: string) => (store.has(key) ? { key, size: store.get(key)!.byteLength } : null)),
174-
get: vi.fn(async (key: string, opts?: { range?: { offset: number; length: number } }) => {
175-
const buf = store.get(key);
176-
if (!buf) return null;
177-
const slice = opts?.range
178-
? buf.slice(opts.range.offset, opts.range.offset + opts.range.length)
179-
: buf;
180-
return { arrayBuffer: async () => slice };
181-
}),
182-
put: vi.fn(async (key: string, body: ArrayBuffer | Uint8Array) => {
183-
// Normalize to a self-contained ArrayBuffer (Uint8Array views may
184-
// wrap a larger backing buffer; store only the referenced bytes).
185-
if (body instanceof Uint8Array) {
186-
const copy = new ArrayBuffer(body.byteLength);
187-
new Uint8Array(copy).set(body);
188-
store.set(key, copy);
189-
} else {
190-
store.set(key, body);
191-
}
192-
}),
193-
createMultipartUpload: vi.fn(async (key: string) => {
194-
const uploadId = `upload-${++uploadCounter}`;
195-
const state = { key, parts: new Map<number, Uint8Array>(), aborted: false };
196-
multiparts.set(uploadId, state);
197-
return {
198-
uploadId,
199-
key,
200-
uploadPart: vi.fn(async (partNumber: number, body: Uint8Array) => {
201-
if (state.aborted) throw new Error(`upload ${uploadId} aborted`);
202-
// Copy — caller may recycle its buffer immediately.
203-
const copy = new Uint8Array(body.byteLength);
204-
copy.set(body);
205-
state.parts.set(partNumber, copy);
206-
return { partNumber, etag: `etag-${uploadId}-${partNumber}` };
207-
}),
208-
complete: vi.fn(async (parts: Array<{ partNumber: number; etag: string }>) => {
209-
if (state.aborted) throw new Error(`upload ${uploadId} aborted`);
210-
// Concatenate parts in order.
211-
const sortedPartNums = parts.map((p) => p.partNumber).sort((a, b) => a - b);
212-
// Enforce R2's non-terminal-parts-must-be-same-size constraint
213-
// (error 10048 in real R2). Catches regressions in flush logic.
214-
if (sortedPartNums.length >= 2) {
215-
const sizes = sortedPartNums.map((pn) => state.parts.get(pn)!.byteLength);
216-
const nonTerminalSizes = sizes.slice(0, -1);
217-
const first = nonTerminalSizes[0]!;
218-
for (const s of nonTerminalSizes) {
219-
if (s !== first) {
220-
throw new Error(`completeMultipartUpload: All non-trailing parts must have the same length. (10048) — got sizes ${sizes.join(', ')}`);
221-
}
222-
}
223-
}
224-
const total = sortedPartNums.reduce((n, pn) => n + state.parts.get(pn)!.byteLength, 0);
225-
const assembled = new Uint8Array(total);
226-
let off = 0;
227-
for (const pn of sortedPartNums) {
228-
const part = state.parts.get(pn)!;
229-
assembled.set(part, off);
230-
off += part.byteLength;
231-
}
232-
const buf = new ArrayBuffer(total);
233-
new Uint8Array(buf).set(assembled);
234-
store.set(state.key, buf);
235-
multiparts.delete(uploadId);
236-
}),
237-
abort: vi.fn(async () => {
238-
state.aborted = true;
239-
multiparts.delete(uploadId);
240-
}),
241-
};
242-
}),
243-
_store: store,
244-
_multiparts: multiparts,
245-
};
246-
}
247-
248161
describe('writeShardStreaming → streamShardRows round-trip', () => {
249162
test('write + read preserves rows exactly', async () => {
250163
const r2 = makeR2() as unknown as R2Bucket;
Lines changed: 280 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,280 @@
1+
// Tests for the same-tier "consolidate the dust" writer path
2+
// (`planDustTiling` + `writeShard`'s same-tier branch).
3+
//
4+
// Regression suite for the 2026-07-09 prod wedge: the previous
5+
// prev-rung × k source rule required shards that the min-cover never
6+
// materializes (a closing rung's LAST constituent closes and is
7+
// superseded in the same tick), so every consolidation wedged with
8+
// `no_inputs` from the first 10min rung on up.
9+
import { describe, test, expect } from 'vitest';
10+
import { parquetWriteBuffer } from 'hyparquet-writer';
11+
import type { Duration, ExpectedShard } from 'pyrmts';
12+
import { planDustTiling, writeShard, shardKey } from './cascade';
13+
import { streamShardRows, writeShardRows } from './aggregate';
14+
import type { AvailV3Row } from './transform';
15+
import type { LucIndex } from './luc';
16+
import { makeR2 } from './mock-r2';
17+
18+
const MIN = 60_000;
19+
const ms = (iso: string) => Date.parse(iso);
20+
21+
function availRow(s2_cell: string, dtMs: number, bikes: Record<number, number>): AvailV3Row {
22+
return {
23+
s2_cell, dt: BigInt(dtMs),
24+
bikes: JSON.stringify(bikes),
25+
ebikes: '{}', docks: '{}', disabled: '{}', pending: '{}',
26+
};
27+
}
28+
29+
async function collect(gen: AsyncGenerator<AvailV3Row>): Promise<AvailV3Row[]> {
30+
const out: AvailV3Row[] = [];
31+
for await (const r of gen) out.push(r);
32+
return out;
33+
}
34+
35+
describe('planDustTiling', () => {
36+
const probeFrom = (existing: Set<string>) =>
37+
async (rung: Duration, startMs: number) =>
38+
existing.has(`${rung}@${startMs}`) ? { size: 1000 } : null;
39+
40+
test('steady-state midnight consolidation: dust tiles + one finest-rung tail hole', async () => {
41+
// /1m@12h closing at 2026-07-09T00:00Z. The dust that accumulated
42+
// over [12:00, 24:00) — each shard was written when it entered the
43+
// min-cover; the instantly-superseded "last constituents"
44+
// (6h@18:00, 3h@21:00, 1h@23:00, 30min@23:30, 10min@23:50,
45+
// 5min@23:55) never existed.
46+
const S = ms('2026-07-08T12:00:00Z');
47+
const E = ms('2026-07-09T00:00:00Z');
48+
const existing = new Set([
49+
`6h@${S}`,
50+
`3h@${S + 360 * MIN}`, // 18:00
51+
`1h@${S + 540 * MIN}`, // 21:00
52+
`1h@${S + 600 * MIN}`, // 22:00
53+
`30min@${S + 660 * MIN}`, // 23:00
54+
`10min@${S + 690 * MIN}`, // 23:30
55+
`10min@${S + 700 * MIN}`, // 23:40
56+
`5min@${S + 710 * MIN}`, // 23:50
57+
]);
58+
const finer: Duration[] = ['5min', '10min', '30min', '1h', '3h', '6h'];
59+
const { tiles, holes, aborted } = await planDustTiling(finer, S, E, probeFrom(existing));
60+
expect(aborted).toBe(false);
61+
expect(tiles.map((t) => `${t.rung}@${(t.startMs - S) / MIN}min`)).toEqual([
62+
'6h@0min', '3h@360min', '1h@540min', '1h@600min',
63+
'30min@660min', '10min@690min', '10min@700min', '5min@710min',
64+
]);
65+
// One hole: the 5min last-constituent [23:55, 24:00).
66+
expect(holes).toEqual([[S + 715 * MIN, E]]);
67+
});
68+
69+
test('complete prev-rung × k dust needs no holes', async () => {
70+
const S = ms('2026-07-09T00:30:00Z');
71+
const existing = new Set([`5min@${S}`, `5min@${S + 5 * MIN}`]);
72+
const { tiles, holes, aborted } = await planDustTiling(['5min'], S, S + 10 * MIN, probeFrom(existing));
73+
expect(aborted).toBe(false);
74+
expect(tiles.map((t) => `${t.rung}@${t.startMs}`)).toEqual([`5min@${S}`, `5min@${S + 5 * MIN}`]);
75+
expect(holes).toEqual([]);
76+
});
77+
78+
test('nothing existing coalesces into a single full-period hole', async () => {
79+
const S = ms('2026-07-09T00:00:00Z');
80+
const E = S + 720 * MIN;
81+
const { tiles, holes, aborted } = await planDustTiling(
82+
['5min', '10min', '30min', '1h', '3h', '6h'], S, E, probeFrom(new Set()));
83+
expect(aborted).toBe(false);
84+
expect(tiles).toEqual([]);
85+
expect(holes).toEqual([[S, E]]);
86+
});
87+
88+
test('alternating existing/missing (degraded backlog) aborts at MAX_TILES', async () => {
89+
// Mimics the post-wedge backlog: only first-half 5min shards exist.
90+
const S = ms('2026-07-09T00:00:00Z');
91+
const E = S + 720 * MIN;
92+
const existing = new Set<string>();
93+
for (let t = S; t < E; t += 10 * MIN) existing.add(`5min@${t}`);
94+
const { aborted } = await planDustTiling(['5min'], S, E, probeFrom(existing));
95+
expect(aborted).toBe(true);
96+
});
97+
});
98+
99+
// ─── writeShard same-tier integration (mock R2) ─────────────────────
100+
101+
const lucStub = { chains: new Map([['st1', ['aa', 'ab']]]) } as LucIndex;
102+
103+
/** Raw loader-format minute parquet: one row per station with
104+
* `{metric}_n` / `{metric}_sum` columns and `dt` in SECONDS. */
105+
function putRawMinute(r2: ReturnType<typeof makeR2>, tIso: string, bikesValue: number): void {
106+
const t = new Date(tIso);
107+
const dateStr = t.toISOString().slice(0, 10);
108+
const hh = String(t.getUTCHours()).padStart(2, '0');
109+
const mm = String(t.getUTCMinutes()).padStart(2, '0');
110+
const key = `gbfs/avail/agg=1m/cons=1m/${dateStr}/${hh}${mm}.parquet`;
111+
const buf = parquetWriteBuffer({
112+
columnData: [
113+
{ name: 'station_id', data: ['st1'], type: 'STRING' },
114+
{ name: 'dt', data: [Math.floor(t.getTime() / 1000)], type: 'INT32' },
115+
{ name: 'bikes_n', data: [1], type: 'INT32' },
116+
{ name: 'bikes_sum', data: [bikesValue], type: 'INT32' },
117+
],
118+
});
119+
void r2.put(key, buf);
120+
}
121+
122+
describe('writeShard same-tier consolidation', () => {
123+
test('wedge repro: /1m@10min from one 5min tile + raw-WAL hole fill', async () => {
124+
// The exact prod-wedge shape: 5min@00:30 exists (was dust for one
125+
// tick), 5min@00:35 never materialized. Old prev-rung × 2 rule →
126+
// no_inputs forever. New rule: tile [00:30, 00:35) from the shard,
127+
// fill [00:35, 00:40) from raw minutes.
128+
const r2 = makeR2();
129+
const S = ms('2026-07-09T00:30:00Z');
130+
const rows: AvailV3Row[] = [];
131+
for (let i = 0; i < 5; i++) {
132+
rows.push(availRow('aa', S + i * MIN, { 5: 1 }));
133+
rows.push(availRow('ab', S + i * MIN, { 5: 1 }));
134+
}
135+
await writeShardRows(r2 as never, shardKey('1m', '5min', new Date(S)), rows);
136+
for (let i = 5; i < 10; i++) {
137+
putRawMinute(r2, new Date(S + i * MIN).toISOString(), 7);
138+
}
139+
140+
const result = await writeShard(
141+
r2 as never, lucStub, '1m', '10min',
142+
new Date(S), new Date(S), new Date(S + 10 * MIN), new Map());
143+
expect(result.status).toBe('wrote');
144+
// 1 tile + 5 raw minutes, all present.
145+
expect(result.inputsExpected).toBe(6);
146+
expect(result.inputsPresent).toBe(6);
147+
148+
const written = await collect(streamShardRows(r2 as never, result.key));
149+
// 2 cells × 10 minutes; first 5 minutes from the tile ({5:1}),
150+
// last 5 from raw ({7:1}).
151+
const byCell = (cell: string) => written
152+
.filter((r) => r.s2_cell === cell)
153+
.map((r) => [Number(r.dt - BigInt(S)) / MIN, r.bikes]);
154+
const expected = [
155+
...[0, 1, 2, 3, 4].map((m) => [m, '{"5":1}']),
156+
...[5, 6, 7, 8, 9].map((m) => [m, '{"7":1}']),
157+
];
158+
expect(byCell('aa')).toEqual(expected);
159+
expect(byCell('ab')).toEqual(expected);
160+
});
161+
162+
test('/1m no_inputs when the hole has no raw data', async () => {
163+
const r2 = makeR2();
164+
const S = ms('2026-07-09T00:30:00Z');
165+
await writeShardRows(r2 as never, shardKey('1m', '5min', new Date(S)),
166+
[availRow('aa', S, { 5: 1 })]);
167+
// No raw minutes for [00:35, 00:40).
168+
const result = await writeShard(
169+
r2 as never, lucStub, '1m', '10min',
170+
new Date(S), new Date(S), new Date(S + 10 * MIN), new Map());
171+
expect(result.status).toBe('no_inputs');
172+
});
173+
174+
test('/2m@30min: same-tier tiles + cross-tier hole fill, clipped + rebinned', async () => {
175+
// /2m@10min tiles exist for [00:00, 00:20); the hole [00:20, 00:30)
176+
// fills from a /1m@30min cover shard spanning the WHOLE period —
177+
// its rows outside the hole must be clipped (the tiles already
178+
// cover them), and its 1min rows must rebin to 2min buckets.
179+
const r2 = makeR2();
180+
const S = ms('2026-07-09T00:00:00Z');
181+
for (const tileStart of [S, S + 10 * MIN]) {
182+
const rows: AvailV3Row[] = [];
183+
for (let t = tileStart; t < tileStart + 10 * MIN; t += 2 * MIN) {
184+
rows.push(availRow('aa', t, { 4: 2 }));
185+
}
186+
await writeShardRows(r2 as never, shardKey('2m', '10min', new Date(tileStart)), rows);
187+
}
188+
const srcKey = shardKey('1m', '30min', new Date(S));
189+
const srcRows: AvailV3Row[] = [];
190+
for (let t = S; t < S + 30 * MIN; t += MIN) {
191+
srcRows.push(availRow('aa', t, { 9: 1 }));
192+
}
193+
await writeShardRows(r2 as never, srcKey, srcRows);
194+
const expectedByTier = new Map<string, ExpectedShard[]>([['1m', [{
195+
tier: '1m', shardDur: '30min' as Duration,
196+
periodStart: new Date(S), periodEnd: new Date(S + 30 * MIN),
197+
effectiveStart: new Date(S), effectiveEnd: new Date(S + 30 * MIN),
198+
key: srcKey,
199+
}]]]);
200+
201+
const result = await writeShard(
202+
r2 as never, lucStub, '2m', '30min',
203+
new Date(S), new Date(S), new Date(S + 30 * MIN), expectedByTier);
204+
expect(result.status).toBe('wrote');
205+
// 2 tiles + 1 hole-fill source.
206+
expect(result.inputsExpected).toBe(3);
207+
expect(result.inputsPresent).toBe(3);
208+
209+
const written = await collect(streamShardRows(r2 as never, result.key));
210+
// [00:00, 00:20): tile rows pass through untouched ({4:2} per 2min).
211+
// [00:20, 00:30): hole-filled from /1m — two 1min rows fold into
212+
// each 2min bucket ({9:2}). No double counting in the tiled range.
213+
expect(written.map((r) => [Number(r.dt - BigInt(S)) / MIN, r.bikes])).toEqual([
214+
...[0, 2, 4, 6, 8, 10, 12, 14, 16, 18].map((m) => [m, '{"4":2}']),
215+
...[20, 22, 24, 26, 28].map((m) => [m, '{"9":2}']),
216+
]);
217+
});
218+
219+
test('fragmented dust falls back to pure cross-tier fill (wedge-scar heal)', async () => {
220+
// Wedge-day scar: /2m@12h whose window has only alternating 10min
221+
// fragments (36 tiles + 36 holes > MAX_TILES). The tiling is
222+
// abandoned wholesale and the period fills purely from the /1m
223+
// cover shard — output must reflect ONLY the cross-tier source
224+
// (no double-count from the discarded fragments).
225+
const r2 = makeR2();
226+
const S = ms('2026-07-09T00:00:00Z');
227+
const E = S + 720 * MIN;
228+
for (let t = S; t < E; t += 20 * MIN) {
229+
// Fragment content deliberately distinct ({1:1}) from the /1m
230+
// source ({9:1}) so double-counting would be visible.
231+
await writeShardRows(r2 as never, shardKey('2m', '10min', new Date(t)),
232+
[availRow('aa', t, { 1: 1 })]);
233+
}
234+
const srcKey = shardKey('1m', '12h', new Date(S));
235+
const srcRows: AvailV3Row[] = [];
236+
for (let t = S; t < E; t += MIN) {
237+
srcRows.push(availRow('aa', t, { 9: 1 }));
238+
}
239+
await writeShardRows(r2 as never, srcKey, srcRows);
240+
const expectedByTier = new Map<string, ExpectedShard[]>([['1m', [{
241+
tier: '1m', shardDur: '12h' as Duration,
242+
periodStart: new Date(S), periodEnd: new Date(E),
243+
effectiveStart: new Date(S), effectiveEnd: new Date(E),
244+
key: srcKey,
245+
}]]]);
246+
247+
const result = await writeShard(
248+
r2 as never, lucStub, '2m', '12h',
249+
new Date(S), new Date(S), new Date(E), expectedByTier);
250+
expect(result.status).toBe('wrote');
251+
expect(result.inputsExpected).toBe(1);
252+
expect(result.inputsPresent).toBe(1);
253+
254+
const written = await collect(streamShardRows(r2 as never, result.key));
255+
// 360 2min bins, each folding two 1min source rows → {9:2}; the
256+
// fragments' {1:1} must NOT appear anywhere.
257+
expect(written.length).toBe(360);
258+
expect(new Set(written.map((r) => r.bikes))).toEqual(new Set(['{"9":2}']));
259+
});
260+
261+
test('/2m no_inputs when the hole-fill source shard is missing on R2', async () => {
262+
const r2 = makeR2();
263+
const S = ms('2026-07-09T00:00:00Z');
264+
for (const tileStart of [S, S + 10 * MIN]) {
265+
await writeShardRows(r2 as never, shardKey('2m', '10min', new Date(tileStart)),
266+
[availRow('aa', tileStart, { 4: 2 })]);
267+
}
268+
const srcKey = shardKey('1m', '30min', new Date(S));
269+
const expectedByTier = new Map<string, ExpectedShard[]>([['1m', [{
270+
tier: '1m', shardDur: '30min' as Duration,
271+
periodStart: new Date(S), periodEnd: new Date(S + 30 * MIN),
272+
effectiveStart: new Date(S), effectiveEnd: new Date(S + 30 * MIN),
273+
key: srcKey, // never written to R2
274+
}]]]);
275+
const result = await writeShard(
276+
r2 as never, lucStub, '2m', '30min',
277+
new Date(S), new Date(S), new Date(S + 30 * MIN), expectedByTier);
278+
expect(result.status).toBe('no_inputs');
279+
});
280+
});

0 commit comments

Comments
 (0)