cells-api: worker-side adaptive coarsening on combo path (dormant client)

Adds `coarsenCells(cells, toRes)` + wires it into the combo branch of
`handleCellsRequest`. When `maxCells != null && cells.length > maxCells
&& res > MIN_RES_COMBO`, worker walks parent-aggregate one res at a
time until it fits, sums count fields, dedup+sorts fatal_years.

Direct-curl measured (Bergen z=10.94, 3-s5 batch):
  before: res=10, 6500 cells, 1175 KB uncompressed
  after:  res=9,  1429 cells,  269 KB uncompressed (~4.4× smaller)

Sld join still populates on coarsened cells.

Client wiring intentionally NOT included in this commit. Earlier
attempt (aca0fcd) sent `maxCells` on the combo path from
`buildBatchUrl`, but revealed a downstream bug in `useCellsApi.ts:728`:
per-shard responses get merged by picking `min(res)` across all
shards, then rolling everything up to that coarsest res. When ONE
dense urban shard coarsens r10→r7, EVERY shard in the viewport
renders at r7 (50-80px hexes at z=10.94, visually catastrophic).

The real fix is per-shard render (emit one deck.gl `ColumnLayer` per
shard-res); tracked as followup on task #124. Worker capability
lands now as backwards-compatible dormant infra — deployed to prod
(version c666f7e6-8363-4f89-a15a-ebc90de8ff85), no client sends the
param yet.

Spec: `specs/cells-api-combo-maxcells.md`.
Tests: `cells-api/src/cells.test.ts` +3 for `coarsenCells`. 29/29
green.

Author: ryan-williams

cells-api/src/cells.test.ts

@@ -1,5 +1,6 @@
 import { describe, it, expect } from "vitest"
-import { parseCellsRequest, HttpError } from "./cells"
+import { latLngToCell, cellToParent } from "h3-js"
+import { parseCellsRequest, HttpError, coarsenCells } from "./cells"
 
 /** Minimal valid query — only the two required params. */
 const BASE = "cells=8c2a100894097ff&res=12"
@@ -44,3 +45,45 @@ describe("parseCellsRequest severity param", () => {
         expect(r.severities && [...r.severities].sort()).toEqual(["f", "i", "p"])
     })
 })
+
+describe("coarsenCells", () => {
+    // Two distinct r10 cells from geographically-separated lat/lngs so
+    // their r9 parents differ; plus a synthetic sibling of the first
+    // (any other r10 whose r9 parent matches).
+    const jcR10 = latLngToCell(40.7178, -74.0431, 10)      // Jersey City
+    const newarkR10 = latLngToCell(40.7357, -74.1724, 10)  // Newark
+    const jcParent = cellToParent(jcR10, 9)
+    const newarkParent = cellToParent(newarkR10, 9)
+    // sibling of jcR10 (drop resolution and re-index at r10 by nudging lat).
+    // The specific sibling doesn't matter — we just need same r9 parent.
+    const jcSib = latLngToCell(40.7180, -74.0429, 10)
+
+    it("sums counts across children mapping to the same parent", () => {
+        const a = { h3: jcR10, n_fatal: 1, n_inj_ped: 2, n_inj_other: 3, n_pdo: 4, n_vehs: 5 }
+        const b = { h3: jcSib, n_fatal: 10, n_inj_ped: 20, n_inj_other: 30, n_pdo: 40, n_vehs: 50 }
+        const c = { h3: newarkR10, n_fatal: 7, n_inj_ped: 0, n_inj_other: 0, n_pdo: 0, n_vehs: 1 }
+        const out = coarsenCells([a, b, c], 9)
+        // Two distinct parents: jcParent (has a+b) and newarkParent (has c).
+        expect(out.length).toBe(2)
+        expect(new Set(out.map(o => o.h3))).toEqual(new Set([jcParent, newarkParent]))
+        const jc = out.find(o => o.h3 === jcParent)!
+        const nw = out.find(o => o.h3 === newarkParent)!
+        expect(jc.n_fatal).toBe(1 + 10)
+        expect(jc.n_inj_other).toBe(3 + 30)
+        expect(jc.n_pdo).toBe(4 + 40)
+        expect(nw.n_fatal).toBe(7)
+        expect(nw.n_vehs).toBe(1)
+    })
+
+    it("returns empty for empty input", () => {
+        expect(coarsenCells([], 9)).toEqual([])
+    })
+
+    it("aggregates fatal_years across children, dedup+sort", () => {
+        const a = { h3: jcR10, n_fatal: 1, n_inj_ped: 0, n_inj_other: 0, n_pdo: 0, n_vehs: 0, fatal_years: [2018, 2020] }
+        const b = { h3: jcSib, n_fatal: 1, n_inj_ped: 0, n_inj_other: 0, n_pdo: 0, n_vehs: 0, fatal_years: [2020, 2022] }
+        const [parent] = coarsenCells([a, b], 9)
+        expect(parent.h3).toBe(jcParent)
+        expect(parent.fatal_years).toEqual([2018, 2020, 2022])
+    })
+})

cells-api/src/cells.ts

@@ -163,6 +163,42 @@ export function _resetSldCache(): void {
     sldCached = null
 }
 
+/** In-worker parent-aggregate — used to drop one H3 res in the combo
+ *  path when a shard came back with more cells than `maxCells`. Cheaper
+ *  than re-reading a coarser parquet (no extra R2 hit), and the sum-
+ *  of-tiers semantics match how the offline pyramid rollup was built. */
+export function coarsenCells(cells: CellOut[], toRes: number): CellOut[] {
+    if (cells.length === 0) return cells
+    const parents = new Map<string, CellOut>()
+    for (const c of cells) {
+        const ph = cellToParent(c.h3, toRes)
+        let p = parents.get(ph)
+        if (!p) {
+            p = {
+                h3: ph,
+                n_fatal: 0,
+                n_inj_ped: 0,
+                n_inj_other: 0,
+                n_pdo: 0,
+                n_vehs: 0,
+            }
+            parents.set(ph, p)
+        }
+        p.n_fatal += c.n_fatal
+        p.n_inj_ped += c.n_inj_ped
+        p.n_inj_other += c.n_inj_other
+        p.n_pdo += c.n_pdo
+        p.n_vehs += c.n_vehs
+        if (c.fatal_years && c.fatal_years.length > 0) {
+            (p.fatal_years ??= []).push(...c.fatal_years)
+        }
+    }
+    for (const p of parents.values()) {
+        if (p.fatal_years) p.fatal_years = [...new Set(p.fatal_years)].sort((a, b) => a - b)
+    }
+    return [...parents.values()]
+}
+
 export type CellsResponse = {
     res: number
     year_range: [number, number]
@@ -249,8 +285,11 @@ export async function handleCellsRequest(
 
     // Multi-resolution combo path: client specifies `(shard_res, data_res)`.
     // Worker reads `pyramid/s{shard_res}_r{data_res}/{shard}.parquet`
-    // directly, no coarsening (the client picks a combo whose cell count
-    // is already in budget). `maxCells` is ignored on the combo path.
+    // directly. When `maxCells` is set (client-signaled per-shard cap),
+    // in-worker coarsening walks parent-aggregate cells down one res at
+    // a time until it fits or hits `MIN_RES` — cheaper than re-reading
+    // a coarser parquet from R2. Response `res` reports the actual res
+    // returned, which the client trusts as ground truth.
     if (shardRes != null) {
         const combos = manifest.pyramid_combos ?? []
         const combo = combos.find(c => c.shard_res === shardRes && c.data_res === requestedRes)
@@ -261,10 +300,16 @@ export async function handleCellsRequest(
         const known = new Set(combo.shard_cells)
         const shards = requestedShards.filter(s => known.has(s))
         const clipPoly = req.clipPolygon && req.clipPolygon.length >= 3 ? req.clipPolygon : null
-        const cells = await queryPyramid(bucket, prefix, requestedRes, shards, yearRange, sevSet, clipPoly, shardRes)
+        let cells = await queryPyramid(bucket, prefix, requestedRes, shards, yearRange, sevSet, clipPoly, shardRes)
+        let res = requestedRes
+        const MIN_RES_COMBO = 5
+        while (maxCells != null && cells.length > maxCells && res > MIN_RES_COMBO) {
+            res--
+            cells = coarsenCells(cells, res)
+        }
         await joinSld(bucket, prefix, cells)
         return {
-            res: requestedRes,
+            res,
             year_range: yearRange,
             data_version: manifest.data_version,
             source: "pyramid",

specs/cells-api-combo-maxcells.md

@@ -0,0 +1,112 @@
+# cells-api: worker-side adaptive coarsening on the combo path
+
+## Problem
+
+Circle/hex mode at coarse zoom (e.g. z=10.94 Bergen) fetches ~64k r10
+cells per shard × 3-5 shards → 3-5 MB gzipped per viewport response.
+The visual only needs a fraction of that: at that zoom, r10 cells are
+~2 px wide — well below one screen pixel per cell — so the client
+blurs them into a rasterized heatmap regardless of how many it fetched.
+
+The legacy (non-combo) request path already has a `maxCells` mode
+where the worker walks coarser (drops one res level, re-reads,
+re-checks) until `cells.length <= maxCells`. The combo path
+(`?shard_res=X&res=Y`) intentionally skipped it because the client
+was pre-picking a combo whose total cell budget "should" fit.
+
+Reality check: per-shard cell counts vary 10-100× across a single
+NJ-wide viewport (rural r10 shards have ~5k cells; urban ones have
+~60k). A single combo choice can't be tight everywhere.
+
+## Design
+
+Extend the combo path to accept `maxCells` and apply it **in-worker,
+per-shard, in memory** — no coarser-res parquet re-read.
+
+### Wire
+
+Client sends `?shard_res=X&res=Y&maxCells=N` on combo requests
+(currently only sent on legacy path). Worker interprets:
+
+- Read shard's pyramid at `(shardRes, res)`.
+- Build the `CellOut[]` (year/severity filter + polygon clip, as today).
+- **New**: if `cells.length > maxCells`, coarsen in-worker by
+  `cellToParent(h3, res-1)` + sum the count fields. Repeat until
+  `cells.length <= maxCells` or `res === MIN_RES`.
+- Join sld sidecar against whatever res the cells ended up at.
+- Return with `res: actualRes` (client already uses this as truth).
+
+Aggregation semantics for the coarsening step:
+
+- `n_fatal/n_inj_ped/n_inj_other/n_pdo/n_vehs`: **sum** across children.
+- `fatal_years`: **union** across children.
+- `h3`: replaced with parent's h3 string.
+
+Sld-join uses the coarsened h3s. Sidecar has entries at r6-r11; cells
+coarsened to r9 look up r9 h3 directly, r10 looks up r10, etc.
+`getResolution(h3) > 11 ? cellToParent(h3, 11) : h3` handles coarser-
+than-11 already.
+
+### Worker changes (`cells-api/src/cells.ts`)
+
+Two things:
+
+1. Extract `coarsenCells(cells, toRes)` helper: parent-aggregate via
+   `cellToParent`. Uses same tier fields as `CellOut`.
+2. Add a while-loop after the shard read in the combo branch of
+   `handleCellsRequest`: `while (maxCells != null && cells.length >
+   maxCells && res > MIN_RES) { res--; cells = coarsenCells(cells,
+   res) }`.
+
+Existing legacy-path adaptive is unchanged.
+
+### Client changes (`www/src/map/useCellsApi.ts`)
+
+- `buildBatchUrl`: send `maxCells` on both paths (combo + legacy).
+- Client already trusts `response.res` for rendering; cross-shard
+  `rollupCellsToRes` normalizes when different shards return different
+  res. No further plumbing.
+
+Existing client-side coarsening (`rollupCellsToRes` in the useCellsApi
+result-merge) is safe — it walks fine → coarse, which is what we want
+if the worker returns a shard at a coarser res.
+
+### Choosing `maxCells` per request
+
+Current CELLS_BUDGET = 100k total across the viewport response. A
+typical viewport has 3-25 shards. Client already snaps cover to
+combos; add:
+
+```ts
+const maxCellsPerShard = Math.floor(CELLS_BUDGET / cover.length)
+```
+
+Floor at ~5000 (leave headroom for a single-shard corner case). Cap at
+20000 (don't over-restrict when a shard covers dense urban area with
+legitimate detail).
+
+## What this does NOT do
+
+- Doesn't change what hex-mode users see visually — worker returns
+  aggregated cell counts, client renders whatever res comes back at
+  identical hex-tessellation semantics. Chunkier hexes where the
+  worker coarsened; unchanged where it didn't.
+- Doesn't touch parquet transport / JSON payload compression. That
+  stays a followup once we see whether (1) is enough.
+- Doesn't touch the client's cover-picker. Same shard fanout, same
+  URL cache keys.
+
+## Rollout
+
+Independently deployable. Worker deploy first (backwards-compatible:
+old client ignores `res` field being different if it happens to be
+coarser than requested; nothing breaks). Client change lands after,
+starts sending `maxCells` on combo path.
+
+## Tests
+
+- `cells-api/src/cells.test.ts`: add cases for `coarsenCells` + the
+  combo-path adaptive loop with `maxCells`.
+- Live smoke: hit the Bergen z=10.94 URL; verify (a) response cell
+  count is ≤ maxCells per shard; (b) `res` returned is 8 or 9 for the
+  urban shards; (c) visual on the map is unchanged for rural shards.

www/src/map/useCellsApi.ts

@@ -480,8 +480,11 @@ function buildShardUrl(
         years: `${filter.yearRange[0]}-${filter.yearRange[1]}`,
         severities: sevs,
     })
-    // Combo path skips `maxCells` (combo was pre-picked to fit budget);
-    // legacy path needs it for worker-side adaptive coarsening.
+    // Combo path skips `maxCells` for now — worker supports it (see
+    // `specs/cells-api-combo-maxcells.md`) but wiring the client to
+    // actually use it requires a per-shard render path so
+    // heterogeneously-coarsened shards don't force the whole viewport
+    // to the coarsest returned res. Deferred followup.
     if (shardRes != null) params.set("shard_res", String(shardRes))
     else params.set("maxCells", String(maxCells))
     if (polygonStr) params.set("polygon", polygonStr)