Skip to content

Commit 65f1c4b

Browse files
ringsaturnclaude
ringsaturn
and
claude
authored
mrms: publish the radar mosaic as a live rolling window (#17)
The MRMS source becomes the one dataset that is observations and live. `--run latest` resolves to the window whose last hour holds the bucket's newest frame (fetch.latest_mrms_slot; three whole hours plus the hour in progress with `--hours 4`), and because the same run is rebuilt every five minutes, `build-bin --round HHMM` writes each build into a round subdirectory of its run with a window.json beside the manifest and the pointer naming the round — an object rewritten under an unchanged ?v= would hand a viewer's range requests the wrong bytes. The previous run's frames are hard-linked across by object key when the window crosses an hour. publish-mrms.yml runs one job an hour looping scripts/mrms_rounds.sh: a round every five minutes to five to the hour, skipped when the live window already ends at the bucket's newest frame, `make upload-r2 … ROUND=` then `prune-r2-rounds` (the run's newest two rounds, never the pointer's) and `prune-r2 KEEP=2`. The Makefile threads ROUND through upload-r2, check-pointer and the warm script, and gains live-window. The shell lists mrms among the live feeds, polls its pointer every two minutes, treats a changed manifestCrc32 rather than a changed run id as a new run, and on a rolling window keeps the playhead by observation time or follows the end when it was at the end. pickBundleVariant scales the needed width by the bundle's longitude span (off the poster metadata), so a regional grid takes its half tier on a far-out view. Claude-Session: https://claude.ai/code/session_0135a5JP5eKsaLJTiyWS4vmN Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
1 parent 21b39be commit 65f1c4b

30 files changed

Lines changed: 1027 additions & 93 deletions

‎.github/workflows/publish-mrms.yml‎

Lines changed: 173 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,173 @@
1+
name: publish mrms
2+
3+
# NOAA MRMS, the radar mosaic over the contiguous United States: an
4+
# observation every two minutes, about a minute behind real time on its
5+
# bucket. The live feed is a rolling window — the last three whole hours
6+
# plus the hour in progress, 0.02°, the native two-minute step — rebuilt
7+
# every five minutes into a round subdirectory of its run
8+
# (<model>.<run>/<HHMM>/) and pointed at by latest-mrms.json.
9+
#
10+
# Not the three-job pipeline of publish.yml: a cron here fires once a
11+
# minute at best and tens of minutes late under load, so one job an hour
12+
# runs the rounds itself (scripts/mrms_rounds.sh), sleeping to each
13+
# five-minute boundary until five to the hour, and the next hour's cron
14+
# takes over. The cron's lateness costs the first round of the hour only.
15+
#
16+
# Machine time: twenty-four jobs of up to fifty-five minutes a day, about
17+
# 660 hours a month — free on a public repository's standard runners, far
18+
# past the free allowance were this repository ever private (then the
19+
# round would have to stretch to twenty minutes, or the job move to a
20+
# self-hosted runner).
21+
22+
on:
23+
schedule:
24+
- cron: "0 * * * *"
25+
workflow_dispatch:
26+
inputs:
27+
once:
28+
description: "Run one round and stop, instead of the rest of the hour"
29+
type: boolean
30+
default: false
31+
round_minutes:
32+
description: "Minutes between rounds"
33+
type: string
34+
default: "5"
35+
hours:
36+
description: "Window length in hours (the whole hours on screen plus the one in progress)"
37+
type: string
38+
default: "4"
39+
profile:
40+
description: "Quantization/compression profile"
41+
type: choice
42+
options: [balanced, quality, compact]
43+
default: balanced
44+
keep:
45+
description: "Published runs to keep on R2 (2 = the live run and the one before it)"
46+
type: string
47+
default: "2"
48+
force:
49+
description: "Build a round even when the live one already ends at the bucket's newest frame"
50+
type: boolean
51+
default: false
52+
dry_run:
53+
description: "Build and report what the uploads and prunes would do, but write nothing to R2"
54+
type: boolean
55+
default: false
56+
57+
permissions:
58+
contents: read
59+
60+
concurrency:
61+
group: publish-mrms
62+
cancel-in-progress: false
63+
64+
env:
65+
# Convert through the native encoder (the xuepy wheel, a dependency of
66+
# xuebuild) and insist on it: a round that quietly fell back to the
67+
# subprocess pipeline would overrun its five minutes.
68+
XUE_ENCODER: native
69+
70+
jobs:
71+
publish:
72+
# Scheduled runs on forks would publish into a bucket they cannot reach.
73+
if: github.event_name != 'schedule' || github.repository == 'ringsaturn/xue'
74+
runs-on: ubuntu-24.04
75+
# The rounds stop at five to the hour on their own; this is the
76+
# backstop for a round that hangs.
77+
timeout-minutes: 65
78+
env:
79+
ONCE: ${{ inputs.once || false }}
80+
ROUND_MINUTES: ${{ inputs.round_minutes || '5' }}
81+
HOURS: ${{ inputs.hours || '4' }}
82+
PROFILE: ${{ inputs.profile || 'balanced' }}
83+
KEEP: ${{ inputs.keep || '2' }}
84+
FORCE: ${{ inputs.force || false }}
85+
DRY_RUN: ${{ inputs.dry_run && '--dryrun' || '' }}
86+
steps:
87+
- uses: actions/checkout@v4
88+
89+
- name: Validate inputs
90+
run: |
91+
set -euo pipefail
92+
printf '%s' "$ROUND_MINUTES" | grep -Eq '^[1-9][0-9]?$' \
93+
|| { echo "::error::round_minutes must be a positive integer of minutes"; exit 1; }
94+
printf '%s' "$HOURS" | grep -Eq '^[1-9][0-9]?$' \
95+
|| { echo "::error::hours must be a positive integer"; exit 1; }
96+
printf '%s' "$KEEP" | grep -Eq '^[1-9][0-9]*$' \
97+
|| { echo "::error::keep must be a positive integer"; exit 1; }
98+
99+
# The AWS CLI is preinstalled on the runner; R2 is its S3 endpoint.
100+
# The credentials are step-scoped rather than job env, so third-party
101+
# actions never see them; the listing proves they work before the
102+
# first round rather than at its upload.
103+
- name: Check R2 credentials
104+
env:
105+
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
106+
AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
107+
AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
108+
AWS_REQUEST_CHECKSUM_CALCULATION: when_required
109+
AWS_RESPONSE_CHECKSUM_VALIDATION: when_required
110+
run: |
111+
for name in CLOUDFLARE_ACCOUNT_ID AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY; do
112+
[ -n "${!name}" ] || { echo "::error::$name is empty; set the R2 repository secrets"; exit 1; }
113+
done
114+
aws --version
115+
aws s3 ls "s3://dataset/xue/" --region auto \
116+
--endpoint-url "https://${CLOUDFLARE_ACCOUNT_ID}.r2.cloudflarestorage.com" > /dev/null \
117+
|| { echo "::error::R2 credentials failed a test listing of the dataset bucket"; exit 1; }
118+
119+
- uses: astral-sh/setup-uv@v5
120+
with:
121+
enable-cache: true
122+
python-version: "3.14"
123+
124+
# No GDAL and no ffmpeg: the source ships no video, and the xuepy
125+
# wheel carries the GDAL the native encoder extracts and inspects
126+
# with (`xuepy>=0.15`, the first wheel that knows the source).
127+
- name: Sync Python environment
128+
run: uv sync
129+
130+
- name: Run the rounds
131+
env:
132+
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
133+
AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
134+
AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
135+
run: |
136+
set -o pipefail
137+
scripts/mrms_rounds.sh 2>&1 | tee rounds.log
138+
status=$?
139+
{
140+
echo "### mrms rounds"
141+
echo
142+
echo '```'
143+
grep -E '^round [0-9]{4}:|rounds this hour' rounds.log || true
144+
echo '```'
145+
} >> "$GITHUB_STEP_SUMMARY"
146+
exit $status
147+
148+
# The pointer is served with no-cache, so the edge revalidates it; a
149+
# stale read here means the last flip has not propagated rather than
150+
# failed, so it warns instead of failing an otherwise complete hour.
151+
- name: Verify the live pointer
152+
if: ${{ !inputs.dry_run }}
153+
run: |
154+
set -uo pipefail
155+
[ -f web/public/data/latest-mrms.json ] || { echo "No round built this hour." >> "$GITHUB_STEP_SUMMARY"; exit 0; }
156+
url="https://dataset.ringsaturn.me/xue/latest-mrms.json"
157+
published=$(curl -fsSL "$url" | jq -r '.manifestPath' || true)
158+
if [ -n "$published" ] && [ -f "web/public/data/$published" ]; then
159+
echo "$url serves $published." >> "$GITHUB_STEP_SUMMARY"
160+
else
161+
echo "::warning::$url serves '${published:-nothing}', not the round this job last built"
162+
fi
163+
164+
- uses: actions/upload-artifact@v4
165+
if: ${{ !cancelled() }}
166+
with:
167+
name: publish-report-mrms
168+
path: |
169+
rounds.log
170+
build-report.json
171+
web/public/data/latest-mrms.json
172+
retention-days: 14
173+
if-no-files-found: ignore

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,13 @@ web/public/data/gfs.*/
1313
web/public/data/ecmwf.*/
1414
web/public/data/sflux.*/
1515
web/public/data/hrrr.*/
16+
web/public/data/mrms.*/
1617
web/public/data/manifest.json
1718
web/public/data/latest.json
1819
web/public/data/latest-ecmwf.json
1920
web/public/data/latest-sflux.json
2021
web/public/data/latest-hrrr.json
22+
web/public/data/latest-mrms.json
2123
web/public/data/tc.*/
2224
web/public/data/latest-tc.json
2325
web/public/data/showcase/

‎CLAUDE.md‎

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -241,11 +241,30 @@ publishing data at the new version.**
241241
(`_snap_regional_steps`, the regional analogue of the global `360 /
242242
width` rule, mirrored in `grid.rs`) — GDAL derives the step from the
243243
first and last coordinates, and MRMS writes its last one a hair short.
244-
No live pointer yet — the rolling window is the next step: `build-bin
244+
MRMS is the one source that is an observation *and* live. `build-bin
245245
--model mrms --run <hour> --hours 3` builds a past window into
246-
`mrms.<run>/` and nothing points at it; what is published are showcase
247-
cases (`ida-2021`, `quad-state-tornado-2021`), and the shell registers
248-
`mrms` in `FORECAST_MODELS` as an observation dataset off the live list.
246+
`mrms.<run>/` (the showcase cases `ida-2021`, `quad-state-tornado-2021`
247+
are cropped ones); the live feed is a **rolling window**: `--run latest`
248+
resolves (`fetch.py::latest_mrms_slot`, `resolve_run`) to the run whose
249+
last hour holds the bucket's newest frame, `--hours 4` (three whole
250+
hours plus the hour in progress, `window_hours + 1`), and since the same
251+
run is rebuilt every five minutes, each build takes `--round HHMM` and
252+
lands in `mrms.<run>/<HHMM>/` with a `window.json` (`fetch.py::
253+
window_summary`) beside its manifest and the pointer naming the round —
254+
never overwrite an object under an unchanged `?v=`: a viewer's range
255+
requests against it decode the wrong bytes. `publish-mrms.yml` is one
256+
job an hour looping `scripts/mrms_rounds.sh` (a round every five
257+
minutes to five to the hour: newest frame vs the live `window.json`
258+
→ build → `make upload-r2 … ROUND=` → `prune-r2-rounds` keeps the run's
259+
newest two rounds, `prune-r2 KEEP=2` the previous run), not the
260+
three-job `publish.yml`. The shell lists `mrms` among
261+
`FORECAST_MODEL_IDS`, polls its pointer every two minutes instead of
262+
five, treats a changed `manifestCrc32` (not a changed run id) as a new
263+
run, and on a rolling window keeps the playhead by observation time —
264+
or follows the end when it was at the end (`checkForNewRun` /
265+
`resumeOnNewRun` in `main.ts`). `pickBundleVariant` scales the needed
266+
width by the bundle's longitude span (read off the poster metadata), so
267+
a regional grid can take its half tier on a far-out view.
249268
A source with a `regrid` (`hrrr`) is computed on a map projection:
250269
`_grid_info` reads
251270
the Lambert conformal parameters out of GDAL's WKT (`reproject.py`, and

‎Makefile‎

Lines changed: 59 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,15 @@ PROFILE ?= balanced
1010
# sflux (GFS surface flux, native ~13 km, hourly, adds the dswrf layer), or
1111
# hrrr (NOAA HRRR, 3 km over the contiguous US, a cycle every hour, to F18);
1212
# mrms (the NOAA radar mosaic, an observation every two minutes) builds a
13-
# window named by its first hour and has no live pointer yet.
13+
# window named by its first hour — `RUN=latest` the live rolling window.
1414
MODEL ?= gfs
15+
# One round of a rolling window (`build-bin --round`): the run's artifacts
16+
# and manifest live in <model>.<run>/<ROUND>/ and the pointer names that
17+
# round, so a later round of the same run never overwrites what a viewer is
18+
# still reading (a range request against a rewritten object decodes the
19+
# wrong bytes). Empty for a forecast run, which is built once.
20+
ROUND ?=
21+
RUN_DIR = $(MODEL).$(RUN)$(if $(ROUND),/$(ROUND))
1522
# Published runs of one model to keep on R2 (`make prune-r2`). One means the
1623
# live run only: the bucket carries no history.
1724
KEEP ?= 1
@@ -37,7 +44,7 @@ AWS_REQUEST_CHECKSUM_CALCULATION ?= when_required
3744
AWS_RESPONSE_CHECKSUM_VALIDATION ?= when_required
3845
export AWS_DEFAULT_REGION AWS_REQUEST_CHECKSUM_CALCULATION AWS_RESPONSE_CHECKSUM_VALIDATION
3946

40-
.PHONY: check install wasm test test-rust test-e2e encoder-rust encoder-rust-test encoder-wheel bench bench-video bench-lossy mvp serve format-pdf deploy-build upload-r2 upload-r2-bundles upload-r2-manifest check-pointer upload-r2-pointer warm-r2 prune-r2 live-run live-manifest deploy-pages deploy showcase showcase-check showcase-refresh upload-r2-showcase tc-build live-tc-index upload-r2-tc prune-r2-tc clean
47+
.PHONY: check install wasm test test-rust test-e2e encoder-rust encoder-rust-test encoder-wheel bench bench-video bench-lossy mvp serve format-pdf deploy-build upload-r2 upload-r2-bundles upload-r2-manifest check-pointer upload-r2-pointer warm-r2 prune-r2 prune-r2-rounds live-run live-manifest live-window deploy-pages deploy showcase showcase-check showcase-refresh upload-r2-showcase tc-build live-tc-index upload-r2-tc prune-r2-tc clean
4148

4249
check:
4350
$(PYTHON) scripts/check_dependencies.py
@@ -153,14 +160,18 @@ deploy-build:
153160
# taking the run live with `upload-r2-manifest` — and the three targets share
154161
# their pieces: a partial manifest (manifest.part.*.json, the assembler's
155162
# input) never reaches the bucket from any of them.
163+
#
164+
# A rolling window's round (ROUND=HHMM) is the same path one directory
165+
# deeper: the round's artifacts and manifest are synced into
166+
# <model>.<run>/<ROUND>/ and the pointer names the round.
156167
upload-r2:
157-
@set -e; dir=web/public/data/$(MODEL).$(RUN); \
168+
@set -e; dir=web/public/data/$(RUN_DIR); \
158169
[ -d "$$dir" ] || { echo "no built run at $$dir, pass RUN=YYYYMMDDHH"; exit 1; }; \
159-
$(MAKE) --no-print-directory check-pointer MODEL=$(MODEL) RUN=$(RUN); \
160-
$(S3) sync $$dir s3://$(R2_BUCKET)/$(R2_PREFIX)/$(MODEL).$(RUN)/ --no-progress $(DRY_RUN) \
170+
$(MAKE) --no-print-directory check-pointer MODEL=$(MODEL) RUN=$(RUN) ROUND=$(ROUND); \
171+
$(S3) sync $$dir s3://$(R2_BUCKET)/$(R2_PREFIX)/$(RUN_DIR)/ --no-progress $(DRY_RUN) \
161172
--exclude "manifest.part.*.json" \
162173
--cache-control "public, max-age=31536000, immutable"; \
163-
[ -n "$(DRY_RUN)" ] || $(MAKE) --no-print-directory warm-r2 MODEL=$(MODEL) RUN=$(RUN) \
174+
[ -n "$(DRY_RUN)" ] || $(MAKE) --no-print-directory warm-r2 MODEL=$(MODEL) RUN=$(RUN) ROUND=$(ROUND) \
164175
|| echo "warming the edge cache failed; the run goes live cold"; \
165176
$(MAKE) --no-print-directory upload-r2-pointer MODEL=$(MODEL) RUN=$(RUN) DRY_RUN=$(DRY_RUN)
166177

@@ -199,12 +210,16 @@ upload-r2-manifest:
199210
# both, and a pointer that disagrees with its manifest strands every viewer
200211
# on a 404.
201212
check-pointer:
202-
@set -e; dir=web/public/data/$(MODEL).$(RUN); \
213+
@set -e; dir=web/public/data/$(RUN_DIR); \
203214
pointer_run=$$(jq -r .run web/public/data/$(LATEST_FILE)); \
204215
[ "$$pointer_run" = "$(RUN)" ] || { \
205216
echo "$(LATEST_FILE) names run $$pointer_run, not $(RUN) — a later build rewrote it;"; \
206217
echo "rebuild run $(RUN) (or upload run $$pointer_run) so the pointer matches the assets"; \
207218
exit 1; }; \
219+
pointer_path=$$(jq -r .manifestPath web/public/data/$(LATEST_FILE)); \
220+
[ "$$pointer_path" = "$(RUN_DIR)/manifest.json" ] || { \
221+
echo "$(LATEST_FILE) names $$pointer_path, not $(RUN_DIR)/manifest.json — pass the ROUND it was built with"; \
222+
exit 1; }; \
208223
pointer_crc=$$(jq -r .manifestCrc32 web/public/data/$(LATEST_FILE)); \
209224
manifest_crc=$$($(PYTHON) -c "import sys, zlib; print(f'{zlib.crc32(open(sys.argv[1], \"rb\").read()) & 0xFFFFFFFF:08x}')" $$dir/manifest.json); \
210225
[ "$$pointer_crc" = "$$manifest_crc" ] || { \
@@ -221,7 +236,7 @@ upload-r2-pointer:
221236
# the site's Origin header, so the edge (and, with tiered cache, the upper
222237
# tier every other data center fills from) holds it before anyone asks.
223238
warm-r2:
224-
scripts/warm_edge_cache.sh $(MODEL) $(RUN)
239+
ROUND=$(ROUND) scripts/warm_edge_cache.sh $(MODEL) $(RUN)
225240

226241
# Historical showcase cases: past runs cropped to one weather event, defined
227242
# in showcase/cases/*.json and built into web/public/data/showcase/. Pass
@@ -339,6 +354,31 @@ prune-r2:
339354
fi; \
340355
done
341356

357+
# A rolling window keeps its rounds beside one another inside the run
358+
# directory. After the pointer has moved on to a new round, the rounds
359+
# before the newest ROUNDS_KEEP are deleted — never the one the pointer
360+
# names, whatever its name sorts as. A viewer on a replaced round has the
361+
# next round plus a poll interval to be brought forward before its objects
362+
# go; the top-level prune (`prune-r2`, KEEP=2 for this source) is what
363+
# keeps the previous run's last round alongside.
364+
ROUNDS_KEEP ?= 2
365+
prune-r2-rounds:
366+
@set -e; \
367+
pointer=$$($(S3) cp s3://$(R2_BUCKET)/$(R2_PREFIX)/$(LATEST_FILE) - --only-show-errors 2>/dev/null || true); \
368+
[ -n "$$pointer" ] || { echo "no live pointer for $(MODEL), refusing to prune"; exit 1; }; \
369+
live=$$(printf '%s' "$$pointer" | jq -r .manifestPath | xargs dirname); \
370+
run=$$(printf '%s' "$$pointer" | jq -r .run); \
371+
echo "live $(MODEL) round: $$live"; \
372+
listing=$$($(S3) ls s3://$(R2_BUCKET)/$(R2_PREFIX)/$(MODEL).$$run/) \
373+
|| { echo "listing the run failed, refusing to prune"; exit 1; }; \
374+
for round in $$(printf '%s\n' "$$listing" | awk '/ PRE /{print $$2}' \
375+
| sed 's:/$$::' | sort -r | tail -n +$$(($(ROUNDS_KEEP) + 1))); do \
376+
if [ "$(MODEL).$$run/$$round" != "$$live" ]; then \
377+
echo "Deleting $(MODEL).$$run/$$round..."; \
378+
$(S3) rm s3://$(R2_BUCKET)/$(R2_PREFIX)/$(MODEL).$$run/$$round/ --recursive --only-show-errors $(DRY_RUN); \
379+
fi; \
380+
done
381+
342382
# Print the run the live pointer names, or nothing when there is no pointer.
343383
live-run:
344384
@$(S3) cp s3://$(R2_BUCKET)/$(R2_PREFIX)/$(LATEST_FILE) - --only-show-errors | jq -r .run || true
@@ -347,9 +387,17 @@ live-run:
347387
# base a top-up merges new bundles onto (`xuebuild assemble-run
348388
# --base-manifest`). Prints nothing when there is no live run.
349389
live-manifest:
350-
@set -e; live=$$($(MAKE) -s --no-print-directory live-run MODEL=$(MODEL)); \
351-
[ -n "$$live" ] || exit 0; \
352-
$(S3) cp s3://$(R2_BUCKET)/$(R2_PREFIX)/$(MODEL).$$live/manifest.json - --only-show-errors
390+
@set -e; path=$$($(S3) cp s3://$(R2_BUCKET)/$(R2_PREFIX)/$(LATEST_FILE) - --only-show-errors 2>/dev/null | jq -r .manifestPath || true); \
391+
[ -n "$$path" ] || exit 0; \
392+
$(S3) cp s3://$(R2_BUCKET)/$(R2_PREFIX)/$$path - --only-show-errors
393+
394+
# The window.json beside the live manifest of a rolling window (what the
395+
# round holds: frame count, first and newest slot), as the bucket holds it.
396+
# Prints nothing when there is no live run or the run has no window record.
397+
live-window:
398+
@set -e; path=$$($(S3) cp s3://$(R2_BUCKET)/$(R2_PREFIX)/$(LATEST_FILE) - --only-show-errors 2>/dev/null | jq -r .manifestPath || true); \
399+
[ -n "$$path" ] || exit 0; \
400+
$(S3) cp s3://$(R2_BUCKET)/$(R2_PREFIX)/$$(dirname $$path)/window.json - --only-show-errors 2>/dev/null || true
353401

354402
# Publish dist-deploy/ (built via deploy-build) to the Cloudflare Pages project.
355403
deploy-pages:

0 commit comments

Comments
 (0)