feat(stage6): static-JSON-shard UI rewrite on GitHub Pages — US1–US8

.github/workflows/deploy-ui.yml

@@ -0,0 +1,76 @@
+name: deploy-ui
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'site/**'
+      - '.github/workflows/deploy-ui.yml'
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  pages: write
+
+concurrency:
+  group: deploy-ui-production
+  cancel-in-progress: false
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Node 20 + pnpm
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+      - name: Enable pnpm
+        run: npm install --global pnpm@10
+
+      - name: Install site deps
+        working-directory: site
+        run: pnpm install --frozen-lockfile
+
+      - name: Run JS unit tests (Vitest)
+        working-directory: site
+        run: pnpm test:unit --run
+
+      - name: Set up Python 3.14
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.14'
+
+      - name: Install link-check deps
+        run: pip install requests pyyaml
+
+      - name: References link check (FR-017)
+        # HEAD every URL in the references registry; non-zero exit blocks
+        # the deploy when a citation 4xx's or times out (T086 / T088).
+        run: PYTHONPATH=src python -m ohbm2026.ui_data.link_check specs/008-ui-rewrite/contracts/references.yaml
+
+      - name: Build site
+        working-directory: site
+        env:
+          BASE_PATH: ''
+          VITE_BUILD_SHA: ${{ github.sha }}
+          VITE_BUILD_AT: ${{ github.event.head_commit.timestamp }}
+          VITE_DATA_PACKAGE_URL: ${{ vars.OHBM2026_UI_DATA_PACKAGE_URL }}
+        run: |
+          export VITE_BUILD_SHA_SHORT="${VITE_BUILD_SHA:0:7}"
+          pnpm build
+
+      - name: Publish to gh-pages root
+        uses: peaceiris/actions-gh-pages@v3
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: site/build
+          publish_branch: gh-pages
+          destination_dir: .
+          keep_files: true
+          enable_jekyll: false
+          commit_message: 'deploy(ui): ${{ github.sha }}'

.github/workflows/pr-preview-cleanup.yml

@@ -0,0 +1,71 @@
+name: pr-preview-cleanup
+
+on:
+  pull_request:
+    types: [closed]
+  workflow_dispatch:
+    inputs:
+      pr_number:
+        description: PR number to clean up
+        required: true
+
+permissions:
+  contents: write
+  deployments: write
+  pull-requests: read
+
+jobs:
+  cleanup:
+    runs-on: ubuntu-latest
+    if: github.event.pull_request.head.repo.full_name == github.repository || github.event_name == 'workflow_dispatch'
+    env:
+      PR_NUMBER: ${{ github.event.pull_request.number || github.event.inputs.pr_number }}
+    steps:
+      - name: Checkout gh-pages
+        uses: actions/checkout@v4
+        with:
+          ref: gh-pages
+          fetch-depth: 1
+
+      - name: Remove pr-${{ env.PR_NUMBER }} preview directory
+        run: |
+          if [ -d "pr-${PR_NUMBER}" ]; then
+            rm -rf "pr-${PR_NUMBER}"
+            git config user.name "github-actions[bot]"
+            git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+            git add -A
+            if git diff --staged --quiet; then
+              echo "Nothing staged after removal — skipping commit."
+            else
+              git commit -m "cleanup(ui): remove pr-${PR_NUMBER} preview"
+              git push
+            fi
+          else
+            echo "No preview directory pr-${PR_NUMBER} to remove."
+          fi
+
+      - name: Mark pr-preview-${{ env.PR_NUMBER }} deployment inactive
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const envName = `pr-preview-${process.env.PR_NUMBER}`;
+            const { data: deployments } = await github.rest.repos.listDeployments({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              environment: envName,
+              per_page: 100,
+            });
+            if (deployments.length === 0) {
+              core.info(`No deployments found for environment ${envName}`);
+              return;
+            }
+            for (const dep of deployments) {
+              await github.rest.repos.createDeploymentStatus({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                deployment_id: dep.id,
+                state: 'inactive',
+                description: 'PR closed; preview cleaned up.',
+              });
+            }
+            core.info(`Marked ${deployments.length} deployment(s) inactive for ${envName}.`);

.github/workflows/pr-preview.yml

@@ -0,0 +1,69 @@
+name: pr-preview
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+    paths:
+      - 'site/**'
+      - '.github/workflows/pr-preview.yml'
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  pages: write
+  deployments: write
+  pull-requests: read
+
+concurrency:
+  group: deploy-ui-preview-pr-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+jobs:
+  preview:
+    runs-on: ubuntu-latest
+    if: github.event.pull_request.head.repo.full_name == github.repository
+    environment:
+      name: pr-preview-${{ github.event.pull_request.number }}
+      url: https://${{ github.repository_owner }}.github.io/${{ github.event.repository.name }}/pr-${{ github.event.pull_request.number }}/
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Node 20 + pnpm
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+      - name: Enable pnpm
+        run: npm install --global pnpm@10
+
+      - name: Install site deps
+        working-directory: site
+        run: pnpm install --frozen-lockfile
+
+      - name: Run JS unit tests (Vitest)
+        working-directory: site
+        run: pnpm test:unit --run
+
+      - name: Build site (PR preview)
+        working-directory: site
+        env:
+          BASE_PATH: /pr-${{ github.event.pull_request.number }}
+          VITE_BUILD_SHA: ${{ github.event.pull_request.head.sha }}
+          VITE_BUILD_AT: ${{ github.event.pull_request.head.repo.pushed_at }}
+          VITE_DATA_PACKAGE_URL: ${{ vars.OHBM2026_UI_DATA_PACKAGE_URL }}
+        run: |
+          export VITE_BUILD_SHA_SHORT="${VITE_BUILD_SHA:0:7}"
+          pnpm build
+
+      - name: Publish to gh-pages /pr-${{ github.event.pull_request.number }}
+        uses: peaceiris/actions-gh-pages@v3
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: site/build
+          publish_branch: gh-pages
+          destination_dir: pr-${{ github.event.pull_request.number }}
+          keep_files: false
+          enable_jekyll: false
+          commit_message: 'preview(ui): pr-${{ github.event.pull_request.number }} ${{ github.sha }}'

.specify/feature.json

@@ -1 +1 @@
-{"feature_directory": "specs/007-package-reorg"}
+{"feature_directory": "specs/008-ui-rewrite"}

CLAUDE.md

@@ -97,7 +97,7 @@ Subcommands group into stages (see README for full options):
 - enrichment: `enrich-abstracts`, `title-audit` (note: `enrich`, `analyze-figures`, `extract-claims`, `reference-metadata` REMOVED in Stage 2 rewire — FR-014. Operators run `enrich-abstracts` and use `--invalidate <component>` for targeted refresh.)
 - embeddings: `embed-matrix` (Stage 3 canonical — per-component bundles for one or more `(model, component)` pairs; supports voyage / minilm / openai / pubmedbert; NeuroScape is a derivation step). Per-model debugging subcommands kept for backward compat: `embed-minilm`, `embed-hf`, `embed-openai`, `embed-voyage`, `embed-stage2`, `apply-published-stage2`. Multi-component recipes (full-manuscript, methods+results, title+results+conclusion) are composed downstream via `neuroscape.compose_recipe([...], model_key=...)`.
 - analysis: `semantic-analysis`, `cluster-benchmark`, `umap-plot`, `compare-projections`, `optimize-projections`
-- UI: `export-ui`, `build-ui`
+- UI data package: `scripts/build_ui_data.py` (Stage 6 canonical; emits `site/static/data/` consumed by the SvelteKit site under `site/`).
 
 Poster layout/sequencing is **not** in `ohbmcli` AND is **parked** as of Stage 5 (see `ohbm2026.layout` parked package and `scripts/layout/` for the 15 companion scripts: `scripts/layout/optimize_poster_layout.py`, `scripts/layout/analyze_poster_layout.py`, `scripts/layout/benchmark_poster_sequencing.py`, etc.). Always pass explicit input paths and a fresh `--output-root`/`--output-dir`; do not rely on stale baked-in defaults.
 
@@ -123,7 +123,7 @@ All library code lives in `src/ohbm2026/`:
 - `artifacts.py` — shared artifact-naming/state-key helpers used across stages.
 - `category_evaluation.py`, `category_rollup.py` — compare learned cluster families against submitter taxonomies.
 - `layout/` (**parked** as of Stage 5 — specs/007-package-reorg/) — poster_layout, poster_sequencing, nocd_experiments. Preserved verbatim for future revival; not actively maintained. Tests under `tests/test_poster_*.py` + `tests/test_nocd_experiments.py` still run with import-paths updated to `ohbm2026.layout.*`. The 15 companion scripts live under `scripts/layout/`. Revive when a new organizer cycle needs poster-layout work.
-- `ui.py` — static UI export (`export-ui` writes a fresh bundle; `build-ui` also mirrors to `export/ui-site/`).
+- `ui_data/` — **Stage 6** UI data-package builders. `manifest.py`, `abstracts.py`, `authors.py`, `cells.py`, `topics.py`, `neighbors.py`, `enrichment.py`, `vectors.py` produce per-shard envelopes (every shard carries a top-level `build_info` block — FR-019 + CA-008). `state_key.py` discovers the corpus + Stage 4 rollup state-keys at build time (CA-007). `builder.py` orchestrates + enforces the 8 cross-shard invariants from `specs/008-ui-rewrite/data-model.md` §8. `link_check.py` HEAD-validates `specs/008-ui-rewrite/contracts/references.yaml` (every external citation from the About page; non-zero exit blocks the deploy — FR-017). CLI entry: `scripts/build_ui_data.py`. Schema: every emitted JSON shard validates against `specs/008-ui-rewrite/contracts/ui_data.linkml.yaml` via `scripts/validate_ui_data.sh`. The SvelteKit site lives at `site/` (self-contained pnpm project; gh-pages deploy via `.github/workflows/{deploy-ui,pr-preview,pr-preview-cleanup}.yml`; runtime data fetched from the Dropbox tarball at `vars.OHBM2026_UI_DATA_PACKAGE_URL`).
 - `cli.py` — single dispatch entrypoint that wires the above into subcommands.
 
 Tests in `tests/` mirror the module names and use `unittest`.
@@ -168,28 +168,33 @@ Current canonical defaults (the UI consumes these):
 <!-- SPECKIT START -->
 For additional context about technologies to be used, project structure,
 shell commands, and other important information, read the current plan
-at `specs/007-package-reorg/plan.md`. The companion design artifacts
+at `specs/008-ui-rewrite/plan.md`. The companion design artifacts
 under the same directory — `research.md`, `data-model.md`, `contracts/`,
-and `quickstart.md` — pin Stage 5 package-reorganization design: three
-independent commit series that collapse the legacy `src/ohbm2026/
-enrichment.py` (1,361 LOC, 62 def/class symbols, ~22+ unused) into
-focused `enrich/` submodules (`text.py`, `cache_paths.py`,
-`markdown_render.py`, `openai_compat.py`); park `poster_layout.py`,
-`poster_sequencing.py`, and `nocd_experiments.py` under
-`src/ohbm2026/layout/` (no scheduled maintenance; revive when a new
-organizer cycle needs it) along with their 15 `scripts/` companions
-under `scripts/layout/`; and split the monolithic `src/ohbm2026/ui.py`
-(1,361 LOC) into a `ui/` package with leaf/mid/trunk submodules
-(`text.py`, `figures.py`, `references.py`, `manifest.py`,
-`payload_legacy.py`, `payload_stage4.py`, `cli.py`). No backward-compat
-shim at any `__init__.py`; every consumer imports from the explicit
-submodule that owns the symbol (Stage 4 / Q2 / T108b precedent). Tests
-are explicitly waived for the refactor body per the user's "OK to skip
-tests" guidance; the existing test suite (583 / 1 pre-existing failure)
-stays green, with `tests/test_enrichment.py` deleted because it covers
-only the legacy Stage 2 path that Stage 2.1 already replaced.
+and `quickstart.md` — pin Stage 6 UI-rewrite design: a static
+SvelteKit + Vite site served from GitHub Pages, built and deployed by
+a GitHub Action with per-PR preview management, consuming a static-
+JSON-shard data package (DuckDB-WASM ruled out at 3,244-abstract scale
+per the Session-2026-05-17 clarification). Adds previously-missing
+capabilities: program-assigned poster ids (not submission ids),
+restored author + affiliation details, 3D UMAP alongside 2D + lasso,
+typo-tolerant lexical + semantic search (MiniLM-L6 ONNX via
+transformers.js with int8-quantized corpus vectors), interactive
+facet recomputation, a saved-list shopping cart with mailto-based
+export, an optional walkthrough (shepherd.js), an About page with
+build-time verified references, and mobile-first responsive layout.
+Accepted abstracts only; withdrawn submissions never surface
+anywhere. Three workflows: `deploy-ui.yml` (main → root), `pr-
+preview.yml` (PR → `/pr-<N>/`), `pr-preview-cleanup.yml` (close →
+remove). 8 user stories; MVP = US1 (search + browse on every device).
+No new secrets beyond the default `GITHUB_TOKEN`. The Python data-
+builder package lives at `src/ohbm2026/ui_data/`; the site lives at
+`site/`.
 
 Previous-stage plans:
+- Stage 5 package reorganization: `specs/007-package-reorg/plan.md`
+  (collapsed `enrichment.py` into `enrich/{text, markdown_render}.py`;
+  parked `layout/`; split `ui.py` into `ui/{payload, cli}.py`;
+  consolidated `load_json`/`write_json` into `ohbm2026/util/json_io.py`).
 - Stage 4 analysis & annotation: `specs/006-analysis-annotation/plan.md`
   (canonical `ohbmcli analyze-matrix` producing 48 bundles + canonical
   rollup `annotations__<state-key>.{parquet,sqlite}`; joblib-parallel

README.md

@@ -103,6 +103,68 @@ The latest end state of the project is:
    - two semantic cluster lenses:
      - `25-cluster benchmark`
      - `claims 28-cluster benchmark`
+9. **Stage 6 — UI** — static SvelteKit site served from GitHub Pages, with per-PR preview deploys surfaced in the PR's Deployments box (NOT bot comments). US1–US7 + US8 all shipped on `008-ui-rewrite`. Production at `abstractatlas.brainkb.org`; PR previews at `/pr-<N>/`. See `specs/008-ui-rewrite/` for spec/plan/tasks and `specs/008-ui-rewrite/quickstart.md` for the local-dev recipe.
+
+## Stage 6: UI
+
+The Stage 6 site lives under `site/` (a self-contained SvelteKit project; SvelteKit 2 + Vite 6 + Svelte 5). The data-package builder lives under `src/ohbm2026/ui_data/`. Capabilities: typo-tolerant lexical search, transformers.js-backed semantic search (MiniLM-L6 ONNX in a Web Worker against an int8-quantised corpus matrix), 2D + 3D UMAP with lasso + cluster colour-coding, interactive facets, cart + email-my-list, a guided tour (shepherd.js), and an About page whose external citations are HEAD-checked at build time (`link_check.py`).
+
+Build the data package + the site locally:
+
+```bash
+PYTHONPATH=src .venv/bin/python scripts/build_ui_data.py \
+  --corpus data/primary/abstracts.json \
+  --withdrawn data/primary/abstracts_withdrawn.json \
+  --authors data/primary/authors.json \
+  --enriched data/primary/abstracts_enriched.sqlite \
+  --analysis-root data/outputs/analysis \
+  --discover-rollup \
+  --output site/static/data
+
+cd site && pnpm install && pnpm dev    # http://localhost:5173
+```
+
+Run the JS test suite:
+
+```bash
+cd site && pnpm test:unit --run        # Vitest unit tests
+cd site && UI_DATA_AVAILABLE=1 pnpm exec playwright test --project=chromium   # e2e (needs the data package built first)
+```
+
+### Refreshing the deployed data package
+
+CI doesn't materialize the Stage 1–4 inputs. Instead, the maintainer builds the data package locally and hosts the tarball at a URL the deploy workflow reads from the `OHBM2026_UI_DATA_PACKAGE_URL` repo variable (sha256-pinned via `OHBM2026_UI_DATA_PACKAGE_SHA256`).
+
+**Dropbox in-place write is required to preserve the share link** ([Dropbox docs](https://help.dropbox.com/share/force-download)). The recipe below uses `tar -czf <canonical-path>` which opens the destination with `O_WRONLY|O_CREAT|O_TRUNC` — preserves the inode, which Dropbox observes as an overwrite (link stays). Do NOT use `cp newfile.tar.gz <canonical-path>` from a different filesystem or `rm canonical && cp …` — both look like delete+create to Dropbox and break the share URL.
+
+```bash
+# 1. Build the data package in place (writes to site/static/data/).
+#    Each shard is written via O_TRUNC so the inode stays stable for any
+#    later in-place tar write; mtimes are pinned to a fixed timestamp for
+#    byte-identical tarballs across reruns.
+PYTHONPATH=src .venv/bin/python scripts/build_ui_data.py \
+  --corpus data/primary/abstracts.json \
+  --withdrawn data/primary/abstracts_withdrawn.json \
+  --authors data/primary/authors.json \
+  --enriched data/primary/abstracts_enriched.sqlite \
+  --analysis-root data/outputs/analysis \
+  --discover-rollup \
+  --output site/static/data
+
+# 2. Write the tarball directly to the canonical Dropbox path. The first
+#    time you do this, share-link the resulting file via the Dropbox UI
+#    and record the `?dl=0` URL as `OHBM2026_UI_DATA_PACKAGE_URL`. Every
+#    refresh hereafter re-uses the same path → same inode → same share URL.
+tar -czf ~/dbm/shares/ohbm2026/ui-data.tar.gz -C site/static data
+
+# 3. Update the sha256 repo variable so CI verifies the new bytes.
+NEW_SHA=$(shasum -a 256 ~/dbm/shares/ohbm2026/ui-data.tar.gz | awk '{print $1}')
+gh variable set OHBM2026_UI_DATA_PACKAGE_SHA256 --body "$NEW_SHA"
+```
+
+The URL repo variable never changes after the first share-link is generated; only the sha256 var bumps per refresh. Next PR-preview deploy fetches + sha256-verifies the new package.
+
+Per-PR previews surface in the **PR's Deployments box** (top-of-PR, via the `environment:` declaration in `.github/workflows/pr-preview.yml`) — NOT as a bot comment. The short committish (first 7 chars of git SHA) bakes into the page `<title>` + the persistent footer affordance via the `VITE_BUILD_SHA` env var injected by the deploy workflows, so reviewers can verify each PR-preview reflects the latest pushed commit at-a-glance (FR-022 + SC-011).
 
 ## External Requirements
 

docs/reproducibility-vision.md

@@ -196,7 +196,31 @@ The usual order is:
 4. embedding commands such as `embed-minilm`, `embed-voyage`, or
    `apply-published-stage2`
 5. `ohbmcli cluster-benchmark` and related semantic-analysis steps
-6. `ohbmcli build-ui`
+6. **Stage 6 UI data package** (canonical site):
+   ```
+   PYTHONPATH=src .venv/bin/python scripts/build_ui_data.py \
+     --corpus data/primary/abstracts.json \
+     --withdrawn data/primary/abstracts_withdrawn.json \
+     --authors data/primary/authors.json \
+     --enriched data/primary/abstracts_enriched.sqlite \
+     --analysis-root data/outputs/analysis \
+     --discover-rollup \
+     --minilm-root data/outputs/embeddings/minilm \
+     --output site/static/data
+   ```
+   Validate the package against the LinkML schema:
+   ```
+   scripts/validate_ui_data.sh
+   ```
+   Then HEAD-check the references registry (FR-017):
+   ```
+   PYTHONPATH=src .venv/bin/python -m ohbm2026.ui_data.link_check \
+     specs/008-ui-rewrite/contracts/references.yaml
+   ```
+   Re-tar onto the Dropbox shared link (in-place to preserve the inode +
+   share URL) and bump `OHBM2026_UI_DATA_PACKAGE_SHA256` so the deploy
+   workflow picks up the new bundle. The SvelteKit site itself is built +
+   deployed by `.github/workflows/deploy-ui.yml` on merge to `main`.
 
 ### Level 3: Rebuild from the upstream abstract source