-
Notifications
You must be signed in to change notification settings - Fork 1
feat(polish): expanded feedback build_info + SC sweep + lexical warm-up #14
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,171 @@ | ||
| /** | ||
| * T096 — Success-Criteria sweep against production. | ||
| * | ||
| * Each `test()` here verifies one SC item from spec.md. Runs against the | ||
| * live production URL by default; override with `TARGET_BASE`. | ||
| * | ||
| * Coverage in this spec (locally observable, no live data needed beyond a | ||
| * deployed site): | ||
| * | ||
| * SC-002 search latency — type into the search bar; assert the | ||
| * visible result count updates within 500 ms. | ||
| * SC-003 cell-switch timing — change the model dropdown; assert the | ||
| * UMAP cell shard swap completes < 1.5 s. | ||
| * SC-004 mobile viewport — 360 × 640 viewport has no horizontal | ||
| * scroll on the home page. | ||
| * SC-005 accepted-only — the abstracts shard contains zero | ||
| * `accepted_for === "Withdrawn"` records. | ||
| * SC-011 footer build SHA — home + about + abstract-permalink all | ||
| * render the same 7-char short SHA in | ||
| * the footer. | ||
| * | ||
| * Out of scope (need separate infra): | ||
| * SC-001 first interactive paint — measured by Lighthouse-CI workflow. | ||
| * SC-006 data-package size — measured against the local build dir. | ||
| * SC-007 link check — runs in deploy-ui.yml as a hard gate. | ||
| * SC-008 PR-preview timing — observed manually across several PRs. | ||
| * SC-009 cart reload — covered by cart_email + cart unit tests. | ||
| * SC-010 typo recall — separate `eval_typo_recall.py` script. | ||
| * SC-012 AI-attribution — separate Playwright assertion (TBD). | ||
| */ | ||
|
|
||
| import { test, expect, chromium, devices } from '@playwright/test'; | ||
|
|
||
| test.setTimeout(180_000); | ||
|
|
||
| const BASE = (process.env.TARGET_BASE || 'https://abstractatlas.brainkb.org').replace(/\/$/, ''); | ||
|
|
||
| test.describe('SC sweep', () => { | ||
| test('SC-002 — search latency: typing returns a filtered count in < 500 ms (warm path)', async () => { | ||
| // In a real session the semantic worker pre-warms during page load, | ||
| // so by the time the user types, it's ready. We mirror that here: | ||
| // wait for the ✨ Semantic toggle to drop its `loading` class | ||
| // (i.e., the worker reached `ready`) before measuring keystroke | ||
| // latency. Cold-start latency is bounded by the MiniLM ONNX | ||
| // download (~23 MB) and is reported separately below for context. | ||
| const browser = await chromium.launch(); | ||
| const ctx = await browser.newContext({ viewport: { width: 1440, height: 900 } }); | ||
| const page = await ctx.newPage(); | ||
| await page.goto(`${BASE}/`, { waitUntil: 'load' }); | ||
| await page.waitForSelector('[data-testid="result-card"]', { timeout: 30000 }); | ||
| // Wait for the semantic worker to settle. | ||
| await page | ||
| .waitForFunction( | ||
| () => { | ||
| const btn = document.querySelector<HTMLButtonElement>('[data-testid="toggle-semantic"]'); | ||
| return !!btn && !btn.classList.contains('loading') && !btn.disabled; | ||
| }, | ||
| { timeout: 30000 } | ||
| ) | ||
| .catch(() => null); | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Swallowing the timeout error from ); |
||
| await page.waitForTimeout(500); // belt-and-braces idle | ||
|
|
||
| const before = await page.locator('[data-testid="result-count"]').textContent(); | ||
| const start = Date.now(); | ||
| await page.getByTestId('search-input').fill('memory'); | ||
| await expect | ||
| .poll( | ||
| async () => { | ||
| const t = (await page.locator('[data-testid="result-count"]').textContent())?.trim(); | ||
| return t !== before && /^\d+$/.test(t || ''); | ||
| }, | ||
| { timeout: 1500, intervals: [50, 100, 200] } | ||
| ) | ||
| .toBe(true); | ||
| const elapsed = Date.now() - start; | ||
| console.log(`SC-002 search latency (warm): ${elapsed} ms`); | ||
| expect(elapsed).toBeLessThanOrEqual(500); | ||
| await browser.close(); | ||
| }); | ||
|
|
||
| test('SC-003 — cell-switch timing: model dropdown change re-renders UMAP in < 1500 ms', async () => { | ||
| const browser = await chromium.launch(); | ||
| const ctx = await browser.newContext({ viewport: { width: 1440, height: 900 } }); | ||
| const page = await ctx.newPage(); | ||
| await page.goto(`${BASE}/`, { waitUntil: 'load' }); | ||
| await page.waitForSelector('[data-testid="umap-chart-2d"]', { timeout: 30000 }); | ||
| // Wait for initial UMAP render | ||
| await page.waitForTimeout(2000); | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. |
||
| // Switch model — neuroscape → voyage. | ||
| const start = Date.now(); | ||
| await page | ||
| .locator('[data-testid="model-selector-model"]') | ||
| .selectOption({ value: 'voyage' }) | ||
| .catch(() => null); | ||
| await page.waitForFunction( | ||
| () => { | ||
| const el = document.querySelector('h3 code'); | ||
| return el?.textContent?.startsWith('voyage_'); | ||
| }, | ||
| { timeout: 5000 } | ||
| ); | ||
| const elapsed = Date.now() - start; | ||
| console.log(`SC-003 cell-switch timing: ${elapsed} ms`); | ||
| expect(elapsed).toBeLessThanOrEqual(1500); | ||
| await browser.close(); | ||
| }); | ||
|
|
||
| test('SC-004 — 360×640 mobile: home page has no horizontal scroll', async () => { | ||
| const browser = await chromium.launch(); | ||
| const ctx = await browser.newContext({ ...devices['Pixel 5'], viewport: { width: 360, height: 640 } }); | ||
| const page = await ctx.newPage(); | ||
| await page.goto(`${BASE}/`, { waitUntil: 'load' }); | ||
| await page.waitForSelector('[data-testid="search-input"]', { timeout: 30000 }); | ||
| await page.waitForTimeout(1500); | ||
| const scroll = await page.evaluate(() => ({ | ||
| docW: document.documentElement.scrollWidth, | ||
| viewW: document.documentElement.clientWidth | ||
| })); | ||
| console.log(`SC-004 mobile overflow: docW=${scroll.docW} viewW=${scroll.viewW}`); | ||
| expect(scroll.docW).toBeLessThanOrEqual(scroll.viewW + 1); | ||
| await browser.close(); | ||
| }); | ||
|
|
||
| test('SC-005 — accepted-only: no Withdrawn records leak into the deployed corpus', async () => { | ||
| const browser = await chromium.launch(); | ||
| const ctx = await browser.newContext({ viewport: { width: 1440, height: 900 } }); | ||
| const page = await ctx.newPage(); | ||
| await page.goto(`${BASE}/`, { waitUntil: 'load' }); | ||
| await page.waitForSelector('[data-testid="result-card"]', { timeout: 30000 }); | ||
| await page.waitForTimeout(2000); | ||
| // `+page.svelte` exposes a debug `window.__abstracts` snapshot. | ||
| const withdrawnCount = await page.evaluate(() => { | ||
| const arr = (window as unknown as { __abstracts?: { accepted_for: string }[] }).__abstracts; | ||
| if (!Array.isArray(arr)) return -1; | ||
| return arr.filter((a) => a.accepted_for === 'Withdrawn').length; | ||
| }); | ||
| console.log(`SC-005 withdrawn-count: ${withdrawnCount}`); | ||
| expect(withdrawnCount).toBe(0); | ||
| await browser.close(); | ||
| }); | ||
|
|
||
| test('SC-011 — footer build_info SHA consistent across home / about / abstract permalink', async () => { | ||
| const browser = await chromium.launch(); | ||
| const ctx = await browser.newContext({ viewport: { width: 1440, height: 900 } }); | ||
| const page = await ctx.newPage(); | ||
| await page.goto(`${BASE}/`, { waitUntil: 'load' }); | ||
| await page.waitForSelector('[data-testid="build-info-short-sha"]', { timeout: 30000 }); | ||
| const homeSha = (await page.locator('[data-testid="build-info-short-sha"]').first().textContent())?.trim(); | ||
| expect(homeSha).toMatch(/^[0-9a-f]{7,12}$/); | ||
|
|
||
| await page.goto(`${BASE}/about/`, { waitUntil: 'load' }); | ||
| await page.waitForSelector('[data-testid="build-info-short-sha"]', { timeout: 30000 }); | ||
| const aboutSha = (await page.locator('[data-testid="build-info-short-sha"]').first().textContent())?.trim(); | ||
| expect(aboutSha).toBe(homeSha); | ||
|
|
||
| // Sample a real poster_id from the home page to test the permalink route. | ||
| await page.goto(`${BASE}/`, { waitUntil: 'load' }); | ||
| await page.waitForSelector('[data-testid="result-card"]', { timeout: 30000 }); | ||
| const posterId = await page.locator('[data-testid="result-card"]').first().getAttribute('data-poster-id'); | ||
| if (posterId) { | ||
| await page.goto(`${BASE}/abstract/${encodeURIComponent(posterId)}/`, { waitUntil: 'load' }); | ||
| await page.waitForSelector('[data-testid="build-info-short-sha"]', { timeout: 30000 }); | ||
| const permalinkSha = ( | ||
| await page.locator('[data-testid="build-info-short-sha"]').first().textContent() | ||
| )?.trim(); | ||
| expect(permalinkSha).toBe(homeSha); | ||
| console.log(`SC-011 SHA consistent across routes: ${homeSha}`); | ||
| } | ||
| await browser.close(); | ||
| }); | ||
| }); | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Using the
pagefixture is the standard and recommended way to write Playwright tests. It automatically handles browser and context creation/cleanup, and ensures the test respects the global configuration (likebaseURLorviewport) defined inplaywright.config.ts. Manually launching the browser withchromium.launch()is less efficient and requires manual cleanup.Note: This pattern repeats in all tests in this file; consider updating all of them and removing the manual
browser.close()calls.