|
| 1 | +/** |
| 2 | + * Runs the real Esku build in a real browser, feeding it a known recording instead of a |
| 3 | + * camera, and reads the diagnostics panel the way a person would. |
| 4 | + * |
| 5 | + * Why this exists: every other measurement in this project replays landmarks offline, and |
| 6 | + * offline replay carries the dataset's own frame rate baked in. That blind spot cost a long |
| 7 | + * investigation — the app scored 0.696 in `tools/train/simulate_app.py` while writing nothing |
| 8 | + * in a browser, because every segmenter threshold was a frame count tuned at SWL-LSE's 20 fps. |
| 9 | + * Nothing offline could see it. This can. |
| 10 | + * |
| 11 | + * Deliberately *not* a vitest test: CI has no Chrome, no camera and no corpus, and the numbers |
| 12 | + * that matter here depend on how fast the machine is. It asserts only what must hold on any |
| 13 | + * device, and reports the rest for a human to read. |
| 14 | + * |
| 15 | + * Usage and setup: see README.md in this directory. |
| 16 | + */ |
| 17 | +import { readFileSync } from 'node:fs'; |
| 18 | +import { basename, join } from 'node:path'; |
| 19 | +import process from 'node:process'; |
| 20 | +import puppeteer from 'puppeteer-core'; |
| 21 | + |
| 22 | +const HERE = import.meta.dirname; |
| 23 | +const ROOT = join(HERE, '..', '..'); |
| 24 | + |
| 25 | +const BASE = process.env.BASE ?? 'http://localhost:4199/esku/'; |
| 26 | +const CHROME = process.env.CHROME ?? join(HERE, 'chrome-linux64', 'chrome'); |
| 27 | +const SECONDS = Number(process.env.SECONDS ?? 20); |
| 28 | +const PLAYBACK = Number(process.env.PLAYBACK ?? 1); |
| 29 | + |
| 30 | +/** MediaPipe logs this at info level on every start; it is not an error. */ |
| 31 | +const BENIGN = /XNNPACK delegate|Created TensorFlow Lite/; |
| 32 | + |
| 33 | +/** |
| 34 | + * The shipped thresholds, read from the source rather than copied. |
| 35 | + * |
| 36 | + * Duplicating them here would let this tool quietly disagree with the app it is measuring, |
| 37 | + * which is the exact class of bug it was built to catch. A miss throws instead of guessing. |
| 38 | + */ |
| 39 | +function shippedFloor() { |
| 40 | + const source = readFileSync( |
| 41 | + join(ROOT, 'src', 'domain', 'recognition', 'services', 'SignSegmenter.ts'), |
| 42 | + 'utf-8', |
| 43 | + ); |
| 44 | + const read = (key) => { |
| 45 | + const found = source.match(new RegExp(`^\\s+${key}:\\s*([0-9.]+),`, 'm')); |
| 46 | + if (!found) throw new Error(`cannot read ${key} from SignSegmenter.ts — did it get renamed?`); |
| 47 | + return Number(found[1]); |
| 48 | + }; |
| 49 | + const minSignMs = read('minSignMs'); |
| 50 | + const minFrames = read('minFrames'); |
| 51 | + // A window may only close once it has run minSignMs, and is then thrown away unless it |
| 52 | + // holds minFrames samples. Below this rate the two rules cannot both be satisfied. |
| 53 | + return { minSignMs, minFrames, requiredFps: minFrames / (minSignMs / 1000) }; |
| 54 | +} |
| 55 | + |
| 56 | +/** |
| 57 | + * Replaces the camera with a video, before any app code runs. |
| 58 | + * |
| 59 | + * `captureStream()` on a playing <video> yields a MediaStream indistinguishable from a camera |
| 60 | + * to getUserMedia's callers — no fake-device flags, no y4m conversion, no file serving. The |
| 61 | + * clip travels as a data URL so it is same-origin by construction. Decode state is reported |
| 62 | + * back, because a missing codec must not be able to masquerade as "recognised nothing". |
| 63 | + */ |
| 64 | +function fakeCamera({ dataUrl, rate }) { |
| 65 | + const shim = async () => { |
| 66 | + const video = document.createElement('video'); |
| 67 | + video.src = dataUrl; |
| 68 | + video.muted = true; |
| 69 | + video.loop = true; |
| 70 | + video.playsInline = true; |
| 71 | + video.playbackRate = rate; |
| 72 | + await new Promise((resolve, reject) => { |
| 73 | + video.addEventListener('loadeddata', resolve, { once: true }); |
| 74 | + video.addEventListener('error', () => reject(new Error('clip failed to decode')), { |
| 75 | + once: true, |
| 76 | + }); |
| 77 | + setTimeout(() => reject(new Error('clip timed out loading')), 20000); |
| 78 | + }); |
| 79 | + await video.play(); |
| 80 | + window.__feed = { width: video.videoWidth, height: video.videoHeight }; |
| 81 | + return video.captureStream(); |
| 82 | + }; |
| 83 | + navigator.mediaDevices.getUserMedia = shim; |
| 84 | + navigator.getUserMedia = (_constraints, ok, fail) => shim().then(ok, fail); |
| 85 | +} |
| 86 | + |
| 87 | +/** Reads the panel as label/value pairs, which survives wording changes better than text. */ |
| 88 | +function readPanel() { |
| 89 | + const rows = {}; |
| 90 | + for (const row of document.querySelectorAll('#diag-body .diagnostics__row')) { |
| 91 | + const label = row.querySelector('dt')?.textContent?.trim(); |
| 92 | + const value = row.querySelector('dd')?.textContent?.trim(); |
| 93 | + if (label) rows[label] = value ?? ''; |
| 94 | + } |
| 95 | + return { |
| 96 | + rows, |
| 97 | + transcript: document.querySelector('#transcript')?.textContent?.trim() ?? '', |
| 98 | + status: document.querySelector('#status')?.textContent?.trim() ?? '', |
| 99 | + feed: window.__feed ?? null, |
| 100 | + }; |
| 101 | +} |
| 102 | + |
| 103 | +const firstNumber = (text) => Number(text?.match(/-?\d+(\.\d+)?/)?.[0] ?? Number.NaN); |
| 104 | +const nthNumber = (text, n) => Number(text?.match(/-?\d+(\.\d+)?/g)?.[n] ?? Number.NaN); |
| 105 | + |
| 106 | +async function measure(browser, clip, floor) { |
| 107 | + const page = await browser.newPage(); |
| 108 | + const problems = []; |
| 109 | + page.on('pageerror', (error) => problems.push(`pageerror: ${error.message}`)); |
| 110 | + page.on('console', (message) => { |
| 111 | + if (message.type() === 'error' && !BENIGN.test(message.text())) { |
| 112 | + problems.push(`console: ${message.text().slice(0, 200)}`); |
| 113 | + } |
| 114 | + }); |
| 115 | + page.on('requestfailed', (request) => |
| 116 | + problems.push(`request failed: ${request.url()} (${request.failure()?.errorText})`), |
| 117 | + ); |
| 118 | + |
| 119 | + const bytes = readFileSync(clip).toString('base64'); |
| 120 | + await page.evaluateOnNewDocument(fakeCamera, { |
| 121 | + dataUrl: `data:video/mp4;base64,${bytes}`, |
| 122 | + rate: PLAYBACK, |
| 123 | + }); |
| 124 | + await page.goto(BASE, { waitUntil: 'networkidle2', timeout: 180000 }); |
| 125 | + |
| 126 | + // The app not booting at all is the loudest regression this can catch — a wrong base path |
| 127 | + // 404s the bundle, and the SPA fallback answers 200 for it, so the page looks served and is |
| 128 | + // empty. Waiting for the control gives that a clear verdict instead of a puppeteer stack. |
| 129 | + const booted = await page |
| 130 | + .waitForSelector('#toggle', { timeout: 30000 }) |
| 131 | + .then(() => true) |
| 132 | + .catch(() => false); |
| 133 | + if (!booted) { |
| 134 | + problems.push('the app never mounted: #toggle absent (wrong BASE? bundle 404?)'); |
| 135 | + await page.close(); |
| 136 | + return { word: basename(clip, '.mp4'), problems, booted: false, framesWithHands: 0 }; |
| 137 | + } |
| 138 | + |
| 139 | + await page.click('#toggle'); |
| 140 | + |
| 141 | + const startedAt = Date.now(); |
| 142 | + await new Promise((resolve) => setTimeout(resolve, SECONDS * 1000)); |
| 143 | + const elapsedS = (Date.now() - startedAt) / 1000; |
| 144 | + |
| 145 | + await page.click('#diag-toggle'); |
| 146 | + const report = await page.evaluate(readPanel); |
| 147 | + await page.close(); |
| 148 | + |
| 149 | + const framesSeen = firstNumber(report.rows.Fotogramas); |
| 150 | + return { |
| 151 | + word: basename(clip, '.mp4'), |
| 152 | + problems, |
| 153 | + feed: report.feed, |
| 154 | + status: report.status, |
| 155 | + transcript: report.transcript, |
| 156 | + framesSeen, |
| 157 | + framesWithHands: nthNumber(report.rows.Fotogramas, 1), |
| 158 | + fps: framesSeen / elapsedS, |
| 159 | + windowsClosed: firstNumber(report.rows['Ventanas cerradas']), |
| 160 | + windowsShort: nthNumber(report.rows['Ventanas cerradas'], 1), |
| 161 | + engineLoaded: report.rows['Motor cargado'] === 'sí', |
| 162 | + invocations: firstNumber(report.rows['Veces consultado']), |
| 163 | + words: firstNumber(report.rows['Palabras del vocabulario']), |
| 164 | + vetoedBy: report.rows['Bloqueado por'], |
| 165 | + rawTop: report.rows['Mejores opciones, sin filtrar'], |
| 166 | + signature: report.rows['Lo que recibió el modelo (esperado entre paréntesis)'], |
| 167 | + fastEnough: framesSeen / elapsedS >= floor.requiredFps, |
| 168 | + }; |
| 169 | +} |
| 170 | + |
| 171 | +const clips = process.argv.slice(2); |
| 172 | +if (clips.length === 0) { |
| 173 | + console.error('usage: node harness.mjs <clip.mp4> [more.mp4 ...] (see README.md)'); |
| 174 | + process.exit(2); |
| 175 | +} |
| 176 | + |
| 177 | +const floor = shippedFloor(); |
| 178 | +console.log( |
| 179 | + `shipped floor: minSignMs ${floor.minSignMs} over minFrames ${floor.minFrames} ` + |
| 180 | + `=> a device must sustain ${floor.requiredFps.toFixed(1)} fps for any sign to survive\n`, |
| 181 | +); |
| 182 | + |
| 183 | +const browser = await puppeteer.launch({ |
| 184 | + executablePath: CHROME, |
| 185 | + headless: true, |
| 186 | + args: [ |
| 187 | + '--no-sandbox', |
| 188 | + '--use-fake-ui-for-media-stream', |
| 189 | + '--autoplay-policy=no-user-gesture-required', |
| 190 | + // MediaPipe wants a GPU delegate. Headless here gets software WebGL — about 1.3 fps, well |
| 191 | + // under the floor above, so recognition cannot be asserted from this machine. Measured: |
| 192 | + // WSL's /dev/dxg does not help, WebGL falls back to software regardless. |
| 193 | + ...(process.env.GL === 'auto' |
| 194 | + ? [] |
| 195 | + : ['--use-gl=angle', '--use-angle=swiftshader', '--enable-unsafe-swiftshader']), |
| 196 | + ], |
| 197 | +}); |
| 198 | + |
| 199 | +const results = []; |
| 200 | +try { |
| 201 | + for (const clip of clips) results.push(await measure(browser, clip, floor)); |
| 202 | +} finally { |
| 203 | + await browser.close(); |
| 204 | +} |
| 205 | + |
| 206 | +for (const r of results) { |
| 207 | + console.log(`${'='.repeat(70)}\n${r.word}\n${'='.repeat(70)}`); |
| 208 | + if (r.booted === false) { |
| 209 | + console.log(`problems : ${r.problems.join('\n ')}\n`); |
| 210 | + continue; |
| 211 | + } |
| 212 | + console.log(`feed : ${r.feed ? `${r.feed.width}x${r.feed.height}` : 'NO DECODE'}`); |
| 213 | + console.log(`frames : ${r.framesSeen} (${r.framesWithHands} with a hand)`); |
| 214 | + console.log(`frame rate : ${r.fps.toFixed(1)} fps ${r.fastEnough ? '' : '<-- below floor'}`); |
| 215 | + console.log(`engine loaded : ${r.engineLoaded ? 'yes' : 'NO'}`); |
| 216 | + console.log(`windows : ${r.windowsClosed} closed, ${r.windowsShort} discarded as short`); |
| 217 | + console.log(`engine asked : ${r.invocations}`); |
| 218 | + console.log(`words written : ${r.words} vetoed by: ${r.vetoedBy}`); |
| 219 | + console.log(`raw scores : ${r.rawTop}`); |
| 220 | + console.log(`fed the model : ${r.signature}`); |
| 221 | + console.log(`transcript : ${JSON.stringify(r.transcript)}`); |
| 222 | + if (r.problems.length) console.log(`problems : ${r.problems.join('\n ')}`); |
| 223 | + console.log(); |
| 224 | +} |
| 225 | + |
| 226 | +/** |
| 227 | + * Only what holds on any device is a failure. |
| 228 | + * |
| 229 | + * Recognition itself is not assertable here: it needs a frame rate this machine cannot reach, |
| 230 | + * and pretending otherwise would either produce a permanently red check or invite someone to |
| 231 | + * lower a shipped threshold to make it green. What *is* assertable catches real regressions — |
| 232 | + * a base-path break that 404s the weights, a landmark pipeline that stops producing hands, a |
| 233 | + * clip that silently fails to decode. |
| 234 | + */ |
| 235 | +const failures = []; |
| 236 | +for (const r of results) { |
| 237 | + if (r.problems.length) failures.push(`${r.word}: ${r.problems.join('; ')}`); |
| 238 | + if (r.booted === false) continue; |
| 239 | + if (!r.feed?.width) failures.push(`${r.word}: clip never decoded`); |
| 240 | + if (!r.engineLoaded) failures.push(`${r.word}: vocabulary weights never loaded`); |
| 241 | + if (!(r.framesWithHands > 0)) failures.push(`${r.word}: no frame ever had a hand in it`); |
| 242 | + // Only meaningful once the device is fast enough for a window to be able to survive. |
| 243 | + if (r.fastEnough && r.invocations === 0) { |
| 244 | + failures.push( |
| 245 | + `${r.word}: ${r.fps.toFixed(1)} fps is above the floor yet the engine was never asked`, |
| 246 | + ); |
| 247 | + } |
| 248 | +} |
| 249 | + |
| 250 | +if (failures.length) { |
| 251 | + console.log('FAILED'); |
| 252 | + for (const failure of failures) console.log(` - ${failure}`); |
| 253 | + process.exit(1); |
| 254 | +} |
| 255 | + |
| 256 | +const slow = results.filter((r) => !r.fastEnough); |
| 257 | +console.log('PASSED — invariants hold'); |
| 258 | +if (slow.length) { |
| 259 | + console.log( |
| 260 | + ` note: ${slow.length}/${results.length} run(s) below ${floor.requiredFps.toFixed(1)} fps, ` + |
| 261 | + 'so recognition was measured, not asserted. Read the numbers above.', |
| 262 | + ); |
| 263 | +} |
0 commit comments