feat: measure what each vision model costs per frame

Author: Endika

src/application/use-cases/RecognizeSignsUseCase.ts

@@ -291,6 +291,7 @@ export class RecognizeSignsUseCase {
       wordsEmitted: this.wordsEmitted,
       lettersEmitted: this.lettersEmitted,
       lastSignature: this.lastSignature,
+      frameCost: this.source.frameCost?.() ?? null,
     };
   }
 

src/domain/landmarks/services/ILandmarkSource.ts

@@ -1,3 +1,4 @@
+import type { FrameCost } from '../value-objects/FrameCost';
 import type { LandmarkFrame } from '../value-objects/LandmarkFrame';
 
 export type LandmarkListener = (frame: LandmarkFrame) => void;
@@ -11,6 +12,8 @@ export interface ILandmarkSource {
   start(listener: LandmarkListener): Promise<void>;
   stop(): void;
   isRunning(): boolean;
+  /** Optional: only a source with real models has a per-model cost to report. */
+  frameCost?(): FrameCost | null;
 }
 
 export class CameraUnavailableError extends Error {

src/domain/landmarks/value-objects/FrameCost.ts

@@ -0,0 +1,64 @@
+/**
+ * What one frame of the vision pipeline costs, per model.
+ *
+ * Frame rate is not a comfort metric in this app: below the segmenter's floor no window can
+ * satisfy both `minSignMs` and `minFrames`, so the app writes nothing at all. Deciding which
+ * model to make cheaper needs its share of the budget measured on a real device, not guessed
+ * from model file sizes — the face model is the smallest of the three on disk.
+ */
+export interface FrameCost {
+  /** Median wall-clock milliseconds of one `detectForVideo` pass. */
+  readonly handsMs: number;
+  readonly poseMs: number;
+  readonly faceMs: number;
+  /** How many frames the medians are taken over. */
+  readonly samples: number;
+}
+
+/**
+ * Median of the samples taken so far, per model.
+ *
+ * Medians, not means: a frame that lands on a garbage collection or a lost animation frame is
+ * several times the typical cost, and this project has already been misled once by comparing
+ * against means over data with rare extreme outliers.
+ */
+export class FrameCostMeter {
+  private readonly hands: number[] = [];
+  private readonly pose: number[] = [];
+  private readonly face: number[] = [];
+
+  constructor(private readonly window = 120) {}
+
+  record(handsMs: number, poseMs: number, faceMs: number): void {
+    this.push(this.hands, handsMs);
+    this.push(this.pose, poseMs);
+    this.push(this.face, faceMs);
+  }
+
+  reset(): void {
+    this.hands.length = 0;
+    this.pose.length = 0;
+    this.face.length = 0;
+  }
+
+  read(): FrameCost | null {
+    if (this.hands.length === 0) return null;
+    return {
+      handsMs: median(this.hands),
+      poseMs: median(this.pose),
+      faceMs: median(this.face),
+      samples: this.hands.length,
+    };
+  }
+
+  private push(into: number[], value: number): void {
+    into.push(value);
+    if (into.length > this.window) into.shift();
+  }
+}
+
+function median(values: readonly number[]): number {
+  const sorted = [...values].sort((a, b) => a - b);
+  const middle = sorted.length >> 1;
+  return sorted.length % 2 === 1 ? sorted[middle]! : (sorted[middle - 1]! + sorted[middle]!) / 2;
+}

src/domain/landmarks/value-objects/__tests__/FrameCost.test.ts

@@ -0,0 +1,42 @@
+import { describe, expect, it } from 'vitest';
+import { FrameCostMeter } from '../FrameCost';
+
+describe('FrameCostMeter', () => {
+  it('reads nothing before a frame has been measured', () => {
+    expect(new FrameCostMeter().read()).toBeNull();
+  });
+
+  it('reports the median of each model, not the mean', () => {
+    const meter = new FrameCostMeter();
+    meter.record(10, 5, 2);
+    meter.record(12, 6, 3);
+    // One frame landing on a garbage collection must not move the reading
+    meter.record(400, 200, 100);
+
+    expect(meter.read()).toEqual({ handsMs: 12, poseMs: 6, faceMs: 3, samples: 3 });
+  });
+
+  it('averages the two middle samples when the count is even', () => {
+    const meter = new FrameCostMeter();
+    meter.record(10, 4, 1);
+    meter.record(20, 6, 3);
+
+    expect(meter.read()).toMatchObject({ handsMs: 15, poseMs: 5, faceMs: 2 });
+  });
+
+  it('keeps only the most recent frames, so a slow start stops counting', () => {
+    const meter = new FrameCostMeter(4);
+    for (const ms of [100, 100, 100, 100]) meter.record(ms, ms, ms);
+    for (const ms of [10, 10, 10, 10]) meter.record(ms, ms, ms);
+
+    expect(meter.read()).toEqual({ handsMs: 10, poseMs: 10, faceMs: 10, samples: 4 });
+  });
+
+  it('forgets everything on reset, since the cameras do not cost the same', () => {
+    const meter = new FrameCostMeter();
+    meter.record(10, 5, 2);
+    meter.reset();
+
+    expect(meter.read()).toBeNull();
+  });
+});

src/domain/recognition/value-objects/RecognitionDiagnostics.ts

@@ -1,3 +1,5 @@
+import type { FrameCost } from '@domain/landmarks/value-objects/FrameCost';
+
 /**
  * What the pipeline actually did, so a silent app can be told apart from a wrong one.
  *
@@ -65,6 +67,8 @@ export interface RecognitionDiagnostics {
   readonly lettersEmitted: number;
   /** What the model was actually fed for the last window. Null until one is classified. */
   readonly lastSignature: SignatureProfile | null;
+  /** Per-model cost of a frame. Null until the camera has produced one. */
+  readonly frameCost: FrameCost | null;
 }
 
 export const EMPTY_DIAGNOSTICS: RecognitionDiagnostics = {
@@ -82,4 +86,5 @@ export const EMPTY_DIAGNOSTICS: RecognitionDiagnostics = {
   wordsEmitted: 0,
   lettersEmitted: 0,
   lastSignature: null,
+  frameCost: null,
 };

src/infrastructure/vision/MediaPipeLandmarkSource.ts

@@ -3,6 +3,7 @@ import {
   type ILandmarkSource,
   type LandmarkListener,
 } from '@domain/landmarks/services/ILandmarkSource';
+import { type FrameCost, FrameCostMeter } from '@domain/landmarks/value-objects/FrameCost';
 import {
   createHandLandmarks,
   type Handedness,
@@ -44,6 +45,7 @@ export class MediaPipeLandmarkSource implements ILandmarkSource {
   private running = false;
   private facing: CameraFacing = 'user';
   private listener: LandmarkListener | null = null;
+  private readonly cost = new FrameCostMeter();
 
   constructor(
     private readonly video: HTMLVideoElement,
@@ -54,6 +56,10 @@ export class MediaPipeLandmarkSource implements ILandmarkSource {
     return this.running;
   }
 
+  frameCost(): FrameCost | null {
+    return this.cost.read();
+  }
+
   get camera(): CameraFacing {
     return this.facing;
   }
@@ -139,8 +145,14 @@ export class MediaPipeLandmarkSource implements ILandmarkSource {
         this.lastVideoTime = this.video.currentTime;
         const timestampMs = performance.now();
         const result = this.landmarker.detectForVideo(this.video, timestampMs);
+        const afterHands = performance.now();
         const pose = this.pose?.detectForVideo(this.video, timestampMs);
+        const afterPose = performance.now();
         const face = this.face?.detectForVideo(this.video, timestampMs);
+        const afterFace = performance.now();
+
+        // VIDEO mode returns the result, so the call has waited for its own GPU work
+        this.cost.record(afterHands - timestampMs, afterPose - afterHands, afterFace - afterPose);
 
         listener({
           timestampMs,
@@ -160,6 +172,8 @@ export class MediaPipeLandmarkSource implements ILandmarkSource {
     if (this.frameHandle !== null) cancelAnimationFrame(this.frameHandle);
     this.frameHandle = null;
     this.lastVideoTime = -1;
+    // Front and rear cameras do not cost the same, and useCamera() switches through here
+    this.cost.reset();
 
     // Releasing the tracks is what turns the camera indicator off. Leaving them live would
     // keep recording in the background, which this app must never appear to do.

src/presentation/components/DiagnosticsPanel.ts

@@ -1,3 +1,4 @@
+import type { FrameCost } from '@domain/landmarks/value-objects/FrameCost';
 import {
   EMPTY_DIAGNOSTICS,
   type RecognitionDiagnostics,
@@ -26,6 +27,16 @@ const VETO_LABELS: Record<WindowVeto, string> = {
   duplicate: 'repetido: mismo signo que el anterior',
 };
 
+function frameCostLabel(cost: FrameCost | null): string {
+  if (!cost) return '—';
+  const total = cost.handsMs + cost.poseMs + cost.faceMs;
+  const share = (ms: number) => `${ms.toFixed(0)} ms`;
+  return (
+    `${share(total)} · manos ${share(cost.handsMs)} · pose ${share(cost.poseMs)}` +
+    ` · cara ${share(cost.faceMs)}`
+  );
+}
+
 /**
  * Shows what the pipeline did, so "it writes nothing" becomes a specific failure.
  *
@@ -78,6 +89,7 @@ export class DiagnosticsPanel {
     // closing windows at all, is the engine being asked, and what did it actually score.
     const rows: [string, string][] = [
       ['Fotogramas', `${d.framesSeen} (${d.framesWithHands} con mano)`],
+      ['Coste por fotograma', frameCostLabel(d.frameCost)],
       ['Segmentador', d.segmenterActive ? `activo, ${d.pendingFrames} fotogramas` : 'en reposo'],
       ['Ventanas cerradas', `${d.windowsClosed} (${d.windowsTooShort} descartadas por cortas)`],
       ['Última ventana', d.lastWindowFrames ? `${d.lastWindowFrames} fotogramas` : '—'],