fix: measure sign windows in time, so recognition survives any frame rate

Author: Endika

src/application/use-cases/__tests__/RecognizeSignsUseCase.test.ts

@@ -84,14 +84,30 @@ class CountingWindowClassifier implements ISignClassifier {
   }
 }
 
+const FRAME_MS = 33;
+
 function movingFrames(count: number, from = 0) {
   return Array.from({ length: count }, (_, i) =>
-    buildFrame((from + i) * 33, buildHand({ offset: { x: (from + i) * 0.06, y: 0 } })),
+    buildFrame((from + i) * FRAME_MS, buildHand({ offset: { x: (from + i) * 0.06, y: 0 } })),
   );
 }
 
-function stillFrames(count: number, at: number) {
-  return Array.from({ length: count }, () => buildFrame(0, buildHand({ offset: { x: at, y: 0 } })));
+/**
+ * One complete sign: moving, then held still long enough to read as a boundary.
+ *
+ * The frame count is chosen for *duration* — 40 frames at 33 ms is 1.3 s, past the
+ * segmenter's `minSignMs`. Timestamps have to be real and continuous: these frames used to
+ * be stamped 0 and the segmenter counted frames, so the lie was invisible. It is not any
+ * more, and a collapsed timestamp span now silently means "no sign happened".
+ */
+function scriptedSign(moving = 40, still = 6) {
+  const heldAt = (moving - 1) * 0.06;
+  return [
+    ...movingFrames(moving),
+    ...Array.from({ length: still }, (_, i) =>
+      buildFrame((moving + i) * FRAME_MS, buildHand({ offset: { x: heldAt, y: 0 } })),
+    ),
+  ];
 }
 
 const tick = () => new Promise((resolve) => setTimeout(resolve, 0));
@@ -121,7 +137,7 @@ describe('RecognizeSignsUseCase', () => {
     await tick();
 
     // The sign now completes entirely while that first call is still in flight.
-    for (const frame of [...movingFrames(20), ...stillFrames(14, 1.2)]) source.push(frame);
+    for (const frame of scriptedSign()) source.push(frame);
     expect(vocabulary.calls).toBe(0);
 
     slow.blocking = false;
@@ -133,7 +149,7 @@ describe('RecognizeSignsUseCase', () => {
 
   it('transcribes a recognised sign as a word', async () => {
     await recognize.start(() => {});
-    for (const frame of [...movingFrames(20), ...stillFrames(14, 1.2)]) source.push(frame);
+    for (const frame of scriptedSign()) source.push(frame);
     await tick();
 
     expect(recognize.current.toText().toLowerCase()).toContain('dolor');
@@ -146,7 +162,7 @@ describe('RecognizeSignsUseCase', () => {
       await recognize.start((update) => {
         last = update;
       });
-      for (const frame of [...movingFrames(20), ...stillFrames(14, 1.2)]) source.push(frame);
+      for (const frame of scriptedSign()) source.push(frame);
       await tick();
       return last!.diagnostics;
     }
@@ -213,7 +229,7 @@ describe('RecognizeSignsUseCase', () => {
     slow.blocking = true;
     source.push(buildFrame(0, buildHand()));
     await tick();
-    for (const frame of [...movingFrames(20), ...stillFrames(14, 1.2)]) source.push(frame);
+    for (const frame of scriptedSign()) source.push(frame);
 
     recognize.stop();
     slow.blocking = false;

src/domain/recognition/services/SignSegmenter.ts

@@ -3,8 +3,14 @@ import { HandPoint, pointAt } from '@domain/landmarks/value-objects/Landmark';
 import { dominantHand, type LandmarkFrame } from '@domain/landmarks/value-objects/LandmarkFrame';
 
 export interface SegmenterOptions {
-  /** Movement above this (palm widths per frame) counts as "signing". */
-  readonly motionThreshold: number;
+  /**
+   * Movement above this (palm widths per **second**) counts as "signing".
+   *
+   * Per second, not per frame: a frame-relative floor is a speed floor divided by the frame
+   * rate, so the same unhurried sign counts as motion on a slow phone and as stillness on a
+   * fast one.
+   */
+  readonly motionRate: number;
   /**
    * Fraction of the window's peak speed below which the signer is decelerating.
    *
@@ -14,17 +20,27 @@ export interface SegmenterOptions {
    * subsumes the old one rather than sitting beside it.
    */
   readonly decelerationDrop: number;
-  /** Frames the deceleration must persist before it counts as a boundary, not a wobble. */
-  readonly decelerationHold: number;
+  /** How long the deceleration must persist before it counts as a boundary, not a wobble. */
+  readonly decelerationHoldMs: number;
   /**
-   * Frames a window must reach before a deceleration may close it.
+   * How long a window must run before a deceleration may close it.
    *
    * Signs decelerate internally — a two-part sign slows at its hinge — so without a floor
    * near the typical sign length the rule chops signs in half. Measured: dropping this from
-   * 24 to 18 costs 8 points of isolated top-1 to buy 3 points of continuous recovery.
+   * 1150 ms to 850 ms costs 8 points of isolated top-1 to buy 3 points of continuous recovery.
    */
-  readonly minSignFrames: number;
+  readonly minSignMs: number;
   /** Shortest accepted sign; below this it is camera noise, not a sign. */
+  readonly minMs: number;
+  /**
+   * Fewest frames a window may contain, whatever its duration.
+   *
+   * A separate concern from `minMs`, and both are needed. `minMs` rejects what is too brief
+   * to be a sign; this rejects what has too few samples to *describe* one — sixteen feature
+   * slots resampled from two frames is a vector the model has never seen. On a device slow
+   * enough to trip this, recognition is genuinely impossible, and the discarded-window
+   * counter is what makes that visible instead of silent.
+   */
   readonly minFrames: number;
   /**
    * Longest a window may run before it is emitted anyway.
@@ -34,7 +50,7 @@ export interface SegmenterOptions {
    * doing all the work while stillness was the only other rule, which is precisely why
    * windows were cut at a fixed length instead of at sign boundaries.
    */
-  readonly maxFrames: number;
+  readonly maxMs: number;
 }
 
 /**
@@ -43,43 +59,59 @@ export interface SegmenterOptions {
  * `tools/train/sweep.py` replays SWL-LSE's isolated recordings; `tools/train/continuous.py`
  * splices them into unbroken streams and asks how many signs survive. The shipped
  * stillness rule scored 0.739 on the first and **0.146 on the second** — it never closes
- * without a pause, so it only closed at `maxFrames`, and with a median sign of 30 frames
+ * without a pause, so it only closed at the cap, and with a median sign of 30 frames
  * nearly every 48-frame window straddled a boundary. That is why the app read a signing
  * video and wrote nothing.
  *
  * These settings give 0.696 isolated and 0.384 continuous at higher precision (0.755 against
  * 0.710). Four points of the validated path bought twenty-four of the one people actually
  * use. Note the continuous benchmark splices isolated recordings and so cannot reproduce
  * real co-articulation: read it as an upper bound.
+ *
+ * **Expressed in time, not frames, and that was a bug for a while.** Every threshold here
+ * used to be a frame count, swept against SWL-LSE — which is 20.00 fps in all 300 of its
+ * recordings. The live pipeline runs three MediaPipe models per frame and reaches whatever
+ * the device allows, so those counts silently meant a different duration on every phone.
+ * Below the dataset's rate a sign never reached that 24-frame floor and every window was
+ * discarded as too short: measured in a real browser, 23 frames in 16 s, six windows
+ * discarded, the vocabulary engine asked **zero** times, nothing written. The same build
+ * recognised "Dolor" at 68% as soon as the input was slowed to restore the sampling density.
+ * `LandmarkFrame` has always carried `timestampMs`; it just was not read.
+ *
+ * The values below are the swept frame counts converted at 20.00 fps — and N frames span
+ * N-1 intervals, so the 24-frame floor is 1150 ms, not 1200. Converting exactly is what lets
+ * `simulate_app.py` reproduce the pre-conversion scores and prove the port faithful.
  */
 export const DEFAULT_SEGMENTER_OPTIONS: SegmenterOptions = {
-  motionThreshold: 0.03,
+  motionRate: 0.6,
   decelerationDrop: 0.45,
-  decelerationHold: 1,
-  minSignFrames: 24,
+  decelerationHoldMs: 50,
+  minSignMs: 1150,
+  minMs: 150,
   minFrames: 4,
-  maxFrames: 48,
+  maxMs: 2350,
 };
 
 /**
  * Turns a continuous landmark stream into discrete sign windows.
  *
- * A window closes where the signer decelerates off their own peak speed, with `maxFrames`
+ * A window closes where the signer decelerates off their own peak speed, with `maxMs`
  * as a backstop. Deceleration rather than stillness because stillness is a property of
  * dictionary recordings, not of signing: measured on spliced continuous streams, waiting
  * for a pause recovered 14.6% of signs against 38.4% for this rule.
  *
- * `minSignFrames` is what keeps it honest — signs decelerate internally too, so a boundary
+ * `minSignMs` is what keeps it honest — signs decelerate internally too, so a boundary
  * is only believed once the window is already about as long as a sign.
  *
- * Deliberately a pure state machine over frames — no timers, no clock, no I/O — so its
- * behaviour is reproducible in tests by pushing a scripted frame sequence.
+ * Deliberately a pure state machine over the frames' own `timestampMs` — no timers, no
+ * clock, no I/O — so its behaviour is reproducible in tests by pushing a scripted sequence,
+ * and identical at any frame rate.
  */
 export class SignSegmenter {
   private readonly options: SegmenterOptions;
   private window: LandmarkFrame[] = [];
-  private slowFrames = 0;
-  private peakMotion = 0;
+  private slowMs = 0;
+  private peakRate = 0;
   private active = false;
   private shortWindows = 0;
 
@@ -98,7 +130,7 @@ export class SignSegmenter {
   }
 
   /**
-   * Windows completed and then discarded for being shorter than `minFrames`.
+   * Windows completed and then discarded for being too brief or too sparsely sampled.
    *
    * Invisible from outside otherwise: `push` returns null both when nothing ended and when
    * something ended and was judged too short to be a sign. Those are opposite diagnoses.
@@ -112,34 +144,38 @@ export class SignSegmenter {
     const hand = dominantHand(frame);
     if (!hand) return this.handleHandLost();
 
-    const motion = this.motionSince(frame);
+    const sinceLastMs = this.gapBefore(frame);
+    const rate = this.motionRateSince(frame, sinceLastMs);
     this.window.push(frame);
 
-    if (motion > this.options.motionThreshold) {
+    if (rate > this.options.motionRate) {
       this.active = true;
-      this.peakMotion = Math.max(this.peakMotion, motion);
+      this.peakRate = Math.max(this.peakRate, rate);
     }
 
     if (!this.active) {
       // Still idle: keep only a short tail so the next sign's start is not truncated.
-      if (this.window.length > this.options.minFrames) this.window.shift();
+      while (this.window.length > 1 && this.span() > this.options.minMs) this.window.shift();
       return null;
     }
 
-    if (this.window.length >= this.options.maxFrames) return this.close();
+    if (this.span() >= this.options.maxMs) return this.close();
 
     const decelerating =
-      this.window.length >= this.options.minSignFrames &&
-      this.peakMotion > 0 &&
-      motion < this.peakMotion * this.options.decelerationDrop;
+      this.span() >= this.options.minSignMs &&
+      this.peakRate > 0 &&
+      rate < this.peakRate * this.options.decelerationDrop;
 
     if (!decelerating) {
-      this.slowFrames = 0;
+      this.slowMs = 0;
       return null;
     }
 
-    this.slowFrames += 1;
-    return this.slowFrames >= this.options.decelerationHold ? this.close() : null;
+    // Accumulated rather than counted: one frame is 50 ms at the frame rate this was tuned
+    // at and 17 ms at 60 fps, so counting frames would make the hold three times stricter on
+    // a fast device — the same units mistake one level down.
+    this.slowMs += sinceLastMs;
+    return this.slowMs >= this.options.decelerationHoldMs ? this.close() : null;
   }
 
   /** A hand leaving frame ends the sign as surely as stillness does. */
@@ -153,26 +189,44 @@ export class SignSegmenter {
 
   private close(): readonly LandmarkFrame[] | null {
     const completed = this.window;
+    const spanMs = spanOf(completed);
     this.reset();
-    if (completed.length >= this.options.minFrames) return completed;
+
+    const longEnough = spanMs >= this.options.minMs;
+    const sampledEnough = completed.length >= this.options.minFrames;
+    if (longEnough && sampledEnough) return completed;
+
     this.shortWindows += 1;
     return null;
   }
 
   reset(): void {
     this.window = [];
-    this.slowFrames = 0;
-    this.peakMotion = 0;
+    this.slowMs = 0;
+    this.peakRate = 0;
     this.active = false;
   }
 
+  /** Wall-clock span of the frames buffered so far. */
+  private span(): number {
+    return spanOf(this.window);
+  }
+
+  /** Time since the previous buffered frame, from the frames themselves. */
+  private gapBefore(frame: LandmarkFrame): number {
+    const previous = this.window.at(-1);
+    return previous ? frame.timestampMs - previous.timestampMs : 0;
+  }
+
   /**
-   * Mean fingertip displacement from the previous frame, in palm widths so that moving the
-   * phone closer does not read as faster signing.
+   * Mean fingertip speed since the previous frame, in palm widths per second.
+   *
+   * Palm widths so that moving the phone closer does not read as faster signing; per second
+   * so that a slow phone does not read as faster signing either.
    */
-  private motionSince(frame: LandmarkFrame): number {
+  private motionRateSince(frame: LandmarkFrame, sinceLastMs: number): number {
     const previous = this.window.at(-1);
-    if (!previous) return 0;
+    if (!previous || sinceLastMs <= 0) return 0;
 
     const current = dominantHand(frame);
     const before = dominantHand(previous);
@@ -190,6 +244,11 @@ export class SignSegmenter {
       (sum, tip) => sum + distance(pointAt(current, tip), pointAt(before, tip)),
       0,
     );
-    return total / tips.length / scale;
+    return total / tips.length / scale / (sinceLastMs / 1000);
   }
 }
+
+function spanOf(frames: readonly LandmarkFrame[]): number {
+  if (frames.length < 2) return 0;
+  return frames.at(-1)!.timestampMs - frames[0]!.timestampMs;
+}