feat: report the feature vector the model was actually fed (#39)

Author: Endika

src/application/use-cases/RecognizeSignsUseCase.ts

@@ -10,6 +10,7 @@ import {
 import type {
   RawScore,
   RecognitionDiagnostics,
+  SignatureProfile,
   WindowVeto,
 } from '@domain/recognition/value-objects/RecognitionDiagnostics';
 import { Transcript } from '@domain/transcript/entities/Transcript';
@@ -64,6 +65,7 @@ export class RecognizeSignsUseCase {
   private lastRawTop: readonly RawScore[] = [];
   private lastVeto: WindowVeto | null = null;
   private wordsEmitted = 0;
+  private lastSignature: SignatureProfile | null = null;
   /** The segmenter outlives a session, so short-window counts are read as a delta. */
   private shortWindowsAtStart = 0;
 
@@ -169,6 +171,7 @@ export class RecognizeSignsUseCase {
         this.vocabularyInvocations += 1;
         const words = await this.classifyWindow(pending);
         this.lastRawTop = this.collectRawScores();
+        this.lastSignature = this.collectSignatureProfile();
         const top = words[0] ?? null;
         const word = this.windowStabilizer.accept(top);
         if (word) this.append(word, frame.timestampMs);
@@ -234,6 +237,15 @@ export class RecognizeSignsUseCase {
     return [];
   }
 
+  private collectSignatureProfile(): SignatureProfile | null {
+    for (const engine of this.classifiers) {
+      if (engine.granularity === 'window' && engine.lastSignatureProfile) {
+        return engine.lastSignatureProfile;
+      }
+    }
+    return null;
+  }
+
   private resetDiagnostics(): void {
     this.framesSeen = 0;
     this.framesWithHands = 0;
@@ -243,6 +255,7 @@ export class RecognizeSignsUseCase {
     this.lastRawTop = [];
     this.lastVeto = null;
     this.wordsEmitted = 0;
+    this.lastSignature = null;
     this.shortWindowsAtStart = this.segmenter.discardedShortWindows;
   }
 
@@ -262,6 +275,7 @@ export class RecognizeSignsUseCase {
       lastRawTop: this.lastRawTop,
       lastVeto: this.lastVeto,
       wordsEmitted: this.wordsEmitted,
+      lastSignature: this.lastSignature,
     };
   }
 

src/domain/recognition/services/ISignClassifier.ts

@@ -1,6 +1,6 @@
 import type { LandmarkFrame } from '@domain/landmarks/value-objects/LandmarkFrame';
 import type { SignCandidate } from '../value-objects/Gloss';
-import type { RawScore } from '../value-objects/RecognitionDiagnostics';
+import type { RawScore, SignatureProfile } from '../value-objects/RecognitionDiagnostics';
 
 /**
  * The single port every recognition engine implements, so the application layer never
@@ -34,4 +34,6 @@ export interface ISignClassifier {
    * returns an empty array in both cases.
    */
   readonly lastScores?: readonly RawScore[];
+  /** The feature vector the last `classify` was fed, summarised per body part. */
+  readonly lastSignatureProfile?: SignatureProfile | null;
 }

src/domain/recognition/services/__tests__/vocabularySignature.test.ts

@@ -3,7 +3,11 @@ import { join } from 'node:path';
 import type { LandmarkFrame } from '@domain/landmarks/value-objects/LandmarkFrame';
 import { describe, expect, it } from 'vitest';
 import { buildFrame, buildHand } from '@/test/handFixtures';
-import { VOCABULARY_SIGNATURE_LENGTH, vocabularySignature } from '../vocabularySignature';
+import {
+  profileSignature,
+  VOCABULARY_SIGNATURE_LENGTH,
+  vocabularySignature,
+} from '../vocabularySignature';
 
 /** Written by `tools/train/make_parity.py` from the same synthetic input. */
 const parity = JSON.parse(
@@ -79,4 +83,24 @@ describe('vocabularySignature', () => {
     // certainly not a different sign.
     expect(short).toHaveLength(stretched.length);
   });
+
+  describe('profileSignature', () => {
+    it('summarises each body part of a well-formed window', () => {
+      const profile = profileSignature(vocabularySignature(parityFrames()));
+
+      expect(profile.torso.emptyFrames).toBe(0);
+      expect(profile.rightHand.meanMagnitude).toBeGreaterThan(0);
+      expect(profile.face.meanMagnitude).toBeGreaterThan(0);
+    });
+
+    it('reports the hands as empty when pose is missing, which is what breaks the model', () => {
+      // Hand coordinates are expressed relative to the torso, so losing pose does not degrade
+      // the hand block — it zeroes it. Invisible on screen, fatal to the prediction.
+      const noPose = parityFrames().map(({ pose: _pose, ...frame }) => frame);
+      const profile = profileSignature(vocabularySignature(noPose));
+
+      expect(profile.torso.emptyFrames).toBe(1);
+      expect(profile.rightHand.emptyFrames).toBe(1);
+    });
+  });
 });

src/domain/recognition/services/vocabularySignature.ts

@@ -1,6 +1,10 @@
 import { PosePoint } from '@domain/landmarks/value-objects/BodyLandmarks';
 import { HandPoint, type Landmark } from '@domain/landmarks/value-objects/Landmark';
 import type { LandmarkFrame } from '@domain/landmarks/value-objects/LandmarkFrame';
+import type {
+  SignatureBlock,
+  SignatureProfile,
+} from '@domain/recognition/value-objects/RecognitionDiagnostics';
 
 /**
  * What the trained LSE vocabulary model reads.
@@ -25,6 +29,14 @@ const FACE_FLOATS = 6;
 const FRAME_FLOATS = HAND_FLOATS * 2 + TORSO_FLOATS + FACE_FLOATS;
 export const VOCABULARY_SIGNATURE_LENGTH = VOCABULARY_FRAMES * FRAME_FLOATS;
 
+/** Where each part lives inside one frame's slice, for reading a built signature back. */
+const BLOCKS = {
+  rightHand: [0, HAND_FLOATS],
+  leftHand: [HAND_FLOATS, HAND_FLOATS * 2],
+  torso: [HAND_FLOATS * 2, HAND_FLOATS * 2 + TORSO_FLOATS],
+  face: [HAND_FLOATS * 2 + TORSO_FLOATS, FRAME_FLOATS],
+} as const;
+
 /**
  * Face Mesh indices for the landmarks that carry grammar while signing. MediaPipe's
  * FaceLandmarker returns 478 points; 0–467 are the same mesh these indices refer to.
@@ -70,6 +82,44 @@ export function vocabularySignature(window: readonly LandmarkFrame[]): Float32Ar
   return signature;
 }
 
+/**
+ * Reads a built signature back as four per-part summaries.
+ *
+ * The model scores near-noise in the browser while measuring 0.741 offline, and the feature
+ * code has verified parity with the trainer — so what differs is the input, not the maths.
+ * Comparing these four numbers against the same statistics over SWL-LSE's test split says
+ * which part is wrong without guessing: a part that is empty here and never empty in
+ * training, or an order-of-magnitude gap, is the answer.
+ */
+export function profileSignature(signature: Float32Array): SignatureProfile {
+  const read = (from: number, to: number): SignatureBlock => {
+    let empty = 0;
+    let total = 0;
+    let count = 0;
+    for (let slot = 0; slot < VOCABULARY_FRAMES; slot += 1) {
+      const base = slot * FRAME_FLOATS;
+      let magnitude = 0;
+      for (let i = from; i < to; i += 1) magnitude += Math.abs(signature[base + i] ?? 0);
+      if (magnitude === 0) empty += 1;
+      else {
+        total += magnitude / (to - from);
+        count += 1;
+      }
+    }
+    return {
+      emptyFrames: empty / VOCABULARY_FRAMES,
+      meanMagnitude: count ? total / count : 0,
+    };
+  };
+
+  return {
+    rightHand: read(...BLOCKS.rightHand),
+    leftHand: read(...BLOCKS.leftHand),
+    torso: read(...BLOCKS.torso),
+    face: read(...BLOCKS.face),
+  };
+}
+
 function sampleIndex(slot: number, length: number): number {
   if (length === 1) return 0;
   return Math.round((slot / (VOCABULARY_FRAMES - 1)) * (length - 1));

src/domain/recognition/value-objects/RecognitionDiagnostics.ts

@@ -22,6 +22,27 @@ export interface RawScore {
  */
 export type WindowVeto = 'classifier' | 'stabilizer' | 'duplicate';
 
+export interface SignatureBlock {
+  /** 0..1 of the 16 sampled frames where this part contributed nothing at all. */
+  readonly emptyFrames: number;
+  /** Mean absolute value across the part's floats, over the frames that had it. */
+  readonly meanMagnitude: number;
+}
+
+/**
+ * The feature vector the browser actually built, summarised per body part.
+ *
+ * Measured over SWL-LSE's test split for comparison — right hand 3.37, left 3.61, torso
+ * 0.515, face 0.157, and torso empty on 0.0% of frames. A part that reads far from its
+ * number here is receiving something training never saw.
+ */
+export interface SignatureProfile {
+  readonly rightHand: SignatureBlock;
+  readonly leftHand: SignatureBlock;
+  readonly torso: SignatureBlock;
+  readonly face: SignatureBlock;
+}
+
 export interface RecognitionDiagnostics {
   readonly framesSeen: number;
   readonly framesWithHands: number;
@@ -39,6 +60,8 @@ export interface RecognitionDiagnostics {
   readonly lastRawTop: readonly RawScore[];
   readonly lastVeto: WindowVeto | null;
   readonly wordsEmitted: number;
+  /** What the model was actually fed for the last window. Null until one is classified. */
+  readonly lastSignature: SignatureProfile | null;
 }
 
 export const EMPTY_DIAGNOSTICS: RecognitionDiagnostics = {
@@ -54,4 +77,5 @@ export const EMPTY_DIAGNOSTICS: RecognitionDiagnostics = {
   lastRawTop: [],
   lastVeto: null,
   wordsEmitted: 0,
+  lastSignature: null,
 };

src/infrastructure/recognition/VocabularySignClassifier.ts

@@ -1,6 +1,7 @@
 import type { LandmarkFrame } from '@domain/landmarks/value-objects/LandmarkFrame';
 import type { ISignClassifier } from '@domain/recognition/services/ISignClassifier';
 import {
+  profileSignature,
   VOCABULARY_SIGNATURE_LENGTH,
   vocabularySignature,
 } from '@domain/recognition/services/vocabularySignature';
@@ -9,7 +10,10 @@ import {
   createGloss,
   type SignCandidate,
 } from '@domain/recognition/value-objects/Gloss';
-import type { RawScore } from '@domain/recognition/value-objects/RecognitionDiagnostics';
+import type {
+  RawScore,
+  SignatureProfile,
+} from '@domain/recognition/value-objects/RecognitionDiagnostics';
 import {
   affine,
   type GruDirection,
@@ -68,6 +72,7 @@ export class VocabularySignClassifier implements ISignClassifier {
   private manifest: VocabularyManifest | null = null;
   private tensors: Map<string, Float32Array> | null = null;
   private rawTop: readonly RawScore[] = [];
+  private profile: SignatureProfile | null = null;
 
   constructor(
     private readonly manifestUrl: string,
@@ -79,6 +84,11 @@ export class VocabularySignClassifier implements ISignClassifier {
     return this.rawTop;
   }
 
+  /** What the last window looked like as features, per body part. */
+  get lastSignatureProfile(): SignatureProfile | null {
+    return this.profile;
+  }
+
   isReady(): boolean {
     return this.tensors !== null;
   }
@@ -119,7 +129,9 @@ export class VocabularySignClassifier implements ISignClassifier {
     const tensors = this.tensors;
     if (!manifest || !tensors || window.length === 0) return [];
 
-    const probabilities = softmax(this.forward(vocabularySignature(window), manifest, tensors));
+    const signature = vocabularySignature(window);
+    this.profile = profileSignature(signature);
+    const probabilities = softmax(this.forward(signature, manifest, tensors));
 
     const ranked = manifest.concepts
       .map((concept, i) => ({

src/presentation/components/DiagnosticsPanel.ts

@@ -1,9 +1,24 @@
 import {
   EMPTY_DIAGNOSTICS,
   type RecognitionDiagnostics,
+  type SignatureBlock,
   type WindowVeto,
 } from '@domain/recognition/value-objects/RecognitionDiagnostics';
 
+/**
+ * The same statistics measured over SWL-LSE's test split — `tools/train`, 598 recordings.
+ *
+ * Shown beside the live numbers because the model scores near-noise in the browser while
+ * measuring 0.741 offline, and feature parity with the trainer is already verified. So the
+ * input differs, and the part whose numbers do not match is where.
+ */
+const EXPECTED: Record<string, { empty: number; magnitude: number }> = {
+  'Mano derecha': { empty: 0.238, magnitude: 3.37 },
+  'Mano izquierda': { empty: 0.377, magnitude: 3.61 },
+  Torso: { empty: 0.0, magnitude: 0.515 },
+  Cara: { empty: 0.002, magnitude: 0.157 },
+};
+
 /**
  * Shows what the pipeline did, so "it writes nothing" becomes a specific failure.
  *
@@ -90,6 +105,37 @@ export class DiagnosticsPanel {
         <dt>Mejores opciones, sin filtrar</dt>
         <dd>${scores}</dd>
       </div>
+      <div class="diagnostics__row diagnostics__row--wide">
+        <dt>Lo que recibió el modelo (esperado entre paréntesis)</dt>
+        <dd>${this.describeSignature()}</dd>
+      </div>
     `;
   }
+
+  /** Live feature magnitudes next to the training reference, one line per body part. */
+  private describeSignature(): string {
+    const profile = this.latest.lastSignature;
+    if (!profile) return 'todavía nada';
+
+    const parts: [string, SignatureBlock][] = [
+      ['Mano derecha', profile.rightHand],
+      ['Mano izquierda', profile.leftHand],
+      ['Torso', profile.torso],
+      ['Cara', profile.face],
+    ];
+
+    return parts
+      .map(([label, block]) => {
+        const reference = EXPECTED[label]!;
+        const empty = `${(block.emptyFrames * 100).toFixed(0)}% vacío (${(reference.empty * 100).toFixed(0)}%)`;
+        const size = `${block.meanMagnitude.toFixed(2)} (${reference.magnitude.toFixed(2)})`;
+        // Flagged rather than left to be eyeballed: an order of magnitude is the signal.
+        const off =
+          block.emptyFrames - reference.empty > 0.3 ||
+          block.meanMagnitude > reference.magnitude * 3 ||
+          block.meanMagnitude < reference.magnitude / 3;
+        return `<div${off ? ' class="diagnostics__off"' : ''}>${label}: ${empty} · ${size}</div>`;
+      })
+      .join('');
+  }
 }

src/presentation/styles/global.css

@@ -470,3 +470,9 @@ body {
 .diagnostics__row--wide dd {
   text-align: left;
 }
+
+/* A block whose numbers are nowhere near training: the one line worth finding fast. */
+.diagnostics__off {
+  color: var(--danger);
+  font-weight: 600;
+}