feat: teach the app your own signs and recognise them offline

Author: Endika Iglesias

README.md

@@ -15,8 +15,8 @@ Three engines answer through one port, so the app does not care which one produc
 | Engine | What it reads | Where it comes from |
 | --- | --- | --- |
 | **Alphabet** | Fingerspelled letters (dactilológico). Spell anything, letter by letter. | Geometric handshape rules — no training data needed. |
-| **Vocabulary** | Whole LSE signs, one word each. | ONNX model trained on [SWL-LSE](https://zenodo.org/records/13691887): 300 classes, ~180 concepts after merging variants. Health domain. |
-| **Taught** | Any sign you record yourself, in any sign language. | Nearest-prototype match over 3+ examples you give it, stored on-device. |
+| **Vocabulary** | Whole LSE signs, one word each. | ONNX model trained on [SWL-LSE](https://zenodo.org/records/13691887): 300 classes, ~180 concepts after merging variants. Health domain. **Not built yet.** |
+| **Taught** | Any sign you record yourself, in any sign language. | Nearest-prototype match over 3+ recordings, stored in IndexedDB on your device. Working now. |
 
 ### What it does not do
 
@@ -46,8 +46,15 @@ src/
 
 The domain is where the interesting logic lives and it is fully testable without a camera:
 `SignSegmenter` decides where one sign ends and the next begins, `CandidateStabilizer` stops
-a jittering classifier from spelling `AAAABAAAA`, and `handShape` reduces 21 landmarks to
-scale- and handedness-invariant ratios.
+a jittering classifier from spelling `AAAABAAAA`, `handShape` reduces 21 landmarks to scale-
+and handedness-invariant ratios, and `windowSignature` collapses a whole sign into one
+fixed-length vector so two performances of it can be compared.
+
+**Signature similarity is Euclidean, not cosine, and that is load-bearing.** Every hand
+shares the same gross structure, so cosine scored a fist against an open hand at 0.965 and an
+index point against a Y at 0.962 — no threshold separates those. Distance over already
+normalised coordinates gives 0.10 and 0.25 for the same pairs. If taught-sign recognition
+ever starts matching everything, check that this has not been "simplified" back to cosine.
 
 **Normalisation must match training.** `normalizeHand` mirrors what `tools/train` applies to
 the dataset. A model trained on normalised coordinates and fed raw ones predicts noise

src/application/use-cases/ManageCustomSignsUseCase.ts

@@ -0,0 +1,21 @@
+import type { CustomSign } from '@domain/recognition/entities/CustomSign';
+import type { ICustomSignRepository } from '@domain/recognition/repositories/ICustomSignRepository';
+
+/**
+ * Listing and deleting taught signs.
+ *
+ * Deletion is not a nicety: these recordings are derived from the user's body and may encode
+ * health vocabulary, so being able to remove one is part of the app's privacy posture, not a
+ * convenience feature.
+ */
+export class ManageCustomSignsUseCase {
+  constructor(private readonly repository: ICustomSignRepository) {}
+
+  async list(): Promise<CustomSign[]> {
+    return this.repository.findAll();
+  }
+
+  async delete(id: string): Promise<void> {
+    await this.repository.delete(id);
+  }
+}

src/application/use-cases/RecognizeSignsUseCase.ts

@@ -37,6 +37,8 @@ export class RecognizeSignsUseCase {
   private listener: RecognitionListener | null = null;
   /** Guards against overlapping async classify calls piling up behind a slow frame. */
   private busy = false;
+  /** Set while teaching: the next completed sign is handed over instead of transcribed. */
+  private capture: ((window: readonly LandmarkFrame[]) => void) | null = null;
 
   constructor(
     private readonly source: ILandmarkSource,
@@ -75,6 +77,27 @@ export class RecognizeSignsUseCase {
     return this.transcript;
   }
 
+  /**
+   * Resolves with the next completed sign instead of transcribing it.
+   *
+   * Recording reuses the live segmenter rather than a separate timed capture, so a taught
+   * example is delimited exactly the way a recognised sign will be. Capturing on a stopwatch
+   * would train the app on windows it never sees at recognition time.
+   */
+  captureWindow(): Promise<readonly LandmarkFrame[]> {
+    return new Promise((resolve) => {
+      this.capture = resolve;
+    });
+  }
+
+  cancelCapture(): void {
+    this.capture = null;
+  }
+
+  get isCapturing(): boolean {
+    return this.capture !== null;
+  }
+
   private async onFrame(frame: LandmarkFrame): Promise<void> {
     if (frame.hands.length === 0) {
       // A hand leaving frame is a deliberate boundary: it lets the same letter repeat.
@@ -83,6 +106,18 @@ export class RecognizeSignsUseCase {
 
     const closedWindow = this.segmenter.push(frame);
 
+    if (this.capture) {
+      // Teaching: hand the finished sign over, and transcribe nothing meanwhile — the user
+      // is demonstrating a sign, not dictating.
+      if (closedWindow) {
+        const deliver = this.capture;
+        this.capture = null;
+        deliver(closedWindow);
+      }
+      this.emit([], frame);
+      return;
+    }
+
     if (this.busy) return;
     this.busy = true;
     try {

src/application/use-cases/TeachCustomSignUseCase.ts

@@ -0,0 +1,60 @@
+import type { LandmarkFrame } from '@domain/landmarks/value-objects/LandmarkFrame';
+import {
+  type CustomSign,
+  DuplicateSignTextError,
+  MIN_PROTOTYPES_PER_SIGN,
+  NotEnoughExamplesError,
+} from '@domain/recognition/entities/CustomSign';
+import type { ICustomSignRepository } from '@domain/recognition/repositories/ICustomSignRepository';
+import { windowSignature } from '@domain/recognition/services/windowSignature';
+
+export class EmptySignTextError extends Error {
+  constructor() {
+    super('A taught sign needs a word to write when it is recognised');
+    this.name = 'EmptySignTextError';
+  }
+}
+
+/**
+ * Turns a handful of recordings of one sign into something the app can recognise.
+ *
+ * Clock and id generator are injected so a test can assert on exact stored values instead of
+ * whatever the wall clock happened to say.
+ */
+export class TeachCustomSignUseCase {
+  constructor(
+    private readonly repository: ICustomSignRepository,
+    private readonly now: () => number = () => Date.now(),
+    private readonly newId: () => string = () => crypto.randomUUID(),
+  ) {}
+
+  async execute(
+    text: string,
+    examples: readonly (readonly LandmarkFrame[])[],
+  ): Promise<CustomSign> {
+    const trimmed = text.trim();
+    if (trimmed.length === 0) throw new EmptySignTextError();
+
+    // Empty recordings are dropped first: three attempts where two caught no hand is not
+    // three examples, and accepting them would produce a sign that matches almost anything.
+    const usable = examples.filter((example) => example.some((frame) => frame.hands.length > 0));
+    if (usable.length < MIN_PROTOTYPES_PER_SIGN) {
+      throw new NotEnoughExamplesError(usable.length);
+    }
+
+    const existing = await this.repository.findAll();
+    if (existing.some((sign) => sign.text.toLowerCase() === trimmed.toLowerCase())) {
+      throw new DuplicateSignTextError(trimmed);
+    }
+
+    const sign: CustomSign = {
+      id: this.newId(),
+      text: trimmed,
+      prototypes: usable.map(windowSignature),
+      createdAtMs: this.now(),
+    };
+
+    await this.repository.save(sign);
+    return sign;
+  }
+}

src/application/use-cases/__tests__/TeachCustomSignUseCase.test.ts

@@ -0,0 +1,84 @@
+import {
+  DuplicateSignTextError,
+  NotEnoughExamplesError,
+} from '@domain/recognition/entities/CustomSign';
+import { SIGNATURE_LENGTH } from '@domain/recognition/services/windowSignature';
+import { InMemoryCustomSignRepository } from '@infrastructure/persistence/InMemoryCustomSignRepository';
+import { beforeEach, describe, expect, it } from 'vitest';
+import { buildFrame, buildHand } from '@/test/handFixtures';
+import { EmptySignTextError, TeachCustomSignUseCase } from '../TeachCustomSignUseCase';
+
+function record(nudge = 0) {
+  return Array.from({ length: 10 }, (_, i) =>
+    buildFrame(i * 33, buildHand({ curls: [0.9, 0, 0.9, 0.9, 0.9], offset: { x: nudge, y: 0 } })),
+  );
+}
+
+const EMPTY_RECORDING = [buildFrame(0, null), buildFrame(33, null)];
+
+describe('TeachCustomSignUseCase', () => {
+  let repository: InMemoryCustomSignRepository;
+  let teach: TeachCustomSignUseCase;
+
+  beforeEach(() => {
+    repository = new InMemoryCustomSignRepository();
+    teach = new TeachCustomSignUseCase(
+      repository,
+      () => 1_700_000_000_000,
+      () => 'fixed-id',
+    );
+  });
+
+  it('stores one prototype per usable recording', async () => {
+    const sign = await teach.execute('ibuprofeno', [record(0), record(0.02), record(0.04)]);
+
+    expect(sign.prototypes).toHaveLength(3);
+    expect(sign.prototypes[0]).toHaveLength(SIGNATURE_LENGTH);
+    expect(await repository.findAll()).toHaveLength(1);
+  });
+
+  it('records the injected id and timestamp', async () => {
+    const sign = await teach.execute('ibuprofeno', [record(0), record(0.02), record(0.04)]);
+
+    expect(sign.id).toBe('fixed-id');
+    expect(sign.createdAtMs).toBe(1_700_000_000_000);
+  });
+
+  it('trims the text it will write', async () => {
+    const sign = await teach.execute('  ibuprofeno  ', [record(0), record(0.02), record(0.04)]);
+    expect(sign.text).toBe('ibuprofeno');
+  });
+
+  it('rejects a sign with no word to write', async () => {
+    await expect(teach.execute('   ', [record(0), record(0.02), record(0.04)])).rejects.toThrow(
+      EmptySignTextError,
+    );
+  });
+
+  it('rejects fewer than three examples', async () => {
+    await expect(teach.execute('ibuprofeno', [record(0), record(0.02)])).rejects.toThrow(
+      NotEnoughExamplesError,
+    );
+  });
+
+  it('does not count recordings that caught no hand', async () => {
+    // Three attempts where two saw nothing is one example, not three. Accepting them would
+    // store zero-filled prototypes that match almost anything.
+    await expect(
+      teach.execute('ibuprofeno', [record(0), EMPTY_RECORDING, EMPTY_RECORDING]),
+    ).rejects.toThrow(NotEnoughExamplesError);
+  });
+
+  it('rejects a word that is already taught, whatever the casing', async () => {
+    await teach.execute('ibuprofeno', [record(0), record(0.02), record(0.04)]);
+
+    await expect(
+      teach.execute('Ibuprofeno', [record(0), record(0.02), record(0.04)]),
+    ).rejects.toThrow(DuplicateSignTextError);
+  });
+
+  it('leaves the repository untouched when it rejects', async () => {
+    await expect(teach.execute('', [record(0), record(0.02), record(0.04)])).rejects.toThrow();
+    expect(await repository.findAll()).toEqual([]);
+  });
+});

src/bootstrap/Container.ts

@@ -1,6 +1,10 @@
+import { ManageCustomSignsUseCase } from '@application/use-cases/ManageCustomSignsUseCase';
 import { RecognizeSignsUseCase } from '@application/use-cases/RecognizeSignsUseCase';
+import { TeachCustomSignUseCase } from '@application/use-cases/TeachCustomSignUseCase';
 import type { ISignClassifier } from '@domain/recognition/services/ISignClassifier';
+import { IndexedDBCustomSignRepository } from '@infrastructure/persistence/indexeddb/IndexedDBCustomSignRepository';
 import { HandshapeAlphabetClassifier } from '@infrastructure/recognition/HandshapeAlphabetClassifier';
+import { PrototypeSignClassifier } from '@infrastructure/recognition/PrototypeSignClassifier';
 import { MediaPipeLandmarkSource } from '@infrastructure/vision/MediaPipeLandmarkSource';
 
 /**
@@ -9,6 +13,10 @@ import { MediaPipeLandmarkSource } from '@infrastructure/vision/MediaPipeLandmar
  */
 export class Container {
   readonly source: MediaPipeLandmarkSource;
+  readonly customSigns = new IndexedDBCustomSignRepository();
+  readonly taught = new PrototypeSignClassifier(this.customSigns);
+  readonly teach = new TeachCustomSignUseCase(this.customSigns);
+  readonly manageCustomSigns = new ManageCustomSignsUseCase(this.customSigns);
   readonly classifiers: readonly ISignClassifier[];
   readonly recognize: RecognizeSignsUseCase;
 
@@ -22,7 +30,7 @@ export class Container {
       maxHands: 2,
     });
 
-    this.classifiers = [new HandshapeAlphabetClassifier()];
+    this.classifiers = [new HandshapeAlphabetClassifier(), this.taught];
     this.recognize = new RecognizeSignsUseCase(this.source, this.classifiers);
   }
 }

src/domain/recognition/services/__tests__/windowSignature.test.ts

@@ -0,0 +1,111 @@
+import { describe, expect, it } from 'vitest';
+import { buildFrame, buildHand } from '@/test/handFixtures';
+import { SIGNATURE_LENGTH, similarity, windowSignature } from '../windowSignature';
+
+type Curls = [number, number, number, number, number];
+
+/** A held handshape, repeated for `frames` — a static sign. */
+function hold(curls: Curls, frames: number, handedness: 'left' | 'right' = 'right') {
+  return Array.from({ length: frames }, (_, i) =>
+    buildFrame(i * 33, buildHand({ curls, handedness })),
+  );
+}
+
+/** A handshape travelling across the frame — a dynamic sign. */
+function travel(curls: Curls, frames: number, step = 0.04) {
+  return Array.from({ length: frames }, (_, i) =>
+    buildFrame(i * 33, buildHand({ curls, offset: { x: i * step, y: 0 } })),
+  );
+}
+
+const FIST: Curls = [0.9, 0.9, 0.9, 0.9, 0.9];
+const OPEN: Curls = [0, 0, 0, 0, 0];
+const POINT: Curls = [0.9, 0, 0.9, 0.9, 0.9];
+
+describe('windowSignature', () => {
+  it('always produces the same length, whatever the sign lasted', () => {
+    expect(windowSignature(hold(OPEN, 3))).toHaveLength(SIGNATURE_LENGTH);
+    expect(windowSignature(hold(OPEN, 40))).toHaveLength(SIGNATURE_LENGTH);
+  });
+
+  it('returns a zero signature for an empty window rather than throwing', () => {
+    const signature = windowSignature([]);
+    expect(signature).toHaveLength(SIGNATURE_LENGTH);
+    expect(signature.every((value) => value === 0)).toBe(true);
+  });
+
+  it('matches the same sign performed at different speeds', () => {
+    // The whole point of resampling: duration must not be what distinguishes two signs.
+    const fast = windowSignature(travel(POINT, 6));
+    const slow = windowSignature(travel(POINT, 30));
+    expect(similarity(fast, slow)).toBeGreaterThan(0.95);
+  });
+
+  it('separates two genuinely different handshapes by a wide margin', () => {
+    // Asserting a real number, not just an ordering: an earlier cosine metric ranked these
+    // "correctly" while scoring them 0.965, which no threshold could have used.
+    const fist = windowSignature(hold(FIST, 12));
+    const open = windowSignature(hold(OPEN, 12));
+    expect(similarity(fist, open)).toBeLessThan(0.3);
+  });
+
+  it('keeps the closest distinct pair below the recognition threshold', () => {
+    // A fist and an index point share four curled fingers, so they are the hardest pair on
+    // this fixture set. If even they scored above 0.86 the classifier would be guessing.
+    const fist = windowSignature(hold(FIST, 12));
+    const point = windowSignature(hold(POINT, 12));
+    expect(similarity(fist, point)).toBeLessThan(0.86);
+  });
+
+  it('still matches a re-recording of the same sign with natural variation', () => {
+    const first = windowSignature(hold(POINT, 12));
+    const wobbly = Array.from({ length: 14 }, (_, i) =>
+      buildFrame(
+        i * 33,
+        buildHand({
+          curls: [0.85, 0.06, 0.96, 0.84, 0.95],
+          offset: { x: 0.03, y: 0.02 },
+        }),
+      ),
+    );
+    expect(similarity(first, windowSignature(wobbly))).toBeGreaterThan(0.86);
+  });
+
+  it('is unaffected by where in frame the sign happens', () => {
+    const here = windowSignature(hold(POINT, 12));
+    const shifted = hold(POINT, 12).map((frame) =>
+      buildFrame(frame.timestampMs, buildHand({ curls: POINT, offset: { x: 0.25, y: -0.2 } })),
+    );
+    expect(similarity(here, windowSignature(shifted))).toBeGreaterThan(0.99);
+  });
+
+  it('keeps the two hands in separate slots', () => {
+    // A right-handed sign and its left-handed twin must not collide, or a two-handed sign
+    // could never be told apart from a one-handed one.
+    const right = windowSignature(hold(POINT, 10, 'right'));
+    const left = windowSignature(hold(POINT, 10, 'left'));
+    expect(similarity(right, left)).toBeLessThan(0.99);
+  });
+});
+
+describe('similarity', () => {
+  it('scores a vector against itself at the top of the range', () => {
+    const signature = windowSignature(hold(POINT, 10));
+    expect(similarity(signature, signature)).toBeCloseTo(1, 5);
+  });
+
+  it('scores zero against a signature where nothing was tracked', () => {
+    expect(similarity(new Float32Array(SIGNATURE_LENGTH), windowSignature(hold(OPEN, 5)))).toBe(0);
+  });
+
+  it('does not call two empty signatures a perfect match', () => {
+    // They sit at distance zero from each other, so the raw metric would say 1.0 — "I saw
+    // no hand" agreeing with "I saw no hand" is not a recognised sign.
+    const empty = new Float32Array(SIGNATURE_LENGTH);
+    expect(similarity(empty, new Float32Array(SIGNATURE_LENGTH))).toBe(0);
+  });
+
+  it('refuses to compare signatures of different lengths', () => {
+    expect(similarity(new Float32Array(4), new Float32Array(8))).toBe(0);
+  });
+});