fix: stop drawing pose landmarks the model never saw

Author: Endika

src/domain/landmarks/value-objects/BodyLandmarks.ts

@@ -18,7 +18,13 @@ export const PosePoint = {
   rightHip: 24,
 } as const;
 
-/** Drawn as the torso: the shoulder line and the box down to the hips. */
+/**
+ * Drawn as the torso: the shoulder line, and the box down to the hips when the hips are
+ * genuinely in shot.
+ *
+ * A signer is usually framed from the chest up, so the hip edges are dropped far more often
+ * than they are drawn. `isVisible` decides, per edge, at render time.
+ */
 export const TORSO_CONNECTIONS: readonly (readonly [number, number])[] = [
   [PosePoint.leftShoulder, PosePoint.rightShoulder],
   [PosePoint.leftShoulder, PosePoint.leftHip],
@@ -34,11 +40,49 @@ export const ARM_CONNECTIONS: readonly (readonly [number, number])[] = [
   [PosePoint.rightElbow, PosePoint.rightWrist],
 ];
 
-/** Neck: the head sitting on the shoulder line, drawn as its own segment. */
-export const NECK_CONNECTIONS: readonly (readonly [number, number])[] = [
-  [PosePoint.nose, PosePoint.leftShoulder],
-  [PosePoint.nose, PosePoint.rightShoulder],
-];
+/**
+ * Below this, a pose landmark is the model's guess rather than something it saw.
+ *
+ * Filming a signer from the chest up leaves the hips invisible, and MediaPipe answers with
+ * extrapolated coordinates instead of nothing. Drawing those produces a torso box running
+ * off the bottom of the picture and arms pointing at the frame edges.
+ */
+export const MIN_POSE_VISIBILITY = 0.6;
+
+export function isVisible(point: Landmark | undefined): boolean {
+  // Absent visibility means the estimator does not report it (hands, face) — trust those.
+  return point !== undefined && (point.visibility ?? 1) >= MIN_POSE_VISIBILITY;
+}
+
+/**
+ * The neck: one segment from the middle of the shoulders up to the head.
+ *
+ * MediaPipe has no neck landmark, so it has to be derived. Drawing nose-to-each-shoulder
+ * instead — the obvious shortcut — paints a wide triangle across the chest that looks
+ * nothing like a neck and hides the signing space behind it.
+ */
+export function neckSegment(pose: readonly Landmark[]): readonly [Landmark, Landmark] | null {
+  const left = pose[PosePoint.leftShoulder];
+  const right = pose[PosePoint.rightShoulder];
+  const nose = pose[PosePoint.nose];
+  if (!isVisible(left) || !isVisible(right) || !isVisible(nose) || !left || !right || !nose) {
+    return null;
+  }
+
+  const centre: Landmark = {
+    x: (left.x + right.x) / 2,
+    y: (left.y + right.y) / 2,
+    z: (left.z + right.z) / 2,
+  };
+  // Stop short of the nose: the neck ends at the chin, and running the line into the face
+  // mesh just clutters it.
+  const chin: Landmark = {
+    x: centre.x + (nose.x - centre.x) * 0.6,
+    y: centre.y + (nose.y - centre.y) * 0.6,
+    z: centre.z + (nose.z - centre.z) * 0.6,
+  };
+  return [centre, chin];
+}
 
 export interface PoseLandmarks {
   readonly points: readonly Landmark[];

src/domain/landmarks/value-objects/Landmark.ts

@@ -3,6 +3,15 @@ export interface Landmark {
   readonly x: number;
   readonly y: number;
   readonly z: number;
+  /**
+   * How sure the estimator is that this point is actually in frame, 0..1.
+   *
+   * Pose reports it; hands and face do not. It matters because MediaPipe's pose model
+   * *extrapolates* landmarks it cannot see rather than omitting them — film someone from
+   * the chest up and it will still hand you hip coordinates, invented, somewhere below the
+   * bottom of the picture.
+   */
+  readonly visibility?: number | undefined;
 }
 
 export type Handedness = 'left' | 'right';

src/domain/landmarks/value-objects/__tests__/BodyLandmarks.test.ts

@@ -0,0 +1,72 @@
+import { describe, expect, it } from 'vitest';
+import { isVisible, neckSegment, PosePoint } from '../BodyLandmarks';
+import type { Landmark } from '../Landmark';
+
+/** A pose where everything is confidently seen unless overridden. */
+function buildPose(overrides: Record<number, Partial<Landmark>> = {}): Landmark[] {
+  return Array.from({ length: 33 }, (_, i) => ({
+    x: 0.5,
+    y: 0.5,
+    z: 0,
+    visibility: 0.95,
+    ...overrides[i],
+  }));
+}
+
+describe('isVisible', () => {
+  it('accepts a confidently seen landmark', () => {
+    expect(isVisible({ x: 0, y: 0, z: 0, visibility: 0.9 })).toBe(true);
+  });
+
+  it('rejects a landmark the model only guessed at', () => {
+    // Framing a signer from the chest up leaves the hips invisible, and MediaPipe answers
+    // with extrapolated coordinates rather than nothing.
+    expect(isVisible({ x: 0, y: 0, z: 0, visibility: 0.1 })).toBe(false);
+  });
+
+  it('trusts landmarks from estimators that report no visibility at all', () => {
+    // Hands and face do not report it; treating absent as invisible would erase them.
+    expect(isVisible({ x: 0, y: 0, z: 0 })).toBe(true);
+  });
+
+  it('rejects a missing landmark', () => {
+    expect(isVisible(undefined)).toBe(false);
+  });
+});
+
+describe('neckSegment', () => {
+  it('runs from the middle of the shoulders toward the head', () => {
+    const pose = buildPose({
+      [PosePoint.leftShoulder]: { x: 0.4, y: 0.6 },
+      [PosePoint.rightShoulder]: { x: 0.6, y: 0.6 },
+      [PosePoint.nose]: { x: 0.5, y: 0.2 },
+    });
+    const segment = neckSegment(pose);
+
+    expect(segment?.[0].x).toBeCloseTo(0.5, 5);
+    expect(segment?.[0].y).toBeCloseTo(0.6, 5);
+    // Upward, and stopping short of the nose so it does not run into the face mesh.
+    expect(segment?.[1].y).toBeLessThan(0.6);
+    expect(segment?.[1].y).toBeGreaterThan(0.2);
+  });
+
+  it('is one segment, not a triangle across the chest', () => {
+    // Drawing nose-to-each-shoulder was the obvious shortcut and looked nothing like a neck.
+    const segment = neckSegment(buildPose());
+    expect(segment).toHaveLength(2);
+  });
+
+  it('declines when a shoulder is not really visible', () => {
+    const pose = buildPose({ [PosePoint.rightShoulder]: { visibility: 0.1 } });
+    expect(neckSegment(pose)).toBeNull();
+  });
+
+  it('declines when the head is not really visible', () => {
+    const pose = buildPose({ [PosePoint.nose]: { visibility: 0.2 } });
+    expect(neckSegment(pose)).toBeNull();
+  });
+
+  it('declines on an empty pose rather than throwing', () => {
+    expect(neckSegment([])).toBeNull();
+  });
+});

src/presentation/components/LandmarkOverlay.ts

@@ -1,6 +1,8 @@
 import {
   ARM_CONNECTIONS,
-  NECK_CONNECTIONS,
+  isVisible,
+  neckSegment,
+  PosePoint,
   TORSO_CONNECTIONS,
 } from '@domain/landmarks/value-objects/BodyLandmarks';
 import { HAND_CONNECTIONS, type Landmark } from '@domain/landmarks/value-objects/Landmark';
@@ -82,10 +84,23 @@ export class LandmarkOverlay {
     const pose = frame.pose?.points ?? [];
     const face = frame.face?.points ?? [];
 
+    const neck = pose.length > 0 ? neckSegment(pose) : null;
+
     if (pose.length > 0) {
       this.drawEdges(pose, TORSO_CONNECTIONS, PART_COLOURS.torso, 3.5 * unit, project);
       this.drawEdges(pose, ARM_CONNECTIONS, PART_COLOURS.arms, 3.5 * unit, project);
-      this.drawEdges(pose, NECK_CONNECTIONS, PART_COLOURS.neck, 2.5 * unit, project);
+    }
+
+    if (neck) {
+      this.context.strokeStyle = PART_COLOURS.neck;
+      this.context.lineWidth = 3 * unit;
+      const [from, to] = neck;
+      const start = project(from.x, from.y);
+      const end = project(to.x, to.y);
+      this.context.beginPath();
+      this.context.moveTo(start.x, start.y);
+      this.context.lineTo(end.x, end.y);
+      this.context.stroke();
     }
 
     if (face.length > 0) {
@@ -101,12 +116,17 @@ export class LandmarkOverlay {
 
     this.context.globalAlpha = 1;
 
+    // Reported per part from what was actually drawable, so a chip going red means "not
+    // seen" rather than "the pose model happened to return an array".
+    const seen = (index: number) => isVisible(pose[index]);
     return {
       hands: frame.hands.length > 0,
       face: face.length > 0,
-      neck: pose.length > 0,
-      torso: pose.length > 0,
-      arms: pose.length > 0,
+      neck: neck !== null,
+      torso: seen(PosePoint.leftShoulder) && seen(PosePoint.rightShoulder),
+      arms:
+        (seen(PosePoint.leftElbow) && seen(PosePoint.leftWrist)) ||
+        (seen(PosePoint.rightElbow) && seen(PosePoint.rightWrist)),
     };
   }
 
@@ -122,7 +142,10 @@ export class LandmarkOverlay {
     for (const [from, to] of edges) {
       const a = points[from];
       const b = points[to];
-      if (!a || !b) continue;
+      // Both ends must be genuinely seen. MediaPipe extrapolates pose landmarks it cannot
+      // find rather than omitting them, so drawing unconditionally paints invented hips
+      // below the picture and arms reaching for the frame edges.
+      if (!isVisible(a) || !isVisible(b) || !a || !b) continue;
       const start = project(a.x, a.y);
       const end = project(b.x, b.y);
       this.context.beginPath();