Skip to content

Commit a46665d

Browse files
larszuclaude
andauthored
Gesichtserkennung: Rückfall auf den Prozessor statt still abzuschalten, Fehler sichtbar (#137)
Der GPU-Delegate von MediaPipe ließ sich auf manchen Rechnern anlegen und scheiterte erst am ersten Bild; die Erkennung schaltete sich dann ohne Meldung ab. Jetzt: CPU-Rückfall, fehlgeschlagenes Laden wird nicht festgehalten, Hinweis im Panel-⚙, Details im Feedback-Protokoll. Bridge-Bilder für die Erkennung auf 640 px ausgedünnt, CPU-Takt 5/s. Claude-Session: https://claude.ai/code/session_015SeZtGvNXy2CQn9CDiA1Na Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
1 parent f4b3ddd commit a46665d

5 files changed

Lines changed: 118 additions & 23 deletions

File tree

‎src/face.ts‎

Lines changed: 61 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -10,36 +10,67 @@ type Box = [number, number, number, number];
1010
export interface Face { id: number; box: Box }
1111
let nextId = 1;
1212

13+
/**
14+
* Which way the detector runs, and why it stopped. The GPU delegate can be created and still
15+
* fail on the first picture (some drivers, ANGLE/D3D11, software WebGL); then detection falls
16+
* back to the CPU instead of switching itself off silently. A failed load is not kept, so
17+
* switching face tracking on again retries.
18+
*/
19+
export const faceStatus: { delegate: 'GPU' | 'CPU' | null; error: string } = { delegate: null, error: '' };
20+
1321
let detector: Promise<FaceDetector> | null = null;
14-
function getDetector() {
22+
function getDetector(delegate: 'GPU' | 'CPU' = 'GPU') {
1523
detector ??= (async () => {
1624
const base = new URL('.', location.href).href;
1725
const fileset = await FilesetResolver.forVisionTasks(`${base}mediapipe`);
18-
const opts = (delegate: 'GPU' | 'CPU') => ({
19-
baseOptions: { modelAssetPath: `${base}models/blaze_face_short_range.tflite`, delegate },
26+
const opts = (d: 'GPU' | 'CPU') => ({
27+
baseOptions: { modelAssetPath: `${base}models/blaze_face_short_range.tflite`, delegate: d },
2028
runningMode: 'VIDEO' as const, minDetectionConfidence: 0.5,
2129
});
22-
try { return await FaceDetector.createFromOptions(fileset, opts('GPU')); } catch { return FaceDetector.createFromOptions(fileset, opts('CPU')); }
30+
if (delegate === 'GPU') {
31+
try { const det = await FaceDetector.createFromOptions(fileset, opts('GPU')); faceStatus.delegate = 'GPU'; return det; } catch (e) { console.warn('face tracking: GPU delegate unavailable, using CPU', e); }
32+
}
33+
const det = await FaceDetector.createFromOptions(fileset, opts('CPU'));
34+
faceStatus.delegate = 'CPU';
35+
return det;
2336
})();
37+
detector.catch(() => { detector = null; });
2438
return detector;
2539
}
2640

41+
/** Drop the GPU detector after a runtime failure and continue on the CPU. */
42+
async function fallBackToCpu(e: unknown) {
43+
console.warn('face tracking: GPU detection failed, switching to CPU', e);
44+
const old = detector; detector = null;
45+
try { (await old)?.close(); } catch { /* already gone */ }
46+
return getDetector('CPU');
47+
}
48+
2749
const scratch = document.createElement('canvas');
28-
/** Something MediaPipe can read: the element itself or a copy of raw frames. */
29-
function imageOf(src: Source): TexImageSource | null {
30-
if (src.element) return src.element;
50+
/** Raw frames are decimated to this width for detection (the model looks at 128 × 128 anyway). */
51+
const DETECT_W = 640;
52+
/**
53+
* Something MediaPipe can read: the element itself or a decimated copy of raw frames, and the
54+
* factor from its pixels back to source pixels.
55+
*/
56+
function imageOf(src: Source): { img: TexImageSource; scale: number } | null {
57+
if (src.element) return { img: src.element, scale: 1 };
3158
if (!src.data) return null;
32-
const w = src.width, h = src.height;
33-
let px: Uint8ClampedArray;
34-
if (src.yuv) {
35-
const dec = src.decoder(), d = src.data;
36-
px = new Uint8ClampedArray(w * h * 4);
37-
for (let i = 0; i < px.length; i += 4) { const c = dec(d, i); px[i] = c[0] * 255; px[i + 1] = c[1] * 255; px[i + 2] = c[2] * 255; px[i + 3] = 255; }
38-
} else if (src.depth === 8) px = new Uint8ClampedArray(src.data.buffer, src.data.byteOffset, w * h * 4);
39-
else { px = new Uint8ClampedArray(w * h * 4); for (let i = 0; i < px.length; i++) px[i] = src.data[i] / 257; }
59+
const step = Math.max(1, Math.ceil(src.width / DETECT_W));
60+
const w = Math.floor(src.width / step), h = Math.floor(src.height / step), W = src.width;
61+
const px = new Uint8ClampedArray(w * h * 4), d = src.data;
62+
const dec = src.yuv ? src.decoder() : null, k = src.depth === 8 ? 1 : 1 / 257;
63+
for (let y = 0; y < h; y++) {
64+
for (let x = 0; x < w; x++) {
65+
const i = ((y * step) * W + x * step) * 4, o = (y * w + x) * 4;
66+
if (dec) { const c = dec(d, i); px[o] = c[0] * 255; px[o + 1] = c[1] * 255; px[o + 2] = c[2] * 255; }
67+
else { px[o] = d[i] * k; px[o + 1] = d[i + 1] * k; px[o + 2] = d[i + 2] * k; }
68+
px[o + 3] = 255;
69+
}
70+
}
4071
scratch.width = w; scratch.height = h;
4172
scratch.getContext('2d')!.putImageData(new ImageData(px as Uint8ClampedArray<ArrayBuffer>, w, h), 0, 0);
42-
return scratch;
73+
return { img: scratch, scale: step };
4374
}
4475

4576
const center = (b: Box) => [(b[0] + b[2]) / 2, (b[1] + b[3]) / 2];
@@ -71,17 +102,24 @@ export function overlap(a: Box, b: Box) {
71102
}
72103

73104
async function detect(s: Source): Promise<Box[]> {
74-
const det = await getDetector();
75-
const img = imageOf(s);
76-
if (!img) return [];
105+
let det = await getDetector();
106+
const im = imageOf(s);
107+
if (!im) return [];
108+
const { img, scale } = im;
77109
lastTs = Math.max(lastTs + 1, performance.now());
78-
const res = det.detectForVideo(img as HTMLVideoElement, lastTs);
110+
let res;
111+
try { res = det.detectForVideo(img as HTMLVideoElement, lastTs); } catch (e) {
112+
if (faceStatus.delegate !== 'GPU') throw e;
113+
det = await fallBackToCpu(e);
114+
lastTs += 1;
115+
res = det.detectForVideo(img as HTMLVideoElement, lastTs);
116+
}
79117
// BlazeFace boxes run from the brows to the chin: extend upwards to include the forehead
80118
// (skin measurement), only a little sideways and down
81119
const side = 0.08, top = 0.3, bottom = 0.02;
82120
return res.detections.map((d) => d.boundingBox).filter((b): b is NonNullable<typeof b> => !!b).map((b) =>
83121
[b.originX - b.width * side, b.originY - b.height * top, b.originX + b.width * (1 + side), b.originY + b.height * (1 + bottom)]
84-
.map((v, i) => Math.round(Math.max(0, Math.min(i % 2 ? s.height : s.width, v)))) as Box);
122+
.map((v, i) => Math.round(Math.max(0, Math.min(i % 2 ? s.height : s.width, v * scale)))) as Box);
85123
}
86124

87125
/** One-off check: the face that lies inside a hand-drawn rectangle, if any. */
@@ -107,8 +145,10 @@ export async function trackFaces(sources: Source[], onChange: () => void) {
107145
s.faces = matchFaces(s.faces, boxes);
108146
if (s.faces.map((f) => f.id).join() !== before) onChange(); // face list shows up in the UI
109147
}
148+
faceStatus.error = '';
110149
} catch (e) {
111150
console.warn('face tracking', e);
151+
faceStatus.error = e instanceof Error ? e.message : String(e);
112152
tracked.forEach((s) => (s.faceMode = 'off'));
113153
onChange();
114154
} finally {

‎src/i18n/de/panel.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,4 +160,6 @@ export default {
160160
'panel.draw.castTowards': 'Richtung {hue}',
161161
'panel.draw.castNeutral': ' (neutral)',
162162
'panel.draw.justOutside': 'knapp außerhalb {gamut} (≤ 5 %)',
163+
'panel.facesFailed': 'Gesichtserkennung angehalten: {error}. Hilfe → Feedback senden schickt die Details mit.',
164+
'panel.facesCpu': 'Gesichtserkennung läuft auf dem Prozessor (die Grafikkarte hat abgelehnt) – funktioniert, etwas langsamer.',
163165
} satisfies Translation<typeof en>;

‎src/i18n/en/panel.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -159,4 +159,6 @@ export default {
159159
'panel.draw.castTowards': 'towards {hue}',
160160
'panel.draw.castNeutral': ' (neutral)',
161161
'panel.draw.justOutside': 'just outside {gamut} (≤ 5 %)',
162+
'panel.facesFailed': 'Face detection stopped: {error}. Help → Send Feedback includes the details.',
163+
'panel.facesCpu': 'Face detection runs on the processor (the graphics card refused) – works, a little slower.',
162164
} as const satisfies Messages;

‎src/main.ts‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1027,8 +1027,11 @@ function panelSettings(p: PanelState): Node[] {
10271027
if (s) {
10281028
row(t('panel.faces'), select(s.faceMode, [['off', t('panel.offCap')], ['detect', t('panel.facesDetect')], ['all', t('panel.facesAll')]], (v) => {
10291029
s.faceMode = v as Source['faceMode'];
1030-
if (s.faceMode === 'off') { s.faces = []; s.faceSel.clear(); }
1030+
if (s.faceMode === 'off') { s.faces = []; s.faceSel.clear(); } else { faceLoadError = ''; if (faceMod) faceMod.faceStatus.error = ''; }
10311031
refreshHeads(); }, t('panel.facesTitle')));
1032+
const fe = faceLoadError || faceMod?.faceStatus.error;
1033+
if (fe) row('', h('p', { class: 'hint warn', role: 'alert' }, t('panel.facesFailed', { error: fe })));
1034+
else if (s.faceTrack && faceMod?.faceStatus.delegate === 'CPU') row('', h('p', { class: 'hint' }, t('panel.facesCpu')));
10321035
}
10331036
}
10341037
if (p.scope === 'stats') {
@@ -1090,9 +1093,20 @@ function roiChip(p: PanelState): Node | string {
10901093
}
10911094

10921095
// face tracking (MediaPipe is only loaded once someone switches it on)
1096+
let faceLoadError = '';
1097+
let faceMod: typeof import('./face') | null = null;
1098+
let faceTick = 0;
10931099
setInterval(() => {
10941100
if (!sources.some((s) => s.faceTrack)) return;
1095-
import('./face').then((m) => m.trackFaces(sources, refreshHeads));
1101+
// on the CPU (GPU delegate refused) detection costs more main-thread time: 5 instead of 10 per second
1102+
if (faceMod?.faceStatus.delegate === 'CPU' && faceTick++ % 2) return;
1103+
import('./face').then((m) => { faceMod = m; return m.trackFaces(sources, refreshHeads); }).catch((e) => {
1104+
// the MediaPipe module or its WASM did not load (offline browser, blocked file …)
1105+
console.warn('face tracking: module not loaded', e);
1106+
faceLoadError = (e as Error).message || String(e);
1107+
for (const s of sources) s.faceMode = 'off';
1108+
refreshHeads();
1109+
});
10961110
}, 100);
10971111

10981112
function toggleSolo(idx: number) {

‎test/faceFallback.test.ts‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
import { describe, expect, it, vi } from 'vitest';
2+
3+
// The GPU delegate is created fine but fails on the first picture (seen on a Windows PC):
4+
// detection must continue on the CPU instead of switching itself off.
5+
const created: string[] = [];
6+
vi.mock('@mediapipe/tasks-vision', () => ({
7+
FilesetResolver: { forVisionTasks: async () => ({}) },
8+
FaceDetector: {
9+
createFromOptions: async (_f: unknown, o: { baseOptions: { delegate: string } }) => {
10+
const d = o.baseOptions.delegate; created.push(d);
11+
return {
12+
close: () => {},
13+
detectForVideo: () => {
14+
if (d === 'GPU') throw new Error('WebGL context lost');
15+
return { detections: [{ boundingBox: { originX: 10, originY: 20, width: 100, height: 100 } }] };
16+
},
17+
};
18+
},
19+
},
20+
}));
21+
vi.stubGlobal('document', { createElement: () => ({}) });
22+
vi.stubGlobal('location', { href: 'http://localhost/' });
23+
const { trackFaces, faceStatus } = await import('../src/face');
24+
25+
describe('face tracking fallback', () => {
26+
it('falls back to the CPU when the GPU delegate fails on the first picture', async () => {
27+
const src = { faceTrack: true, ready: true, element: {}, width: 1920, height: 1080, faces: [] as unknown[], faceMode: 'all' };
28+
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
29+
await trackFaces([src as never], () => {});
30+
warn.mockRestore();
31+
expect(created).toEqual(['GPU', 'CPU']);
32+
expect(faceStatus.delegate).toBe('CPU');
33+
expect(faceStatus.error).toBe('');
34+
expect(src.faceMode).toBe('all');
35+
expect(src.faces).toHaveLength(1);
36+
});
37+
});

0 commit comments

Comments
 (0)