@@ -10,36 +10,67 @@ type Box = [number, number, number, number];
1010export interface Face { id : number ; box : Box }
1111let nextId = 1 ;
1212
13+ /**
14+ * Which way the detector runs, and why it stopped. The GPU delegate can be created and still
15+ * fail on the first picture (some drivers, ANGLE/D3D11, software WebGL); then detection falls
16+ * back to the CPU instead of switching itself off silently. A failed load is not kept, so
17+ * switching face tracking on again retries.
18+ */
19+ export const faceStatus : { delegate : 'GPU' | 'CPU' | null ; error : string } = { delegate : null , error : '' } ;
20+
1321let detector : Promise < FaceDetector > | null = null ;
14- function getDetector ( ) {
22+ function getDetector ( delegate : 'GPU' | 'CPU' = 'GPU' ) {
1523 detector ??= ( async ( ) => {
1624 const base = new URL ( '.' , location . href ) . href ;
1725 const fileset = await FilesetResolver . forVisionTasks ( `${ base } mediapipe` ) ;
18- const opts = ( delegate : 'GPU' | 'CPU' ) => ( {
19- baseOptions : { modelAssetPath : `${ base } models/blaze_face_short_range.tflite` , delegate } ,
26+ const opts = ( d : 'GPU' | 'CPU' ) => ( {
27+ baseOptions : { modelAssetPath : `${ base } models/blaze_face_short_range.tflite` , delegate : d } ,
2028 runningMode : 'VIDEO' as const , minDetectionConfidence : 0.5 ,
2129 } ) ;
22- try { return await FaceDetector . createFromOptions ( fileset , opts ( 'GPU' ) ) ; } catch { return FaceDetector . createFromOptions ( fileset , opts ( 'CPU' ) ) ; }
30+ if ( delegate === 'GPU' ) {
31+ try { const det = await FaceDetector . createFromOptions ( fileset , opts ( 'GPU' ) ) ; faceStatus . delegate = 'GPU' ; return det ; } catch ( e ) { console . warn ( 'face tracking: GPU delegate unavailable, using CPU' , e ) ; }
32+ }
33+ const det = await FaceDetector . createFromOptions ( fileset , opts ( 'CPU' ) ) ;
34+ faceStatus . delegate = 'CPU' ;
35+ return det ;
2336 } ) ( ) ;
37+ detector . catch ( ( ) => { detector = null ; } ) ;
2438 return detector ;
2539}
2640
41+ /** Drop the GPU detector after a runtime failure and continue on the CPU. */
42+ async function fallBackToCpu ( e : unknown ) {
43+ console . warn ( 'face tracking: GPU detection failed, switching to CPU' , e ) ;
44+ const old = detector ; detector = null ;
45+ try { ( await old ) ?. close ( ) ; } catch { /* already gone */ }
46+ return getDetector ( 'CPU' ) ;
47+ }
48+
2749const scratch = document . createElement ( 'canvas' ) ;
28- /** Something MediaPipe can read: the element itself or a copy of raw frames. */
29- function imageOf ( src : Source ) : TexImageSource | null {
30- if ( src . element ) return src . element ;
50+ /** Raw frames are decimated to this width for detection (the model looks at 128 × 128 anyway). */
51+ const DETECT_W = 640 ;
52+ /**
53+ * Something MediaPipe can read: the element itself or a decimated copy of raw frames, and the
54+ * factor from its pixels back to source pixels.
55+ */
56+ function imageOf ( src : Source ) : { img : TexImageSource ; scale : number } | null {
57+ if ( src . element ) return { img : src . element , scale : 1 } ;
3158 if ( ! src . data ) return null ;
32- const w = src . width , h = src . height ;
33- let px : Uint8ClampedArray ;
34- if ( src . yuv ) {
35- const dec = src . decoder ( ) , d = src . data ;
36- px = new Uint8ClampedArray ( w * h * 4 ) ;
37- for ( let i = 0 ; i < px . length ; i += 4 ) { const c = dec ( d , i ) ; px [ i ] = c [ 0 ] * 255 ; px [ i + 1 ] = c [ 1 ] * 255 ; px [ i + 2 ] = c [ 2 ] * 255 ; px [ i + 3 ] = 255 ; }
38- } else if ( src . depth === 8 ) px = new Uint8ClampedArray ( src . data . buffer , src . data . byteOffset , w * h * 4 ) ;
39- else { px = new Uint8ClampedArray ( w * h * 4 ) ; for ( let i = 0 ; i < px . length ; i ++ ) px [ i ] = src . data [ i ] / 257 ; }
59+ const step = Math . max ( 1 , Math . ceil ( src . width / DETECT_W ) ) ;
60+ const w = Math . floor ( src . width / step ) , h = Math . floor ( src . height / step ) , W = src . width ;
61+ const px = new Uint8ClampedArray ( w * h * 4 ) , d = src . data ;
62+ const dec = src . yuv ? src . decoder ( ) : null , k = src . depth === 8 ? 1 : 1 / 257 ;
63+ for ( let y = 0 ; y < h ; y ++ ) {
64+ for ( let x = 0 ; x < w ; x ++ ) {
65+ const i = ( ( y * step ) * W + x * step ) * 4 , o = ( y * w + x ) * 4 ;
66+ if ( dec ) { const c = dec ( d , i ) ; px [ o ] = c [ 0 ] * 255 ; px [ o + 1 ] = c [ 1 ] * 255 ; px [ o + 2 ] = c [ 2 ] * 255 ; }
67+ else { px [ o ] = d [ i ] * k ; px [ o + 1 ] = d [ i + 1 ] * k ; px [ o + 2 ] = d [ i + 2 ] * k ; }
68+ px [ o + 3 ] = 255 ;
69+ }
70+ }
4071 scratch . width = w ; scratch . height = h ;
4172 scratch . getContext ( '2d' ) ! . putImageData ( new ImageData ( px as Uint8ClampedArray < ArrayBuffer > , w , h ) , 0 , 0 ) ;
42- return scratch ;
73+ return { img : scratch , scale : step } ;
4374}
4475
4576const center = ( b : Box ) => [ ( b [ 0 ] + b [ 2 ] ) / 2 , ( b [ 1 ] + b [ 3 ] ) / 2 ] ;
@@ -71,17 +102,24 @@ export function overlap(a: Box, b: Box) {
71102}
72103
73104async function detect ( s : Source ) : Promise < Box [ ] > {
74- const det = await getDetector ( ) ;
75- const img = imageOf ( s ) ;
76- if ( ! img ) return [ ] ;
105+ let det = await getDetector ( ) ;
106+ const im = imageOf ( s ) ;
107+ if ( ! im ) return [ ] ;
108+ const { img, scale } = im ;
77109 lastTs = Math . max ( lastTs + 1 , performance . now ( ) ) ;
78- const res = det . detectForVideo ( img as HTMLVideoElement , lastTs ) ;
110+ let res ;
111+ try { res = det . detectForVideo ( img as HTMLVideoElement , lastTs ) ; } catch ( e ) {
112+ if ( faceStatus . delegate !== 'GPU' ) throw e ;
113+ det = await fallBackToCpu ( e ) ;
114+ lastTs += 1 ;
115+ res = det . detectForVideo ( img as HTMLVideoElement , lastTs ) ;
116+ }
79117 // BlazeFace boxes run from the brows to the chin: extend upwards to include the forehead
80118 // (skin measurement), only a little sideways and down
81119 const side = 0.08 , top = 0.3 , bottom = 0.02 ;
82120 return res . detections . map ( ( d ) => d . boundingBox ) . filter ( ( b ) : b is NonNullable < typeof b > => ! ! b ) . map ( ( b ) =>
83121 [ b . originX - b . width * side , b . originY - b . height * top , b . originX + b . width * ( 1 + side ) , b . originY + b . height * ( 1 + bottom ) ]
84- . map ( ( v , i ) => Math . round ( Math . max ( 0 , Math . min ( i % 2 ? s . height : s . width , v ) ) ) ) as Box ) ;
122+ . map ( ( v , i ) => Math . round ( Math . max ( 0 , Math . min ( i % 2 ? s . height : s . width , v * scale ) ) ) ) as Box ) ;
85123}
86124
87125/** One-off check: the face that lies inside a hand-drawn rectangle, if any. */
@@ -107,8 +145,10 @@ export async function trackFaces(sources: Source[], onChange: () => void) {
107145 s . faces = matchFaces ( s . faces , boxes ) ;
108146 if ( s . faces . map ( ( f ) => f . id ) . join ( ) !== before ) onChange ( ) ; // face list shows up in the UI
109147 }
148+ faceStatus . error = '' ;
110149 } catch ( e ) {
111150 console . warn ( 'face tracking' , e ) ;
151+ faceStatus . error = e instanceof Error ? e . message : String ( e ) ;
112152 tracked . forEach ( ( s ) => ( s . faceMode = 'off' ) ) ;
113153 onChange ( ) ;
114154 } finally {
0 commit comments