|
| 1 | +import { useEffect, useRef, useState } from 'react' |
| 2 | +import * as tf from '@tensorflow/tfjs' |
| 3 | +import * as poseDetection from '@tensorflow-models/pose-detection' |
| 4 | +import { t } from '../lib/i18n.js' |
| 5 | + |
| 6 | +// Bone pairs for MoveNet's 17-keypoint layout — only the ones useful for checking |
| 7 | +// lifting form (arms, torso, legs); face points are detected but not drawn. |
| 8 | +const EDGES = [ |
| 9 | + ['left_shoulder', 'right_shoulder'], ['left_shoulder', 'left_elbow'], ['left_elbow', 'left_wrist'], |
| 10 | + ['right_shoulder', 'right_elbow'], ['right_elbow', 'right_wrist'], |
| 11 | + ['left_shoulder', 'left_hip'], ['right_shoulder', 'right_hip'], ['left_hip', 'right_hip'], |
| 12 | + ['left_hip', 'left_knee'], ['left_knee', 'left_ankle'], |
| 13 | + ['right_hip', 'right_knee'], ['right_knee', 'right_ankle'], |
| 14 | +] |
| 15 | +const MIN_SCORE = 0.3 |
| 16 | + |
| 17 | +// Live skeleton overlay on the front camera — issue #17's MVP: prove the pose-detection |
| 18 | +// pipeline works end to end before adding any rep-counting or form-correction logic. |
| 19 | +// Everything runs on-device (tfjs + MoveNet); the video frame never leaves the browser. |
| 20 | +export default function FormTracker() { |
| 21 | + const videoRef = useRef(null) |
| 22 | + const canvasRef = useRef(null) |
| 23 | + const [status, setStatus] = useState('loading') // loading | ready | denied | error |
| 24 | + |
| 25 | + useEffect(() => { |
| 26 | + let stream, detector, raf, stopped = false |
| 27 | + ;(async () => { |
| 28 | + try { |
| 29 | + stream = await navigator.mediaDevices.getUserMedia({ video: { facingMode: 'user' } }) |
| 30 | + if (stopped) return |
| 31 | + const v = videoRef.current |
| 32 | + v.srcObject = stream |
| 33 | + await v.play() |
| 34 | + await tf.ready() |
| 35 | + detector = await poseDetection.createDetector(poseDetection.SupportedModels.MoveNet, { |
| 36 | + modelType: poseDetection.movenet.modelType.SINGLEPOSE_LIGHTNING, |
| 37 | + }) |
| 38 | + if (stopped) return |
| 39 | + setStatus('ready') |
| 40 | + |
| 41 | + const draw = async () => { |
| 42 | + if (stopped) return |
| 43 | + const c = canvasRef.current |
| 44 | + if (v.readyState >= 2) { |
| 45 | + if (c.width !== v.videoWidth) { c.width = v.videoWidth; c.height = v.videoHeight } |
| 46 | + const poses = await detector.estimatePoses(v) |
| 47 | + const kp = poses[0]?.keypoints || [] |
| 48 | + const byName = Object.fromEntries(kp.map(p => [p.name, p])) |
| 49 | + const ctx = c.getContext('2d') |
| 50 | + ctx.clearRect(0, 0, c.width, c.height) |
| 51 | + ctx.strokeStyle = '#5eead4' |
| 52 | + ctx.lineWidth = Math.max(2, c.width / 160) |
| 53 | + for (const [a, b] of EDGES) { |
| 54 | + const pa = byName[a], pb = byName[b] |
| 55 | + if (pa && pb && pa.score > MIN_SCORE && pb.score > MIN_SCORE) { |
| 56 | + ctx.beginPath(); ctx.moveTo(pa.x, pa.y); ctx.lineTo(pb.x, pb.y); ctx.stroke() |
| 57 | + } |
| 58 | + } |
| 59 | + ctx.fillStyle = '#facc15' |
| 60 | + const r = Math.max(3, c.width / 120) |
| 61 | + for (const p of kp) if (p.score > MIN_SCORE) { ctx.beginPath(); ctx.arc(p.x, p.y, r, 0, Math.PI * 2); ctx.fill() } |
| 62 | + } |
| 63 | + raf = requestAnimationFrame(draw) |
| 64 | + } |
| 65 | + draw() |
| 66 | + } catch (e) { |
| 67 | + setStatus(e && e.name === 'NotAllowedError' ? 'denied' : 'error') |
| 68 | + } |
| 69 | + })() |
| 70 | + return () => { |
| 71 | + stopped = true |
| 72 | + if (raf) cancelAnimationFrame(raf) |
| 73 | + stream?.getTracks().forEach(tr => tr.stop()) |
| 74 | + detector?.dispose() |
| 75 | + } |
| 76 | + }, []) |
| 77 | + |
| 78 | + return <> |
| 79 | + <h3>{t('Form check')}</h3> |
| 80 | + {status === 'denied' && <div className="muted">{t('Camera access was denied — allow it in your browser settings to see your form.')}</div>} |
| 81 | + {status === 'error' && <div className="muted">{t('Could not start the camera on this device.')}</div>} |
| 82 | + {status !== 'denied' && status !== 'error' && <div style={{ position: 'relative', marginTop: 8 }}> |
| 83 | + <video ref={videoRef} playsInline muted style={{ width: '100%', borderRadius: 12, display: 'block', transform: 'scaleX(-1)' }} /> |
| 84 | + <canvas ref={canvasRef} style={{ position: 'absolute', inset: 0, width: '100%', height: '100%', transform: 'scaleX(-1)' }} /> |
| 85 | + {status === 'loading' && <div className="muted" style={{ position: 'absolute', inset: 0, display: 'flex', alignItems: 'center', justifyContent: 'center' }}>{t('Starting camera…')}</div>} |
| 86 | + </div>} |
| 87 | + <div className="muted small" style={{ marginTop: 10 }}>{t('Skeleton overlay only, for now — nothing is recorded or uploaded.')}</div> |
| 88 | + </> |
| 89 | +} |
0 commit comments