Smooth head rotations

Author: iwoplaza

apps/typegpu-docs/src/examples/rendering/pufferfish/index.ts

@@ -95,15 +95,13 @@ function draw(timestamp: number) {
   const invProjMat = mat4.identity(d.mat4x4f());
   const scale = Math.max(1, canvas.height / canvas.width);
   const aspect = canvas.width / canvas.height;
-  const puffScale = 1.2 - pufferfishController.sizeSpring.value * 0.2;
+  const puffScale = 1.5 - pufferfishController.sizeSpring.value * 0.2;
   mat4.scale(invProjMat, [aspect * scale, scale, 1], invProjMat);
   mat4.scale(invProjMat, d.vec3f(puffScale, puffScale, 1), invProjMat);
 
   const invModelMat = mat4.identity(d.mat4x4f());
-  const headPitch = Math.sin(timestamp * 0.001) * 0.2;
-  const headYaw = Math.sin(timestamp * 0.0023) * 0.2;
-  mat4.rotateY(invModelMat, headYaw, invModelMat);
-  mat4.rotateX(invModelMat, headPitch, invModelMat);
+  mat4.rotateY(invModelMat, pufferfishController.headYaw, invModelMat);
+  mat4.rotateX(invModelMat, -pufferfishController.headPitch, invModelMat);
 
   const videoTexture = device.importExternalTexture({ source: video });
   const frequentGroup = root.createBindGroup(frequentLayout, {

apps/typegpu-docs/src/examples/rendering/pufferfish/pufferfish-controller.ts

@@ -128,14 +128,31 @@ function extractFaceLandmarks(landmarks: NormalizedLandmark[]): FaceLandmarks {
   };
 }
 
+function encroach(
+  from: number,
+  to: number,
+  factorPerSecond: number,
+  deltaSeconds: number,
+): number {
+  const diff = to - from;
+  const factor = factorPerSecond ** deltaSeconds;
+  return from + diff * (1 - factor);
+}
+
 export class PufferfishController {
   #faceLandmarker: FaceLandmarker;
   sizeSpring: Spring;
   faceLandmarks: FaceLandmarks | undefined;
+  smoothFaceLandmarks: FaceLandmarks | undefined;
+
+  headYaw: number;
+  headPitch: number;
 
   constructor(faceLandmarker: FaceLandmarker) {
     this.#faceLandmarker = faceLandmarker;
-    this.sizeSpring = new Spring({ damping: 10, mass: 1, stiffness: 1000 });
+    this.sizeSpring = new Spring({ damping: 16, mass: 1, stiffness: 1000 });
+    this.headYaw = 0;
+    this.headPitch = 0;
   }
 
   updatePuffScore(
@@ -157,6 +174,41 @@ export class PufferfishController {
 
   update(dt: number) {
     this.sizeSpring.update(dt);
+
+    const current = this.faceLandmarks;
+    const prev = this.smoothFaceLandmarks || current;
+    if (!prev || !current) {
+      return;
+    }
+
+    const fromFaceOval = prev.faceOval;
+    const toFaceOval = current.faceOval;
+
+    const conv = 0.002;
+    this.smoothFaceLandmarks = {
+      ...prev,
+      // We only smooth out the face oval for now, as it's the only landmark we use
+      faceOval: {
+        xMin: encroach(fromFaceOval.xMin, toFaceOval.xMin, conv, dt),
+        xMax: encroach(fromFaceOval.xMax, toFaceOval.xMax, conv, dt),
+        yMin: encroach(fromFaceOval.yMin, toFaceOval.yMin, conv, dt),
+        yMax: encroach(fromFaceOval.yMax, toFaceOval.yMax, conv, dt),
+      },
+    };
+
+    const nose = current.nose;
+
+    // Calculate the center of the face oval
+    const ovalX = (toFaceOval.xMin + toFaceOval.xMax) / 2;
+    const ovalY = (toFaceOval.yMin + toFaceOval.yMax) / 2;
+
+    // Determining the head's orientation based on how much the nose is off-center
+    const targetYaw = -(nose.x - ovalX) * 10;
+    const targetPitch = (nose.y - ovalY) * 10;
+
+    // Smoothly approaching the target orientation
+    this.headYaw = encroach(this.headYaw, targetYaw, 0.01, dt);
+    this.headPitch = encroach(this.headPitch, targetPitch, 0.01, dt);
   }
 }
 

apps/typegpu-docs/src/examples/rendering/pufferfish/shader.ts

@@ -54,28 +54,6 @@ const Shape = d.struct({
 });
 type Shape = d.Infer<typeof Shape>;
 
-const rotationMatrixY = (angle: number): d.m3x3f => {
-  'use gpu';
-  const c = std.cos(angle);
-  const s = std.sin(angle);
-
-  return d.mat3x3f(c, 0.0, s, 0.0, 1.0, 0.0, -s, 0.0, c);
-};
-
-const rotationMatrixX = (angle: number): d.m3x3f => {
-  'use gpu';
-  const c = std.cos(angle);
-  const s = std.sin(angle);
-  return d.mat3x3f(1.0, 0.0, 0.0, 0.0, c, -s, 0.0, s, c);
-};
-
-const rotationMatrixZ = (angle: number): d.m3x3f => {
-  'use gpu';
-  const c = std.cos(angle);
-  const s = std.sin(angle);
-  return d.mat3x3f(c, -s, 0.0, s, c, 0.0, 0.0, 0.0, 1.0);
-};
-
 const sdEllipsoid = (p: d.v3f, r: d.v3f): number => {
   'use gpu';
   const k0 = std.length(p.div(r));
@@ -330,6 +308,9 @@ export const fullColorFragment = tgpu['~unstable'].fragmentFn({
    */
   const faceSize = 1.8;
   const faceUv = suv.mul(faceSize).mul(0.5).add(0.5);
+  // Flip the x axis
+  faceUv.x = 1.0 - faceUv.x;
+
   if (distance < MAX_DIST) {
     // Hit the pufferfish
     // const localPos = hitPos.sub(bodyPosition);
@@ -393,11 +374,11 @@ export const sdfDebugFragment = tgpu['~unstable'].fragmentFn({
   out: d.vec4f,
 })((input) => {
   'use gpu';
-  const uv = input.uv.mul(2).sub(1).mul(1.5);
+  const uv = input.uv.mul(2).sub(1);
   const bodyPosition = bodyPositionAccess.$;
   // Ray setup
-  const initialRo = d.vec3f(uv.x, uv.y, 0);
-  let ro = uniformsAccess.$.invProjMat.mul(d.vec4f(initialRo, 1)).xyz;
+  const suv = uniformsAccess.$.invProjMat.mul(d.vec4f(uv, 0, 1)).xy;
+  let ro = d.vec3f(suv, 0);
   let rd = uniformsAccess.$.invProjMat.mul(d.vec4f(0, 0, 1, 0)).xyz;
 
   // Transforming around the pufferfish