Update Detector classes to run their tfjs detectors off of a sing…

…le static `Detector.currentFrame` instance variable
Valkryst · Dec 6, 2023 · 0f9b979 · 0f9b979
1 parent 754c765
commit 0f9b979
Show file tree

Hide file tree

Showing 6 changed files with 40 additions and 6 deletions.
diff --git a/js/detector/body_detector.js b/js/detector/body_detector.js
@@ -29,7 +29,6 @@ export class BodyDetector extends Detector {
         BodyDetector.instance = this;
 
         this.mesh = mesh;
-        this.inputElement = document.getElementById("jellron-video-canvas");
 
         poseDetection.createDetector(
             poseDetection.SupportedModels.MoveNet,
@@ -56,7 +55,7 @@ export class BodyDetector extends Detector {
 
             let rawBodies = [];
             try {
-                rawBodies = await this.detector.estimatePoses(this.inputElement);
+                rawBodies = await this.detector.estimatePoses(Detector.getCurrentFrame());
             } catch (e) {
                 /*
                  * Depending on the state of the video element, this can throw a "Requested texture size [0x0] is

diff --git a/js/detector/detector.js b/js/detector/detector.js
@@ -1,6 +1,10 @@
 import {RunnableInterval} from "../runnable_interval.js";
+import {validateInstanceOf} from "../utility/validation.js";
 
 export class Detector extends RunnableInterval {
+    /** Current frame to be processed. */
+    static currentFrame = null;
+
     /** Creates a new Detector. */
     constructor() {
         super();
@@ -25,4 +29,23 @@ export class Detector extends RunnableInterval {
     isReady() {
         return this.detector != null;
     }
+
+    /**
+     * Retrieves the current frame.
+     *
+     * @returns {Tensor} Current frame.
+     */
+    static getCurrentFrame() {
+        return Detector.currentFrame;
+    }
+
+    /**
+     * Sets the current frame.
+     *
+     * @param frame New current frame.
+     */
+    static setCurrentFrame(frame) {
+        Detector.currentFrame?.dispose();
+        Detector.currentFrame = frame;
+    }
 }
diff --git a/js/detector/face_detector.js b/js/detector/face_detector.js
@@ -29,7 +29,6 @@ export class FaceDetector extends Detector {
         FaceDetector.instance = this;
 
         this.mesh = mesh;
-        this.inputElement = document.getElementById("jellron-video-canvas");
 
         faceLandmarksDetection.createDetector(
             faceLandmarksDetection.SupportedModels.MediaPipeFaceMesh,
@@ -56,7 +55,7 @@ export class FaceDetector extends Detector {
 
             let rawFaces = [];
             try {
-                rawFaces = await this.detector.estimateFaces(this.inputElement);
+                rawFaces = await this.detector.estimateFaces(Detector.getCurrentFrame());
             } catch (e) {
                 /*
                  * Depending on the state of the video element, this can throw a "Requested texture size [0x0] is

diff --git a/js/detector/hand_detector.js b/js/detector/hand_detector.js
@@ -29,7 +29,6 @@ export class HandDetector extends Detector {
         HandDetector.instance = this;
 
         this.mesh = mesh;
-        this.inputElement = document.getElementById("jellron-video-canvas");
 
         handPoseDetection.createDetector(
             handPoseDetection.SupportedModels.MediaPipeHands,
@@ -56,7 +55,7 @@ export class HandDetector extends Detector {
 
             let rawHands = [];
             try {
-                rawHands = await this.detector.estimateHands(this.inputElement);
+                rawHands = await this.detector.estimateHands(Detector.getCurrentFrame());
             } catch (e) {
                 /*
                  * Depending on the state of the video element, this can throw a "Requested texture size [0x0] is

diff --git a/js/renderer/keypoint_renderer.js b/js/renderer/keypoint_renderer.js
@@ -1,3 +1,4 @@
+import {Detector} from "../detector/detector.js";
 import {Keypoint} from "../keypoint.js";
 import {Mesh} from "../mesh.js";
 import {Renderer} from "./renderer.js";
@@ -37,6 +38,7 @@ export class KeypointRenderer extends Renderer {
         }
 
         const scene = new Scene();
+        const videoCanvas = document.getElementById("jellron-video-canvas");
 
         this.dispatchEvent(new CustomEvent("started"));
         this.intervalId = setInterval(async () => {
@@ -110,6 +112,9 @@ export class KeypointRenderer extends Renderer {
 
             this.glContext.render(scene, this.getCamera());
 
+            // We set this here because the KeypointRenderer runs at a lower framerate than VideoRenderer.
+            Detector.setCurrentFrame(tf.browser.fromPixels(videoCanvas));
+
             this.lastRuntime = performance.now() - currentTime;
             this.dispatchEvent(new CustomEvent("rendered", {detail: {runtime: this.lastRuntime}}));
         }, 1000 / KeypointRenderer.fps);

diff --git a/js/renderer/renderer.js b/js/renderer/renderer.js
@@ -52,6 +52,15 @@ export class Renderer extends RunnableInterval {
         return camera;
     }
 
+    /**
+     * Retrieves the canvas to use for rendering.
+     *
+     * @returns {HTMLCanvasElement} Canvas to use for rendering.
+     */
+    getCanvas() {
+        return this.canvas;
+    }
+
     /**
      * Sets the canvas to use for rendering.
      *