ml5-handpose.recho.js

/**
 * @title ASCII Hands with Ml5.js HandPose
 * @author Bairui Su
 * @created 2025-09-21
 * @pull_request 115
 * @github pearmini
 * @thumbnail_start 40
 * @label ASCII Art
 */

/**
 * ============================================================================
 * =                     ASCII Hands with Ml5.js HandPose                     =
 * ============================================================================
 *
 * Wave your hands to see the effect!
 *
 * This example shows how to use **Ml5** HandPose to detect hands and draw them
 * on the screen. Also, it shows how to use **p5** to create a capture video
 * element and remove it after the video is no longer needed.
 *
 * Note that we use ml5 **async constructor** to create the handPose object,
 * which is what I worked on this summer as a ml5 researcher to support p5 2.0
 * async setup. The async constructor is a new feature in ml5 1.0.0.
 *
 * You can check out the original pull request here:
 * https://github.com/ml5js/ml5-next-gen/pull/258
 *
 * You can check out the original issue here:
 * https://github.com/ml5js/ml5-next-gen/issues/244
 */

const width = 96;
const height = 30;
const scaleX = d3.scaleLinear([0, 640], [0, width]);
const scaleY = d3.scaleLinear([0, 480], [0, height]);
const video = await createCapture(640, 480);
const handPose = await ml5.handPose(video); // Async constructor!!!

//➜ ................................................................................................
//➜ .............@..................................................................................
//➜ ....................@...........................................................................
//➜ ....@.......@...................................................................................
//➜ ....@..............@.........@..........................@.......................................
//➜ ...........@..................................................@.................................
//➜ ....@.............@........@...................@.........@....@.................................
//➜ ........................@.............................................@.........................
//➜ ................................................@........@....@.................................
//➜ ....@....@.....@.................................@...................@..........................
//➜ ...................@................................................@...........................
//➜ ..................................@......................@......................................
//➜ .....................................................@.......@..................................
//➜ .............................@...................................@..............................
//➜ ................................................................................................
//➜ ......................@................@...@....................................................
//➜ ..............@.................................................................................
//➜ ....@............................................@..............................................
//➜ ......................................................@.........................................
//➜ .............................................................@..................................
//➜ ................................................................................................
//➜ ................................................................................................
//➜ ................................................................................................
//➜ ................................................................................................
//➜ ................................................................................................
//➜ ................................................................................................
//➜ ................................................................................................
//➜ ................................................................................................
//➜ ................................................................................................
//➜ ................................................................................................
handPose.detectStart(video, (hands) => {
  const buffer = d3.range(width * height).map(() => ".");

  for (let i = 0; i < hands.length; i++) {
    const hand = hands[i];
    for (let j = 0; j < hand.keypoints.length; j++) {
      const points = hand.keypoints[j];
      const x = ~~scaleX(points.x);
      const y = ~~scaleY(points.y);
      if (x > 0 && x < width && y > 0 && y < height) buffer[y * width + x] = "@";
    }
  }

  let output = "";
  for (let i = 0; i < height; ++i) {
    for (let j = 0; j < width; ++j) output += buffer[i * width + j];
    output += i === height - 1 ? "" : "\n";
  }
  echo.clear();
  echo(output);
});

{
  echo.dispose(() => removeCapture(video));
}

// Use p5 to create a capture video element.
function createCapture(width, height) {
  return new Promise((resolve) => {
    new p5((p) => {
      p.setup = () => {
        p.noCanvas();
        p.noLoop();
        const video = p.createCapture(p.VIDEO);
        video.size(width, height);
        video.hide();
        resolve(video);
      };
    });
  });
}

// Remove the capture video element.
function removeCapture(video) {
  if (video) {
    if (video.elt && video.elt.srcObject) {
      video.elt.srcObject.getTracks().forEach((track) => track.stop());
    }
    video.remove();
  }
}

const ml5 = await recho.require("https://unpkg.com/ml5@1/dist/ml5.js");
const p5 = await recho.require("https://unpkg.com/p5@1.2.0/lib/p5.js");
const d3 = await recho.require("d3-array", "d3-scale");