From a3bcde5a0db453c0f1152436772a45bb5ee9794d Mon Sep 17 00:00:00 2001 From: Maya Rajan Date: Thu, 13 Mar 2025 12:38:59 -0400 Subject: [PATCH 01/35] updates --- scratch-packages/scratch-vm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scratch-packages/scratch-vm b/scratch-packages/scratch-vm index 8aca21fd7..cc723570e 160000 --- a/scratch-packages/scratch-vm +++ b/scratch-packages/scratch-vm @@ -1 +1 @@ -Subproject commit 8aca21fd7b82334d39c804f89d02096121242551 +Subproject commit cc723570e845e1c04dcc7a71e925905b61a7b431 From 39fbf93ca1e45182dd66e6952fedfe5616c61848 Mon Sep 17 00:00:00 2001 From: Maya Rajan Date: Wed, 14 May 2025 18:40:46 -0400 Subject: [PATCH 02/35] temporarily remove api --- extensions/src/textClassification/index.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/extensions/src/textClassification/index.ts b/extensions/src/textClassification/index.ts index 3be3d7505..716645b69 100644 --- a/extensions/src/textClassification/index.ts +++ b/extensions/src/textClassification/index.ts @@ -8,7 +8,7 @@ import timer from "./timer"; import voices, { Voice } from "./voices"; import Sentiment from "sentiment"; import { ToxicityClassifier, load as loadToxicity } from "@tensorflow-models/toxicity"; -import { getTranslationToEnglish } from "./services/translation"; +// import { getTranslationToEnglish } from "./services/translation"; import { Predictor, build, failure, success } from "./model"; const { legacyBlock, } = legacyFullSupport.for(); @@ -309,13 +309,15 @@ export default class TextClassification extends extension(details, "legacySuppor } private async getConfidence(text: string) { - const translation = await getTranslationToEnglish(text); + // const translation = await getTranslationToEnglish(text); + const translation = text; const { score } = await this.customPredictor(translation); return score; } private async getEmbeddings(text) { - const newText = await getTranslationToEnglish(text); //translates text from any language to english + // const newText = await getTranslationToEnglish(text); //translates text from any language to english + const newText = text; if (this.labels.length === 0 || !this.labels[0] || !this.customPredictor) return; const { label } = await this.customPredictor(newText); return label; From d0fc9c97631ab9287c173fbcc5330c660d87e982 Mon Sep 17 00:00:00 2001 From: Maya Rajan Date: Thu, 14 Aug 2025 13:46:28 -0400 Subject: [PATCH 03/35] load tfjs --- extensions/src/poseBody/index.ts | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/extensions/src/poseBody/index.ts b/extensions/src/poseBody/index.ts index 7f9fb3395..10baa08d4 100644 --- a/extensions/src/poseBody/index.ts +++ b/extensions/src/poseBody/index.ts @@ -76,12 +76,26 @@ export default class PoseBody extends Extension { */ bodyOptions = info.menus.PART.items; + + loadScript(src) { + return new Promise((resolve, reject) => { + const script = document.createElement('script'); + script.src = src; + script.async = true; + script.onload = () => resolve(src); + script.onerror = () => reject(new Error(`Failed to load script: ${src}`)); + document.head.appendChild(script); + }); + } + /** * Acts like class PoseBody's constructor (instead of a child class constructor) * @param env */ - init(env: Environment) { + async init(env: Environment) { + await this.loadScript("https://cdn.jsdelivr.net/npm/@tensorflow/tfjs@4.22.0/dist/tf.min.js"); + if (this.runtime.ioDevices) { this._loop(); } @@ -154,6 +168,7 @@ export default class PoseBody extends Extension { * @returns */ async ensureBodyModelLoaded() { + this.bodyModel ??= await posenet.load(); return this.bodyModel; } From a99a7377e6a4c2ff1be838c4d5332a3c9fad046b Mon Sep 17 00:00:00 2001 From: Maya Rajan Date: Thu, 14 Aug 2025 17:11:26 -0400 Subject: [PATCH 04/35] new changes --- extensions/src/poseBody/index.ts | 14 +--- extensions/src/poseBody/package.json | 4 +- extensions/src/poseBody/pnpm-lock.yaml | 97 ++++++++++++++++---------- 3 files changed, 63 insertions(+), 52 deletions(-) diff --git a/extensions/src/poseBody/index.ts b/extensions/src/poseBody/index.ts index 10baa08d4..1082dbcd1 100644 --- a/extensions/src/poseBody/index.ts +++ b/extensions/src/poseBody/index.ts @@ -1,4 +1,5 @@ import { ArgumentType, BlockType, Extension, Block, DefineBlock, Environment, ExtensionMenuDisplayDetails, RuntimeEvent, ValueOf } from "$common"; +import "@tensorflow/tfjs-backend-webgl"; import * as posenet from '@tensorflow-models/posenet'; import { legacyFullSupport, info } from "./legacy"; @@ -77,24 +78,11 @@ export default class PoseBody extends Extension { bodyOptions = info.menus.PART.items; - loadScript(src) { - return new Promise((resolve, reject) => { - const script = document.createElement('script'); - script.src = src; - script.async = true; - script.onload = () => resolve(src); - script.onerror = () => reject(new Error(`Failed to load script: ${src}`)); - document.head.appendChild(script); - }); - } - /** * Acts like class PoseBody's constructor (instead of a child class constructor) * @param env */ async init(env: Environment) { - - await this.loadScript("https://cdn.jsdelivr.net/npm/@tensorflow/tfjs@4.22.0/dist/tf.min.js"); if (this.runtime.ioDevices) { this._loop(); diff --git a/extensions/src/poseBody/package.json b/extensions/src/poseBody/package.json index 3c76f8705..9245e4b33 100644 --- a/extensions/src/poseBody/package.json +++ b/extensions/src/poseBody/package.json @@ -11,6 +11,8 @@ "author": "", "license": "ISC", "dependencies": { - "@tensorflow-models/posenet": "2.2.1" + "@tensorflow-models/posenet": "2.2.2", + "@tensorflow/tfjs-backend-webgl": "3.0.0-rc.1", + "@tensorflow/tfjs-core": "3.0.0-rc.1" } } \ No newline at end of file diff --git a/extensions/src/poseBody/pnpm-lock.yaml b/extensions/src/poseBody/pnpm-lock.yaml index ef5650a0d..940de8abd 100644 --- a/extensions/src/poseBody/pnpm-lock.yaml +++ b/extensions/src/poseBody/pnpm-lock.yaml @@ -9,43 +9,55 @@ importers: .: dependencies: '@tensorflow-models/posenet': - specifier: 2.2.1 - version: 2.2.1(@tensorflow/tfjs-converter@3.21.0(@tensorflow/tfjs-core@3.21.0))(@tensorflow/tfjs-core@3.21.0) + specifier: 2.2.2 + version: 2.2.2(@tensorflow/tfjs-converter@3.21.0(@tensorflow/tfjs-core@3.0.0-rc.1))(@tensorflow/tfjs-core@3.0.0-rc.1) + '@tensorflow/tfjs-backend-webgl': + specifier: 3.0.0-rc.1 + version: 3.0.0-rc.1(@tensorflow/tfjs-core@3.0.0-rc.1) + '@tensorflow/tfjs-core': + specifier: 3.0.0-rc.1 + version: 3.0.0-rc.1 packages: - '@tensorflow-models/posenet@2.2.1': - resolution: {integrity: sha512-n9/g6DfjAyrBTf/zt1haRCyWsgALxUCzg9/Ks3Y2mbYavRZVSCSTRPy/qlE5Hr4tLfyckGfDN14zmGTthNcg/g==} + '@tensorflow-models/posenet@2.2.2': + resolution: {integrity: sha512-0SXIksRet/IdX7WVH+JSD6W3upkGHix1hwtd3xykIoIMGR7zQ4SC5+wZcNt9ofASyxNYQoI+tUULUo4LNw0c3w==} peerDependencies: - '@tensorflow/tfjs-converter': ^1.3.0 - '@tensorflow/tfjs-core': ^1.3.0 + '@tensorflow/tfjs-converter': ^3.0.0-rc.1 + '@tensorflow/tfjs-core': ^3.0.0-rc.1 + + '@tensorflow/tfjs-backend-cpu@3.0.0-rc.1': + resolution: {integrity: sha512-9+13fs3vVpfOpDY+Aa/JXdda2SuBN/clxxoNYIX3pMY0GAGwpUaAkSECLuOOhnngHcl9pxuaJtgqrCcrq+A3/A==} + engines: {yarn: '>= 1.3.2'} + peerDependencies: + '@tensorflow/tfjs-core': 3.0.0-rc.1 + + '@tensorflow/tfjs-backend-webgl@3.0.0-rc.1': + resolution: {integrity: sha512-qEjtL/71HflBHJgWXhtj+8BVYB9Q+NHw282QGWi9QLA9SlWe3BRJhbGCaD28/Ddl+mSzBWFQnK+sKdZ04LL1Sg==} + engines: {yarn: '>= 1.3.2'} + peerDependencies: + '@tensorflow/tfjs-core': 3.0.0-rc.1 '@tensorflow/tfjs-converter@3.21.0': resolution: {integrity: sha512-12Y4zVDq3yW+wSjSDpSv4HnpL2sDZrNiGSg8XNiDE4HQBdjdA+a+Q3sZF/8NV9y2yoBhL5L7V4mMLDdbZBd9/Q==} peerDependencies: '@tensorflow/tfjs-core': 3.21.0 - '@tensorflow/tfjs-core@3.21.0': - resolution: {integrity: sha512-YSfsswOqWfd+M4bXIhT3hwtAb+IV8+ODwIxwdFR/7jTAPZP1wMVnSlpKnXHAN64HFOiP+Tm3HmKusEZ0+09A0w==} + '@tensorflow/tfjs-core@3.0.0-rc.1': + resolution: {integrity: sha512-h9a0TyWNJFqgmyfSklwI2q0SC93WFr8FBHwHorvsfm2mDK0ZrSs6bhWqa44tWRJjCrJNRq9J4GoTZQM292RTwg==} engines: {yarn: '>= 1.3.2'} - '@types/long@4.0.2': - resolution: {integrity: sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==} - '@types/offscreencanvas@2019.3.0': resolution: {integrity: sha512-esIJx9bQg+QYF0ra8GnvfianIY8qWB0GBx54PK5Eps6m+xTj86KLavHv6qDhzKcu5UUOgNfJ2pWaIIV7TRUd9Q==} - '@types/seedrandom@2.4.34': - resolution: {integrity: sha512-ytDiArvrn/3Xk6/vtylys5tlY6eo7Ane0hvcx++TKo6RxQXuVfW0AF/oeWqAj9dN29SyhtawuXstgmPlwNcv/A==} + '@types/seedrandom@2.4.27': + resolution: {integrity: sha512-YvMLqFak/7rt//lPBtEHv3M4sRNA+HGxrhFZ+DQs9K2IkYJbNwVIb8avtJfhDiuaUBX/AW0jnjv48FV8h3u9bQ==} '@types/webgl-ext@0.0.30': resolution: {integrity: sha512-LKVgNmBxN0BbljJrVUwkxwRYqzsAEPcZOe6S2T6ZaBDIrFp0qu4FNlpc5sM1tGbXUYFgdVQIoeLk1Y1UoblyEg==} - '@webgpu/types@0.1.16': - resolution: {integrity: sha512-9E61voMP4+Rze02jlTXud++Htpjyyk8vw5Hyw9FGRrmhHQg2GqbuOfwf5Klrb8vTxc2XWI3EfO7RUHMpxTj26A==} - - long@4.0.0: - resolution: {integrity: sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==} + '@types/webgl2@0.0.5': + resolution: {integrity: sha512-oGaKsBbxQOY5+aJFV3KECDhGaXt+yZJt2y/OZsnQGLRkH6Fvr7rv4pCt3SRH1somIHfej/c4u7NSpCyd9x+1Ow==} node-fetch@2.6.13: resolution: {integrity: sha512-StxNAxh15zr77QvvkmveSQ8uCQ4+v5FkvNTj0OESmiHu+VRi/gXArXtkWMElOsOUNLtUEvI4yS+rdtOHZTwlQA==} @@ -56,8 +68,8 @@ packages: encoding: optional: true - seedrandom@3.0.5: - resolution: {integrity: sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==} + seedrandom@2.4.3: + resolution: {integrity: sha512-2CkZ9Wn2dS4mMUWQaXLsOAfGD+irMlLEeSP3cMxpGbgyOOzJGFa+MWCOMTOCMyZinHRPxyOj/S/C57li/1to6Q==} tr46@0.0.3: resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==} @@ -70,45 +82,54 @@ packages: snapshots: - '@tensorflow-models/posenet@2.2.1(@tensorflow/tfjs-converter@3.21.0(@tensorflow/tfjs-core@3.21.0))(@tensorflow/tfjs-core@3.21.0)': + '@tensorflow-models/posenet@2.2.2(@tensorflow/tfjs-converter@3.21.0(@tensorflow/tfjs-core@3.0.0-rc.1))(@tensorflow/tfjs-core@3.0.0-rc.1)': dependencies: - '@tensorflow/tfjs-converter': 3.21.0(@tensorflow/tfjs-core@3.21.0) - '@tensorflow/tfjs-core': 3.21.0 + '@tensorflow/tfjs-converter': 3.21.0(@tensorflow/tfjs-core@3.0.0-rc.1) + '@tensorflow/tfjs-core': 3.0.0-rc.1 - '@tensorflow/tfjs-converter@3.21.0(@tensorflow/tfjs-core@3.21.0)': + '@tensorflow/tfjs-backend-cpu@3.0.0-rc.1(@tensorflow/tfjs-core@3.0.0-rc.1)': dependencies: - '@tensorflow/tfjs-core': 3.21.0 + '@tensorflow/tfjs-core': 3.0.0-rc.1 + '@types/seedrandom': 2.4.27 + seedrandom: 2.4.3 - '@tensorflow/tfjs-core@3.21.0': + '@tensorflow/tfjs-backend-webgl@3.0.0-rc.1(@tensorflow/tfjs-core@3.0.0-rc.1)': dependencies: - '@types/long': 4.0.2 + '@tensorflow/tfjs-backend-cpu': 3.0.0-rc.1(@tensorflow/tfjs-core@3.0.0-rc.1) + '@tensorflow/tfjs-core': 3.0.0-rc.1 '@types/offscreencanvas': 2019.3.0 - '@types/seedrandom': 2.4.34 + '@types/seedrandom': 2.4.27 + '@types/webgl-ext': 0.0.30 + '@types/webgl2': 0.0.5 + seedrandom: 2.4.3 + + '@tensorflow/tfjs-converter@3.21.0(@tensorflow/tfjs-core@3.0.0-rc.1)': + dependencies: + '@tensorflow/tfjs-core': 3.0.0-rc.1 + + '@tensorflow/tfjs-core@3.0.0-rc.1': + dependencies: + '@types/offscreencanvas': 2019.3.0 + '@types/seedrandom': 2.4.27 '@types/webgl-ext': 0.0.30 - '@webgpu/types': 0.1.16 - long: 4.0.0 node-fetch: 2.6.13 - seedrandom: 3.0.5 + seedrandom: 2.4.3 transitivePeerDependencies: - encoding - '@types/long@4.0.2': {} - '@types/offscreencanvas@2019.3.0': {} - '@types/seedrandom@2.4.34': {} + '@types/seedrandom@2.4.27': {} '@types/webgl-ext@0.0.30': {} - '@webgpu/types@0.1.16': {} - - long@4.0.0: {} + '@types/webgl2@0.0.5': {} node-fetch@2.6.13: dependencies: whatwg-url: 5.0.0 - seedrandom@3.0.5: {} + seedrandom@2.4.3: {} tr46@0.0.3: {} From aa0d3d819d7059927fece1dc623385bb3212eaa1 Mon Sep 17 00:00:00 2001 From: Maya Rajan Date: Thu, 14 Aug 2025 18:16:40 -0400 Subject: [PATCH 05/35] cpu instead of webgl --- extensions/src/poseBody/index.ts | 2 +- extensions/src/poseBody/package.json | 2 +- extensions/src/poseBody/pnpm-lock.yaml | 23 +---------------------- 3 files changed, 3 insertions(+), 24 deletions(-) diff --git a/extensions/src/poseBody/index.ts b/extensions/src/poseBody/index.ts index 1082dbcd1..bd794c2a0 100644 --- a/extensions/src/poseBody/index.ts +++ b/extensions/src/poseBody/index.ts @@ -1,5 +1,5 @@ import { ArgumentType, BlockType, Extension, Block, DefineBlock, Environment, ExtensionMenuDisplayDetails, RuntimeEvent, ValueOf } from "$common"; -import "@tensorflow/tfjs-backend-webgl"; +import "@tensorflow/tfjs-backend-cpu"; import * as posenet from '@tensorflow-models/posenet'; import { legacyFullSupport, info } from "./legacy"; diff --git a/extensions/src/poseBody/package.json b/extensions/src/poseBody/package.json index 9245e4b33..81155bc9c 100644 --- a/extensions/src/poseBody/package.json +++ b/extensions/src/poseBody/package.json @@ -12,7 +12,7 @@ "license": "ISC", "dependencies": { "@tensorflow-models/posenet": "2.2.2", - "@tensorflow/tfjs-backend-webgl": "3.0.0-rc.1", + "@tensorflow/tfjs-backend-cpu": "3.0.0-rc.1", "@tensorflow/tfjs-core": "3.0.0-rc.1" } } \ No newline at end of file diff --git a/extensions/src/poseBody/pnpm-lock.yaml b/extensions/src/poseBody/pnpm-lock.yaml index 940de8abd..9e5f3ec11 100644 --- a/extensions/src/poseBody/pnpm-lock.yaml +++ b/extensions/src/poseBody/pnpm-lock.yaml @@ -11,7 +11,7 @@ importers: '@tensorflow-models/posenet': specifier: 2.2.2 version: 2.2.2(@tensorflow/tfjs-converter@3.21.0(@tensorflow/tfjs-core@3.0.0-rc.1))(@tensorflow/tfjs-core@3.0.0-rc.1) - '@tensorflow/tfjs-backend-webgl': + '@tensorflow/tfjs-backend-cpu': specifier: 3.0.0-rc.1 version: 3.0.0-rc.1(@tensorflow/tfjs-core@3.0.0-rc.1) '@tensorflow/tfjs-core': @@ -32,12 +32,6 @@ packages: peerDependencies: '@tensorflow/tfjs-core': 3.0.0-rc.1 - '@tensorflow/tfjs-backend-webgl@3.0.0-rc.1': - resolution: {integrity: sha512-qEjtL/71HflBHJgWXhtj+8BVYB9Q+NHw282QGWi9QLA9SlWe3BRJhbGCaD28/Ddl+mSzBWFQnK+sKdZ04LL1Sg==} - engines: {yarn: '>= 1.3.2'} - peerDependencies: - '@tensorflow/tfjs-core': 3.0.0-rc.1 - '@tensorflow/tfjs-converter@3.21.0': resolution: {integrity: sha512-12Y4zVDq3yW+wSjSDpSv4HnpL2sDZrNiGSg8XNiDE4HQBdjdA+a+Q3sZF/8NV9y2yoBhL5L7V4mMLDdbZBd9/Q==} peerDependencies: @@ -56,9 +50,6 @@ packages: '@types/webgl-ext@0.0.30': resolution: {integrity: sha512-LKVgNmBxN0BbljJrVUwkxwRYqzsAEPcZOe6S2T6ZaBDIrFp0qu4FNlpc5sM1tGbXUYFgdVQIoeLk1Y1UoblyEg==} - '@types/webgl2@0.0.5': - resolution: {integrity: sha512-oGaKsBbxQOY5+aJFV3KECDhGaXt+yZJt2y/OZsnQGLRkH6Fvr7rv4pCt3SRH1somIHfej/c4u7NSpCyd9x+1Ow==} - node-fetch@2.6.13: resolution: {integrity: sha512-StxNAxh15zr77QvvkmveSQ8uCQ4+v5FkvNTj0OESmiHu+VRi/gXArXtkWMElOsOUNLtUEvI4yS+rdtOHZTwlQA==} engines: {node: 4.x || >=6.0.0} @@ -93,16 +84,6 @@ snapshots: '@types/seedrandom': 2.4.27 seedrandom: 2.4.3 - '@tensorflow/tfjs-backend-webgl@3.0.0-rc.1(@tensorflow/tfjs-core@3.0.0-rc.1)': - dependencies: - '@tensorflow/tfjs-backend-cpu': 3.0.0-rc.1(@tensorflow/tfjs-core@3.0.0-rc.1) - '@tensorflow/tfjs-core': 3.0.0-rc.1 - '@types/offscreencanvas': 2019.3.0 - '@types/seedrandom': 2.4.27 - '@types/webgl-ext': 0.0.30 - '@types/webgl2': 0.0.5 - seedrandom: 2.4.3 - '@tensorflow/tfjs-converter@3.21.0(@tensorflow/tfjs-core@3.0.0-rc.1)': dependencies: '@tensorflow/tfjs-core': 3.0.0-rc.1 @@ -123,8 +104,6 @@ snapshots: '@types/webgl-ext@0.0.30': {} - '@types/webgl2@0.0.5': {} - node-fetch@2.6.13: dependencies: whatwg-url: 5.0.0 From f29c9467eb087c5155f479ecd6042ba2f0f39056 Mon Sep 17 00:00:00 2001 From: Maya Rajan Date: Thu, 14 Aug 2025 18:23:34 -0400 Subject: [PATCH 06/35] remove async --- extensions/src/poseBody/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/src/poseBody/index.ts b/extensions/src/poseBody/index.ts index bd794c2a0..348296488 100644 --- a/extensions/src/poseBody/index.ts +++ b/extensions/src/poseBody/index.ts @@ -82,7 +82,7 @@ export default class PoseBody extends Extension { * Acts like class PoseBody's constructor (instead of a child class constructor) * @param env */ - async init(env: Environment) { + init(env: Environment) { if (this.runtime.ioDevices) { this._loop(); From a29e843efa9c67ffde0d31654f2815eb84a10e5b Mon Sep 17 00:00:00 2001 From: Maya Rajan Date: Thu, 14 Aug 2025 18:36:39 -0400 Subject: [PATCH 07/35] back to webgl --- extensions/src/poseBody/index.ts | 2 +- extensions/src/poseBody/package.json | 2 +- extensions/src/poseBody/pnpm-lock.yaml | 23 ++++++++++++++++++++++- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/extensions/src/poseBody/index.ts b/extensions/src/poseBody/index.ts index 348296488..21c7f9b41 100644 --- a/extensions/src/poseBody/index.ts +++ b/extensions/src/poseBody/index.ts @@ -1,5 +1,5 @@ import { ArgumentType, BlockType, Extension, Block, DefineBlock, Environment, ExtensionMenuDisplayDetails, RuntimeEvent, ValueOf } from "$common"; -import "@tensorflow/tfjs-backend-cpu"; +import "@tensorflow/tfjs-backend-webgl"; import * as posenet from '@tensorflow-models/posenet'; import { legacyFullSupport, info } from "./legacy"; diff --git a/extensions/src/poseBody/package.json b/extensions/src/poseBody/package.json index 81155bc9c..9245e4b33 100644 --- a/extensions/src/poseBody/package.json +++ b/extensions/src/poseBody/package.json @@ -12,7 +12,7 @@ "license": "ISC", "dependencies": { "@tensorflow-models/posenet": "2.2.2", - "@tensorflow/tfjs-backend-cpu": "3.0.0-rc.1", + "@tensorflow/tfjs-backend-webgl": "3.0.0-rc.1", "@tensorflow/tfjs-core": "3.0.0-rc.1" } } \ No newline at end of file diff --git a/extensions/src/poseBody/pnpm-lock.yaml b/extensions/src/poseBody/pnpm-lock.yaml index 9e5f3ec11..940de8abd 100644 --- a/extensions/src/poseBody/pnpm-lock.yaml +++ b/extensions/src/poseBody/pnpm-lock.yaml @@ -11,7 +11,7 @@ importers: '@tensorflow-models/posenet': specifier: 2.2.2 version: 2.2.2(@tensorflow/tfjs-converter@3.21.0(@tensorflow/tfjs-core@3.0.0-rc.1))(@tensorflow/tfjs-core@3.0.0-rc.1) - '@tensorflow/tfjs-backend-cpu': + '@tensorflow/tfjs-backend-webgl': specifier: 3.0.0-rc.1 version: 3.0.0-rc.1(@tensorflow/tfjs-core@3.0.0-rc.1) '@tensorflow/tfjs-core': @@ -32,6 +32,12 @@ packages: peerDependencies: '@tensorflow/tfjs-core': 3.0.0-rc.1 + '@tensorflow/tfjs-backend-webgl@3.0.0-rc.1': + resolution: {integrity: sha512-qEjtL/71HflBHJgWXhtj+8BVYB9Q+NHw282QGWi9QLA9SlWe3BRJhbGCaD28/Ddl+mSzBWFQnK+sKdZ04LL1Sg==} + engines: {yarn: '>= 1.3.2'} + peerDependencies: + '@tensorflow/tfjs-core': 3.0.0-rc.1 + '@tensorflow/tfjs-converter@3.21.0': resolution: {integrity: sha512-12Y4zVDq3yW+wSjSDpSv4HnpL2sDZrNiGSg8XNiDE4HQBdjdA+a+Q3sZF/8NV9y2yoBhL5L7V4mMLDdbZBd9/Q==} peerDependencies: @@ -50,6 +56,9 @@ packages: '@types/webgl-ext@0.0.30': resolution: {integrity: sha512-LKVgNmBxN0BbljJrVUwkxwRYqzsAEPcZOe6S2T6ZaBDIrFp0qu4FNlpc5sM1tGbXUYFgdVQIoeLk1Y1UoblyEg==} + '@types/webgl2@0.0.5': + resolution: {integrity: sha512-oGaKsBbxQOY5+aJFV3KECDhGaXt+yZJt2y/OZsnQGLRkH6Fvr7rv4pCt3SRH1somIHfej/c4u7NSpCyd9x+1Ow==} + node-fetch@2.6.13: resolution: {integrity: sha512-StxNAxh15zr77QvvkmveSQ8uCQ4+v5FkvNTj0OESmiHu+VRi/gXArXtkWMElOsOUNLtUEvI4yS+rdtOHZTwlQA==} engines: {node: 4.x || >=6.0.0} @@ -84,6 +93,16 @@ snapshots: '@types/seedrandom': 2.4.27 seedrandom: 2.4.3 + '@tensorflow/tfjs-backend-webgl@3.0.0-rc.1(@tensorflow/tfjs-core@3.0.0-rc.1)': + dependencies: + '@tensorflow/tfjs-backend-cpu': 3.0.0-rc.1(@tensorflow/tfjs-core@3.0.0-rc.1) + '@tensorflow/tfjs-core': 3.0.0-rc.1 + '@types/offscreencanvas': 2019.3.0 + '@types/seedrandom': 2.4.27 + '@types/webgl-ext': 0.0.30 + '@types/webgl2': 0.0.5 + seedrandom: 2.4.3 + '@tensorflow/tfjs-converter@3.21.0(@tensorflow/tfjs-core@3.0.0-rc.1)': dependencies: '@tensorflow/tfjs-core': 3.0.0-rc.1 @@ -104,6 +123,8 @@ snapshots: '@types/webgl-ext@0.0.30': {} + '@types/webgl2@0.0.5': {} + node-fetch@2.6.13: dependencies: whatwg-url: 5.0.0 From 9ec434ab5cd18caba6a178c481dc0d3d7da14183 Mon Sep 17 00:00:00 2001 From: Maya Rajan Date: Wed, 5 Nov 2025 18:21:29 -0600 Subject: [PATCH 08/35] face and body landmarks --- extensions/src/poseBody/index.ts | 28 ++++++++++++++++++++++++++++ extensions/src/poseBody/legacy.ts | 31 +++++++++++++++++++++++++++++++ extensions/src/poseFace/index.ts | 20 ++++++++++++++++++++ extensions/src/poseFace/legacy.ts | 31 +++++++++++++++++++++++++++++++ 4 files changed, 110 insertions(+) diff --git a/extensions/src/poseBody/index.ts b/extensions/src/poseBody/index.ts index 21c7f9b41..41ad83265 100644 --- a/extensions/src/poseBody/index.ts +++ b/extensions/src/poseBody/index.ts @@ -37,6 +37,7 @@ type Details = { * Contains descriptions of the blocks of the Block Sensing extension */ type Blocks = { + returnPart(coord: string, bodyPart: string): number; goToPart(bodyPart: string): void; // these video blocks are present in a few different extensions, perhaps making a file just for these? videoToggle(state: string): void; @@ -219,6 +220,32 @@ export default class PoseBody extends Extension { } }); + const returnPart = legacyDefinition.returnPart({ + operation: (coord: string, bodyPart: string, util) => { + + if (this.hasPose()) { + const { x, y } = this.tfCoordsToScratch(this.poseState.keypoints.find(point => point.part === bodyPart).position); + if (coord === 'x') { + return x; + } else { + return y; + } + } + }, + argumentMethods: { + 0: { + handler(coord: string) { + return ['x', 'y'].includes(coord) ? coord : 'x'; + } + }, + 1: { + handler: (bodyPart: string) => { + return handlerOptions.includes(bodyPart) ? bodyPart : 'nose'; + } + } + } + }); + const videoToggle = legacyDefinition.videoToggle({ operation: (video_state) => { this.toggleVideo(video_state); @@ -240,6 +267,7 @@ export default class PoseBody extends Extension { return { goToPart, + returnPart, videoToggle, setVideoTransparency } diff --git a/extensions/src/poseBody/legacy.ts b/extensions/src/poseBody/legacy.ts index fc5c13460..5e911fff7 100644 --- a/extensions/src/poseBody/legacy.ts +++ b/extensions/src/poseBody/legacy.ts @@ -17,6 +17,24 @@ export const info = { } } }, + { + "opcode": "returnPart", + "text": "get [COORD] of [PART]", + "blockType": "reporter", + "isTerminal": false, + "arguments": { + "COORD": { + "type": "string", + "defaultValue": "x", + "menu": "COORD" + }, + "PART": { + "type": "string", + "defaultValue": "rightShoulder", + "menu": "PART" + } + } + }, { "opcode": "videoToggle", "text": "turn video [VIDEO_STATE]", @@ -115,6 +133,19 @@ export const info = { } ] }, + "COORD": { + "acceptReporters": false, + "items": [ + { + "text": "x", + "value": "x" + }, + { + "text": "y", + "value": "y" + } + ] + }, "ATTRIBUTE": { "acceptReporters": true, "items": [ diff --git a/extensions/src/poseFace/index.ts b/extensions/src/poseFace/index.ts index ab49c8839..609f521d5 100644 --- a/extensions/src/poseFace/index.ts +++ b/extensions/src/poseFace/index.ts @@ -45,6 +45,7 @@ type Details = { */ type Blocks = { affdexGoToPart(facePart: string): void; + affdexReturnPart(coord: string, facePart: string): number; affdexWhenExpression(expression: string): boolean; affdexExpressionAmount(expression: string): number; affdexIsExpression(expression: string): boolean; @@ -199,6 +200,18 @@ export default class PoseFace extends Extension { (util.target as any).setXY(x, y, false); } + returnPart(coord, part, util) { + if (!this.affdexState || !this.affdexState.featurePoints) return; + + const featurePoint = this.affdexState.featurePoints[part]; + const { x, y } = this.convertCoordsToScratch(featurePoint); + if (coord === 'x') { + return x; + } else { + return y; + } + } + /** * If an expression is being expressed * @param expression @@ -302,6 +315,12 @@ export default class PoseFace extends Extension { } }); + const affdexReturnPart = legacyDefinition.affdexReturnPart({ + operation: (coord: string, part: string, util: BlockUtility) => { + return this.returnPart(coord, part, util) + } + }); + const affdexWhenExpression = legacyDefinition.affdexWhenExpression({ operation: (expression: string) => { return this.isExpression(expression); @@ -403,6 +422,7 @@ export default class PoseFace extends Extension { return { affdexGoToPart, + affdexReturnPart, affdexWhenExpression, affdexExpressionAmount, affdexIsExpression, diff --git a/extensions/src/poseFace/legacy.ts b/extensions/src/poseFace/legacy.ts index 8c6b0f4ba..887cd6366 100644 --- a/extensions/src/poseFace/legacy.ts +++ b/extensions/src/poseFace/legacy.ts @@ -17,6 +17,24 @@ export const info = { } } }, + { + "opcode": "affdexReturnPart", + "text": "get [COORD] from [AFFDEX_POINT]", + "blockType": "reporter", + "isTerminal": false, + "arguments": { + "COORD": { + "type": "string", + "defaultValue": "x", + "menu": "COORD" + }, + "AFFDEX_POINT": { + "type": "string", + "defaultValue": "0", + "menu": "AFFDEX_POINT" + } + } + }, { "opcode": "affdexWhenExpression", "text": "when [EXPRESSION] detected", @@ -446,6 +464,19 @@ export const info = { "value": "on-flipped" } ] + }, + "COORD": { + "acceptReporters": false, + "items": [ + { + "text": "x", + "value": "x" + }, + { + "text": "y", + "value": "y" + }, + ] } } } as const; From 84f0b3fda8e27288b79205c0cbd3e62ed1dd2c97 Mon Sep 17 00:00:00 2001 From: Maya Rajan Date: Thu, 6 Nov 2025 00:27:04 -0500 Subject: [PATCH 09/35] hand add --- extensions/src/poseHand/index.ts | 36 +++++++++++++++++++++++++++++++ extensions/src/poseHand/legacy.ts | 36 +++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) diff --git a/extensions/src/poseHand/index.ts b/extensions/src/poseHand/index.ts index 5e566d404..7aca223d6 100644 --- a/extensions/src/poseHand/index.ts +++ b/extensions/src/poseHand/index.ts @@ -37,6 +37,7 @@ type Details = { */ type Blocks = { goToHandPart(handPart: string, fingerPart: number): void; + returnHandPart(coord: string, handPart: string, fingerPart: number): number; // these video blocks are present in a few different extensions, perhaps making a file just for these? videoToggle(state: string): void; setVideoTransparency(transparency: number): void; @@ -219,6 +220,40 @@ export default class PoseHand extends Extension { } }); + const returnHandPart = legacyDefinition.returnHandPart({ + operation: (coord: string, handPart: string, fingerPart: number, util): number => { + if (this.isConnected()) { + console.log('connected 2'); + const [x, y, z] = this.handPoseState[0].annotations[handPart][fingerPart]; + const { x: scratchX, y: scratchY } = this.tfCoordsToScratch({ x, y, z }); + if (coord === 'x') { + return scratchX; + } else { + return scratchY; + } + } else { + return 0; + } + }, + argumentMethods: { + 0: { + handler: (coord: string) => { + return ['x', 'y'].includes(coord) ? coord : "x"; + } + }, + 1: { + handler: (finger: string) => { + return handlerFingerOptions.includes(finger) ? finger : "thumb"; + } + }, + 2: { + handler: (part: number) => { + return Math.max(Math.min(part, 3), 0) + } + } + } + }); + const videoToggle = legacyDefinition.videoToggle({ operation: (video_state) => { this.toggleVideo(video_state); @@ -241,6 +276,7 @@ export default class PoseHand extends Extension { return { goToHandPart, + returnHandPart, videoToggle, setVideoTransparency } diff --git a/extensions/src/poseHand/legacy.ts b/extensions/src/poseHand/legacy.ts index c739e853a..bbeee4dc5 100644 --- a/extensions/src/poseHand/legacy.ts +++ b/extensions/src/poseHand/legacy.ts @@ -22,6 +22,29 @@ export const info = { } } }, + { + "opcode": "returnHandPart", + "text": "get [COORD] from [HAND_PART] [HAND_SUB_PART]", + "blockType": "reporter", + "isTerminal": false, + "arguments": { + "COORD": { + "type": "string", + "defaultValue": "x", + "menu": "COORD" + }, + "HAND_PART": { + "type": "string", + "defaultValue": "thumb", + "menu": "HAND_PART" + }, + "HAND_SUB_PART": { + "type": "number", + "defaultValue": 3, + "menu": "HAND_SUB_PART" + } + } + }, { "opcode": "videoToggle", "text": "turn video [VIDEO_STATE]", @@ -119,6 +142,19 @@ export const info = { } ] }, + "COORD": { + "acceptReporters": false, + "items": [ + { + "text": "x", + "value": "x" + }, + { + "text": "y", + "value": "y" + } + ] + }, "VIDEO_STATE": { "acceptReporters": true, "items": [ From f5daafb2736c54da78d1f0b4689974790d4756d0 Mon Sep 17 00:00:00 2001 From: Maya Rajan Date: Thu, 6 Nov 2025 00:41:45 -0500 Subject: [PATCH 10/35] small issues --- extensions/src/poseBody/index.ts | 2 +- extensions/src/poseHand/index.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/extensions/src/poseBody/index.ts b/extensions/src/poseBody/index.ts index 41ad83265..23b6c6d3d 100644 --- a/extensions/src/poseBody/index.ts +++ b/extensions/src/poseBody/index.ts @@ -221,7 +221,7 @@ export default class PoseBody extends Extension { }); const returnPart = legacyDefinition.returnPart({ - operation: (coord: string, bodyPart: string, util) => { + operation: (coord: string, bodyPart: string) => { if (this.hasPose()) { const { x, y } = this.tfCoordsToScratch(this.poseState.keypoints.find(point => point.part === bodyPart).position); diff --git a/extensions/src/poseHand/index.ts b/extensions/src/poseHand/index.ts index 7aca223d6..defd95f28 100644 --- a/extensions/src/poseHand/index.ts +++ b/extensions/src/poseHand/index.ts @@ -221,7 +221,7 @@ export default class PoseHand extends Extension { }); const returnHandPart = legacyDefinition.returnHandPart({ - operation: (coord: string, handPart: string, fingerPart: number, util): number => { + operation: (coord: string, handPart: string, fingerPart: number) => { if (this.isConnected()) { console.log('connected 2'); const [x, y, z] = this.handPoseState[0].annotations[handPart][fingerPart]; From aae24376275a143be1ae1d396206145b76adcf06 Mon Sep 17 00:00:00 2001 From: Maya Rajan Date: Thu, 6 Nov 2025 12:19:18 -0500 Subject: [PATCH 11/35] switch hand to mediapipe --- extensions/src/poseHand/index.ts | 95 ++++++++++++++++++-------- extensions/src/poseHand/package.json | 2 +- extensions/src/poseHand/pnpm-lock.yaml | 71 ++----------------- 3 files changed, 73 insertions(+), 95 deletions(-) diff --git a/extensions/src/poseHand/index.ts b/extensions/src/poseHand/index.ts index 5e566d404..2480dfb54 100644 --- a/extensions/src/poseHand/index.ts +++ b/extensions/src/poseHand/index.ts @@ -1,7 +1,6 @@ import { ArgumentType, BlockType, Extension, Block, DefineBlock, Environment, ExtensionMenuDisplayDetails, RuntimeEvent } from "$common"; import { legacyFullSupport, info } from "./legacy"; - -import * as handpose from '@tensorflow-models/handpose'; +import { HandLandmarker, FilesetResolver } from '@mediapipe/tasks-vision'; const { legacyExtension, legacyDefinition } = legacyFullSupport.for(); // TODO: Add extension's health check (peripheral) @@ -82,12 +81,23 @@ export default class PoseHand extends Extension { * @param env */ init(env: Environment) { - + this.loadMediaPipeModel(); if (this.runtime.ioDevices) { this._loop(); } } + /** + * Converts the coordinates from the MediaPipe hand estimate to Scratch coordinates + * @param x + * @param y + * @param z + * @returns enum + */ + mediapipeCoordsToScratch(x, y, z) { + return this.tfCoordsToScratch({ x: this.DIMENSIONS[0] * x, y: this.DIMENSIONS[1] * y, z }); + } + /** * Converts the coordinates from the hand pose estimate to Scratch coordinates * @param x @@ -113,8 +123,7 @@ export default class PoseHand extends Extension { * @returns {boolean} true if connected, false if not connected */ isConnected() { - console.log('connected'); - return !!this.handPoseState && this.handPoseState.length > 0; + return !!this.handPoseState && this.handPoseState.landmarks.length > 0; } /** @@ -125,38 +134,34 @@ export default class PoseHand extends Extension { async _loop() { while (true) { const frame = this.runtime.ioDevices.video.getFrame({ - format: 'image-data', + format: 'canvas', dimensions: this.DIMENSIONS }); const time = +new Date(); - if (frame) { - this.handPoseState = await this.estimateHandPoseOnImage(frame); + if (this.handModel && frame) { + this.handPoseState = this.handModel.detect(frame); } const estimateThrottleTimeout = (+new Date() - time) / 4; await new Promise(r => setTimeout(r, estimateThrottleTimeout)); } } - /** - * Estimates where the hand is on the video frame. - * @param imageElement - * @returns {Promise} - */ - async estimateHandPoseOnImage(imageElement) { - const handModel = await this.getLoadedHandModel(); - return await handModel.estimateHands(imageElement, { - flipHorizontal: false - }); - } - /** - * Gets the hand model from handpose - * @returns hand model - */ - async getLoadedHandModel() { - this.handModel ??= await handpose.load(); - return this.handModel; + + async loadMediaPipeModel() { + const vision = await FilesetResolver.forVisionTasks( + // path/to/wasm/root + "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@latest/wasm" + ); + this.handModel = await HandLandmarker.createFromOptions( + vision, + { + baseOptions: { + modelAssetPath: "https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/latest/hand_landmarker.task" + }, + numHands: 2 + }); } /** @@ -196,12 +201,44 @@ export default class PoseHand extends Extension { const handlerFingerOptions: Array = this.fingerOptions.map(finger => finger.value); + const handOptions = { + "thumb": { + 3: 4, + 1: 2, + 0: 1, + 2: 3 + }, + "indexFinger": { + 3: 8, + 1: 6, + 0: 5, + 2: 7 + }, + "middleFinger": { + 3: 12, + 1: 10, + 0: 9, + 2: 11 + }, + "ringFinger": { + 3: 16, + 1: 14, + 0: 13, + 2: 15 + }, + "pinky": { + 3: 20, + 1: 18, + 0: 17, + 2: 19 + }, + } + const goToHandPart = legacyDefinition.goToHandPart({ operation: (handPart: string, fingerPart: number, util) => { if (this.isConnected()) { - console.log('connected 2'); - const [x, y, z] = this.handPoseState[0].annotations[handPart][fingerPart]; - const { x: scratchX, y: scratchY } = this.tfCoordsToScratch({ x, y, z }); + const { x, y, z } = this.handPoseState.landmarks[0][handOptions[handPart][fingerPart]]; + const { x: scratchX, y: scratchY } = this.mediapipeCoordsToScratch(x, y, z); (util.target as any).setXY(scratchX, scratchY, false); } }, diff --git a/extensions/src/poseHand/package.json b/extensions/src/poseHand/package.json index 32b84239f..3b07806ea 100644 --- a/extensions/src/poseHand/package.json +++ b/extensions/src/poseHand/package.json @@ -11,6 +11,6 @@ "author": "", "license": "ISC", "dependencies": { - "@tensorflow-models/handpose": "^0.0.3" + "@mediapipe/tasks-vision": "^0.1.0-alpha-12" } } \ No newline at end of file diff --git a/extensions/src/poseHand/pnpm-lock.yaml b/extensions/src/poseHand/pnpm-lock.yaml index d1084d72f..5b191fb2e 100644 --- a/extensions/src/poseHand/pnpm-lock.yaml +++ b/extensions/src/poseHand/pnpm-lock.yaml @@ -8,74 +8,15 @@ importers: .: dependencies: - '@tensorflow-models/handpose': - specifier: ^0.0.3 - version: 0.0.3(@tensorflow/tfjs-converter@1.7.4(@tensorflow/tfjs-core@1.7.4))(@tensorflow/tfjs-core@1.7.4) + '@mediapipe/tasks-vision': + specifier: ^0.1.0-alpha-12 + version: 0.1.0-alpha-9 packages: - '@tensorflow-models/handpose@0.0.3': - resolution: {integrity: sha512-U5SBwxeQUXVawACDn+e0r4XJEDEah/J1HlWAqApXcm8DXjCtGKxQm/8BmFsg6ebbtAQ/R1bripohaQ655fv29w==} - peerDependencies: - '@tensorflow/tfjs-converter': ^1.6.1 - '@tensorflow/tfjs-core': ^1.6.1 - - '@tensorflow/tfjs-converter@1.7.4': - resolution: {integrity: sha512-B/Ux9I3osI0CXoESGR0Xe5C6BsEfC04+g2xn5zVaW9KEuVEnGEgnuBQxgijRFzkqTwoyLv4ptAmjyIghVARX0Q==} - peerDependencies: - '@tensorflow/tfjs-core': 1.7.4 - - '@tensorflow/tfjs-core@1.7.4': - resolution: {integrity: sha512-3G4VKJ6nPs7iCt6gs3bjRj8chihKrYWenf63R0pm7D9MhlrVoX/tpN4LYVMGgBL7jHPxMLKdOkoAZJrn/J88HQ==} - engines: {yarn: '>= 1.3.2'} - - '@types/offscreencanvas@2019.3.0': - resolution: {integrity: sha512-esIJx9bQg+QYF0ra8GnvfianIY8qWB0GBx54PK5Eps6m+xTj86KLavHv6qDhzKcu5UUOgNfJ2pWaIIV7TRUd9Q==} - - '@types/seedrandom@2.4.27': - resolution: {integrity: sha512-YvMLqFak/7rt//lPBtEHv3M4sRNA+HGxrhFZ+DQs9K2IkYJbNwVIb8avtJfhDiuaUBX/AW0jnjv48FV8h3u9bQ==} - - '@types/webgl-ext@0.0.30': - resolution: {integrity: sha512-LKVgNmBxN0BbljJrVUwkxwRYqzsAEPcZOe6S2T6ZaBDIrFp0qu4FNlpc5sM1tGbXUYFgdVQIoeLk1Y1UoblyEg==} - - '@types/webgl2@0.0.4': - resolution: {integrity: sha512-PACt1xdErJbMUOUweSrbVM7gSIYm1vTncW2hF6Os/EeWi6TXYAYMPp+8v6rzHmypE5gHrxaxZNXgMkJVIdZpHw==} - - node-fetch@2.1.2: - resolution: {integrity: sha512-IHLHYskTc2arMYsHZH82PVX8CSKT5lzb7AXeyO06QnjGDKtkv+pv3mEki6S7reB/x1QPo+YPxQRNEVgR5V/w3Q==} - engines: {node: 4.x || >=6.0.0} - - seedrandom@2.4.3: - resolution: {integrity: sha512-2CkZ9Wn2dS4mMUWQaXLsOAfGD+irMlLEeSP3cMxpGbgyOOzJGFa+MWCOMTOCMyZinHRPxyOj/S/C57li/1to6Q==} + '@mediapipe/tasks-vision@0.1.0-alpha-9': + resolution: {integrity: sha512-hpgY13d3zwbKEyEEG03m1rpEYM56CGPcMGgHwveZ4+SfDjmMLaev14bvlVpog51UNDE7abJTRL6Q4OtIJsSxXQ==} snapshots: - '@tensorflow-models/handpose@0.0.3(@tensorflow/tfjs-converter@1.7.4(@tensorflow/tfjs-core@1.7.4))(@tensorflow/tfjs-core@1.7.4)': - dependencies: - '@tensorflow/tfjs-converter': 1.7.4(@tensorflow/tfjs-core@1.7.4) - '@tensorflow/tfjs-core': 1.7.4 - - '@tensorflow/tfjs-converter@1.7.4(@tensorflow/tfjs-core@1.7.4)': - dependencies: - '@tensorflow/tfjs-core': 1.7.4 - - '@tensorflow/tfjs-core@1.7.4': - dependencies: - '@types/offscreencanvas': 2019.3.0 - '@types/seedrandom': 2.4.27 - '@types/webgl-ext': 0.0.30 - '@types/webgl2': 0.0.4 - node-fetch: 2.1.2 - seedrandom: 2.4.3 - - '@types/offscreencanvas@2019.3.0': {} - - '@types/seedrandom@2.4.27': {} - - '@types/webgl-ext@0.0.30': {} - - '@types/webgl2@0.0.4': {} - - node-fetch@2.1.2: {} - - seedrandom@2.4.3: {} + '@mediapipe/tasks-vision@0.1.0-alpha-9': {} From 55b3da747a4582ccbb21b26978a67e4c34116cd1 Mon Sep 17 00:00:00 2001 From: Maya Rajan Date: Thu, 6 Nov 2025 13:51:40 -0500 Subject: [PATCH 12/35] edit return --- extensions/src/poseHand/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extensions/src/poseHand/index.ts b/extensions/src/poseHand/index.ts index 4f06745b6..0d92088f0 100644 --- a/extensions/src/poseHand/index.ts +++ b/extensions/src/poseHand/index.ts @@ -261,8 +261,8 @@ export default class PoseHand extends Extension { operation: (coord: string, handPart: string, fingerPart: number) => { if (this.isConnected()) { console.log('connected 2'); - const [x, y, z] = this.handPoseState[0].annotations[handPart][fingerPart]; - const { x: scratchX, y: scratchY } = this.tfCoordsToScratch({ x, y, z }); + const { x, y, z } = this.handPoseState.landmarks[0][handOptions[handPart][fingerPart]]; + const { x: scratchX, y: scratchY } = this.mediapipeCoordsToScratch(x, y, z); if (coord === 'x') { return scratchX; } else { From 7451e1d290ed19122cda8812bda7ae5c2bf32698 Mon Sep 17 00:00:00 2001 From: Maya Rajan Date: Thu, 6 Nov 2025 14:15:39 -0500 Subject: [PATCH 13/35] update --- extensions/src/poseHand/package.json | 2 +- extensions/src/poseHand/pnpm-lock.yaml | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/extensions/src/poseHand/package.json b/extensions/src/poseHand/package.json index 3b07806ea..ff504c1fb 100644 --- a/extensions/src/poseHand/package.json +++ b/extensions/src/poseHand/package.json @@ -11,6 +11,6 @@ "author": "", "license": "ISC", "dependencies": { - "@mediapipe/tasks-vision": "^0.1.0-alpha-12" + "@mediapipe/tasks-vision": "^0.10.0" } } \ No newline at end of file diff --git a/extensions/src/poseHand/pnpm-lock.yaml b/extensions/src/poseHand/pnpm-lock.yaml index 5b191fb2e..08cc65caa 100644 --- a/extensions/src/poseHand/pnpm-lock.yaml +++ b/extensions/src/poseHand/pnpm-lock.yaml @@ -9,14 +9,14 @@ importers: .: dependencies: '@mediapipe/tasks-vision': - specifier: ^0.1.0-alpha-12 - version: 0.1.0-alpha-9 + specifier: ^0.10.0 + version: 0.10.21 packages: - '@mediapipe/tasks-vision@0.1.0-alpha-9': - resolution: {integrity: sha512-hpgY13d3zwbKEyEEG03m1rpEYM56CGPcMGgHwveZ4+SfDjmMLaev14bvlVpog51UNDE7abJTRL6Q4OtIJsSxXQ==} + '@mediapipe/tasks-vision@0.10.21': + resolution: {integrity: sha512-TuhKH+credq4zLksGbYrnvJ1aLIWMc5r0UHwzxzql4BHECJwIAoBR61ZrqwGOW6ZmSBIzU1t4VtKj8hbxFaKeA==} snapshots: - '@mediapipe/tasks-vision@0.1.0-alpha-9': {} + '@mediapipe/tasks-vision@0.10.21': {} From 876acd19e43cd0c2a76fe7df68603c86a077a9d9 Mon Sep 17 00:00:00 2001 From: Maya Rajan Date: Fri, 7 Nov 2025 16:58:04 -0500 Subject: [PATCH 14/35] improve accuracy --- extensions/src/poseHand/index.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/extensions/src/poseHand/index.ts b/extensions/src/poseHand/index.ts index 0d92088f0..f97f61224 100644 --- a/extensions/src/poseHand/index.ts +++ b/extensions/src/poseHand/index.ts @@ -159,8 +159,10 @@ export default class PoseHand extends Extension { vision, { baseOptions: { - modelAssetPath: "https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/latest/hand_landmarker.task" + modelAssetPath: "https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/latest/hand_landmarker.task", + delegate: "GPU", }, + numHands: 2 }); } From b375cf45d0a82c889f752239f12211e49349062f Mon Sep 17 00:00:00 2001 From: Maya Rajan Date: Fri, 7 Nov 2025 17:25:20 -0500 Subject: [PATCH 15/35] undo --- extensions/src/poseHand/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/src/poseHand/index.ts b/extensions/src/poseHand/index.ts index f97f61224..ba17e9638 100644 --- a/extensions/src/poseHand/index.ts +++ b/extensions/src/poseHand/index.ts @@ -160,7 +160,7 @@ export default class PoseHand extends Extension { { baseOptions: { modelAssetPath: "https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/latest/hand_landmarker.task", - delegate: "GPU", + //delegate: "GPU", }, numHands: 2 From aeb08ad94dc58bff5d294478867017fc492fe11b Mon Sep 17 00:00:00 2001 From: Maya Rajan Date: Fri, 7 Nov 2025 18:12:47 -0500 Subject: [PATCH 16/35] video attempt --- extensions/src/poseHand/index.ts | 55 ++++++++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/extensions/src/poseHand/index.ts b/extensions/src/poseHand/index.ts index 2480dfb54..ec0cca88d 100644 --- a/extensions/src/poseHand/index.ts +++ b/extensions/src/poseHand/index.ts @@ -83,7 +83,8 @@ export default class PoseHand extends Extension { init(env: Environment) { this.loadMediaPipeModel(); if (this.runtime.ioDevices) { - this._loop(); + // this._loop(); + // this.handModel } } @@ -95,7 +96,7 @@ export default class PoseHand extends Extension { * @returns enum */ mediapipeCoordsToScratch(x, y, z) { - return this.tfCoordsToScratch({ x: this.DIMENSIONS[0] * x, y: this.DIMENSIONS[1] * y, z }); + return this.tfCoordsToScratch({ x: (this.DIMENSIONS[0] - (this.DIMENSIONS[0] * x)), y: this.DIMENSIONS[1] * y, z }); } /** @@ -123,6 +124,7 @@ export default class PoseHand extends Extension { * @returns {boolean} true if connected, false if not connected */ isConnected() { + console.log(this.handPoseState); return !!this.handPoseState && this.handPoseState.landmarks.length > 0; } @@ -160,8 +162,15 @@ export default class PoseHand extends Extension { baseOptions: { modelAssetPath: "https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/latest/hand_landmarker.task" }, + runningMode: "VIDEO", numHands: 2 }); + console.log(this.runtime.ioDevices.video); + if (this.runtime.ioDevices.video.provider._video) { + this.handModel.detectForVideo(this.runtime.ioDevices.video.provider._video, Date.now()); + } + + console.log("CONNECTED"); } /** @@ -256,6 +265,48 @@ export default class PoseHand extends Extension { } }); + const returnHandPart = legacyDefinition.returnHandPart({ + operation: (coord: string, handPart: string, fingerPart: number) => { + console.log(this.runtime.ioDevices.video.provider._track); + console.log(this.runtime.ioDevices.video.provider); + let results; + if (this.runtime.ioDevices.video.provider._video) { + results = this.handModel.detectForVideo(this.runtime.ioDevices.video.provider._video, Date.now()); + console.log("results", results); + } + + if (results && results.landmarks.length > 0) { + console.log('connected 2'); + const { x, y, z } = results.landmarks[0][handOptions[handPart][fingerPart]]; + const { x: scratchX, y: scratchY } = this.mediapipeCoordsToScratch(x, y, z); + if (coord === 'x') { + return scratchX; + } else { + return scratchY; + } + } else { + return 0; + } + }, + argumentMethods: { + 0: { + handler: (coord: string) => { + return ['x', 'y'].includes(coord) ? coord : "x"; + } + }, + 1: { + handler: (finger: string) => { + return handlerFingerOptions.includes(finger) ? finger : "thumb"; + } + }, + 2: { + handler: (part: number) => { + return Math.max(Math.min(part, 3), 0) + } + } + } + }); + const videoToggle = legacyDefinition.videoToggle({ operation: (video_state) => { this.toggleVideo(video_state); From 06591446eb1721ae1dc9b5e05d541089730e7814 Mon Sep 17 00:00:00 2001 From: Maya Rajan Date: Fri, 7 Nov 2025 18:29:21 -0500 Subject: [PATCH 17/35] video updates --- extensions/src/poseHand/index.ts | 63 +++++++++++++++++--------------- 1 file changed, 33 insertions(+), 30 deletions(-) diff --git a/extensions/src/poseHand/index.ts b/extensions/src/poseHand/index.ts index 343affdca..9cef07fe3 100644 --- a/extensions/src/poseHand/index.ts +++ b/extensions/src/poseHand/index.ts @@ -83,10 +83,10 @@ export default class PoseHand extends Extension { */ init(env: Environment) { this.loadMediaPipeModel(); - if (this.runtime.ioDevices) { - // this._loop(); - // this.handModel - } + // if (this.runtime.ioDevices) { + // // this._loop(); + // // this.handModel + // } } /** @@ -124,31 +124,31 @@ export default class PoseHand extends Extension { * Checks if the hand pose estimate is ready to be used * @returns {boolean} true if connected, false if not connected */ - isConnected() { - console.log(this.handPoseState); - return !!this.handPoseState && this.handPoseState.landmarks.length > 0; - } + // isConnected() { + // console.log(this.handPoseState); + // return !!this.handPoseState && this.handPoseState.landmarks.length > 0; + // } /** * Runs for the entire time the extension is running. Gets information about the video frame. * Estimates where the hand is on the video frame. Creates a delay to prevent this function from constantly running, * so as to prevent the entire program from slowing down. */ - async _loop() { - while (true) { - const frame = this.runtime.ioDevices.video.getFrame({ - format: 'canvas', - dimensions: this.DIMENSIONS - }); + // async _loop() { + // while (true) { + // const frame = this.runtime.ioDevices.video.getFrame({ + // format: 'canvas', + // dimensions: this.DIMENSIONS + // }); - const time = +new Date(); - if (this.handModel && frame) { - this.handPoseState = this.handModel.detect(frame); - } - const estimateThrottleTimeout = (+new Date() - time) / 4; - await new Promise(r => setTimeout(r, estimateThrottleTimeout)); - } - } + // const time = +new Date(); + // if (this.handModel && frame) { + // this.handPoseState = this.handModel.detect(frame); + // } + // const estimateThrottleTimeout = (+new Date() - time) / 4; + // await new Promise(r => setTimeout(r, estimateThrottleTimeout)); + // } + // } @@ -162,7 +162,7 @@ export default class PoseHand extends Extension { { baseOptions: { modelAssetPath: "https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/latest/hand_landmarker.task", - //delegate: "GPU", + delegate: "GPU", }, runningMode: "VIDEO", numHands: 2 @@ -172,7 +172,6 @@ export default class PoseHand extends Extension { this.handModel.detectForVideo(this.runtime.ioDevices.video.provider._video, Date.now()); } - console.log("CONNECTED"); } /** @@ -246,9 +245,15 @@ export default class PoseHand extends Extension { } const goToHandPart = legacyDefinition.goToHandPart({ + operation: (handPart: string, fingerPart: number, util) => { - if (this.isConnected()) { - const { x, y, z } = this.handPoseState.landmarks[0][handOptions[handPart][fingerPart]]; + let results; + if (this.runtime.ioDevices && this.runtime.ioDevices.video.provider._video) { + results = this.handModel.detectForVideo(this.runtime.ioDevices.video.provider._video, Date.now()); + } + + if (results && results.landmarks.length > 0) { + const { x, y, z } = results.landmarks[0][handOptions[handPart][fingerPart]]; const { x: scratchX, y: scratchY } = this.mediapipeCoordsToScratch(x, y, z); (util.target as any).setXY(scratchX, scratchY, false); } @@ -269,12 +274,10 @@ export default class PoseHand extends Extension { const returnHandPart = legacyDefinition.returnHandPart({ operation: (coord: string, handPart: string, fingerPart: number) => { - console.log(this.runtime.ioDevices.video.provider._track); - console.log(this.runtime.ioDevices.video.provider); + let results; - if (this.runtime.ioDevices.video.provider._video) { + if (this.runtime.ioDevices && this.runtime.ioDevices.video.provider._video) { results = this.handModel.detectForVideo(this.runtime.ioDevices.video.provider._video, Date.now()); - console.log("results", results); } if (results && results.landmarks.length > 0) { From 3add86ae749f62a41e402c9222e547275e55b219 Mon Sep 17 00:00:00 2001 From: Maya Rajan Date: Mon, 10 Nov 2025 17:28:09 -0500 Subject: [PATCH 18/35] going back to images --- extensions/src/poseHand/index.ts | 73 ++++++++++++-------------------- 1 file changed, 28 insertions(+), 45 deletions(-) diff --git a/extensions/src/poseHand/index.ts b/extensions/src/poseHand/index.ts index 9cef07fe3..a14825a83 100644 --- a/extensions/src/poseHand/index.ts +++ b/extensions/src/poseHand/index.ts @@ -83,10 +83,9 @@ export default class PoseHand extends Extension { */ init(env: Environment) { this.loadMediaPipeModel(); - // if (this.runtime.ioDevices) { - // // this._loop(); - // // this.handModel - // } + if (this.runtime.ioDevices) { + this._loop(); + } } /** @@ -97,7 +96,7 @@ export default class PoseHand extends Extension { * @returns enum */ mediapipeCoordsToScratch(x, y, z) { - return this.tfCoordsToScratch({ x: (this.DIMENSIONS[0] - (this.DIMENSIONS[0] * x)), y: this.DIMENSIONS[1] * y, z }); + return this.tfCoordsToScratch({ x: (this.DIMENSIONS[0] * x), y: this.DIMENSIONS[1] * y, z }); } /** @@ -124,31 +123,31 @@ export default class PoseHand extends Extension { * Checks if the hand pose estimate is ready to be used * @returns {boolean} true if connected, false if not connected */ - // isConnected() { - // console.log(this.handPoseState); - // return !!this.handPoseState && this.handPoseState.landmarks.length > 0; - // } + isConnected() { + console.log(this.handPoseState); + return !!this.handPoseState && this.handPoseState.landmarks.length > 0; + } /** * Runs for the entire time the extension is running. Gets information about the video frame. * Estimates where the hand is on the video frame. Creates a delay to prevent this function from constantly running, * so as to prevent the entire program from slowing down. */ - // async _loop() { - // while (true) { - // const frame = this.runtime.ioDevices.video.getFrame({ - // format: 'canvas', - // dimensions: this.DIMENSIONS - // }); + async _loop() { + while (true) { + const frame = this.runtime.ioDevices.video.getFrame({ + format: 'canvas', + dimensions: this.DIMENSIONS + }); - // const time = +new Date(); - // if (this.handModel && frame) { - // this.handPoseState = this.handModel.detect(frame); - // } - // const estimateThrottleTimeout = (+new Date() - time) / 4; - // await new Promise(r => setTimeout(r, estimateThrottleTimeout)); - // } - // } + const time = +new Date(); + if (this.handModel && frame) { + this.handPoseState = this.handModel.detect(frame); + } + const estimateThrottleTimeout = (+new Date() - time) / 4; + await new Promise(r => setTimeout(r, estimateThrottleTimeout)); + } + } @@ -162,17 +161,11 @@ export default class PoseHand extends Extension { { baseOptions: { modelAssetPath: "https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/latest/hand_landmarker.task", - delegate: "GPU", + delegate: "GPU" }, - runningMode: "VIDEO", numHands: 2 }); - console.log(this.runtime.ioDevices.video); - if (this.runtime.ioDevices.video.provider._video) { - this.handModel.detectForVideo(this.runtime.ioDevices.video.provider._video, Date.now()); - } - - } + } /** * Turns the video camera off/on/on and flipped. This is called in the operation of videoToggleBlock @@ -247,13 +240,9 @@ export default class PoseHand extends Extension { const goToHandPart = legacyDefinition.goToHandPart({ operation: (handPart: string, fingerPart: number, util) => { - let results; - if (this.runtime.ioDevices && this.runtime.ioDevices.video.provider._video) { - results = this.handModel.detectForVideo(this.runtime.ioDevices.video.provider._video, Date.now()); - } - if (results && results.landmarks.length > 0) { - const { x, y, z } = results.landmarks[0][handOptions[handPart][fingerPart]]; + if (this.isConnected()) { + const { x, y, z } = this.handPoseState.landmarks[0][handOptions[handPart][fingerPart]]; const { x: scratchX, y: scratchY } = this.mediapipeCoordsToScratch(x, y, z); (util.target as any).setXY(scratchX, scratchY, false); } @@ -274,15 +263,9 @@ export default class PoseHand extends Extension { const returnHandPart = legacyDefinition.returnHandPart({ operation: (coord: string, handPart: string, fingerPart: number) => { - - let results; - if (this.runtime.ioDevices && this.runtime.ioDevices.video.provider._video) { - results = this.handModel.detectForVideo(this.runtime.ioDevices.video.provider._video, Date.now()); - } - - if (results && results.landmarks.length > 0) { + if (this.isConnected()) { console.log('connected 2'); - const { x, y, z } = results.landmarks[0][handOptions[handPart][fingerPart]]; + const { x, y, z } = this.handPoseState.landmarks[0][handOptions[handPart][fingerPart]]; const { x: scratchX, y: scratchY } = this.mediapipeCoordsToScratch(x, y, z); if (coord === 'x') { return scratchX; From 65f3c9316c0a4799aa9c3be23b9a68f9517eba94 Mon Sep 17 00:00:00 2001 From: Maya Rajan Date: Tue, 11 Nov 2025 16:26:51 -0500 Subject: [PATCH 19/35] debug statement --- extensions/src/poseHand/index.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/extensions/src/poseHand/index.ts b/extensions/src/poseHand/index.ts index 9cef07fe3..332d984be 100644 --- a/extensions/src/poseHand/index.ts +++ b/extensions/src/poseHand/index.ts @@ -248,10 +248,17 @@ export default class PoseHand extends Extension { operation: (handPart: string, fingerPart: number, util) => { let results; + const start = performance.now(); + if (this.runtime.ioDevices && this.runtime.ioDevices.video.provider._video) { - results = this.handModel.detectForVideo(this.runtime.ioDevices.video.provider._video, Date.now()); + results = this.handModel.detectForVideo( + this.runtime.ioDevices.video.provider._video, + Date.now() + ); } + const end = performance.now(); + console.log(`detectForVideo took ${(end - start).toFixed(2)} ms`); if (results && results.landmarks.length > 0) { const { x, y, z } = results.landmarks[0][handOptions[handPart][fingerPart]]; const { x: scratchX, y: scratchY } = this.mediapipeCoordsToScratch(x, y, z); From ec1b3b4bb736c56cf24ecaa9c0fdbdfdcf3e6cd4 Mon Sep 17 00:00:00 2001 From: Maya Rajan Date: Tue, 11 Nov 2025 16:29:08 -0500 Subject: [PATCH 20/35] debug statement --- extensions/src/poseHand/index.ts | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/extensions/src/poseHand/index.ts b/extensions/src/poseHand/index.ts index a14825a83..3434b8c9c 100644 --- a/extensions/src/poseHand/index.ts +++ b/extensions/src/poseHand/index.ts @@ -135,20 +135,35 @@ export default class PoseHand extends Extension { */ async _loop() { while (true) { + const loopStart = performance.now(); + const frame = this.runtime.ioDevices.video.getFrame({ format: 'canvas', dimensions: this.DIMENSIONS }); - - const time = +new Date(); + + const detectStart = performance.now(); + if (this.handModel && frame) { this.handPoseState = this.handModel.detect(frame); } - const estimateThrottleTimeout = (+new Date() - time) / 4; + + const detectEnd = performance.now(); + + const estimateThrottleTimeout = (detectEnd - detectStart) / 4; + + const loopEnd = performance.now(); + const detectTime = (detectEnd - detectStart).toFixed(2); + const totalLoopTime = (loopEnd - loopStart).toFixed(2); + + console.log( + `detect() took ${detectTime} ms | total loop iteration: ${totalLoopTime} ms` + ); + await new Promise(r => setTimeout(r, estimateThrottleTimeout)); } } - + async loadMediaPipeModel() { From eea38e2442b71bb19440c9bd5bbfafab1b84bc4b Mon Sep 17 00:00:00 2001 From: Maya Rajan Date: Wed, 12 Nov 2025 16:23:26 -0500 Subject: [PATCH 21/35] trigger build --- extensions/src/poseHand/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/src/poseHand/index.ts b/extensions/src/poseHand/index.ts index 3434b8c9c..22f848cf6 100644 --- a/extensions/src/poseHand/index.ts +++ b/extensions/src/poseHand/index.ts @@ -157,7 +157,7 @@ export default class PoseHand extends Extension { const totalLoopTime = (loopEnd - loopStart).toFixed(2); console.log( - `detect() took ${detectTime} ms | total loop iteration: ${totalLoopTime} ms` + `detect() took ${detectTime} ms | total loop iteration: ${totalLoopTime} ms ` ); await new Promise(r => setTimeout(r, estimateThrottleTimeout)); From 9a669bfc8a62b8d01ba585abac230785c949f12d Mon Sep 17 00:00:00 2001 From: Maya Rajan Date: Wed, 12 Nov 2025 16:47:15 -0500 Subject: [PATCH 22/35] unnecessary print --- extensions/src/poseHand/index.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/extensions/src/poseHand/index.ts b/extensions/src/poseHand/index.ts index 332d984be..d61394386 100644 --- a/extensions/src/poseHand/index.ts +++ b/extensions/src/poseHand/index.ts @@ -167,10 +167,6 @@ export default class PoseHand extends Extension { runningMode: "VIDEO", numHands: 2 }); - console.log(this.runtime.ioDevices.video); - if (this.runtime.ioDevices.video.provider._video) { - this.handModel.detectForVideo(this.runtime.ioDevices.video.provider._video, Date.now()); - } } From f7d3fa8212386a1e3326b22aa2936ef5b4b56f40 Mon Sep 17 00:00:00 2001 From: Maya Rajan Date: Wed, 12 Nov 2025 17:14:11 -0500 Subject: [PATCH 23/35] remove print statements --- extensions/src/poseHand/index.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/extensions/src/poseHand/index.ts b/extensions/src/poseHand/index.ts index 22f848cf6..825604c2d 100644 --- a/extensions/src/poseHand/index.ts +++ b/extensions/src/poseHand/index.ts @@ -124,7 +124,6 @@ export default class PoseHand extends Extension { * @returns {boolean} true if connected, false if not connected */ isConnected() { - console.log(this.handPoseState); return !!this.handPoseState && this.handPoseState.landmarks.length > 0; } @@ -279,7 +278,6 @@ export default class PoseHand extends Extension { const returnHandPart = legacyDefinition.returnHandPart({ operation: (coord: string, handPart: string, fingerPart: number) => { if (this.isConnected()) { - console.log('connected 2'); const { x, y, z } = this.handPoseState.landmarks[0][handOptions[handPart][fingerPart]]; const { x: scratchX, y: scratchY } = this.mediapipeCoordsToScratch(x, y, z); if (coord === 'x') { From b4d3d332549ac165b2f60ebb437c88c3ac25343c Mon Sep 17 00:00:00 2001 From: Maya Rajan Date: Thu, 13 Nov 2025 14:45:48 -0500 Subject: [PATCH 24/35] improve loop --- extensions/src/poseHand/index.ts | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/extensions/src/poseHand/index.ts b/extensions/src/poseHand/index.ts index 825604c2d..ad10d8e8e 100644 --- a/extensions/src/poseHand/index.ts +++ b/extensions/src/poseHand/index.ts @@ -133,34 +133,38 @@ export default class PoseHand extends Extension { * so as to prevent the entire program from slowing down. */ async _loop() { - while (true) { - const loopStart = performance.now(); const frame = this.runtime.ioDevices.video.getFrame({ format: 'canvas', dimensions: this.DIMENSIONS }); - const detectStart = performance.now(); + if (!this.handModel || !frame) { + requestAnimationFrame(this._loop.bind(this)); + return; + } + if (this.handModel && frame) { this.handPoseState = this.handModel.detect(frame); } + + requestAnimationFrame(this._loop.bind(this)); - const detectEnd = performance.now(); + // const detectEnd = performance.now(); - const estimateThrottleTimeout = (detectEnd - detectStart) / 4; + // const estimateThrottleTimeout = (detectEnd - detectStart) / 4; - const loopEnd = performance.now(); - const detectTime = (detectEnd - detectStart).toFixed(2); - const totalLoopTime = (loopEnd - loopStart).toFixed(2); + // const loopEnd = performance.now(); + // const detectTime = (detectEnd - detectStart).toFixed(2); + // const totalLoopTime = (loopEnd - loopStart).toFixed(2); - console.log( - `detect() took ${detectTime} ms | total loop iteration: ${totalLoopTime} ms ` - ); + // console.log( + // `detect() took ${detectTime} ms | total loop iteration: ${totalLoopTime} ms ` + // ); - await new Promise(r => setTimeout(r, estimateThrottleTimeout)); - } + // await new Promise(r => setTimeout(r, estimateThrottleTimeout)); + } From 4aa808021ec3cd50c0c16bab4c88ac8e24b10ac8 Mon Sep 17 00:00:00 2001 From: Maya Rajan Date: Thu, 13 Nov 2025 16:57:25 -0500 Subject: [PATCH 25/35] remove dead code --- extensions/src/poseHand/index.ts | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/extensions/src/poseHand/index.ts b/extensions/src/poseHand/index.ts index ad10d8e8e..08cb80106 100644 --- a/extensions/src/poseHand/index.ts +++ b/extensions/src/poseHand/index.ts @@ -151,20 +151,6 @@ export default class PoseHand extends Extension { requestAnimationFrame(this._loop.bind(this)); - // const detectEnd = performance.now(); - - // const estimateThrottleTimeout = (detectEnd - detectStart) / 4; - - // const loopEnd = performance.now(); - // const detectTime = (detectEnd - detectStart).toFixed(2); - // const totalLoopTime = (loopEnd - loopStart).toFixed(2); - - // console.log( - // `detect() took ${detectTime} ms | total loop iteration: ${totalLoopTime} ms ` - // ); - - // await new Promise(r => setTimeout(r, estimateThrottleTimeout)); - } From b680d2fd7cacaf264f2cd68c1d3ef58484f5a3d1 Mon Sep 17 00:00:00 2001 From: Maya Rajan Date: Tue, 25 Nov 2025 20:00:13 -0500 Subject: [PATCH 26/35] table fix --- extensions/src/tables/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/src/tables/index.ts b/extensions/src/tables/index.ts index 7e1ce764f..9edf83596 100644 --- a/extensions/src/tables/index.ts +++ b/extensions/src/tables/index.ts @@ -93,7 +93,7 @@ export default class Tables extends Extension { changeTableValue(info: { name: string, row: number, column: number, value: number }) { const { name, row, column, value } = info; - this.tables[name][row][column] = value; + this.tables[name][row][column] = value ? value : 0; } defineBlocks(): Tables["BlockDefinitions"] { From 8a348a7bdd9ff38d2df3c791ac46b08cb7e69820 Mon Sep 17 00:00:00 2001 From: Maya Rajan Date: Mon, 1 Dec 2025 17:41:43 -0500 Subject: [PATCH 27/35] column and row names --- extensions/src/tables/View.svelte | 30 ++++++++++++++++++++++++++--- extensions/src/tables/index.ts | 32 +++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/extensions/src/tables/View.svelte b/extensions/src/tables/View.svelte index e86435f87..555917978 100644 --- a/extensions/src/tables/View.svelte +++ b/extensions/src/tables/View.svelte @@ -71,15 +71,39 @@ - {#each [...Array(extension.tables[selected][0].length)] as _, i} - {i + 1} + {#each extension.columnNames[selected] as columnName, i} + + e.currentTarget.contentEditable = "true"} + on:blur={(e) => { + e.currentTarget.contentEditable = "false"; + invoke("renameColumn", { name: selected, column: i, value: e.currentTarget.innerText }); + }} + > + {columnName} + + {/each} {#each extension.tables[selected] as row, i} - {i + 1} + + e.currentTarget.contentEditable = "true"} + on:blur={(e) => { + e.currentTarget.contentEditable = "false"; + invoke("renameRow", { name: selected, row: i, value: e.currentTarget.innerText }); + }} + > + {extension.rowNames[selected][i]} + + {#each row as value, j} update(e, i, j)} data-testid="tableCell"> diff --git a/extensions/src/tables/index.ts b/extensions/src/tables/index.ts index 9edf83596..ae5b134e8 100644 --- a/extensions/src/tables/index.ts +++ b/extensions/src/tables/index.ts @@ -32,6 +32,8 @@ type Blocks = { @validGenericExtension() export default class Tables extends Extension { tables: Record; + columnNames: Record; + rowNames: Record; tableNamesArg: any; defaultNumberArg: any; @@ -46,6 +48,10 @@ export default class Tables extends Extension { if (!this.tables) { this.tables = {}; this.tables.myTable = []; + this.columnNames = {}; + this.rowNames = {}; + this.columnNames.myTable = ["Col 1"]; + this.rowNames.myTable = ["Row 1"]; this.tables.myTable.push([0]); } @@ -80,6 +86,24 @@ export default class Tables extends Extension { ); } + renameColumn(info: { name: string, column: number, value: string }) { + const { name, column, value } = info; + if (this.columnNames[name] && this.columnNames[name][column] !== undefined) { + this.columnNames[name][column] = value; + } else { + alert(`That table or column does not exist.`); + } + } + + renameRow(info: { name: string, row: number, value: string }) { + const { name, row, value } = info; + if (this.rowNames[name] && this.rowNames[name][row] !== undefined) { + this.rowNames[name][row] = value; + } else { + alert(`That table or row does not exist.`); + } + } + newTable(info: { name: string, rows: number, columns: number }) { const { name, rows, columns } = info; @@ -89,6 +113,10 @@ export default class Tables extends Extension { for (let j = 0; j < columns; j++) newRow.push(0); this.tables[name].push(newRow); } + + // Default row & column names + this.rowNames[name] = Array.from({ length: rows }, (_, i) => `Row ${i + 1}`); + this.columnNames[name] = Array.from({ length: columns }, (_, i) => `Col ${i + 1}`); } changeTableValue(info: { name: string, row: number, column: number, value: number }) { @@ -150,6 +178,8 @@ export default class Tables extends Extension { for (let i = 0; i < this.tables[table].length; i++) { this.tables[table][i].push(0); } + const colCount = this.columnNames[table].length; + this.columnNames[table].push(`Col ${colCount + 1}`); } }), // add a row to the given table @@ -167,6 +197,8 @@ export default class Tables extends Extension { newRow.push(0); } this.tables[table].push(newRow); + const rowCount = this.rowNames[table].length; + this.rowNames[table].push(`Row ${rowCount + 1}`); } }), // change the value in a given table cell From c10f6dfb713a02ce3a65bed6c7078dc608a23db4 Mon Sep 17 00:00:00 2001 From: Maya Rajan Date: Fri, 16 Jan 2026 17:38:27 -0500 Subject: [PATCH 28/35] test on main site --- scratch-packages/scratch-gui | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scratch-packages/scratch-gui b/scratch-packages/scratch-gui index 91c188acf..67de78ab6 160000 --- a/scratch-packages/scratch-gui +++ b/scratch-packages/scratch-gui @@ -1 +1 @@ -Subproject commit 91c188acf6f2f1a9e267c8a5a61e196fb18bcc39 +Subproject commit 67de78ab680c09fcb41cbe3430e8c01efc9a731e From 4d31c25f9fc7617d051221701e3b71ee4e28b4c6 Mon Sep 17 00:00:00 2001 From: Maya Rajan Date: Fri, 16 Jan 2026 18:04:55 -0500 Subject: [PATCH 29/35] test on main site fix --- scratch-packages/scratch-gui | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scratch-packages/scratch-gui b/scratch-packages/scratch-gui index 67de78ab6..1bc4fb7f1 160000 --- a/scratch-packages/scratch-gui +++ b/scratch-packages/scratch-gui @@ -1 +1 @@ -Subproject commit 67de78ab680c09fcb41cbe3430e8c01efc9a731e +Subproject commit 1bc4fb7f137372b7b70bc3e8b7d80f2f29bc1824 From 98b7c76c77a39b3d3faad688e204ba90e4d3f05c Mon Sep 17 00:00:00 2001 From: Maya Rajan Date: Mon, 26 Jan 2026 18:39:06 -0500 Subject: [PATCH 30/35] move code to google chooser --- scratch-packages/scratch-gui | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scratch-packages/scratch-gui b/scratch-packages/scratch-gui index 1bc4fb7f1..5714f7017 160000 --- a/scratch-packages/scratch-gui +++ b/scratch-packages/scratch-gui @@ -1 +1 @@ -Subproject commit 1bc4fb7f137372b7b70bc3e8b7d80f2f29bc1824 +Subproject commit 5714f70173b4fc6bf87b1a94bbbe266f5b3e33b1 From 64705ad1401bfeddfd78b25f7f71e3c2fa8b7ceb Mon Sep 17 00:00:00 2001 From: Maya Rajan Date: Wed, 28 Jan 2026 17:10:01 -0500 Subject: [PATCH 31/35] point to main --- scratch-packages/scratch-gui | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scratch-packages/scratch-gui b/scratch-packages/scratch-gui index 5714f7017..dd778ace8 160000 --- a/scratch-packages/scratch-gui +++ b/scratch-packages/scratch-gui @@ -1 +1 @@ -Subproject commit 5714f70173b4fc6bf87b1a94bbbe266f5b3e33b1 +Subproject commit dd778ace8a8e8a5b479311c20d3eab9ddc0afc31 From 7391681773e2f62a37edb2522c3477e8df83630f Mon Sep 17 00:00:00 2001 From: Maya Rajan Date: Wed, 28 Jan 2026 17:57:29 -0500 Subject: [PATCH 32/35] switch order --- scratch-packages/scratch-gui | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scratch-packages/scratch-gui b/scratch-packages/scratch-gui index dd778ace8..59ae0de18 160000 --- a/scratch-packages/scratch-gui +++ b/scratch-packages/scratch-gui @@ -1 +1 @@ -Subproject commit dd778ace8a8e8a5b479311c20d3eab9ddc0afc31 +Subproject commit 59ae0de18e1d02baa0e43444ffbe2bab3cd3b9ab From 0e66f28f5682de40e12fdec2f32bfb3ebcc70c2a Mon Sep 17 00:00:00 2001 From: Maya Rajan Date: Mon, 2 Mar 2026 14:11:08 -0500 Subject: [PATCH 33/35] pen fix --- scratch-packages/scratch-vm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scratch-packages/scratch-vm b/scratch-packages/scratch-vm index cc723570e..9876f8082 160000 --- a/scratch-packages/scratch-vm +++ b/scratch-packages/scratch-vm @@ -1 +1 @@ -Subproject commit cc723570e845e1c04dcc7a71e925905b61a7b431 +Subproject commit 9876f8082effbc6f71d70d2a581291240017275a From 9faaba3308d2246c6596ff8b4734c9e5273b9118 Mon Sep 17 00:00:00 2001 From: Maya Rajan Date: Mon, 2 Mar 2026 14:36:42 -0500 Subject: [PATCH 34/35] switch to const --- scratch-packages/scratch-vm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scratch-packages/scratch-vm b/scratch-packages/scratch-vm index 9876f8082..160e17986 160000 --- a/scratch-packages/scratch-vm +++ b/scratch-packages/scratch-vm @@ -1 +1 @@ -Subproject commit 9876f8082effbc6f71d70d2a581291240017275a +Subproject commit 160e17986c6d90d67f7a8fe6e3f840aa78fcb175 From c67322fd51133b6b8f5dcf5a138be0571e4c4fcf Mon Sep 17 00:00:00 2001 From: Maya Rajan Date: Tue, 3 Mar 2026 16:59:27 -0500 Subject: [PATCH 35/35] new commit --- scratch-packages/scratch-vm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scratch-packages/scratch-vm b/scratch-packages/scratch-vm index 160e17986..cfd70582e 160000 --- a/scratch-packages/scratch-vm +++ b/scratch-packages/scratch-vm @@ -1 +1 @@ -Subproject commit 160e17986c6d90d67f7a8fe6e3f840aa78fcb175 +Subproject commit cfd70582efd1b3a8fcfe9ff0353eb271e2b18a03