Merge branch 'develop' of https://github.com/elizaOS/eliza into develop

Author: Shaw

plugins/plugin-computeruse/src/__tests__/dhash.test.ts

@@ -0,0 +1,157 @@
+/**
+ * Pure-function tests for the dHash + block-grid implementation.
+ *
+ * We synthesize tiny PNGs via `node:zlib` and the PNG chunk format so we
+ * don't need any decoder dependency to exercise both code paths.
+ */
+
+import { deflateSync } from "node:zlib";
+import { describe, expect, it } from "vitest";
+import {
+  blockGrid,
+  decodePng,
+  diffBlocks,
+  frameDhash,
+  hamming,
+} from "../scene/dhash.js";
+
+function crc32(bytes: Buffer): number {
+  let crc = 0xffffffff >>> 0;
+  for (let i = 0; i < bytes.length; i += 1) {
+    crc = (crc ^ bytes[i]!) >>> 0;
+    for (let j = 0; j < 8; j += 1) {
+      const lsb = crc & 1;
+      crc = (crc >>> 1) ^ (lsb ? 0xedb88320 : 0);
+    }
+  }
+  return (crc ^ 0xffffffff) >>> 0;
+}
+
+function pngChunk(type: string, data: Buffer): Buffer {
+  const len = Buffer.alloc(4);
+  len.writeUInt32BE(data.length);
+  const t = Buffer.from(type, "ascii");
+  const crc = Buffer.alloc(4);
+  crc.writeUInt32BE(crc32(Buffer.concat([t, data])));
+  return Buffer.concat([len, t, data, crc]);
+}
+
+/**
+ * Build a 16×16 8-bit RGB PNG painted in a horizontal gradient unless
+ * `solid` is true (in which case every pixel is the same color).
+ */
+function makeTinyPng(seed = 0, solid = false): Buffer {
+  const w = 16;
+  const h = 16;
+  const stride = w * 3;
+  const rows: number[] = [];
+  for (let y = 0; y < h; y += 1) {
+    rows.push(0); // filter = None
+    for (let x = 0; x < w; x += 1) {
+      const v = solid ? (seed * 7) % 255 : ((x + seed) * 16) % 255;
+      rows.push(v, v, v);
+    }
+  }
+  const raw = Buffer.from(rows);
+  const idat = deflateSync(raw);
+  const ihdr = Buffer.alloc(13);
+  ihdr.writeUInt32BE(w, 0);
+  ihdr.writeUInt32BE(h, 4);
+  ihdr[8] = 8; // bit depth
+  ihdr[9] = 2; // color type RGB
+  ihdr[10] = 0;
+  ihdr[11] = 0;
+  ihdr[12] = 0;
+  const signature = Buffer.from([
+    0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a,
+  ]);
+  return Buffer.concat([
+    signature,
+    pngChunk("IHDR", ihdr),
+    pngChunk("IDAT", idat),
+    pngChunk("IEND", Buffer.alloc(0)),
+  ]);
+}
+
+describe("dhash — pure functions", () => {
+  it("decodes a minimal RGB PNG", () => {
+    const png = makeTinyPng();
+    const decoded = decodePng(png);
+    expect(decoded).not.toBeNull();
+    expect(decoded?.width).toBe(16);
+    expect(decoded?.height).toBe(16);
+    expect(decoded?.rgba.length).toBe(16 * 16 * 4);
+  });
+
+  it("returns null for non-PNG input", () => {
+    expect(decodePng(Buffer.from("not a png"))).toBeNull();
+  });
+
+  it("frameDhash is stable for identical frames", () => {
+    const a = frameDhash(makeTinyPng(7));
+    const b = frameDhash(makeTinyPng(7));
+    expect(a).not.toBeNull();
+    expect(b).not.toBeNull();
+    expect(a).toBe(b);
+  });
+
+  it("frameDhash differs for visually different frames", () => {
+    const a = frameDhash(makeTinyPng(0));
+    const b = frameDhash(makeTinyPng(50));
+    expect(a).not.toBeNull();
+    expect(b).not.toBeNull();
+    expect(hamming(a!, b!)).toBeGreaterThan(0);
+  });
+
+  it("hamming(x, x) == 0", () => {
+    expect(hamming(0xdeadbeefcafe0001n, 0xdeadbeefcafe0001n)).toBe(0);
+  });
+
+  it("hamming(a, b) counts changed bits", () => {
+    expect(hamming(0n, 0xffffffffffffffffn)).toBe(64);
+  });
+});
+
+describe("dhash — block grid", () => {
+  it("produces a cols*rows grid", () => {
+    const grid = blockGrid(makeTinyPng(0), 4, 4);
+    expect(grid).not.toBeNull();
+    expect(grid!.cols).toBe(4);
+    expect(grid!.rows).toBe(4);
+    expect(grid!.hashes.length).toBe(16);
+  });
+
+  it("identical frames produce identical block grids and zero dirty blocks", () => {
+    const a = blockGrid(makeTinyPng(11), 4, 4)!;
+    const b = blockGrid(makeTinyPng(11), 4, 4)!;
+    const dirty = diffBlocks(a, b);
+    expect(dirty.length).toBe(0);
+  });
+
+  it("first frame (prev=null) marks every block dirty", () => {
+    const grid = blockGrid(makeTinyPng(0), 4, 4)!;
+    const dirty = diffBlocks(null, grid);
+    expect(dirty.length).toBe(grid.cols * grid.rows);
+  });
+
+  it("changed frames produce a non-zero dirty list", () => {
+    const a = blockGrid(makeTinyPng(0), 4, 4)!;
+    const b = blockGrid(makeTinyPng(120), 4, 4)!;
+    const dirty = diffBlocks(a, b);
+    expect(dirty.length).toBeGreaterThan(0);
+  });
+
+  it("dirty-block bboxes are translated to image pixel space when dims are known", () => {
+    const a = blockGrid(makeTinyPng(0), 4, 4)!;
+    const b = blockGrid(makeTinyPng(120), 4, 4)!;
+    const dirty = diffBlocks(a, b, 16, 16);
+    for (const d of dirty) {
+      expect(d.bbox[2]).toBeGreaterThan(0);
+      expect(d.bbox[3]).toBeGreaterThan(0);
+      expect(d.bbox[0]).toBeGreaterThanOrEqual(0);
+      expect(d.bbox[1]).toBeGreaterThanOrEqual(0);
+      expect(d.bbox[0] + d.bbox[2]).toBeLessThanOrEqual(16);
+      expect(d.bbox[1] + d.bbox[3]).toBeLessThanOrEqual(16);
+    }
+  });
+});

plugins/plugin-computeruse/src/index.ts

@@ -27,6 +27,7 @@ import { promoteSubactionsToActions } from "@elizaos/core";
 import { useComputerAction } from "./actions/use-computer.js";
 import { windowAction } from "./actions/window.js";
 import { computerStateProvider } from "./providers/computer-state.js";
+import { sceneProvider } from "./providers/scene.js";
 import { computerUseRouteHandler } from "./routes/computer-use-compat-routes.js";
 import { ComputerUseService } from "./services/computer-use-service.js";
 
@@ -80,7 +81,7 @@ export const computerUsePlugin: Plugin = {
     ...promoteSubactionsToActions(windowAction),
   ],
 
-  providers: [computerStateProvider],
+  providers: [computerStateProvider, sceneProvider],
 
   routes: computerUseRoutes,
 
@@ -99,6 +100,53 @@ export { ComputerUseService } from "./services/computer-use-service.js";
 
 // iOS computer-use surface. See `docs/IOS_CONSTRAINTS.md` for the honest scope.
 export * from "./mobile/index.js";
+
+// WS6: scene-builder surface — consumed by WS7 (Brain) and WS10 verifiers.
+export {
+  _resetDefaultSceneBuilderForTests,
+  getDefaultSceneBuilder,
+  SceneBuilder,
+  type SceneBuilderDeps,
+  type SceneUpdateEvent,
+} from "./scene/scene-builder.js";
+export type {
+  Scene,
+  SceneApp,
+  SceneAppWindow,
+  SceneAxNode,
+  SceneFocusedWindow,
+  SceneOcrBox,
+  SceneVlmElement,
+} from "./scene/scene-types.js";
+export { serializeSceneForPrompt } from "./scene/serialize.js";
+export {
+  type BlockGrid,
+  blockGrid,
+  decodePng,
+  diffBlocks,
+  type DirtyBlock,
+  frameDhash,
+  hamming,
+} from "./scene/dhash.js";
+export { enumerateApps, joinAppsAndWindows } from "./scene/apps.js";
+export {
+  type AccessibilityProvider,
+  DarwinAccessibilityProvider,
+  LinuxAccessibilityProvider,
+  NullAccessibilityProvider,
+  parseHyprlandClients,
+  parseSwayTree,
+  resolveAccessibilityProvider,
+  setAccessibilityProvider,
+  WindowsAccessibilityProvider,
+} from "./scene/a11y-provider.js";
+export {
+  listProcesses,
+  parsePsOutput,
+  parseWindowsProcessJson,
+  type ProcessInfo,
+} from "./platform/process-list.js";
+export { sceneProvider } from "./providers/scene.js";
 export type {
   DesktopControlCapabilities,
   DesktopControlCapability,

plugins/plugin-computeruse/src/platform/process-list.ts

@@ -0,0 +1,148 @@
+/**
+ * Cross-platform process listing.
+ *
+ * The WS6 scene-builder joins running processes with windows to produce the
+ * `apps[]` field of a Scene. The contract is intentionally minimal — pid,
+ * executable/display name, and a best-effort foreground flag if cheap to
+ * obtain. Anything richer (memory, cpu, parent pid) is out of scope here
+ * because the scene-builder runs every active-poll frame and must stay
+ * cheap.
+ *
+ * Per-OS source:
+ *   - Linux  : `/proc/<pid>/comm` and `/proc/<pid>/status`. Pure FS read,
+ *              no shell out. ~5ms for 300 processes.
+ *   - macOS  : `ps -axo pid=,comm=` — built-in BSD ps.
+ *   - Windows: PowerShell `Get-Process | Select Id, ProcessName`.
+ *   - Android: stub returning `[]` — the JS contract for `UsageStatsManager`
+ *              is owned by WS8's native side. We expose the function shape so
+ *              the scene-builder doesn't have to branch.
+ *
+ * Failure semantics:
+ *   - A single un-readable process is skipped, not propagated.
+ *   - A complete enumeration failure returns `[]` and the scene-builder logs
+ *     once per platform-mode at warn.
+ */
+
+import { execFileSync, execSync } from "node:child_process";
+import { readFileSync, readdirSync } from "node:fs";
+import { currentPlatform } from "./helpers.js";
+
+export interface ProcessInfo {
+  pid: number;
+  name: string;
+}
+
+export function listProcesses(): ProcessInfo[] {
+  const os = currentPlatform();
+  if (os === "linux") return listLinux();
+  if (os === "darwin") return listDarwin();
+  if (os === "win32") return listWindows();
+  return [];
+}
+
+function listLinux(): ProcessInfo[] {
+  const out: ProcessInfo[] = [];
+  let entries: string[];
+  try {
+    entries = readdirSync("/proc");
+  } catch {
+    return out;
+  }
+  for (const entry of entries) {
+    if (!/^\d+$/.test(entry)) continue;
+    const pid = Number.parseInt(entry, 10);
+    if (!Number.isFinite(pid)) continue;
+    let name = "";
+    try {
+      // `comm` is the 15-char truncated executable basename. Good enough for
+      // join-with-windows; a longer name (cmdline arg0 basename) is overkill.
+      name = readFileSync(`/proc/${pid}/comm`, "utf8").trim();
+    } catch {
+      continue;
+    }
+    if (!name) continue;
+    out.push({ pid, name });
+  }
+  return out;
+}
+
+function listDarwin(): ProcessInfo[] {
+  try {
+    const text = execFileSync("ps", ["-axco", "pid=,comm="], {
+      timeout: 4000,
+      encoding: "utf8",
+      stdio: ["ignore", "pipe", "ignore"],
+    });
+    return parsePsOutput(text);
+  } catch {
+    try {
+      // Fallback: BSD ps without `-c` (gives full path in comm).
+      const text = execFileSync("ps", ["-axo", "pid=,comm="], {
+        timeout: 4000,
+        encoding: "utf8",
+        stdio: ["ignore", "pipe", "ignore"],
+      });
+      return parsePsOutput(text);
+    } catch {
+      return [];
+    }
+  }
+}
+
+export function parsePsOutput(text: string): ProcessInfo[] {
+  const out: ProcessInfo[] = [];
+  for (const line of text.split(/\r?\n/)) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+    const m = trimmed.match(/^(\d+)\s+(.+)$/);
+    if (!m) continue;
+    const pid = Number.parseInt(m[1] ?? "0", 10);
+    if (!Number.isFinite(pid) || pid <= 0) continue;
+    const rawName = (m[2] ?? "").trim();
+    if (!rawName) continue;
+    // For `ps -axo pid=,comm=` without `-c` the comm column holds an absolute
+    // path. We strip to the basename so the scene-builder's join key matches
+    // the AppleScript window enumerator's `name of proc`.
+    const name = rawName.split("/").pop() ?? rawName;
+    out.push({ pid, name });
+  }
+  return out;
+}
+
+function listWindows(): ProcessInfo[] {
+  try {
+    const text = execSync(
+      "powershell -NoProfile -Command \"Get-Process | Select-Object Id,ProcessName | ConvertTo-Json -Compress\"",
+      { timeout: 8000, encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] },
+    );
+    return parseWindowsProcessJson(text);
+  } catch {
+    return [];
+  }
+}
+
+interface WinProcessRow {
+  Id?: number;
+  ProcessName?: string;
+}
+
+export function parseWindowsProcessJson(text: string): ProcessInfo[] {
+  let raw: unknown;
+  try {
+    raw = JSON.parse(text);
+  } catch {
+    return [];
+  }
+  const items: WinProcessRow[] = Array.isArray(raw)
+    ? (raw as WinProcessRow[])
+    : [raw as WinProcessRow];
+  const out: ProcessInfo[] = [];
+  for (const row of items) {
+    if (!row || typeof row !== "object") continue;
+    const pid = Number(row.Id);
+    const name = typeof row.ProcessName === "string" ? row.ProcessName : "";
+    if (!Number.isFinite(pid) || pid <= 0 || !name) continue;
+    out.push({ pid, name });
+  }
+  return out;
+}