[feat] Introduce Loop tasks: Map, Reduce, and While

TODO.md

@@ -9,10 +9,10 @@ TODO.md
 - [ ] Datasets Package
   - [x] Documents dataset (mabye rename to DocumentDataset)
   - [ ] Chunks Package (or part of DocumentDataset?)
-  - [ ] Move Model repository to datasets package.
-- [ ] Chunks and nodes are not always the same.
-  - [ ] And we may need to save the chunk's node path. Or paths? or document range? Standard metadata?
-- [ ] Instead of passing doc_id around, pass a document key that is unknonwn (string or object)
+  - [x] Move Model repository to datasets package.
+- [x] Chunks and nodes are not always the same.
+  - [x] And we may need to save the chunk's node path. Or paths? or document range? Standard metadata?
+- [ ] Instead of passing doc_id around, pass a document key that is of type unknown (string or object)
 
 - [ ] Get a better model for question answering.
 - [ ] Get a better model for named entity recognition, the current one recognized everything as a token, not helpful.
@@ -25,16 +25,11 @@ TODO.md
 - [ ] rename the registration stuff to not look ugly: registerHuggingfaceTransformers() and registerHuggingfaceTransformersUsingWorkers() and registerHuggingfaceTransformersInsideWorker()
 - [ ] fix image transferables
 
-onnx-community/ModernBERT-finetuned-squad-ONNX - summarization
+- [ ] Consider different ways to connect tasks to queues. What is a task? What is a job?
+
+- [ ] Input and outputs are all scalar, arrays, or unions. But what about streams? Stream of items in an array, stream of content for a scalar like a string, etc.
 
-- [x] Auto-generated primary keys for TabularStorage
-  - [x] Schema annotation with `x-auto-generated: true`
-  - [x] Type system with `InsertEntity` for optional auto-generated keys
-  - [x] Support for autoincrement (integer) and UUID (string) strategies
-  - [x] Configurable client-provided keys: "never", "if-missing", "always"
-  - [x] Implementations: InMemory, SQLite, Postgres, Supabase, IndexedDB, FsFolder
-  - [x] Comprehensive test suite (342 tests pass)
-  - [x] Documentation updated
+onnx-community/ModernBERT-finetuned-squad-ONNX - summarization
 
 Rework the Document Dataset. Currently there is a Document storage of tabular storage type, and that should be registered as a "dataset:document:source" meaning the source material in node format. And there is already a "dataset:document-chunk" for the chunk/vector storage which should be registered as a "dataset:document:chunk" with a well defined metadata schema. The two combined should be registered as a "dataset:document" which is the complete document with its source and all its chunks and metadata. This is for convenience but not used by tasks or ai tasks.
 

examples/cli/CHANGELOG.md

@@ -1,5 +1,19 @@
 # @workglow/cli
 
+## 0.0.90
+
+### Patch Changes
+
+- Introduce Loop tasks: Map, Reduce, and While
+- Updated dependencies
+  - @workglow/task-graph@0.0.90
+  - @workglow/tasks@0.0.90
+  - @workglow/test@0.0.90
+  - @workglow/util@0.0.90
+  - @workglow/ai@0.0.90
+  - @workglow/ai-provider@0.0.90
+  - @workglow/storage@0.0.90
+
 ## 0.0.89
 
 ### Patch Changes

examples/cli/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@workglow/cli",
   "type": "module",
-  "version": "0.0.89",
+  "version": "0.0.90",
   "description": "Command-line interface example for Workglow, demonstrating how to build and run AI task pipelines from the terminal.",
   "scripts": {
     "dev": "concurrently -c 'auto' -n 'cli:' 'bun:dev-*'",
@@ -45,7 +45,7 @@
     "@sroussey/transformers": "catalog:",
     "@types/react": "=18.3.12",
     "chalk": "^5.6.2",
-    "commander": "^14.0.1",
+    "commander": "^14.0.3",
     "is-unicode-supported": "^2.1.0",
     "react": "=18.3.1",
     "react-reconciler": "=0.31.0",

examples/cli/src/TaskCLI.ts

@@ -4,14 +4,18 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import { DownloadModelTask, getGlobalModelRepository, type ModelConfig } from "@workglow/ai";
+import {
+  DownloadModelTask,
+  getGlobalModelRepository,
+  registerAiTasks,
+  type ModelConfig,
+} from "@workglow/ai";
 import { HF_TRANSFORMERS_ONNX } from "@workglow/ai-provider";
-import { JsonTaskItem, TaskGraph, Workflow } from "@workglow/task-graph";
-import { DelayTask, JsonTask } from "@workglow/tasks";
+import { JsonTaskItem, registerBaseTasks, TaskGraph, Workflow } from "@workglow/task-graph";
+import { DelayTask, JsonTask, registerCommonTasks } from "@workglow/tasks";
 import type { Command } from "commander";
 import { readFile, writeFile } from "fs/promises";
 import { runTasks } from "./TaskGraphToUI";
-
 /**
  * Read image input from file or stdin
  */
@@ -539,8 +543,18 @@ export function AddBaseCommands(program: Command) {
   program
     .command("json")
     .description("run based on json input")
-    .argument("[json]", "json text to rewrite and vectorize")
-    .action(async (json) => {
+    .argument("[json]", "json text")
+    .option("--file <path>", "read JSON from file")
+    .action(async (jsonArg, options) => {
+      registerBaseTasks();
+      registerCommonTasks();
+      registerAiTasks();
+      let json = jsonArg;
+      if (!json && options.file) {
+        json = (await readFile(options.file, "utf-8")).trim();
+      } else if (!json && !process.stdin.isTTY) {
+        json = await readTextInput();
+      }
       if (!json) {
         const exampleJson: JsonTaskItem[] = [
           {

examples/cli/src/components/TaskUI.tsx

@@ -144,7 +144,9 @@ export const TaskUI: FC<{
       setError((prevErr) => (prevErr ? `${prevErr}\nAborted` : "Aborted"));
     };
     onRegenerate();
-    setDependantChildren(graph.getTargetTasks(task.config.id));
+    const targets = graph.getTargetTasks(task.config.id);
+    const unique = [...new Map(targets.map((t) => [t.config.id, t])).values()];
+    setDependantChildren(unique);
 
     task.on("start", onStart);
     task.on("progress", onProgress);

examples/web/CHANGELOG.md

@@ -1,5 +1,22 @@
 # @workglow/web
 
+## 0.0.90
+
+### Patch Changes
+
+- Introduce Loop tasks: Map, Reduce, and While
+- Updated dependencies
+  - @workglow/task-graph@0.0.90
+  - @workglow/tasks@0.0.90
+  - @workglow/test@0.0.90
+  - @workglow/util@0.0.90
+  - @workglow/ai@0.0.90
+  - @workglow/ai-provider@0.0.90
+  - @workglow/debug@0.0.90
+  - @workglow/job-queue@0.0.90
+  - @workglow/sqlite@0.0.90
+  - @workglow/storage@0.0.90
+
 ## 0.0.89
 
 ### Patch Changes

examples/web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@workglow/web",
-  "version": "0.0.89",
+  "version": "0.0.90",
   "type": "module",
   "description": "Web interface example for Workglow, showcasing a React-based UI for building and visualizing AI task pipelines.",
   "scripts": {
@@ -30,8 +30,8 @@
     "@mediapipe/tasks-genai": "catalog:",
     "@radix-ui/react-icons": "^1.3.2",
     "@sroussey/transformers": "catalog:",
-    "@uiw/codemirror-theme-vscode": "^4.25.2",
-    "@uiw/react-codemirror": "^4.25.2",
+    "@uiw/codemirror-theme-vscode": "^4.25.4",
+    "@uiw/react-codemirror": "^4.25.4",
     "@xyflow/react": "^12.10.0",
     "clsx": "^2.1.1",
     "react": "^19.2.4",
@@ -42,9 +42,9 @@
     "tailwind-merge": "2.6.0"
   },
   "devDependencies": {
-    "@types/react": "^19.2.10",
+    "@types/react": "^19.2.14",
     "@types/react-dom": "^19.2.3",
-    "@vitejs/plugin-react": "^5.1.3",
+    "@vitejs/plugin-react": "^5.1.4",
     "autoprefixer": "10.4.24",
     "postcss": "8.5.6",
     "tailwindcss": "3.4.17",

package.json

@@ -32,20 +32,20 @@
     "publish": "bun ./scripts/publish-workspaces.ts"
   },
   "dependencies": {
-    "caniuse-lite": "^1.0.30001767"
+    "caniuse-lite": "^1.0.30001769"
   },
   "catalog": {
     "@sroussey/transformers": "3.8.2",
-    "@mediapipe/tasks-text": "^0.10.22-rc.20250304",
-    "@mediapipe/tasks-vision": "^0.10.22-rc.20250304",
-    "@mediapipe/tasks-audio": "^0.10.22-rc.20250304",
-    "@mediapipe/tasks-genai": "^0.10.22-rc.20250304"
+    "@mediapipe/tasks-text": "^0.10.32",
+    "@mediapipe/tasks-vision": "^0.10.32",
+    "@mediapipe/tasks-audio": "^0.10.32",
+    "@mediapipe/tasks-genai": "^0.10.26"
   },
   "devDependencies": {
     "@sroussey/changesets-cli": "^2.29.7",
-    "@types/bun": "^1.3.8",
-    "@typescript-eslint/eslint-plugin": "^8.54.0",
-    "@typescript-eslint/parser": "^8.54.0",
+    "@types/bun": "^1.3.9",
+    "@typescript-eslint/eslint-plugin": "^8.55.0",
+    "@typescript-eslint/parser": "^8.55.0",
     "concurrently": "^9.2.1",
     "eslint": "^9.39.2",
     "eslint-plugin-jsx-a11y": "^6.10.2",
@@ -54,7 +54,7 @@
     "eslint-plugin-regexp": "^2.10.0",
     "globals": "^17.3.0",
     "prettier": "^3.8.1",
-    "turbo": "^2.8.3",
+    "turbo": "^2.8.6",
     "typescript": "5.9.3",
     "vitest": "^4.0.18"
   },

packages/ai-provider/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @workglow/ai-provider
 
+## 0.0.90
+
+### Patch Changes
+
+- Introduce Loop tasks: Map, Reduce, and While
+- Updated dependencies
+  - @workglow/task-graph@0.0.90
+  - @workglow/util@0.0.90
+  - @workglow/ai@0.0.90
+  - @workglow/job-queue@0.0.90
+  - @workglow/storage@0.0.90
+
 ## 0.0.89
 
 ### Patch Changes

packages/ai-provider/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@workglow/ai-provider",
   "type": "module",
-  "version": "0.0.89",
+  "version": "0.0.90",
   "description": "AI provider integrations for Workglow, supporting OpenAI, Hugging Face Transformers, MediaPipe, and GGML models.",
   "scripts": {
     "watch": "concurrently -c 'auto' 'bun:watch-*'",

packages/ai/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @workglow/ai
 
+## 0.0.90
+
+### Patch Changes
+
+- Introduce Loop tasks: Map, Reduce, and While
+- Updated dependencies
+  - @workglow/task-graph@0.0.90
+  - @workglow/util@0.0.90
+  - @workglow/dataset@0.0.90
+  - @workglow/job-queue@0.0.90
+  - @workglow/storage@0.0.90
+
 ## 0.0.89
 
 ### Patch Changes

packages/ai/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@workglow/ai",
   "type": "module",
-  "version": "0.0.89",
+  "version": "0.0.90",
   "description": "Core AI functionality for Workglow, including task execution, model management, and AI pipeline orchestration.",
   "scripts": {
     "watch": "concurrently -c 'auto' 'bun:watch-*'",

packages/ai/src/model/ModelRepository.ts

@@ -168,7 +168,7 @@ export class ModelRepository {
    * Enumerates all models in the repository
    * @returns Promise resolving to an array of model instances
    */
-  async enumerateAllModels() {
+  async enumerateAllModels(): Promise<ModelRecord[] | undefined> {
     const models = await this.modelTabularRepository.getAll();
     if (!models || models.length === 0) return undefined;
     return models;
@@ -179,7 +179,7 @@ export class ModelRepository {
    * @param modelId - The model_id of the model to find
    * @returns Promise resolving to the found model or undefined if not found
    */
-  async findByName(model_id: string) {
+  async findByName(model_id: string): Promise<ModelRecord | undefined> {
     if (typeof model_id != "string") return undefined;
     const model = await this.modelTabularRepository.get({ model_id });
     return model ?? undefined;

packages/dataset/CHANGELOG.md

@@ -1,5 +1,14 @@
 # @workglow/dataset
 
+## 0.0.90
+
+### Patch Changes
+
+- Introduce Loop tasks: Map, Reduce, and While
+- Updated dependencies
+  - @workglow/util@0.0.90
+  - @workglow/storage@0.0.90
+
 ## 0.0.89
 
 ### Patch Changes

packages/dataset/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@workglow/dataset",
   "type": "module",
-  "version": "0.0.89",
+  "version": "0.0.90",
   "description": "Dataset package for Workglow.",
   "scripts": {
     "watch": "concurrently -c 'auto' 'bun:watch-*'",

packages/debug/CHANGELOG.md

@@ -1,5 +1,14 @@
 # @workglow/debug
 
+## 0.0.90
+
+### Patch Changes
+
+- Introduce Loop tasks: Map, Reduce, and While
+- Updated dependencies
+  - @workglow/task-graph@0.0.90
+  - @workglow/util@0.0.90
+
 ## 0.0.89
 
 ### Patch Changes

packages/debug/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@workglow/debug",
   "type": "module",
-  "version": "0.0.89",
+  "version": "0.0.90",
   "description": "Debug utilities for Workglow, including Chrome DevTools custom formatters for task graphs and workflows.",
   "scripts": {
     "watch": "concurrently -c 'auto' 'bun:watch-*'",

packages/job-queue/CHANGELOG.md

@@ -1,5 +1,14 @@
 # @workglow/job-queue
 
+## 0.0.90
+
+### Patch Changes
+
+- Introduce Loop tasks: Map, Reduce, and While
+- Updated dependencies
+  - @workglow/util@0.0.90
+  - @workglow/storage@0.0.90
+
 ## 0.0.89
 
 ### Patch Changes

packages/job-queue/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@workglow/job-queue",
   "type": "module",
-  "version": "0.0.89",
+  "version": "0.0.90",
   "description": "Job queue management for Workglow, handling task scheduling, execution, and distributed processing.",
   "scripts": {
     "watch": "concurrently -c 'auto' 'bun:watch-*'",

packages/sqlite/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @workglow/sqlite
 
+## 0.0.90
+
+### Patch Changes
+
+- Introduce Loop tasks: Map, Reduce, and While
+
 ## 0.0.89
 
 ### Patch Changes

packages/sqlite/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@workglow/sqlite",
   "type": "module",
-  "version": "0.0.89",
+  "version": "0.0.90",
   "description": "SQLite database providing cross-platform database support for browser, Node.js, and Bun environments.",
   "scripts": {
     "watch": "concurrently -c 'auto' 'bun:watch-*'",
@@ -20,7 +20,7 @@
     "prepare": "node -e \"const pkg=require('./package.json');pkg.exports['.'].bun='./dist/bun.js';pkg.exports['.'].types='./dist/types.d.ts';require('fs').writeFileSync('package.json',JSON.stringify(pkg,null,2))\""
   },
   "peerDependencies": {
-    "@sqlite.org/sqlite-wasm": "^3.51.1-build2",
+    "@sqlite.org/sqlite-wasm": "^3.51.2-build6",
     "better-sqlite3": "^12.6.2"
   },
   "peerDependenciesMeta": {