[refactor] Refactor AI provider registration and introduce HuggingFace and TensorFlow MediaPipe providers

- Replaced `register_HFT_InlineJobFns` and `register_HFT_ClientJobFns` with `HuggingFaceTransformersProvider` and `TensorFlowMediaPipeProvider` for improved modularity and clarity.
- Updated documentation and examples to reflect the new provider registration methods.
- Enhanced worker registration for HuggingFace and TensorFlow MediaPipe tasks, streamlining the integration of AI models.
- Removed deprecated registration functions to clean up the codebase.

Author: sroussey

docs/developers/01_getting_started.md

@@ -138,10 +138,10 @@ import {
 import { DebugLogTask } from "@workglow/tasks";
 import { ConcurrencyLimiter, JobQueueClient, JobQueueServer } from "@workglow/job-queue";
 import { InMemoryQueueStorage } from "@workglow/storage";
-import { HF_TRANSFORMERS_ONNX, register_HFT_InlineJobFns } from "@workglow/ai-provider";
+import { HF_TRANSFORMERS_ONNX, HuggingFaceTransformersProvider } from "@workglow/ai-provider";
 
 // Provider run functions on this thread
-await register_HFT_InlineJobFns();
+await new HuggingFaceTransformersProvider().register({ mode: "inline" });
 
 // Set up a model repo and models
 const modelRepo = new InMemoryModelRepository();
@@ -218,17 +218,17 @@ You can use as much or as little "magic" as you want. The config helpers are the
 
 Tasks are agnostic to the provider. Text embedding can be done with several providers, such as Hugging Face Transformers (ONNX) or MediaPipe locally, or OpenAI etc via API calls.
 
-- **`register_HFT_InlineJobFns()`** - Registers the Hugging Face Transformers local provider. Now you can use an ONNX model name for `TextEmbedding`, etc.
-- **`register_TFMP_InlineJobFns()`** - Registers the MediaPipe TF.js local provider. Now you can use one of the MediaPipe models.
+- **`new HuggingFaceTransformersProvider().register({ mode: "inline" })`** - Registers the Hugging Face Transformers local provider. Now you can use an ONNX model name for `TextEmbedding`, etc.
+- **`new TensorFlowMediaPipeProvider().register({ mode: "inline" })`** - Registers the MediaPipe TF.js local provider. Now you can use one of the MediaPipe models.
 
 ### Registering Provider plus related Job Queue
 
-LLM providers have long running functions. These are handled by a Job Queue. There are some pre-built ones:
+LLM providers have long running functions. These are handled by a Job Queue. The `provider.register()` call creates the queue automatically. For convenience:
 
 #### In memory:
 
-- **`register_HFT_InMemoryQueue`** sets up the Hugging Face Transformers provider (above), and a job queue with `JobQueueServer` and `JobQueueClient` with a `ConcurrencyLimiter` so the ONNX queue only runs one task/job at a time.
-- **`register_TFMP_InMemoryQueue`** does the same for MediaPipe.
+- **`register_HFT_InMemoryQueue`** (from `@workglow/test`) - Equivalent to `new HuggingFaceTransformersProvider().register({ mode: "inline" })`.
+- **`register_TFMP_InMemoryQueue`** (from `@workglow/test`) - Equivalent to `new TensorFlowMediaPipeProvider().register({ mode: "inline" })`.
 
 #### Using SQLite:
 

examples/cli/src/worker_hft.ts

@@ -4,8 +4,8 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import { HFT_WORKER_JOBRUN, HFT_WORKER_JOBRUN_REGISTER } from "@workglow/ai-provider";
-import { globalServiceRegistry } from "@workglow/util";
+import { env } from "@sroussey/transformers";
+import { HFT_WORKER_JOBRUN_REGISTER } from "@workglow/ai-provider";
 
-globalServiceRegistry.get(HFT_WORKER_JOBRUN);
-console.log("worker_htf loaded", HFT_WORKER_JOBRUN_REGISTER);
+env.backends!.onnx!.wasm!.proxy = true;
+HFT_WORKER_JOBRUN_REGISTER();

examples/cli/src/workglow.ts

@@ -1,6 +1,6 @@
 #!/usr/bin/env bun
 
-import { register_HFT_InlineJobFns } from "@workglow/ai-provider";
+import { HFT_TASKS, HuggingFaceTransformersProvider } from "@workglow/ai-provider";
 import { getTaskQueueRegistry } from "@workglow/task-graph";
 import { registerHuggingfaceLocalModels } from "@workglow/test";
 import { program } from "commander";
@@ -11,10 +11,7 @@ program.version("1.0.0").description("A CLI to run tasks.");
 AddBaseCommands(program);
 
 await registerHuggingfaceLocalModels();
-await register_HFT_InlineJobFns();
-
-// await registerMediaPipeTfJsLocalModels();
-// await register_TFMP_InlineJobFns();
+await new HuggingFaceTransformersProvider(HFT_TASKS).register({ mode: "inline" });
 
 await program.parseAsync(process.argv);
 

examples/cli/src/workglow_worker.ts

@@ -1,6 +1,6 @@
 #!/usr/bin/env bun
 
-import { register_HFT_ClientJobFns } from "@workglow/ai-provider";
+import { HuggingFaceTransformersProvider } from "@workglow/ai-provider";
 import { getTaskQueueRegistry } from "@workglow/task-graph";
 import { registerHuggingfaceLocalModels } from "@workglow/test";
 import { program } from "commander";
@@ -11,14 +11,10 @@ program.version("1.0.0").description("A CLI to run tasks.");
 AddBaseCommands(program);
 
 await registerHuggingfaceLocalModels();
-await register_HFT_ClientJobFns(
-  new Worker(new URL("./worker_hft.ts", import.meta.url), { type: "module" })
-);
-
-// await registerMediaPipeTfJsLocalModels();
-// await register_TFMP_ClientJobFns(
-//   new Worker(new URL("./worker_tfmp.ts", import.meta.url), { type: "module" })
-// );
+await new HuggingFaceTransformersProvider().register({
+  mode: "worker",
+  worker: new Worker(new URL("./worker_hft.ts", import.meta.url), { type: "module" }),
+});
 
 await program.parseAsync(process.argv);
 

examples/web/src/App.tsx

@@ -4,7 +4,10 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import { register_HFT_ClientJobFns, register_TFMP_ClientJobFns } from "@workglow/ai-provider";
+import {
+  HuggingFaceTransformersProvider,
+  TensorFlowMediaPipeProvider,
+} from "@workglow/ai-provider";
 import { getTaskQueueRegistry, JsonTaskItem, TaskGraph, Workflow } from "@workglow/task-graph";
 import { JsonTask } from "@workglow/tasks";
 import {
@@ -22,12 +25,14 @@ import { GraphStoreStatus } from "./status/GraphStoreStatus";
 import { OutputRepositoryStatus } from "./status/OutputRepositoryStatus";
 import { QueuesStatus } from "./status/QueueStatus";
 
-await register_TFMP_ClientJobFns(
-  new Worker(new URL("./worker_tfmp.ts", import.meta.url), { type: "module" })
-);
-await register_HFT_ClientJobFns(
-  new Worker(new URL("./worker_hft.ts", import.meta.url), { type: "module" })
-);
+await new TensorFlowMediaPipeProvider().register({
+  mode: "worker",
+  worker: new Worker(new URL("./worker_tfmp.ts", import.meta.url), { type: "module" }),
+});
+await new HuggingFaceTransformersProvider().register({
+  mode: "worker",
+  worker: new Worker(new URL("./worker_hft.ts", import.meta.url), { type: "module" }),
+});
 
 const queueRegistry = getTaskQueueRegistry();
 queueRegistry.clearQueues();

examples/web/src/worker_hft.ts

@@ -5,9 +5,11 @@
  */
 
 import { env } from "@sroussey/transformers";
-import { HFT_WORKER_JOBRUN, HFT_WORKER_JOBRUN_REGISTER } from "@workglow/ai-provider";
-import { globalServiceRegistry } from "@workglow/util";
+import { HFT_WORKER_JOBRUN_REGISTER } from "@workglow/ai-provider";
 
-env.backends.onnx.wasm.proxy = true;
-globalServiceRegistry.get(HFT_WORKER_JOBRUN);
-console.log("worker_htf loaded", HFT_WORKER_JOBRUN_REGISTER);
+const onnx = env?.backends?.onnx;
+if (onnx) {
+  onnx.wasm!.proxy = true;
+}
+
+HFT_WORKER_JOBRUN_REGISTER();

examples/web/src/worker_tfmp.ts

@@ -4,8 +4,6 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import { TFMP_WORKER_JOBRUN, TFMP_WORKER_JOBRUN_REGISTER } from "@workglow/ai-provider";
-import { globalServiceRegistry } from "@workglow/util";
+import { TFMP_WORKER_JOBRUN_REGISTER } from "@workglow/ai-provider";
 
-globalServiceRegistry.get(TFMP_WORKER_JOBRUN);
-console.log("worker_tfmp loaded", TFMP_WORKER_JOBRUN_REGISTER);
+TFMP_WORKER_JOBRUN_REGISTER();

packages/ai-provider/README.md

@@ -41,11 +41,14 @@ npm install @mediapipe/tasks-text @mediapipe/tasks-vision @mediapipe/tasks-audio
 ### 1. Basic Setup
 
 ```typescript
-import { register_HFT_InlineJobFns, register_TFMP_InlineJobFns } from "@workglow/ai-provider";
+import {
+  HuggingFaceTransformersProvider,
+  TensorFlowMediaPipeProvider,
+} from "@workglow/ai-provider";
 
 // Register AI providers
-await register_HFT_InlineJobFns();
-await register_TFMP_InlineJobFns();
+await new HuggingFaceTransformersProvider().register({ mode: "inline" });
+await new TensorFlowMediaPipeProvider().register({ mode: "inline" });
 ```
 
 ### 2. Using AI Tasks in Workflows
@@ -234,37 +237,42 @@ For better performance, especially in browser environments, run AI inference in
 #### Main Thread Setup
 
 ```typescript
-import { register_HFT_ClientJobFns, register_TFMP_ClientJobFns } from "@workglow/ai-provider";
+import {
+  HuggingFaceTransformersProvider,
+  TensorFlowMediaPipeProvider,
+} from "@workglow/ai-provider";
 
 // Register HuggingFace Transformers with worker
-register_HFT_ClientJobFns(
-  new Worker(new URL("./hft-worker.ts", import.meta.url), { type: "module" })
-);
+await new HuggingFaceTransformersProvider().register({
+  mode: "worker",
+  worker: new Worker(new URL("./hft-worker.ts", import.meta.url), { type: "module" }),
+});
 
 // Register MediaPipe with worker
-register_TFMP_ClientJobFns(
-  new Worker(new URL("./tfmp-worker.ts", import.meta.url), { type: "module" })
-);
+await new TensorFlowMediaPipeProvider().register({
+  mode: "worker",
+  worker: new Worker(new URL("./tfmp-worker.ts", import.meta.url), { type: "module" }),
+});
 ```
 
 #### Worker Setup Files
 
 **hft-worker.ts:**
 
 ```typescript
-import { register_HFT_WorkerJobFns } from "@workglow/ai-provider";
+import { HFT_WORKER_JOBRUN_REGISTER } from "@workglow/ai-provider";
 
 // Register HuggingFace Transformers worker functions
-register_HFT_WorkerJobFns();
+HFT_WORKER_JOBRUN_REGISTER();
 ```
 
 **tfmp-worker.ts:**
 
 ```typescript
-import { register_TFMP_WorkerJobFns } from "@workglow/ai-provider";
+import { TFMP_WORKER_JOBRUN_REGISTER } from "@workglow/ai-provider";
 
 // Register MediaPipe worker functions
-register_TFMP_WorkerJobFns();
+TFMP_WORKER_JOBRUN_REGISTER();
 ```
 
 ### Model Management
@@ -286,30 +294,20 @@ await downloadTask.execute();
 ### Custom Job Queue Configuration
 
 ```typescript
-import {
-  JobQueueClient,
-  JobQueueServer,
-  ConcurrencyLimiter,
-  DelayLimiter,
-} from "@workglow/job-queue";
-import { InMemoryQueueStorage } from "@workglow/storage";
-import { register_HFT_InlineJobFns, HF_TRANSFORMERS_ONNX } from "@workglow/ai-provider";
-
-// Configure queue with custom limits
-const customQueue = new JobQueueServer(HF_TRANSFORMERS_ONNX, AiJob, {
-  storage: new InMemoryQueueStorage(HF_TRANSFORMERS_ONNX),
-  queueName: HF_TRANSFORMERS_ONNX,
-  limiter: new ConcurrencyLimiter(2, 1000), // 2 concurrent jobs, 1000ms timeout
-});
+import { HuggingFaceTransformersProvider } from "@workglow/ai-provider";
 
-const client = new JobQueueClient({
-  storage: new InMemoryQueueStorage(HF_TRANSFORMERS_ONNX),
-  queueName: HF_TRANSFORMERS_ONNX,
+// Register with custom queue concurrency (provider creates queue with concurrency: 2)
+await new HuggingFaceTransformersProvider().register({
+  mode: "inline",
+  queue: { concurrency: 2 },
 });
 
-client.attach(customQueue);
-// Register AI providers
-await register_HFT_InlineJobFns(client);
+// Or skip auto-creation and use your own queue:
+await new HuggingFaceTransformersProvider().register({
+  mode: "inline",
+  queue: { autoCreate: false },
+});
+// Then register your custom queue with getTaskQueueRegistry().registerQueue(...)
 ```
 
 ### Error Handling
@@ -352,13 +350,13 @@ await task.execute();
 ## Complete Working Example
 
 ```typescript
-import { HF_TRANSFORMERS_ONNX, register_HFT_InlineJobFns } from "@workglow/ai-provider";
-import { TextGenerationTask, TextEmbeddingTask, AiJob } from "@workglow/ai";
-import { Workflow, getTaskQueueRegistry } from "@workglow/task-graph";
+import { HuggingFaceTransformersProvider } from "@workglow/ai-provider";
+import { TextGenerationTask, TextEmbeddingTask } from "@workglow/ai";
+import { Workflow } from "@workglow/task-graph";
 
 async function main() {
   // 1. Register the AI provider
-  await register_HFT_InlineJobFns();
+  await new HuggingFaceTransformersProvider().register({ mode: "inline" });
 
   // 2. Create and run workflow
   const workflow = new Workflow();

packages/ai-provider/src/hf-transformers/HFT_Worker.ts

@@ -0,0 +1,16 @@
+/**
+ * @license
+ * Copyright 2025 Steven Roussey <sroussey@gmail.com>
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { globalServiceRegistry, parentPort, WORKER_SERVER } from "@workglow/util";
+import { HFT_TASKS } from "./common/HFT_JobRunFns";
+import { HuggingFaceTransformersProvider } from "./HuggingFaceTransformersProvider";
+
+export function HFT_WORKER_JOBRUN_REGISTER() {
+  const workerServer = globalServiceRegistry.get(WORKER_SERVER);
+  new HuggingFaceTransformersProvider(HFT_TASKS).registerOnWorkerServer(workerServer);
+  parentPort.postMessage({ type: "ready" });
+  console.log("HFT_WORKER_JOBRUN registered");
+}

packages/ai-provider/src/hf-transformers/HuggingFaceTransformersProvider.ts

@@ -0,0 +1,79 @@
+/**
+ * @license
+ * Copyright 2025 Steven Roussey <sroussey@gmail.com>
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { AiProvider, type AiProviderRegisterOptions, type AiProviderRunFn } from "@workglow/ai";
+import { HF_TRANSFORMERS_ONNX } from "./common/HFT_Constants";
+import type { HfTransformersOnnxModelConfig } from "./common/HFT_ModelSchema";
+
+/**
+ * AI provider for HuggingFace Transformers ONNX models.
+ *
+ * Supports text, vision, and multimodal tasks via the @sroussey/transformers library.
+ *
+ * Task run functions are injected via the constructor so that the heavy
+ * `@sroussey/transformers` library is only imported where actually needed
+ * (inline mode, worker server), not on the main thread in worker mode.
+ *
+ * @example
+ * ```typescript
+ * // Worker mode (main thread) -- lightweight, no heavy imports:
+ * await new HuggingFaceTransformersProvider().register({
+ *   mode: "worker",
+ *   worker: new Worker(new URL("./worker_hft.ts", import.meta.url), { type: "module" }),
+ * });
+ *
+ * // Inline mode -- caller provides the tasks:
+ * import { HFT_TASKS } from "@workglow/ai-provider";
+ * await new HuggingFaceTransformersProvider(HFT_TASKS).register({ mode: "inline" });
+ *
+ * // Worker side -- caller provides the tasks:
+ * import { HFT_TASKS } from "@workglow/ai-provider";
+ * new HuggingFaceTransformersProvider(HFT_TASKS).registerOnWorkerServer(workerServer);
+ * ```
+ */
+export class HuggingFaceTransformersProvider extends AiProvider<HfTransformersOnnxModelConfig> {
+  readonly name = HF_TRANSFORMERS_ONNX;
+
+  readonly taskTypes = [
+    "DownloadModelTask",
+    "UnloadModelTask",
+    "TextEmbeddingTask",
+    "TextGenerationTask",
+    "TextQuestionAnswerTask",
+    "TextLanguageDetectionTask",
+    "TextClassificationTask",
+    "TextFillMaskTask",
+    "TextNamedEntityRecognitionTask",
+    "TextRewriterTask",
+    "TextSummaryTask",
+    "TextTranslationTask",
+    "ImageSegmentationTask",
+    "ImageToTextTask",
+    "BackgroundRemovalTask",
+    "ImageEmbeddingTask",
+    "ImageClassificationTask",
+    "ObjectDetectionTask",
+  ] as const;
+
+  constructor(tasks?: Record<string, AiProviderRunFn<any, any, HfTransformersOnnxModelConfig>>) {
+    super(tasks);
+  }
+
+  protected override async onInitialize(options: AiProviderRegisterOptions): Promise<void> {
+    if (options.mode === "inline") {
+      const { env } = await import("@sroussey/transformers");
+      // @ts-ignore -- backends.onnx.wasm.proxy is not fully typed
+      env.backends.onnx.wasm.proxy = true;
+    }
+  }
+
+  override async dispose(): Promise<void> {
+    if (this.tasks) {
+      const { clearPipelineCache } = await import("./common/HFT_JobRunFns");
+      clearPipelineCache();
+    }
+  }
+}