[refactor] Transition from Repository to Storage or Dataset Naming Convention

- Updated the codebase to replace references from `repository` to `dataset`, enhancing clarity in data management terminology.
- Refactored various tasks and schemas to align with the new dataset structure, including `DocumentChunkDataset` and `DocumentDataset`.
- Removed deprecated chunk vector storage components and introduced new vector storage implementations for PostgreSQL and SQLite.
- Enhanced the TODO list with new items related to dataset management and improved documentation to reflect these changes.
- Updated tests to ensure compatibility with the new dataset naming and structure.

Author: sroussey

TODO.md

@@ -2,13 +2,13 @@ TODO.md
 
 - [x] Rename repositories in the packages/storage to use the word Storage instead of Repository.
 - [ ] Vector Storage (not chunk storage)
-  - [ ] Rename the files from packages/storage/src/vector-storage to packages/storage/src/vector
-  - [ ] No fixed column names, use the schema to define the columns.
+  - [x] Rename the files from packages/storage/src/vector-storage to packages/storage/src/vector
+  - [x] No fixed column names, use the schema to define the columns.
   - [ ] Option for which column to use if there are multiple, default to the first one.
   - [ ] Use @mceachen/sqlite-vec for sqlite storage.
 - [ ] Datasets Package
-  - [ ] Documents repository (mabye rename to DocumentDataset)
-  - [ ] Chunks repository (maybe rename to ChunkDataset) or DocumentChunksDataset? Or just part of DocumentDataset? Or is it a new thing?
+  - [x] Documents dataset (mabye rename to DocumentDataset)
+  - [ ] Chunks Package (or part of DocumentDataset?)
   - [ ] Move Model repository to datasets package.
 - [ ] Chunk Repository
   - [ ] Add to packages/tasks or packages/ai
@@ -17,6 +17,7 @@ TODO.md
 - [ ] Chunks and nodes are not always the same.
   - [ ] And we may need to save the chunk's node path. Or paths? or document range? Standard metadata?
 - [ ] Use Repository to always envelope the storage operations (for transactions, dealing with IDs, etc).
+- [ ] Instead of passing doc_id around, pass a document key that is unknonwn (string or object)
 
 - [ ] Get a better model for question answering.
 - [ ] Get a better model for named entity recognition, the current one recognized everything as a token, not helpful.

docs/developers/03_extending.md

@@ -138,13 +138,13 @@ When defining task input schemas, you can use `format` annotations to enable aut
 
 The system supports several format annotations out of the box:
 
-| Format                            | Description                         | Helper Function               |
-| --------------------------------- | ----------------------------------- | ----------------------------- |
-| `model`                           | Any AI model configuration          | `TypeModel()`                 |
-| `model:TaskName`                  | Model compatible with specific task | —                             |
-| `repository:tabular`              | Tabular data repository             | `TypeTabularRepository()`     |
-| `repository:document-node-vector` | Vector storage repository           | `TypeChunkVectorRepository()` |
-| `repository:document`             | Document repository                 | `TypeDocumentRepository()`    |
+| Format                         | Description                         | Helper Function               |
+| ------------------------------ | ----------------------------------- | ----------------------------- |
+| `model`                        | Any AI model configuration          | `TypeModel()`                 |
+| `model:TaskName`               | Model compatible with specific task | —                             |
+| `storage:tabular`              | Tabular data dataset                | `TypeTabularRepository()`     |
+| `dataset:document-node-vector` | Vector storage dataset              | `TypeChunkVectorRepository()` |
+| `dataset:document`             | Document dataset                    | `TypeDocumentRepository()`    |
 
 ### Example: Using Format Annotations
 
@@ -316,7 +316,7 @@ await new Workflow()
   })
   .chunkToVector()
   .vectorStoreUpsert({
-    repository: vectorRepo,
+    dataset: vectorDataset,
   })
   .run();
 ```
@@ -330,7 +330,7 @@ const answer = await new Workflow()
     model: "Xenova/all-MiniLM-L6-v2",
   })
   .vectorStoreSearch({
-    repository: vectorRepo,
+    dataset: vectorDataset,
     topK: 10,
   })
   .reranker({

packages/ai-provider/package.json

@@ -17,8 +17,8 @@
   },
   "exports": {
     ".": {
-      "bun": "./dist/index.js",
-      "types": "./dist/types.d.ts",
+      "bun": "./src/index.ts",
+      "types": "./src/types.ts",
       "import": "./dist/index.js"
     }
   },

packages/ai/package.json

@@ -23,8 +23,8 @@
     ".": {
       "react-native": "./dist/browser.js",
       "browser": "./dist/browser.js",
-      "bun": "./dist/bun.js",
-      "types": "./dist/types.d.ts",
+      "bun": "./src/bun.ts",
+      "types": "./src/types.ts",
       "import": "./dist/node.js"
     }
   },

packages/ai/src/model/ModelRegistry.ts

@@ -59,7 +59,7 @@ async function resolveModelFromRegistry(
 
   if (Array.isArray(id)) {
     const results = await Promise.all(id.map((i) => modelRepo.findByName(i)));
-    return results.filter((model) => model !== undefined) as ModelConfig[];
+    return results.filter((model): model is NonNullable<typeof model> => model !== undefined);
   }
 
   const model = await modelRepo.findByName(id);

packages/ai/src/task/ChunkRetrievalTask.ts

@@ -4,7 +4,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import { AnyChunkVectorStorage, TypeChunkVectorRepository } from "@workglow/dataset";
+import { DocumentChunkDataset, TypeDocumentChunkDataset } from "@workglow/dataset";
 import {
   CreateWorkflow,
   IExecuteContext,
@@ -25,7 +25,7 @@ import { TextEmbeddingTask } from "./TextEmbeddingTask";
 const inputSchema = {
   type: "object",
   properties: {
-    repository: TypeChunkVectorRepository({
+    dataset: TypeDocumentChunkDataset({
       title: "Document Chunk Vector Repository",
       description: "The document chunk vector repository instance to search in",
     }),
@@ -72,14 +72,14 @@ const inputSchema = {
       default: false,
     },
   },
-  required: ["repository", "query"],
+  required: ["dataset", "query"],
   if: {
     properties: {
       query: { type: "string" },
     },
   },
   then: {
-    required: ["repository", "query", "model"],
+    required: ["dataset", "query", "model"],
   },
   additionalProperties: false,
 } as const satisfies DataPortSchema;
@@ -162,7 +162,7 @@ export class DocumentNodeRetrievalTask extends Task<
 
   async execute(input: RetrievalTaskInput, context: IExecuteContext): Promise<RetrievalTaskOutput> {
     const {
-      repository,
+      dataset,
       query,
       topK = 5,
       filter,
@@ -172,7 +172,7 @@ export class DocumentNodeRetrievalTask extends Task<
     } = input;
 
     // Repository is resolved by input resolver system before execution
-    const repo = repository as AnyChunkVectorStorage;
+    const repo = dataset as DocumentChunkDataset;
 
     // Determine query vector
     let queryVector: TypedArray;

packages/ai/src/task/ChunkVectorHybridSearchTask.ts

@@ -4,7 +4,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import { AnyChunkVectorStorage, TypeChunkVectorRepository } from "@workglow/dataset";
+import { DocumentChunkDataset, TypeDocumentChunkDataset } from "@workglow/dataset";
 import {
   CreateWorkflow,
   IExecuteContext,
@@ -22,7 +22,7 @@ import {
 const inputSchema = {
   type: "object",
   properties: {
-    repository: TypeChunkVectorRepository({
+    dataset: TypeDocumentChunkDataset({
       title: "Document Chunk Vector Repository",
       description:
         "The document chunk vector repository instance to search in (must support hybridSearch)",
@@ -71,7 +71,7 @@ const inputSchema = {
       default: false,
     },
   },
-  required: ["repository", "queryVector", "queryText"],
+  required: ["dataset", "queryVector", "queryText"],
   additionalProperties: false,
 } as const satisfies DataPortSchema;
 
@@ -160,7 +160,7 @@ export class ChunkVectorHybridSearchTask extends Task<
     context: IExecuteContext
   ): Promise<HybridSearchTaskOutput> {
     const {
-      repository,
+      dataset,
       queryVector,
       queryText,
       topK = 10,
@@ -171,11 +171,11 @@ export class ChunkVectorHybridSearchTask extends Task<
     } = input;
 
     // Repository is resolved by input resolver system before execution
-    const repo = repository as AnyChunkVectorStorage;
+    const repo = dataset as DocumentChunkDataset;
 
     // Check if repository supports hybrid search
     if (!repo.hybridSearch) {
-      throw new Error("Repository does not support hybrid search.");
+      throw new Error("Dataset does not support hybrid search.");
     }
 
     // Convert to Float32Array for repository search (repo expects Float32Array by default)

packages/ai/src/task/ChunkVectorSearchTask.ts

@@ -4,7 +4,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import { AnyChunkVectorStorage, TypeChunkVectorRepository } from "@workglow/dataset";
+import { DocumentChunkDataset, TypeDocumentChunkDataset } from "@workglow/dataset";
 import {
   CreateWorkflow,
   IExecuteContext,
@@ -22,7 +22,7 @@ import {
 const inputSchema = {
   type: "object",
   properties: {
-    repository: TypeChunkVectorRepository({
+    dataset: TypeDocumentChunkDataset({
       title: "Vector Repository",
       description: "The vector repository instance to search in",
     }),
@@ -51,7 +51,7 @@ const inputSchema = {
       default: 0,
     },
   },
-  required: ["repository", "query"],
+  required: ["dataset", "query"],
   additionalProperties: false,
 } as const satisfies DataPortSchema;
 
@@ -103,7 +103,7 @@ export type VectorStoreSearchTaskInput = FromSchema<typeof inputSchema, TypedArr
 export type VectorStoreSearchTaskOutput = FromSchema<typeof outputSchema, TypedArraySchemaOptions>;
 
 /**
- * Task for searching similar vectors in a vector repository.
+ * Task for searching similar vectors in a document chunk dataset.
  * Returns top-K most similar vectors with their metadata and scores.
  */
 export class ChunkVectorSearchTask extends Task<
@@ -114,7 +114,7 @@ export class ChunkVectorSearchTask extends Task<
   public static type = "ChunkVectorSearchTask";
   public static category = "Vector Store";
   public static title = "Vector Store Search";
-  public static description = "Search for similar vectors in a vector repository";
+  public static description = "Search for similar vectors in a document chunk dataset";
   public static cacheable = true;
 
   public static inputSchema(): DataPortSchema {
@@ -129,9 +129,9 @@ export class ChunkVectorSearchTask extends Task<
     input: VectorStoreSearchTaskInput,
     context: IExecuteContext
   ): Promise<VectorStoreSearchTaskOutput> {
-    const { repository, query, topK = 10, filter, scoreThreshold = 0 } = input;
+    const { dataset, query, topK = 10, filter, scoreThreshold = 0 } = input;
 
-    const repo = repository as AnyChunkVectorStorage;
+    const repo = dataset as DocumentChunkDataset;
 
     const results = await repo.similaritySearch(query, {
       topK,

packages/ai/src/task/ChunkVectorUpsertTask.ts

@@ -4,7 +4,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import { AnyChunkVectorStorage, TypeChunkVectorRepository } from "@workglow/dataset";
+import { DocumentChunkDataset, TypeDocumentChunkDataset } from "@workglow/dataset";
 import {
   CreateWorkflow,
   IExecuteContext,
@@ -23,7 +23,7 @@ import { TypeSingleOrArray } from "./base/AiTaskSchemas";
 const inputSchema = {
   type: "object",
   properties: {
-    repository: TypeChunkVectorRepository({
+    dataset: TypeDocumentChunkDataset({
       title: "Document Chunk Vector Repository",
       description: "The document chunk vector repository instance to store vectors in",
     }),
@@ -45,7 +45,7 @@ const inputSchema = {
       additionalProperties: true,
     }),
   },
-  required: ["repository", "doc_id", "vectors", "metadata"],
+  required: ["dataset", "doc_id", "vectors", "metadata"],
   additionalProperties: false,
 } as const satisfies DataPortSchema;
 
@@ -91,7 +91,7 @@ export class ChunkVectorUpsertTask extends Task<
   public static type = "ChunkVectorUpsertTask";
   public static category = "Vector Store";
   public static title = "Vector Store Upsert";
-  public static description = "Store vector embeddings with metadata in a vector repository";
+  public static description = "Store vector embeddings with metadata in a document chunk dataset";
   public static cacheable = false; // Has side effects
 
   public static inputSchema(): DataPortSchema {
@@ -106,15 +106,15 @@ export class ChunkVectorUpsertTask extends Task<
     input: VectorStoreUpsertTaskInput,
     context: IExecuteContext
   ): Promise<VectorStoreUpsertTaskOutput> {
-    const { repository, doc_id, vectors, metadata } = input;
+    const { dataset, doc_id, vectors, metadata } = input;
 
     // Normalize inputs to arrays
     const vectorArray = Array.isArray(vectors) ? vectors : [vectors];
     const metadataArray = Array.isArray(metadata)
       ? metadata
       : Array(vectorArray.length).fill(metadata);
 
-    const repo = repository as AnyChunkVectorStorage;
+    const repo = dataset as DocumentChunkDataset;
 
     await context.updateProgress(1, "Upserting vectors");
 

packages/ai/src/task/HierarchicalChunkerTask.ts

@@ -9,7 +9,6 @@ import {
   estimateTokens,
   getChildren,
   hasChildren,
-  NodeIdGenerator,
   type ChunkNode,
   type DocumentNode,
   type TokenBudget,
@@ -21,7 +20,7 @@ import {
   Task,
   Workflow,
 } from "@workglow/task-graph";
-import { DataPortSchema, FromSchema } from "@workglow/util";
+import { DataPortSchema, FromSchema, uuid4 } from "@workglow/util";
 
 const inputSchema = {
   type: "object",
@@ -207,7 +206,7 @@ export class HierarchicalChunkerTask extends Task<
 
     if (estimateTokens(text) <= tokenBudget.maxTokensPerChunk - tokenBudget.reservedTokens) {
       // Text fits in one chunk
-      const chunkId = await NodeIdGenerator.generateChunkId(doc_id, leafNodeId, 0);
+      const chunkId = uuid4();
       chunks.push({
         chunkId,
         doc_id,
@@ -226,7 +225,7 @@ export class HierarchicalChunkerTask extends Task<
       const endOffset = Math.min(startOffset + maxChars, text.length);
       const chunkText = text.substring(startOffset, endOffset);
 
-      const chunkId = await NodeIdGenerator.generateChunkId(doc_id, leafNodeId, chunkOrdinal);
+      const chunkId = uuid4();
 
       chunks.push({
         chunkId,