diff --git a/dataLayer/schemaDescribe.ts b/dataLayer/schemaDescribe.ts index 88b260e85..2e849eead 100644 --- a/dataLayer/schemaDescribe.ts +++ b/dataLayer/schemaDescribe.ts @@ -158,6 +158,7 @@ async function descTable(describeTableObject: any, attrPerms?: any) { assigned_updated_time: att.assignUpdatedTime, nullable: att.nullable, computed: att.computed ? true : undefined, // only include if computed + embed: att.embed ? { source: att.embed.source, model: att.embed.model } : undefined, properties: att.properties ? att.properties.map((prop) => { return { type: prop.type, name: prop.name }; diff --git a/integrationTests/server/embed-directive.test.ts b/integrationTests/server/embed-directive.test.ts new file mode 100644 index 000000000..b0c17ab0d --- /dev/null +++ b/integrationTests/server/embed-directive.test.ts @@ -0,0 +1,473 @@ +/** + * `@embed` directive integration test. + * + * Spins up a fake Ollama HTTP server inside the test, points Harper's models + * config at it, deploys a schema with `@embed`, and exercises these paths + * end-to-end: + * + * 1. **Happy path** — POST a record → fake-ollama returns a deterministic + * vector → record stores the vector at the `@embed`-decorated field. + * + * 2. **Source-unchanged PATCH** — PATCH a record with a non-source field → + * no new embed call is made (fake-ollama hit count stays flat); the + * existing embedding survives via patch-merge. + * + * 3. **Source-changing PATCH** — PATCH a record with the source field → + * embed fires once, the stored vector matches the NEW content. + * + * 4. **PUT (full-record update)** — embed fires once, stored vector reflects new content. + * + * 5. **Replication-receiver skip** — POST with `x-replicate-from: none` and a + * pre-supplied vector → no embed call; the supplied vector is stored as-is. + * + * 6. **Caching-table `@embed`** — GET fires `getFromSource`; the cache write + * (which bypasses `_writeUpdate`) ends up with a populated vector. + * + * The fake-ollama server returns deterministic 3-element vectors derived from the + * input text so assertions can compare exact values. + */ +import { suite, test, before, after } from 'node:test'; +import { strictEqual, ok } from 'node:assert/strict'; +import { createServer, type IncomingMessage, type ServerResponse, type Server } from 'node:http'; +import type { AddressInfo } from 'node:net'; +import { startHarper, teardownHarper } from '@harperfast/integration-testing'; +// .mjs siblings — TypeScript needs `// @ts-expect-error` because no declaration files exist +// @ts-expect-error utils/client.mjs has no type declarations; runtime resolves fine +import { createApiClient } from '../apiTests/utils/client.mjs'; +// @ts-expect-error utils/lifecycle.mjs has no type declarations; runtime resolves fine +import { restartHttpWorkers } from '../apiTests/utils/lifecycle.mjs'; +import request from 'supertest'; + +const SCHEMA_GRAPHQL = [ + 'type EmbedDoc @table(database: "embedtest") @sealed @export {', + '\tid: ID! @primaryKey', + '\tcontent: String', + '\ttag: String', + '\tembedding: [Float] @embed(source: "content", model: "default")', + '}', + '', + 'type CachedEmbedDoc @table(database: "embedtest") @sealed @export {', + '\tid: ID! @primaryKey', + '\tcontent: String', + '\tembedding: [Float] @embed(source: "content", model: "default")', + '}', + '', +].join('\n'); + +// resources.js wiring for the caching-table @embed path. The Resource base class +// and the `databases` global are provided by Harper's component loader at boot. +const RESOURCES_JS = [ + 'const { CachedEmbedDoc } = databases.embedtest;', + '', + 'export class CachedEmbedSource extends Resource {', + '\tasync get() {', + '\t\tconst id = this.getId();', + '\t\treturn { id, content: `derived content for ${id}` };', + '\t}', + '}', + '', + 'CachedEmbedDoc.sourcedFrom(CachedEmbedSource);', + '', +].join('\n'); + +/** Vectors are stored and returned as a plain JSON number[]. */ +function decodeVector(field: any): number[] | undefined { + return Array.isArray(field) ? field.map(Number) : undefined; +} + +interface FakeOllama { + url: string; + host: string; // form for Harper's `host:` config field + close: () => Promise; + embedCallCount: () => number; + lastEmbedInputs: () => string[][]; + reset: () => void; +} + +/** + * Deterministic fake embedder: maps an input string to a stable 3-element + * vector so assertions can compare exact values. + */ +function deterministicVector(input: string): number[] { + let h1 = 0; + let h2 = 0; + let h3 = 0; + for (let i = 0; i < input.length; i++) { + const c = input.charCodeAt(i); + h1 = (h1 * 31 + c) % 9973; + h2 = (h2 * 37 + c) % 9967; + h3 = (h3 * 41 + c) % 9941; + } + // Normalize to (0, 1) range + return [h1 / 9973, h2 / 9967, h3 / 9941]; +} + +async function startFakeOllama(): Promise { + let embedCalls = 0; + const embedInputs: string[][] = []; + const server: Server = createServer((req: IncomingMessage, res: ServerResponse) => { + if (req.method === 'POST' && req.url === '/api/embed') { + let body = ''; + req.on('data', (chunk) => (body += chunk)); + req.on('end', () => { + try { + const parsed = JSON.parse(body) as { model: string; input: string | string[] }; + const inputs = Array.isArray(parsed.input) ? parsed.input : [parsed.input]; + embedCalls++; + embedInputs.push(inputs); + const embeddings = inputs.map((s) => deterministicVector(s)); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ embeddings, prompt_eval_count: inputs.join(' ').length })); + } catch (err) { + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: String(err) })); + } + }); + return; + } + res.writeHead(404); + res.end(); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const addr = server.address() as AddressInfo; + const url = `http://127.0.0.1:${addr.port}`; + const host = `127.0.0.1:${addr.port}`; + return { + url, + host, + close: () => new Promise((resolve, reject) => server.close((err) => (err ? reject(err) : resolve()))), + embedCallCount: () => embedCalls, + lastEmbedInputs: () => embedInputs, + reset: () => { + embedCalls = 0; + embedInputs.length = 0; + }, + }; +} + +suite('@embed directive end-to-end with fake Ollama', (ctx: any) => { + let fake: FakeOllama; + let client: any; + + before(async () => { + fake = await startFakeOllama(); + // Local debugging: setting HARPER_INTEGRATION_TEST_FORCE_LOOPBACK forces the + // 127.0.0.1 fast-path so the test runs without `harper-integration-test-setup-loopback` + // (macOS dev machines don't have the pool by default). CI uses the pool normally. + if (process.env.HARPER_INTEGRATION_TEST_FORCE_LOOPBACK) { + ctx.harper = { ...ctx.harper, hostname: '127.0.0.1' }; + } + await startHarper(ctx, { + config: { + logging: { auditLog: true }, + models: { + embedding: { + default: { backend: 'ollama', host: fake.host, model: 'fake-embed' }, + }, + }, + }, + env: {}, + }); + client = createApiClient(ctx.harper); + + await client + .req() + .send({ operation: 'add_component', project: 'embedtest' }) + .expect((r: any) => { + const text = JSON.stringify(r.body); + ok(text.includes('Successfully added project') || text.includes('Project already exists'), r.text); + }); + + await client + .req() + .send({ operation: 'set_component_file', project: 'embedtest', file: 'schema.graphql', payload: SCHEMA_GRAPHQL }) + .expect((r: any) => ok(r.body?.message?.includes?.('Successfully set component: schema.graphql'), r.text)) + .expect(200); + + await client + .req() + .send({ operation: 'set_component_file', project: 'embedtest', file: 'resources.js', payload: RESOURCES_JS }) + .expect((r: any) => ok(r.body?.message?.includes?.('Successfully set component: resources.js'), r.text)) + .expect(200); + + await restartHttpWorkers(client, '/openapi'); + fake.reset(); + }); + + after(async () => { + try { + await teardownHarper(ctx); + } finally { + await fake.close(); + } + }); + + test('schema with @embed creates EmbedDoc table', async () => { + const desc = await client.req().send({ operation: 'describe_all' }).expect(200); + const embedDoc = desc.body?.embedtest?.EmbedDoc; + ok(embedDoc, 'EmbedDoc table not created'); + const embeddingAttr = (embedDoc.attributes || []).find((a: any) => a.attribute === 'embedding'); + ok(embeddingAttr, 'embedding attribute should be present'); + strictEqual(embeddingAttr.type, 'array', 'embedding type should be a [Float] array'); + strictEqual(embeddingAttr.indexed?.type, 'HNSW', 'embedding should be auto-HNSW-indexed'); + // describe should surface the @embed config, not just the auto-added HNSW index + ok(embeddingAttr.embed, 'describe should surface @embed config'); + strictEqual(embeddingAttr.embed.source, 'content'); + strictEqual(embeddingAttr.embed.model, 'default'); + }); + + test('happy path: POST → embedder runs → vector stored on record', async () => { + fake.reset(); + const content = 'harper is a database'; + const expected = deterministicVector(content); + + await request(ctx.harper.httpURL) + .post('/EmbedDoc/') + .set(client.headers) + .send({ id: 'doc-happy', content }) + .expect((r: any) => ok([200, 201, 204].includes(r.status), `unexpected status ${r.status}: ${r.text}`)); + + // Verify the fake-ollama received exactly one embed call with the source text + strictEqual(fake.embedCallCount(), 1, 'expected exactly one embed call'); + const inputs = fake.lastEmbedInputs()[0]; + strictEqual(inputs.length, 1); + ok(inputs[0].includes(content), `embed input "${inputs[0]}" should contain source text`); + + // GET the record back and verify the embedding was stored + const getResp = await client.reqRest('/EmbedDoc/doc-happy').expect(200); + const body = getResp.body as { id: string; content: string; embedding: unknown }; + strictEqual(body.id, 'doc-happy'); + strictEqual(body.content, content); + const stored = decodeVector(body.embedding); + ok(stored, `embedding field should be populated, got: ${JSON.stringify(body.embedding)}`); + strictEqual(stored.length, 3, 'expected 3-element vector'); + for (let i = 0; i < 3; i++) { + ok( + Math.abs(stored[i] - expected[i]) < 1e-5, + `vector[${i}] mismatch: stored=${stored[i]} expected=${expected[i]}` + ); + } + }); + + test('PATCH unrelated field does NOT re-run embedder; existing embedding survives', async () => { + // Seed a record first + const content = 'patch baseline content'; + await request(ctx.harper.httpURL) + .post('/EmbedDoc/') + .set(client.headers) + .send({ id: 'doc-patch', content }) + .expect((r: any) => ok([200, 201, 204].includes(r.status), `seed POST status ${r.status}: ${r.text}`)); + + const baselineEmbedCalls = fake.embedCallCount(); + + // PATCH a non-source field. The embed hook's source-presence predicate should skip; + // no new embed call should fire and the existing vector should remain unchanged. + await request(ctx.harper.httpURL) + .patch('/EmbedDoc/doc-patch') + .set(client.headers) + .send({ tag: 'updated' }) + .expect((r: any) => ok([200, 204].includes(r.status), `PATCH status ${r.status}: ${r.text}`)); + + strictEqual( + fake.embedCallCount(), + baselineEmbedCalls, + 'embed should not fire when the source field is not in the PATCH payload' + ); + + // Verify the embedding is still the one from the original content + const expected = deterministicVector(content); + const getResp = await client.reqRest('/EmbedDoc/doc-patch').expect(200); + const body = getResp.body as { tag: string; embedding: unknown }; + strictEqual(body.tag, 'updated'); + const stored = decodeVector(body.embedding); + ok(stored, `embedding should still be populated after non-source PATCH, got: ${JSON.stringify(body.embedding)}`); + strictEqual(stored.length, 3); + for (let i = 0; i < 3; i++) { + ok(Math.abs(stored[i] - expected[i]) < 1e-5, 'existing embedding should survive non-source PATCH'); + } + }); + + test('PATCH source field DOES re-run embedder; stored vector matches new content', async () => { + // Seed a record with one content value. + const initialContent = 'patch-source baseline'; + await request(ctx.harper.httpURL) + .post('/EmbedDoc/') + .set(client.headers) + .send({ id: 'doc-source-patch', content: initialContent }) + .expect((r: any) => ok([200, 201, 204].includes(r.status), `seed POST status ${r.status}: ${r.text}`)); + + const baselineEmbedCalls = fake.embedCallCount(); + + // PATCH the source field — exercises the async `update()` path where the embed + // promise must be awaited before commit. + const updatedContent = 'patch-source updated text'; + await request(ctx.harper.httpURL) + .patch('/EmbedDoc/doc-source-patch') + .set(client.headers) + .send({ content: updatedContent }) + .expect((r: any) => ok([200, 204].includes(r.status), `PATCH status ${r.status}: ${r.text}`)); + + strictEqual( + fake.embedCallCount(), + baselineEmbedCalls + 1, + 'embed should fire exactly once when the source field is in the PATCH payload' + ); + const inputs = fake.lastEmbedInputs().at(-1)!; + ok( + inputs.some((s) => s.includes(updatedContent)), + `embed input ${JSON.stringify(inputs)} should reflect the updated content` + ); + + // Verify the stored vector matches the NEW content, not the seed. + const expected = deterministicVector(updatedContent); + const getResp = await client.reqRest('/EmbedDoc/doc-source-patch').expect(200); + const body = getResp.body as { id: string; content: string; embedding: unknown }; + strictEqual(body.content, updatedContent, 'PATCH should have updated the source field'); + const stored = decodeVector(body.embedding); + ok(stored, `embedding should be populated after source PATCH, got: ${JSON.stringify(body.embedding)}`); + strictEqual(stored.length, 3); + for (let i = 0; i < 3; i++) { + ok( + Math.abs(stored[i] - expected[i]) < 1e-5, + `vector[${i}] mismatch: stored=${stored[i]} expected (from new content)=${expected[i]}` + ); + } + }); + + test('PUT (full-record update) on existing row re-runs embedder', async () => { + // PUT exercises the SAME legacy-URLSearchParams branch in Table.ts that PATCH does + // (REST dispatches as `resource.put(data, query)`; the query is a URLSearchParams, + // so put() takes the back-compat branch that — pre-fix — dropped update()'s promise). + await request(ctx.harper.httpURL) + .post('/EmbedDoc/') + .set(client.headers) + .send({ id: 'doc-put-source', content: 'put baseline' }) + .expect((r: any) => ok([200, 201, 204].includes(r.status), `seed POST status ${r.status}: ${r.text}`)); + + const baselineEmbedCalls = fake.embedCallCount(); + const updatedContent = 'put updated text'; + + await request(ctx.harper.httpURL) + .put('/EmbedDoc/doc-put-source') + .set(client.headers) + .send({ id: 'doc-put-source', content: updatedContent }) + .expect((r: any) => ok([200, 204].includes(r.status), `PUT status ${r.status}: ${r.text}`)); + + strictEqual( + fake.embedCallCount(), + baselineEmbedCalls + 1, + 'embed should fire exactly once on a PUT that includes the source field' + ); + + const expected = deterministicVector(updatedContent); + const getResp = await client.reqRest('/EmbedDoc/doc-put-source').expect(200); + const body = getResp.body as { content: string; embedding: unknown }; + strictEqual(body.content, updatedContent); + const stored = decodeVector(body.embedding); + ok(stored, `embedding should be populated after PUT, got: ${JSON.stringify(body.embedding)}`); + for (let i = 0; i < 3; i++) { + ok(Math.abs(stored[i] - expected[i]) < 1e-5, `PUT vector[${i}] mismatch: ${stored[i]} vs ${expected[i]}`); + } + }); + + test('replication-receiver: POST with x-replicate-from:none + supplied vector → embedder skipped', async () => { + const content = 'replicated record content'; + const suppliedVector = [0.111, 0.222, 0.333]; + const baselineEmbedCalls = fake.embedCallCount(); + + await request(ctx.harper.httpURL) + .post('/EmbedDoc/') + .set({ ...client.headers, 'x-replicate-from': 'none' }) + .send({ id: 'doc-replica', content, embedding: suppliedVector }) + .expect((r: any) => ok([200, 201, 204].includes(r.status), `replica POST status ${r.status}: ${r.text}`)); + + strictEqual( + fake.embedCallCount(), + baselineEmbedCalls, + 'embed should NOT fire on a write with x-replicate-from: none (receiver context)' + ); + + const getResp = await client.reqRest('/EmbedDoc/doc-replica').expect(200); + const body = getResp.body as { id: string; content: string; embedding: unknown }; + strictEqual(body.id, 'doc-replica'); + strictEqual(body.content, content); + const stored = decodeVector(body.embedding); + ok(stored, `embedding should be populated from supplied vector, got: ${JSON.stringify(body.embedding)}`); + strictEqual(stored.length, 3); + // The receiver must preserve the originator's vector — NOT overwrite with what + // it would have computed locally. Compare against suppliedVector, not against + // deterministicVector(content). + for (let i = 0; i < 3; i++) { + ok( + Math.abs(stored[i] - suppliedVector[i]) < 1e-5, + `receiver stored ${stored[i]} but should be the originator's ${suppliedVector[i]}` + ); + } + }); + + test('caching table with @embed: GET fires source → cache write embeds → vector stored', async () => { + // GET on a caching-sourced table triggers `getFromSource`; Harper writes the + // resolved record into the cache via a path that bypasses `_writeUpdate` but + // still runs the embed hook. This test exercises that path. + const id = 'cached-1'; + const expectedContent = `derived content for ${id}`; + const expectedVector = deterministicVector(expectedContent); + const baselineEmbedCalls = fake.embedCallCount(); + + // First GET: cache miss → source resolves → cache write fires embed hook. + const firstGet = await client.reqRest(`/CachedEmbedDoc/${id}`).expect(200); + const firstBody = firstGet.body as { id: string; content: string; embedding: unknown }; + strictEqual(firstBody.id, id); + strictEqual(firstBody.content, expectedContent, 'cache should reflect the source-returned content'); + + // The embedder must have run exactly once during the cache write. + strictEqual( + fake.embedCallCount(), + baselineEmbedCalls + 1, + 'embed should fire exactly once when populating a caching table from source' + ); + const inputs = fake.lastEmbedInputs().at(-1)!; + ok( + inputs.some((s) => s.includes(expectedContent)), + `embed input ${JSON.stringify(inputs)} should reflect the source content` + ); + + // Stored row must have the embedding. The GET response above may not surface the + // embedding column for sourced-table reads, so verify against an authoritative + // search_by_hash on the underlying table. + // `getFromSource` resolves the GET BEFORE the cache write's txn commits (see + // `Table.ts:~4275` design comment — "we don't want to wait for the transaction"). + // On a slow CI runner the txn commit can land AFTER our next search_by_hash, so + // poll a few times with a short backoff rather than assert on first read. + let search: any; + for (let attempt = 0; attempt < 10; attempt++) { + search = await client + .req() + .send({ + operation: 'search_by_hash', + database: 'embedtest', + table: 'CachedEmbedDoc', + hash_values: [id], + get_attributes: ['*'], + }) + .expect(200); + if (Array.isArray(search.body) && search.body.length === 1) break; + await new Promise((resolve) => setTimeout(resolve, 25)); + } + ok(Array.isArray(search.body) && search.body.length === 1, `search_by_hash body: ${JSON.stringify(search.body)}`); + const stored = decodeVector(search.body[0].embedding); + ok(stored, `embedding should be populated on the cached row, got: ${JSON.stringify(search.body[0].embedding)}`); + strictEqual(stored.length, 3); + for (let i = 0; i < 3; i++) { + ok( + Math.abs(stored[i] - expectedVector[i]) < 1e-5, + `cached-row vector[${i}] mismatch: stored=${stored[i]} expected=${expectedVector[i]}` + ); + } + + // Second GET: cache hit. The embedder must NOT fire again. + const callsAfterFirst = fake.embedCallCount(); + await client.reqRest(`/CachedEmbedDoc/${id}`).expect(200); + strictEqual(fake.embedCallCount(), callsAfterFirst, 'cache-hit GET should not re-run the embedder'); + }); +}); diff --git a/resources/Table.ts b/resources/Table.ts index 03cfdf4e5..44ad666c5 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -47,6 +47,7 @@ import { transaction, contextStorage } from './transaction.ts'; import { MAXIMUM_KEY, writeKey, compareKeys } from 'ordered-binary'; import { getWorkerIndex, getWorkerCount } from '../server/threads/manageThreads.js'; import { HAS_BLOBS, auditRetention, removeAuditEntry } from './auditStore.ts'; +import { buildEmbedBefore, createDefaultEmbedder, type EmbedAttribute, type Embedder } from './models/embedHook.ts'; import { autoCast, autoCastBooleanStrict } from '../utility/common_utils.ts'; import { recordUpdater, @@ -85,6 +86,8 @@ export type Attribute = { computed?: any; resolve?: any; computedFromExpression?: any; + embed?: { source: string; model: string }; + version?: any; properties?: Array; elements?: Attribute; sealed?: boolean; @@ -256,6 +259,11 @@ export function makeTable(options) { static updatedTimeProperty = updatedTimeProperty; static propertyResolvers; static userResolvers = {}; + // `@embed` hook registry. `userSetEmbedders` records names set explicitly via + // `setEmbedAttribute` so a schema reload refreshes defaults without clobbering them. + static userEmbedders: { [name: string]: Embedder } = {}; + static userSetEmbedders: Set = new Set(); + static embedAttributes: EmbedAttribute[] = (attributes as any[]).filter((a) => a?.embed); static source?: typeof TableResource; declare static sourceOptions: any; declare static intermediateSource: boolean; @@ -1234,14 +1242,14 @@ export function makeTable(options) { } return when(loading, () => { this.#changes = updates; - this._writeUpdate(id, this.#changes, false); - return this; + // `when` awaits the embed hook (when `@embed` is active) before resolving, + // so the caller's `save()` doesn't run before the write is staged. + return when(this._writeUpdate(id, this.#changes, false), () => this); }); }); } } - this._writeUpdate(id, this.#changes, fullUpdate); - return this; + return when(this._writeUpdate(id, this.#changes, fullUpdate), () => this); } /** @@ -1249,17 +1257,17 @@ export function makeTable(options) { */ save() { if (this.#savingOperation) { - const promiseOrResult = this.#savingOperation.promise || this.#savingOperation.result; - const transaction = txnForContext(this.getContext()); - if (transaction.save) { - try { - return transaction.save(this.#savingOperation) || promiseOrResult; - } finally { - this.#savingOperation = null; - } + try { + return this.#saveOperation(this.#savingOperation); + } finally { + this.#savingOperation = null; } } } + #saveOperation(operation: any) { + const transaction = txnForContext(this.getContext()); + if (transaction.save) return transaction.save(operation) || operation.promise || operation.result; + } addTo(property: any, value: any) { if (typeof value === 'number' || typeof value === 'bigint') { @@ -1509,9 +1517,9 @@ export function makeTable(options) { record: Record & RecordObject ): void | (Record & Partial) | Promise)> { if (record === undefined || record instanceof URLSearchParams) { - // legacy argument position, shift the arguments and go through the update method for back-compat - (this as any).update(target, true); - return this.save() as any; + // legacy argument position, shift the arguments and go through the update method for back-compat. + // `when` settles the embed hook before `save()` so the write is staged first. + return when((this as any).update(target, true), () => this.save() as any) as any; } else { let allowed = true; if (target == undefined) throw new TypeError('Can not put a record without a target'); @@ -1526,17 +1534,21 @@ export function makeTable(options) { } // standard path, handle arrays as multiple updates, and otherwise do a direct update if (Array.isArray(record)) { - return Promise.all( - record.map((element) => { - const id = element[primaryKey]; - this._writeUpdate(id, element, true); - return this.save() as any; - }) - ) as any; + // Capture each element's operation synchronously (before any async `@embed` + // hook resolves): `#savingOperation` is a single field that parallel writes + // would otherwise clobber, so a deferred `save()` would commit the wrong op + // — e.g. one element's save running before a later element's vector is written. + const writes = record.map((element) => { + const id = element[primaryKey]; + const writePromise = this._writeUpdate(id, element, true); + const operation = this.#savingOperation; + return when(writePromise, () => this.#saveOperation(operation)); + }); + this.#savingOperation = null; + return Promise.all(writes) as any; } else { const id = requestTargetToId(target as any); - this._writeUpdate(id, record, true); - return this.save() as any; + return when(this._writeUpdate(id, record, true), () => this.save() as any); } }) as any; } @@ -1575,8 +1587,10 @@ export function makeTable(options) { throw new ClientError('Record already exists', 409); } } - this._writeUpdate(id, record, true); - return record; + // `_writeUpdate` may return a promise when an `@embed` directive + // requires running an embedder before the per-write `commit(...)` + // closure. `when()` passes through synchronous returns. + return when(this._writeUpdate(id, record, true), () => record); }) as any; } @@ -1586,9 +1600,9 @@ export function makeTable(options) { recordUpdate: Partial ): void | (Record & Partial) | Promise)> { if (recordUpdate === undefined || recordUpdate instanceof URLSearchParams) { - // legacy argument position, shift the arguments and go through the update method for back-compat - (this as any).update(target, false); - return this.save() as any; + // legacy argument position, shift the arguments and go through the update method for back-compat. + // `when` settles the embed hook before `save()` so the write is staged first. + return when(this.update(target, false), () => this.save() as any) as any; } else { // standard path, ensure there is no return object return when(this.update(target, recordUpdate), () => { @@ -1603,7 +1617,6 @@ export function makeTable(options) { _writeUpdate(id: Id, recordUpdate: any, fullUpdate: boolean, options?: any) { const context = this.getContext(); const transaction = txnForContext(context); - checkValidId(id); const entry = this.#entry ?? primaryStore.getEntry(id, { transaction: transaction.getReadTxn() }); const writeToSource = () => { @@ -1952,8 +1965,27 @@ export function makeTable(options) { }, }; this.#savingOperation = write; - write.beforeIntermediate = preCommitBlobsForRecordBefore(write, recordUpdate); - return transaction.addWrite(write as any); + // `@embed` hook must run before `addWrite` so the embedder's vector is on the + // record when `commit` runs. (The txn `before` slot runs after commit, which + // suits blob writes but not embedding, where the vector must be present at commit.) + // Known limitation of this write-time placement (a validate-time alternative was + // tried and reverted as a Harper-foreign pattern): the embedder sees this write's + // payload, before table validation — so a write that later fails validation still + // calls the backend, and a tracked-instance mutation (update(id,{}); row.source=…; + // save()) that sets the source via accessors after update() won't re-embed. A + // resource-layer re-embed is the proper fix; tracked as a follow-up. + const embedBefore = buildEmbedBefore( + recordUpdate, + context, + options, + TableResource.embedAttributes, + TableResource.userEmbedders + ); + const proceed = (): any => { + write.beforeIntermediate = preCommitBlobsForRecordBefore(write, recordUpdate); + return transaction.addWrite(write as any); + }; + return embedBefore ? embedBefore().then(proceed) : proceed(); } async delete(target: RequestTargetOrId): Promise { @@ -3368,6 +3400,14 @@ export function makeTable(options) { * When attributes have been changed, we update the accessors that are assigned to this table */ static updatedAttributes() { + // Refresh on every call: schema reload mutates `attributes` in place, so the + // class-construction snapshot would otherwise go stale. + this.embedAttributes = (this.attributes as any[]).filter((a) => a?.embed); + // Drop registry entries for attributes that are no longer `@embed`, so a dropped + // directive doesn't leave a stale embedder or block a default refresh on re-add. + const embedNames = new Set(this.embedAttributes.map((a) => a.name)); + for (const name of Object.keys(this.userEmbedders)) if (!embedNames.has(name)) delete this.userEmbedders[name]; + for (const name of this.userSetEmbedders) if (!embedNames.has(name)) this.userSetEmbedders.delete(name); propertyResolvers = this.propertyResolvers = { $id: (object, context, entry) => ({ value: entry.key }), $updatedtime: (object, context, entry) => entry.version, @@ -3383,6 +3423,11 @@ export function makeTable(options) { attribute.resolve = null; // reset this const relationship = attribute.relationship; const computed = attribute.computed; + // Register the default embedder unless an author override is set. Sits outside + // the resolver chain below so `@embed` fields still flow through auto-HNSW indexing. + if (attribute.embed && !TableResource.userSetEmbedders.has(attribute.name)) { + this.userEmbedders[attribute.name] = createDefaultEmbedder(attribute.embed); + } if (relationship) { if (attribute.indexed) { console.error( @@ -3566,6 +3611,25 @@ export function makeTable(options) { } this.userResolvers[attribute_name] = resolver; } + /** + * Override the default embedder for an `@embed` attribute. Return the vector to + * store at `attribute_name`. The embedder receives the write payload (the fields + * present in the PUT/PATCH body), not the post-merge record, so multi-field + * concatenation only works when all source fields are in the same write. + */ + static setEmbedAttribute(attribute_name: string, embedder: Embedder): void { + const attribute = findAttribute(attributes, attribute_name); + if (!attribute) { + console.error(`The attribute "${attribute_name}" does not exist in the table "${tableName}"`); + return; + } + if (!attribute.embed) { + console.error(`The attribute "${attribute_name}" is not declared with @embed in the table "${tableName}"`); + return; + } + this.userEmbedders[attribute_name] = embedder; + this.userSetEmbedders.add(attribute_name); + } static async deleteHistory(endTime = 0, cleanupDeletedRecords = false) { let completion: Promise; for (const auditRecord of auditStore.getRange({ @@ -4428,6 +4492,19 @@ export function makeTable(options) { } }, }; + // The cache-from-source write bypasses `_writeUpdate`, so wire the embed hook here + // too (always the originating node). It runs after the client GET has resolved with + // fresh source data, so it's a background commit: an embedder failure aborts the cache + // write via the outer error handler (row re-embeds next read) and never reaches the + // caller. Source-resolution errors are handled earlier, with the stale-data fallback. + const embedBefore = buildEmbedBefore( + updatedRecord, + sourceContext, + undefined, + TableResource.embedAttributes, + TableResource.userEmbedders + ); + if (embedBefore) await embedBefore(); sourceWrite.before = preCommitBlobsForRecordBefore(sourceWrite, updatedRecord); dbTxn.addWrite(sourceWrite); }), diff --git a/resources/databases.ts b/resources/databases.ts index 44f185ad3..43f224753 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -1124,6 +1124,7 @@ export function table(tableDefinition: TableDefinition): Tabl // note that non-indexed attributes do not need a dbi if (attributeDescriptor?.attribute && !attributeDescriptor.name) attributeDescriptor.indexed = true; // legacy descriptor + // Include `embed` so a source/model change refreshes the embed registry. const changed = !attributeDescriptor || attributeDescriptor.type !== attribute.type || @@ -1132,7 +1133,8 @@ export function table(tableDefinition: TableDefinition): Tabl attributeDescriptor.version !== attribute.version || attributeDescriptor.enumerable !== attribute.enumerable || JSON.stringify(attributeDescriptor.properties) !== JSON.stringify(attribute.properties) || - JSON.stringify(attributeDescriptor.elements) !== JSON.stringify(attribute.elements); + JSON.stringify(attributeDescriptor.elements) !== JSON.stringify(attribute.elements) || + JSON.stringify(attributeDescriptor.embed) !== JSON.stringify(attribute.embed); if (attribute.indexed) { const dbi = openIndex(dbiKey, rootStore, attribute); if ( diff --git a/resources/graphql.ts b/resources/graphql.ts index f70672838..56697e1d9 100644 --- a/resources/graphql.ts +++ b/resources/graphql.ts @@ -5,6 +5,7 @@ import { getWorkerIndex } from '../server/threads/manageThreads.js'; import { Resources } from './Resources.ts'; import type { NamedTypeNode, StringValueNode } from 'graphql'; import { once } from 'node:events'; +import { ClientError } from '../utility/errors/hdbError.ts'; const PRIMITIVE_TYPES = ['ID', 'Int', 'Float', 'Long', 'String', 'Boolean', 'Date', 'Bytes', 'Any', 'BigInt', 'Blob']; @@ -18,6 +19,7 @@ server.knownGraphQLDirectives.push( 'primaryKey', 'indexed', 'computed', + 'embed', 'relationship', 'createdTime', 'updatedTime', @@ -143,6 +145,32 @@ async function processGraphQLSchema(gqlContent, urlPath, filePath, resources) { } } property.computed = property.computed || true; + } else if (directiveName === 'embed') { + // `@embed(source, model)`: on write, embed `record[source]` into this + // attribute and auto-index it with HNSW. + const embedDefinition: { source?: string; model?: string } = {}; + for (const arg of directive.arguments || []) { + if (arg.value.kind !== 'StringValue') + throw new ClientError( + `@embed(${arg.name.value}: ...) on "${property.name}" expects a string literal`, + 400 + ); + embedDefinition[arg.name.value] = (arg.value as StringValueNode).value; + } + if (!embedDefinition.source || !embedDefinition.model) { + const loc = directive.loc; + throw new ClientError( + `@embed on "${property.name}" requires both "source" and "model" arguments` + + (loc ? ` (line ${loc.startToken?.line ?? '?'}, column ${loc.startToken?.column ?? '?'})` : ''), + 400 + ); + } else { + property.embed = embedDefinition; + // Version carries the model so a model change triggers a reindex (re-index only, not re-embed). + if (property.version == undefined) { + property.version = `embed:${embedDefinition.model}`; + } + } } else if (directiveName === 'relationship') { const relationshipDefinition = {}; for (const arg of directive.arguments) { @@ -168,6 +196,35 @@ async function processGraphQLSchema(gqlContent, urlPath, filePath, resources) { console.warn(`@${directiveName} is an unknown directive, at`, directive.loc); } } + // @embed targets a vector column and auto-indexes it with HNSW; resolved after all + // directives so an explicit @indexed (in any order) is honored. The target must be an + // array (e.g. [Float]) — a scalar would store the vector wrong and HNSW-index a non-vector. + if (property.embed) { + const elementType = + property.type === 'array' ? (property as { elements?: { type?: string } }).elements?.type : undefined; + if (property.type !== 'array' || elementType !== 'Float') + throw new ClientError( + `@embed on "${property.name}" requires a [Float] attribute type; got "${property.type === 'array' ? `[${elementType ?? '?'}]` : property.type}"`, + 400 + ); + if (!property.indexed) property.indexed = { type: 'HNSW' }; + else if ((property.indexed as { type?: string }).type !== 'HNSW') + throw new ClientError( + `@embed on "${property.name}" auto-indexes with HNSW; remove the conflicting @indexed or set @indexed(type: "HNSW")`, + 400 + ); + } + } + // @embed source must reference a declared field; a typo would silently leave + // the vector column unpopulated (the source key never appears in write payloads). + for (const prop of properties as any[]) { + // Object.hasOwn (not `in`): `attributesObject` is a plain object, so `in` would + // match inherited prototype keys (toString, constructor) and pass a bad source. + if (prop.embed && !Object.hasOwn(attributesObject, prop.embed.source)) + throw new ClientError( + `@embed on "${prop.name}" references unknown source field "${prop.embed.source}"`, + 400 + ); } typeDef.type = typeName; } diff --git a/resources/models/embedHook.ts b/resources/models/embedHook.ts new file mode 100644 index 000000000..1aa728a92 --- /dev/null +++ b/resources/models/embedHook.ts @@ -0,0 +1,138 @@ +/** + * `@embed` directive write-time hook. `createDefaultEmbedder` builds the embedder + * a table registers for an `@embed` attribute; `buildEmbedBefore` produces the + * pre-commit callback that runs registered embedders and writes their vectors onto + * the record before it commits. + */ + +// Lazily resolved to avoid a require cycle on the unit-test load path; only needed on failure. +function getLogger(): { error?: (...args: any[]) => void } { + try { + // eslint-disable-next-line @typescript-eslint/no-var-requires + return require('#src/utility/logging/logger').logger ?? {}; + } catch { + return {}; + } +} + +export type EmbedConfig = { + source: string; + model: string; +}; + +export type EmbedAttribute = { + name: string; + embed: EmbedConfig; +}; + +export type Embedder = (record: any) => Promise; + +// Matches the public `Models.embed` signature; a named type so tests can inject a fake. +type EmbedFn = ( + input: string | string[], + opts: { model?: string; inputType?: 'document' | 'query' } +) => Promise; + +// Lazy-imported so this module can be unit-tested without loading the transaction +// stack `Models.ts` pulls in. Overridable via `__setEmbedFnForTest`. +let _embedFn: EmbedFn | undefined; +function resolveEmbedFn(): EmbedFn { + if (_embedFn) return _embedFn; + const { Models } = require('#src/resources/models/Models'); // eslint-disable-line @typescript-eslint/no-var-requires + const models = new Models(); + _embedFn = (input, opts) => models.embed(input, opts); + return _embedFn; +} + +/** Test seam: override the embed function. Pass `undefined` to reset to `Models.embed`. */ +export function __setEmbedFnForTest(fn: EmbedFn | undefined): void { + _embedFn = fn; +} + +export function createDefaultEmbedder(embedConfig: EmbedConfig): Embedder { + const { source, model } = embedConfig; + return async (record: any): Promise => { + const sourceValue = record?.[source]; + if (sourceValue == null) return null; + const vectors = await resolveEmbedFn()(String(sourceValue), { + model, + inputType: 'document', + }); + const v = vectors?.[0]; + if (v == null) return undefined; + // Store as a plain array — typed arrays don't round-trip through the record encoder. + return v instanceof Float32Array ? Array.from(v) : Array.from(v as any); + }; +} + +/** + * Build the pre-commit callback that runs embedders for every `@embed` attribute whose + * source field is present in this write. Returns `undefined` when there's nothing to do + * (no `@embed` attributes, a replication-receiver write, or no source field in the payload), + * so the call site can skip it. + * + * Source-field semantics: embed only when the source field is in the payload. A PATCH that + * omits it leaves the existing vector untouched; an explicit `source: null` clears the vector. + */ +export function buildEmbedBefore( + record: any, + context: any, + options: any, + embedAttributes: EmbedAttribute[] | undefined, + userEmbedders: Record +): (() => Promise) | undefined { + if (!embedAttributes || embedAttributes.length === 0) return undefined; + // Skip when the write already carries the vector: cluster-replication receiver + // (isNotification), REST x-replicate-from:none, or audit-log replay. Note that + // proactive source-subscribe pushes also set isNotification and so skip embedding. + if (options?.isNotification === true || context?.replicateFrom === false || context?.alreadyLogged === true) { + return undefined; + } + if (!record || typeof record !== 'object') return undefined; + let anySourcePresent = false; + for (const attr of embedAttributes) { + const sourceKey = attr.embed?.source; + if (sourceKey && sourceKey in record) { + anySourcePresent = true; + break; + } + } + if (!anySourcePresent) return undefined; + return async (): Promise => { + // Parallel: each embedder mutates a distinct attribute, so there's no ordering hazard. + await Promise.all( + embedAttributes.map(async (attr) => { + const sourceKey = attr.embed?.source; + if (!sourceKey) return; + if (!(sourceKey in record)) return; + const sourceValue = record[sourceKey]; + if (sourceValue == null) { + record[attr.name] = null; + return; + } + // CRDT op payloads (`{__op__, value}`) aren't a meaningful embed source; skip. + if (sourceValue && typeof sourceValue === 'object' && (sourceValue as any).__op__) return; + const embedder = userEmbedders[attr.name]; + if (!embedder) return; + let vector; + try { + vector = await embedder(record); + } catch (err) { + // Backend errors can carry URLs / key tails; log raw, rethrow sanitized. + getLogger().error?.(`Embedder for attribute "${attr.name}" failed:`, err); + throw new Error(`Failed to compute embedding for attribute "${attr.name}"`); + } + record[attr.name] = normalizeVector(vector); + }) + ); + }; +} + +// Custom embedders may return any typed array; flatten to a plain array so it round-trips +// through the record encoder. NaN is left for HNSW to reject at index time. +function normalizeVector(vector: any): number[] | null { + if (vector == null) return null; + if (Array.isArray(vector)) return vector; + if (ArrayBuffer.isView(vector)) return Array.from(vector as any); + return vector; +} diff --git a/unitTests/resources/models/embedBulkWrite.test.js b/unitTests/resources/models/embedBulkWrite.test.js new file mode 100644 index 000000000..469522016 --- /dev/null +++ b/unitTests/resources/models/embedBulkWrite.test.js @@ -0,0 +1,61 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const { setTimeout: delay } = require('node:timers/promises'); +const { setupTestDBPath } = require('../../testUtils'); +const { table } = require('#src/resources/databases'); +const { transaction } = require('#src/resources/transaction'); +const { setMainIsWorker } = require('#js/server/threads/manageThreads'); + +// A loadAsInstance:false table reaches Table.put's array branch directly. With @embed the +// embed hook is async, so the parallel writes must not share the single #savingOperation +// slot: if element A's embed resolves first and save() reads the slot, it must save A's op +// (whose vector is written), NOT a later element's op whose embed is still pending. +describe('@embed bulk array write — staggered embed timing (loadAsInstance:false)', () => { + if (process.env.HARPER_STORAGE_ENGINE === 'lmdb') return; + let T; + before(() => { + setupTestDBPath(); + setMainIsWorker(true); + T = table({ + table: 'EmbedBulkRace', + database: 'test', + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'content', type: 'String' }, + { name: 'embedding', type: 'Array', embed: { source: 'content', model: 'default' }, indexed: { type: 'HNSW' } }, + ], + }); + T.loadAsInstance = false; + // Stagger: the first element's embed resolves immediately; the second's resolves later. + // This forces the "A done, B still pending" interleaving that exposes a shared save slot. + T.setEmbedAttribute('embedding', async (record) => { + const n = String(record.content ?? '').length; + if (String(record.content).startsWith('slow')) await delay(40); + return [n, n, n]; + }); + }); + + it('commits each element with its own vector regardless of embed completion order', async () => { + await transaction((context) => + T.put( + [ + { id: 'fast', content: 'ab' }, // embed resolves immediately -> [2,2,2] + { id: 'slow', content: 'slow-content' }, // embed resolves after a delay -> [12,12,12] + ], + context + ) + ); + + const fast = await T.get('fast'); + const slow = await T.get('slow'); + assert.equal(fast?.content, 'ab'); + assert.deepEqual([...(fast?.embedding ?? [])], [2, 2, 2], 'fast element vector'); + assert.equal(slow?.content, 'slow-content'); + assert.deepEqual( + [...(slow?.embedding ?? [])], + [12, 12, 12], + 'slow element vector must be written before its op is saved' + ); + }); +}); diff --git a/unitTests/resources/models/embedDirective.test.js b/unitTests/resources/models/embedDirective.test.js new file mode 100644 index 000000000..229ba5919 --- /dev/null +++ b/unitTests/resources/models/embedDirective.test.js @@ -0,0 +1,122 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const { setupTestDBPath } = require('../../testUtils'); +const { loadGQLSchema } = require('#src/resources/graphql'); + +// Parse-level behavior of the `@embed` directive: auto-HNSW attach and the +// loud-fail when paired with a conflicting (non-HNSW) explicit index. +describe('@embed directive parsing', () => { + before(() => setupTestDBPath()); + + it('auto-attaches an HNSW index when @embed has no explicit @indexed', async () => { + await loadGQLSchema(`type EmbedAuto @table { + id: ID @primaryKey + content: String + embedding: [Float] @embed(source: "content", model: "default") + }`); + const attr = tables.EmbedAuto.attributes.find((a) => a.name === 'embedding'); + assert.equal(attr.indexed?.type, 'HNSW', 'embedding should be auto-HNSW-indexed'); + }); + + it('accepts @embed with an explicit HNSW @indexed', async () => { + await assert.doesNotReject( + loadGQLSchema(`type EmbedHnsw @table { + id: ID @primaryKey + content: String + embedding: [Float] @embed(source: "content", model: "default") @indexed(type: "HNSW") + }`) + ); + }); + + it('rejects @embed on a non-array (scalar) attribute type (loud-fail, 400)', async () => { + await assert.rejects( + loadGQLSchema(`type EmbedScalar @table { + id: ID @primaryKey + content: String + embedding: String @embed(source: "content", model: "default") + }`), + (err) => { + assert.equal(err.statusCode, 400, 'should be a client error'); + assert.match(err.message, /\[Float\]/, 'message should name the [Float] requirement'); + return true; + } + ); + }); + + it('rejects @embed on a non-Float array element type (e.g. [String]) (loud-fail, 400)', async () => { + await assert.rejects( + loadGQLSchema(`type EmbedStrArr @table { + id: ID @primaryKey + content: String + embedding: [String] @embed(source: "content", model: "default") + }`), + (err) => { + assert.equal(err.statusCode, 400, 'should be a client error'); + assert.match(err.message, /\[Float\]/, 'message should name the [Float] requirement'); + return true; + } + ); + }); + + it('rejects @embed with a non-string-literal argument (loud-fail, 400)', async () => { + await assert.rejects( + loadGQLSchema(`type EmbedNonStr @table { + id: ID @primaryKey + content: String + embedding: [Float] @embed(source: 123, model: "default") + }`), + (err) => { + assert.equal(err.statusCode, 400, 'should be a client error'); + assert.match(err.message, /string literal/, 'message should name the string-literal requirement'); + return true; + } + ); + }); + + it('rejects @embed whose source references an unknown field (loud-fail, 400)', async () => { + await assert.rejects( + loadGQLSchema(`type EmbedBadSource @table { + id: ID @primaryKey + content: String + embedding: [Float] @embed(source: "contnet", model: "default") + }`), + (err) => { + assert.equal(err.statusCode, 400, 'should be a client error'); + assert.match(err.message, /unknown source field|contnet/, 'message should name the unknown source'); + return true; + } + ); + }); + + it('rejects @embed whose source is a prototype key (e.g. "toString") (loud-fail, 400)', async () => { + // `source in attributesObject` would match Object.prototype keys; Object.hasOwn must not. + await assert.rejects( + loadGQLSchema(`type EmbedProto @table { + id: ID @primaryKey + content: String + embedding: [Float] @embed(source: "toString", model: "default") + }`), + (err) => { + assert.equal(err.statusCode, 400, 'should be a client error'); + assert.match(err.message, /unknown source field|toString/, 'message should name the bad source'); + return true; + } + ); + }); + + it('rejects @embed combined with a non-HNSW @indexed (loud-fail, 400)', async () => { + await assert.rejects( + loadGQLSchema(`type EmbedBtree @table { + id: ID @primaryKey + content: String + embedding: [Float] @embed(source: "content", model: "default") @indexed(type: "BTREE") + }`), + (err) => { + assert.equal(err.statusCode, 400, 'should be a client error'); + assert.match(err.message, /HNSW/, 'message should name the HNSW requirement'); + return true; + } + ); + }); +}); diff --git a/unitTests/resources/models/embedHook.test.js b/unitTests/resources/models/embedHook.test.js new file mode 100644 index 000000000..6d68b9a12 --- /dev/null +++ b/unitTests/resources/models/embedHook.test.js @@ -0,0 +1,241 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const { buildEmbedBefore, createDefaultEmbedder, __setEmbedFnForTest } = require('#src/resources/models/embedHook'); + +const VECTOR_F32 = new Float32Array([0.1, 0.2, 0.3]); +// Embedders can return any of the `Embedder` shapes (Float32Array, Array, etc.). +// `buildEmbedBefore` normalizes the output to `Array` for storage — the record +// encoder mangles typed arrays via `updateAndFreeze`, so we flatten at the boundary. +// `VECTOR` is what's STORED; `VECTOR_F32` is what the embedder may RETURN. +const VECTOR = Array.from(VECTOR_F32); + +function fakeEmbedCapturing() { + const calls = []; + const fn = async (input, opts) => { + calls.push({ input, opts }); + return [VECTOR]; + }; + fn.calls = calls; + return fn; +} + +describe('embedHook', () => { + describe('createDefaultEmbedder', () => { + afterEach(() => __setEmbedFnForTest(undefined)); + + it('reads source field, calls Models.embed with document inputType, returns first vector as Array', async () => { + const embedFn = fakeEmbedCapturing(); + __setEmbedFnForTest(embedFn); + const embedder = createDefaultEmbedder({ source: 'content', model: 'default' }); + const vec = await embedder({ content: 'hello world' }); + // Default embedder converts Float32Array → Array so Harper's record + // encoder doesn't mangle it via `updateAndFreeze`. HNSW accepts both. + assert.ok(Array.isArray(vec), 'vec should be a plain Array'); + assert.deepEqual(vec, Array.from(VECTOR)); + assert.equal(embedFn.calls.length, 1); + assert.equal(embedFn.calls[0].input, 'hello world'); + assert.equal(embedFn.calls[0].opts.model, 'default'); + assert.equal(embedFn.calls[0].opts.inputType, 'document'); + }); + + it('returns null when source value is null', async () => { + const embedFn = fakeEmbedCapturing(); + __setEmbedFnForTest(embedFn); + const embedder = createDefaultEmbedder({ source: 'content', model: 'default' }); + assert.equal(await embedder({ content: null }), null); + assert.equal(embedFn.calls.length, 0); + }); + + it('returns null when source value is undefined', async () => { + const embedFn = fakeEmbedCapturing(); + __setEmbedFnForTest(embedFn); + const embedder = createDefaultEmbedder({ source: 'content', model: 'default' }); + assert.equal(await embedder({}), null); + assert.equal(embedFn.calls.length, 0); + }); + + it('stringifies non-string source values before passing to embed()', async () => { + const embedFn = fakeEmbedCapturing(); + __setEmbedFnForTest(embedFn); + const embedder = createDefaultEmbedder({ source: 'count', model: 'default' }); + await embedder({ count: 42 }); + assert.equal(embedFn.calls[0].input, '42'); + }); + }); + + describe('buildEmbedBefore', () => { + const attrs = [{ name: 'embedding', embed: { source: 'content', model: 'default' } }]; + + it('returns undefined when embedAttributes is empty', () => { + assert.equal(buildEmbedBefore({ content: 'x' }, {}, {}, [], {}), undefined); + assert.equal(buildEmbedBefore({ content: 'x' }, {}, {}, undefined, {}), undefined); + }); + + it('returns undefined on cluster-replication receive (options.isNotification === true)', () => { + const before = buildEmbedBefore({ content: 'x' }, {}, { isNotification: true }, attrs, { + embedding: async () => VECTOR, + }); + assert.equal(before, undefined); + }); + + it('returns undefined on REST x-replicate-from: none (context.replicateFrom === false)', () => { + const before = buildEmbedBefore({ content: 'x' }, { replicateFrom: false }, {}, attrs, { + embedding: async () => VECTOR, + }); + assert.equal(before, undefined); + }); + + it('DOES fire on a local-originating write where replicateFrom is undefined', () => { + // Originating writes have undefined replicateFrom (not false); make sure the predicate + // does not over-skip and silently drop local embedding work. + const before = buildEmbedBefore({ content: 'x' }, {}, {}, attrs, { + embedding: async () => VECTOR, + }); + assert.ok(before, 'embedder should fire on local-originating writes'); + }); + + it('returns undefined on replay context (alreadyLogged === true)', () => { + const before = buildEmbedBefore({ content: 'x' }, { alreadyLogged: true }, {}, attrs, { + embedding: async () => VECTOR, + }); + assert.equal(before, undefined); + }); + + it('returns undefined when no embed-source field is in the write payload (patch that omits source)', () => { + const before = buildEmbedBefore({ otherField: 'unchanged' }, {}, {}, attrs, { + embedding: async () => VECTOR, + }); + assert.equal(before, undefined); + }); + + it('returns undefined when record is not an object', () => { + assert.equal(buildEmbedBefore(null, {}, {}, attrs, { embedding: async () => VECTOR }), undefined); + assert.equal(buildEmbedBefore(undefined, {}, {}, attrs, { embedding: async () => VECTOR }), undefined); + }); + + it('runs the embedder and writes vector to the target attribute when source is present', async () => { + const record = { content: 'hello' }; + const before = buildEmbedBefore(record, {}, {}, attrs, { + embedding: async (r) => { + assert.equal(r.content, 'hello'); + return VECTOR; + }, + }); + assert.ok(before); + await before(); + assert.deepEqual(record.embedding, VECTOR); + }); + + it('clears the embedding to null when source is explicitly null', async () => { + const record = { content: null }; + let called = false; + const before = buildEmbedBefore(record, {}, {}, attrs, { + embedding: async () => { + called = true; + return VECTOR; + }, + }); + assert.ok(before); + await before(); + assert.equal(record.embedding, null); + assert.equal(called, false, 'embedder should not run when source is null'); + }); + + it('skips attributes whose source is not in the payload (multi-attribute table)', async () => { + const multiAttrs = [ + { name: 'embA', embed: { source: 'titleField', model: 'default' } }, + { name: 'embB', embed: { source: 'bodyField', model: 'default' } }, + ]; + const record = { bodyField: 'b' }; // only bodyField is in this patch + let embACalls = 0; + let embBCalls = 0; + const before = buildEmbedBefore(record, {}, {}, multiAttrs, { + embA: async () => { + embACalls++; + return VECTOR; + }, + embB: async () => { + embBCalls++; + return VECTOR; + }, + }); + assert.ok(before); + await before(); + assert.equal(embACalls, 0, 'embA source not in payload, skipped'); + assert.equal(embBCalls, 1, 'embB source in payload, fired'); + assert.equal(record.embA, undefined); + assert.deepEqual(record.embB, VECTOR); + }); + + it('skips an attribute that has no registered embedder', async () => { + const record = { content: 'hello' }; + const before = buildEmbedBefore(record, {}, {}, attrs, {}); // no embedder + assert.ok(before); + await before(); + assert.equal(record.embedding, undefined, 'no vector written when no embedder is registered'); + }); + + it('writes null to the target when the embedder returns null/undefined', async () => { + const record = { content: 'hello' }; + const before = buildEmbedBefore(record, {}, {}, attrs, { + embedding: async () => null, + }); + assert.ok(before); + await before(); + assert.equal(record.embedding, null); + }); + + it('normalizes custom-embedder typed-array output to Array for storage', async () => { + // Custom embedders may return a typed array; the hook flattens it to a plain + // array so it round-trips through the record encoder. + const record = { content: 'hello' }; + const before = buildEmbedBefore(record, {}, {}, attrs, { + embedding: async () => VECTOR_F32, // typed-array return from a custom embedder + }); + assert.ok(before); + await before(); + assert.ok(Array.isArray(record.embedding), `expected Array, got ${record.embedding?.constructor?.name}`); + assert.deepEqual(record.embedding, VECTOR); + }); + + it('skips the embed call when the source payload is a CRDT operation', async () => { + // CRDT ops (`{__op__, value}`) get unwrapped at validate-time (`Table.validate`). + // Harper today only supports numeric `add` (`resources/crdt.ts`), which isn't a + // meaningful @embed source. Sending the raw op object to the embedder would + // stringify to "[object Object]" and waste an embedder API call. + const record = { content: { __op__: 'add', value: 5 } }; + let called = false; + const before = buildEmbedBefore(record, {}, {}, attrs, { + embedding: async () => { + called = true; + return VECTOR; + }, + }); + assert.ok(before); + await before(); + assert.equal(called, false, 'embedder should not run when source value is a CRDT op'); + assert.equal(record.embedding, undefined, 'no vector should be written for CRDT-op source'); + }); + + it('propagates a sanitized error when the embedder throws', async () => { + const record = { content: 'hello' }; + const before = buildEmbedBefore(record, {}, {}, attrs, { + embedding: async () => { + throw new Error('https://internal-embed.svc:9000 401 key=sk-abc123 unauthorized'); + }, + }); + assert.ok(before); + // the embedder's raw backend message must NOT propagate as-is to the caller; the + // sanitized error should reference only the attribute name and a generic phrase. + await assert.rejects(before(), (err) => { + assert.ok(!/sk-abc123/.test(err.message), 'API key tail leaked'); + assert.ok(!/internal-embed\.svc/.test(err.message), 'internal hostname leaked'); + assert.ok(/embedding/i.test(err.message), 'error message should mention embedding'); + return true; + }); + // record.embedding should not have been written + assert.equal(record.embedding, undefined); + }); + }); +}); diff --git a/unitTests/resources/models/embedRegistry.test.js b/unitTests/resources/models/embedRegistry.test.js new file mode 100644 index 000000000..f711e9e03 --- /dev/null +++ b/unitTests/resources/models/embedRegistry.test.js @@ -0,0 +1,51 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const { setupTestDBPath } = require('../../testUtils'); +const { table } = require('#src/resources/databases'); + +// Exercises the per-table `@embed` registry on the Table class: default-embedder +// registration, the component-author override (setEmbedAttribute) surviving a schema +// reload, and stale-entry pruning when an attribute's `@embed` is dropped. +describe('@embed registry (setEmbedAttribute + schema reload)', () => { + let T; + before(() => { + setupTestDBPath(); + T = table({ + table: 'EmbedRegTest', + database: 'test', + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'content', type: 'String' }, + { name: 'embedding', type: 'Array', embed: { source: 'content', model: 'default' }, indexed: { type: 'HNSW' } }, + ], + }); + T.updatedAttributes(); + }); + + it('registers a default embedder for an @embed attribute', () => { + assert.equal(typeof T.userEmbedders.embedding, 'function'); + assert.equal(T.userSetEmbedders.has('embedding'), false, 'default registration is not marked as an override'); + }); + + it('a component-author override survives a schema reload', () => { + const custom = async () => [1, 2, 3]; + T.setEmbedAttribute('embedding', custom); + assert.equal(T.userEmbedders.embedding, custom); + assert.ok(T.userSetEmbedders.has('embedding')); + + T.updatedAttributes(); // simulate an in-place schema reload + assert.equal(T.userEmbedders.embedding, custom, 'custom embedder must not be clobbered by the default on reload'); + assert.ok(T.userSetEmbedders.has('embedding')); + }); + + it('dropping @embed prunes the registry (no stale embedder or override flag)', () => { + const attr = T.attributes.find((a) => a.name === 'embedding'); + delete attr.embed; // schema redeployed without the @embed directive + T.updatedAttributes(); + + assert.equal(T.userEmbedders.embedding, undefined, 'stale embedder must be pruned'); + assert.equal(T.userSetEmbedders.has('embedding'), false, 'stale override flag must be pruned'); + assert.equal(T.embedAttributes.length, 0, 'embedAttributes must be refreshed'); + }); +});