diff --git a/convex/catalogFeed.test.ts b/convex/catalogFeed.test.ts index baa950badd..d5fa337ad4 100644 --- a/convex/catalogFeed.test.ts +++ b/convex/catalogFeed.test.ts @@ -1,6 +1,14 @@ -import { CATALOG_FEED_ID, CATALOG_SKILLS_FEED_ID } from "clawhub-schema"; +import { CATALOG_FEED_ID, CATALOG_SKILLS_FEED_ID, type CatalogFeedEntry } from "clawhub-schema"; import { beforeEach, describe, expect, it, vi } from "vitest"; -import { listOfficialEntries, listOfficialSkillEntries, publish } from "./catalogFeed"; +import { + __test, + listChanges, + listOfficialEntries, + listOfficialSkillEntries, + pruneCatalogFeedHistoryInternal, + publish, + storePublication, +} from "./catalogFeed"; vi.mock("./lib/publishers", () => ({ getOwnerPublisher: vi.fn().mockResolvedValue({ handle: "openclaw" }), @@ -28,6 +36,38 @@ const listOfficialSkillEntriesHandler = ( const publishHandler = ( publish as unknown as WrappedHandler<{ expiresAt: string }, Array<{ feedId: string }>> )._handler; +const storePublicationHandler = ( + storePublication as unknown as WrappedHandler< + { + feedId: typeof CATALOG_FEED_ID | typeof CATALOG_SKILLS_FEED_ID; + description: string; + generatedAt: string; + expiresAt: string; + entries: unknown[]; + }, + { sequence: number; entryCount: number } + > +)._handler; +const listChangesHandler = ( + listChanges as unknown as WrappedHandler< + { + feedId: typeof CATALOG_FEED_ID; + fromSequence: number; + toSequence: number; + paginationOpts: { cursor: string | null; numItems: number }; + }, + { + resetRequired: boolean; + page?: Array<{ sequence: number; ordinal: number; payload: string }>; + } + > +)._handler; +const pruneCatalogFeedHistoryHandler = ( + pruneCatalogFeedHistoryInternal as unknown as WrappedHandler< + { batchSize?: number }, + { deleted: number; hasMore: boolean } + > +)._handler; function makePackage(overrides: Record = {}) { return { @@ -110,7 +150,7 @@ function makeGitHubSource(overrides: Record = {}) { }; } -function makeFeedSkillEntry(index: number) { +function makeFeedSkillEntry(index: number): CatalogFeedEntry { const id = `@openclaw/demo-${index.toString().padStart(3, "0")}`; return { type: "skill", @@ -189,6 +229,216 @@ describe("catalog feed projection", () => { vi.clearAllMocks(); }); + it("builds deterministic complete revision changes", () => { + const previous = makeFeedSkillEntry(1); + const replacement = { ...previous, title: "Updated" }; + const removed = makeFeedSkillEntry(2); + expect( + __test.buildCatalogFeedChanges({ + sequence: 7, + previousEntries: [removed, previous], + nextEntries: [replacement], + previousDescription: "Official", + nextDescription: "Official", + }), + ).toEqual([ + { sequence: 7, operation: "remove", entryType: "skill", entryId: removed.id }, + { sequence: 7, operation: "upsert", entry: replacement }, + ]); + expect( + __test.buildCatalogFeedChanges({ + sequence: 8, + previousEntries: [replacement], + nextEntries: [replacement], + previousDescription: "Official", + nextDescription: "Official", + }), + ).toEqual([{ sequence: 8, operation: "metadata", metadata: { description: "Official" } }]); + }); + + it("stores a revision and its journal rows with the current publication", async () => { + const insert = vi.fn(async (table: string, _value: Record) => `${table}:1`); + const patch = vi.fn(); + const eq = vi.fn().mockReturnThis(); + const ctx = { + db: { + query: vi.fn((table: string) => ({ + withIndex: vi.fn((_index: string, apply: (q: { eq: typeof eq }) => unknown) => { + apply({ eq }); + if (table === "catalogFeedRevisions") { + return { order: vi.fn(() => ({ first: vi.fn(async () => null) })) }; + } + return { unique: vi.fn(async () => null) }; + }), + })), + insert, + patch, + replace: vi.fn(), + delete: vi.fn(), + get: vi.fn(), + normalizeId: vi.fn(), + system: { get: vi.fn(), query: vi.fn() }, + }, + }; + const result = await storePublicationHandler(ctx, { + feedId: CATALOG_SKILLS_FEED_ID, + description: "Official", + generatedAt: "2026-07-16T00:00:00.000Z", + expiresAt: "2026-07-16T01:00:00.000Z", + entries: [makeFeedSkillEntry(1)], + }); + + expect(result).toMatchObject({ sequence: 1, entryCount: 1 }); + expect(insert).toHaveBeenCalledWith( + "catalogFeedRevisions", + expect.objectContaining({ + feedId: CATALOG_SKILLS_FEED_ID, + sequence: 1, + changeCount: 2, + cumulativeChangeCount: 2, + }), + ); + const journalRows = insert.mock.calls.filter(([table]) => table === "catalogFeedChanges"); + expect(journalRows).toHaveLength(2); + expect(journalRows.map(([, row]) => JSON.parse(String(row.payload)).operation)).toEqual([ + "upsert", + "metadata", + ]); + expect(patch).not.toHaveBeenCalled(); + }); + + it("reads only the requested bounded change range", async () => { + const builder = { + eq: vi.fn().mockReturnThis(), + gt: vi.fn().mockReturnThis(), + lte: vi.fn().mockReturnThis(), + }; + const paginate = vi.fn(async () => ({ + page: [ + { + sequence: 4, + ordinal: 0, + payload: '{"operation":"metadata"}', + expirationTime: 1, + }, + ], + isDone: true, + continueCursor: "", + })); + const query = vi.fn((table: string) => { + if (table === "catalogFeedRevisions") { + return { + withIndex: vi.fn((_index: string, apply?: (q: typeof builder) => unknown) => { + apply?.(builder); + return { + order: vi.fn((direction: "asc" | "desc") => ({ + first: vi.fn(async () => + direction === "asc" + ? { sequence: 4, changeCount: 2, cumulativeChangeCount: 2 } + : { sequence: 5, changeCount: 1, cumulativeChangeCount: 3 }, + ), + })), + unique: vi.fn(async () => ({ + sequence: 4, + changeCount: 2, + cumulativeChangeCount: 2, + })), + }; + }), + }; + } + return { + withIndex: vi.fn((_index: string, apply: (q: typeof builder) => unknown) => { + apply(builder); + return { paginate }; + }), + }; + }); + const result = await listChangesHandler( + { + db: { query }, + }, + { + feedId: CATALOG_FEED_ID, + fromSequence: 3, + toSequence: 4, + paginationOpts: { cursor: null, numItems: 100 }, + }, + ); + + expect(builder.gt).toHaveBeenCalledWith("sequence", 3); + expect(builder.lte).toHaveBeenCalledWith("sequence", 4); + expect(result).toMatchObject({ + resetRequired: false, + retainedFromSequence: 3, + currentSequence: 5, + changeCount: 2, + page: [{ sequence: 4, ordinal: 0, payload: '{"operation":"metadata"}' }], + }); + + const laterRange = await listChangesHandler( + { db: { query } }, + { + feedId: CATALOG_FEED_ID, + fromSequence: 4, + toSequence: 5, + paginationOpts: { cursor: null, numItems: 100 }, + }, + ); + expect(laterRange).toMatchObject({ resetRequired: false, changeCount: 1 }); + + const reset = await listChangesHandler( + { db: { query } }, + { + feedId: CATALOG_FEED_ID, + fromSequence: 2, + toSequence: 4, + paginationOpts: { cursor: null, numItems: 100 }, + }, + ); + expect(reset).toEqual({ + resetRequired: true, + retainedFromSequence: 3, + currentSequence: 5, + }); + expect(paginate).toHaveBeenCalledTimes(2); + }); + + it("prunes catalog history in bounded continuation batches", async () => { + const expiredRevisions = [{ _id: "catalogFeedRevisions:1" }, { _id: "catalogFeedRevisions:2" }]; + const take = vi.fn(async () => expiredRevisions); + const delete_ = vi.fn(); + const runAfter = vi.fn(); + const result = await pruneCatalogFeedHistoryHandler( + { + db: { + query: vi.fn(() => ({ + withIndex: vi.fn( + (_index: string, apply: (q: { lt: ReturnType }) => unknown) => { + const q = { lt: vi.fn().mockReturnThis() }; + apply(q); + return { take }; + }, + ), + })), + insert: vi.fn(), + patch: vi.fn(), + replace: vi.fn(), + delete: delete_, + get: vi.fn(), + normalizeId: vi.fn(), + system: { get: vi.fn(), query: vi.fn() }, + }, + scheduler: { runAfter }, + }, + { batchSize: 2 }, + ); + + expect(result).toEqual({ deleted: 2, hasMore: true }); + expect(delete_).toHaveBeenCalledTimes(2); + expect(runAfter).toHaveBeenCalledWith(0, expect.anything(), { batchSize: 2 }); + }); + it("projects official releases into ClawHub install candidates", async () => { const result = await listOfficialEntriesHandler( makeCtx([makePackage()], { diff --git a/convex/catalogFeed.ts b/convex/catalogFeed.ts index 9bd884b579..441f96e964 100644 --- a/convex/catalogFeed.ts +++ b/convex/catalogFeed.ts @@ -5,16 +5,20 @@ import { CATALOG_FEED_SOURCE_REF, CATALOG_SKILLS_FEED_DESCRIPTION, CATALOG_SKILLS_FEED_ID, + parseCatalogFeed, PROMOTIONS_FEED_ID, serializeCatalogFeed, + type CatalogFeedChange, type CatalogFeedEntry, type CatalogFeedSkillEntry, } from "clawhub-schema"; +import { paginationOptsValidator } from "convex/server"; import { v } from "convex/values"; import { internal } from "./_generated/api"; import type { Doc } from "./_generated/dataModel"; -import { internalAction, internalMutation, internalQuery } from "./_generated/server"; +import { internalAction, internalQuery } from "./_generated/server"; import type { QueryCtx } from "./_generated/server"; +import { internalMutation } from "./functions"; import { isSkillHighlighted } from "./lib/badges"; import { sha256Hex } from "./lib/clawpack"; import { isPublicSkillDoc } from "./lib/globalStats"; @@ -32,6 +36,8 @@ import { const CATALOG_FEED_DESCRIPTION = "Official OpenClaw plugins published on ClawHub."; const CATALOG_FEED_PAGE_SIZE = 100; const MAX_CATALOG_FEED_ENTRIES = 1000; +const CATALOG_FEED_CHANGE_RETENTION_MS = 30 * 24 * 60 * 60 * 1000; +const CATALOG_FEED_CHANGE_PAGE_SIZE = 500; const CATALOG_FEED_FAMILIES = ["code-plugin", "bundle-plugin"] as const; type CatalogQueryCtx = Pick; @@ -91,6 +97,47 @@ const catalogFeedEntryValidator = v.union( v.object({ type: v.literal("skill"), ...catalogFeedEntryFields }), ); +function catalogFeedEntryKey(entry: Pick) { + return `${entry.type}\0${entry.id}`; +} + +function buildCatalogFeedChanges(args: { + sequence: number; + previousEntries: CatalogFeedEntry[]; + nextEntries: CatalogFeedEntry[]; + previousDescription?: string; + nextDescription: string; +}): CatalogFeedChange[] { + const previousByKey = new Map( + args.previousEntries.map((entry) => [catalogFeedEntryKey(entry), entry]), + ); + const nextByKey = new Map(args.nextEntries.map((entry) => [catalogFeedEntryKey(entry), entry])); + const changes: CatalogFeedChange[] = []; + for (const [key, previous] of [...previousByKey].sort(([left], [right]) => + left.localeCompare(right), + )) { + if (nextByKey.has(key)) continue; + changes.push({ + sequence: args.sequence, + operation: "remove", + entryType: previous.type, + entryId: previous.id, + }); + } + for (const [key, entry] of [...nextByKey].sort(([left], [right]) => left.localeCompare(right))) { + if (JSON.stringify(previousByKey.get(key)) === JSON.stringify(entry)) continue; + changes.push({ sequence: args.sequence, operation: "upsert", entry }); + } + if (args.previousDescription !== args.nextDescription || changes.length === 0) { + changes.push({ + sequence: args.sequence, + operation: "metadata", + metadata: { description: args.nextDescription }, + }); + } + return changes; +} + async function buildEntry( ctx: CatalogQueryCtx, pkg: Doc<"packages">, @@ -380,7 +427,20 @@ export const storePublication = internalMutation({ .query("catalogFeedPublications") .withIndex("by_feed", (q) => q.eq("feedId", args.feedId)) .unique(); + const latestRevision = await ctx.db + .query("catalogFeedRevisions") + .withIndex("by_feed_and_sequence", (q) => q.eq("feedId", args.feedId)) + .order("desc") + .first(); const sequence = (latest?.sequence ?? 0) + 1; + const previousFeed = latest ? parseCatalogFeed(JSON.parse(latest.payload)) : null; + const changes = buildCatalogFeedChanges({ + sequence, + previousEntries: previousFeed?.entries ?? [], + nextEntries: args.entries, + previousDescription: previousFeed?.description, + nextDescription: args.description, + }); const payload = serializeCatalogFeed({ schemaVersion: CATALOG_FEED_SCHEMA_VERSION, id: args.feedId, @@ -392,6 +452,7 @@ export const storePublication = internalMutation({ }); const payloadSha256 = await sha256Hex(new TextEncoder().encode(payload)); const publishedAt = Date.now(); + const expirationTime = publishedAt + CATALOG_FEED_CHANGE_RETENTION_MS; const publication = { feedId: args.feedId, sequence, @@ -404,6 +465,34 @@ export const storePublication = internalMutation({ const publicationId = latest ? (await ctx.db.patch(latest._id, publication), latest._id) : await ctx.db.insert("catalogFeedPublications", publication); + await ctx.db.insert("catalogFeedRevisions", { + feedId: args.feedId, + sequence, + changeCount: changes.length, + cumulativeChangeCount: (latestRevision?.cumulativeChangeCount ?? 0) + changes.length, + generatedAt: args.generatedAt, + expiresAt: args.expiresAt, + description: args.description, + publishedAt, + expirationTime, + }); + for (const [ordinal, change] of changes.entries()) { + const identity = + change.operation === "upsert" + ? { entryType: change.entry.type, entryId: change.entry.id } + : change.operation === "remove" + ? { entryType: change.entryType, entryId: change.entryId } + : {}; + await ctx.db.insert("catalogFeedChanges", { + feedId: args.feedId, + sequence, + ordinal, + operation: change.operation, + ...identity, + payload: JSON.stringify(change), + expirationTime, + }); + } return { publicationId, feedId: args.feedId, @@ -415,6 +504,192 @@ export const storePublication = internalMutation({ }, }); +export const listChanges = internalQuery({ + args: { + feedId: v.union(v.literal(CATALOG_FEED_ID), v.literal(CATALOG_SKILLS_FEED_ID)), + fromSequence: v.number(), + toSequence: v.number(), + paginationOpts: paginationOptsValidator, + }, + handler: async (ctx, args) => { + if ( + !Number.isSafeInteger(args.fromSequence) || + !Number.isSafeInteger(args.toSequence) || + args.fromSequence < 0 || + args.toSequence < args.fromSequence + ) { + throw new Error("Catalog feed change range is invalid"); + } + if ( + !Number.isSafeInteger(args.paginationOpts.numItems) || + args.paginationOpts.numItems < 1 || + args.paginationOpts.numItems > CATALOG_FEED_CHANGE_PAGE_SIZE + ) { + throw new Error( + `Catalog feed change page size must be between 1 and ${CATALOG_FEED_CHANGE_PAGE_SIZE}`, + ); + } + const state = await readCatalogFeedChangeState(ctx, args.feedId); + const window = changeWindowFromState(state); + if ( + args.fromSequence < window.retainedFromSequence || + args.toSequence > window.currentSequence + ) { + return { resetRequired: true as const, ...window }; + } + const changeCount = await countCatalogFeedChanges(ctx, args.feedId, args, state); + if (changeCount === null) { + return { resetRequired: true as const, ...window }; + } + const page = await ctx.db + .query("catalogFeedChanges") + .withIndex("by_feed_and_sequence_and_ordinal", (q) => + q + .eq("feedId", args.feedId) + .gt("sequence", args.fromSequence) + .lte("sequence", args.toSequence), + ) + .paginate(args.paginationOpts); + return { + resetRequired: false as const, + ...window, + changeCount, + ...page, + page: page.page.map(({ sequence, ordinal, payload }) => ({ sequence, ordinal, payload })), + }; + }, +}); + +async function readCatalogFeedChangeState( + ctx: Pick, + feedId: typeof CATALOG_FEED_ID | typeof CATALOG_SKILLS_FEED_ID, +) { + const [oldest, latest] = await Promise.all([ + ctx.db + .query("catalogFeedRevisions") + .withIndex("by_feed_and_sequence", (q) => q.eq("feedId", feedId)) + .order("asc") + .first(), + ctx.db + .query("catalogFeedRevisions") + .withIndex("by_feed_and_sequence", (q) => q.eq("feedId", feedId)) + .order("desc") + .first(), + ]); + if (latest) { + return { + currentSequence: latest.sequence, + retainedFromSequence: Math.max(0, (oldest?.sequence ?? latest.sequence) - 1), + oldestRevision: oldest, + latestRevision: latest, + }; + } + const publication = await ctx.db + .query("catalogFeedPublications") + .withIndex("by_feed", (q) => q.eq("feedId", feedId)) + .unique(); + const currentSequence = publication?.sequence ?? 0; + return { + currentSequence, + retainedFromSequence: currentSequence, + oldestRevision: null, + latestRevision: null, + }; +} + +function changeWindowFromState(state: Awaited>) { + return { + currentSequence: state.currentSequence, + retainedFromSequence: state.retainedFromSequence, + }; +} + +async function countCatalogFeedChanges( + ctx: Pick, + feedId: typeof CATALOG_FEED_ID | typeof CATALOG_SKILLS_FEED_ID, + range: { fromSequence: number; toSequence: number }, + state: Awaited>, +) { + if (range.fromSequence === range.toSequence) return 0; + if (!state.oldestRevision || !state.latestRevision) return null; + + const baseRevision = + range.fromSequence === state.retainedFromSequence + ? state.oldestRevision + : range.fromSequence === state.latestRevision.sequence + ? state.latestRevision + : await ctx.db + .query("catalogFeedRevisions") + .withIndex("by_feed_and_sequence", (q) => + q.eq("feedId", feedId).eq("sequence", range.fromSequence), + ) + .unique(); + const targetRevision = + range.toSequence === state.latestRevision.sequence + ? state.latestRevision + : await ctx.db + .query("catalogFeedRevisions") + .withIndex("by_feed_and_sequence", (q) => + q.eq("feedId", feedId).eq("sequence", range.toSequence), + ) + .unique(); + if (!baseRevision || !targetRevision) return null; + + const baseCount = + range.fromSequence === state.retainedFromSequence + ? baseRevision.cumulativeChangeCount - baseRevision.changeCount + : baseRevision.cumulativeChangeCount; + const changeCount = targetRevision.cumulativeChangeCount - baseCount; + return Number.isSafeInteger(changeCount) && changeCount >= 0 ? changeCount : null; +} + +export const getChangeWindow = internalQuery({ + args: { + feedId: v.union(v.literal(CATALOG_FEED_ID), v.literal(CATALOG_SKILLS_FEED_ID)), + }, + handler: async (ctx, args) => + changeWindowFromState(await readCatalogFeedChangeState(ctx, args.feedId)), +}); + +export const pruneCatalogFeedHistoryInternal = internalMutation({ + args: { batchSize: v.optional(v.number()) }, + handler: async (ctx, args) => { + const batchSize = args.batchSize ?? CATALOG_FEED_CHANGE_PAGE_SIZE; + if ( + !Number.isSafeInteger(batchSize) || + batchSize < 1 || + batchSize > CATALOG_FEED_CHANGE_PAGE_SIZE + ) { + throw new Error( + `Catalog feed prune batch size must be between 1 and ${CATALOG_FEED_CHANGE_PAGE_SIZE}`, + ); + } + const now = Date.now(); + // Retire the revision marker first so readers reset instead of observing a + // revision whose journal rows are only partially retained. + const revisions = await ctx.db + .query("catalogFeedRevisions") + .withIndex("by_expiration_time", (q) => q.lt("expirationTime", now)) + .take(batchSize); + const changes = + revisions.length < batchSize + ? await ctx.db + .query("catalogFeedChanges") + .withIndex("by_expiration_time", (q) => q.lt("expirationTime", now)) + .take(batchSize - revisions.length) + : []; + for (const row of [...revisions, ...changes]) await ctx.db.delete(row._id); + const deleted = changes.length + revisions.length; + const hasMore = deleted === batchSize; + if (hasMore) { + await ctx.scheduler.runAfter(0, internal.catalogFeed.pruneCatalogFeedHistoryInternal, { + batchSize, + }); + } + return { deleted, hasMore }; + }, +}); + export const publish = internalAction({ args: { expiresAt: v.string(), @@ -507,3 +782,5 @@ export const getLatestPublication = internalQuery({ .withIndex("by_feed", (q) => q.eq("feedId", args.feedId)) .unique(), }); + +export const __test = { buildCatalogFeedChanges }; diff --git a/convex/crons.test.ts b/convex/crons.test.ts index b0c2a36b2a..66422ccc32 100644 --- a/convex/crons.test.ts +++ b/convex/crons.test.ts @@ -17,6 +17,7 @@ const mocks = vi.hoisted(() => { const authRefreshTokensPruneRef = Symbol("auth-refresh-tokens-prune"); const publisherInvitesPruneRef = Symbol("publisher-invites-prune"); const promotionsFeedPublishRef = Symbol("promotions-feed-publish"); + const catalogFeedHistoryPruneRef = Symbol("catalog-feed-history-prune"); const securityScanExpiredLeaseRecoveryRef = Symbol("security-scan-expired-lease-recovery"); const securityScanDispatchWatchdogRef = Symbol("security-scan-dispatch-watchdog"); return { @@ -35,6 +36,7 @@ const mocks = vi.hoisted(() => { authRefreshTokensPruneRef, publisherInvitesPruneRef, promotionsFeedPublishRef, + catalogFeedHistoryPruneRef, securityScanExpiredLeaseRecoveryRef, securityScanDispatchWatchdogRef, }; @@ -80,6 +82,9 @@ vi.mock("./_generated/api", () => ({ promotionsFeed: { publishInternal: mocks.promotionsFeedPublishRef, }, + catalogFeed: { + pruneCatalogFeedHistoryInternal: mocks.catalogFeedHistoryPruneRef, + }, vt: { pollPendingScans: Symbol("vt-pending-scans"), backfillActiveSkillsVTCache: Symbol("vt-cache-backfill"), @@ -159,6 +164,17 @@ describe("crons", () => { ); }); + it("prunes catalog feed history in bounded daily batches", async () => { + await import("./crons"); + + expect(mocks.interval).toHaveBeenCalledWith( + "catalog-feed-history-prune", + { hours: 24 }, + mocks.catalogFeedHistoryPruneRef, + { batchSize: 500 }, + ); + }); + it("prunes expired skill scan requests in bounded continuation batches", async () => { await import("./crons"); diff --git a/convex/crons.ts b/convex/crons.ts index e92c59975b..147869c456 100644 --- a/convex/crons.ts +++ b/convex/crons.ts @@ -19,6 +19,13 @@ if (process.env.CLAWHUB_DISABLE_CRONS !== "1" && process.env.CLAWHUB_PREVIEW !== {}, ); + crons.interval( + "catalog-feed-history-prune", + { hours: 24 }, + internal.catalogFeed.pruneCatalogFeedHistoryInternal, + { batchSize: RETENTION_STANDARD_BATCH_SIZE }, + ); + crons.interval( "trending-leaderboard", { minutes: 60 }, diff --git a/convex/http.ts b/convex/http.ts index 7bab99eeaf..dc8ba09ac4 100644 --- a/convex/http.ts +++ b/convex/http.ts @@ -50,7 +50,6 @@ import { listPromotionsV1Http, promotionsGetRouterV1Http, promotionsPostRouterV1Http, - catalogFeedV1Http, catalogSkillsFeedV1Http, promotionsFeedV1Http, usersGetRouterV1Http, @@ -60,6 +59,8 @@ import { whoamiV1Http, contentRightsV1Http, } from "./httpApiV1"; +import { signedCatalogFeedChangesHttp } from "./httpApiV1/catalogFeedChanges"; +import { signedCatalogFeedV1Http } from "./httpApiV1/catalogFeedSigning"; import { preflightHandler } from "./httpPreflight"; import { installRateLimitedRoutes } from "./lib/httpRouteRateLimit"; import { @@ -165,7 +166,13 @@ http.route({ http.route({ path: ApiRoutes.catalogFeed, method: "GET", - handler: catalogFeedV1Http, + handler: signedCatalogFeedV1Http, +}); + +http.route({ + path: ApiRoutes.catalogFeedChanges, + method: "GET", + handler: signedCatalogFeedChangesHttp, }); http.route({ diff --git a/convex/httpApiV1.catalogFeedChanges.test.ts b/convex/httpApiV1.catalogFeedChanges.test.ts new file mode 100644 index 0000000000..4a079bd336 --- /dev/null +++ b/convex/httpApiV1.catalogFeedChanges.test.ts @@ -0,0 +1,176 @@ +import { generateKeyPairSync } from "node:crypto"; +import { CATALOG_FEED_CHANGES_PAYLOAD_TYPE } from "clawhub-schema"; +import { describe, expect, it, vi } from "vitest"; +import { signedCatalogFeedChangesHandler } from "./httpApiV1/catalogFeedChanges"; + +async function signingEnv() { + const { privateKey } = generateKeyPairSync("ed25519", { + privateKeyEncoding: { type: "pkcs8", format: "pem" }, + publicKeyEncoding: { type: "spki", format: "pem" }, + }); + return { + CLAWHUB_FEED_SIGNING_CONFIG: JSON.stringify({ + keyId: "clawhub-feed-2026-q3", + privateKey, + }), + }; +} + +async function signedPayload(response: Response) { + const envelope = (await response.json()) as { + payloadType: string; + payload: string; + signatures: Array>; + }; + expect(Object.keys(envelope).sort()).toEqual(["payload", "payloadType", "signatures"]); + expect(Object.keys(envelope.signatures[0] ?? {}).sort()).toEqual(["keyid", "sig"]); + expect(envelope.payloadType).toBe(CATALOG_FEED_CHANGES_PAYLOAD_TYPE); + return JSON.parse(Buffer.from(envelope.payload, "base64url").toString("utf8")) as Record< + string, + unknown + >; +} + +describe("signed catalog feed changes", () => { + it("pins and signs a paginated change range", async () => { + const env = await signingEnv(); + const runQuery = vi.fn(async (_reference, args: Record) => { + if (!("fromSequence" in args)) { + return { retainedFromSequence: 3, currentSequence: 5 }; + } + const pagination = args.paginationOpts as { cursor: string | null }; + return pagination.cursor === null + ? { + resetRequired: false, + retainedFromSequence: 3, + currentSequence: 5, + changeCount: 2, + page: [ + { + sequence: 4, + ordinal: 0, + payload: JSON.stringify({ + sequence: 4, + operation: "metadata", + metadata: { description: "Official" }, + }), + }, + ], + isDone: false, + continueCursor: "convex-page-2", + } + : { + resetRequired: false, + retainedFromSequence: 3, + currentSequence: 5, + changeCount: 2, + page: [ + { + sequence: 5, + ordinal: 0, + payload: JSON.stringify({ + sequence: 5, + operation: "metadata", + metadata: { description: "Current" }, + }), + }, + ], + isDone: true, + continueCursor: "", + }; + }); + const ctx = { runQuery }; + const firstResponse = await signedCatalogFeedChangesHandler( + ctx as never, + new Request("https://clawhub.ai/api/v1/feeds/plugins/changes?fromSequence=3&limit=1"), + env, + ); + + expect(firstResponse.status).toBe(200); + expect(firstResponse.headers.get("cache-control")).toBe("no-store"); + expect(firstResponse.headers.get("content-type")).toBe( + "application/vnd.dsse+json; charset=utf-8", + ); + const first = await signedPayload(firstResponse); + expect(first).toMatchObject({ + feedId: "clawhub-official", + fromSequence: 3, + toSequence: 5, + requestCursor: null, + pageIndex: 0, + startIndex: 0, + changeCount: 2, + }); + const nextCursor = String(first.nextCursor); + + const secondResponse = await signedCatalogFeedChangesHandler( + ctx as never, + new Request( + `https://clawhub.ai/api/v1/feeds/plugins/changes?cursor=${encodeURIComponent(nextCursor)}`, + ), + env, + ); + expect(secondResponse.status).toBe(200); + const second = await signedPayload(secondResponse); + expect(second).toMatchObject({ + requestCursor: nextCursor, + pageIndex: 1, + startIndex: 1, + changeCount: 2, + nextCursor: null, + }); + expect(runQuery.mock.calls.at(-1)?.[1]).toMatchObject({ + fromSequence: 3, + toSequence: 5, + paginationOpts: { cursor: "convex-page-2", numItems: 1 }, + }); + + runQuery.mockClear(); + const replacement = nextCursor.endsWith("A") ? "B" : "A"; + const tampered = `${nextCursor.slice(0, -1)}${replacement}`; + const tamperedResponse = await signedCatalogFeedChangesHandler( + ctx as never, + new Request( + `https://clawhub.ai/api/v1/feeds/plugins/changes?cursor=${encodeURIComponent(tampered)}`, + ), + env, + ); + expect(tamperedResponse.status).toBe(400); + expect(runQuery).not.toHaveBeenCalled(); + }); + + it("returns a signed reset response when retained history cannot cover the range", async () => { + const env = await signingEnv(); + const runQuery = vi.fn(async (_reference, args: Record) => + !("fromSequence" in args) + ? { retainedFromSequence: 3, currentSequence: 5 } + : { resetRequired: true, retainedFromSequence: 3, currentSequence: 5 }, + ); + const response = await signedCatalogFeedChangesHandler( + { runQuery } as never, + new Request("https://clawhub.ai/api/v1/feeds/plugins/changes?fromSequence=2"), + env, + ); + + expect(response.status).toBe(409); + expect(await signedPayload(response)).toMatchObject({ + fromSequence: 2, + currentSequence: 5, + resetRequired: true, + snapshotUrl: "https://clawhub.ai/api/v1/feeds/plugins", + }); + }); + + it("fails closed before querying when the signer is unavailable", async () => { + const runQuery = vi.fn(); + const response = await signedCatalogFeedChangesHandler( + { runQuery } as never, + new Request("https://clawhub.ai/api/v1/feeds/plugins/changes?fromSequence=0"), + {}, + ); + + expect(response.status).toBe(503); + expect(response.headers.get("cache-control")).toBe("no-store"); + expect(runQuery).not.toHaveBeenCalled(); + }); +}); diff --git a/convex/httpApiV1.catalogFeedSigning.test.ts b/convex/httpApiV1.catalogFeedSigning.test.ts new file mode 100644 index 0000000000..f1da091cc2 --- /dev/null +++ b/convex/httpApiV1.catalogFeedSigning.test.ts @@ -0,0 +1,183 @@ +import { createPublicKey, generateKeyPairSync, verify as verifyDetached } from "node:crypto"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { internal } from "./_generated/api"; +import { + OPENCLAW_CATALOG_FEED_PAYLOAD_TYPE, + resolveFeedSigningConfig, + signCatalogFeedPayload, + signedCatalogFeedV1Handler, +} from "./httpApiV1/catalogFeedSigning"; + +type QueryCtx = { + runQuery: ReturnType; +}; + +const publication = { + feedId: "clawhub-official", + sequence: 4, + generatedAt: "2026-06-23T00:00:00.000Z", + expiresAt: "2026-06-30T00:00:00.000Z", + payload: + '{"schemaVersion":1,"id":"clawhub-official","generatedAt":"2026-06-23T00:00:00.000Z","sequence":4,"expiresAt":"2026-06-30T00:00:00.000Z","entries":[]}', + payloadSha256: "catalog-payload-sha256", + publishedAt: Date.parse("2026-06-23T00:00:00.000Z"), +}; + +async function signingFixture() { + const { privateKey, publicKey } = generateKeyPairSync("ed25519", { + privateKeyEncoding: { type: "pkcs8", format: "pem" }, + publicKeyEncoding: { type: "spki", format: "pem" }, + }); + const env = { + CLAWHUB_FEED_SIGNING_CONFIG: JSON.stringify({ + keyId: "clawhub-feed-2026-q3", + privateKey, + }), + }; + const config = await resolveFeedSigningConfig(env); + if (!config) throw new Error("expected signing config"); + return { config, env, publicKey }; +} + +function dsseInput(payloadBytes: Buffer) { + const typeBytes = Buffer.from(OPENCLAW_CATALOG_FEED_PAYLOAD_TYPE, "utf8"); + return Buffer.concat([ + Buffer.from( + `DSSEv1 ${typeBytes.length} ${OPENCLAW_CATALOG_FEED_PAYLOAD_TYPE} ${payloadBytes.length} `, + "utf8", + ), + payloadBytes, + ]); +} + +describe("signed catalog feed", () => { + let ctx: QueryCtx; + + beforeEach(() => { + ctx = { runQuery: vi.fn().mockResolvedValue(publication) }; + }); + + it("signs the exact stored publication bytes with DSSE Ed25519", async () => { + const { config, publicKey } = await signingFixture(); + const signed = await signCatalogFeedPayload(publication.payload, config); + const payloadBytes = Buffer.from(signed.envelope.payload, "base64url"); + + expect(payloadBytes.toString("utf8")).toBe(publication.payload); + expect(Object.keys(signed.envelope).sort()).toEqual(["payload", "payloadType", "signatures"]); + expect(signed.envelope).toMatchObject({ + payloadType: OPENCLAW_CATALOG_FEED_PAYLOAD_TYPE, + signatures: [ + { + keyid: "clawhub-feed-2026-q3", + }, + ], + }); + expect(Object.keys(signed.envelope.signatures[0] ?? {}).sort()).toEqual(["keyid", "sig"]); + expect( + verifyDetached( + null, + dsseInput(payloadBytes), + createPublicKey(publicKey), + Buffer.from(signed.envelope.signatures[0]?.sig ?? "", "base64url"), + ), + ).toBe(true); + }); + + it("serves a stable signed representation with envelope validators", async () => { + const { env } = await signingFixture(); + const first = await signedCatalogFeedV1Handler( + ctx as never, + new Request("https://clawhub.ai/api/v1/feeds/plugins"), + env, + ); + + expect(first.status).toBe(200); + const body = await first.text(); + const envelope = JSON.parse(body) as { payload: string }; + expect(Buffer.from(envelope.payload, "base64url").toString("utf8")).toBe(publication.payload); + expect(first.headers.get("content-type")).toBe("application/vnd.dsse+json; charset=utf-8"); + expect(first.headers.get("etag")).toMatch(/^"sha256:[a-f0-9]{64}"$/u); + expect(first.headers.get("x-content-sha256")).toMatch(/^[a-f0-9]{64}$/u); + expect(first.headers.get("x-catalog-payload-sha256")).toBe(publication.payloadSha256); + expect(first.headers.get("x-openclaw-feed-signing-key-id")).toBe("clawhub-feed-2026-q3"); + expect(first.headers.get("last-modified")).toBeNull(); + expect(ctx.runQuery).toHaveBeenCalledWith(internal.catalogFeed.getLatestPublication, { + feedId: "clawhub-official", + }); + + const etag = first.headers.get("etag") ?? ""; + const notModified = await signedCatalogFeedV1Handler( + ctx as never, + new Request("https://clawhub.ai/api/v1/feeds/plugins", { + headers: { "If-None-Match": etag }, + }), + env, + ); + expect(notModified.status).toBe(304); + expect(await notModified.text()).toBe(""); + + const repeated = await signedCatalogFeedV1Handler( + ctx as never, + new Request("https://clawhub.ai/api/v1/feeds/plugins"), + env, + ); + expect(await repeated.text()).toBe(body); + expect(repeated.headers.get("etag")).toBe(etag); + + const ignoredLastModified = await signedCatalogFeedV1Handler( + ctx as never, + new Request("https://clawhub.ai/api/v1/feeds/plugins", { + headers: { "If-Modified-Since": "Wed, 31 Dec 2098 23:59:59 GMT" }, + }), + env, + ); + expect(ignoredLastModified.status).toBe(200); + }); + + it.each([ + {}, + { CLAWHUB_FEED_SIGNING_CONFIG: "not-json" }, + { CLAWHUB_FEED_SIGNING_CONFIG: "[]" }, + { + CLAWHUB_FEED_SIGNING_CONFIG: JSON.stringify({ + keyId: "clawhub-feed-2026-q3", + }), + }, + { + CLAWHUB_FEED_SIGNING_CONFIG: JSON.stringify({ + keyId: "clawhub-feed-2026-q3", + privateKey: "not-a-private-key", + }), + }, + { + CLAWHUB_FEED_SIGNING_CONFIG: JSON.stringify({ + keyId: "invalid key id\r\nheader", + privateKey: "not-a-private-key", + unexpected: true, + }), + }, + ])("fails closed before reading a publication when signing config is invalid", async (env) => { + const response = await signedCatalogFeedV1Handler( + ctx as never, + new Request("https://clawhub.ai/api/v1/feeds/plugins"), + env, + ); + + expect(response.status).toBe(503); + expect(response.headers.get("cache-control")).toBe("no-store"); + expect(ctx.runQuery).not.toHaveBeenCalled(); + }); + + it("does not cache a missing stored publication", async () => { + const { env } = await signingFixture(); + ctx.runQuery.mockResolvedValue(null); + const response = await signedCatalogFeedV1Handler( + ctx as never, + new Request("https://clawhub.ai/api/v1/feeds/plugins"), + env, + ); + + expect(response.status).toBe(503); + expect(response.headers.get("cache-control")).toBe("no-store"); + }); +}); diff --git a/convex/httpApiV1/catalogFeedChanges.ts b/convex/httpApiV1/catalogFeedChanges.ts new file mode 100644 index 0000000000..fabe41491e --- /dev/null +++ b/convex/httpApiV1/catalogFeedChanges.ts @@ -0,0 +1,354 @@ +import { + ApiRoutes, + CATALOG_FEED_CHANGES_MAX_RECORDS, + CATALOG_FEED_CHANGES_PAYLOAD_TYPE, + CATALOG_FEED_ID, + CATALOG_FEED_SCHEMA_VERSION, + parseCatalogFeedChangePage, + parseCatalogFeedResetRequired, + type CatalogFeedChange, +} from "clawhub-schema"; +import { internal } from "../_generated/api"; +import type { ActionCtx } from "../_generated/server"; +import { httpAction } from "../functions"; +import { corsHeaders, mergeHeaders } from "../lib/httpHeaders"; +import { + type FeedSigningConfig, + resolveFeedSigningConfig, + signFeedPayload, +} from "./catalogFeedSigning"; +import { catalogFeedUnavailableResponse } from "./catalogFeedV1"; + +const CATALOG_FEED_CURSOR_PAYLOAD_TYPE = "openclaw.official-catalog-change-cursor.v1"; +const CURSOR_MAX_LENGTH = 4096; +const PROJECTION_TTL_MS = 5 * 60 * 1000; +const MAX_SIGNED_PAGE_BYTES = 1024 * 1024; + +type CatalogChangesCursor = { + schemaVersion: 1; + operation: "changes"; + feedId: typeof CATALOG_FEED_ID; + fromSequence: number; + toSequence: number; + databaseCursor: string; + limit: number; + pageIndex: number; + startIndex: number; + changeCount: number; + generatedAt: string; + expiresAt: string; +}; + +function base64UrlEncode(bytes: Uint8Array) { + let binary = ""; + for (let offset = 0; offset < bytes.length; offset += 0x8000) { + binary += String.fromCharCode(...bytes.subarray(offset, offset + 0x8000)); + } + return btoa(binary).replaceAll("+", "-").replaceAll("/", "_").replace(/=+$/u, ""); +} + +function base64UrlDecode(value: string) { + if (!/^[A-Za-z0-9_-]+$/u.test(value)) throw new Error("Invalid base64url value"); + const padded = value + .replaceAll("-", "+") + .replaceAll("_", "/") + .padEnd(Math.ceil(value.length / 4) * 4, "="); + return Uint8Array.from(atob(padded), (character) => character.charCodeAt(0)); +} + +function timingSafeEqual(left: string, right: string) { + const leftBytes = new TextEncoder().encode(left); + const rightBytes = new TextEncoder().encode(right); + let difference = leftBytes.length ^ rightBytes.length; + const length = Math.max(leftBytes.length, rightBytes.length); + for (let index = 0; index < length; index += 1) { + difference |= (leftBytes[index] ?? 0) ^ (rightBytes[index] ?? 0); + } + return difference === 0; +} + +function hasExactKeys(record: Record, keys: readonly string[]) { + return Object.keys(record).sort().join("\0") === [...keys].sort().join("\0"); +} + +function isNonNegativeInteger(value: unknown): value is number { + return typeof value === "number" && Number.isSafeInteger(value) && value >= 0; +} + +function parseCursorPayload(value: unknown): CatalogChangesCursor | null { + if (!value || typeof value !== "object" || Array.isArray(value)) return null; + const record = value as Record; + if ( + !hasExactKeys(record, [ + "schemaVersion", + "operation", + "feedId", + "fromSequence", + "toSequence", + "databaseCursor", + "limit", + "pageIndex", + "startIndex", + "changeCount", + "generatedAt", + "expiresAt", + ]) || + record.schemaVersion !== 1 || + record.operation !== "changes" || + record.feedId !== CATALOG_FEED_ID || + !isNonNegativeInteger(record.fromSequence) || + !isNonNegativeInteger(record.toSequence) || + record.toSequence < record.fromSequence || + typeof record.databaseCursor !== "string" || + !record.databaseCursor || + record.databaseCursor.length > CURSOR_MAX_LENGTH || + !isNonNegativeInteger(record.limit) || + record.limit < 1 || + record.limit > CATALOG_FEED_CHANGES_MAX_RECORDS || + !isNonNegativeInteger(record.pageIndex) || + record.pageIndex < 1 || + !isNonNegativeInteger(record.startIndex) || + record.startIndex < record.pageIndex || + !isNonNegativeInteger(record.changeCount) || + record.startIndex >= record.changeCount || + typeof record.generatedAt !== "string" || + typeof record.expiresAt !== "string" || + !Number.isFinite(Date.parse(record.generatedAt)) || + !Number.isFinite(Date.parse(record.expiresAt)) || + Date.parse(record.expiresAt) <= Date.parse(record.generatedAt) + ) { + return null; + } + return record as CatalogChangesCursor; +} + +async function encodeCursor(cursor: CatalogChangesCursor, config: FeedSigningConfig) { + const signed = await signFeedPayload( + CATALOG_FEED_CURSOR_PAYLOAD_TYPE, + JSON.stringify(cursor), + config, + ); + const encoded = base64UrlEncode(new TextEncoder().encode(signed.body)); + if (encoded.length > CURSOR_MAX_LENGTH) throw new Error("Catalog change cursor is too large"); + return encoded; +} + +async function decodeCursor(raw: string, config: FeedSigningConfig) { + if (!raw || raw.length > CURSOR_MAX_LENGTH) return null; + try { + const envelopeBody = new TextDecoder().decode(base64UrlDecode(raw)); + const envelope = JSON.parse(envelopeBody) as Record; + if ( + !hasExactKeys(envelope, ["payloadType", "payload", "signatures"]) || + envelope.payloadType !== CATALOG_FEED_CURSOR_PAYLOAD_TYPE || + typeof envelope.payload !== "string" || + !Array.isArray(envelope.signatures) || + envelope.signatures.length !== 1 + ) { + return null; + } + const payload = new TextDecoder().decode(base64UrlDecode(envelope.payload)); + const expected = await signFeedPayload(CATALOG_FEED_CURSOR_PAYLOAD_TYPE, payload, config); + if (!timingSafeEqual(expected.body, envelopeBody)) return null; + return parseCursorPayload(JSON.parse(payload)); + } catch { + return null; + } +} + +function textResponse(message: string, status: number) { + return new Response(message, { + status, + headers: mergeHeaders( + { "Content-Type": "text/plain; charset=utf-8", "Cache-Control": "no-store" }, + corsHeaders(), + ), + }); +} + +async function signedChangesResponse(payload: unknown, config: FeedSigningConfig, status = 200) { + if ((payload as { resetRequired?: unknown }).resetRequired === true) { + parseCatalogFeedResetRequired(payload); + } else { + parseCatalogFeedChangePage(payload); + } + const signed = await signFeedPayload( + CATALOG_FEED_CHANGES_PAYLOAD_TYPE, + JSON.stringify(payload), + config, + ); + if (new TextEncoder().encode(signed.body).length > MAX_SIGNED_PAGE_BYTES) { + return textResponse("Catalog change page is too large; retry with a smaller limit", 413); + } + return new Response(signed.body, { + status, + headers: mergeHeaders( + { + "Content-Type": "application/vnd.dsse+json; charset=utf-8", + "Cache-Control": "no-store", + "X-Content-SHA256": signed.sha256, + "X-OpenClaw-Feed-Signing-Key-ID": config.keyId, + "X-Content-Type-Options": "nosniff", + }, + corsHeaders(), + ), + }); +} + +function hasOnlySearchParams(url: URL, allowed: ReadonlySet) { + return [...url.searchParams.keys()].every((key) => allowed.has(key)); +} + +function projectionTimes() { + const generatedAtMs = Date.now(); + return { + generatedAt: new Date(generatedAtMs).toISOString(), + expiresAt: new Date(generatedAtMs + PROJECTION_TTL_MS).toISOString(), + }; +} + +export async function signedCatalogFeedChangesHandler( + ctx: ActionCtx, + request: Request, + env: Record = process.env, +) { + let config: FeedSigningConfig | null; + try { + config = await resolveFeedSigningConfig(env); + } catch { + return catalogFeedUnavailableResponse("Signed catalog changes are unavailable"); + } + if (!config) return catalogFeedUnavailableResponse("Signed catalog changes are unavailable"); + + const url = new URL(request.url); + const rawCursor = url.searchParams.get("cursor"); + let fromSequence: number; + let toSequence: number; + let databaseCursor: string | null; + let limit: number; + let pageIndex: number; + let startIndex: number; + let expectedChangeCount: number | null; + let generatedAt: string; + let expiresAt: string; + + if (rawCursor !== null) { + if (!hasOnlySearchParams(url, new Set(["cursor"]))) { + return textResponse("Cursor requests cannot include change parameters", 400); + } + const cursor = await decodeCursor(rawCursor, config); + if (!cursor) return textResponse("Invalid catalog changes cursor", 400); + if (Date.parse(cursor.expiresAt) <= Date.now()) { + return textResponse("Catalog changes cursor expired", 409); + } + ({ + fromSequence, + toSequence, + databaseCursor, + limit, + pageIndex, + startIndex, + changeCount: expectedChangeCount, + generatedAt, + expiresAt, + } = cursor); + } else { + if (!hasOnlySearchParams(url, new Set(["fromSequence", "limit"]))) { + return textResponse("Invalid catalog changes parameter", 400); + } + const rawFromSequence = url.searchParams.get("fromSequence"); + const rawLimit = url.searchParams.get("limit"); + limit = rawLimit === null ? 100 : Number(rawLimit); + if ( + rawFromSequence === null || + !/^\d+$/u.test(rawFromSequence) || + !Number.isSafeInteger(Number(rawFromSequence)) || + (rawLimit !== null && !/^\d+$/u.test(rawLimit)) || + !Number.isSafeInteger(limit) || + limit < 1 || + limit > CATALOG_FEED_CHANGES_MAX_RECORDS + ) { + return textResponse("Invalid catalog changes request", 400); + } + fromSequence = Number(rawFromSequence); + const window = await ctx.runQuery(internal.catalogFeed.getChangeWindow, { + feedId: CATALOG_FEED_ID, + }); + if (fromSequence > window.currentSequence) { + return textResponse("Catalog changes fromSequence is ahead of the current feed", 400); + } + toSequence = window.currentSequence; + databaseCursor = null; + pageIndex = 0; + startIndex = 0; + expectedChangeCount = null; + ({ generatedAt, expiresAt } = projectionTimes()); + } + + const result = await ctx.runQuery(internal.catalogFeed.listChanges, { + feedId: CATALOG_FEED_ID, + fromSequence, + toSequence, + paginationOpts: { cursor: databaseCursor, numItems: limit }, + }); + if (result.resetRequired) { + return await signedChangesResponse( + { + schemaVersion: CATALOG_FEED_SCHEMA_VERSION, + feedId: CATALOG_FEED_ID, + fromSequence, + currentSequence: result.currentSequence, + generatedAt, + expiresAt, + resetRequired: true, + snapshotUrl: new URL(ApiRoutes.catalogFeed, request.url).toString(), + }, + config, + 409, + ); + } + if (expectedChangeCount !== null && result.changeCount !== expectedChangeCount) { + return catalogFeedUnavailableResponse("Catalog change range changed unexpectedly"); + } + + const changes = result.page.map(({ payload }) => JSON.parse(payload) as CatalogFeedChange); + const changeCount = result.changeCount; + const nextCursor = result.isDone + ? null + : await encodeCursor( + { + schemaVersion: 1, + operation: "changes", + feedId: CATALOG_FEED_ID, + fromSequence, + toSequence, + databaseCursor: result.continueCursor, + limit, + pageIndex: pageIndex + 1, + startIndex: startIndex + changes.length, + changeCount, + generatedAt, + expiresAt, + }, + config, + ); + return await signedChangesResponse( + { + schemaVersion: CATALOG_FEED_SCHEMA_VERSION, + feedId: CATALOG_FEED_ID, + fromSequence, + toSequence, + generatedAt, + expiresAt, + requestCursor: rawCursor, + pageIndex, + startIndex, + changeCount, + changes, + nextCursor, + }, + config, + ); +} + +export const signedCatalogFeedChangesHttp = httpAction(signedCatalogFeedChangesHandler); diff --git a/convex/httpApiV1/catalogFeedSigning.ts b/convex/httpApiV1/catalogFeedSigning.ts new file mode 100644 index 0000000000..1e735b01fa --- /dev/null +++ b/convex/httpApiV1/catalogFeedSigning.ts @@ -0,0 +1,192 @@ +import { CATALOG_FEED_ID } from "clawhub-schema"; +import { internal } from "../_generated/api"; +import type { ActionCtx } from "../_generated/server"; +import { httpAction } from "../functions"; +import { + catalogFeedResponseHeaders, + catalogFeedUnavailableResponse, + matchesEtag, +} from "./catalogFeedV1"; + +export const OPENCLAW_CATALOG_FEED_PAYLOAD_TYPE = + "openclaw.official-external-plugin-catalog-feed.v1"; + +export type FeedSigningConfig = { + keyId: string; + privateKey: CryptoKey; +}; + +type SignedFeedEnvelope = { + payloadType: string; + payload: string; + signatures: readonly { + keyid: string; + sig: string; + }[]; +}; + +function decodePkcs8PrivateKey(raw: string) { + const normalized = raw.replaceAll("\\n", "\n").trim(); + const match = + /^-----BEGIN PRIVATE KEY-----\s*([A-Za-z0-9+/=\s]+)\s*-----END PRIVATE KEY-----$/m.exec( + normalized, + ); + if (!match?.[1]) throw new Error("ClawHub feed signing key must be PKCS#8 PEM"); + const binary = atob(match[1].replaceAll(/\s/gu, "")); + return Uint8Array.from(binary, (character) => character.charCodeAt(0)); +} + +export async function resolveFeedSigningConfig( + env: Record, +): Promise { + const rawConfig = env.CLAWHUB_FEED_SIGNING_CONFIG?.trim(); + if (!rawConfig) return null; + if (rawConfig.length > 32_768) throw new Error("ClawHub feed signing config is too large"); + let parsed: unknown; + try { + parsed = JSON.parse(rawConfig); + } catch { + throw new Error("ClawHub feed signing config must be valid JSON"); + } + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new Error("ClawHub feed signing config must be an object"); + } + const record = parsed as Record; + if ( + Object.keys(record).sort().join(",") !== "keyId,privateKey" || + typeof record.keyId !== "string" || + typeof record.privateKey !== "string" + ) { + throw new Error("ClawHub feed signing config must contain only keyId and privateKey"); + } + const keyId = record.keyId.trim(); + const privateKeyValue = record.privateKey.trim(); + if (!keyId || !privateKeyValue) { + throw new Error("ClawHub feed signing config fields must not be empty"); + } + if (!/^[A-Za-z0-9._:-]{1,128}$/u.test(keyId)) { + throw new Error("ClawHub feed signing key id is invalid"); + } + if (privateKeyValue.length > 16_384) { + throw new Error("ClawHub feed signing private key is too large"); + } + + const privateKey = await crypto.subtle.importKey( + "pkcs8", + decodePkcs8PrivateKey(privateKeyValue), + { name: "Ed25519" }, + false, + ["sign"], + ); + if (privateKey.algorithm.name !== "Ed25519") { + throw new Error("ClawHub feed signing key must be Ed25519"); + } + return { keyId, privateKey }; +} + +function concatBytes(left: Uint8Array, right: Uint8Array) { + const result = new Uint8Array(left.length + right.length); + result.set(left); + result.set(right, left.length); + return result; +} + +function dssePreAuthenticationEncoding(payloadType: string, payloadBytes: Uint8Array) { + const encoder = new TextEncoder(); + const payloadTypeBytes = encoder.encode(payloadType); + const prefix = encoder.encode( + `DSSEv1 ${payloadTypeBytes.length} ${payloadType} ${payloadBytes.length} `, + ); + return concatBytes(prefix, payloadBytes); +} + +function base64UrlEncode(bytes: Uint8Array) { + let binary = ""; + for (let offset = 0; offset < bytes.length; offset += 0x8000) { + binary += String.fromCharCode(...bytes.subarray(offset, offset + 0x8000)); + } + return btoa(binary).replaceAll("+", "-").replaceAll("/", "_").replace(/=+$/u, ""); +} + +function toHex(bytes: Uint8Array) { + let result = ""; + for (const byte of bytes) result += byte.toString(16).padStart(2, "0"); + return result; +} + +export async function signFeedPayload( + payloadType: string, + payload: string, + config: FeedSigningConfig, +): Promise<{ envelope: SignedFeedEnvelope; body: string; sha256: string }> { + const payloadBytes = new TextEncoder().encode(payload); + const signature = new Uint8Array( + await crypto.subtle.sign( + { name: "Ed25519" }, + config.privateKey, + dssePreAuthenticationEncoding(payloadType, payloadBytes), + ), + ); + const envelope: SignedFeedEnvelope = { + payloadType, + payload: base64UrlEncode(payloadBytes), + signatures: [ + { + keyid: config.keyId, + sig: base64UrlEncode(signature), + }, + ], + }; + const body = JSON.stringify(envelope); + const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(body)); + return { + envelope, + body, + sha256: toHex(new Uint8Array(digest)), + }; +} + +export async function signCatalogFeedPayload(payload: string, config: FeedSigningConfig) { + return await signFeedPayload(OPENCLAW_CATALOG_FEED_PAYLOAD_TYPE, payload, config); +} + +export async function signedCatalogFeedV1Handler( + ctx: ActionCtx, + request: Request, + env: Record = process.env, +) { + let signingConfig: FeedSigningConfig | null; + try { + signingConfig = await resolveFeedSigningConfig(env); + } catch { + return catalogFeedUnavailableResponse("Signed catalog feed is unavailable"); + } + if (!signingConfig) { + return catalogFeedUnavailableResponse("Signed catalog feed is unavailable"); + } + + const publication = await ctx.runQuery(internal.catalogFeed.getLatestPublication, { + feedId: CATALOG_FEED_ID, + }); + if (!publication) return catalogFeedUnavailableResponse(); + + const signed = await signCatalogFeedPayload(publication.payload, signingConfig); + const etag = `"sha256:${signed.sha256}"`; + const headers = new Headers( + catalogFeedResponseHeaders(publication, { + representationSha256: signed.sha256, + additionalHeaders: { + "Content-Type": "application/vnd.dsse+json; charset=utf-8", + "X-Catalog-Payload-SHA256": publication.payloadSha256, + "X-OpenClaw-Feed-Signing-Key-ID": signingConfig.keyId, + }, + }), + ); + headers.delete("Last-Modified"); + if (matchesEtag(request, etag)) { + return new Response(null, { status: 304, headers }); + } + return new Response(signed.body, { status: 200, headers }); +} + +export const signedCatalogFeedV1Http = httpAction(signedCatalogFeedV1Handler); diff --git a/convex/httpApiV1/catalogFeedV1.ts b/convex/httpApiV1/catalogFeedV1.ts index b98046cb2a..ab0df36daa 100644 --- a/convex/httpApiV1/catalogFeedV1.ts +++ b/convex/httpApiV1/catalogFeedV1.ts @@ -3,7 +3,7 @@ import { internal } from "../_generated/api"; import type { ActionCtx } from "../_generated/server"; import { corsHeaders, mergeHeaders } from "../lib/httpHeaders"; -function matchesEtag(request: Request, etag: string) { +export function matchesEtag(request: Request, etag: string) { const header = request.headers.get("if-none-match"); if (!header) return false; return header @@ -16,7 +16,7 @@ function matchesEtag(request: Request, etag: string) { }); } -function matchesLastModified(request: Request, publishedAt: number) { +export function matchesLastModified(request: Request, publishedAt: number) { const header = request.headers.get("if-modified-since"); if (!header) return false; const since = Date.parse(header); @@ -24,6 +24,48 @@ function matchesLastModified(request: Request, publishedAt: number) { return Math.floor(publishedAt / 1000) * 1000 <= since; } +export function catalogFeedUnavailableResponse(message = "Catalog feed is not published") { + return new Response(message, { + status: 503, + headers: mergeHeaders( + { + "Content-Type": "text/plain; charset=utf-8", + "Cache-Control": "no-store", + }, + corsHeaders(), + ), + }); +} + +export function catalogFeedResponseHeaders( + publication: { + sequence: number; + payloadSha256: string; + publishedAt: number; + }, + options?: { + representationSha256?: string; + additionalHeaders?: Record; + }, +) { + const representationSha256 = options?.representationSha256 ?? publication.payloadSha256; + return mergeHeaders( + { + "Content-Type": "application/json; charset=utf-8", + "Cache-Control": "public, max-age=60, s-maxage=300, stale-while-revalidate=86400", + "Surrogate-Control": "max-age=300, stale-while-revalidate=86400", + ETag: `"sha256:${representationSha256}"`, + "Last-Modified": new Date(publication.publishedAt).toUTCString(), + "X-Catalog-Feed-Sequence": String(publication.sequence), + "X-Content-SHA256": representationSha256, + "X-Content-Type-Options": "nosniff", + Vary: "Accept-Encoding", + ...options?.additionalHeaders, + }, + corsHeaders(), + ); +} + export async function catalogFeedV1Handler( ctx: ActionCtx, request: Request, @@ -34,33 +76,11 @@ export async function catalogFeedV1Handler( ) { const publication = await ctx.runQuery(internal.catalogFeed.getLatestPublication, { feedId }); if (!publication) { - return new Response("Catalog feed is not published", { - status: 503, - headers: mergeHeaders( - { - "Content-Type": "text/plain; charset=utf-8", - "Cache-Control": "no-store", - }, - corsHeaders(), - ), - }); + return catalogFeedUnavailableResponse(); } const etag = `"sha256:${publication.payloadSha256}"`; - const headers = mergeHeaders( - { - "Content-Type": "application/json; charset=utf-8", - "Cache-Control": "public, max-age=60, s-maxage=300, stale-while-revalidate=86400", - "Surrogate-Control": "max-age=300, stale-while-revalidate=86400", - ETag: etag, - "Last-Modified": new Date(publication.publishedAt).toUTCString(), - "X-Catalog-Feed-Sequence": String(publication.sequence), - "X-Content-SHA256": publication.payloadSha256, - "X-Content-Type-Options": "nosniff", - Vary: "Accept-Encoding", - }, - corsHeaders(), - ); + const headers = catalogFeedResponseHeaders(publication); if ( matchesEtag(request, etag) || diff --git a/convex/lib/retentionPolicy.ts b/convex/lib/retentionPolicy.ts index e24afafa7e..743da16bb7 100644 --- a/convex/lib/retentionPolicy.ts +++ b/convex/lib/retentionPolicy.ts @@ -207,6 +207,18 @@ export const RETENTION_POLICIES = { packageModerationEventLogs: permanent("Package moderation event audit log."), officialPluginMigrations: permanent("Official plugin migration state."), catalogFeedPublications: permanent("Current published hosted catalog feed snapshot."), + catalogFeedRevisions: ephemeral("Revision boundaries for bounded catalog feed deltas.", { + expirationField: "expirationTime", + expirationIndex: "by_expiration_time", + prune: "catalogFeed.pruneCatalogFeedHistoryInternal", + retention: "Thirty days after publication.", + }), + catalogFeedChanges: ephemeral("Bounded catalog feed change journal.", { + expirationField: "expirationTime", + expirationIndex: "by_expiration_time", + prune: "catalogFeed.pruneCatalogFeedHistoryInternal", + retention: "Thirty days after publication.", + }), stars: permanent("User star records."), promotions: permanent("Curated promotional offers; ended records stay for launch-page history."), auditLogs: permanent("Audit logs are durable compliance/security history."), diff --git a/convex/schema.ts b/convex/schema.ts index 8db4c1d461..7be50d21f2 100644 --- a/convex/schema.ts +++ b/convex/schema.ts @@ -2664,6 +2664,33 @@ const catalogFeedPublications = defineTable({ publishedAt: v.number(), }).index("by_feed", ["feedId"]); +const catalogFeedRevisions = defineTable({ + feedId: v.string(), + sequence: v.number(), + changeCount: v.number(), + cumulativeChangeCount: v.number(), + generatedAt: v.string(), + expiresAt: v.string(), + description: v.string(), + publishedAt: v.number(), + expirationTime: v.number(), +}) + .index("by_feed_and_sequence", ["feedId", "sequence"]) + .index("by_expiration_time", ["expirationTime"]); + +const catalogFeedChanges = defineTable({ + feedId: v.string(), + sequence: v.number(), + ordinal: v.number(), + operation: v.union(v.literal("upsert"), v.literal("remove"), v.literal("metadata")), + entryType: v.optional(v.union(v.literal("plugin"), v.literal("skill"))), + entryId: v.optional(v.string()), + payload: v.string(), + expirationTime: v.number(), +}) + .index("by_feed_and_sequence_and_ordinal", ["feedId", "sequence", "ordinal"]) + .index("by_expiration_time", ["expirationTime"]); + const stars = defineTable({ skillId: v.id("skills"), userId: v.id("users"), @@ -3383,6 +3410,8 @@ export default defineSchema({ packageModerationEventLogs, officialPluginMigrations, catalogFeedPublications, + catalogFeedRevisions, + catalogFeedChanges, stars, promotions, auditLogs, diff --git a/packages/schema/dist/catalogFeedDistribution.d.ts b/packages/schema/dist/catalogFeedDistribution.d.ts new file mode 100644 index 0000000000..8e481854a9 --- /dev/null +++ b/packages/schema/dist/catalogFeedDistribution.d.ts @@ -0,0 +1,319 @@ +import { type inferred } from "arktype"; +export declare const CATALOG_FEED_QUERY_PAYLOAD_TYPE = "openclaw.official-external-plugin-catalog-query-results.v1"; +export declare const CATALOG_FEED_CHANGES_PAYLOAD_TYPE = "openclaw.official-external-plugin-catalog-changes.v1"; +export declare const CATALOG_FEED_QUERY_MAX_ENTRIES = 200; +export declare const CATALOG_FEED_CHANGES_MAX_RECORDS = 500; +export declare const CATALOG_FEED_DESCRIPTION_MAX_BYTES = 1024; +export declare const CatalogFeedQuerySchema: import("arktype/internal/variants/object.ts").ObjectType<{ + text?: string | undefined; + types?: ("plugin" | "skill")[] | undefined; + states?: ("available" | "blocked" | "deprecated" | "disabled" | "recommended")[] | undefined; + publisherIds?: string[] | undefined; +}, {}>; +export type CatalogFeedQuery = (typeof CatalogFeedQuerySchema)[inferred]; +export declare const CatalogFeedMetadataSchema: import("arktype/internal/variants/object.ts").ObjectType<{ + description: string | null; +}, {}>; +export type CatalogFeedMetadata = (typeof CatalogFeedMetadataSchema)[inferred]; +export declare const CatalogFeedUpsertChangeSchema: import("arktype/internal/variants/object.ts").ObjectType<{ + sequence: number; + operation: "upsert"; + entry: { + id: string; + title: string; + version: string; + state: "available" | "blocked" | "deprecated" | "disabled" | "recommended"; + featured?: boolean | undefined; + publisher: { + id: string; + trust: "community" | "official"; + }; + install: { + candidates: { + sourceRef: string; + package: string; + version: string; + integrity: string; + github?: { + repo: string; + path: string; + commit: string; + contentHash: string; + } | undefined; + }[]; + }; + type: "plugin"; + } | { + id: string; + title: string; + version: string; + state: "available" | "blocked" | "deprecated" | "disabled" | "recommended"; + featured?: boolean | undefined; + publisher: { + id: string; + trust: "community" | "official"; + }; + install: { + candidates: { + sourceRef: string; + package: string; + version: string; + integrity: string; + github?: { + repo: string; + path: string; + commit: string; + contentHash: string; + } | undefined; + }[]; + }; + type: "skill"; + }; +}, {}>; +export declare const CatalogFeedRemoveChangeSchema: import("arktype/internal/variants/object.ts").ObjectType<{ + sequence: number; + operation: "remove"; + entryId: string; + entryType: "plugin" | "skill"; +}, {}>; +export declare const CatalogFeedMetadataChangeSchema: import("arktype/internal/variants/object.ts").ObjectType<{ + sequence: number; + operation: "metadata"; + metadata: { + description: string | null; + }; +}, {}>; +export declare const CatalogFeedChangeSchema: import("arktype/internal/variants/object.ts").ObjectType<{ + sequence: number; + operation: "upsert"; + entry: { + id: string; + title: string; + version: string; + state: "available" | "blocked" | "deprecated" | "disabled" | "recommended"; + featured?: boolean | undefined; + publisher: { + id: string; + trust: "community" | "official"; + }; + install: { + candidates: { + sourceRef: string; + package: string; + version: string; + integrity: string; + github?: { + repo: string; + path: string; + commit: string; + contentHash: string; + } | undefined; + }[]; + }; + type: "plugin"; + } | { + id: string; + title: string; + version: string; + state: "available" | "blocked" | "deprecated" | "disabled" | "recommended"; + featured?: boolean | undefined; + publisher: { + id: string; + trust: "community" | "official"; + }; + install: { + candidates: { + sourceRef: string; + package: string; + version: string; + integrity: string; + github?: { + repo: string; + path: string; + commit: string; + contentHash: string; + } | undefined; + }[]; + }; + type: "skill"; + }; +} | { + sequence: number; + operation: "remove"; + entryId: string; + entryType: "plugin" | "skill"; +} | { + sequence: number; + operation: "metadata"; + metadata: { + description: string | null; + }; +}, {}>; +export type CatalogFeedChange = (typeof CatalogFeedChangeSchema)[inferred]; +export declare const CatalogFeedQueryPageSchema: import("arktype/internal/variants/object.ts").ObjectType<{ + schemaVersion: number; + feedId: string; + sequence: number; + generatedAt: string; + expiresAt: string; + query: { + text?: string | undefined; + types?: ("plugin" | "skill")[] | undefined; + states?: ("available" | "blocked" | "deprecated" | "disabled" | "recommended")[] | undefined; + publisherIds?: string[] | undefined; + }; + requestCursor: string | null; + pageIndex: number; + startIndex: number; + resultCount: number; + entries: ({ + id: string; + title: string; + version: string; + state: "available" | "blocked" | "deprecated" | "disabled" | "recommended"; + featured?: boolean | undefined; + publisher: { + id: string; + trust: "community" | "official"; + }; + install: { + candidates: { + sourceRef: string; + package: string; + version: string; + integrity: string; + github?: { + repo: string; + path: string; + commit: string; + contentHash: string; + } | undefined; + }[]; + }; + type: "plugin"; + } | { + id: string; + title: string; + version: string; + state: "available" | "blocked" | "deprecated" | "disabled" | "recommended"; + featured?: boolean | undefined; + publisher: { + id: string; + trust: "community" | "official"; + }; + install: { + candidates: { + sourceRef: string; + package: string; + version: string; + integrity: string; + github?: { + repo: string; + path: string; + commit: string; + contentHash: string; + } | undefined; + }[]; + }; + type: "skill"; + })[]; + nextCursor: string | null; +}, {}>; +export type CatalogFeedQueryPage = (typeof CatalogFeedQueryPageSchema)[inferred]; +export declare const CatalogFeedChangePageSchema: import("arktype/internal/variants/object.ts").ObjectType<{ + schemaVersion: number; + feedId: string; + fromSequence: number; + toSequence: number; + generatedAt: string; + expiresAt: string; + requestCursor: string | null; + pageIndex: number; + startIndex: number; + changeCount: number; + changes: ({ + sequence: number; + operation: "upsert"; + entry: { + id: string; + title: string; + version: string; + state: "available" | "blocked" | "deprecated" | "disabled" | "recommended"; + featured?: boolean | undefined; + publisher: { + id: string; + trust: "community" | "official"; + }; + install: { + candidates: { + sourceRef: string; + package: string; + version: string; + integrity: string; + github?: { + repo: string; + path: string; + commit: string; + contentHash: string; + } | undefined; + }[]; + }; + type: "plugin"; + } | { + id: string; + title: string; + version: string; + state: "available" | "blocked" | "deprecated" | "disabled" | "recommended"; + featured?: boolean | undefined; + publisher: { + id: string; + trust: "community" | "official"; + }; + install: { + candidates: { + sourceRef: string; + package: string; + version: string; + integrity: string; + github?: { + repo: string; + path: string; + commit: string; + contentHash: string; + } | undefined; + }[]; + }; + type: "skill"; + }; + } | { + sequence: number; + operation: "remove"; + entryId: string; + entryType: "plugin" | "skill"; + } | { + sequence: number; + operation: "metadata"; + metadata: { + description: string | null; + }; + })[]; + nextCursor: string | null; +}, {}>; +export type CatalogFeedChangePage = (typeof CatalogFeedChangePageSchema)[inferred]; +export declare const CatalogFeedResetRequiredSchema: import("arktype/internal/variants/object.ts").ObjectType<{ + schemaVersion: number; + feedId: string; + fromSequence: number; + currentSequence: number; + generatedAt: string; + expiresAt: string; + resetRequired: true; + snapshotUrl: string; +}, {}>; +export type CatalogFeedResetRequired = (typeof CatalogFeedResetRequiredSchema)[inferred]; +export declare function normalizeCatalogFeedQuery(value: unknown): CatalogFeedQuery; +export declare function parseCatalogFeedQueryPage(value: unknown): CatalogFeedQueryPage; +export declare function parseCatalogFeedQueryPages(values: readonly unknown[]): CatalogFeedQueryPage[]; +export declare function parseCatalogFeedChangePage(value: unknown): CatalogFeedChangePage; +export declare function parseCatalogFeedChangePages(values: readonly unknown[]): CatalogFeedChangePage[]; +export declare function parseCatalogFeedResetRequired(value: unknown): CatalogFeedResetRequired; diff --git a/packages/schema/dist/catalogFeedDistribution.js b/packages/schema/dist/catalogFeedDistribution.js new file mode 100644 index 0000000000..88a9c46e6f --- /dev/null +++ b/packages/schema/dist/catalogFeedDistribution.js @@ -0,0 +1,458 @@ +import { type } from "arktype"; +import { CatalogFeedEntrySchema, CatalogFeedStateSchema, CATALOG_FEED_SCHEMA_VERSION, } from "./catalogFeed.js"; +export const CATALOG_FEED_QUERY_PAYLOAD_TYPE = "openclaw.official-external-plugin-catalog-query-results.v1"; +export const CATALOG_FEED_CHANGES_PAYLOAD_TYPE = "openclaw.official-external-plugin-catalog-changes.v1"; +export const CATALOG_FEED_QUERY_MAX_ENTRIES = 200; +export const CATALOG_FEED_CHANGES_MAX_RECORDS = 500; +export const CATALOG_FEED_DESCRIPTION_MAX_BYTES = 1_024; +export const CatalogFeedQuerySchema = type({ + "+": "reject", + text: "string?", + types: type('"plugin"|"skill"').array().optional(), + states: CatalogFeedStateSchema.array().optional(), + publisherIds: type("string").array().optional(), +}); +export const CatalogFeedMetadataSchema = type({ + "+": "reject", + description: "string|null", +}); +export const CatalogFeedUpsertChangeSchema = type({ + "+": "reject", + sequence: "number", + operation: '"upsert"', + entry: CatalogFeedEntrySchema, +}); +export const CatalogFeedRemoveChangeSchema = type({ + "+": "reject", + sequence: "number", + operation: '"remove"', + entryId: "string", + entryType: '"plugin"|"skill"', +}); +export const CatalogFeedMetadataChangeSchema = type({ + "+": "reject", + sequence: "number", + operation: '"metadata"', + metadata: CatalogFeedMetadataSchema, +}); +export const CatalogFeedChangeSchema = type(CatalogFeedUpsertChangeSchema.or(CatalogFeedRemoveChangeSchema).or(CatalogFeedMetadataChangeSchema)); +export const CatalogFeedQueryPageSchema = type({ + "+": "reject", + schemaVersion: "number", + feedId: "string", + sequence: "number", + generatedAt: "string", + expiresAt: "string", + query: CatalogFeedQuerySchema, + requestCursor: "string|null", + pageIndex: "number", + startIndex: "number", + resultCount: "number", + entries: CatalogFeedEntrySchema.array(), + nextCursor: "string|null", +}); +export const CatalogFeedChangePageSchema = type({ + "+": "reject", + schemaVersion: "number", + feedId: "string", + fromSequence: "number", + toSequence: "number", + generatedAt: "string", + expiresAt: "string", + requestCursor: "string|null", + pageIndex: "number", + startIndex: "number", + changeCount: "number", + changes: CatalogFeedChangeSchema.array(), + nextCursor: "string|null", +}); +export const CatalogFeedResetRequiredSchema = type({ + "+": "reject", + schemaVersion: "number", + feedId: "string", + fromSequence: "number", + currentSequence: "number", + generatedAt: "string", + expiresAt: "string", + resetRequired: "true", + snapshotUrl: "string", +}); +const utf8Length = (value) => new TextEncoder().encode(value).length; +function requireBoundedString(value, name, maxBytes) { + const length = utf8Length(value); + if (length < 1 || length > maxBytes) { + throw new Error(`${name} must be between 1 and ${maxBytes} UTF-8 bytes`); + } +} +function requireNonNegativeInteger(value, name) { + if (!Number.isSafeInteger(value) || value < 0) { + throw new Error(`${name} must be a non-negative safe integer`); + } +} +function requireValidWindow(generatedAt, expiresAt) { + requireBoundedString(generatedAt, "Catalog feed projection generatedAt", 64); + requireBoundedString(expiresAt, "Catalog feed projection expiresAt", 64); + const isRfc3339Instant = (value) => { + const match = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.\d+)?(?:Z|([+-])(\d{2}):(\d{2}))$/u.exec(value); + if (!match) + return false; + const [year, month, day, hour, minute, second, offsetHour, offsetMinute] = [ + match[1], + match[2], + match[3], + match[4], + match[5], + match[6], + match[8] ?? "0", + match[9] ?? "0", + ].map(Number); + const leapYear = year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0); + const daysInMonth = [31, leapYear ? 29 : 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]; + return (month >= 1 && + month <= 12 && + day >= 1 && + day <= daysInMonth[month - 1] && + hour <= 23 && + minute <= 59 && + second <= 59 && + (offsetHour === 0 || (offsetHour <= 23 && offsetMinute <= 59)) && + Number.isFinite(Date.parse(value))); + }; + if (!isRfc3339Instant(generatedAt) || !isRfc3339Instant(expiresAt)) { + throw new Error("Catalog feed projection timestamps must use RFC 3339 syntax"); + } + const generatedAtMs = Date.parse(generatedAt); + const expiresAtMs = Date.parse(expiresAt); + if (!Number.isFinite(generatedAtMs) || !Number.isFinite(expiresAtMs)) { + throw new Error("Catalog feed projection timestamps must be valid dates"); + } + if (expiresAtMs <= generatedAtMs) { + throw new Error("Catalog feed projection expiresAt must be after generatedAt"); + } +} +function normalizeQueryTextWhitespace(value) { + let result = ""; + let pendingSpace = false; + for (const character of value.normalize("NFC")) { + const codePoint = character.codePointAt(0); + if ((codePoint >= 0x09 && codePoint <= 0x0d) || codePoint === 0x20) { + pendingSpace = result.length > 0; + continue; + } + if (pendingSpace) + result += " "; + result += character; + pendingSpace = false; + } + return result; +} +function sortedUnique(values) { + const encoder = new TextEncoder(); + const compareUtf8 = (left, right) => { + const leftBytes = encoder.encode(left); + const rightBytes = encoder.encode(right); + const length = Math.min(leftBytes.length, rightBytes.length); + for (let index = 0; index < length; index += 1) { + const difference = leftBytes[index] - rightBytes[index]; + if (difference !== 0) + return difference; + } + return leftBytes.length - rightBytes.length; + }; + return [...new Set(values)].sort(compareUtf8); +} +export function normalizeCatalogFeedQuery(value) { + const query = CatalogFeedQuerySchema.assert(value); + const normalized = {}; + if (query.text !== undefined) { + const text = normalizeQueryTextWhitespace(query.text); + requireBoundedString(text, "Catalog feed query text", 256); + normalized.text = text; + } + if (query.types !== undefined) { + const types = sortedUnique(query.types); + if (types.length === 0) + throw new Error("Catalog feed query types must not be empty"); + normalized.types = types; + } + if (query.states !== undefined) { + const states = sortedUnique(query.states); + if (states.length === 0) + throw new Error("Catalog feed query states must not be empty"); + normalized.states = states; + } + if (query.publisherIds !== undefined) { + const publisherIds = sortedUnique(query.publisherIds); + if (publisherIds.length < 1 || publisherIds.length > 100) { + throw new Error("Catalog feed query publisherIds must contain between 1 and 100 values"); + } + for (const publisherId of publisherIds) { + requireBoundedString(publisherId, "Catalog feed publisher id", 256); + } + normalized.publisherIds = publisherIds; + } + if (Object.keys(normalized).length === 0) { + throw new Error("Catalog feed query must include at least one filter"); + } + return normalized; +} +function queriesEqual(left, right) { + const arraysEqual = (first, second) => first === undefined + ? second === undefined + : second !== undefined && + first.length === second.length && + first.every((value, index) => value === second[index]); + return (left.text === right.text && + arraysEqual(left.types, right.types) && + arraysEqual(left.states, right.states) && + arraysEqual(left.publisherIds, right.publisherIds)); +} +function requireProjectionHeader(value) { + if (value.schemaVersion !== CATALOG_FEED_SCHEMA_VERSION) { + throw new Error(`Unsupported catalog feed projection schema version: ${value.schemaVersion}`); + } + requireBoundedString(value.feedId, "Catalog feed id", 256); + requireValidWindow(value.generatedAt, value.expiresAt); +} +function requirePageBounds(value) { + requireNonNegativeInteger(value.pageIndex, "Catalog feed projection pageIndex"); + requireNonNegativeInteger(value.startIndex, "Catalog feed projection startIndex"); + requireNonNegativeInteger(value.totalCount, "Catalog feed projection total count"); + if (value.requestCursor === null && (value.pageIndex !== 0 || value.startIndex !== 0)) { + throw new Error("Catalog feed projection first page must start at page and item index zero"); + } + if (value.requestCursor !== null && value.pageIndex === 0) { + throw new Error("Catalog feed projection continuation must have a positive page index"); + } + if (value.requestCursor !== null && value.startIndex < value.pageIndex) { + throw new Error("Catalog feed projection continuation offset cannot precede its page index"); + } + if (value.startIndex + value.itemCount > value.totalCount) { + throw new Error("Catalog feed projection page exceeds its declared total count"); + } + if (value.nextCursor === null && value.startIndex + value.itemCount !== value.totalCount) { + throw new Error("Catalog feed projection terminal page must end at its declared total count"); + } + if (value.nextCursor !== null && + (value.itemCount === 0 || value.startIndex + value.itemCount >= value.totalCount)) { + throw new Error("Catalog feed projection continuation must make progress before total count"); + } + if (value.nextCursor !== null && value.nextCursor === value.requestCursor) { + throw new Error("Catalog feed projection next cursor must differ from the request cursor"); + } + for (const cursor of [value.requestCursor, value.nextCursor]) { + if (cursor !== null && utf8Length(cursor) > 4096) { + throw new Error("Catalog feed projection cursor exceeds 4096 UTF-8 bytes"); + } + } +} +export function parseCatalogFeedQueryPage(value) { + const page = CatalogFeedQueryPageSchema.assert(value); + requireProjectionHeader(page); + requireNonNegativeInteger(page.sequence, "Catalog feed projection sequence"); + const normalizedQuery = normalizeCatalogFeedQuery(page.query); + if (!queriesEqual(normalizedQuery, page.query)) { + throw new Error("Catalog feed query page must carry the normalized query"); + } + if (page.entries.length > CATALOG_FEED_QUERY_MAX_ENTRIES) { + throw new Error(`Catalog feed query page exceeds ${CATALOG_FEED_QUERY_MAX_ENTRIES} entries`); + } + const entryKeys = new Set(); + for (const entry of page.entries) { + requireBoundedString(entry.id, "Catalog feed query entry id", 256); + const entryKey = `${entry.type}\0${entry.id}`; + if (entryKeys.has(entryKey)) { + throw new Error("Catalog feed query page contains duplicate entry identities"); + } + entryKeys.add(entryKey); + if (page.query.types && !page.query.types.includes(entry.type)) { + throw new Error("Catalog feed query entry does not match the requested types"); + } + if (page.query.states && !page.query.states.includes(entry.state)) { + throw new Error("Catalog feed query entry does not match the requested states"); + } + if (page.query.publisherIds && !page.query.publisherIds.includes(entry.publisher.id)) { + throw new Error("Catalog feed query entry does not match the requested publisherIds"); + } + } + requirePageBounds({ + requestCursor: page.requestCursor, + pageIndex: page.pageIndex, + startIndex: page.startIndex, + itemCount: page.entries.length, + totalCount: page.resultCount, + nextCursor: page.nextCursor, + }); + return page; +} +export function parseCatalogFeedQueryPages(values) { + if (values.length === 0) { + throw new Error("Catalog feed query page chain must not be empty"); + } + const pages = values.map(parseCatalogFeedQueryPage); + const first = pages[0]; + if (first.requestCursor !== null || first.pageIndex !== 0 || first.startIndex !== 0) { + throw new Error("Catalog feed query page chain must start at page and item index zero"); + } + const entryKeys = new Set(); + const consumedCursors = new Set(); + let expectedStartIndex = 0; + let expectedRequestCursor = null; + for (const [pageIndex, page] of pages.entries()) { + if (page.feedId !== first.feedId || + page.sequence !== first.sequence || + page.expiresAt !== first.expiresAt || + page.resultCount !== first.resultCount || + !queriesEqual(page.query, first.query)) { + throw new Error("Catalog feed query page chain changed its pinned projection"); + } + if (page.pageIndex !== pageIndex || + page.startIndex !== expectedStartIndex || + page.requestCursor !== expectedRequestCursor) { + throw new Error("Catalog feed query page chain contains a cursor, page, or offset gap"); + } + if (page.requestCursor !== null) { + if (consumedCursors.has(page.requestCursor)) { + throw new Error("Catalog feed query page chain reuses a continuation cursor"); + } + consumedCursors.add(page.requestCursor); + } + for (const entry of page.entries) { + const entryKey = `${entry.type}\0${entry.id}`; + if (entryKeys.has(entryKey)) { + throw new Error("Catalog feed query page chain contains duplicate entry identities"); + } + entryKeys.add(entryKey); + } + expectedStartIndex += page.entries.length; + expectedRequestCursor = page.nextCursor; + } + if (expectedRequestCursor !== null) { + throw new Error("Catalog feed query page chain must include its terminal page"); + } + return pages; +} +export function parseCatalogFeedChangePage(value) { + const page = CatalogFeedChangePageSchema.assert(value); + requireProjectionHeader(page); + requireNonNegativeInteger(page.fromSequence, "Catalog feed change fromSequence"); + requireNonNegativeInteger(page.toSequence, "Catalog feed change toSequence"); + if (page.toSequence < page.fromSequence) { + throw new Error("Catalog feed change toSequence must not precede fromSequence"); + } + if (page.changes.length > CATALOG_FEED_CHANGES_MAX_RECORDS) { + throw new Error(`Catalog feed change page exceeds ${CATALOG_FEED_CHANGES_MAX_RECORDS} records`); + } + let priorSequence = page.fromSequence; + for (const change of page.changes) { + requireNonNegativeInteger(change.sequence, "Catalog feed change sequence"); + if (change.sequence <= page.fromSequence || change.sequence > page.toSequence) { + throw new Error("Catalog feed change sequence is outside the requested range"); + } + if (change.sequence < priorSequence) { + throw new Error("Catalog feed changes must be ordered by sequence"); + } + if (change.sequence > priorSequence + 1 && + (page.pageIndex === 0 || priorSequence > page.fromSequence)) { + throw new Error("Catalog feed change page contains a missing revision"); + } + priorSequence = change.sequence; + if (change.operation === "remove") { + requireBoundedString(change.entryId, "Catalog feed removed entry id", 256); + } + else if (change.operation === "upsert") { + requireBoundedString(change.entry.id, "Catalog feed upsert entry id", 256); + } + else if (change.metadata.description !== null) { + requireBoundedString(change.metadata.description, "Catalog feed metadata description", CATALOG_FEED_DESCRIPTION_MAX_BYTES); + } + } + if (page.nextCursor === null) { + const terminalSequence = page.changes.at(-1)?.sequence ?? page.fromSequence; + if (terminalSequence !== page.toSequence) { + throw new Error("Catalog feed terminal change page must reach toSequence"); + } + } + requirePageBounds({ + requestCursor: page.requestCursor, + pageIndex: page.pageIndex, + startIndex: page.startIndex, + itemCount: page.changes.length, + totalCount: page.changeCount, + nextCursor: page.nextCursor, + }); + return page; +} +export function parseCatalogFeedChangePages(values) { + if (values.length === 0) { + throw new Error("Catalog feed change page chain must not be empty"); + } + const pages = values.map(parseCatalogFeedChangePage); + const first = pages[0]; + if (first.requestCursor !== null || first.pageIndex !== 0 || first.startIndex !== 0) { + throw new Error("Catalog feed change page chain must start at page and item index zero"); + } + let expectedStartIndex = 0; + let expectedRequestCursor = null; + let priorSequence = first.fromSequence; + const consumedCursors = new Set(); + for (const [pageIndex, page] of pages.entries()) { + if (page.feedId !== first.feedId || + page.fromSequence !== first.fromSequence || + page.toSequence !== first.toSequence || + page.expiresAt !== first.expiresAt || + page.changeCount !== first.changeCount) { + throw new Error("Catalog feed change page chain changed its pinned range"); + } + if (page.pageIndex !== pageIndex || + page.startIndex !== expectedStartIndex || + page.requestCursor !== expectedRequestCursor) { + throw new Error("Catalog feed change page chain contains a cursor, page, or offset gap"); + } + if (page.requestCursor !== null) { + if (consumedCursors.has(page.requestCursor)) { + throw new Error("Catalog feed change page chain reuses a continuation cursor"); + } + consumedCursors.add(page.requestCursor); + } + for (const change of page.changes) { + if (change.sequence !== priorSequence && change.sequence !== priorSequence + 1) { + throw new Error("Catalog feed change page chain contains a missing revision"); + } + priorSequence = change.sequence; + } + expectedStartIndex += page.changes.length; + expectedRequestCursor = page.nextCursor; + } + if (expectedRequestCursor !== null) { + throw new Error("Catalog feed change page chain must include its terminal page"); + } + if (priorSequence !== first.toSequence) { + throw new Error("Catalog feed change page chain must cover every revision"); + } + return pages; +} +export function parseCatalogFeedResetRequired(value) { + const reset = CatalogFeedResetRequiredSchema.assert(value); + requireProjectionHeader(reset); + requireNonNegativeInteger(reset.fromSequence, "Catalog feed reset fromSequence"); + requireNonNegativeInteger(reset.currentSequence, "Catalog feed reset currentSequence"); + if (reset.currentSequence <= reset.fromSequence) { + throw new Error("Catalog feed reset currentSequence must follow fromSequence"); + } + if (utf8Length(reset.snapshotUrl) > 2048) { + throw new Error("Catalog feed reset snapshotUrl exceeds 2048 UTF-8 bytes"); + } + let snapshotUrl; + try { + snapshotUrl = new URL(reset.snapshotUrl); + } + catch { + throw new Error("Catalog feed reset snapshotUrl must be absolute HTTPS"); + } + if (snapshotUrl.protocol !== "https:" || snapshotUrl.username || snapshotUrl.password) { + throw new Error("Catalog feed reset snapshotUrl must be absolute HTTPS without credentials"); + } + return reset; +} +//# sourceMappingURL=catalogFeedDistribution.js.map \ No newline at end of file diff --git a/packages/schema/dist/catalogFeedDistribution.js.map b/packages/schema/dist/catalogFeedDistribution.js.map new file mode 100644 index 0000000000..697f9d73bd --- /dev/null +++ b/packages/schema/dist/catalogFeedDistribution.js.map @@ -0,0 +1 @@ +{"version":3,"file":"catalogFeedDistribution.js","sourceRoot":"","sources":["../src/catalogFeedDistribution.ts"],"names":[],"mappings":"AAAA,OAAO,EAAiB,IAAI,EAAE,MAAM,SAAS,CAAC;AAC9C,OAAO,EACL,sBAAsB,EACtB,sBAAsB,EACtB,2BAA2B,GAC5B,MAAM,kBAAkB,CAAC;AAE1B,MAAM,CAAC,MAAM,+BAA+B,GAC1C,4DAA4D,CAAC;AAC/D,MAAM,CAAC,MAAM,iCAAiC,GAC5C,sDAAsD,CAAC;AACzD,MAAM,CAAC,MAAM,8BAA8B,GAAG,GAAG,CAAC;AAClD,MAAM,CAAC,MAAM,gCAAgC,GAAG,GAAG,CAAC;AACpD,MAAM,CAAC,MAAM,kCAAkC,GAAG,KAAK,CAAC;AAExD,MAAM,CAAC,MAAM,sBAAsB,GAAG,IAAI,CAAC;IACzC,GAAG,EAAE,QAAQ;IACb,IAAI,EAAE,SAAS;IACf,KAAK,EAAE,IAAI,CAAC,kBAAkB,CAAC,CAAC,KAAK,EAAE,CAAC,QAAQ,EAAE;IAClD,MAAM,EAAE,sBAAsB,CAAC,KAAK,EAAE,CAAC,QAAQ,EAAE;IACjD,YAAY,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAC,QAAQ,EAAE;CAChD,CAAC,CAAC;AAGH,MAAM,CAAC,MAAM,yBAAyB,GAAG,IAAI,CAAC;IAC5C,GAAG,EAAE,QAAQ;IACb,WAAW,EAAE,aAAa;CAC3B,CAAC,CAAC;AAGH,MAAM,CAAC,MAAM,6BAA6B,GAAG,IAAI,CAAC;IAChD,GAAG,EAAE,QAAQ;IACb,QAAQ,EAAE,QAAQ;IAClB,SAAS,EAAE,UAAU;IACrB,KAAK,EAAE,sBAAsB;CAC9B,CAAC,CAAC;AACH,MAAM,CAAC,MAAM,6BAA6B,GAAG,IAAI,CAAC;IAChD,GAAG,EAAE,QAAQ;IACb,QAAQ,EAAE,QAAQ;IAClB,SAAS,EAAE,UAAU;IACrB,OAAO,EAAE,QAAQ;IACjB,SAAS,EAAE,kBAAkB;CAC9B,CAAC,CAAC;AACH,MAAM,CAAC,MAAM,+BAA+B,GAAG,IAAI,CAAC;IAClD,GAAG,EAAE,QAAQ;IACb,QAAQ,EAAE,QAAQ;IAClB,SAAS,EAAE,YAAY;IACvB,QAAQ,EAAE,yBAAyB;CACpC,CAAC,CAAC;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,IAAI,CACzC,6BAA6B,CAAC,EAAE,CAAC,6BAA6B,CAAC,CAAC,EAAE,CAChE,+BAA+B,CAChC,CACF,CAAC;AAGF,MAAM,CAAC,MAAM,0BAA0B,GAAG,IAAI,CAAC;IAC7C,GAAG,EAAE,QAAQ;IACb,aAAa,EAAE,QAAQ;IACvB,MAAM,EAAE,QAAQ;IAChB,QAAQ,EAAE,QAAQ;IAClB,WAAW,EAAE,QAAQ;IACrB,SAAS,EAAE,QAAQ;IACnB,KAAK,EAAE,sBAAsB;IAC7B,aAAa,EAAE,aAAa;IAC5B,SAAS,EAAE,QAAQ;IACnB,UAAU,EAAE,QAAQ;IACpB,WAAW,EAAE,QAAQ;IACrB,OAAO,EAAE,sBAAsB,CAAC,KAAK,EAAE;IACvC,UAAU,EAAE,aAAa;CAC1B,CAAC,CAAC;AAGH,MAAM,CAAC,MAAM,2BAA2B,GAAG,IAAI,CAAC;IAC9C,GAAG,EAAE,QAAQ;IACb,aAAa,EAAE,QAAQ;IACvB,MAAM,EAAE,QAAQ;IAChB,YAAY,EAAE,QAAQ;IACtB,UAAU,EAAE,QAAQ;IACpB,WAAW,EAAE,QAAQ;IACrB,SAAS,EAAE,QAAQ;IACnB,aAAa,EAAE,aAAa;IAC5B,SAAS,EAAE,QAAQ;IACnB,UAAU,EAAE,QAAQ;IACpB,WAAW,EAAE,QAAQ;IACrB,OAAO,EAAE,uBAAuB,CAAC,KAAK,EAAE;IACxC,UAAU,EAAE,aAAa;CAC1B,CAAC,CAAC;AAGH,MAAM,CAAC,MAAM,8BAA8B,GAAG,IAAI,CAAC;IACjD,GAAG,EAAE,QAAQ;IACb,aAAa,EAAE,QAAQ;IACvB,MAAM,EAAE,QAAQ;IAChB,YAAY,EAAE,QAAQ;IACtB,eAAe,EAAE,QAAQ;IACzB,WAAW,EAAE,QAAQ;IACrB,SAAS,EAAE,QAAQ;IACnB,aAAa,EAAE,MAAM;IACrB,WAAW,EAAE,QAAQ;CACtB,CAAC,CAAC;AAGH,MAAM,UAAU,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;AAE7E,SAAS,oBAAoB,CAAC,KAAa,EAAE,IAAY,EAAE,QAAgB;IACzE,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;IACjC,IAAI,MAAM,GAAG,CAAC,IAAI,MAAM,GAAG,QAAQ,EAAE,CAAC;QACpC,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,0BAA0B,QAAQ,cAAc,CAAC,CAAC;IAC3E,CAAC;AACH,CAAC;AAED,SAAS,yBAAyB,CAAC,KAAa,EAAE,IAAY;IAC5D,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QAC9C,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,sCAAsC,CAAC,CAAC;IACjE,CAAC;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,WAAmB,EAAE,SAAiB;IAChE,oBAAoB,CAAC,WAAW,EAAE,qCAAqC,EAAE,EAAE,CAAC,CAAC;IAC7E,oBAAoB,CAAC,SAAS,EAAE,mCAAmC,EAAE,EAAE,CAAC,CAAC;IACzE,MAAM,gBAAgB,GAAG,CAAC,KAAa,EAAE,EAAE;QACzC,MAAM,KAAK,GACT,yFAAyF,CAAC,IAAI,CAC5F,KAAK,CACN,CAAC;QACJ,IAAI,CAAC,KAAK;YAAE,OAAO,KAAK,CAAC;QACzB,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,YAAY,CAAC,GAAG;YACzE,KAAK,CAAC,CAAC,CAAC;YACR,KAAK,CAAC,CAAC,CAAC;YACR,KAAK,CAAC,CAAC,CAAC;YACR,KAAK,CAAC,CAAC,CAAC;YACR,KAAK,CAAC,CAAC,CAAC;YACR,KAAK,CAAC,CAAC,CAAC;YACR,KAAK,CAAC,CAAC,CAAC,IAAI,GAAG;YACf,KAAK,CAAC,CAAC,CAAC,IAAI,GAAG;SAChB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACd,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,GAAG,KAAK,CAAC,IAAI,IAAI,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC;QAC1E,MAAM,WAAW,GAAG,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;QACrF,OAAO,CACL,KAAK,IAAI,CAAC;YACV,KAAK,IAAI,EAAE;YACX,GAAG,IAAI,CAAC;YACR,GAAG,IAAI,WAAW,CAAC,KAAK,GAAG,CAAC,CAAE;YAC9B,IAAI,IAAI,EAAE;YACV,MAAM,IAAI,EAAE;YACZ,MAAM,IAAI,EAAE;YACZ,CAAC,UAAU,KAAK,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE,IAAI,YAAY,IAAI,EAAE,CAAC,CAAC;YAC9D,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CACnC,CAAC;IACJ,CAAC,CAAC;IACF,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,EAAE,CAAC;QACnE,MAAM,IAAI,KAAK,CAAC,6DAA6D,CAAC,CAAC;IACjF,CAAC;IACD,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IAC9C,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IAC1C,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QACrE,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;IAC5E,CAAC;IACD,IAAI,WAAW,IAAI,aAAa,EAAE,CAAC;QACjC,MAAM,IAAI,KAAK,CAAC,6DAA6D,CAAC,CAAC;IACjF,CAAC;AACH,CAAC;AAED,SAAS,4BAA4B,CAAC,KAAa;IACjD,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,IAAI,YAAY,GAAG,KAAK,CAAC;IACzB,KAAK,MAAM,SAAS,IAAI,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;QAC/C,MAAM,SAAS,GAAG,SAAS,CAAC,WAAW,CAAC,CAAC,CAAE,CAAC;QAC5C,IAAI,CAAC,SAAS,IAAI,IAAI,IAAI,SAAS,IAAI,IAAI,CAAC,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;YACnE,YAAY,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;YACjC,SAAS;QACX,CAAC;QACD,IAAI,YAAY;YAAE,MAAM,IAAI,GAAG,CAAC;QAChC,MAAM,IAAI,SAAS,CAAC;QACpB,YAAY,GAAG,KAAK,CAAC;IACvB,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,YAAY,CAAC,MAAyB;IAC7C,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;IAClC,MAAM,WAAW,GAAG,CAAC,IAAY,EAAE,KAAa,EAAE,EAAE;QAClD,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACvC,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACzC,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,MAAM,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;QAC7D,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;YAC/C,MAAM,UAAU,GAAG,SAAS,CAAC,KAAK,CAAE,GAAG,UAAU,CAAC,KAAK,CAAE,CAAC;YAC1D,IAAI,UAAU,KAAK,CAAC;gBAAE,OAAO,UAAU,CAAC;QAC1C,CAAC;QACD,OAAO,SAAS,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC;IAC9C,CAAC,CAAC;IACF,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;AAChD,CAAC;AAED,MAAM,UAAU,yBAAyB,CAAC,KAAc;IACtD,MAAM,KAAK,GAAG,sBAAsB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACnD,MAAM,UAAU,GAAqB,EAAE,CAAC;IACxC,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,MAAM,IAAI,GAAG,4BAA4B,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACtD,oBAAoB,CAAC,IAAI,EAAE,yBAAyB,EAAE,GAAG,CAAC,CAAC;QAC3D,UAAU,CAAC,IAAI,GAAG,IAAI,CAAC;IACzB,CAAC;IACD,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,KAAK,CAA2C,CAAC;QAClF,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;QACtF,UAAU,CAAC,KAAK,GAAG,KAAK,CAAC;IAC3B,CAAC;IACD,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,MAAM,CAA4C,CAAC;QACrF,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;QACxF,UAAU,CAAC,MAAM,GAAG,MAAM,CAAC;IAC7B,CAAC;IACD,IAAI,KAAK,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;QACrC,MAAM,YAAY,GAAG,YAAY,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QACtD,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,IAAI,YAAY,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YACzD,MAAM,IAAI,KAAK,CAAC,uEAAuE,CAAC,CAAC;QAC3F,CAAC;QACD,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE,CAAC;YACvC,oBAAoB,CAAC,WAAW,EAAE,2BAA2B,EAAE,GAAG,CAAC,CAAC;QACtE,CAAC;QACD,UAAU,CAAC,YAAY,GAAG,YAAY,CAAC;IACzC,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzC,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;IACzE,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,SAAS,YAAY,CAAC,IAAsB,EAAE,KAAuB;IACnE,MAAM,WAAW,GAAG,CAAC,KAAyB,EAAE,MAA0B,EAAE,EAAE,CAC5E,KAAK,KAAK,SAAS;QACjB,CAAC,CAAC,MAAM,KAAK,SAAS;QACtB,CAAC,CAAC,MAAM,KAAK,SAAS;YACpB,KAAK,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM;YAC9B,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IAC7D,OAAO,CACL,IAAI,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI;QACxB,WAAW,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC;QACpC,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC;QACtC,WAAW,CAAC,IAAI,CAAC,YAAY,EAAE,KAAK,CAAC,YAAY,CAAC,CACnD,CAAC;AACJ,CAAC;AAED,SAAS,uBAAuB,CAAC,KAKhC;IACC,IAAI,KAAK,CAAC,aAAa,KAAK,2BAA2B,EAAE,CAAC;QACxD,MAAM,IAAI,KAAK,CAAC,uDAAuD,KAAK,CAAC,aAAa,EAAE,CAAC,CAAC;IAChG,CAAC;IACD,oBAAoB,CAAC,KAAK,CAAC,MAAM,EAAE,iBAAiB,EAAE,GAAG,CAAC,CAAC;IAC3D,kBAAkB,CAAC,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;AACzD,CAAC;AAED,SAAS,iBAAiB,CAAC,KAO1B;IACC,yBAAyB,CAAC,KAAK,CAAC,SAAS,EAAE,mCAAmC,CAAC,CAAC;IAChF,yBAAyB,CAAC,KAAK,CAAC,UAAU,EAAE,oCAAoC,CAAC,CAAC;IAClF,yBAAyB,CAAC,KAAK,CAAC,UAAU,EAAE,qCAAqC,CAAC,CAAC;IACnF,IAAI,KAAK,CAAC,aAAa,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,SAAS,KAAK,CAAC,IAAI,KAAK,CAAC,UAAU,KAAK,CAAC,CAAC,EAAE,CAAC;QACtF,MAAM,IAAI,KAAK,CAAC,2EAA2E,CAAC,CAAC;IAC/F,CAAC;IACD,IAAI,KAAK,CAAC,aAAa,KAAK,IAAI,IAAI,KAAK,CAAC,SAAS,KAAK,CAAC,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CAAC,sEAAsE,CAAC,CAAC;IAC1F,CAAC;IACD,IAAI,KAAK,CAAC,aAAa,KAAK,IAAI,IAAI,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC;QACvE,MAAM,IAAI,KAAK,CAAC,2EAA2E,CAAC,CAAC;IAC/F,CAAC;IACD,IAAI,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC,SAAS,GAAG,KAAK,CAAC,UAAU,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CAAC,+DAA+D,CAAC,CAAC;IACnF,CAAC;IACD,IAAI,KAAK,CAAC,UAAU,KAAK,IAAI,IAAI,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC,SAAS,KAAK,KAAK,CAAC,UAAU,EAAE,CAAC;QACzF,MAAM,IAAI,KAAK,CAAC,4EAA4E,CAAC,CAAC;IAChG,CAAC;IACD,IACE,KAAK,CAAC,UAAU,KAAK,IAAI;QACzB,CAAC,KAAK,CAAC,SAAS,KAAK,CAAC,IAAI,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,UAAU,CAAC,EACjF,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,4EAA4E,CAAC,CAAC;IAChG,CAAC;IACD,IAAI,KAAK,CAAC,UAAU,KAAK,IAAI,IAAI,KAAK,CAAC,UAAU,KAAK,KAAK,CAAC,aAAa,EAAE,CAAC;QAC1E,MAAM,IAAI,KAAK,CAAC,yEAAyE,CAAC,CAAC;IAC7F,CAAC;IACD,KAAK,MAAM,MAAM,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC;QAC7D,IAAI,MAAM,KAAK,IAAI,IAAI,UAAU,CAAC,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC;YACjD,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;QAC7E,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,yBAAyB,CAAC,KAAc;IACtD,MAAM,IAAI,GAAG,0BAA0B,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACtD,uBAAuB,CAAC,IAAI,CAAC,CAAC;IAC9B,yBAAyB,CAAC,IAAI,CAAC,QAAQ,EAAE,kCAAkC,CAAC,CAAC;IAC7E,MAAM,eAAe,GAAG,yBAAyB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC9D,IAAI,CAAC,YAAY,CAAC,eAAe,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAC/C,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;IAC7E,CAAC;IACD,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,8BAA8B,EAAE,CAAC;QACzD,MAAM,IAAI,KAAK,CAAC,mCAAmC,8BAA8B,UAAU,CAAC,CAAC;IAC/F,CAAC;IACD,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;IACpC,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;QACjC,oBAAoB,CAAC,KAAK,CAAC,EAAE,EAAE,6BAA6B,EAAE,GAAG,CAAC,CAAC;QACnE,MAAM,QAAQ,GAAG,GAAG,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,EAAE,EAAE,CAAC;QAC9C,IAAI,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,6DAA6D,CAAC,CAAC;QACjF,CAAC;QACD,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACxB,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/D,MAAM,IAAI,KAAK,CAAC,6DAA6D,CAAC,CAAC;QACjF,CAAC;QACD,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;YAClE,MAAM,IAAI,KAAK,CAAC,8DAA8D,CAAC,CAAC;QAClF,CAAC;QACD,IAAI,IAAI,CAAC,KAAK,CAAC,YAAY,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,QAAQ,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,CAAC;YACrF,MAAM,IAAI,KAAK,CAAC,oEAAoE,CAAC,CAAC;QACxF,CAAC;IACH,CAAC;IACD,iBAAiB,CAAC;QAChB,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,UAAU,EAAE,IAAI,CAAC,UAAU;QAC3B,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM;QAC9B,UAAU,EAAE,IAAI,CAAC,WAAW;QAC5B,UAAU,EAAE,IAAI,CAAC,UAAU;KAC5B,CAAC,CAAC;IACH,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,0BAA0B,CAAC,MAA0B;IACnE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;IACrE,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC;IACpD,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC;IACxB,IAAI,KAAK,CAAC,aAAa,KAAK,IAAI,IAAI,KAAK,CAAC,SAAS,KAAK,CAAC,IAAI,KAAK,CAAC,UAAU,KAAK,CAAC,EAAE,CAAC;QACpF,MAAM,IAAI,KAAK,CAAC,sEAAsE,CAAC,CAAC;IAC1F,CAAC;IACD,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;IACpC,MAAM,eAAe,GAAG,IAAI,GAAG,EAAU,CAAC;IAC1C,IAAI,kBAAkB,GAAG,CAAC,CAAC;IAC3B,IAAI,qBAAqB,GAAkB,IAAI,CAAC;IAChD,KAAK,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;QAChD,IACE,IAAI,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM;YAC5B,IAAI,CAAC,QAAQ,KAAK,KAAK,CAAC,QAAQ;YAChC,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC,SAAS;YAClC,IAAI,CAAC,WAAW,KAAK,KAAK,CAAC,WAAW;YACtC,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,EACtC,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,6DAA6D,CAAC,CAAC;QACjF,CAAC;QACD,IACE,IAAI,CAAC,SAAS,KAAK,SAAS;YAC5B,IAAI,CAAC,UAAU,KAAK,kBAAkB;YACtC,IAAI,CAAC,aAAa,KAAK,qBAAqB,EAC5C,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,sEAAsE,CAAC,CAAC;QAC1F,CAAC;QACD,IAAI,IAAI,CAAC,aAAa,KAAK,IAAI,EAAE,CAAC;YAChC,IAAI,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC;gBAC5C,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAC;YAChF,CAAC;YACD,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC1C,CAAC;QACD,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjC,MAAM,QAAQ,GAAG,GAAG,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,EAAE,EAAE,CAAC;YAC9C,IAAI,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC5B,MAAM,IAAI,KAAK,CAAC,mEAAmE,CAAC,CAAC;YACvF,CAAC;YACD,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC1B,CAAC;QACD,kBAAkB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;QAC1C,qBAAqB,GAAG,IAAI,CAAC,UAAU,CAAC;IAC1C,CAAC;IACD,IAAI,qBAAqB,KAAK,IAAI,EAAE,CAAC;QACnC,MAAM,IAAI,KAAK,CAAC,8DAA8D,CAAC,CAAC;IAClF,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,0BAA0B,CAAC,KAAc;IACvD,MAAM,IAAI,GAAG,2BAA2B,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACvD,uBAAuB,CAAC,IAAI,CAAC,CAAC;IAC9B,yBAAyB,CAAC,IAAI,CAAC,YAAY,EAAE,kCAAkC,CAAC,CAAC;IACjF,yBAAyB,CAAC,IAAI,CAAC,UAAU,EAAE,gCAAgC,CAAC,CAAC;IAC7E,IAAI,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,8DAA8D,CAAC,CAAC;IAClF,CAAC;IACD,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,gCAAgC,EAAE,CAAC;QAC3D,MAAM,IAAI,KAAK,CAAC,oCAAoC,gCAAgC,UAAU,CAAC,CAAC;IAClG,CAAC;IACD,IAAI,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC;IACtC,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;QAClC,yBAAyB,CAAC,MAAM,CAAC,QAAQ,EAAE,8BAA8B,CAAC,CAAC;QAC3E,IAAI,MAAM,CAAC,QAAQ,IAAI,IAAI,CAAC,YAAY,IAAI,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;YAC9E,MAAM,IAAI,KAAK,CAAC,6DAA6D,CAAC,CAAC;QACjF,CAAC;QACD,IAAI,MAAM,CAAC,QAAQ,GAAG,aAAa,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;QACtE,CAAC;QACD,IACE,MAAM,CAAC,QAAQ,GAAG,aAAa,GAAG,CAAC;YACnC,CAAC,IAAI,CAAC,SAAS,KAAK,CAAC,IAAI,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC,EAC3D,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;QAC1E,CAAC;QACD,aAAa,GAAG,MAAM,CAAC,QAAQ,CAAC;QAChC,IAAI,MAAM,CAAC,SAAS,KAAK,QAAQ,EAAE,CAAC;YAClC,oBAAoB,CAAC,MAAM,CAAC,OAAO,EAAE,+BAA+B,EAAE,GAAG,CAAC,CAAC;QAC7E,CAAC;aAAM,IAAI,MAAM,CAAC,SAAS,KAAK,QAAQ,EAAE,CAAC;YACzC,oBAAoB,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE,8BAA8B,EAAE,GAAG,CAAC,CAAC;QAC7E,CAAC;aAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,WAAW,KAAK,IAAI,EAAE,CAAC;YAChD,oBAAoB,CAClB,MAAM,CAAC,QAAQ,CAAC,WAAW,EAC3B,mCAAmC,EACnC,kCAAkC,CACnC,CAAC;QACJ,CAAC;IACH,CAAC;IACD,IAAI,IAAI,CAAC,UAAU,KAAK,IAAI,EAAE,CAAC;QAC7B,MAAM,gBAAgB,GAAG,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,IAAI,IAAI,CAAC,YAAY,CAAC;QAC5E,IAAI,gBAAgB,KAAK,IAAI,CAAC,UAAU,EAAE,CAAC;YACzC,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;QAC7E,CAAC;IACH,CAAC;IACD,iBAAiB,CAAC;QAChB,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,UAAU,EAAE,IAAI,CAAC,UAAU;QAC3B,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM;QAC9B,UAAU,EAAE,IAAI,CAAC,WAAW;QAC5B,UAAU,EAAE,IAAI,CAAC,UAAU;KAC5B,CAAC,CAAC;IACH,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,2BAA2B,CAAC,MAA0B;IACpE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;IACtE,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC;IACrD,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC;IACxB,IAAI,KAAK,CAAC,aAAa,KAAK,IAAI,IAAI,KAAK,CAAC,SAAS,KAAK,CAAC,IAAI,KAAK,CAAC,UAAU,KAAK,CAAC,EAAE,CAAC;QACpF,MAAM,IAAI,KAAK,CAAC,uEAAuE,CAAC,CAAC;IAC3F,CAAC;IACD,IAAI,kBAAkB,GAAG,CAAC,CAAC;IAC3B,IAAI,qBAAqB,GAAkB,IAAI,CAAC;IAChD,IAAI,aAAa,GAAG,KAAK,CAAC,YAAY,CAAC;IACvC,MAAM,eAAe,GAAG,IAAI,GAAG,EAAU,CAAC;IAC1C,KAAK,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;QAChD,IACE,IAAI,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM;YAC5B,IAAI,CAAC,YAAY,KAAK,KAAK,CAAC,YAAY;YACxC,IAAI,CAAC,UAAU,KAAK,KAAK,CAAC,UAAU;YACpC,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC,SAAS;YAClC,IAAI,CAAC,WAAW,KAAK,KAAK,CAAC,WAAW,EACtC,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;QAC7E,CAAC;QACD,IACE,IAAI,CAAC,SAAS,KAAK,SAAS;YAC5B,IAAI,CAAC,UAAU,KAAK,kBAAkB;YACtC,IAAI,CAAC,aAAa,KAAK,qBAAqB,EAC5C,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,uEAAuE,CAAC,CAAC;QAC3F,CAAC;QACD,IAAI,IAAI,CAAC,aAAa,KAAK,IAAI,EAAE,CAAC;YAChC,IAAI,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC;gBAC5C,MAAM,IAAI,KAAK,CAAC,6DAA6D,CAAC,CAAC;YACjF,CAAC;YACD,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC1C,CAAC;QACD,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAClC,IAAI,MAAM,CAAC,QAAQ,KAAK,aAAa,IAAI,MAAM,CAAC,QAAQ,KAAK,aAAa,GAAG,CAAC,EAAE,CAAC;gBAC/E,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAC;YAChF,CAAC;YACD,aAAa,GAAG,MAAM,CAAC,QAAQ,CAAC;QAClC,CAAC;QACD,kBAAkB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;QAC1C,qBAAqB,GAAG,IAAI,CAAC,UAAU,CAAC;IAC1C,CAAC;IACD,IAAI,qBAAqB,KAAK,IAAI,EAAE,CAAC;QACnC,MAAM,IAAI,KAAK,CAAC,+DAA+D,CAAC,CAAC;IACnF,CAAC;IACD,IAAI,aAAa,KAAK,KAAK,CAAC,UAAU,EAAE,CAAC;QACvC,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAC;IAC9E,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,6BAA6B,CAAC,KAAc;IAC1D,MAAM,KAAK,GAAG,8BAA8B,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC3D,uBAAuB,CAAC,KAAK,CAAC,CAAC;IAC/B,yBAAyB,CAAC,KAAK,CAAC,YAAY,EAAE,iCAAiC,CAAC,CAAC;IACjF,yBAAyB,CAAC,KAAK,CAAC,eAAe,EAAE,oCAAoC,CAAC,CAAC;IACvF,IAAI,KAAK,CAAC,eAAe,IAAI,KAAK,CAAC,YAAY,EAAE,CAAC;QAChD,MAAM,IAAI,KAAK,CAAC,6DAA6D,CAAC,CAAC;IACjF,CAAC;IACD,IAAI,UAAU,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,IAAI,EAAE,CAAC;QACzC,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;IAC7E,CAAC;IACD,IAAI,WAAgB,CAAC;IACrB,IAAI,CAAC;QACH,WAAW,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IAC3C,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;IAC3E,CAAC;IACD,IAAI,WAAW,CAAC,QAAQ,KAAK,QAAQ,IAAI,WAAW,CAAC,QAAQ,IAAI,WAAW,CAAC,QAAQ,EAAE,CAAC;QACtF,MAAM,IAAI,KAAK,CAAC,2EAA2E,CAAC,CAAC;IAC/F,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"} \ No newline at end of file diff --git a/packages/schema/dist/index.d.ts b/packages/schema/dist/index.d.ts index daba3ca2c9..4b4c0f313f 100644 --- a/packages/schema/dist/index.d.ts +++ b/packages/schema/dist/index.d.ts @@ -1,6 +1,7 @@ export type { ArkValidator } from "./ark.js"; export { formatArkErrors, parseArk } from "./ark.js"; export * from "./catalogFeed.js"; +export * from "./catalogFeedDistribution.js"; export * from "./catalogMetadata.js"; export * from "./docsLinks.js"; export * from "./license.js"; diff --git a/packages/schema/dist/index.js b/packages/schema/dist/index.js index 34f9019cb2..5448628ca0 100644 --- a/packages/schema/dist/index.js +++ b/packages/schema/dist/index.js @@ -1,5 +1,6 @@ export { formatArkErrors, parseArk } from "./ark.js"; export * from "./catalogFeed.js"; +export * from "./catalogFeedDistribution.js"; export * from "./catalogMetadata.js"; export * from "./docsLinks.js"; export * from "./license.js"; diff --git a/packages/schema/dist/index.js.map b/packages/schema/dist/index.js.map index 2e8222e1b1..7aadc2eb7f 100644 --- a/packages/schema/dist/index.js.map +++ b/packages/schema/dist/index.js.map @@ -1 +1 @@ -{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AACrD,cAAc,kBAAkB,CAAC;AACjC,cAAc,sBAAsB,CAAC;AACrC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,cAAc,CAAC;AAC7B,cAAc,uBAAuB,CAAC;AACtC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,eAAe,CAAC;AAC9B,cAAc,uBAAuB,CAAC;AACtC,cAAc,qBAAqB,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AACzD,cAAc,cAAc,CAAC;AAC7B,cAAc,gBAAgB,CAAC"} \ No newline at end of file +{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AACrD,cAAc,kBAAkB,CAAC;AACjC,cAAc,8BAA8B,CAAC;AAC7C,cAAc,sBAAsB,CAAC;AACrC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,cAAc,CAAC;AAC7B,cAAc,uBAAuB,CAAC;AACtC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,eAAe,CAAC;AAC9B,cAAc,uBAAuB,CAAC;AACtC,cAAc,qBAAqB,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AACzD,cAAc,cAAc,CAAC;AAC7B,cAAc,gBAAgB,CAAC"} \ No newline at end of file diff --git a/packages/schema/dist/routes.d.ts b/packages/schema/dist/routes.d.ts index dc19578d4e..b354d132eb 100644 --- a/packages/schema/dist/routes.d.ts +++ b/packages/schema/dist/routes.d.ts @@ -24,6 +24,7 @@ export declare const ApiRoutes: { readonly bundlePlugins: "/api/v1/bundle-plugins"; readonly promotions: "/api/v1/promotions"; readonly catalogFeed: "/api/v1/feeds/plugins"; + readonly catalogFeedChanges: "/api/v1/feeds/plugins/changes"; readonly catalogSkillsFeed: "/api/v1/feeds/skills"; readonly promotionsFeed: "/api/v1/feeds/promotions"; readonly stars: "/api/v1/stars"; diff --git a/packages/schema/dist/routes.js b/packages/schema/dist/routes.js index e08aa86aa3..433aee22e3 100644 --- a/packages/schema/dist/routes.js +++ b/packages/schema/dist/routes.js @@ -24,6 +24,7 @@ export const ApiRoutes = { bundlePlugins: "/api/v1/bundle-plugins", promotions: "/api/v1/promotions", catalogFeed: "/api/v1/feeds/plugins", + catalogFeedChanges: "/api/v1/feeds/plugins/changes", catalogSkillsFeed: "/api/v1/feeds/skills", promotionsFeed: "/api/v1/feeds/promotions", stars: "/api/v1/stars", diff --git a/packages/schema/dist/routes.js.map b/packages/schema/dist/routes.js.map index 0b6b07fee3..753dfd27fe 100644 --- a/packages/schema/dist/routes.js.map +++ b/packages/schema/dist/routes.js.map @@ -1 +1 @@ -{"version":3,"file":"routes.js","sourceRoot":"","sources":["../src/routes.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,eAAe,GAAG;IAC7B,QAAQ,EAAE,eAAe;IACzB,MAAM,EAAE,aAAa;IACrB,KAAK,EAAE,YAAY;IACnB,YAAY,EAAE,oBAAoB;IAClC,SAAS,EAAE,iBAAiB;IAC5B,YAAY,EAAE,qBAAqB;IACnC,UAAU,EAAE,kBAAkB;IAC9B,mBAAmB,EAAE,4BAA4B;IACjD,cAAc,EAAE,uBAAuB;IACvC,gBAAgB,EAAE,yBAAyB;CACnC,CAAC;AAEX,MAAM,CAAC,MAAM,SAAS,GAAG;IACvB,MAAM,EAAE,gBAAgB;IACxB,OAAO,EAAE,iBAAiB;IAC1B,QAAQ,EAAE,kBAAkB;IAC5B,gBAAgB,EAAE,4BAA4B;IAC9C,MAAM,EAAE,gBAAgB;IACxB,UAAU,EAAE,uBAAuB;IACnC,OAAO,EAAE,iBAAiB;IAC1B,aAAa,EAAE,wBAAwB;IACvC,QAAQ,EAAE,kBAAkB;IAC5B,WAAW,EAAE,sBAAsB;IACnC,aAAa,EAAE,wBAAwB;IACvC,UAAU,EAAE,oBAAoB;IAChC,WAAW,EAAE,uBAAuB;IACpC,iBAAiB,EAAE,sBAAsB;IACzC,cAAc,EAAE,0BAA0B;IAC1C,KAAK,EAAE,eAAe;IACtB,SAAS,EAAE,mBAAmB;IAC9B,UAAU,EAAE,oBAAoB;IAChC,KAAK,EAAE,eAAe;IACtB,aAAa,EAAE,wBAAwB;IACvC,MAAM,EAAE,gBAAgB;IACxB,YAAY,EAAE,uBAAuB;CAC7B,CAAC"} \ No newline at end of file +{"version":3,"file":"routes.js","sourceRoot":"","sources":["../src/routes.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,eAAe,GAAG;IAC7B,QAAQ,EAAE,eAAe;IACzB,MAAM,EAAE,aAAa;IACrB,KAAK,EAAE,YAAY;IACnB,YAAY,EAAE,oBAAoB;IAClC,SAAS,EAAE,iBAAiB;IAC5B,YAAY,EAAE,qBAAqB;IACnC,UAAU,EAAE,kBAAkB;IAC9B,mBAAmB,EAAE,4BAA4B;IACjD,cAAc,EAAE,uBAAuB;IACvC,gBAAgB,EAAE,yBAAyB;CACnC,CAAC;AAEX,MAAM,CAAC,MAAM,SAAS,GAAG;IACvB,MAAM,EAAE,gBAAgB;IACxB,OAAO,EAAE,iBAAiB;IAC1B,QAAQ,EAAE,kBAAkB;IAC5B,gBAAgB,EAAE,4BAA4B;IAC9C,MAAM,EAAE,gBAAgB;IACxB,UAAU,EAAE,uBAAuB;IACnC,OAAO,EAAE,iBAAiB;IAC1B,aAAa,EAAE,wBAAwB;IACvC,QAAQ,EAAE,kBAAkB;IAC5B,WAAW,EAAE,sBAAsB;IACnC,aAAa,EAAE,wBAAwB;IACvC,UAAU,EAAE,oBAAoB;IAChC,WAAW,EAAE,uBAAuB;IACpC,kBAAkB,EAAE,+BAA+B;IACnD,iBAAiB,EAAE,sBAAsB;IACzC,cAAc,EAAE,0BAA0B;IAC1C,KAAK,EAAE,eAAe;IACtB,SAAS,EAAE,mBAAmB;IAC9B,UAAU,EAAE,oBAAoB;IAChC,KAAK,EAAE,eAAe;IACtB,aAAa,EAAE,wBAAwB;IACvC,MAAM,EAAE,gBAAgB;IACxB,YAAY,EAAE,uBAAuB;CAC7B,CAAC"} \ No newline at end of file diff --git a/packages/schema/src/catalogFeedDistribution.test.ts b/packages/schema/src/catalogFeedDistribution.test.ts new file mode 100644 index 0000000000..e396f7b49a --- /dev/null +++ b/packages/schema/src/catalogFeedDistribution.test.ts @@ -0,0 +1,366 @@ +import { describe, expect, it } from "vitest"; +import { + CATALOG_FEED_CHANGES_PAYLOAD_TYPE, + CATALOG_FEED_QUERY_PAYLOAD_TYPE, + normalizeCatalogFeedQuery, + parseCatalogFeedChangePages, + parseCatalogFeedChangePage, + parseCatalogFeedQueryPages, + parseCatalogFeedQueryPage, + parseCatalogFeedResetRequired, + type CatalogFeedChangePage, + type CatalogFeedQueryPage, +} from "./catalogFeedDistribution.js"; + +const entry = { + type: "plugin" as const, + id: "@openclaw/demo", + title: "Demo", + version: "1.2.3", + state: "available" as const, + publisher: { id: "openclaw", trust: "official" as const }, + install: { + candidates: [ + { + sourceRef: "public-clawhub", + package: "@openclaw/demo", + version: "1.2.3", + integrity: "sha256:abc", + }, + ], + }, +}; + +function queryPage(overrides: Partial = {}): CatalogFeedQueryPage { + return { + schemaVersion: 1, + feedId: "clawhub-official", + sequence: 4, + generatedAt: "2026-07-16T00:00:00.000Z", + expiresAt: "2026-07-16T00:05:00.000Z", + query: { text: "demo", types: ["plugin"] }, + requestCursor: null, + pageIndex: 0, + startIndex: 0, + resultCount: 1, + entries: [entry], + nextCursor: null, + ...overrides, + }; +} + +function changePage(overrides: Partial = {}): CatalogFeedChangePage { + return { + schemaVersion: 1, + feedId: "clawhub-official", + fromSequence: 3, + toSequence: 4, + generatedAt: "2026-07-16T00:00:00.000Z", + expiresAt: "2026-07-16T00:05:00.000Z", + requestCursor: null, + pageIndex: 0, + startIndex: 0, + changeCount: 1, + changes: [{ sequence: 4, operation: "upsert", entry }], + nextCursor: null, + ...overrides, + }; +} + +describe("catalog feed distribution schema", () => { + it("binds distinct payload types to query and change representations", () => { + expect(CATALOG_FEED_QUERY_PAYLOAD_TYPE).toContain("query-results.v1"); + expect(CATALOG_FEED_CHANGES_PAYLOAD_TYPE).toContain("changes.v1"); + expect(new Set([CATALOG_FEED_QUERY_PAYLOAD_TYPE, CATALOG_FEED_CHANGES_PAYLOAD_TYPE]).size).toBe( + 2, + ); + }); + + it("normalizes every bounded catalog query filter", () => { + expect( + normalizeCatalogFeedQuery({ + text: "\tCafe\u0301\r\n tools ", + types: ["skill", "plugin", "skill"], + states: ["blocked", "available", "blocked"], + publisherIds: ["zeta", "alpha", "zeta"], + }), + ).toEqual({ + text: "Caf\u00e9 tools", + types: ["plugin", "skill"], + states: ["available", "blocked"], + publisherIds: ["alpha", "zeta"], + }); + expect(() => normalizeCatalogFeedQuery({})).toThrow("at least one filter"); + expect(() => normalizeCatalogFeedQuery({ text: " " })).toThrow("1 and 256"); + expect(() => normalizeCatalogFeedQuery({ types: [] })).toThrow("must not be empty"); + expect(() => normalizeCatalogFeedQuery({ publisherIds: [] })).toThrow("between 1 and 100"); + }); + + it("accepts normalized query pages and rejects ambiguous pagination", () => { + expect(parseCatalogFeedQueryPage(queryPage()).entries).toHaveLength(1); + expect(() => + parseCatalogFeedQueryPage(queryPage({ query: { types: ["skill", "plugin"] } })), + ).toThrow("normalized query"); + expect(() => parseCatalogFeedQueryPage(queryPage({ pageIndex: 1 }))).toThrow("first page"); + expect(() => + parseCatalogFeedQueryPage(queryPage({ resultCount: 2, nextCursor: null })), + ).toThrow("terminal page"); + expect(() => + parseCatalogFeedQueryPage(queryPage({ entries: Array.from({ length: 201 }, () => entry) })), + ).toThrow("exceeds 200 entries"); + expect(() => parseCatalogFeedQueryPage(queryPage({ query: { types: ["skill"] } }))).toThrow( + "requested types", + ); + expect(() => + parseCatalogFeedQueryPage(queryPage({ entries: [entry, entry], resultCount: 2 })), + ).toThrow("duplicate entry identities"); + const skillWithPluginId = { ...entry, type: "skill" as const }; + expect( + parseCatalogFeedQueryPage( + queryPage({ + query: { text: "demo", types: ["plugin", "skill"] }, + entries: [entry, skillWithPluginId], + resultCount: 2, + }), + ).entries, + ).toEqual([entry, skillWithPluginId]); + expect(() => parseCatalogFeedQueryPage(queryPage({ nextCursor: "after-end" }))).toThrow( + "make progress", + ); + expect(() => + parseCatalogFeedQueryPage( + queryPage({ entries: [], resultCount: 1, nextCursor: "empty-page" }), + ), + ).toThrow("make progress"); + expect(() => + parseCatalogFeedQueryPage( + queryPage({ + requestCursor: "same", + nextCursor: "same", + pageIndex: 1, + startIndex: 1, + resultCount: 3, + }), + ), + ).toThrow("must differ"); + expect(() => + parseCatalogFeedQueryPage( + queryPage({ + requestCursor: "page-two", + pageIndex: 2, + startIndex: 1, + resultCount: 2, + }), + ), + ).toThrow("cannot precede"); + }); + + it("validates complete query page chains", () => { + const query = { text: "demo", types: ["plugin", "skill"] as ("plugin" | "skill")[] }; + const skillWithPluginId = { ...entry, type: "skill" as const }; + const first = queryPage({ query, resultCount: 2, nextCursor: "query-page-2" }); + const second = queryPage({ + query, + requestCursor: "query-page-2", + pageIndex: 1, + startIndex: 1, + resultCount: 2, + entries: [skillWithPluginId], + }); + expect(parseCatalogFeedQueryPages([first, second])).toHaveLength(2); + expect(() => parseCatalogFeedQueryPages([first])).toThrow("terminal page"); + expect(() => + parseCatalogFeedQueryPages([first, { ...second, requestCursor: "wrong-cursor" }]), + ).toThrow("cursor, page, or offset gap"); + expect(() => + parseCatalogFeedQueryPages([ + { ...first, resultCount: 4 }, + { ...second, resultCount: 4, nextCursor: "query-page-3" }, + { + ...second, + requestCursor: "query-page-3", + nextCursor: "query-page-2", + pageIndex: 2, + startIndex: 2, + resultCount: 4, + entries: [{ ...skillWithPluginId, id: "@openclaw/demo-2" }], + }, + { + ...second, + requestCursor: "query-page-2", + pageIndex: 3, + startIndex: 3, + resultCount: 4, + entries: [{ ...skillWithPluginId, id: "@openclaw/demo-3" }], + }, + ]), + ).toThrow("reuses a continuation cursor"); + }); + + it("accepts ordered changes and keeps blocked entries as upserts", () => { + const blockedEntry = { ...entry, state: "blocked" as const }; + const parsed = parseCatalogFeedChangePage( + changePage({ + fromSequence: 2, + changeCount: 3, + changes: [ + { sequence: 3, operation: "remove", entryId: "old", entryType: "plugin" }, + { sequence: 4, operation: "upsert", entry: blockedEntry }, + { sequence: 4, operation: "metadata", metadata: { description: "Official" } }, + ], + }), + ); + expect(parsed.changes[1]).toMatchObject({ operation: "upsert", entry: { state: "blocked" } }); + expect(() => + parseCatalogFeedChangePage( + changePage({ + fromSequence: 1, + toSequence: 3, + changeCount: 3, + changes: [ + { sequence: 2, operation: "upsert", entry }, + { sequence: 3, operation: "metadata", metadata: { description: "Official" } }, + { sequence: 2, operation: "remove", entryId: "old", entryType: "plugin" }, + ], + }), + ), + ).toThrow("ordered by sequence"); + expect(() => + parseCatalogFeedChangePage( + changePage({ + changes: [{ sequence: 4, operation: "upsert", entry: { ...entry, id: "x".repeat(257) } }], + }), + ), + ).toThrow("upsert entry id"); + expect(() => + parseCatalogFeedChangePage( + changePage({ + changes: [ + { sequence: 4, operation: "metadata", metadata: { description: "x".repeat(1_025) } }, + ], + }), + ), + ).toThrow("metadata description"); + expect(() => + parseCatalogFeedChangePage( + changePage({ + toSequence: 5, + changes: [{ sequence: 4, operation: "upsert", entry }], + }), + ), + ).toThrow("reach toSequence"); + expect(() => + parseCatalogFeedChangePage( + changePage({ + fromSequence: 2, + toSequence: 4, + changes: [{ sequence: 4, operation: "upsert", entry }], + }), + ), + ).toThrow("missing revision"); + expect(() => + parseCatalogFeedChangePage( + changePage({ fromSequence: 3, toSequence: 4, changeCount: 0, changes: [] }), + ), + ).toThrow("reach toSequence"); + expect( + parseCatalogFeedChangePage( + changePage({ fromSequence: 4, toSequence: 4, changeCount: 0, changes: [] }), + ).changes, + ).toEqual([]); + }); + + it("validates revision continuity across change page boundaries", () => { + const first = changePage({ + fromSequence: 2, + toSequence: 4, + changeCount: 2, + changes: [{ sequence: 3, operation: "upsert", entry }], + nextCursor: "change-page-2", + }); + const second = changePage({ + fromSequence: 2, + toSequence: 4, + requestCursor: "change-page-2", + pageIndex: 1, + startIndex: 1, + changeCount: 2, + changes: [{ sequence: 4, operation: "metadata", metadata: { description: "Official" } }], + }); + expect(parseCatalogFeedChangePages([first, second])).toHaveLength(2); + expect(() => + parseCatalogFeedChangePages([first, { ...second, expiresAt: "2026-07-16T00:06:00.000Z" }]), + ).toThrow("changed its pinned range"); + expect(() => parseCatalogFeedChangePages([first])).toThrow("terminal page"); + expect(() => + parseCatalogFeedChangePages([ + { ...first, toSequence: 5 }, + { + ...second, + toSequence: 5, + changes: [{ sequence: 5, operation: "metadata", metadata: { description: "Official" } }], + }, + ]), + ).toThrow("missing revision"); + expect(() => + parseCatalogFeedChangePages([ + { ...first, toSequence: 5, changeCount: 4 }, + { ...second, toSequence: 5, changeCount: 4, nextCursor: "change-page-3" }, + { + ...second, + toSequence: 5, + requestCursor: "change-page-3", + nextCursor: "change-page-2", + pageIndex: 2, + startIndex: 2, + changeCount: 4, + changes: [{ sequence: 5, operation: "metadata", metadata: { description: "Official" } }], + }, + { + ...second, + toSequence: 5, + requestCursor: "change-page-2", + pageIndex: 3, + startIndex: 3, + changeCount: 4, + changes: [{ sequence: 5, operation: "metadata", metadata: { description: "Official" } }], + }, + ]), + ).toThrow("reuses a continuation cursor"); + }); + + it("rejects invalid ranges, expiry windows, and reset locations", () => { + expect(() => parseCatalogFeedChangePage(changePage({ toSequence: 2 }))).toThrow( + "must not precede", + ); + expect(() => + parseCatalogFeedQueryPage(queryPage({ expiresAt: "2026-07-15T00:00:00.000Z" })), + ).toThrow("after generatedAt"); + expect(() => parseCatalogFeedQueryPage(queryPage({ generatedAt: "July 16, 2026" }))).toThrow( + "RFC 3339", + ); + expect(() => + parseCatalogFeedQueryPage(queryPage({ generatedAt: "2026-02-30T00:00:00Z" })), + ).toThrow("RFC 3339"); + const reset = { + schemaVersion: 1, + feedId: "clawhub-official", + fromSequence: 2, + currentSequence: 4, + generatedAt: "2026-07-16T00:00:00.000Z", + expiresAt: "2026-07-16T00:05:00.000Z", + resetRequired: true as const, + snapshotUrl: "https://clawhub.ai/api/v1/feeds/plugins", + }; + expect(parseCatalogFeedResetRequired(reset).currentSequence).toBe(4); + expect(() => parseCatalogFeedResetRequired({ ...reset, currentSequence: 2 })).toThrow( + "must follow", + ); + const credentialedUrl = new URL(reset.snapshotUrl); + credentialedUrl.username = "test-user"; + credentialedUrl.password = ["test", "password"].join("-"); + expect(() => + parseCatalogFeedResetRequired({ ...reset, snapshotUrl: credentialedUrl.href }), + ).toThrow("without credentials"); + }); +}); diff --git a/packages/schema/src/catalogFeedDistribution.ts b/packages/schema/src/catalogFeedDistribution.ts new file mode 100644 index 0000000000..ee1c8d1c7f --- /dev/null +++ b/packages/schema/src/catalogFeedDistribution.ts @@ -0,0 +1,523 @@ +import { type inferred, type } from "arktype"; +import { + CatalogFeedEntrySchema, + CatalogFeedStateSchema, + CATALOG_FEED_SCHEMA_VERSION, +} from "./catalogFeed.js"; + +export const CATALOG_FEED_QUERY_PAYLOAD_TYPE = + "openclaw.official-external-plugin-catalog-query-results.v1"; +export const CATALOG_FEED_CHANGES_PAYLOAD_TYPE = + "openclaw.official-external-plugin-catalog-changes.v1"; +export const CATALOG_FEED_QUERY_MAX_ENTRIES = 200; +export const CATALOG_FEED_CHANGES_MAX_RECORDS = 500; +export const CATALOG_FEED_DESCRIPTION_MAX_BYTES = 1_024; + +export const CatalogFeedQuerySchema = type({ + "+": "reject", + text: "string?", + types: type('"plugin"|"skill"').array().optional(), + states: CatalogFeedStateSchema.array().optional(), + publisherIds: type("string").array().optional(), +}); +export type CatalogFeedQuery = (typeof CatalogFeedQuerySchema)[inferred]; + +export const CatalogFeedMetadataSchema = type({ + "+": "reject", + description: "string|null", +}); +export type CatalogFeedMetadata = (typeof CatalogFeedMetadataSchema)[inferred]; + +export const CatalogFeedUpsertChangeSchema = type({ + "+": "reject", + sequence: "number", + operation: '"upsert"', + entry: CatalogFeedEntrySchema, +}); +export const CatalogFeedRemoveChangeSchema = type({ + "+": "reject", + sequence: "number", + operation: '"remove"', + entryId: "string", + entryType: '"plugin"|"skill"', +}); +export const CatalogFeedMetadataChangeSchema = type({ + "+": "reject", + sequence: "number", + operation: '"metadata"', + metadata: CatalogFeedMetadataSchema, +}); +export const CatalogFeedChangeSchema = type( + CatalogFeedUpsertChangeSchema.or(CatalogFeedRemoveChangeSchema).or( + CatalogFeedMetadataChangeSchema, + ), +); +export type CatalogFeedChange = (typeof CatalogFeedChangeSchema)[inferred]; + +export const CatalogFeedQueryPageSchema = type({ + "+": "reject", + schemaVersion: "number", + feedId: "string", + sequence: "number", + generatedAt: "string", + expiresAt: "string", + query: CatalogFeedQuerySchema, + requestCursor: "string|null", + pageIndex: "number", + startIndex: "number", + resultCount: "number", + entries: CatalogFeedEntrySchema.array(), + nextCursor: "string|null", +}); +export type CatalogFeedQueryPage = (typeof CatalogFeedQueryPageSchema)[inferred]; + +export const CatalogFeedChangePageSchema = type({ + "+": "reject", + schemaVersion: "number", + feedId: "string", + fromSequence: "number", + toSequence: "number", + generatedAt: "string", + expiresAt: "string", + requestCursor: "string|null", + pageIndex: "number", + startIndex: "number", + changeCount: "number", + changes: CatalogFeedChangeSchema.array(), + nextCursor: "string|null", +}); +export type CatalogFeedChangePage = (typeof CatalogFeedChangePageSchema)[inferred]; + +export const CatalogFeedResetRequiredSchema = type({ + "+": "reject", + schemaVersion: "number", + feedId: "string", + fromSequence: "number", + currentSequence: "number", + generatedAt: "string", + expiresAt: "string", + resetRequired: "true", + snapshotUrl: "string", +}); +export type CatalogFeedResetRequired = (typeof CatalogFeedResetRequiredSchema)[inferred]; + +const utf8Length = (value: string) => new TextEncoder().encode(value).length; + +function requireBoundedString(value: string, name: string, maxBytes: number) { + const length = utf8Length(value); + if (length < 1 || length > maxBytes) { + throw new Error(`${name} must be between 1 and ${maxBytes} UTF-8 bytes`); + } +} + +function requireNonNegativeInteger(value: number, name: string) { + if (!Number.isSafeInteger(value) || value < 0) { + throw new Error(`${name} must be a non-negative safe integer`); + } +} + +function requireValidWindow(generatedAt: string, expiresAt: string) { + requireBoundedString(generatedAt, "Catalog feed projection generatedAt", 64); + requireBoundedString(expiresAt, "Catalog feed projection expiresAt", 64); + const isRfc3339Instant = (value: string) => { + const match = + /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.\d+)?(?:Z|([+-])(\d{2}):(\d{2}))$/u.exec( + value, + ); + if (!match) return false; + const [year, month, day, hour, minute, second, offsetHour, offsetMinute] = [ + match[1], + match[2], + match[3], + match[4], + match[5], + match[6], + match[8] ?? "0", + match[9] ?? "0", + ].map(Number); + const leapYear = year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0); + const daysInMonth = [31, leapYear ? 29 : 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]; + return ( + month >= 1 && + month <= 12 && + day >= 1 && + day <= daysInMonth[month - 1]! && + hour <= 23 && + minute <= 59 && + second <= 59 && + (offsetHour === 0 || (offsetHour <= 23 && offsetMinute <= 59)) && + Number.isFinite(Date.parse(value)) + ); + }; + if (!isRfc3339Instant(generatedAt) || !isRfc3339Instant(expiresAt)) { + throw new Error("Catalog feed projection timestamps must use RFC 3339 syntax"); + } + const generatedAtMs = Date.parse(generatedAt); + const expiresAtMs = Date.parse(expiresAt); + if (!Number.isFinite(generatedAtMs) || !Number.isFinite(expiresAtMs)) { + throw new Error("Catalog feed projection timestamps must be valid dates"); + } + if (expiresAtMs <= generatedAtMs) { + throw new Error("Catalog feed projection expiresAt must be after generatedAt"); + } +} + +function normalizeQueryTextWhitespace(value: string) { + let result = ""; + let pendingSpace = false; + for (const character of value.normalize("NFC")) { + const codePoint = character.codePointAt(0)!; + if ((codePoint >= 0x09 && codePoint <= 0x0d) || codePoint === 0x20) { + pendingSpace = result.length > 0; + continue; + } + if (pendingSpace) result += " "; + result += character; + pendingSpace = false; + } + return result; +} + +function sortedUnique(values: readonly string[]) { + const encoder = new TextEncoder(); + const compareUtf8 = (left: string, right: string) => { + const leftBytes = encoder.encode(left); + const rightBytes = encoder.encode(right); + const length = Math.min(leftBytes.length, rightBytes.length); + for (let index = 0; index < length; index += 1) { + const difference = leftBytes[index]! - rightBytes[index]!; + if (difference !== 0) return difference; + } + return leftBytes.length - rightBytes.length; + }; + return [...new Set(values)].sort(compareUtf8); +} + +export function normalizeCatalogFeedQuery(value: unknown): CatalogFeedQuery { + const query = CatalogFeedQuerySchema.assert(value); + const normalized: CatalogFeedQuery = {}; + if (query.text !== undefined) { + const text = normalizeQueryTextWhitespace(query.text); + requireBoundedString(text, "Catalog feed query text", 256); + normalized.text = text; + } + if (query.types !== undefined) { + const types = sortedUnique(query.types) as NonNullable; + if (types.length === 0) throw new Error("Catalog feed query types must not be empty"); + normalized.types = types; + } + if (query.states !== undefined) { + const states = sortedUnique(query.states) as NonNullable; + if (states.length === 0) throw new Error("Catalog feed query states must not be empty"); + normalized.states = states; + } + if (query.publisherIds !== undefined) { + const publisherIds = sortedUnique(query.publisherIds); + if (publisherIds.length < 1 || publisherIds.length > 100) { + throw new Error("Catalog feed query publisherIds must contain between 1 and 100 values"); + } + for (const publisherId of publisherIds) { + requireBoundedString(publisherId, "Catalog feed publisher id", 256); + } + normalized.publisherIds = publisherIds; + } + if (Object.keys(normalized).length === 0) { + throw new Error("Catalog feed query must include at least one filter"); + } + return normalized; +} + +function queriesEqual(left: CatalogFeedQuery, right: CatalogFeedQuery) { + const arraysEqual = (first?: readonly string[], second?: readonly string[]) => + first === undefined + ? second === undefined + : second !== undefined && + first.length === second.length && + first.every((value, index) => value === second[index]); + return ( + left.text === right.text && + arraysEqual(left.types, right.types) && + arraysEqual(left.states, right.states) && + arraysEqual(left.publisherIds, right.publisherIds) + ); +} + +function requireProjectionHeader(value: { + schemaVersion: number; + feedId: string; + generatedAt: string; + expiresAt: string; +}) { + if (value.schemaVersion !== CATALOG_FEED_SCHEMA_VERSION) { + throw new Error(`Unsupported catalog feed projection schema version: ${value.schemaVersion}`); + } + requireBoundedString(value.feedId, "Catalog feed id", 256); + requireValidWindow(value.generatedAt, value.expiresAt); +} + +function requirePageBounds(value: { + requestCursor: string | null; + pageIndex: number; + startIndex: number; + itemCount: number; + totalCount: number; + nextCursor: string | null; +}) { + requireNonNegativeInteger(value.pageIndex, "Catalog feed projection pageIndex"); + requireNonNegativeInteger(value.startIndex, "Catalog feed projection startIndex"); + requireNonNegativeInteger(value.totalCount, "Catalog feed projection total count"); + if (value.requestCursor === null && (value.pageIndex !== 0 || value.startIndex !== 0)) { + throw new Error("Catalog feed projection first page must start at page and item index zero"); + } + if (value.requestCursor !== null && value.pageIndex === 0) { + throw new Error("Catalog feed projection continuation must have a positive page index"); + } + if (value.requestCursor !== null && value.startIndex < value.pageIndex) { + throw new Error("Catalog feed projection continuation offset cannot precede its page index"); + } + if (value.startIndex + value.itemCount > value.totalCount) { + throw new Error("Catalog feed projection page exceeds its declared total count"); + } + if (value.nextCursor === null && value.startIndex + value.itemCount !== value.totalCount) { + throw new Error("Catalog feed projection terminal page must end at its declared total count"); + } + if ( + value.nextCursor !== null && + (value.itemCount === 0 || value.startIndex + value.itemCount >= value.totalCount) + ) { + throw new Error("Catalog feed projection continuation must make progress before total count"); + } + if (value.nextCursor !== null && value.nextCursor === value.requestCursor) { + throw new Error("Catalog feed projection next cursor must differ from the request cursor"); + } + for (const cursor of [value.requestCursor, value.nextCursor]) { + if (cursor !== null && utf8Length(cursor) > 4096) { + throw new Error("Catalog feed projection cursor exceeds 4096 UTF-8 bytes"); + } + } +} + +export function parseCatalogFeedQueryPage(value: unknown): CatalogFeedQueryPage { + const page = CatalogFeedQueryPageSchema.assert(value); + requireProjectionHeader(page); + requireNonNegativeInteger(page.sequence, "Catalog feed projection sequence"); + const normalizedQuery = normalizeCatalogFeedQuery(page.query); + if (!queriesEqual(normalizedQuery, page.query)) { + throw new Error("Catalog feed query page must carry the normalized query"); + } + if (page.entries.length > CATALOG_FEED_QUERY_MAX_ENTRIES) { + throw new Error(`Catalog feed query page exceeds ${CATALOG_FEED_QUERY_MAX_ENTRIES} entries`); + } + const entryKeys = new Set(); + for (const entry of page.entries) { + requireBoundedString(entry.id, "Catalog feed query entry id", 256); + const entryKey = `${entry.type}\0${entry.id}`; + if (entryKeys.has(entryKey)) { + throw new Error("Catalog feed query page contains duplicate entry identities"); + } + entryKeys.add(entryKey); + if (page.query.types && !page.query.types.includes(entry.type)) { + throw new Error("Catalog feed query entry does not match the requested types"); + } + if (page.query.states && !page.query.states.includes(entry.state)) { + throw new Error("Catalog feed query entry does not match the requested states"); + } + if (page.query.publisherIds && !page.query.publisherIds.includes(entry.publisher.id)) { + throw new Error("Catalog feed query entry does not match the requested publisherIds"); + } + } + requirePageBounds({ + requestCursor: page.requestCursor, + pageIndex: page.pageIndex, + startIndex: page.startIndex, + itemCount: page.entries.length, + totalCount: page.resultCount, + nextCursor: page.nextCursor, + }); + return page; +} + +export function parseCatalogFeedQueryPages(values: readonly unknown[]): CatalogFeedQueryPage[] { + if (values.length === 0) { + throw new Error("Catalog feed query page chain must not be empty"); + } + const pages = values.map(parseCatalogFeedQueryPage); + const first = pages[0]!; + if (first.requestCursor !== null || first.pageIndex !== 0 || first.startIndex !== 0) { + throw new Error("Catalog feed query page chain must start at page and item index zero"); + } + const entryKeys = new Set(); + const consumedCursors = new Set(); + let expectedStartIndex = 0; + let expectedRequestCursor: string | null = null; + for (const [pageIndex, page] of pages.entries()) { + if ( + page.feedId !== first.feedId || + page.sequence !== first.sequence || + page.expiresAt !== first.expiresAt || + page.resultCount !== first.resultCount || + !queriesEqual(page.query, first.query) + ) { + throw new Error("Catalog feed query page chain changed its pinned projection"); + } + if ( + page.pageIndex !== pageIndex || + page.startIndex !== expectedStartIndex || + page.requestCursor !== expectedRequestCursor + ) { + throw new Error("Catalog feed query page chain contains a cursor, page, or offset gap"); + } + if (page.requestCursor !== null) { + if (consumedCursors.has(page.requestCursor)) { + throw new Error("Catalog feed query page chain reuses a continuation cursor"); + } + consumedCursors.add(page.requestCursor); + } + for (const entry of page.entries) { + const entryKey = `${entry.type}\0${entry.id}`; + if (entryKeys.has(entryKey)) { + throw new Error("Catalog feed query page chain contains duplicate entry identities"); + } + entryKeys.add(entryKey); + } + expectedStartIndex += page.entries.length; + expectedRequestCursor = page.nextCursor; + } + if (expectedRequestCursor !== null) { + throw new Error("Catalog feed query page chain must include its terminal page"); + } + return pages; +} + +export function parseCatalogFeedChangePage(value: unknown): CatalogFeedChangePage { + const page = CatalogFeedChangePageSchema.assert(value); + requireProjectionHeader(page); + requireNonNegativeInteger(page.fromSequence, "Catalog feed change fromSequence"); + requireNonNegativeInteger(page.toSequence, "Catalog feed change toSequence"); + if (page.toSequence < page.fromSequence) { + throw new Error("Catalog feed change toSequence must not precede fromSequence"); + } + if (page.changes.length > CATALOG_FEED_CHANGES_MAX_RECORDS) { + throw new Error(`Catalog feed change page exceeds ${CATALOG_FEED_CHANGES_MAX_RECORDS} records`); + } + let priorSequence = page.fromSequence; + for (const change of page.changes) { + requireNonNegativeInteger(change.sequence, "Catalog feed change sequence"); + if (change.sequence <= page.fromSequence || change.sequence > page.toSequence) { + throw new Error("Catalog feed change sequence is outside the requested range"); + } + if (change.sequence < priorSequence) { + throw new Error("Catalog feed changes must be ordered by sequence"); + } + if ( + change.sequence > priorSequence + 1 && + (page.pageIndex === 0 || priorSequence > page.fromSequence) + ) { + throw new Error("Catalog feed change page contains a missing revision"); + } + priorSequence = change.sequence; + if (change.operation === "remove") { + requireBoundedString(change.entryId, "Catalog feed removed entry id", 256); + } else if (change.operation === "upsert") { + requireBoundedString(change.entry.id, "Catalog feed upsert entry id", 256); + } else if (change.metadata.description !== null) { + requireBoundedString( + change.metadata.description, + "Catalog feed metadata description", + CATALOG_FEED_DESCRIPTION_MAX_BYTES, + ); + } + } + if (page.nextCursor === null) { + const terminalSequence = page.changes.at(-1)?.sequence ?? page.fromSequence; + if (terminalSequence !== page.toSequence) { + throw new Error("Catalog feed terminal change page must reach toSequence"); + } + } + requirePageBounds({ + requestCursor: page.requestCursor, + pageIndex: page.pageIndex, + startIndex: page.startIndex, + itemCount: page.changes.length, + totalCount: page.changeCount, + nextCursor: page.nextCursor, + }); + return page; +} + +export function parseCatalogFeedChangePages(values: readonly unknown[]): CatalogFeedChangePage[] { + if (values.length === 0) { + throw new Error("Catalog feed change page chain must not be empty"); + } + const pages = values.map(parseCatalogFeedChangePage); + const first = pages[0]!; + if (first.requestCursor !== null || first.pageIndex !== 0 || first.startIndex !== 0) { + throw new Error("Catalog feed change page chain must start at page and item index zero"); + } + let expectedStartIndex = 0; + let expectedRequestCursor: string | null = null; + let priorSequence = first.fromSequence; + const consumedCursors = new Set(); + for (const [pageIndex, page] of pages.entries()) { + if ( + page.feedId !== first.feedId || + page.fromSequence !== first.fromSequence || + page.toSequence !== first.toSequence || + page.expiresAt !== first.expiresAt || + page.changeCount !== first.changeCount + ) { + throw new Error("Catalog feed change page chain changed its pinned range"); + } + if ( + page.pageIndex !== pageIndex || + page.startIndex !== expectedStartIndex || + page.requestCursor !== expectedRequestCursor + ) { + throw new Error("Catalog feed change page chain contains a cursor, page, or offset gap"); + } + if (page.requestCursor !== null) { + if (consumedCursors.has(page.requestCursor)) { + throw new Error("Catalog feed change page chain reuses a continuation cursor"); + } + consumedCursors.add(page.requestCursor); + } + for (const change of page.changes) { + if (change.sequence !== priorSequence && change.sequence !== priorSequence + 1) { + throw new Error("Catalog feed change page chain contains a missing revision"); + } + priorSequence = change.sequence; + } + expectedStartIndex += page.changes.length; + expectedRequestCursor = page.nextCursor; + } + if (expectedRequestCursor !== null) { + throw new Error("Catalog feed change page chain must include its terminal page"); + } + if (priorSequence !== first.toSequence) { + throw new Error("Catalog feed change page chain must cover every revision"); + } + return pages; +} + +export function parseCatalogFeedResetRequired(value: unknown): CatalogFeedResetRequired { + const reset = CatalogFeedResetRequiredSchema.assert(value); + requireProjectionHeader(reset); + requireNonNegativeInteger(reset.fromSequence, "Catalog feed reset fromSequence"); + requireNonNegativeInteger(reset.currentSequence, "Catalog feed reset currentSequence"); + if (reset.currentSequence <= reset.fromSequence) { + throw new Error("Catalog feed reset currentSequence must follow fromSequence"); + } + if (utf8Length(reset.snapshotUrl) > 2048) { + throw new Error("Catalog feed reset snapshotUrl exceeds 2048 UTF-8 bytes"); + } + let snapshotUrl: URL; + try { + snapshotUrl = new URL(reset.snapshotUrl); + } catch { + throw new Error("Catalog feed reset snapshotUrl must be absolute HTTPS"); + } + if (snapshotUrl.protocol !== "https:" || snapshotUrl.username || snapshotUrl.password) { + throw new Error("Catalog feed reset snapshotUrl must be absolute HTTPS without credentials"); + } + return reset; +} diff --git a/packages/schema/src/index.ts b/packages/schema/src/index.ts index daba3ca2c9..4b4c0f313f 100644 --- a/packages/schema/src/index.ts +++ b/packages/schema/src/index.ts @@ -1,6 +1,7 @@ export type { ArkValidator } from "./ark.js"; export { formatArkErrors, parseArk } from "./ark.js"; export * from "./catalogFeed.js"; +export * from "./catalogFeedDistribution.js"; export * from "./catalogMetadata.js"; export * from "./docsLinks.js"; export * from "./license.js"; diff --git a/packages/schema/src/routes.ts b/packages/schema/src/routes.ts index 31efd59d1e..8e988093c1 100644 --- a/packages/schema/src/routes.ts +++ b/packages/schema/src/routes.ts @@ -25,6 +25,7 @@ export const ApiRoutes = { bundlePlugins: "/api/v1/bundle-plugins", promotions: "/api/v1/promotions", catalogFeed: "/api/v1/feeds/plugins", + catalogFeedChanges: "/api/v1/feeds/plugins/changes", catalogSkillsFeed: "/api/v1/feeds/skills", promotionsFeed: "/api/v1/feeds/promotions", stars: "/api/v1/stars", diff --git a/specs/hosted-catalog-feed.md b/specs/hosted-catalog-feed.md index d11a66aff9..b8080d879f 100644 --- a/specs/hosted-catalog-feed.md +++ b/specs/hosted-catalog-feed.md @@ -94,14 +94,23 @@ outside the promotion's declared provider. `convex/catalogFeed.ts` builds both feeds from indexed package/skill queries and stores one current publication row per feed in `catalogFeedPublications`. -Keeping one row per feed avoids an unbounded publication log while preserving -the sequence and exact payload needed for validators. +It also keeps a bounded 30-day revision and change journal. Each revision stores +its own change count and the cumulative retained count, allowing a reader to pin +an exact sequence range and page it without scanning or counting the journal. +Retention removes a revision marker before its change rows so a concurrent +reader receives a reset response instead of a partial revision. The `Publish Hosted Catalog Feed` workflow refreshes the snapshot every six hours and can be run manually. It requires the existing `Production` environment -`CONVEX_DEPLOY_KEY`. The workflow currently publishes an unsigned feed; signed -envelopes require a separate production key-management decision and must not be -advertised to OpenClaw clients until the signing key and trust root are deployed. +`CONVEX_DEPLOY_KEY`. Publication stores the canonical unsigned payload bytes; +the plugin HTTP action wraps those exact bytes in a deterministic DSSE/Ed25519 +envelope. The private key stays in Convex environment secret storage and the +matching public key is bundled in OpenClaw. + +The response uses the standard DSSE JSON envelope fields: `payloadType`, +`payload`, and `signatures` containing `keyid` and `sig`. Ed25519 is selected by +the trusted key profile rather than repeated as a nonstandard signature field. +The signed representation uses the `application/vnd.dsse+json` media type. `convex/promotionsFeed.ts` builds the promotions snapshot from the bounded active set and stores it in the same publication table. Production backend deploys @@ -114,15 +123,43 @@ snapshot inside its 24-hour `expiresAt` horizon. ## Edge delivery -The HTTP endpoints are `/api/v1/feeds/plugins`, `/api/v1/feeds/skills`, and -`/api/v1/feeds/promotions`. Each returns its stored bytes unchanged and -provides: +The snapshot endpoints are `/api/v1/feeds/plugins`, `/api/v1/feeds/skills`, and +`/api/v1/feeds/promotions`. Each snapshot representation provides: - `ETag: "sha256:"` -- `Last-Modified` - `Cache-Control: public, max-age=60, s-maxage=300, stale-while-revalidate=86400` - `Surrogate-Control: max-age=300, stale-while-revalidate=86400` -- `304 Not Modified` for matching `If-None-Match` or `If-Modified-Since` +- `304 Not Modified` for matching `If-None-Match` + +Unsigned skills and promotions also expose `Last-Modified` and accept +`If-Modified-Since`. The signed plugin representation intentionally does not: +rotating its signer changes the envelope without changing the underlying +publication time, so only its representation ETag is a valid cache validator. + +The plugin feed route requires one atomic `CLAWHUB_FEED_SIGNING_CONFIG` JSON +secret containing exactly `keyId` and `privateKey`. It returns `503` with +`Cache-Control: no-store` when the value is absent or invalid. Its ETag and +`X-Content-SHA256` describe the signed envelope representation; +`X-Catalog-Payload-SHA256` preserves the stored publication payload digest. +Skills and promotions remain on their existing unsigned representations until +their payload types and matching OpenClaw consumers are specified. + +`GET /api/v1/feeds/plugins/changes?fromSequence=&limit=<1..500>` returns the +signed plugin changes after `fromSequence` through a `toSequence` pinned when +the first page is requested. It reports the exact `changeCount`; every revision +in the range has at least one ordered upsert, remove, or metadata record. A +continuation request supplies only the opaque `cursor`. The five-minute signed +cursor binds the feed id, range, exact count, Convex cursor, limit, page index, +start index, and expiry, so clients cannot alter pagination or drift onto a +newer publication mid-chain. + +Change pages use the +`openclaw.official-external-plugin-catalog-changes.v1` DSSE payload type, +`Cache-Control: no-store`, at most 500 records, and a 1 MiB signed-response +limit. If retention can no longer cover the pinned range, the same endpoint +returns a signed `409 resetRequired` payload whose same-origin `snapshotUrl` +points to `/api/v1/feeds/plugins`. Invalid or expired cursors are never treated +as unsigned pagination state. Nitro exposes `/v1/feeds/plugins`, `/v1/feeds/skills`, and `/v1/feeds/promotions` through the same environment-aware Convex proxy used for @@ -140,3 +177,79 @@ schema version. Do not make the feed request-time dynamic. Refresh the stored publication first, then let Vercel or the configured CDN cache the immutable response by ETag. + +## Feed signing runbook + +ClawHub owns the private key and stable key id. OpenClaw owns distribution of +the matching public trust anchor. Do not reuse release, package, TLS, account, +or other platform signing keys for feed signing, and do not publish a trust +bootstrap endpoint from the same origin as the feed. + +### Initial provisioning + +1. Generate a dedicated Ed25519 key pair on an approved operator machine: + + ```bash + openssl genpkey -algorithm ED25519 -out clawhub-feed-private.pem + openssl pkey -in clawhub-feed-private.pem -pubout -out clawhub-feed-public.pem + ``` + +2. Choose a stable, non-secret key id such as `clawhub-feed-2026-q3`. Record the + owner, creation time, intended deployment, and rotation contact in the + operator secret inventory. +3. Store the key id and private PEM as one atomic JSON value in each intended + Convex deployment. Do not create separate mutable variables for the pair: a + partially rotated pair can publish an unverifiable cacheable envelope. For + production, use the Convex dashboard or pipe compact JSON to the CLI without + placing the private key in shell history: + + ```powershell + @{ + keyId = "clawhub-feed-2026-q3" + privateKey = Get-Content -Raw .\clawhub-feed-private.pem + } | ConvertTo-Json -Compress | + bunx convex env set CLAWHUB_FEED_SIGNING_CONFIG --prod + ``` + +4. Confirm only the variable names, not their values: + + ```bash + bunx convex env list --names-only --prod + ``` + +5. Provide `clawhub-feed-public.pem` and the key id to the OpenClaw maintainer + bundling the `clawhub-public` trust profile. The private PEM never leaves + ClawHub's operator-controlled secret path. +6. Deploy ClawHub, publish a fresh `clawhub-official` snapshot, and verify that + `/api/v1/feeds/plugins` returns an envelope whose decoded payload bytes equal + the stored publication and whose signature verifies with the handed-off + public key. +7. Land and release the matching OpenClaw bundled public key. Older unsigned + clients may fall back to their bundled catalog when they first encounter the + envelope; they must not reinterpret it as unsigned feed content. +8. Securely delete operator-machine private-key files after the approved secret + backup and recovery process is complete. + +### Normal rotation + +1. Generate a new dedicated key and key id. +2. Bundle the new public key in OpenClaw while the old key remains trusted. +3. After that trust update is available, replace + `CLAWHUB_FEED_SIGNING_CONFIG` once with JSON containing the new matched pair, + then deploy. Never stage key id and private key separately. +4. Verify a higher-sequence publication under the new key before retiring the + old private key. +5. Remove the old public key in a later OpenClaw release after the supported + client overlap window. + +The first signer emits one signature, so it cannot provide an old-and-new +dual-sign overlap by itself. If operational policy requires dual signing, add +multi-key signer support before beginning that rotation. + +### Emergency revocation + +Remove `CLAWHUB_FEED_SIGNING_CONFIG` or replace it with a new matched pair in +Convex immediately. An absent signing configuration makes the plugin feed return +`503 no-store`. Notify OpenClaw maintainers to remove the compromised public key +through the authenticated release channel. Do not recover by advertising a new +public key from a feed-adjacent endpoint.