diff --git a/CHANGES.md b/CHANGES.md index f44810d85..adda848b5 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -54,12 +54,20 @@ To be released. `@context` that redefines the `as:` prefix or the bare `Public` term is preserved as is. The rewrite is also applied before `eddsa-jcs-2022` Object Integrity Proof signing so the signed bytes - match what is sent on the wire. [[#710]] + match what is sent on the wire. [[#710], [#721]] + + - Improved interoperability with [Pixelfed] by serializing outgoing + activities' `attachment` fields as arrays even when there is only one + attachment. JSON-LD compaction would otherwise emit a scalar value for + single attachments, but Pixelfed currently expects an array and may reject + incoming posts; see [pixelfed/pixelfed#6588]. [[#721]] [Agent Skills]: https://agentskills.io/ [skills-npm]: https://github.com/antfu/skills-npm [Lemmy]: https://join-lemmy.org/ [LemmyNet/lemmy#6465]: https://github.com/LemmyNet/lemmy/issues/6465 +[Pixelfed]: https://pixelfed.org/ +[pixelfed/pixelfed#6588]: https://github.com/pixelfed/pixelfed/issues/6588 [#430]: https://github.com/fedify-dev/fedify/issues/430 [#644]: https://github.com/fedify-dev/fedify/issues/644 [#680]: https://github.com/fedify-dev/fedify/pull/680 @@ -67,6 +75,7 @@ To be released. [#710]: https://github.com/fedify-dev/fedify/pull/710 [#711]: https://github.com/fedify-dev/fedify/issues/711 [#712]: https://github.com/fedify-dev/fedify/pull/712 +[#721]: https://github.com/fedify-dev/fedify/pull/721 ### @fedify/lint diff --git a/docs/manual/send.md b/docs/manual/send.md index 51b4805cf..acde31a25 100644 --- a/docs/manual/send.md +++ b/docs/manual/send.md @@ -1112,7 +1112,27 @@ additional information to the activity. The activity transformers are applied before they are signed with the sender's private key and sent to the recipients. -It can be configured by setting +Fedify also applies a small set of internal JSON-LD wire-format compatibility +fixes after serializing the transformed activity. Unlike activity transformers, +these fixes operate on the compact JSON-LD document rather than the `Activity` +object, so they can preserve representation details such as array-valued +properties that JSON-LD compaction would otherwise collapse. These internal +fixes are applied automatically after serialization for unsigned activities and +for proofs Fedify creates while sending. + +Activities that already carry cryptographic proofs are sent unchanged by +default, so the compact JSON-LD bytes stay consistent with the existing +signature. If you pre-sign an activity locally with `signObject()` or +`createProof()` and then pass it to `Context.sendActivity()`, set +`normalizeExistingProofs: true` so the outgoing wire form matches the +normalized bytes covered by the proof. + +When an outgoing document uses custom or inline JSON-LD contexts, Fedify may +canonicalize the document before and after a representation fix to confirm +that the fix preserves JSON-LD semantics. Documents that are too deeply +nested for safe traversal are sent unchanged instead. + +Activity transformers can be configured by setting the [`activityTransformers`](./federation.md#activitytransformers) option. By default, the following activity transformers are enabled: diff --git a/packages/fedify/src/compat/outgoing-jsonld.test.ts b/packages/fedify/src/compat/outgoing-jsonld.test.ts new file mode 100644 index 000000000..453520564 --- /dev/null +++ b/packages/fedify/src/compat/outgoing-jsonld.test.ts @@ -0,0 +1,287 @@ +import { mockDocumentLoader, test } from "@fedify/fixture"; +import { Create, Document, Note, PUBLIC_COLLECTION } from "@fedify/vocab"; +import { assertEquals } from "@std/assert/assert-equals"; +import { assertStrictEquals } from "@std/assert/assert-strict-equals"; +import { + isPreloadedContextAttachmentSafe, + normalizeAttachmentArrays, + normalizeOutgoingActivityJsonLd, +} from "./outgoing-jsonld.ts"; + +test("normalizeAttachmentArrays() wraps scalar attachments", async () => { + const input = { + "@context": "https://www.w3.org/ns/activitystreams", + type: "Create", + object: { + type: "Note", + attachment: { + type: "Document", + mediaType: "image/png", + url: "https://example.com/image.png", + }, + }, + }; + const output = await normalizeAttachmentArrays(input) as Record< + string, + unknown + >; + const object = output.object as Record; + assertEquals(object.attachment, [ + { + type: "Document", + mediaType: "image/png", + url: "https://example.com/image.png", + }, + ]); +}); + +test("normalizeAttachmentArrays() skips canonicalization for known-safe contexts", async () => { + const input = { + "@context": [ + "https://www.w3.org/ns/activitystreams", + "https://w3id.org/security/data-integrity/v1", + ], + type: "Note", + attachment: { + type: "Document", + mediaType: "image/png", + url: "https://example.com/image.png", + }, + }; + const output = await normalizeAttachmentArrays(input, () => { + throw new Error("context loader should not be called"); + }) as Record; + assertEquals(output.attachment, [ + { + type: "Document", + mediaType: "image/png", + url: "https://example.com/image.png", + }, + ]); +}); + +test("isPreloadedContextAttachmentSafe() checks scoped contexts", () => { + assertEquals( + isPreloadedContextAttachmentSafe({ + "@context": { + attachment: { + "@id": "as:attachment", + "@type": "@id", + }, + Example: { + "@context": { + attachment: { + "@id": "as:attachment", + "@type": "@id", + }, + }, + }, + }, + }), + true, + ); + assertEquals( + isPreloadedContextAttachmentSafe({ + "@context": { + Example: { + "@context": { + attachment: "https://example.com/custom-attachment", + }, + }, + }, + }), + false, + ); +}); + +test("normalizeAttachmentArrays() does not wrap JSON-LD list objects", async () => { + const attachment = { + "@list": [ + { type: "Document", url: "https://example.com/image.png" }, + ], + }; + const input = { + "@context": "https://www.w3.org/ns/activitystreams", + type: "Note", + attachment, + }; + const output = await normalizeAttachmentArrays(input, () => { + throw new Error("context loader should not be called"); + }) as Record; + assertEquals(output.attachment, attachment); +}); + +test("normalizeAttachmentArrays() does not traverse JSON-LD value payloads", async () => { + const input = { + "@context": "https://www.w3.org/ns/activitystreams", + type: "Note", + attachment: { type: "Document" }, + content: { + "@type": "@json", + "@value": { + "@context": { + attachment: "https://example.com/custom-attachment", + }, + attachment: "https://example.com/metadata", + }, + }, + }; + const output = await normalizeAttachmentArrays(input, () => { + throw new Error("context loader should not be called"); + }) as Record; + assertEquals(output.attachment, [{ type: "Document" }]); + assertEquals(output.content, { + "@type": "@json", + "@value": { + "@context": { + attachment: "https://example.com/custom-attachment", + }, + attachment: "https://example.com/metadata", + }, + }); +}); + +test("normalizeAttachmentArrays() leaves attachment arrays unchanged", async () => { + const attachment = [ + { + type: "Document", + mediaType: "image/png", + url: "https://example.com/image.png", + }, + ]; + const input = { + "@context": "https://www.w3.org/ns/activitystreams", + type: "Note", + attachment, + }; + const output = await normalizeAttachmentArrays(input) as Record< + string, + unknown + >; + assertEquals(output.attachment, attachment); +}); + +test("normalizeAttachmentArrays() leaves documents without attachments unchanged", async () => { + const input = { + "@context": "https://www.w3.org/ns/activitystreams", + type: "Note", + content: "Hello", + }; + assertEquals(await normalizeAttachmentArrays(input), input); +}); + +test("normalizeAttachmentArrays() leaves @context subtrees untouched", async () => { + const input = { + "@context": [ + "https://www.w3.org/ns/activitystreams", + { attachment: "https://example.com/custom-attachment" }, + ], + type: "Note", + attachment: "https://example.com/attachment", + }; + const output = await normalizeAttachmentArrays(input) as Record< + string, + unknown + >; + const context = output["@context"] as unknown[]; + assertEquals(context[1], { + attachment: "https://example.com/custom-attachment", + }); + assertEquals(output.attachment, ["https://example.com/attachment"]); +}); + +test("normalizeAttachmentArrays() bails out when wrapping changes semantics", async () => { + const input = { + "@context": { + attachment: { + "@id": "https://example.com/custom-attachment", + "@type": "@json", + }, + }, + attachment: { + custom: true, + }, + }; + const output = await normalizeAttachmentArrays(input) as Record< + string, + unknown + >; + assertEquals(output.attachment, { custom: true }); +}); + +test("normalizeAttachmentArrays() does not poison the global prototype via a __proto__ key", async () => { + const input = JSON.parse(`{ + "@context": "https://www.w3.org/ns/activitystreams", + "type": "Note", + "attachment": { "type": "Document" }, + "__proto__": { "polluted": true } + }`); + await normalizeAttachmentArrays(input); + assertEquals( + (Object.prototype as Record).polluted, + undefined, + ); +}); + +test("normalizeAttachmentArrays() stops before blowing the stack on pathological nesting", async () => { + let deep: Record = { attachment: { type: "Document" } }; + for (let i = 0; i < 256; i++) deep = { object: deep }; + const input = { + "@context": "https://www.w3.org/ns/activitystreams", + type: "Create", + object: deep, + }; + const output = await normalizeAttachmentArrays(input); + assertStrictEquals(output, input); +}); + +test("normalizeAttachmentArrays() skips canonicalization for pathological nesting", async () => { + let deep: Record = { type: "Note" }; + for (let i = 0; i < 256; i++) deep = { object: deep }; + const input = { + "@context": [ + "https://www.w3.org/ns/activitystreams", + "https://example.com/context", + ], + type: "Note", + attachment: { type: "Document" }, + object: deep, + }; + const output = await normalizeAttachmentArrays(input, () => { + throw new Error("context loader should not be called"); + }) as Record; + assertEquals(output.attachment, { type: "Document" }); +}); + +test("normalizeOutgoingActivityJsonLd() applies outgoing JSON-LD workarounds", async () => { + const activity = new Create({ + id: new URL("https://example.com/activities/1"), + actor: new URL("https://example.com/alice"), + object: new Note({ + id: new URL("https://example.com/notes/1"), + tos: [PUBLIC_COLLECTION], + attachments: [ + new Document({ + mediaType: "image/png", + url: new URL("https://example.com/image.png"), + }), + ], + }), + tos: [PUBLIC_COLLECTION], + }); + const compact = await activity.toJsonLd({ format: "compact" }) as Record< + string, + unknown + >; + assertEquals(compact.to, "as:Public"); + const compactObject = compact.object as Record; + assertEquals(Array.isArray(compactObject.attachment), false); + + const normalized = await normalizeOutgoingActivityJsonLd( + compact, + mockDocumentLoader, + ) as Record; + assertEquals(normalized.to, PUBLIC_COLLECTION.href); + const normalizedObject = normalized.object as Record; + assertEquals(Array.isArray(normalizedObject.attachment), true); +}); diff --git a/packages/fedify/src/compat/outgoing-jsonld.ts b/packages/fedify/src/compat/outgoing-jsonld.ts new file mode 100644 index 000000000..69c43387c --- /dev/null +++ b/packages/fedify/src/compat/outgoing-jsonld.ts @@ -0,0 +1,304 @@ +import { type DocumentLoader, preloadedContexts } from "@fedify/vocab-runtime"; +import jsonld from "@fedify/vocab-runtime/jsonld"; +import { getLogger } from "@logtape/logtape"; +import { preloadedOnlyDocumentLoader } from "./preloaded-context-loader.ts"; +import { normalizePublicAudience } from "./public-audience.ts"; + +const logger = getLogger(["fedify", "compat", "outgoing-jsonld"]); + +const ATTACHMENT_FIELDS = new Set([ + "attachment", + "https://www.w3.org/ns/activitystreams#attachment", +]); + +const AS_CONTEXT_URL = "https://www.w3.org/ns/activitystreams"; +const KNOWN_SAFE_CONTEXT_URLS: ReadonlySet = getKnownSafeContextUrls(); + +// Keep the traversal bounded for adversarial JSON-LD passed through proof +// verification fallback paths. +const MAX_TRAVERSAL_DEPTH = 64; + +function isJsonLdListObject(value: unknown): boolean { + return typeof value === "object" && value != null && + Object.hasOwn(value, "@list"); +} + +function isJsonLdValueObject(value: unknown): boolean { + return typeof value === "object" && value != null && + Object.hasOwn(value, "@value"); +} + +function* getContextObjects( + value: unknown, + seen: WeakSet = new WeakSet(), +): Iterable> { + if (Array.isArray(value)) { + if (seen.has(value)) return; + seen.add(value); + for (const item of value) yield* getContextObjects(item, seen); + return; + } + if (typeof value === "object" && value != null) { + if (seen.has(value)) return; + seen.add(value); + const record = value as Record; + yield record; + for (const definition of Object.values(record)) { + if (typeof definition !== "object" || definition == null) continue; + const nestedContext = (definition as Record)["@context"]; + if (nestedContext == null) continue; + yield* getContextObjects(nestedContext, seen); + } + } +} + +function isActivityStreamsAttachmentTerm(value: unknown): boolean { + return typeof value === "object" && value != null && + (value as Record)["@id"] === "as:attachment" && + (value as Record)["@type"] === "@id"; +} + +/** @internal */ +export function isPreloadedContextAttachmentSafe(document: unknown): boolean { + if (typeof document !== "object" || document == null) return true; + const context = (document as Record)["@context"]; + for (const contextObject of getContextObjects(context)) { + if (!Object.hasOwn(contextObject, "attachment")) continue; + if (isActivityStreamsAttachmentTerm(contextObject.attachment)) continue; + return false; + } + return true; +} + +function getKnownSafeContextUrls(): ReadonlySet { + const urls = new Set(); + for (const [url, document] of Object.entries(preloadedContexts)) { + if (isPreloadedContextAttachmentSafe(document)) { + urls.add(url); + } else { + logger.warn( + "Preloaded JSON-LD context {contextUrl} redefines the " + + "`attachment` term incompatibly; attachment array normalization " + + "will require canonicalization for documents using it.", + { contextUrl: url }, + ); + } + } + return urls; +} + +/** + * Wraps scalar ActivityStreams attachment properties in arrays. + */ +function wrapScalarAttachments( + jsonLd: unknown, + depth: number = 0, +): unknown { + if (depth >= MAX_TRAVERSAL_DEPTH) return jsonLd; + + if (Array.isArray(jsonLd)) { + let normalized: unknown[] | null = null; + for (let i = 0; i < jsonLd.length; i++) { + const item = jsonLd[i]; + const next = wrapScalarAttachments(item, depth + 1); + if (normalized == null && next !== item) { + normalized = jsonLd.slice(0, i); + } + if (normalized != null) { + normalized[i] = next; + } + } + return normalized ?? jsonLd; + } + + if (typeof jsonLd !== "object" || jsonLd == null) return jsonLd; + + const record = jsonLd as Record; + const keys = Object.keys(record); + let normalized: Record | null = null; + for (let i = 0; i < keys.length; i++) { + const key = keys[i]; + const value = record[key]; + const next = key === "@context" || + (key === "@value" && isJsonLdValueObject(jsonLd)) + ? value + : wrapScalarAttachments(value, depth + 1); + const shouldWrap = ATTACHMENT_FIELDS.has(key) && + next != null && + !Array.isArray(next) && + !isJsonLdListObject(next); + const output = shouldWrap ? [next] : next; + + if (normalized == null && output !== value) { + const cloned: Record = Object.create(null); + for (let j = 0; j < i; j++) { + const previousKey = keys[j]; + cloned[previousKey] = record[previousKey]; + } + normalized = cloned; + } + if (normalized != null) { + normalized[key] = output; + } + } + + return normalized ?? jsonLd; +} + +function hasNestedContext(value: unknown, depth: number = 0): boolean { + if (depth >= MAX_TRAVERSAL_DEPTH) return true; + if (Array.isArray(value)) { + return value.some((item) => hasNestedContext(item, depth + 1)); + } + if (typeof value !== "object" || value == null) return false; + const record = value as Record; + for (const key of Object.keys(record)) { + if (key === "@context") return true; + if (key === "@value" && isJsonLdValueObject(value)) continue; + if (hasNestedContext(record[key], depth + 1)) return true; + } + return false; +} + +function exceedsTraversalDepth(value: unknown, depth: number = 0): boolean { + if (depth >= MAX_TRAVERSAL_DEPTH) return true; + if (Array.isArray(value)) { + return value.some((item) => exceedsTraversalDepth(item, depth + 1)); + } + if (typeof value !== "object" || value == null) return false; + const record = value as Record; + for (const key of Object.keys(record)) { + if ( + key === "@context" || (key === "@value" && isJsonLdValueObject(value)) + ) { + continue; + } + if (exceedsTraversalDepth(record[key], depth + 1)) return true; + } + return false; +} + +function hasKnownSafeContext(jsonLd: unknown): boolean { + if (typeof jsonLd !== "object" || jsonLd == null) return false; + const record = jsonLd as Record; + if (!Object.hasOwn(record, "@context")) return false; + const context = record["@context"]; + const entries = typeof context === "string" + ? [context] + : Array.isArray(context) + ? context + : null; + if (entries == null || entries.length < 1) return false; + let hasActivityStreamsContext = false; + for (const entry of entries) { + if (typeof entry !== "string") return false; + if (!KNOWN_SAFE_CONTEXT_URLS.has(entry)) return false; + if (entry === AS_CONTEXT_URL) hasActivityStreamsContext = true; + } + if (!hasActivityStreamsContext) return false; + for (const key of Object.keys(record)) { + if (key === "@context") continue; + if (hasNestedContext(record[key])) return false; + } + return true; +} + +function getLogSafeJsonLdMetadata(jsonLd: unknown): Record { + if (typeof jsonLd !== "object" || jsonLd == null) return {}; + const record = jsonLd as Record; + const context = record["@context"]; + return { + id: typeof record.id === "string" + ? record.id + : typeof record["@id"] === "string" + ? record["@id"] + : undefined, + type: typeof record.type === "string" + ? record.type + : typeof record["@type"] === "string" + ? record["@type"] + : undefined, + context: typeof context === "string" + ? context + : Array.isArray(context) + ? context.filter((entry) => typeof entry === "string").slice(0, 4) + : context == null + ? undefined + : "[inline context]", + }; +} + +/** + * Ensures ActivityStreams attachment properties are represented as arrays + * when doing so preserves the JSON-LD semantics. + * + * JSON-LD compaction collapses single-item arrays into scalar values by + * default. Some ActivityPub implementations, Pixelfed among them, parse + * `attachment` as a plain JSON array rather than a JSON-LD property and reject + * otherwise valid objects whose single attachment is emitted as a scalar. + * + * When no `contextLoader` is supplied, the helper falls back to a restricted + * loader that resolves only Fedify's preloaded JSON-LD contexts and rejects + * every other URL without network access. Documents with custom, inline, or + * otherwise uncached contexts should pass a real `contextLoader` if they need + * the semantic-preservation check to succeed; otherwise canonicalization + * failures leave the original document unchanged. + */ +export async function normalizeAttachmentArrays( + jsonLd: unknown, + contextLoader?: DocumentLoader, +): Promise { + const normalized = wrapScalarAttachments(jsonLd); + if (normalized === jsonLd) return jsonLd; + if (exceedsTraversalDepth(jsonLd)) { + logger.debug( + "Skipping attachment array normalization because the JSON-LD document " + + "exceeds the safe traversal depth; leaving it unchanged.", + ); + return jsonLd; + } + if (hasKnownSafeContext(jsonLd)) return normalized; + const loader = contextLoader ?? preloadedOnlyDocumentLoader; + try { + const [before, after] = await Promise.all([ + jsonld.canonize(jsonLd, { + format: "application/n-quads", + documentLoader: loader, + }), + jsonld.canonize(normalized, { + format: "application/n-quads", + documentLoader: loader, + }), + ]); + if (before === after) return normalized; + logger.warn( + "Wrapping scalar attachment values in arrays would change the " + + "canonical form of the JSON-LD document; leaving it unchanged. " + + "This usually means the active JSON-LD context redefines the " + + "`attachment` term. Document: {id}; type: {type}; context: " + + "{context}.", + getLogSafeJsonLdMetadata(jsonLd), + ); + } catch (error) { + logger.debug( + "Failed to verify attachment array normalization equivalence via " + + "JSON-LD canonicalization; leaving the JSON-LD document " + + "unchanged.\n{error}", + { error }, + ); + } + return jsonLd; +} + +/** + * Applies Fedify's internal JSON-LD wire-format interoperability workarounds + * to locally generated outgoing activities before they are signed, enqueued, + * or sent. + */ +export async function normalizeOutgoingActivityJsonLd( + jsonLd: unknown, + contextLoader?: DocumentLoader, +): Promise { + jsonLd = await normalizePublicAudience(jsonLd, contextLoader); + return await normalizeAttachmentArrays(jsonLd, contextLoader); +} diff --git a/packages/fedify/src/compat/preloaded-context-loader.ts b/packages/fedify/src/compat/preloaded-context-loader.ts new file mode 100644 index 000000000..5d1abd327 --- /dev/null +++ b/packages/fedify/src/compat/preloaded-context-loader.ts @@ -0,0 +1,25 @@ +import { type DocumentLoader, preloadedContexts } from "@fedify/vocab-runtime"; + +/** + * A restricted JSON-LD document loader that resolves only contexts bundled + * with Fedify. + * + * This is intentionally narrower than `getDocumentLoader()`: normalization + * helpers are also reached from verification paths that operate on inbound, + * attacker-controlled JSON-LD, so the default fallback must never fetch + * attacker-supplied context URLs. + */ +export const preloadedOnlyDocumentLoader: DocumentLoader = (url: string) => { + if (Object.hasOwn(preloadedContexts, url)) { + return Promise.resolve({ + contextUrl: null, + documentUrl: url, + document: preloadedContexts[url], + }); + } + return Promise.reject( + new Error( + "Refusing to fetch a non-preloaded JSON-LD context: " + url, + ), + ); +}; diff --git a/packages/fedify/src/compat/public-audience.ts b/packages/fedify/src/compat/public-audience.ts index 868dc4829..c11345775 100644 --- a/packages/fedify/src/compat/public-audience.ts +++ b/packages/fedify/src/compat/public-audience.ts @@ -2,6 +2,7 @@ import { PUBLIC_COLLECTION } from "@fedify/vocab"; import { type DocumentLoader, preloadedContexts } from "@fedify/vocab-runtime"; import jsonld from "@fedify/vocab-runtime/jsonld"; import { getLogger } from "@logtape/logtape"; +import { preloadedOnlyDocumentLoader } from "./preloaded-context-loader.ts"; const logger = getLogger(["fedify", "compat", "public-audience"]); @@ -13,35 +14,6 @@ const PUBLIC_ADDRESSING_FIELDS = new Set([ "audience", ]); -// Default fallback document loader for `normalizePublicAudience()` when the -// caller omits `contextLoader`. It resolves only URLs that Fedify already -// ships as preloaded contexts; any other URL is rejected rather than -// fetched. `@fedify/vocab-runtime`'s full `getDocumentLoader()` would -// happily issue network requests for non-preloaded URLs after its -// `validatePublicUrl()` check, which is the SSRF vector raised in the -// review thread when the helper runs on adversarial input. Rejecting -// here turns that path into a canonicalization failure, which -// `normalizePublicAudience()` catches and handles by returning the -// document unchanged. -const preloadedOnlyDocumentLoader: DocumentLoader = (url: string) => { - // `Object.hasOwn()` rather than `url in preloadedContexts`: the loader - // runs on attacker-controlled URLs, and `in` would happily treat - // inherited `Object.prototype` names like `toString` as "preloaded" - // and then hand `Object.prototype.toString` to `jsonld.canonize`. - if (Object.hasOwn(preloadedContexts, url)) { - return Promise.resolve({ - contextUrl: null, - documentUrl: url, - document: preloadedContexts[url], - }); - } - return Promise.reject( - new Error( - "Refusing to fetch a non-preloaded JSON-LD context: " + url, - ), - ); -}; - const AS_CONTEXT_URL = "https://www.w3.org/ns/activitystreams"; // Caps recursion depth on the addressing-field walkers to keep a diff --git a/packages/fedify/src/federation/context.ts b/packages/fedify/src/federation/context.ts index 630bf1edd..0a82afdcf 100644 --- a/packages/fedify/src/federation/context.ts +++ b/packages/fedify/src/federation/context.ts @@ -901,6 +901,20 @@ export interface SendActivityOptions { */ readonly fanout?: "auto" | "skip" | "force"; + /** + * Whether to apply Fedify's outgoing JSON-LD wire-format compatibility fixes + * to activities that already carry Object Integrity Proofs. + * + * By default, Fedify preserves existing proofs byte-for-byte because it + * cannot know whether they were created for the normalized outgoing wire + * form. Set this to `true` when sending an activity that was pre-signed + * locally with `signObject()` or `createProof()`, so the emitted + * compact JSON-LD matches the bytes covered by the proof. + * + * @since 2.2.0 + */ + readonly normalizeExistingProofs?: boolean; + /** * The base URIs to exclude from the recipients' inboxes. It is useful * for excluding the recipients having the same shared inbox with the sender. diff --git a/packages/fedify/src/federation/middleware.test.ts b/packages/fedify/src/federation/middleware.test.ts index 91bd5da93..a32b804d9 100644 --- a/packages/fedify/src/federation/middleware.test.ts +++ b/packages/fedify/src/federation/middleware.test.ts @@ -17,6 +17,7 @@ import { assertThrows, } from "@std/assert"; import fetchMock from "fetch-mock"; +import serialize from "json-canon"; import createFixture from "../../../fixture/src/fixtures/example.com/create.json" with { type: "json", }; @@ -2895,6 +2896,36 @@ test("FederationImpl.sendActivity()", async (t) => { vocab.PUBLIC_COLLECTION.href, ); + verified = null; + await federation.sendActivity( + [{ privateKey: rsaPrivateKey3, keyId: rsaPublicKey3.id! }], + inboxes, + new vocab.Create({ + id: new URL("https://example.com/activity/attachment"), + actor: new URL("https://example.com/person2"), + object: new vocab.Note({ + id: new URL("https://example.com/note/attachment"), + attachments: [ + new vocab.Document({ + mediaType: "image/png", + url: new URL("https://example.com/image.png"), + }), + ], + }), + }), + { context }, + ); + assertEquals(verified, ["ld", "http"]); + const postedWithAttachment = await request?.json() as Record< + string, + unknown + >; + const postedObject = postedWithAttachment.object as Record< + string, + unknown + >; + assertEquals(Array.isArray(postedObject.attachment), true); + verified = null; await federation.sendActivity( [{ privateKey: rsaPrivateKey3, keyId: rsaPublicKey3.id! }], @@ -2933,6 +2964,78 @@ test("FederationImpl.sendActivity()", async (t) => { "application/activity+json", ); + const preSignedActivity = new vocab.Create({ + id: new URL("https://example.com/activity/pre-signed-attachment"), + actor: new URL("https://example.com/person2"), + object: new vocab.Note({ + id: new URL("https://example.com/note/pre-signed-attachment"), + attachments: [ + new vocab.Document({ + mediaType: "image/png", + url: new URL("https://example.com/pre-signed-image.png"), + }), + ], + }), + }); + const preSignedJson = await preSignedActivity.toJsonLd({ + format: "compact", + contextLoader: mockDocumentLoader, + }) as Record; + const preSignedObject = preSignedJson.object as Record; + assertEquals(Array.isArray(preSignedObject.attachment), false); + const created = Temporal.Now.instant(); + const proofConfig = { + "@context": preSignedJson["@context"], + type: "DataIntegrityProof", + cryptosuite: "eddsa-jcs-2022", + verificationMethod: ed25519Multikey.id!.href, + proofPurpose: "assertionMethod", + created: created.toString(), + }; + const encoder = new TextEncoder(); + const proofDigest = await crypto.subtle.digest( + "SHA-256", + encoder.encode(serialize(proofConfig)), + ); + const msgDigest = await crypto.subtle.digest( + "SHA-256", + encoder.encode(serialize(preSignedJson)), + ); + const digest = new Uint8Array( + proofDigest.byteLength + msgDigest.byteLength, + ); + digest.set(new Uint8Array(proofDigest), 0); + digest.set(new Uint8Array(msgDigest), proofDigest.byteLength); + const proofValue = new Uint8Array( + await crypto.subtle.sign("Ed25519", ed25519PrivateKey, digest), + ); + verified = null; + await federation.sendActivity( + [ + { privateKey: ed25519PrivateKey, keyId: ed25519Multikey.id! }, + ], + inboxes, + preSignedActivity.clone({ + proofs: [ + new vocab.DataIntegrityProof({ + cryptosuite: "eddsa-jcs-2022", + verificationMethod: ed25519Multikey.id!, + proofPurpose: "assertionMethod", + proofValue, + created, + }), + ], + }), + { context }, + ); + assertEquals(verified, ["proof"]); + const postedPreSigned = await request?.json() as Record; + const postedPreSignedObject = postedPreSigned.object as Record< + string, + unknown + >; + assertEquals(Array.isArray(postedPreSignedObject.attachment), false); + verified = null; await federation.sendActivity( [ @@ -3638,6 +3741,44 @@ test("ContextImpl.sendActivity()", async (t) => { "application/activity+json", ); + const actorEdKey = (await ctx.getActorKeyPairs("1")).find((key) => + key.privateKey.algorithm.name === "Ed25519" + ); + assert(actorEdKey != null); + assert(actorEdKey.multikey.id != null); + const signedWithNormalizedProof = await signObject( + new vocab.Create({ + id: new URL("https://example.com/activity/signed-attachment"), + actor: ctx.getActorUri("1"), + object: new vocab.Note({ + id: new URL("https://example.com/note/signed-attachment"), + attachments: [ + new vocab.Document({ + mediaType: "image/png", + url: new URL("https://example.com/signed-image.png"), + }), + ], + }), + }), + actorEdKey.privateKey, + actorEdKey.multikey.id, + { contextLoader: documentLoader }, + ); + verified = null; + await ctx.sendActivity( + [{ privateKey: actorEdKey.privateKey, keyId: actorEdKey.multikey.id }], + { + id: new URL("https://example.com/recipient"), + inboxId: new URL("https://example.com/inbox"), + }, + signedWithNormalizedProof, + { normalizeExistingProofs: true }, + ); + assertEquals(verified, ["proof"]); + const postedSigned = await request?.json() as Record; + const postedSignedObject = postedSigned.object as Record; + assertEquals(Array.isArray(postedSignedObject.attachment), true); + await assertRejects(() => ctx.sendActivity( { identifier: "not-found" }, @@ -3805,6 +3946,70 @@ test("ContextImpl.sendActivity()", async (t) => { queue.clear(); + await t.step( + 'fanout: "force" preserves pre-signed proof normalization', + async () => { + const ctxForProof = new ContextImpl({ + data: undefined, + federation, + url: new URL("https://example.com/"), + documentLoader: documentLoader, + contextLoader: documentLoader, + }); + const actorEdKey = (await ctxForProof.getActorKeyPairs("1")).find((key) => + key.privateKey.algorithm.name === "Ed25519" + ); + assert(actorEdKey != null); + assert(actorEdKey.multikey.id != null); + const signedWithNormalizedProof = await signObject( + new vocab.Create({ + id: new URL("https://example.com/activity/signed-attachment-fanout"), + actor: ctxForProof.getActorUri("1"), + object: new vocab.Note({ + id: new URL("https://example.com/note/signed-attachment-fanout"), + attachments: [ + new vocab.Document({ + mediaType: "image/png", + url: new URL("https://example.com/signed-fanout-image.png"), + }), + ], + }), + }), + actorEdKey.privateKey, + actorEdKey.multikey.id, + { contextLoader: documentLoader }, + ); + await ctx2.sendActivity( + [{ privateKey: actorEdKey.privateKey, keyId: actorEdKey.multikey.id }], + { + id: new URL("https://example.com/recipient"), + inboxId: new URL("https://example.com/inbox"), + }, + signedWithNormalizedProof, + { fanout: "force", normalizeExistingProofs: true }, + ); + assertEquals(queue.messages.length, 1); + assert(queue.messages[0].type === "fanout"); + const fanoutMsg = queue.messages[0]; + assertEquals(fanoutMsg.normalizeExistingProofs, true); + + queue.clear(); + await federation2.processQueuedTask(undefined, fanoutMsg); + assertEquals(queue.messages.length, 1); + const outboxMsg = queue.messages[0] as Message; + assert(outboxMsg.type === "outbox"); + + verified = null; + await federation2.processQueuedTask(undefined, outboxMsg); + assertEquals(verified, ["proof"]); + const postedSigned = await request?.json() as Record; + const postedSignedObject = postedSigned.object as Record; + assertEquals(Array.isArray(postedSignedObject.attachment), true); + }, + ); + + queue.clear(); + await t.step('fanout: "skip"', async () => { const activity = new vocab.Create({ id: new URL("https://example.com/activity/1"), diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index e86370a7e..8a9560bc2 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -48,7 +48,7 @@ import { ATTR_URL_FULL, } from "@opentelemetry/semantic-conventions"; import metadata from "../../deno.json" with { type: "json" }; -import { normalizePublicAudience } from "../compat/public-audience.ts"; +import { normalizeOutgoingActivityJsonLd } from "../compat/outgoing-jsonld.ts"; import { getDefaultActivityTransformers } from "../compat/transformers.ts"; import type { ActivityTransformer } from "../compat/types.ts"; import { getNodeInfo, type GetNodeInfoOptions } from "../nodeinfo/client.ts"; @@ -623,6 +623,7 @@ export class FederationImpl await this.sendActivity(keys, message.inboxes, activity, { collectionSync: message.collectionSync, orderingKey: message.orderingKey, + normalizeExistingProofs: message.normalizeExistingProofs, context, }); } @@ -1084,6 +1085,7 @@ export class FederationImpl this.#getLoaderOptions(ctx.origin), ); const activityId = activity.id.href; + let hasProof = false; let proofCreated = false; let rsaKey: { keyId: URL; privateKey: CryptoKey } | null = null; for (const { keyId, privateKey } of keys) { @@ -1095,16 +1097,17 @@ export class FederationImpl // If Object Integrity Proofs were already created before fanout (e.g., in // sendActivityInternal()), skip signing to avoid duplicates. for await (const _ of activity.getProofs({ contextLoader })) { - proofCreated = true; + hasProof = true; break; } - if (!proofCreated) { + if (!hasProof) { for (const { keyId, privateKey } of keys) { if (privateKey.algorithm.name === "Ed25519") { activity = await signObject(activity, privateKey, keyId, { contextLoader, tracerProvider: this.tracerProvider, }); + hasProof = true; proofCreated = true; } } @@ -1113,7 +1116,13 @@ export class FederationImpl format: "compact", contextLoader, }); - jsonLd = await normalizePublicAudience(jsonLd, contextLoader); + // Existing proofs are preserved by default because they may have been + // created over the compact JSON-LD bytes exactly as supplied. Fedify can + // safely normalize unsigned activities, proofs it just created, or + // locally pre-signed activities when callers opt in. + if (proofCreated || !hasProof || options.normalizeExistingProofs) { + jsonLd = await normalizeOutgoingActivityJsonLd(jsonLd, contextLoader); + } if (rsaKey == null) { logger.warn( "No supported key found to create a Linked Data signature for " + @@ -1134,7 +1143,7 @@ export class FederationImpl tracerProvider: this.tracerProvider, }); } - if (!proofCreated) { + if (!hasProof) { logger.warn( "No supported key found to create a proof for the activity {activityId}. " + "The activity will be sent without a proof. " + @@ -2370,6 +2379,7 @@ export class ContextImpl implements Context { orderingKey: options.orderingKey, collectionSync, immediate: options.immediate, + normalizeExistingProofs: options.normalizeExistingProofs, }; span.setAttribute("activitypub.inboxes", expandedRecipients.length); for (const activityTransformer of this.federation.activityTransformers) { @@ -2388,6 +2398,7 @@ export class ContextImpl implements Context { // Pre-sign with Object Integrity Proofs before fanout so that all // recipients receive the same signed activity. Uses Multikey IDs so that // verifiers can look up the correct key type in the actor document. + let proofCreated = false; if (actorKeyPairs != null) { const contextLoader = this.contextLoader; for (const kp of actorKeyPairs) { @@ -2399,6 +2410,7 @@ export class ContextImpl implements Context { contextLoader, tracerProvider: this.tracerProvider, }); + proofCreated = true; } } const inboxes = extractInboxes({ @@ -2423,7 +2435,11 @@ export class ContextImpl implements Context { options.fanout === "skip" || (options.fanout ?? "auto") === "auto" && globalThis.Object.keys(inboxes).length < FANOUT_THRESHOLD ) { - await this.federation.sendActivity(keys, inboxes, activity, opts); + await this.federation.sendActivity(keys, inboxes, activity, { + ...opts, + normalizeExistingProofs: proofCreated || + options.normalizeExistingProofs, + }); return true; } const keyJwkPairs = await Promise.all( @@ -2452,6 +2468,7 @@ export class ContextImpl implements Context { activityType: getTypeId(activity).href, collectionSync: opts.collectionSync, orderingKey: options.orderingKey, + normalizeExistingProofs: proofCreated || options.normalizeExistingProofs, traceContext: carrier, }; if (!this.federation.manuallyStartQueue) { @@ -3331,6 +3348,7 @@ interface SendActivityInternalOptions { readonly immediate?: boolean; readonly collectionSync?: string; readonly orderingKey?: string; + readonly normalizeExistingProofs?: boolean; readonly context: Context; } diff --git a/packages/fedify/src/federation/queue.ts b/packages/fedify/src/federation/queue.ts index 96ad36c07..993d5f7b4 100644 --- a/packages/fedify/src/federation/queue.ts +++ b/packages/fedify/src/federation/queue.ts @@ -30,6 +30,15 @@ export interface FanoutMessage { readonly activityType: string; readonly collectionSync?: string; readonly orderingKey?: string; + /** + * Whether to apply outgoing JSON-LD wire-format normalization to queued + * activities that already carry Object Integrity Proofs. + * + * `true` is used for proofs Fedify created before fanout, or when callers + * explicitly request normalization for locally pre-signed activities. + * `false`/`undefined` preserves existing proofs as-is. + */ + readonly normalizeExistingProofs?: boolean; readonly traceContext: Readonly>; } diff --git a/packages/fedify/src/sig/proof.test.ts b/packages/fedify/src/sig/proof.test.ts index 464fd9d67..b214e0aab 100644 --- a/packages/fedify/src/sig/proof.test.ts +++ b/packages/fedify/src/sig/proof.test.ts @@ -1,9 +1,10 @@ import { mockDocumentLoader, test } from "@fedify/fixture"; -import { normalizePublicAudience } from "../compat/public-audience.ts"; +import { normalizeOutgoingActivityJsonLd } from "../compat/outgoing-jsonld.ts"; import { Create, type CryptographicKey, DataIntegrityProof, + Document, Multikey, Note, Place, @@ -274,8 +275,8 @@ test("signObject()", async () => { ); // The proof hashed during signObject() must cover the same JSON-LD bytes - // that the activity serializes to on the wire — otherwise the public - // audience normalization applied before sending would break verifyProof() + // that the activity serializes to on the wire — otherwise the outgoing + // JSON-LD normalization applied before sending would break verifyProof() // for the eddsa-jcs-2022 cryptosuite, which canonicalises the JCS form // byte-for-byte rather than running URDNA2015. const publicActivity = new Create({ @@ -285,6 +286,12 @@ test("signObject()", async () => { id: new URL("https://server.example/objects/2"), attribution: new URL("https://server.example/users/alice"), content: "Hello public", + attachments: [ + new Document({ + mediaType: "image/png", + url: new URL("https://server.example/objects/2/image.png"), + }), + ], }), tos: [PUBLIC_COLLECTION], }); @@ -296,11 +303,13 @@ test("signObject()", async () => { ); const [proof] = await Array.fromAsync(signed.getProofs(options)); assertInstanceOf(proof, DataIntegrityProof); - const signedJson = await normalizePublicAudience( + const signedJson = await normalizeOutgoingActivityJsonLd( await signed.toJsonLd(options), mockDocumentLoader, ) as Record; assertEquals(signedJson.to, PUBLIC_COLLECTION.href); + const signedJsonObject = signedJson.object as Record; + assertEquals(Array.isArray(signedJsonObject.attachment), true); const verifyCache: Record = {}; const verifyOptions: VerifyProofOptions = { contextLoader: mockDocumentLoader, @@ -318,16 +327,22 @@ test("signObject()", async () => { // Round-trip regression guard: `signObject()` returns a vocab object // whose default `toJsonLd({ format: "compact" })` output still compacts - // the public audience to the `as:Public` CURIE, even though the bytes - // signed by `createProof()` were first normalized to the expanded URI. + // the public audience to the `as:Public` CURIE and single attachments to + // scalars, even though the bytes signed by `createProof()` were first + // normalized to the outgoing wire form. // `verifyProof()` must accept either form so the in-memory pipeline // (sign, reserialize, verify) continues to work without every caller - // having to know about the public-audience compat helper. + // having to know about the outgoing JSON-LD compat helper. const signedJsonWithCurie = await signed.toJsonLd(options) as Record< string, unknown >; assertEquals(signedJsonWithCurie.to, "as:Public"); + const signedJsonWithCurieObject = signedJsonWithCurie.object as Record< + string, + unknown + >; + assertEquals(Array.isArray(signedJsonWithCurieObject.attachment), false); const verifyingKeyFromCurie = await verifyProof( signedJsonWithCurie, proof, @@ -507,16 +522,16 @@ test("verifyProof()", async () => { ); // verifyProof() runs on inbound, potentially adversarial JSON-LD, so - // normalizePublicAudience() must not hand an attacker-controlled + // normalizeOutgoingActivityJsonLd() must not hand an attacker-controlled // `@context` URL to a network-capable document loader. The attacker // input below would otherwise take the canonicalization path (its - // `@context` is not drawn entirely from Fedify's preloaded set), but - // because we do not pass `contextLoader`, normalizePublicAudience() - // falls back to the internal preloaded-only loader, which rejects - // the attacker URL; canonicalization errors out and the normalized - // candidate is dropped. verify then tries the on-wire form against - // a proof that was signed over a different activity and returns - // null cleanly without any network request. + // `@context` is not drawn entirely from Fedify's preloaded set). + // verifyProof() deliberately does not pass its own `contextLoader` to + // normalizeOutgoingActivityJsonLd(), so that helper falls back to the + // internal preloaded-only loader, rejects the attacker URL, and drops + // the normalized candidate. verify then tries the on-wire form against + // a proof that was signed over a different activity and returns null + // cleanly without any network request. const attackerInput = { "@context": [ "https://www.w3.org/ns/activitystreams", @@ -533,13 +548,19 @@ test("verifyProof()", async () => { to: "as:Public", }, }; + const contextLoaderCalls: string[] = []; assertEquals( await verifyProof(attackerInput, proof, { + contextLoader: async (url) => { + contextLoaderCalls.push(url); + return await mockDocumentLoader(url); + }, documentLoader: mockDocumentLoader, keyCache: options.keyCache, }), null, ); + assertFalse(contextLoaderCalls.includes("https://attacker.example/ctx")); }); test("verifyObject()", async () => { diff --git a/packages/fedify/src/sig/proof.ts b/packages/fedify/src/sig/proof.ts index ef9c888b0..cb7885631 100644 --- a/packages/fedify/src/sig/proof.ts +++ b/packages/fedify/src/sig/proof.ts @@ -11,7 +11,8 @@ import { SpanStatusCode, trace, type TracerProvider } from "@opentelemetry/api"; import { encodeHex } from "byte-encodings/hex"; import serialize from "json-canon"; import metadata from "../../deno.json" with { type: "json" }; -import { normalizePublicAudience } from "../compat/public-audience.ts"; +import { normalizeOutgoingActivityJsonLd } from "../compat/outgoing-jsonld.ts"; +import { preloadedOnlyDocumentLoader } from "../compat/preloaded-context-loader.ts"; import { fetchKey, type FetchKeyResult, @@ -132,7 +133,10 @@ export async function createProof( contextLoader, context, }); - compactMsg = await normalizePublicAudience(compactMsg, contextLoader); + compactMsg = await normalizeOutgoingActivityJsonLd( + compactMsg, + contextLoader, + ); const msgCanon = serialize(compactMsg); const encoder = new TextEncoder(); const msgBytes = encoder.encode(msgCanon); @@ -373,18 +377,15 @@ async function verifyProofInternal( if ("https://w3id.org/security#proof" in msg) { delete msg["https://w3id.org/security#proof"]; } - // Try the on-wire form first, then fall back to the public-audience - // normalized form so that signatures created by `createProof` (which - // signs the normalized bytes) still verify when the caller passes the - // default `toJsonLd({ format: "compact" })` output that still carries - // the `as:Public` CURIE. `normalizePublicAudience()` defaults to a - // preloaded-only document loader when `options.contextLoader` is - // omitted, so the normalization attempt is safe to run on inbound, - // potentially adversarial JSON-LD: an attacker-supplied `@context` - // URL cannot steer canonicalization into a network fetch. - const candidates: unknown[] = [msg]; - const normalized = await normalizePublicAudience(msg, options.contextLoader); - if (normalized !== msg) candidates.push(normalized); + // Try the on-wire form first. Only if that fails do we fall back to + // Fedify's outgoing JSON-LD compatibility form so that signatures created + // by `createProof` (which signs the normalized bytes) still verify when the + // caller passes the default `toJsonLd({ format: "compact" })` output. + // + // This fallback must stay on normalizeOutgoingActivityJsonLd()'s + // preloaded-only default loader: it runs on inbound, potentially adversarial + // JSON-LD, and must not let attacker-supplied `@context` URLs steer + // canonicalization into a network fetch through `options.contextLoader`. let fetchedKey: FetchKeyResult | null; try { fetchedKey = await publicKeyPromise; @@ -435,20 +436,30 @@ async function verifyProofInternal( const SHA256_LENGTH = 32; const digest = new Uint8Array(proofDigest.byteLength + SHA256_LENGTH); digest.set(new Uint8Array(proofDigest), 0); - for (const candidate of candidates) { + const proofValue = proof.proofValue; + const verifyCandidate = async (candidate: unknown): Promise => { const msgBytes = encoder.encode(serialize(candidate)); const msgDigest = await crypto.subtle.digest("SHA-256", msgBytes); digest.set(new Uint8Array(msgDigest), proofDigest.byteLength); - const verified = await crypto.subtle.verify( + return await crypto.subtle.verify( "Ed25519", publicKey.publicKey, // `.slice()` narrows `Uint8Array` (which can be // backed by a `SharedArrayBuffer`) to `Uint8Array`, // which is what `crypto.subtle.verify` expects. - proof.proofValue.slice(), + proofValue.slice(), digest, ); - if (verified) return publicKey; + }; + if (await verifyCandidate(msg)) return publicKey; + // This fallback runs on inbound, attacker-controlled JSON-LD, so the loader + // must not fetch custom `@context` URLs from the network. + const normalized = await normalizeOutgoingActivityJsonLd( + msg, + preloadedOnlyDocumentLoader, + ); + if (normalized !== msg && await verifyCandidate(normalized)) { + return publicKey; } if (fetchedKey.cached) { logger.debug(