feat(canon): canonicalization core (VOKF-1 / S1)

packages/canon/package.json

@@ -6,11 +6,25 @@
   "type": "module",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
-  "exports": { ".": { "types": "./dist/index.d.ts", "import": "./dist/index.js" } },
-  "files": ["dist"],
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist"
+  ],
   "scripts": {
     "build": "tsc -p tsconfig.json",
     "test": "vitest run",
     "lint": "tsc -p tsconfig.json --noEmit"
+  },
+  "dependencies": {
+    "canonicalize": "^2.0.0",
+    "yaml": "^2.6.0"
+  },
+  "devDependencies": {
+    "@types/node": "^20.14.0"
   }
 }

packages/canon/src/body.ts

@@ -0,0 +1,46 @@
+/**
+ * Body canonicalization and content_hash — Verifiable OKF SPEC §5.
+ *
+ * The canonical body is defined deterministically so that signer and verifier
+ * always agree byte-for-byte. No markdown-semantic normalization is performed
+ * in v0 (no link/heading/whitespace collapsing inside lines).
+ */
+import { createHash } from "node:crypto";
+
+const utf8Encoder = new TextEncoder();
+
+/** Decode bytes as UTF-8, rejecting invalid UTF-8 (SPEC §5 step 2). */
+export function decodeUtf8(bytes: Uint8Array): string {
+  return new TextDecoder("utf-8", { fatal: true }).decode(bytes);
+}
+
+/**
+ * Canonicalize an (already UTF-8-decoded) body string into canonical bytes.
+ * Steps 3–6 of SPEC §5:
+ *   3. normalize CRLF / CR → LF
+ *   4. strip trailing spaces/tabs at the end of each line
+ *   5. ensure exactly one trailing LF
+ *   6. re-encode UTF-8
+ */
+export function canonicalizeBody(body: string): Uint8Array {
+  const lf = body.replace(/\r\n/g, "\n").replace(/\r/g, "\n"); // step 3
+  const stripped = lf
+    .split("\n")
+    .map((line) => line.replace(/[ \t]+$/u, "")) // step 4
+    .join("\n");
+  const oneTrailingLf = stripped.replace(/\n+$/u, "") + "\n"; // step 5
+  return utf8Encoder.encode(oneTrailingLf); // step 6
+}
+
+/**
+ * Canonicalize body bytes directly: validates UTF-8 (§5 step 2) then applies
+ * steps 3–6. Throws `TypeError` on invalid UTF-8.
+ */
+export function canonicalizeBodyBytes(bodyBytes: Uint8Array): Uint8Array {
+  return canonicalizeBody(decodeUtf8(bodyBytes));
+}
+
+/** content_hash over canonical body bytes: "sha256:" + lowercase hex SHA-256 (§5). */
+export function contentHash(canonicalBody: Uint8Array): string {
+  return "sha256:" + createHash("sha256").update(canonicalBody).digest("hex");
+}

packages/canon/src/frontmatter.ts

@@ -0,0 +1,65 @@
+/**
+ * Frontmatter splitting and parsing — Verifiable OKF.
+ *
+ * Splits an OKF concept (YAML frontmatter + markdown body) at the closing
+ * `---` delimiter (SPEC §5 step 1), and parses the frontmatter into the
+ * JSON data model used for JCS (SPEC §6): plain objects / arrays / strings /
+ * numbers / booleans / null. Timestamps remain RFC 3339 strings.
+ */
+import YAML from "yaml";
+import { decodeUtf8 } from "./body.js";
+
+export interface SplitConcept {
+  /** Whether a leading `---` frontmatter block was found at offset 0. */
+  readonly hasFrontmatter: boolean;
+  /** Raw YAML text between the delimiters (empty if no frontmatter). */
+  readonly frontmatterText: string;
+  /** Body string: everything after the closing `---` delimiter and its newline. */
+  readonly body: string;
+}
+
+// Leading `---` line, frontmatter (non-greedy), closing `---` line + its newline (or EOF).
+const FRONTMATTER_RE = /^---[ \t]*\r?\n([\s\S]*?)\r?\n---[ \t]*(?:\r?\n|$)/u;
+
+/** Split a concept into frontmatter text and body. Validates UTF-8 for byte input. */
+export function splitConcept(input: Uint8Array | string): SplitConcept {
+  const text = typeof input === "string" ? input : decodeUtf8(input);
+  const m = FRONTMATTER_RE.exec(text);
+  if (!m || m.index !== 0) {
+    return { hasFrontmatter: false, frontmatterText: "", body: text };
+  }
+  return {
+    hasFrontmatter: true,
+    frontmatterText: m[1] ?? "",
+    body: text.slice(m[0].length),
+  };
+}
+
+/** Recursively coerce values into the JSON data model (Date → RFC 3339 UTC `Z`). */
+function toJsonModel(value: unknown): unknown {
+  if (value instanceof Date) return value.toISOString().replace(/\.\d{3}Z$/u, "Z");
+  if (Array.isArray(value)) return value.map(toJsonModel);
+  if (value && typeof value === "object") {
+    const out: Record<string, unknown> = {};
+    for (const [k, v] of Object.entries(value as Record<string, unknown>)) {
+      out[k] = toJsonModel(v);
+    }
+    return out;
+  }
+  return value;
+}
+
+/**
+ * Parse frontmatter YAML into the JSON data model. Uses YAML's `core` schema so
+ * ISO timestamps stay strings (not Date), then defensively coerces any Date.
+ * Returns `{}` for empty frontmatter.
+ */
+export function parseFrontmatter(frontmatterText: string): Record<string, unknown> {
+  if (frontmatterText.trim() === "") return {};
+  const parsed = YAML.parse(frontmatterText, { schema: "core" }) as unknown;
+  const model = toJsonModel(parsed);
+  if (model === null || typeof model !== "object" || Array.isArray(model)) {
+    throw new TypeError("frontmatter must be a YAML mapping");
+  }
+  return model as Record<string, unknown>;
+}

packages/canon/src/index.ts

@@ -1,8 +1,46 @@
 /**
- * @verifiable-okf/canon — Verifiable OKF canonicalization core (SPEC §5/§6).
- *
- * Scaffold genesis: the real canonicalization / JCS / signing-payload API lands
- * in the S1 `canon` PR. This file currently exposes only the package version so
- * the monorepo builds and CI is green from a clean checkout.
+ * @verifiable-okf/canon — the shared, version-pinned canonicalization core
+ * (Verifiable OKF SPEC §5/§6). Consumed identically by signer and verifier;
+ * any divergence here breaks every signature, so this package is tested hardest.
  */
 export const CANON_VERSION = "0.1.0";
+
+export { decodeUtf8, canonicalizeBody, canonicalizeBodyBytes, contentHash } from "./body.js";
+export { splitConcept, parseFrontmatter, type SplitConcept } from "./frontmatter.js";
+export { jcs } from "./jcs.js";
+export { buildSigningPayload, stripSignatureValue } from "./payload.js";
+
+import { canonicalizeBody, contentHash } from "./body.js";
+import { splitConcept, parseFrontmatter } from "./frontmatter.js";
+import { buildSigningPayload } from "./payload.js";
+
+export interface CanonicalConcept {
+  readonly hasFrontmatter: boolean;
+  readonly frontmatter: Record<string, unknown>;
+  readonly canonicalBody: Uint8Array;
+  readonly contentHash: string;
+  /** Present only when frontmatter exists (signing binds the frontmatter). */
+  readonly signingPayload?: Uint8Array;
+}
+
+/**
+ * Convenience: split a raw concept, canonicalize the body, compute its
+ * content_hash, parse the frontmatter, and build the signing payload.
+ * Validates UTF-8 for byte input.
+ */
+export function canonicalizeConcept(input: Uint8Array | string): CanonicalConcept {
+  const { hasFrontmatter, frontmatterText, body } = splitConcept(input);
+  const canonicalBody = canonicalizeBody(body);
+  const ch = contentHash(canonicalBody);
+  if (!hasFrontmatter) {
+    return { hasFrontmatter: false, frontmatter: {}, canonicalBody, contentHash: ch };
+  }
+  const frontmatter = parseFrontmatter(frontmatterText);
+  return {
+    hasFrontmatter: true,
+    frontmatter,
+    canonicalBody,
+    contentHash: ch,
+    signingPayload: buildSigningPayload(frontmatter, canonicalBody),
+  };
+}

packages/canon/src/jcs.ts

@@ -0,0 +1,21 @@
+/**
+ * JSON Canonicalization Scheme (RFC 8785) — Verifiable OKF SPEC §6.
+ *
+ * Wraps the `canonicalize` implementation behind a stable, version-pinned
+ * entry point so signer and verifier never diverge. RFC 8785 governs key
+ * ordering (by UTF-16 code unit), number serialization, and string escaping.
+ */
+import canonicalizeDefault from "canonicalize";
+
+// `canonicalize` is a CJS module whose default export is the function; its
+// bundled types don't expose a call signature, so we re-type it here.
+const canonicalize = canonicalizeDefault as unknown as (value: unknown) => string | undefined;
+
+/** RFC 8785 canonical JSON string of a JSON-data-model value. */
+export function jcs(value: unknown): string {
+  const out = canonicalize(value);
+  if (typeof out !== "string") {
+    throw new TypeError("JCS: value is not JSON-serializable");
+  }
+  return out;
+}

packages/canon/src/payload.ts

@@ -0,0 +1,48 @@
+/**
+ * Signing-payload construction — Verifiable OKF SPEC §6.
+ *
+ *   signing_payload = JCS(frontmatter*) || 0x0A || canonical_body
+ *
+ * where `frontmatter*` is the parsed frontmatter mapping with ONLY
+ * `provenance.signature.value` removed (everything else, including the rest of
+ * the provenance block and `signature.scheme`, is retained).
+ */
+import { jcs } from "./jcs.js";
+
+const LF = 0x0a;
+const utf8Encoder = new TextEncoder();
+
+function isObject(v: unknown): v is Record<string, unknown> {
+  return typeof v === "object" && v !== null && !Array.isArray(v);
+}
+
+/**
+ * Return a deep copy of the frontmatter with only `provenance.signature.value`
+ * removed. The input is never mutated.
+ */
+export function stripSignatureValue(
+  frontmatter: Record<string, unknown>,
+): Record<string, unknown> {
+  const clone = structuredClone(frontmatter);
+  const provenance = clone["provenance"];
+  if (isObject(provenance)) {
+    const signature = provenance["signature"];
+    if (isObject(signature)) {
+      delete signature["value"];
+    }
+  }
+  return clone;
+}
+
+/** Build the signing payload bytes (SPEC §6). */
+export function buildSigningPayload(
+  frontmatter: Record<string, unknown>,
+  canonicalBody: Uint8Array,
+): Uint8Array {
+  const jcsBytes = utf8Encoder.encode(jcs(stripSignatureValue(frontmatter)));
+  const out = new Uint8Array(jcsBytes.length + 1 + canonicalBody.length);
+  out.set(jcsBytes, 0);
+  out[jcsBytes.length] = LF;
+  out.set(canonicalBody, jcsBytes.length + 1);
+  return out;
+}

packages/canon/test/body.test.ts

@@ -0,0 +1,51 @@
+import { describe, it, expect } from "vitest";
+import {
+  canonicalizeBody,
+  canonicalizeBodyBytes,
+  contentHash,
+} from "../src/body.js";
+
+const dec = new TextDecoder();
+const s = (u: Uint8Array) => dec.decode(u);
+
+describe("canonicalizeBody (SPEC §5)", () => {
+  it("normalizes CRLF and CR to LF", () => {
+    expect(s(canonicalizeBody("a\r\nb\rc"))).toBe("a\nb\nc\n");
+  });
+
+  it("strips trailing spaces and tabs at end of each line", () => {
+    expect(s(canonicalizeBody("a \t\nb\t \nc"))).toBe("a\nb\nc\n");
+  });
+
+  it("collapses trailing newlines to exactly one LF", () => {
+    expect(s(canonicalizeBody("a\n\n\n"))).toBe("a\n");
+  });
+
+  it("adds a trailing LF when missing", () => {
+    expect(s(canonicalizeBody("no newline"))).toBe("no newline\n");
+  });
+
+  it("maps empty body to a single LF", () => {
+    expect(s(canonicalizeBody(""))).toBe("\n");
+  });
+
+  it("hashes CRLF and trailing-whitespace variants identically", () => {
+    const a = contentHash(canonicalizeBody("hello world\nsecond\n"));
+    const b = contentHash(canonicalizeBody("hello world  \r\nsecond\t\r\n\r\n"));
+    expect(a).toBe(b);
+  });
+
+  it("produces a well-formed content_hash", () => {
+    expect(contentHash(canonicalizeBody("x"))).toMatch(/^sha256:[0-9a-f]{64}$/);
+  });
+
+  it("is idempotent (canon∘canon = canon)", () => {
+    const once = canonicalizeBody("a \r\nb\n\n");
+    const twice = canonicalizeBody(s(once));
+    expect(s(twice)).toBe(s(once));
+  });
+
+  it("rejects invalid UTF-8 bytes", () => {
+    expect(() => canonicalizeBodyBytes(new Uint8Array([0xff, 0xfe, 0x00]))).toThrow();
+  });
+});

packages/canon/test/frontmatter.test.ts

@@ -0,0 +1,59 @@
+import { describe, it, expect } from "vitest";
+import { splitConcept, parseFrontmatter } from "../src/frontmatter.js";
+import { jcs } from "../src/jcs.js";
+
+const CONCEPT = `---
+title: Example
+resource: https://example.com/x
+provenance:
+  spec_version: "verifiable-okf/0.3"
+  issuer: "did:web:example.com"
+---
+# Body
+
+Hello world.
+`;
+
+describe("splitConcept (SPEC §5 step 1)", () => {
+  it("splits frontmatter and body at the closing delimiter", () => {
+    const r = splitConcept(CONCEPT);
+    expect(r.hasFrontmatter).toBe(true);
+    expect(r.frontmatterText).toContain("title: Example");
+    expect(r.body).toBe("# Body\n\nHello world.\n");
+  });
+
+  it("treats input without leading frontmatter as all-body", () => {
+    const r = splitConcept("# Just markdown\n");
+    expect(r.hasFrontmatter).toBe(false);
+    expect(r.body).toBe("# Just markdown\n");
+  });
+
+  it("handles CRLF delimiters", () => {
+    const r = splitConcept("---\r\ntitle: X\r\n---\r\nbody\r\n");
+    expect(r.hasFrontmatter).toBe(true);
+    expect(r.frontmatterText).toContain("title: X");
+    expect(r.body).toBe("body\r\n");
+  });
+});
+
+describe("parseFrontmatter (JSON data model for JCS)", () => {
+  it("parses a mapping into plain JSON types", () => {
+    const fm = parseFrontmatter('title: X\ncount: 3\nflag: true');
+    expect(fm).toEqual({ title: "X", count: 3, flag: true });
+  });
+
+  it("keeps RFC 3339 timestamps as strings", () => {
+    const fm = parseFrontmatter('valid_until: "2026-06-15T00:00:00Z"');
+    expect(fm["valid_until"]).toBe("2026-06-15T00:00:00Z");
+  });
+
+  it("yields the same JCS regardless of source key order", () => {
+    const a = jcs(parseFrontmatter('a: 1\nb: 2\nc: 3'));
+    const b = jcs(parseFrontmatter('c: 3\na: 1\nb: 2'));
+    expect(a).toBe(b);
+  });
+
+  it("returns {} for empty frontmatter", () => {
+    expect(parseFrontmatter("")).toEqual({});
+  });
+});